bad-md-processor.js

bad-md-processor.js

import sqlite3 from 'sqlite3';
import fs from 'node:fs/promises'
import { glob } from 'glob'
import {fromMarkdown} from 'mdast-util-from-markdown'
import {frontmatter} from 'micromark-extension-frontmatter'
import {frontmatterFromMarkdown, frontmatterToMarkdown} from 'mdast-util-frontmatter'
import { syntax } from 'micromark-extension-wiki-link'
import * as wikiLink from 'mdast-util-wiki-link'

// const db = new sqlite3.Database(':memory:');
const db = new sqlite3.Database('./test.sqlite');
async function main(){
    db.serialize(async () => {
      db.run("CREATE TABLE IF NOT EXISTS pkm ( \
        file_name TEXT, \
        file_path TEXT, \
        raw_markdown TEXT, \
        markdown_syntax_tree JSON)");

      const jsfiles = await glob('**/*.md', { ignore: 'node_modules/**' })
      await jsfiles.forEach( async(file_path) => {
        let file_name = file_path.split('/');
        file_name = file_name[file_name.length - 1]
        let file_extension = file_path.split('.');
        file_extension = file_extension[file_extension.length - 1]
        const doc = await fs.readFile(file_path)
        const tree = fromMarkdown(doc, {
          extensions: [syntax(), frontmatter(['yaml', 'toml'])],
          mdastExtensions: [wikiLink.fromMarkdown(), frontmatterFromMarkdown(['yaml', 'toml'])]
        })
        const stmt = await db.prepare('INSERT INTO pkm \
        (file_name, file_path, raw_markdown, markdown_syntax_tree) \
        VALUES (?, ?, ?, json(?))');
        // define the values to be inserted into the statement
        const values = [file_name, file_path, doc, JSON.stringify(tree)];
        // execute the statement with the values
        stmt.run(values, function(err) {
          if (err) {
            console.error(err.message);
          } else {
            console.log(`Row(s) inserted ${this.changes}`);
          }
        });
        console.log(file_path)
        console.log(file_name)
        console.log(file_extension)
        console.log(doc.toString())
        console.log(tree)
      });
      
  });
  // db.close();
}

// 
main()