Single script for doc generation

*doc-generator.md

Doc Generator

Single file doc generator for javascript projects where no other programming language is available.

This was a learning project for working with reactive streams for CLI apps and something I needed at work to generate wiki sites from an existing docs folder.

There's an attached Dockerfile if you wish to deploy it on a container, the script is for flat docs folders with all docs on the first level of the folder a.k.a nested documents aren't supported right now.

Dockerfile

FROM node:16-alpine as build-stage

WORKDIR /app

COPY . .
RUN yarn add github-slugger \
rehype \
rehype-document \
rehype-format \
rehype-highlight \
rehype-rewrite \
rehype-shiki \
rehype-slug \
rehype-stringify \
remark \
remark-gfm \
remark-html \
remark-parse \
remark-rehype \
unified \
xstream; node generate.mjs ./docs

FROM nginx:stable-alpine as production-stage

COPY --from=build-stage /app/dist /usr/share/nginx/html
EXPOSE 80

CMD ["nginx", "-g", "daemon off;"]

generate.mjs

import { unified } from 'unified'
import remarkParse from 'remark-parse'
import { rehype } from 'rehype'
import Slugger from 'github-slugger'
import remarkRehype from 'remark-rehype'
import rehypeSlug from 'rehype-slug'
import rehypeDocument from 'rehype-document'
import rehypeRewrite from 'rehype-rewrite'
import rehypeFormat from 'rehype-format'
import remarkGFM from 'remark-gfm'
import rehypeStringify from 'rehype-stringify'
import rehypeShiki from 'rehype-shiki'

import { promises as fs } from 'fs'
import _xs from 'xstream'
import path from 'path'
import { mkdir } from 'fs/promises'
const xs = _xs.default

const HIGHLIGHT_THEME = 'min-light'

const slugify = new Slugger()
slugify.reset()

const stylesText = `
    html {font-size: 100%;} /*16px*/

    body {
        background: white;
        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-weight: 400;
        line-height: 1.75;
        color: #212529;
    }

    p {margin-bottom: 1rem;}

    h1, h2, h3, h4, h5 {
        margin: 3rem 0 1.38rem;
        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
        font-weight: 600;
        line-height: 1.3;
    }

    h1 {
        margin-top: 0;
        font-size: 3.052rem;
    }

    h2 {font-size: 2.441rem;}

    h3 {font-size: 1.953rem;}

    h4 {font-size: 1.563rem;}

    h5 {font-size: 1.25rem;}

    small, .text_small {font-size: 0.8rem;}

    a {
        color:inherit;
    }

    pre,code {
        background: #f1f3f5 !important;
        border-radius:4px;
    }
    
    pre{
        padding:10px;
    }

    code {
        padding:4px;
    }

    .container{
        display:flex;
    }

    .sidebar {
        display:block;
        position: fixed;
        top: 0;
        z-index:1;
        width:300px;
    }

    .sidebar > a {
        display:block;
        padding:4px; 
        color: #868e96;
        text-decoration:none;
    }

    .sidebar > a:hover {
        color: #000;
    }

    .nav-heading-1 {
        margin-top:1rem;
        margin-left:0px;
    }

    .nav-heading-2{
        margin-left:1rem;
    }

    .container > .content {
        margin:0 auto;
        width:100%;
        max-width: 980px;
    }

    @media screen and (max-width:1250px){
        .sidebar {
            position:static;
        }
        .container{
            flex-wrap:wrap;
        }
    }

`
const args = process.argv.slice(2)
const folder = args[0]
const parseMarkdown = unified().use(remarkParse).parse

await mkdir('dist', { recursive: true })

// Find files needed for processing
const files$ = xs
  .fromPromise(fs.readdir(folder))
  .map(file => xs.fromArray(file))
  .flatten()
  .filter(x => String(x).endsWith('.md'))

// Generate meta for each file, this also reads the file's data
const fileMeta$ = files$
  .map(x => xs.fromPromise(processFile(x)))
  .fold((all, x) => (all.push(x), all), [])
  .map(x =>
    xs
      .combine(...x)
      .map(x => xs.fromArray(x))
      .flatten()
  )
  .flatten()

// extract the headings h1,h2 in each file
const headings$ = fileMeta$
  .map(x => ({
    meta: x,
    ast: parseMarkdown(x.fileData),
  }))
  .map(({ meta, ast }) => {
    const headings = mapHeadingASTToRehype(ast.children, meta)
    return headings
  })
  .fold((all, x) => ((all = all.concat(x)), all), [])

// convert markdown file into HTML
const mdToHtml$ = fileMeta$
  .map(x => {
    const processor = mdProcessor({ title: x.slug })

    const p = processor(x.fileData).then(d => {
      return {
        ...x,
        html: String(d),
      }
    })

    return xs.fromPromise(p)
  })
  .fold((all, x) => (all.push(x), all), [])
  .map(x =>
    xs
      .combine(...x)
      .map(x => xs.fromArray(x))
      .flatten()
  )
  .flatten()

// Write the data to disk
const write$ = xs
  .combine(mdToHtml$, headings$)
  .map(([fileMeta, headings]) => {
    return {
      ...fileMeta,
      finalHTML: getHTMLRewrites({
        index: headings,
        html: fileMeta.html,
      }),
    }
  })
  .map(x => {
    return xs.fromPromise(fs.writeFile(x.target, x.finalHTML, 'utf-8'))
  })
  .fold((all, x) => (all.push(x), all), [])
  .map(x => xs.combine(...x))
  .flatten()

write$.addListener({
  error: err => console.error(err),
})

async function processFile(file) {
  const _slug = file.replace(/.md$/, '.html')
  const _path = path.join(folder, file)
  const _target = path.join('dist', _slug)
  const fileData = await fs.readFile(_path, 'utf-8')
  return {
    slug: _slug,
    path: _path,
    target: _target,
    fileData,
  }
}

function mapHeadingASTToRehype(ast, meta) {
  return ast
    .filter(x => x.type == 'heading' && (x.depth === 1 || x.depth === 2))
    .map(x => {
      const node = {
        type: 'element',
        properties: {
          className: `nav-heading-${x.depth}`,
          href: `/${meta.slug}`,
        },
        tagName: `a`,
        children: [],
      }
      const valueNode = x.children.find(x => x.type == 'text')
      if (valueNode && valueNode.value) {
        node.children.push({
          type: 'text',
          value: valueNode.value,
        })
        const slug = slugify.slug(valueNode.value.toLowerCase())
        node.properties.href += `#${slug}`
      }
      return node
    })
}

function mdProcessor(props) {
  return unified()
    .use(remarkParse)
    .use(remarkGFM)
    .use(remarkRehype)
    .use(rehypeDocument, { title: props.title })
    .use(rehypeFormat)
    .use(rehypeSlug)
    .use(rehypeShiki, {
      theme: HIGHLIGHT_THEME,
    })
    .use(rehypeStringify).process
}

function getHTMLRewrites(params) {
  const { index, html } = params

  const styleRewrite = rehype()
    .use(rehypeRewrite, getStyleRewriter())
    .processSync(html)
    .toString()

  const indexRewrite = rehype()
    .use(rehypeRewrite, getIndexRewriter(index))
    .processSync(styleRewrite)
    .toString()

  return rehype()
    .use(rehypeFormat)
    .use(rehypeStringify)
    .processSync(indexRewrite)
    .toString()
}

function getStyleRewriter() {
  return {
    selector: 'head',
    rewrite: node => {
      node.children = [
        ...node.children,
        {
          type: 'element',
          tagName: 'style',
          children: [
            {
              type: 'text',
              value: stylesText,
            },
          ],
        },
      ]
    },
  }
}

function getIndexRewriter(index) {
  return {
    selector: 'body',
    rewrite: node => {
      if (node.type === 'element') {
        node.children = [
          {
            type: 'element',
            tagName: 'section',
            properties: {
              className: 'container',
            },
            children: [
              {
                type: 'element',
                tagName: 'header',
                properties: {
                  className: 'sidebar',
                },
                children: index,
              },
              {
                type: 'element',
                tagName: 'section',
                properties: {
                  className: 'content',
                },
                children: node.children,
              },
            ],
          },
        ]
      }
    },
  }
}