potch.me site generator 2023

blog.js

// if one file is too unwieldy you probably shouldn't have rolled your own blog yet again, POTCH
const start = Date.now();
const fs = require("fs").promises;
const path = require("path");
const crypto = require("crypto");

const Handlebars = require("handlebars");
const markdown = require("markdown-it");
const frontmatter = require("front-matter");
const prism = require("markdown-it-prism");
const mdfm = require("markdown-it-front-matter");
const mdContainer = require("markdown-it-container");
const prettier = require("prettier");

const RESET = "\x1b[0m";
const RED = "\x1b[;31m";
const BOLD = "\x1b[1m";
const GREEN = "\x1b[;32m";
const YELLOW = "\x1b[;33m";
const BLUE = "\x1b[;34m";
const MAGENTA = "\x1b[;35m";
const CYAN = "\x1b[;36m";
const WHITE = "\x1b[;37m";

const color = (text, ...colors) => `${colors.join("")}${text}${RESET}`;

// async means we can do more stuff while waiting for IO!
const CHUNK_SIZE = 8;

// basic settings
const OUTPUT_PATH = "build";
const INPUT_PATH = "src";
const PROD_SERVER = "https://potch.me";

// command line arguments
const CWD = process.cwd();
const argv = process.argv;

let changeEvent;
if (argv.includes("--changed")) {
  changeEvent = argv[argv.indexOf("--changed") + 1];
  console.log(color("[blog]", WHITE, BOLD) + " onchange:", changeEvent);
}
const FORCE_REBUILD = argv.includes("--force") || changeEvent === "blog.js";
if (FORCE_REBUILD)
  console.log(color("[blog]", WHITE, BOLD) + " doing full rebuild");

let lastCheckpoint = start;
const checkpoint = (label) => {
  console.log(
    `${YELLOW}[${Date.now() - start}ms +${
      Date.now() - lastCheckpoint
    }ms]${RESET} ${label}`
  );
  lastCheckpoint = Date.now();
};

const task = async (name, obj, args = {}) => [name, await obj, args];

const isPublished = (d) => (d.meta ? d.meta.published : true);

const slugify = (s) => s.replace(/\s+/g, "-");

const md = new markdown({ html: true, langPrefix: "pretty-code language-" });
// we need to recognize the frontmatter but md can ignore it
md.use(mdfm, function noop() {});
md.use(prism);
md.use(mdContainer, "wide");
md.use(mdContainer, "fullbg");
md.use(mdContainer, "fullpad");
md.use(mdContainer, "full");
md.use(mdContainer, "centered");

// simple 'x days ago' thingus
Handlebars.registerHelper({
  iso: function iso(d) {
    return new Date(d).toISOString();
  },
  isoDate: function iso(d) {
    return new Date(d).toISOString().split("T")[0];
  },
  ago: function ago(d) {
    d = Date.parse(d);
    if (!d) return "";
    d = ((Date.now() - d) / 1000) | 0;
    if (d < 60) return "seconds";
    // Minutes
    if (d < 120) return "a minute";
    if (d < 3600) return ((d / 60) | 0) + " minutes";
    // Hours
    d = (d / 3600) | 0;
    if (d < 2) return "an hour";
    if (d < 24) return d + " hours";
    // Days
    d = (d / 24) | 0;
    if (d < 2) return "a day";
    if (d < 30) return d + " days";
    // Months
    if (d < 60) return "a month";
    if (d < 360 * 1.5) return ((d / 30) | 0) + " months";
    // Years
    if (d < 365 * 2) return "a year and a half";
    return Math.round(d / 365) + " years";
  },
  slugify,
});

const hashContent = (s) => {
  const hash = crypto.createHash("sha256");
  hash.setEncoding("hex");
  hash.write(s);
  hash.end();
  return hash.read();
};

const loadTemplate = async (path) => (await fs.readFile(path)).toString("utf8");

// get all files in a path recursively
async function crawl(crawlPath, { exclude = [] } = {}) {
  let files = [];
  exclude = new Set(exclude);
  // stack based recursion works nicely with async/await
  // a symlink loop will wreck me oh well
  let toCrawl = [crawlPath];
  while (toCrawl.length) {
    let crawlPath = toCrawl.pop();
    let dirFiles = await fs.readdir(crawlPath, { withFileTypes: true });
    dirFiles
      .filter((f) => !exclude.has(f.name) && !f.name.startsWith("."))
      .forEach((f) => {
        const p = path.join(crawlPath, f.name);
        if (f.isDirectory()) {
          toCrawl.push(p);
        } else {
          files.push(p);
        }
      });
  }

  return files;
}

async function srcIsNewer(srcPath, destPath) {
  return Promise.allSettled([fs.stat(srcPath), fs.stat(destPath)]).then(
    ([a, b]) => b.status !== "fulfilled" || a.value.mtime > b.value.mtime
  );
}

// deep compare objects for triple equality
function deepEQ(a, b) {
  if (a instanceof Date || b instanceof Date) {
    return Date(a).valueOf() === Date(b).valueOf();
  }
  if (typeof a !== typeof b) {
    return false;
  }
  if (typeof a === "object") {
    for (let k of Object.keys(a)) {
      if (!deepEQ(a[k], b[k])) {
        return false;
      }
    }
    return true;
  }
  if (a === b) {
    return true;
  }
  return false;
}

function diff(from = [], to = [], key) {
  const out = [];
  to.forEach((b) => {
    let a = from.find((d) => d[key] === b[key]);
    if (b.hash && a) {
      if (a.hash !== b.hash) {
        out.push(b);
      }
    } else {
      if (!deepEQ(a, b)) {
        out.push(b);
      }
    }
  });
  return out;
}

// turn a file into a document with all sorts of metadata
const indexers = {
  async md(basePath, filePath, index) {
    let raw = (await fs.readFile(filePath)).toString("utf8");
    let stats = await fs.stat(filePath);
    // parse the frontmatter. frontmatter may be the one good use of yaml
    let meta = frontmatter(raw).attributes;

    // allow for lazy authoring by extracting post titles and blurbs from markdown
    let tokens = md.parse(raw, {});
    let titlePosition;
    for (let i = 0; i < tokens.length; i++) {
      let token = tokens[i];
      // extract the first <h1> as the title to put it at the top
      if (!meta.title && token.type === "heading_open" && token.tag === "h1") {
        meta.title = tokens[i + 1].content;
        titlePosition = token.map;
      }
      // use the first paragraph as a blurb if none exists
      if (
        !meta.blurb &&
        meta.blurb !== false &&
        token.type === "paragraph_open"
      ) {
        meta.blurb = md.renderInline(tokens[i + 1].content);
      }
    }

    // strip title from render
    let toContent = raw.split("\n");
    if (titlePosition) {
      toContent.splice(titlePosition[0], titlePosition[1] - titlePosition[0]);
    }
    toContent = toContent.join("\n");

    let docPath = path.relative(basePath, filePath);
    docPath = path.join(
      path.dirname(docPath),
      path.basename(docPath, path.extname(docPath)) + ".html"
    );

    const route = docPath.endsWith("/index.html")
      ? docPath.replace(/\/index\.html$/, "/")
      : docPath;

    const template = meta.template || "default";
    const tags = meta.tags || [];

    // document object
    let doc = {
      type: "md",
      content: md.render(toContent),
      path: docPath,
      name: docPath,
      route,
      meta,
      raw,
      changed: stats.mtime,
      created: stats.birthtime,
      template,
      node: {
        id: "docs|" + docPath,
        hash: hashContent(raw),
        dependencies: ["templates|" + template],
        affects: ["index", ...tags.map((t) => "tags|" + t)],
      },
    };
    if (meta.date) {
      doc.date = meta.date;
    } else {
      doc.date = doc.created;
    }

    index.docs.push(doc);
    index.tree.push(doc.node);
    tags.forEach((tag) => {
      index.tree.push({
        id: "tags|" + tag,
        dependencies: ["templates|tag"],
      });
    });
  },

  async hbs(basePath, filePath, index) {
    const source = await loadTemplate(filePath);
    let name = path.basename(filePath, ".hbs");

    if (path.extname(name) === ".partial") {
      name = path.basename(name, ".partial");
      const doc = {
        name,
        node: {
          id: "partials|" + name,
          hash: hashContent(source),
          dependencies: [],
        },
        source,
      };
      index.partials.push(doc);
      index.tree.push(doc.node);
      Handlebars.registerPartial(name, source);
    } else {
      const source = await loadTemplate(filePath);
      const name = path.basename(filePath, ".hbs");
      const partials = (source.match(/\{\{>([^\}]+)\}\}/g) || []).map((p) =>
        p.match(/\{\{>([^\}]+)\}\}/)[1].trim()
      );

      const doc = {
        name,
        node: {
          id: "templates|" + name,
          hash: hashContent(source),
          dependencies: partials.map((p) => "partials|" + p),
        },
        source,
      };
      index.templates.push(doc);
      index.tree.push(doc.node);
    }
  },
};

const builders = {
  docs({ obj }) {
    return [task("render", obj)];
  },

  index({ index }) {
    const publishedDocs = index.docs.filter(isPublished);
    publishedDocs.sort((a, b) => (a.date > b.date ? -1 : 1));
    const posts = publishedDocs.slice(0, 10);

    return [
      task(
        "render",
        {
          posts,
          path: "index.html",
        },
        { template: "index" }
      ),
      task(
        "render",
        {
          posts,
          path: "rss.xml",
          urlBase: PROD_SERVER,
        },
        { template: "rss", skipFormatting: true }
      ),
    ];
  },

  tags({ name, index }) {
    const docs = index.docs
      .filter(isPublished)
      .filter((d) => d.meta && d.meta.tags && d.meta.tags.includes(name));
    docs.sort((a, b) => (a.date > b.date ? -1 : 1));
    return [
      task(
        "render",
        {
          posts: docs,
          path: `tag/${slugify(name)}.html`,
          tag: name,
        },
        { template: "tag" }
      ),
    ];
  },
};

const runners = {
  // turn .md into .html
  async render(doc, args = {}, { templates }) {
    console.log(color("[render]", CYAN, BOLD), doc.path);

    let template = templates[doc.template || args.template];
    if (!template) {
      template = templates.default;
      console.warn(`unknown template "${args.template}", using default`);
    }
    let outPath = path.join(CWD, OUTPUT_PATH, doc.path);

    // turn input path into output path
    let outDir = outPath.split(path.sep);
    outDir.pop();
    outDir = outDir.join(path.sep);

    let output = template({ ...doc, cacheBust: Date.now() });
    if (!args.skipFormatting) {
      try {
        output = prettier.format(output, {
          parser: "html",
          jsxBracketSameLine: true,
          htmlWhitespaceSensitivity: "css",
          printWidth: 100,
        });
      } catch (e) {
        console.warn(e);
      }
    }

    // write!
    return fs
      .mkdir(outDir, { recursive: true })
      .then((dir) => fs.writeFile(outPath, output));
  },

  async copy(srcPath, { basePath }) {
    let destPath = path.join(
      CWD,
      OUTPUT_PATH,
      path.relative(basePath, srcPath)
    );

    // turn input path into output path
    let destDir = destPath.split(path.sep);
    destDir.pop();
    destDir = destDir.join(path.sep);

    const shouldCopy = await srcIsNewer(srcPath, destPath);

    if (shouldCopy || FORCE_REBUILD) {
      console.log(color("[copy]", CYAN, BOLD), path.relative(CWD, srcPath));
      return fs
        .mkdir(destDir, { recursive: true })
        .then((dir) => fs.copyFile(srcPath, destPath));
    }
  },
};

// main thing-doer
async function go() {
  const basePath = path.join(CWD, INPUT_PATH);

  // array of "to-do" tasks
  let tasks = [];
  let index = {
    docs: [],
    templates: [],
    partials: [],
    tree: [
      {
        id: "index",
        dependencies: ["templates|index"],
      },
    ],
  };

  checkpoint("indexing");

  let files = await crawl(basePath);
  let oldIndex = [];
  try {
    const oldIndexFile = await fs.readFile("./index.json");
    oldIndex = JSON.parse(oldIndexFile.toString("utf8"));
  } catch (e) {
    console.warn("failed to load old index", e.message);
  }

  // chunk-ily index all the files into documents
  for (let i = 0; i < files.length; i += CHUNK_SIZE) {
    let chunk = [];
    for (let j = 0; j < CHUNK_SIZE && i + j < files.length; j++) {
      let file = files[i + j];
      const ext = path.extname(file).slice(1);
      if (indexers[ext]) {
        let indexTask = indexers[ext](basePath, file, index);
        chunk.push(indexTask);
      } else {
        chunk.push(true);
        tasks.push(task("copy", file, { basePath: basePath }));
      }
    }
    await Promise.all(chunk);
  }

  checkpoint("building tree");

  const tree = index.tree.reduce(
    (o, n) => Object.assign(o, { [n.id]: { affects: [], ...n } }),
    {}
  );
  index.tree.forEach((n) => {
    n.dependencies.forEach((d) => tree[d].affects.push(n.id));
  });
  const templates = index.templates.reduce(
    (o, t) => Object.assign(o, { [t.name]: Handlebars.compile(t.source) }),
    {}
  );

  checkpoint("diffing");

  let changedNodes;
  if (FORCE_REBUILD) {
    changedNodes = index.tree.map((n) => tree[n.id]);
  } else {
    changedNodes = diff(oldIndex.tree, index.tree, "id").map((n) => tree[n.id]);
  }

  const expandChanges = (changes) => {
    return changes
      .map((node) => {
        return node
          ? [
              node.id,
              expandChanges(node.affects.map((n) => tree[n])),
              node.affects.filter((n) => !tree[n]),
            ]
          : [];
      })
      .flat(Infinity);
  };

  const expandedChanges = [...new Set(expandChanges(changedNodes))];

  console.log(
    color("[blog]", WHITE, BOLD) +
      ` ${GREEN}${expandedChanges.length}${RESET} changed items`
  );

  expandedChanges.forEach((change) => {
    const [type, name] = change.split("|");
    let obj;
    if (builders[type]) {
      if (type in index) {
        obj = index[type].find((o) => o.name === name);
      }
      tasks.push(...builders[type]({ name, obj, index }));
    }
  });

  checkpoint("building");

  // flush all async tasks
  tasks = await Promise.all(tasks);
  console.log(
    color("[blog]", WHITE, BOLD),
    `Executing ${GREEN}${tasks.length}${RESET} tasks`
  );

  // turn those tasks into action!
  let active = [];
  for (let i = 0; i < tasks.length; i++) {
    let task = tasks[i];

    let [type, obj, args] = task;
    let pending;
    if (type in runners) {
      pending = runners[type](obj, args, { templates, index });
    } else {
      console.warn(`unknown task type "${type}", skipping`);
    }

    active.push(pending);
    pending.then(() => (active = active.filter((p) => p !== pending)));
    if (active.length >= CHUNK_SIZE) {
      await Promise.any(active);
    }
  }

  await Promise.all(active);

  await fs.writeFile("./index.json", JSON.stringify(index));
}

checkpoint("loaded");

go()
  .then(() => {
    checkpoint(color("done!\n", GREEN, BOLD));
  })
  .catch((e) => console.error(e));