Skip to content

Instantly share code, notes, and snippets.

@potch
Created December 12, 2023 05:10
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save potch/d22da24165c811a9ef90973d48185dd3 to your computer and use it in GitHub Desktop.
Save potch/d22da24165c811a9ef90973d48185dd3 to your computer and use it in GitHub Desktop.
potch.me site generator 2023
// if one file is too unwieldy you probably shouldn't have rolled your own blog yet again, POTCH
const start = Date.now();
const fs = require("fs").promises;
const path = require("path");
const crypto = require("crypto");
const Handlebars = require("handlebars");
const markdown = require("markdown-it");
const frontmatter = require("front-matter");
const prism = require("markdown-it-prism");
const mdfm = require("markdown-it-front-matter");
const mdContainer = require("markdown-it-container");
const prettier = require("prettier");
const RESET = "\x1b[0m";
const RED = "\x1b[;31m";
const BOLD = "\x1b[1m";
const GREEN = "\x1b[;32m";
const YELLOW = "\x1b[;33m";
const BLUE = "\x1b[;34m";
const MAGENTA = "\x1b[;35m";
const CYAN = "\x1b[;36m";
const WHITE = "\x1b[;37m";
const color = (text, ...colors) => `${colors.join("")}${text}${RESET}`;
// async means we can do more stuff while waiting for IO!
const CHUNK_SIZE = 8;
// basic settings
const OUTPUT_PATH = "build";
const INPUT_PATH = "src";
const PROD_SERVER = "https://potch.me";
// command line arguments
const CWD = process.cwd();
const argv = process.argv;
let changeEvent;
if (argv.includes("--changed")) {
changeEvent = argv[argv.indexOf("--changed") + 1];
console.log(color("[blog]", WHITE, BOLD) + " onchange:", changeEvent);
}
const FORCE_REBUILD = argv.includes("--force") || changeEvent === "blog.js";
if (FORCE_REBUILD)
console.log(color("[blog]", WHITE, BOLD) + " doing full rebuild");
let lastCheckpoint = start;
const checkpoint = (label) => {
console.log(
`${YELLOW}[${Date.now() - start}ms +${
Date.now() - lastCheckpoint
}ms]${RESET} ${label}`
);
lastCheckpoint = Date.now();
};
const task = async (name, obj, args = {}) => [name, await obj, args];
const isPublished = (d) => (d.meta ? d.meta.published : true);
const slugify = (s) => s.replace(/\s+/g, "-");
const md = new markdown({ html: true, langPrefix: "pretty-code language-" });
// we need to recognize the frontmatter but md can ignore it
md.use(mdfm, function noop() {});
md.use(prism);
md.use(mdContainer, "wide");
md.use(mdContainer, "fullbg");
md.use(mdContainer, "fullpad");
md.use(mdContainer, "full");
md.use(mdContainer, "centered");
// simple 'x days ago' thingus
Handlebars.registerHelper({
iso: function iso(d) {
return new Date(d).toISOString();
},
isoDate: function iso(d) {
return new Date(d).toISOString().split("T")[0];
},
ago: function ago(d) {
d = Date.parse(d);
if (!d) return "";
d = ((Date.now() - d) / 1000) | 0;
if (d < 60) return "seconds";
// Minutes
if (d < 120) return "a minute";
if (d < 3600) return ((d / 60) | 0) + " minutes";
// Hours
d = (d / 3600) | 0;
if (d < 2) return "an hour";
if (d < 24) return d + " hours";
// Days
d = (d / 24) | 0;
if (d < 2) return "a day";
if (d < 30) return d + " days";
// Months
if (d < 60) return "a month";
if (d < 360 * 1.5) return ((d / 30) | 0) + " months";
// Years
if (d < 365 * 2) return "a year and a half";
return Math.round(d / 365) + " years";
},
slugify,
});
const hashContent = (s) => {
const hash = crypto.createHash("sha256");
hash.setEncoding("hex");
hash.write(s);
hash.end();
return hash.read();
};
const loadTemplate = async (path) => (await fs.readFile(path)).toString("utf8");
// get all files in a path recursively
async function crawl(crawlPath, { exclude = [] } = {}) {
let files = [];
exclude = new Set(exclude);
// stack based recursion works nicely with async/await
// a symlink loop will wreck me oh well
let toCrawl = [crawlPath];
while (toCrawl.length) {
let crawlPath = toCrawl.pop();
let dirFiles = await fs.readdir(crawlPath, { withFileTypes: true });
dirFiles
.filter((f) => !exclude.has(f.name) && !f.name.startsWith("."))
.forEach((f) => {
const p = path.join(crawlPath, f.name);
if (f.isDirectory()) {
toCrawl.push(p);
} else {
files.push(p);
}
});
}
return files;
}
async function srcIsNewer(srcPath, destPath) {
return Promise.allSettled([fs.stat(srcPath), fs.stat(destPath)]).then(
([a, b]) => b.status !== "fulfilled" || a.value.mtime > b.value.mtime
);
}
// deep compare objects for triple equality
function deepEQ(a, b) {
if (a instanceof Date || b instanceof Date) {
return Date(a).valueOf() === Date(b).valueOf();
}
if (typeof a !== typeof b) {
return false;
}
if (typeof a === "object") {
for (let k of Object.keys(a)) {
if (!deepEQ(a[k], b[k])) {
return false;
}
}
return true;
}
if (a === b) {
return true;
}
return false;
}
function diff(from = [], to = [], key) {
const out = [];
to.forEach((b) => {
let a = from.find((d) => d[key] === b[key]);
if (b.hash && a) {
if (a.hash !== b.hash) {
out.push(b);
}
} else {
if (!deepEQ(a, b)) {
out.push(b);
}
}
});
return out;
}
// turn a file into a document with all sorts of metadata
const indexers = {
async md(basePath, filePath, index) {
let raw = (await fs.readFile(filePath)).toString("utf8");
let stats = await fs.stat(filePath);
// parse the frontmatter. frontmatter may be the one good use of yaml
let meta = frontmatter(raw).attributes;
// allow for lazy authoring by extracting post titles and blurbs from markdown
let tokens = md.parse(raw, {});
let titlePosition;
for (let i = 0; i < tokens.length; i++) {
let token = tokens[i];
// extract the first <h1> as the title to put it at the top
if (!meta.title && token.type === "heading_open" && token.tag === "h1") {
meta.title = tokens[i + 1].content;
titlePosition = token.map;
}
// use the first paragraph as a blurb if none exists
if (
!meta.blurb &&
meta.blurb !== false &&
token.type === "paragraph_open"
) {
meta.blurb = md.renderInline(tokens[i + 1].content);
}
}
// strip title from render
let toContent = raw.split("\n");
if (titlePosition) {
toContent.splice(titlePosition[0], titlePosition[1] - titlePosition[0]);
}
toContent = toContent.join("\n");
let docPath = path.relative(basePath, filePath);
docPath = path.join(
path.dirname(docPath),
path.basename(docPath, path.extname(docPath)) + ".html"
);
const route = docPath.endsWith("/index.html")
? docPath.replace(/\/index\.html$/, "/")
: docPath;
const template = meta.template || "default";
const tags = meta.tags || [];
// document object
let doc = {
type: "md",
content: md.render(toContent),
path: docPath,
name: docPath,
route,
meta,
raw,
changed: stats.mtime,
created: stats.birthtime,
template,
node: {
id: "docs|" + docPath,
hash: hashContent(raw),
dependencies: ["templates|" + template],
affects: ["index", ...tags.map((t) => "tags|" + t)],
},
};
if (meta.date) {
doc.date = meta.date;
} else {
doc.date = doc.created;
}
index.docs.push(doc);
index.tree.push(doc.node);
tags.forEach((tag) => {
index.tree.push({
id: "tags|" + tag,
dependencies: ["templates|tag"],
});
});
},
async hbs(basePath, filePath, index) {
const source = await loadTemplate(filePath);
let name = path.basename(filePath, ".hbs");
if (path.extname(name) === ".partial") {
name = path.basename(name, ".partial");
const doc = {
name,
node: {
id: "partials|" + name,
hash: hashContent(source),
dependencies: [],
},
source,
};
index.partials.push(doc);
index.tree.push(doc.node);
Handlebars.registerPartial(name, source);
} else {
const source = await loadTemplate(filePath);
const name = path.basename(filePath, ".hbs");
const partials = (source.match(/\{\{>([^\}]+)\}\}/g) || []).map((p) =>
p.match(/\{\{>([^\}]+)\}\}/)[1].trim()
);
const doc = {
name,
node: {
id: "templates|" + name,
hash: hashContent(source),
dependencies: partials.map((p) => "partials|" + p),
},
source,
};
index.templates.push(doc);
index.tree.push(doc.node);
}
},
};
const builders = {
docs({ obj }) {
return [task("render", obj)];
},
index({ index }) {
const publishedDocs = index.docs.filter(isPublished);
publishedDocs.sort((a, b) => (a.date > b.date ? -1 : 1));
const posts = publishedDocs.slice(0, 10);
return [
task(
"render",
{
posts,
path: "index.html",
},
{ template: "index" }
),
task(
"render",
{
posts,
path: "rss.xml",
urlBase: PROD_SERVER,
},
{ template: "rss", skipFormatting: true }
),
];
},
tags({ name, index }) {
const docs = index.docs
.filter(isPublished)
.filter((d) => d.meta && d.meta.tags && d.meta.tags.includes(name));
docs.sort((a, b) => (a.date > b.date ? -1 : 1));
return [
task(
"render",
{
posts: docs,
path: `tag/${slugify(name)}.html`,
tag: name,
},
{ template: "tag" }
),
];
},
};
const runners = {
// turn .md into .html
async render(doc, args = {}, { templates }) {
console.log(color("[render]", CYAN, BOLD), doc.path);
let template = templates[doc.template || args.template];
if (!template) {
template = templates.default;
console.warn(`unknown template "${args.template}", using default`);
}
let outPath = path.join(CWD, OUTPUT_PATH, doc.path);
// turn input path into output path
let outDir = outPath.split(path.sep);
outDir.pop();
outDir = outDir.join(path.sep);
let output = template({ ...doc, cacheBust: Date.now() });
if (!args.skipFormatting) {
try {
output = prettier.format(output, {
parser: "html",
jsxBracketSameLine: true,
htmlWhitespaceSensitivity: "css",
printWidth: 100,
});
} catch (e) {
console.warn(e);
}
}
// write!
return fs
.mkdir(outDir, { recursive: true })
.then((dir) => fs.writeFile(outPath, output));
},
async copy(srcPath, { basePath }) {
let destPath = path.join(
CWD,
OUTPUT_PATH,
path.relative(basePath, srcPath)
);
// turn input path into output path
let destDir = destPath.split(path.sep);
destDir.pop();
destDir = destDir.join(path.sep);
const shouldCopy = await srcIsNewer(srcPath, destPath);
if (shouldCopy || FORCE_REBUILD) {
console.log(color("[copy]", CYAN, BOLD), path.relative(CWD, srcPath));
return fs
.mkdir(destDir, { recursive: true })
.then((dir) => fs.copyFile(srcPath, destPath));
}
},
};
// main thing-doer
async function go() {
const basePath = path.join(CWD, INPUT_PATH);
// array of "to-do" tasks
let tasks = [];
let index = {
docs: [],
templates: [],
partials: [],
tree: [
{
id: "index",
dependencies: ["templates|index"],
},
],
};
checkpoint("indexing");
let files = await crawl(basePath);
let oldIndex = [];
try {
const oldIndexFile = await fs.readFile("./index.json");
oldIndex = JSON.parse(oldIndexFile.toString("utf8"));
} catch (e) {
console.warn("failed to load old index", e.message);
}
// chunk-ily index all the files into documents
for (let i = 0; i < files.length; i += CHUNK_SIZE) {
let chunk = [];
for (let j = 0; j < CHUNK_SIZE && i + j < files.length; j++) {
let file = files[i + j];
const ext = path.extname(file).slice(1);
if (indexers[ext]) {
let indexTask = indexers[ext](basePath, file, index);
chunk.push(indexTask);
} else {
chunk.push(true);
tasks.push(task("copy", file, { basePath: basePath }));
}
}
await Promise.all(chunk);
}
checkpoint("building tree");
const tree = index.tree.reduce(
(o, n) => Object.assign(o, { [n.id]: { affects: [], ...n } }),
{}
);
index.tree.forEach((n) => {
n.dependencies.forEach((d) => tree[d].affects.push(n.id));
});
const templates = index.templates.reduce(
(o, t) => Object.assign(o, { [t.name]: Handlebars.compile(t.source) }),
{}
);
checkpoint("diffing");
let changedNodes;
if (FORCE_REBUILD) {
changedNodes = index.tree.map((n) => tree[n.id]);
} else {
changedNodes = diff(oldIndex.tree, index.tree, "id").map((n) => tree[n.id]);
}
const expandChanges = (changes) => {
return changes
.map((node) => {
return node
? [
node.id,
expandChanges(node.affects.map((n) => tree[n])),
node.affects.filter((n) => !tree[n]),
]
: [];
})
.flat(Infinity);
};
const expandedChanges = [...new Set(expandChanges(changedNodes))];
console.log(
color("[blog]", WHITE, BOLD) +
` ${GREEN}${expandedChanges.length}${RESET} changed items`
);
expandedChanges.forEach((change) => {
const [type, name] = change.split("|");
let obj;
if (builders[type]) {
if (type in index) {
obj = index[type].find((o) => o.name === name);
}
tasks.push(...builders[type]({ name, obj, index }));
}
});
checkpoint("building");
// flush all async tasks
tasks = await Promise.all(tasks);
console.log(
color("[blog]", WHITE, BOLD),
`Executing ${GREEN}${tasks.length}${RESET} tasks`
);
// turn those tasks into action!
let active = [];
for (let i = 0; i < tasks.length; i++) {
let task = tasks[i];
let [type, obj, args] = task;
let pending;
if (type in runners) {
pending = runners[type](obj, args, { templates, index });
} else {
console.warn(`unknown task type "${type}", skipping`);
}
active.push(pending);
pending.then(() => (active = active.filter((p) => p !== pending)));
if (active.length >= CHUNK_SIZE) {
await Promise.any(active);
}
}
await Promise.all(active);
await fs.writeFile("./index.json", JSON.stringify(index));
}
checkpoint("loaded");
go()
.then(() => {
checkpoint(color("done!\n", GREEN, BOLD));
})
.catch((e) => console.error(e));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment