Miniflare Garbage Collection Script

gc.mjs

import fs from "node:fs";
import path from "node:path";
import { Miniflare, sanitisePath } from "miniflare";

async function main() {
  // Extract and validate command line args
  const argv = process.argv.slice(2);
  if (argv.length < 2 || (argv[0] !== "kv" && argv[0] !== "cache")) {
    console.log(`usage: node gc.mjs kv,cache [persist] [namespaces..]`);
    process.exitCode = 1;
    return;
  }

  const isCache = argv[0] === "cache";
  const uniqueKey = isCache
    ? "miniflare-CacheObject"
    : "miniflare-KVNamespaceObject";
  const persist = argv[1];
  const namespaces = argv.slice(2);

  if (isCache) {
    const invalidNamespaces = namespaces
      .filter((name) => name !== "default" && !name.startsWith("named:"))
      .join(", ");
    if (invalidNamespaces.length > 0) {
      console.log(
        `Cache namespaces must be either "default" or start with "named:", got ${invalidNamespaces}`
      );
      process.exitCode = 1;
      return;
    }
  }

  // Initialise Miniflare as an SQLite client to get referenced blobs
  const mf = new Miniflare({
    modules: true,
    script: `export class BlobGetterObject {
      constructor(state) { this.state = state; }
      fetch() { return Response.json(Array.from(this.state.storage.sql.prepare("SELECT blob_id FROM _mf_entries")())); }
    }`,
    compatibilityFlags: ["experimental"], // Required to use `storage.sql`
    durableObjects: {
      BLOB_GETTER: {
        className: "BlobGetterObject",
        // Make this Durable Object behave like Miniflare's internal objects
        unsafeUniqueKey: uniqueKey,
      },
    },
    durableObjectsPersist: persist,
    cache: false, // Disable built-in cache internal object
  });

  const BLOB_GETTER = await mf.getDurableObjectNamespace("BLOB_GETTER");
  for (const namespace of namespaces) {
    // Make sure the namespace and database exist. Checking the database exists
    // is especially important as otherwise we'd end up with no referenced blobs
    // and delete everything.
    const sanitisedNamespace = sanitisePath(namespace);
    const blobPath = path.join(persist, sanitisedNamespace, "blobs");
    if (!fs.existsSync(blobPath)) {
      console.log(
        `Unable to find ${namespace}'s blobs in ${persist}, skipping...`
      );
      continue;
    }
    const id = BLOB_GETTER.idFromName(namespace);
    const dbPath = path.join(persist, uniqueKey, `${id}.sqlite`);
    if (!fs.existsSync(dbPath)) {
      console.log(
        `Unable to find ${namespace}'s database in ${persist}, skipping...`
      );
      continue;
    }

    // Get blobs referenced by this namespace
    const stub = BLOB_GETTER.get(id);
    const res = await stub.fetch("http://placeholder");
    const referencedBlobs = new Set(
      (await res.json()).map(({ blob_id }) => blob_id)
    );

    // Garbage collect all unreferenced blobs
    let count = 0;
    for (const blobId of fs.readdirSync(blobPath)) {
      if (!referencedBlobs.has(blobId)) {
        count++;
        fs.unlinkSync(path.join(blobPath, blobId));
      }
    }
    console.log(`Garbage collected ${count} blob(s) from ${namespace}`);
  }

  await mf.dispose();
}

await main();