differential bundle loading

bundleParts.js

// @ts-check
import '@endo/init';
import { createHash } from 'crypto';
import { ZipReader } from '@endo/zip';

/**
 * @file What files in a bundle are not already in already loaded bundles?
 *
 * For each b1-* style bundle in the bundles table of a swingstore,
 * load only the files that are not already in a loaded bundle.
 *
 * See also:
 *
 * share Zoe/ERTP libraries among contracts #2391
 * https://github.com/Agoric/agoric-sdk/issues/2391
 */

/**
 * SQL short-hand
 *
 * @param {import('better-sqlite3').Database} db
 */
export const dbTool = db => {
  const prepare = (strings, ...params) => {
    const dml = strings.join('?');
    return { stmt: db.prepare(dml), params };
  };
  const sql = (strings, ...args) => {
    const { stmt, params } = prepare(strings, ...args);
    return stmt.all(...params);
  };
  sql.get = (strings, ...args) => {
    const { stmt, params } = prepare(strings, ...args);
    return stmt.get(...params);
  };
  sql.run = (strings, ...args) => {
    const { stmt, params } = prepare(strings, ...args);
    stmt.run(...params);
  };
  return sql;
};

/** @param {Uint8Array} bytes */
const computeSha512 = bytes => {
  const hash = createHash('sha512');
  hash.update(bytes);
  return hash.digest().toString('hex');
};

/** @param {ReturnType<typeof dbTool>} sql */
const bundleLoader = sql => {
  const createTables = async () => {
    sql.run`create table if not exists bundleFiles
      (name text, sha512 text primary key, content blob, info text)`;
    sql.run`create temp table if not exists bundleTmp
     (name text, sha512 text primary key, content blob, info text)`;
  };

  /** @param {ZipReader} zipRd */
  const saveNewFiles = async zipRd => {
    const files = { new: 0, total: zipRd.files.size };
    const bytes = { new: 0, total: 0 };
    for (const [name, { content }] of zipRd.files.entries()) {
      bytes.total += content.length;
      const sha512 = computeSha512(content);
      const {
        qty,
      } = sql.get`select count(*) qty from bundleFiles where sha512 = ${sha512}`;
      if (qty) continue;
      files.new += 1;
      bytes.new += content.length;
      sql.run`insert into bundleFiles (name, sha512, content) values (${name}, ${sha512}, ${content})`;
      console.log('added', name, content.length);
    }
    files.pct = Math.round((files.new / files.total) * 10000) / 100;
    bytes.pct = Math.round((bytes.new / bytes.total) * 10000) / 100;
    return { files, bytes };
  };

  return { createTables, saveNewFiles };
};

/** @param {ZipReader} zipRd */
const getEntry = async zipRd => {
  const de = new TextDecoder();
  const cmapBytes = zipRd.files.get('compartment-map.json').content;
  const cmap = JSON.parse(de.decode(cmapBytes));
  return cmap.entry;
};

/** @param {string[]} argv */
const main = async (argv, { betterSqlite3 }) => {
  const { default: Database } = betterSqlite3;

  const [_node, _script, dbFn] = argv;
  if (!dbFn) throw Error('missing dbFn');
  console.log(Database);
  const db = new Database(dbFn);
  const sql = dbTool(db);

  const loader = bundleLoader(sql);
  await loader.createTables();
  sql.run`delete from bundleFiles`; // @@@@

  const de = new TextDecoder();
  const loaded = sql`select * from bundles`;
  for (const { bundleID, bundle } of loaded) {
    if (!bundleID.startsWith('b1-')) continue;
    const zipBuf = bundle;
    const zipRd = new ZipReader(zipBuf);
    const entry = await getEntry(zipRd);
    console.log(entry, bundleID);

    const stats = await loader.saveNewFiles(zipRd);
    console.log(entry, stats);
  }
};

main([...process.argv], {
  betterSqlite3: await import('better-sqlite3'),
}).catch(err => {
  console.error(err);
  process.exit(1);
});

summary.csv

summary_query.sql

with agg as (
select skip, entry, count(*) files, sum(size) bytes from bundleDetail group by entry, skip
)
, noskip as (select * from agg where skip = 0)
, skip as (select * from agg where skip = 1)
select noskip.entry, noskip.files, noskip.files + skip.files files_tot
     , noskip.bytes, cmap_size, noskip.bytes + skip.bytes bytes_tot
     , 100.0 * skip.bytes / (noskip.bytes + skip.bytes) pct_saved
from noskip
join skip on noskip.entry = skip.entry
join (select entry, size cmap_size from bundleDetail where name = 'compartment-map.json') c on c.entry = noskip.entry
order by pct_saved
;