NPM Offline Cache Experiment

npm-offline-cache.js

// Unfortunately this doesn't work as expected (NPM is broken as usual)

const os = require("os");
const fs = require("fs");
const path = require("path");
const { execSync, exec } = require('child_process');
const async = require("async");
const objectScan = require("object-scan");
const get = require("lodash.get");
const difference = require("lodash.difference");
const appRoot = require('app-root-path');


const extract = (cwd) => {
  const deps = JSON.parse(fs.readFileSync(path.join(cwd, "package-lock.json")), "utf8");
  return objectScan(["dependencies.*", "**.dependencies.*"])(deps)
    .map(dep => dep
      .split(/(?<!\\)\./g)
      .map(part => part.replace(/\\\./g, "\.")))
    .reduce((p, c) => {
      const d = `${c[c.length - 1]}@${get(deps, c).version}`;
      return Object.assign(p, {[d]: `${d.replace(/[@\/]/g, "-").replace(/^-+|-+$/g, '')}.tgz`});
    }, {});
};


const pack = (cwd) => {
  const offlineFolder = path.join(cwd, "offline");
  if (!fs.existsSync(offlineFolder)) {
    fs.mkdirSync(offlineFolder);
  }

  const required = extract(cwd);
  console.log(`Required: ${Object.keys(required).length}`);
  const existing = fs.readdirSync(offlineFolder);
  console.log(`Cached: ${existing.length}`);
  const missing = Object.keys(required)
    .filter(dep => existing.indexOf(required[dep]) === -1);
  console.log(`Missing: ${missing.length}`);
  const unnecessary = difference(existing, Object.keys(required).map(dep => required[dep]));
  console.log(`Unnecessary: ${unnecessary.length}`);

  // delete unnecessary cache
  unnecessary
    .map(fileName => path.join(offlineFolder, fileName))
    .forEach(fs.unlinkSync);

  return new Promise(resolve => {
    async.parallelLimit(
      missing
        .map(dep => cb => {
        exec(`cd "${offlineFolder}" && npm pack ${dep}`, (err, r) => {
          if (err !== null) {
            console.log(err);
            throw err;
          }
          const fileName = r.trim();
          if (fileName !== required[dep]) {
            console.log(`Unexpected File Name: ${fileName} vs ${required[dep]}`);
            throw new Error(`Unexpected File Name: ${fileName} vs ${required[dep]}`);
          }
          console.log("packed " + fileName);
          cb();
        })
      }),
      os.cpus().length,
      (err) => {
        console.log(err);
        resolve(err);
      }
    )
  });
};

const loadCache = (cwd) => {
  execSync("npm cache clean --force");
  execSync("rm -rf node_modules");
  const offlineFolder = path.join(cwd, "offline");
  async.parallelLimit(
    fs.readdirSync(offlineFolder)
      .map(dep => `${path.join(offlineFolder, dep)}`)
      .map(path => (cb) => {
        const cmd = `npm cache add ${path}`;
        console.log(cmd);
        exec(cmd, cb);
      }),
    os.cpus().length,
    (err) => {
      console.log(err);
      execSync(`npm i --offline`);
    }
  );
};

pack(appRoot.path).then(() => {
  loadCache(appRoot.path);
});