On-disk simple node.js cache function using fs promises, for things like calling an external API and re-using its data for multiple pages in a Next.js getStaticProps call

cachalot-v2-prototype.ts

/* eslint-disable no-await-in-loop */
import * as fs from 'fs/promises';
import * as path from 'path';

const checkIfExist = async (p: string) => fs.access(p).then(() => true).catch(() => false);

function msg(...args: any[]) {
  console.info('🐋:', ...args);
}

function error(...args: any[]) {
  console.error('🐋❗️:', ...args);
}

async function getCachePath(): Promise<string | null> {
  const cachalotDirName = '.cachalot';

  // loop parent directories until we find a .git directory or package.json
  let dir = process.cwd();
  let err = null;

  while (dir !== '/' || err !== null) {
    const gitDir = path.join(dir, '.git');
    const packageJson = path.join(dir, 'package.json');
    const cachalotDir = path.join(dir, cachalotDirName);
    if (await checkIfExist(gitDir) || await checkIfExist(packageJson)) {
      if (!await checkIfExist(cachalotDir)) {
        try {
          await fs.mkdir(cachalotDir);
        } catch (e) {
          err = e;
        }
      }
      return cachalotDir;
    }
    dir = dir.substring(0, dir.lastIndexOf('/'));
  }

  // if we have an error, try to create a cache in /tmp
  if (err) {
    const tmpDir = '/tmp';
    try {
      if (await checkIfExist(tmpDir)) {
        const cachalotDir = path.join(tmpDir, cachalotDirName);
        if (!await checkIfExist(cachalotDir)) {
          await fs.mkdir(cachalotDir);
        }

        return cachalotDir;
      }
    } catch (e) {
      error('could not create tmp cache directory', e);
      return null;
    }
  }

  return null;
}

/**
 * Cache the result of a function for a given TTL in seconds
 * @param name - name of the cache
 * @param ttl - time to live in seconds
 * @param fn - function to cache
 * @returns {Promise<any>}
 */
export default async function cachalot(
  name: string,
  ttl: number,
  fn: () => Promise<any>,
): Promise<any> {
  const cachePath = await getCachePath();
  if (!cachePath) {
    return fn();
  }
  const cacheFile = path.join(cachePath, `${name}.json`);

  const runAndWrite = async () => fs.writeFile(cacheFile, '{}', 'utf8').then(() => fn())
    .then((newValue) => {
      fs.writeFile(cacheFile, JSON.stringify({ timestamp: Date.now(), value: newValue }));
      return newValue;
    }).catch((e) => {
      error('error when writing cache', e);
      return null;
    });

  const attemptRead = () => fs.readFile(cacheFile, 'utf8')
    .then((data) => {
      let timestamp;
      let value;
      try {
        const parsedData = JSON.parse(data);
        timestamp = parsedData.timestamp;
        value = parsedData.value;

        if (!timestamp || !value) {
          return Promise.reject(new Error('Empty cache file'));
        }
      } catch (e) {
        return Promise.reject(new Error('Invalid cache file format'));
      }

      if (Date.now() - timestamp < (ttl * 1000)) {
        return value;
      }
      return fs.rm(cacheFile)
        .then(() => Promise.reject(new Error('Cache expired')))
        .catch(() => Promise.reject(new Error('Cache removal failure')));
    });

  return attemptRead().catch(async (e1) => {
    // randomize a wait time 0-100ms to avoid collisions
    await new Promise((resolve) => { setTimeout(resolve, Math.floor(Math.random() * 100)); });
    if (await checkIfExist(cacheFile)) {
      // wait for 1000ms and then try again
      msg(name, 'cache miss', e1.message, 'but file exists, waiting for 1000ms and retrying');
      await new Promise((resolve) => { setTimeout(resolve, 1000); });
      return attemptRead().then(() => {
        msg(name, 'cache hit after waiting');
      }).catch((e2) => {
        msg(name, 'cache waiting timeout, running & writing', e2 && e2.message);
        runAndWrite();
      });
    }
    msg(name, 'cache miss, running & writing');
    return runAndWrite();
  });
}

cachalot.ts

/*

On-disk simple node.js cache function using fs promises
For things like calling an external API and re-using its data for multiple pages in a Next.js getStaticProps call

*/
/* eslint-disable no-await-in-loop */
import * as fs from 'fs/promises';
import * as path from 'path';

const checkIfExist = async (p: string) => fs.access(p).then(() => true).catch(() => false);

async function getCachePath(): Promise<string | null> {
  // loop parent directories until we find a .git directory or package.json
  let dir = process.cwd();
  const cachalotDirName = '.cachalot';
  while (dir !== '/') {
    const gitDir = path.join(dir, '.git');
    const packageJson = path.join(dir, 'package.json');
    const cachalotDir = path.join(dir, cachalotDirName);
    if (await checkIfExist(gitDir) || await checkIfExist(packageJson)) {
      if (!await checkIfExist(cachalotDir)) {
        fs.mkdir(cachalotDir);
      }
      return cachalotDir;
    }
    dir = dir.substring(0, dir.lastIndexOf('/'));
  }
  return null;
}

/**
 * Cache the result of a function for a given TTL in seconds
 * @param name - name of the cache
 * @param ttl - time to live in seconds
 * @param fn - function to cache
 * @returns {Promise<any>}
 */
export default async function cachalot(
  name: string,
  ttl: number,
  fn: () => Promise<any>,
): Promise<any> {
  const cachePath = await getCachePath();
  if (!cachePath) {
    return fn();
  }
  const cacheFile = path.join(cachePath, `${name}.json`);
  return fs.readFile(cacheFile, 'utf8')
    .then((data) => {
      const { timestamp, value } = JSON.parse(data);
      if (Date.now() - timestamp < (ttl * 1000)) {
        return value;
      }
      return fn().then((newValue) => {
        fs.writeFile(cacheFile, JSON.stringify({ timestamp: Date.now(), value: newValue }));
        return newValue;
      });
    }).catch(() => fn().then((newValue) => {
      fs.writeFile(cacheFile, JSON.stringify({ timestamp: Date.now(), value: newValue }));
      return newValue;
    }));
}