Prevent cache stampedes by tracking pending requests

CacheStampedeAvoider.js

/**
 * Example cache class that uses promises to resolve.
 */
class Cache {
  constructor() {
    this.store = {};
  }
  async get(key) {
    return this.store[key];
  }

  async set(key, value) {
    return this.store[key] = value;
  }
}

/**
 * Very simple cache stampede avoidance. By implementing as a class we can have multiple
 * of these in play for different resources.
 */
class StampedeAvoider {
  constructor(cache, factory) {
    this.cache = cache;
    this.factory = factory;
    this.hits = 0;
    this.misses = 0;
    this.fetches = 0;
    this.pending = {};
  }

  async set(key, value) {
    return await this.cache.set(key, value);
  }

  async get(key, value) {
    /** If this is a cache hit we're already done. */
    const entry = await this.cache.get(key);
    if (entry !== undefined) { // sanity check required
      this.hits++;
      return entry;
    }

    /**
     * It may make some sense to do this check first to reduce round trips to the cache,
     * but in doing so, awaiting the cache response allows a window of time where a stampede could still occur.
     * so instead we do this check only if we know we need a cache update.
     */
    if (this.pending[key]) {
      this.misses++;
      return this.pending[key]; // We have a pending request, return the promise to wait
    }

    /** If we get here we've got a cache miss and no pending requests yet. */
    this.misses++;
    this.fetches++;

    /**
     * Create a new pending promise, that resolves when the factory is complete.
     * Since we're putting the same value in the cache as we're returning, we
     * don't have to wait for the cache write to start resolving requests.
     */
    try {
      this.pending[key] = this.factory(key); // Don't await here, just set the promise      
      const result = await this.pending[key]; // Now we wait, other requests can arrive during this time.
      /**
       * Unless we threw an error, we'll put the cache entry away.
       * If a request arrives during this window, we still have a resolved promise in pending that can fire.
       */
      await this.set(key, result);
      return result;
    } finally {
      /**
       * Regardless of error or completion, we can now delete the pending promise.
       * If we completed succesfully the cache is now set, so further requests shouldn't
       * need the pending promise.
       */
      delete this.pending[key]; // Cache should be set by now.
    }

  }

}

// Example setup
const cache = new Cache();
const stampedeAvoider = new StampedeAvoider(cache, async function(key) {
  /** The factory function here returns the final value we expect to cache */
  console.log(`Simulating expensive fetch for key ${key}`);
  return Math.random();
});

// Test
async function test() {
  const a = await stampedeAvoider.get(42);
  const b = stampedeAvoider.get('test');
  const c = stampedeAvoider.get('test');
  await c;
  const d = stampedeAvoider.get('test');

  await Promise.all([a, b, c, d]);

  console.log(`Hits: ${stampedeAvoider.hits} Misses: ${stampedeAvoider.misses} Fetches: ${stampedeAvoider.fetches}`);
  console.log(`${a} ${await b} ${await c} ${await d}`);
}

test();