Fastly Cache Warmer

fastly-cache-warmer.js

'use strict';

['START_URL', 'PATH_REGEX'].forEach((env) => {
    if (!(env in process.env)) {
        console.error(`The '${env}' environment variable is missing.`);
        process.exit(1);
    }
});

let SimpleCrawler = require('simplecrawler');
let crawler = new SimpleCrawler(process.env.START_URL);
let padString = (n, c, p = 6) => (Array(p).join(c) + n).substr(('' + n).length);
let hitOrMissSummary = { HIT: 0, MISS: 0 };

crawler.on('fetchcomplete', (queueItem, _responseBuffer, response) => {
    crawler.queue.countItems({ fetched: true }, (err, count) => {
        if (err) {
            throw err;
        }

        crawler.queue.getLength((err, length) => {
            if (err) {
                throw err;
            }

            let hitOrMiss = response.headers['x-cache'];
            hitOrMissSummary[hitOrMiss] += 1;

            console.log(
                '(%s/%s) X-Cache: %s, Status: %d, Path: %s',
                padString(count, '0'),
                padString(length, '0'),
                padString(hitOrMiss, ' ', 5),
                queueItem.stateData.code,
                queueItem.path
            );
        });
    });
});

crawler.addFetchCondition((item) => {
    return item.path.match(new RegExp(process.env.PATH_REGEX));
});

crawler.on('complete', () => {
    console.log(hitOrMissSummary);
});

crawler.start();

package.json

{
  "name": "fastly-cache-warmer",
  "version": "0.0.1",
  "author": "Tom Cant <tomcant@gmail.com>",
  "dependencies": {
    "simplecrawler": "^1.1.8"
  }
}

START_URL=https://example.com/ PATH_REGEX=.* node index.js

@tomcant
Any solution to warm graphql queries?
Example:
START_URL=https://example.com/graphql?query=%7B%20products\(%20filter%3A%20%7Bcategory_id%3A%20%7Beq%3A%20%22225%22%7D%20targetsegment%3A%20%7Beq%3A%20%2212515%22%7D%20%7D%20sort%3A%20%7B%20price%3A%20DESC%20%7D%20pageSize%3A%201%20currentPage%3A%201%20\)%20%7B%20total_count%20aggregations%20%7B%20attribute_code%20label%20%7D%20items%20%7B%20name%20%7D%20%7D%20%7D