kerns/README.md

## README.md

      
    Raw
  

              README.md
            
          
    Warmer

I needed this for priming some image-heavy pages that were slow to cold-load after a cleared cache.
It's ideal for pages with some form of lazyloading / async content pulled from a CDN or a CMS where the first request can take an unacceptably long time to fulfill.
It also generates full page screenshots which can be used in QA or rudimentary visual regression testing.
Setup


npm i puppeteer yargs chalk-animation
Create a file accessible over the network containing a simple list of URLs to warm.
Put make-warm.js and warm.sh together in a folder and make sure they are executable.

Usage

The script needs one argument (the network location of the file we want to fetch). You can optionally add a username and password if your site is behind authentication.
./warm [http://location/of/cache-data.whatevs] [optional username] [optional password]
Help

Ideas improvements bugs 👉 @kerns

  
## cache-data.twig
{# Creating a list of URLs to Warm

This example uses Twig + Craft, because Craft 👌

You'll need to set this up individually based on your CMS and how it is configured.
Don't take anything below for granted. Your output just needs to generate a file in
the following (very simple) format – one link per new line!:

---
http://some-urls-you-need-warmed.com/some-entry
http://some-urls-you-need-warmed.com/some-other-entry
http://some-urls-you-need-warmed.com/another-entry
---

#}
{% spaceless %}

{# Setup our Craft entry query #}
{% set availableCars = craft.entries().section('autos').availability("not sold") %}

{# Loop through our entries and spit out a list of URLs to warm. #}
{% for entry in availableCars %}
{{entry.url}}
{% endfor %}
{# Array of additional urls to warm. siteUrl is prepended and includes a trailing /, so the first (blank) entry is the homepage. #}
{% set array = [
                "",
                "showroom",
                "contact",
              ] %}
{% for item in array %}
{{siteUrl ~ item}}
{% endfor %}
{% endspaceless %}

## make-warm.js
const fs = require("fs"); // https://github.com/nodejs/node/blob/master/doc/api/fs.md
const puppeteer = require("puppeteer"); // https://github.com/puppeteer/puppeteer
const chalkAnimation = require("chalk-animation"); // https://github.com/bokub/chalk-animation
const { argv } = require("yargs"); // https://github.com/yargs/yargs

// Function for parsing data from our list of urls to visit.
function readURLFile(path) {
  return fs
    .readFileSync(path, "utf-8")
    .split("\n")
    .map((elt) => {
      const url = elt.replace("\r", "");
      return `${url.toLowerCase()}`;
    });
}

// Instantiate our browser and start hitting urls.
(async () => {
  const startDate = new Date().getTime();
  const USER_AGENT =
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3239.108 Safari/537.36";
  const urls = readURLFile("./cache-data.twig"); // <- List of urls, warm.sh fetches this using wget from the file you set up.
  const browser = await puppeteer.launch();

  for (let url of urls) {
    const warmingAnimation = chalkAnimation.karaoke(`🌞 Warming: ${url}`);

    let page = await browser.newPage();
    await page.setUserAgent(USER_AGENT);

    try {
      // If user and pass arguments passed, submit authentication credentials
      if (argv.user && argv.pass) {
        // We need to authenticate
        await page.authenticate({ username: argv.user, password: argv.pass });
      }

      await page.goto(url, { waitUntil: "networkidle2" }); // Wait for network traffic to stop.

      // Optional step if you have a modal or some GDPR consent that needs to be dismissed on the first or every page.
      try {
        await page.click("#elc-accept-all-link"); // <- Change this to the ID of the button or element that needs to be clicked to accept/dismiss cookie consent. etc
        // console.log("🍪 Accepting Cookies\n");
      } catch (error) {
        // console.log("🤭 Cookies Already Accepted\n")
      }

      await autoScroll(page); // Get to the bottom of it.
      await page.waitFor(2000, { waitUntil: "networkidle2" }); // Be like, super sure everything is loaded and css transitions have run their course.

      warmingAnimation.stop(); // Stop the warming animation
      warmingAnimation.f = `🌞 Warming: ${url}`.length + 10; // Set the frame manually to [string length + 10] (this is a hack, thank you @bokub)
      warmingAnimation.render(); // Render one last frame
      chalkAnimation.rainbow(`😎 Success\n`);

      // Set the filename of our screenshot
      let fileName = url.replace(/(\.|\/|:|%|#)/g, "_");
      if (fileName.length > 100) {
        fileName = fileName.substring(0, 100);
      }

      await page.screenshot({
        path: `./cache/${fileName}.jpeg`,
        fullPage: true,
      });
    } catch (err) {
      console.log(`💀 An error occured fetching ${url}`);
    } finally {
      await page.close();
    }
  }

  await browser.close();
  console.log(
    `Time elapsed: ${Math.round(
      (new Date().getTime() - startDate) / 1000
    )} seconds.`
  );
})();

// Function to make sure we trigger rendering of async loaded elements at the bottom of the page.
async function autoScroll(page) {
  await page.evaluate(async () => {
    await new Promise((resolve, reject) => {
      var totalHeight = 0;
      var distance = 100;
      var timer = setInterval(() => {
        var scrollHeight = document.body.scrollHeight;
        window.scrollBy(0, distance);
        totalHeight += distance;

        if (totalHeight >= scrollHeight) {
          clearInterval(timer);
          resolve();
        }
      }, 100);
    });
  });
}

## warm.sh
#!/bin/bash -e
# -e means exit if any command fails

echo "💥 Removing old files..."
rm -rf ./cache/*
rm -rf cache-data.twig

sleep 1

echo "🥶 Huddling cold data..."

sleep 1

# Check for username AND password args...
# If the site is behind authentication, our make-warm script will also need them.
if [[ (-n "$2" && -n "$3") ]]; then
  wget --user $2 --password $3 --quiet --no-check-certificate $1
  sleep 1
  node make-warm.js --user=$2 --pass=$3

# If not, keep it vanilla and get to warming.
else
  wget --quiet --no-check-certificate $1
  sleep 1
  node make-warm.js
fi

echo "✅ Done."
	{# Creating a list of URLs to Warm

	This example uses Twig + Craft, because Craft 👌

	You'll need to set this up individually based on your CMS and how it is configured.
	Don't take anything below for granted. Your output just needs to generate a file in
	the following (very simple) format – one link per new line!:

	---
	http://some-urls-you-need-warmed.com/some-entry
	http://some-urls-you-need-warmed.com/some-other-entry
	http://some-urls-you-need-warmed.com/another-entry
	---

	#}
	{% spaceless %}

	{# Setup our Craft entry query #}
	{% set availableCars = craft.entries().section('autos').availability("not sold") %}

	{# Loop through our entries and spit out a list of URLs to warm. #}
	{% for entry in availableCars %}
	{{entry.url}}
	{% endfor %}
	{# Array of additional urls to warm. siteUrl is prepended and includes a trailing /, so the first (blank) entry is the homepage. #}
	{% set array = [
	"",
	"showroom",
	"contact",
	] %}
	{% for item in array %}
	{{siteUrl ~ item}}
	{% endfor %}
	{% endspaceless %}
	const fs = require("fs"); // https://github.com/nodejs/node/blob/master/doc/api/fs.md
	const puppeteer = require("puppeteer"); // https://github.com/puppeteer/puppeteer
	const chalkAnimation = require("chalk-animation"); // https://github.com/bokub/chalk-animation
	const { argv } = require("yargs"); // https://github.com/yargs/yargs

	// Function for parsing data from our list of urls to visit.
	function readURLFile(path) {
	return fs
	.readFileSync(path, "utf-8")
	.split("\n")
	.map((elt) => {
	const url = elt.replace("\r", "");
	return `${url.toLowerCase()}`;
	});
	}

	// Instantiate our browser and start hitting urls.
	(async () => {
	const startDate = new Date().getTime();
	const USER_AGENT =
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3239.108 Safari/537.36";
	const urls = readURLFile("./cache-data.twig"); // <- List of urls, warm.sh fetches this using wget from the file you set up.
	const browser = await puppeteer.launch();

	for (let url of urls) {
	const warmingAnimation = chalkAnimation.karaoke(`🌞 Warming: ${url}`);

	let page = await browser.newPage();
	await page.setUserAgent(USER_AGENT);

	try {
	// If user and pass arguments passed, submit authentication credentials
	if (argv.user && argv.pass) {
	// We need to authenticate
	await page.authenticate({ username: argv.user, password: argv.pass });
	}

	await page.goto(url, { waitUntil: "networkidle2" }); // Wait for network traffic to stop.

	// Optional step if you have a modal or some GDPR consent that needs to be dismissed on the first or every page.
	try {
	await page.click("#elc-accept-all-link"); // <- Change this to the ID of the button or element that needs to be clicked to accept/dismiss cookie consent. etc
	// console.log("🍪 Accepting Cookies\n");
	} catch (error) {
	// console.log("🤭 Cookies Already Accepted\n")
	}

	await autoScroll(page); // Get to the bottom of it.
	await page.waitFor(2000, { waitUntil: "networkidle2" }); // Be like, super sure everything is loaded and css transitions have run their course.

	warmingAnimation.stop(); // Stop the warming animation
	warmingAnimation.f = `🌞 Warming: ${url}`.length + 10; // Set the frame manually to [string length + 10] (this is a hack, thank you @bokub)
	warmingAnimation.render(); // Render one last frame
	chalkAnimation.rainbow(`😎 Success\n`);

	// Set the filename of our screenshot
	let fileName = url.replace(/(\.\|\/\|:\|%\|#)/g, "_");
	if (fileName.length > 100) {
	fileName = fileName.substring(0, 100);
	}

	await page.screenshot({
	path: `./cache/${fileName}.jpeg`,
	fullPage: true,
	});
	} catch (err) {
	console.log(`💀 An error occured fetching ${url}`);
	} finally {
	await page.close();
	}
	}

	await browser.close();
	console.log(
	`Time elapsed: ${Math.round(
	(new Date().getTime() - startDate) / 1000
	)} seconds.`
	);
	})();

	// Function to make sure we trigger rendering of async loaded elements at the bottom of the page.
	async function autoScroll(page) {
	await page.evaluate(async () => {
	await new Promise((resolve, reject) => {
	var totalHeight = 0;
	var distance = 100;
	var timer = setInterval(() => {
	var scrollHeight = document.body.scrollHeight;
	window.scrollBy(0, distance);
	totalHeight += distance;

	if (totalHeight >= scrollHeight) {
	clearInterval(timer);
	resolve();
	}
	}, 100);
	});
	});
	}
	#!/bin/bash -e
	# -e means exit if any command fails

	echo "💥 Removing old files..."
	rm -rf ./cache/*
	rm -rf cache-data.twig

	sleep 1

	echo "🥶 Huddling cold data..."

	sleep 1

	# Check for username AND password args...
	# If the site is behind authentication, our make-warm script will also need them.
	if [[ (-n "$2" && -n "$3") ]]; then
	wget --user $2 --password $3 --quiet --no-check-certificate $1
	sleep 1
	node make-warm.js --user=$2 --pass=$3

	# If not, keep it vanilla and get to warming.
	else
	wget --quiet --no-check-certificate $1
	sleep 1
	node make-warm.js
	fi

	echo "✅ Done."