Calculating cls in batch for AMP vs non-AMP

batch_cls_calc.js

const puppeteer = require('puppeteer');
const devices = require('puppeteer/DeviceDescriptors');
var fs = require('fs');
const mergeImg = require('merge-img');
const ampToolboxCacheUrl = require('amp-toolbox-cache-url').createCacheUrl;

const Good3G = {
    'offline': false,
    'downloadThroughput': 1.5 * 1024 * 1024 / 8,
    'uploadThroughput': 750 * 1024 / 8,
    'latency': 40
  };
  
const phone = devices['Nexus 5X'];
const agent = "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z‡ Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";


function calcJank() {
  window.cumulativeLayoutShiftScore = 0;

  const observer = new PerformanceObserver((list) => {
   for (const entry of list.getEntries()) {
     if (!entry.hadRecentInput) {
       console.log("New observer entry for cls: " + entry.value);
       window.cumulativeLayoutShiftScore += entry.value;
     }
   }
  });

  observer.observe({type: 'layout-shift', buffered: true});

  document.addEventListener('visibilitychange', () => {
   if (document.visibilityState === 'hidden') {
     observer.takeRecords();
     observer.disconnect();
     console.log('CLS:', window.cumulativeLayoutShiftScore);
   }
  });
}


async function getCLS(url, page, client) {
  try {
	// inject a function with the code from https://web.dev/cls/#measure-cls-in-javascript
    await page.evaluateOnNewDocument(calcJank);  
    await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000});
	await page.waitFor(2000); // let's give it a bit more time, to be sure everything's loaded
	//await page.evaluate(calcJank);
    let cls = await page.evaluate(function() {return window.cumulativeLayoutShiftScore;});
    return cls;
  } catch (error) {
    //console.log(error);
  }
}

async function doBatch(filename) {
    const browser = await puppeteer.launch({ 
      args: ['--no-sandbox'],
      timeout: 10000
    });
    const page = await browser.newPage();
	let validAMP = false;
	page.on('console', msg => {if(msg.text().indexOf("AMP validation successful") >= 0) validAMP = true;});
	await page.emulate(phone);
	page.setUserAgent(agent);
	const client = await page.target().createCDPSession();
    await client.send('Network.enable');
    await client.send('ServiceWorker.enable');
	
    await client.send('Network.emulateNetworkConditions', Good3G);
    await client.send('Emulation.setCPUThrottlingRate', { rate: 4 });
    
	
	let urls = fs.readFileSync('input.csv').toString().split("\n");
	
	for(var i = 0; i < Math.min(urls.length, 1000); i++) {
		const canonical = urls[i];
		let host = "";
		try {
			host = new URL(canonical).host;
		}
		catch(e){continue;}
		const img_canonical = host + '_canonical.jpg';
		const img_amp = host + '_amp.jpg';
		console.log("Processing: " + canonical);
		let cls_canonical = await getCLS(canonical, page, client);
		await page.screenshot({path: img_canonical, type: "jpeg", quality: 60});
		let amp_cdn = null;
		let amp_host = null;
		try {
			amp_host = await page.evaluate(function() {return document.querySelector("link[rel='amphtml']").href});
			console.log("Got AMP Origin link: " + amp_host);
			amp_cdn = await ampToolboxCacheUrl('cdn.ampproject.org', amp_host);
			console.log("Got AMP CDN link: " + amp_cdn);
		}
		catch(e) {
			console.log("No amp - skipping");
			fs.unlinkSync(img_canonical);
			continue;
		};
		if(!amp_host || !amp_cdn) {
			console.log("Couldn't retrieve an AMP URL!");
			fs.unlinkSync(img_canonical);
			continue;
		}
		
		if(cls_canonical === "undefined") {
			console.log("Couldn't calculate CLS for canonical site, skipping!");
			fs.unlinkSync(img_canonical);
			continue;
		}
		let cls_amp_cdn = await getCLS(amp_cdn, page, client);
		
		// 404 and 500 are still valid loads, so this is easiest to see if CDN returned something valid
		if((await page.content()).indexOf("amp-img") === -1) {
			console.log("Couldn't calculate CLS for AMP site, skipping!");
			//console.log((await page.content()).substring(0,300));
			fs.unlinkSync(img_canonical);
			continue;  
		}
		
		//CDN redirects if URL not in cache
		if(page.url() != amp_cdn) {
			console.log("We got redirected to " + page.url() + " therefore skipping....");
			fs.unlinkSync(img_canonical);
			continue;
		}
		
		await page.screenshot({path: img_amp, type: "jpeg", quality: 60});
		if(cls_amp_cdn === "undefined") {
			console.log("Couldn't calculate CLS for AMP site, skipping!");
			fs.unlinkSync(img_canonical);
			continue;
		}
		try {
			if(cls_canonical < cls_amp_cdn) {
			  await mergeImg([img_canonical, img_amp])
			    .then((img) => {
			      img.write(host + '.jpg', () => console.log('writte comparison image'));
			    });
			}
			fs.unlinkSync(img_canonical);
			fs.unlinkSync(img_amp);
		}
		catch(e){console.log(e.message);}
		let cls_amp_host = await getCLS(amp_host, page, client);
		let out = canonical + "," + amp_host + "," + amp_cdn + "," + cls_canonical + "," + cls_amp_host + "," + cls_amp_cdn;
		console.log(out);
		fs.appendFile('output.csv', out + "\n", function (err) {});
	}
}

doBatch('input.csv').then(res => {console.log("Done!");process.exit(0);});