xiaochengh/cls_font_nofont_hack.js

## cls_font_nofont_hack.js
const puppeteer = require('puppeteer');
const fs = require('fs');
const fsExtra = require('fs-extra')
const mustache = require('mustache')
const atob = require('atob');
const btoa = require('btoa');

let REPEAT = 10;

let TEMPLATE =  fs.readFileSync('template-hacked.html', 'utf8');

require('dotenv').config();

const Good3G = {
  'offline': false,
  'downloadThroughput': 1.5 * 1024 * 1024 / 8,
  'uploadThroughput': 750 * 1024 / 8,
  'latency': 40
};

const phone = puppeteer.devices['Nexus 5X'];

function injectJs() {

  window.cls = 0;
  let po = new PerformanceObserver((list) => {
    for (const entry of list.getEntries()) {
      window.cls += entry.value;
    }
  });

  po.observe({type: 'layout-shift', buffered: true});
}


function fallbackFontOverrideRules(fonts) {
  return '<style>' + fonts.map(font => `
@font-face {
  font-family: ${font.family};
  src: ${font.src};
  ascent-override: ${font.ascentOverride};
  descent-override: ${font.descentOverride};
  line-gap-override: ${font.lineGapOverride};
  advance-override: ${font.advanceOverride};
}
`).join('\n') + '</style>';
}


// Copy-pasted from https://gist.github.com/jsoverson/638b63542863596c562ccefc3ae52d8f
async function injectFallbackFontOverrides(page, fonts) {
  const client = await page.target().createCDPSession();

  await client.send('Network.enable');

  await client.send('Network.setRequestInterception', {
    patterns: [
      {urlPattern: '*', resourceType: 'Document', interceptionStage: 'HeadersReceived'},
    ]
  });

  client.on('Network.requestIntercepted', async ({ interceptionId, request, responseHeaders, resourceType }) => {
    console.log(`Intercepted ${request.url} {interception id: ${interceptionId}}`);

    const response = await client.send('Network.getResponseBodyForInterception',{ interceptionId });

    const contentTypeHeader = Object.keys(responseHeaders).find(k => k.toLowerCase() === 'content-type');
    let contentType = responseHeaders[contentTypeHeader];

    const bodyData = response.base64Encoded ? atob(response.body) : response.body;
    const newBody = bodyData.replace('<head>', '<head>' + fallbackFontOverrideRules(fonts));

    if (newBody === bodyData)
      console.log('Failed to inject font metric overrides to ', request.url);

    const newHeaders = [
      'Date: ' + (new Date()).toUTCString(),
      'Connection: closed',
      'Content-Length: ' + newBody.length,
      'Content-Type: ' + contentType
    ];

    console.log(`Continuing interception ${interceptionId}`)
    client.send('Network.continueInterceptedRequest', {
      interceptionId,
      rawResponse: btoa('HTTP/1.1 200 OK' + '\r\n' + newHeaders.join('\r\n') + '\r\n\r\n' + newBody)
    });
  });
}

async function getBrowser(options) {
  options = options || {};
  let args = ['--no-sandbox'];
  if(!options.allowWebFonts) {
    args.push("--disable-remote-fonts");
  }
  if (options.hackFallbackFonts) {
    args.push("--enable-blink-features=CSSFontMetricsOverride");
  }
  console.log(`Brwoser args = ${args}`);
  const browser = await puppeteer.launch({
    args: args,
    //headless: false,
    executablePath: process.env.CHROMIUM_PATH,
    timeout: 10000
  });
  return browser;
}


async function getNewPage(browser) {
  const page = await browser.newPage();
  await page.emulate(phone);
  await page.setCacheEnabled(false); // no cache, so that we can reuse the same page several times
  return page;
}


async function getCLS(page, url, options) {
  await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000});
  await page.waitFor(2000); // let's give it a bit more time, to be sure everything's loaded

  console.log("Injecting JS...");
  await Promise.race([
      page.evaluate(injectJs),
      page.waitFor(5000)
  ]);
  page.waitFor(2000);

  console.log("Gathering data...");
  let cls = await Promise.race([
      page.evaluate(function() {return window.cls}),
      page.waitFor(5000)
  ]);
  return cls;
}

async function doOne(url, options) {
  let browser = await getBrowser(options);
  console.log(`Browser version: ${await browser.version()}`);
  let page = await getNewPage(browser);
  if (options.hackFallbackFonts)
    injectFallbackFontOverrides(page, options.hackFallbackFonts);
  let cls = 0;
  for (let i = 0; i < REPEAT; ++i)
    cls += await getCLS(page, url, options);
  cls /= REPEAT;
  await page.screenshot({path: 'output/' + options.screenshotPath});
  await browser.close();
  return cls;
}

async function doBatch(pages) {
  // reset output file and images dir
  fsExtra.emptyDirSync("output");
  fs.mkdirSync("output/images");
  fs.mkdirSync("output/images/withfonts");
  fs.mkdirSync("output/images/nofonts");
  fs.mkdirSync("output/images/withfonts-hacked");
  fs.mkdirSync("output/images/nofonts-hacked");

  let results = [];
  for (let page of pages) {
    let url = page.url;
    let runs = page.runs.map(run => typeof run === 'string' ? {name: run} : run);

    console.log(`Processing ${url} with runs: ${runs.map(run => run.name).join(', ')}`);

    let result = {url: url, runs: {}};
    for (let run of runs) {
      let options = run.options || {};
      if (run.name.indexOf('withfonts') !== -1)
        options.allowWebFonts = true;
      options.screenshotPath = `images/${run.name}/${new URL(url).hostname}.png`;
      let cls = await doOne(url, options);
      console.log(`${run.name} CLS = ${cls}`);

      result.runs[run.name] = {name: run.name, cls: cls, screenshot: options.screenshotPath};
    }

    results.push(result);
  }

  // write out result html
  var rendered = mustache.render(TEMPLATE, {items: results});
  fs.writeFileSync('output/cls-with-hacks.html', rendered)
}

let pages = require('./input.json');
doBatch(pages).then(res => {console.log("Done!");process.exit(0);});

## input.json
[
  {
    "url": "https://khabarfarsi.com/ua/81595867",
    "runs": [
      "withfonts",
      "nofonts",
      {
        "name": "nofonts-hacked",
	"options": {
          "hackFallbackFonts": [
            {
              "family": "arial",
	      "src": "local(Arial)",
	      "ascentOverride": "calc(2400 / 2300 * 100%)",
	      "descentOverride": "calc(1400 / 2300 * 100%)",
	      "lineGapOverride": "0%",
	      "advanceOverride": "0.066"
	    }
	  ]
	}
      },
      {
        "name": "withfonts-hacked",
	"options": {
          "hackFallbackFonts": [
            {
              "family": "arial",
	      "src": "local(Arial)",
	      "ascentOverride": "calc(2400 / 2300 * 100%)",
	      "descentOverride": "calc(1400 / 2300 * 100%)",
	      "lineGapOverride": "0%",
	      "advanceOverride": "0.066"
	    }
	  ]
	}
      }
    ]
  }
]

## template-hacked.html
<html>

<head>
	<style>
		#mainTable tr:nth-child(odd){
		  background-color: lightblue;
		}

		thead {
			font-weight: bold;
			background-color: lightgrey;
		}

		.screenshot {
			margin:20px;
			width:150px;
		}

		.heatmap {
			width:400px;
		}

		td {
			word-break:break-all;
			width: 16%;
		}
	</style>
</head>

  <body onload="renderHello()">

	<h1>Analysis results for Font-related layout shifts</h1>

	<table>
		<thead>
			<tr>
				<td width="10%">URL</td>
				<td>CLS With Web-Fonts</td>
				<td>CLS With Fallbacks Only</td>
				<td>CLS With Fallbacks + Overrides</td>
                                <td>CLS With Webfonts + Fallback Overrides</td>
				<td style="width: 200px">With Web-Fonts</td>
                                <td style="width: 200px">With Fallbacks + Overrides</td>
				<td style="width: 200px">With Fallbacks Only</td>
			</tr>
		</thead>
		<tbody id="mainTable">
		{{#items}}
		  <tr>
			<td>{{url}}</td>
			<td>{{runs.withfonts.cls}}</td>
			<td>{{runs.nofonts.cls}}</td>
			<td>{{runs.nofonts-hacked.cls}}</td>
			<td>{{runs.withfonts-hacked.cls}}</td>
			<td style="width: 200px"><img src="{{runs.withfonts.screenshot}}" width="200px"></td>
			<td style="width: 200px"><img src="{{runs.nofonts-hacked.screenshot}}" width="200px"></td>
			<td style="width: 200px"><img src="{{runs.nofonts.screenshot}}" width="200px"></td>
		</tr>
		{{/items}}
		</tbody>
	</table>


  </body>
</html>
	const puppeteer = require('puppeteer');
	const fs = require('fs');
	const fsExtra = require('fs-extra')
	const mustache = require('mustache')
	const atob = require('atob');
	const btoa = require('btoa');

	let REPEAT = 10;

	let TEMPLATE = fs.readFileSync('template-hacked.html', 'utf8');

	require('dotenv').config();

	const Good3G = {
	'offline': false,
	'downloadThroughput': 1.5 * 1024 * 1024 / 8,
	'uploadThroughput': 750 * 1024 / 8,
	'latency': 40
	};

	const phone = puppeteer.devices['Nexus 5X'];

	function injectJs() {

	window.cls = 0;
	let po = new PerformanceObserver((list) => {
	for (const entry of list.getEntries()) {
	window.cls += entry.value;
	}
	});

	po.observe({type: 'layout-shift', buffered: true});
	}


	function fallbackFontOverrideRules(fonts) {
	return '<style>' + fonts.map(font => `
	@font-face {
	font-family: ${font.family};
	src: ${font.src};
	ascent-override: ${font.ascentOverride};
	descent-override: ${font.descentOverride};
	line-gap-override: ${font.lineGapOverride};
	advance-override: ${font.advanceOverride};
	}
	`).join('\n') + '</style>';
	}


	// Copy-pasted from https://gist.github.com/jsoverson/638b63542863596c562ccefc3ae52d8f
	async function injectFallbackFontOverrides(page, fonts) {
	const client = await page.target().createCDPSession();

	await client.send('Network.enable');

	await client.send('Network.setRequestInterception', {
	patterns: [
	{urlPattern: '*', resourceType: 'Document', interceptionStage: 'HeadersReceived'},
	]
	});

	client.on('Network.requestIntercepted', async ({ interceptionId, request, responseHeaders, resourceType }) => {
	console.log(`Intercepted ${request.url} {interception id: ${interceptionId}}`);

	const response = await client.send('Network.getResponseBodyForInterception',{ interceptionId });

	const contentTypeHeader = Object.keys(responseHeaders).find(k => k.toLowerCase() === 'content-type');
	let contentType = responseHeaders[contentTypeHeader];

	const bodyData = response.base64Encoded ? atob(response.body) : response.body;
	const newBody = bodyData.replace('<head>', '<head>' + fallbackFontOverrideRules(fonts));

	if (newBody === bodyData)
	console.log('Failed to inject font metric overrides to ', request.url);

	const newHeaders = [
	'Date: ' + (new Date()).toUTCString(),
	'Connection: closed',
	'Content-Length: ' + newBody.length,
	'Content-Type: ' + contentType
	];

	console.log(`Continuing interception ${interceptionId}`)
	client.send('Network.continueInterceptedRequest', {
	interceptionId,
	rawResponse: btoa('HTTP/1.1 200 OK' + '\r\n' + newHeaders.join('\r\n') + '\r\n\r\n' + newBody)
	});
	});
	}

	async function getBrowser(options) {
	options = options \|\| {};
	let args = ['--no-sandbox'];
	if(!options.allowWebFonts) {
	args.push("--disable-remote-fonts");
	}
	if (options.hackFallbackFonts) {
	args.push("--enable-blink-features=CSSFontMetricsOverride");
	}
	console.log(`Brwoser args = ${args}`);
	const browser = await puppeteer.launch({
	args: args,
	//headless: false,
	executablePath: process.env.CHROMIUM_PATH,
	timeout: 10000
	});
	return browser;
	}


	async function getNewPage(browser) {
	const page = await browser.newPage();
	await page.emulate(phone);
	await page.setCacheEnabled(false); // no cache, so that we can reuse the same page several times
	return page;
	}


	async function getCLS(page, url, options) {
	await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000});
	await page.waitFor(2000); // let's give it a bit more time, to be sure everything's loaded

	console.log("Injecting JS...");
	await Promise.race([
	page.evaluate(injectJs),
	page.waitFor(5000)
	]);
	page.waitFor(2000);

	console.log("Gathering data...");
	let cls = await Promise.race([
	page.evaluate(function() {return window.cls}),
	page.waitFor(5000)
	]);
	return cls;
	}

	async function doOne(url, options) {
	let browser = await getBrowser(options);
	console.log(`Browser version: ${await browser.version()}`);
	let page = await getNewPage(browser);
	if (options.hackFallbackFonts)
	injectFallbackFontOverrides(page, options.hackFallbackFonts);
	let cls = 0;
	for (let i = 0; i < REPEAT; ++i)
	cls += await getCLS(page, url, options);
	cls /= REPEAT;
	await page.screenshot({path: 'output/' + options.screenshotPath});
	await browser.close();
	return cls;
	}

	async function doBatch(pages) {
	// reset output file and images dir
	fsExtra.emptyDirSync("output");
	fs.mkdirSync("output/images");
	fs.mkdirSync("output/images/withfonts");
	fs.mkdirSync("output/images/nofonts");
	fs.mkdirSync("output/images/withfonts-hacked");
	fs.mkdirSync("output/images/nofonts-hacked");

	let results = [];
	for (let page of pages) {
	let url = page.url;
	let runs = page.runs.map(run => typeof run === 'string' ? {name: run} : run);

	console.log(`Processing ${url} with runs: ${runs.map(run => run.name).join(', ')}`);

	let result = {url: url, runs: {}};
	for (let run of runs) {
	let options = run.options \|\| {};
	if (run.name.indexOf('withfonts') !== -1)
	options.allowWebFonts = true;
	options.screenshotPath = `images/${run.name}/${new URL(url).hostname}.png`;
	let cls = await doOne(url, options);
	console.log(`${run.name} CLS = ${cls}`);

	result.runs[run.name] = {name: run.name, cls: cls, screenshot: options.screenshotPath};
	}

	results.push(result);
	}

	// write out result html
	var rendered = mustache.render(TEMPLATE, {items: results});
	fs.writeFileSync('output/cls-with-hacks.html', rendered)
	}

	let pages = require('./input.json');
	doBatch(pages).then(res => {console.log("Done!");process.exit(0);});
	[
	{
	"url": "https://khabarfarsi.com/ua/81595867",
	"runs": [
	"withfonts",
	"nofonts",
	{
	"name": "nofonts-hacked",
	"options": {
	"hackFallbackFonts": [
	{
	"family": "arial",
	"src": "local(Arial)",
	"ascentOverride": "calc(2400 / 2300 * 100%)",
	"descentOverride": "calc(1400 / 2300 * 100%)",
	"lineGapOverride": "0%",
	"advanceOverride": "0.066"
	}
	]
	}
	},
	{
	"name": "withfonts-hacked",
	"options": {
	"hackFallbackFonts": [
	{
	"family": "arial",
	"src": "local(Arial)",
	"ascentOverride": "calc(2400 / 2300 * 100%)",
	"descentOverride": "calc(1400 / 2300 * 100%)",
	"lineGapOverride": "0%",
	"advanceOverride": "0.066"
	}
	]
	}
	}
	]
	}
	]
	<html>

	<head>
	<style>
	#mainTable tr:nth-child(odd){
	background-color: lightblue;
	}

	thead {
	font-weight: bold;
	background-color: lightgrey;
	}

	.screenshot {
	margin:20px;
	width:150px;
	}

	.heatmap {
	width:400px;
	}

	td {
	word-break:break-all;
	width: 16%;
	}
	</style>
	</head>

	<body onload="renderHello()">

	<h1>Analysis results for Font-related layout shifts</h1>

	<table>
	<thead>
	<tr>
	<td width="10%">URL</td>
	<td>CLS With Web-Fonts</td>
	<td>CLS With Fallbacks Only</td>
	<td>CLS With Fallbacks + Overrides</td>
	<td>CLS With Webfonts + Fallback Overrides</td>
	<td style="width: 200px">With Web-Fonts</td>
	<td style="width: 200px">With Fallbacks + Overrides</td>
	<td style="width: 200px">With Fallbacks Only</td>
	</tr>
	</thead>
	<tbody id="mainTable">
	{{#items}}
	<tr>
	<td>{{url}}</td>
	<td>{{runs.withfonts.cls}}</td>
	<td>{{runs.nofonts.cls}}</td>
	<td>{{runs.nofonts-hacked.cls}}</td>
	<td>{{runs.withfonts-hacked.cls}}</td>
	<td style="width: 200px"><img src="{{runs.withfonts.screenshot}}" width="200px"></td>
	<td style="width: 200px"><img src="{{runs.nofonts-hacked.screenshot}}" width="200px"></td>
	<td style="width: 200px"><img src="{{runs.nofonts.screenshot}}" width="200px"></td>
	</tr>
	{{/items}}
	</tbody>
	</table>


	</body>
	</html>