Skip to content

Instantly share code, notes, and snippets.

@jsanta
Created November 9, 2021 15:47
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save jsanta/df581338debf95f9e092927ec73f5dad to your computer and use it in GitHub Desktop.
Save jsanta/df581338debf95f9e092927ec73f5dad to your computer and use it in GitHub Desktop.
Puppeteer enterpise plugin for Scully. Basically a huge copy & paste from the original PuppeteerRender plugin, but with some extra attributes for configuration.
/**
* Patched to include timeout support
*/
// tslint:disable: no-string-literal
// const puppeteer = require('puppeteer');
import { HandledRoute, scullyConfig, log, logError, logWarn, createFolderFor, registerPlugin, getPluginConfig } from '@scullyio/scully';
import { scullySystem } from '@scullyio/scully/src/lib/pluginManagement/pluginRepository';
import { launchedBrowser, reLaunch } from '@scullyio/scully/src/lib/renderPlugins/launchedBrowser';
import { ssl, showBrowser } from '@scullyio/scully/src/lib/utils';
import { yellow, green } from 'chalk';
import { readFileSync } from 'fs-extra';
import { jsonc } from 'jsonc';
import { join } from 'path';
import { Browser, Page, Serializable } from 'puppeteer';
import { interval, Subject } from 'rxjs';
import { filter, switchMap, take } from 'rxjs/operators';
import { performance } from 'perf_hooks';
const errorredPages = new Map<string, number>();
let version = '0.0.0';
try {
const pkg = join(__dirname, '../../package.json');
// log(pkg)
version = jsonc.parse(readFileSync(pkg).toString()).version || '0.0.0';
version += '-queplan';
} catch (e) {
// this is only for internals builds
// version = jsonc.parse(readFileSync(join(__dirname, '../../../package.json')).toString()).version || '0.0.0';
}
export interface CustomRoute {
route: string;
renderPlugin: string;
}
export interface CustomRenderConfig {
puppeteerTimeout: number,
scullyTimeout: number,
scullyRetries: number,
scullyExitOnError: boolean,
routes?: Array<CustomRoute>,
hasConfig?: boolean;
}
export const CustomPuppeteerPlugin = 'customPuppeteerRender';
let pluginConfig: CustomRenderConfig = {
puppeteerTimeout: 30000, // default 30s
scullyTimeout: 25, // default 25s
scullyRetries: 3, // default 3 retries
scullyExitOnError: true,
routes: [ {
route: '/',
renderPlugin: 'customPuppeteerRender'
} ]
};
let cnt = 1;
export const customRenderPluginHandler = async (route: HandledRoute): Promise<string> => {
log(`-> customPuppeteerRender: ${route.route}`);
const timeOutValueInSeconds = pluginConfig.scullyTimeout || 25;
const pageLoaded = new Subject<void>();
const path = route.rawRoute
? route.rawRoute
: scullyConfig.hostUrl
? `${scullyConfig.hostUrl}${route.route}`
: `http${ssl ? 's' : ''}://${scullyConfig.hostName}:${scullyConfig.appPort}${route.route}`;
let pageHtml: string;
let browser: Browser;
let page: Page;
try {
const t0 = performance.now();
browser = await launchedBrowser().catch((e) => {
logError('Pupeteer died?', e);
// captureException(e);
return Promise.reject(e);
});
// open a new page
page = await browser.newPage();
let resolve;
const pageReady = new Promise((r) => (resolve = r));
if (scullyConfig.bareProject) {
await page.setRequestInterception(true);
page.on('request', registerRequest);
page.on('requestfinished', unRegisterRequest);
page.on('requestfailed', unRegisterRequest);
const requests = new Set();
// eslint-disable-next-line no-inner-declarations
function registerRequest(request) {
request.continue();
requests.add(requests);
}
// eslint-disable-next-line no-inner-declarations
function unRegisterRequest(request) {
// request.continue();
requests.delete(requests);
}
pageLoaded
.pipe(
switchMap(() => interval(50)),
filter(() => requests.size === 0),
filter(() => route.config && route.config.manualIdleCheck),
take(1)
)
.subscribe({
complete: () => {
log('page should be idle');
resolve();
},
});
}
if (scullyConfig.ignoreResourceTypes && scullyConfig.ignoreResourceTypes.length > 0) {
await page.setRequestInterception(true);
page.on('request', checkIfRequestShouldBeIgnored);
// eslint-disable-next-line no-inner-declarations
function checkIfRequestShouldBeIgnored(request) {
if (scullyConfig.ignoreResourceTypes.includes(request.resourceType())) {
request.abort();
} else {
request.continue();
}
}
}
/** this will be called from the browser, but runs in node */
await page.exposeFunction('onCustomEvent', () => {
resolve();
});
windowSet(page, 'scullyVersion', version);
if (route.config && route.config.manualIdleCheck) {
// windowSet(page, 'ScullyIO-ManualIdle', true);
route.exposeToPage = route.exposeToPage || {};
route.exposeToPage.manualIdle = true;
}
if (scullyConfig.inlineStateOnly) {
route.injectToPage = route.injectToPage || {};
route.injectToPage.inlineStateOnly = true;
}
if (route.exposeToPage !== undefined) {
windowSet(page, 'ScullyIO-exposed', route.exposeToPage);
}
if (route.injectToPage !== undefined) {
windowSet(page, 'ScullyIO-injected', route.injectToPage);
}
/** Inject this into the running page, runs in browser */
await page.evaluateOnNewDocument(() => {
/** set "running" mode */
window['ScullyIO'] = 'running';
window.addEventListener('AngularReady', () => {
window['onCustomEvent']();
});
});
// enter url in page
const pageTimeout = pluginConfig.puppeteerTimeout || (scullyConfig.puppeteerLaunchOptions || { timeout: 0 }).timeout;
if (pageTimeout && pageTimeout > 0) {
await page.goto(path, { waitUntil: 'domcontentloaded', timeout: pageTimeout });
} else {
await page.goto(path);
}
pageLoaded.next();
await Promise.race([pageReady, waitForIt(timeOutValueInSeconds * 1000)]);
/** when done, add in some scully content. */
await page.evaluate(() => {
const head = document.head;
/** add a small script tag to set "generated" mode */
const d = document.createElement('script');
d.innerHTML = `window['ScullyIO']='generated';`;
if (window['ScullyIO-injected']) {
/** and add the injected data there too. */
d.innerHTML += `window['ScullyIO-injected']=${JSON.stringify(window['ScullyIO-injected'])};`;
}
const m = document.createElement('meta');
m.name = 'generator';
m.content = `Scully ${window['scullyVersion']}`;
head.appendChild(d);
head.insertBefore(m, head.firstChild);
/** inject the scully version into the body attribute */
document.body.setAttribute('scully-version', window['scullyVersion']);
});
pageHtml = await page.content();
/** Check for undefined content and re-try */
if ([undefined, null, '', 'undefined', 'null'].includes(pageHtml)) {
throw new Error(`Route "${route.route}" render to ${path}`);
}
const firstTitle = await page.evaluate(() => {
const d = document.querySelector('h1');
return (d && d.innerText) || '';
});
if (firstTitle === '404 - URL not provided in the app Scully is serving') {
logWarn(`Route "${yellow(route.route)}" not provided by angular app`);
}
/** save thumbnail to disk code */
if (scullyConfig.thumbnails) {
const file = join(scullyConfig.outDir, route.route, '/thumbnail.jpg');
createFolderFor(file);
await page.screenshot({ path: file });
}
// pageHtml = await page.evaluate(`(async () => {
// return new XMLSerializer().serializeToString(document.doctype) + document.documentElement.outerHTML
// })()`);
/**
* when the browser is shown, use a 2 minute timeout, otherwise
* wait for page-read || timeout @ 25 seconds.
*/
if (showBrowser) {
// if (false) {
page.evaluate(
"log('\\n\\n------------------------------\\nScully is done, page left open for 120 seconds for inspection\\n------------------------------\\n\\n')"
);
//* don't close the browser, but leave it open for inspection for 120 secs
waitForIt(120 * 1000).then(() => page.close());
} else {
await page.close();
}
const t1 = performance.now();
log(`\n${green('Render[' + (cnt++) + '] ' + route.route + ' took ' + (t1 - t0) + 'ms')}`);
} catch (err) {
pluginConfig.scullyRetries = (!pluginConfig.scullyRetries) ? 3 : pluginConfig.scullyRetries;
const { message } = err;
// tslint:disable-next-line: no-unused-expression
page && typeof page.close === 'function' && (await page.close());
logWarn(`Puppeteer error while rendering "${yellow(route.route)}"`, err, ` we will retry rendering this page up to ${pluginConfig.scullyRetries} times.`);
if (message && message.includes('closed')) {
/** signal the launched to relaunch puppeteer, as it has likely died here. */
reLaunch('closed');
// return puppeteerRender(route);
}
if (errorredPages.has(route.route) && errorredPages.get(route.route) > pluginConfig.scullyRetries) {
logError(`Puppeteer error while rendering "${yellow(route.route)}"`, err, ` we retried rendering this page ${pluginConfig.scullyRetries} times.`);
/** we tried this page before, something is really off. Exit stage left. */
// captureException(err);
// Do not exit, already skipped the url, unless scullyExitOnError = true
if (pluginConfig.scullyExitOnError) {
process.exit(15);
}
} else {
const count = errorredPages.get(route.route) || 0;
errorredPages.set(route.route, count + 1);
/** give it a couple of secs */
await waitForIt(5 * 1000);
/** retry! */
return customRenderPluginHandler(route);
}
}
return pageHtml;
};
export function waitForIt(milliSeconds: number) {
return new Promise<void>((resolve) => setTimeout(() => resolve(), milliSeconds));
}
const windowSet = (page: Page, name: string, value: Serializable) =>
page.evaluateOnNewDocument(`
Object.defineProperty(window, '${name}', {
get() {
return JSON.parse('${JSON.stringify(value)}')
}
})
`);
registerPlugin('enterprise', CustomPuppeteerPlugin, customRenderPluginHandler);
export function getCustomPuppeteerRender(_pluginConfig?: CustomRenderConfig) {
pluginConfig = (!pluginConfig.hasConfig) ? Object.assign({ hasConfig: true }, pluginConfig, _pluginConfig) : pluginConfig;
return CustomPuppeteerPlugin;
}
@jsanta
Copy link
Author

jsanta commented Nov 9, 2021

Usage example:

// Inside your scully.config.ts file
const customRenderConfig: CustomRenderConfig = {
  puppeteerTimeout: 120000,
  scullyTimeout: 45,
  scullyRetries: 5,
  scullyExitOnError: false,
};
const CustomPuppeteerPlugin = getCustomPuppeteerRender(customRenderConfig);

// ... other plugin configurations

export const config: ScullyConfig = {
  puppeteerLaunchOptions: {
    args: [
      '--disable-gpu',
      '--renderer',
      '--no-sandbox',
      '--no-service-autorun',
      '--no-experiments',
      '--no-default-browser-check',
      '--disable-dev-shm-usage',
      '--disable-setuid-sandbox',
      '--no-first-run',
      '--no-zygote',
      '--single-process',
      '--disable-extensions',
      '--user-agent=\'Scully Navigator\'',
      '--window-size=1200,720'
    ],
    // executablePath: '/usr/local/bin/chromium',
    timeout: 120000
  },
  projectRoot: './apps/some_project/src',
  projectName: 'some_project/',
  outDir: './dist/static/some_project/',
  ignoreResourceTypes: [
    'image',
    'media',
    'font',
    'stylesheet'
  ],
  defaultPostRenderers,
  proxyConfig: './apps/some_project/proxy.conf-scully.json',
  appPort: 4200,
  staticPort: 8080,
  routes: {
    '/': {
      type: 'default',
      renderPlugin: getCustomPuppeteerRender(customRenderConfig),
      postRenderers: [...defaultPostRenderers]
    }
  },
  extraRoutes: [],
  handle404: 'index'
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment