screenshot-webpage.js

const fs = require('fs-extra');
const gutil = require('gulp-util');
const puppeteer = require('puppeteer');

function delay(timeout) {
  return new Promise(function(resolve) {
    setTimeout(resolve, timeout);
  })
};

function getDirPath(name) {
  return `./screenshots/${name}`;
};

function getFileName(index) {
  return `${index + 1}.jpg`;
};

function getUrl(name) {
  return `http://127.0.0.1:8001/${name}.html`;
};

async function makeScreenshotsByName(name, selector) {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();

  await page.setViewport({
    width: 1536,
    height: 30000, // Fix after https://github.com/GoogleChrome/puppeteer/issues/959 is closed
    deviceScaleFactor: 2
  });

  const url = getUrl(name);

  await page.goto(url, { timeout: 90000 });

  const dirPath = getDirPath(name);

  await fs.ensureDirSync(dirPath);

  await delay(3000);

  const blocks = await page.$$(selector);

  for (var index = 0; index < blocks.length; index++) {
    const path = `${dirPath}/${getFileName(index)}`;

    await blocks[index].screenshot({
      path: path,
      type: 'jpeg',
      quality: 100,
    });

    gutil.log(`Saved screenshot for '${gutil.colors.cyan(name)}' to '${gutil.colors.magenta(path)}'`);
  }

  await browser.close();
};

const MAX_CONCURRENT_BROWSERS_COUNT = 6;

async function makeScreenshots(description) {
  if (description.length > MAX_CONCURRENT_BROWSERS_COUNT) {
    await makeScreenshots(description.slice(0, MAX_CONCURRENT_BROWSERS_COUNT));
    await makeScreenshots(description.slice(MAX_CONCURRENT_BROWSERS_COUNT));
  } else {
    await Promise.all(description.map(([name, selector]) => makeScreenshotsByName(name, selector)));
  }
};

module.exports = {
  makeScreenshots,
};

search-index.js

const fs = require("fs");
const path = require("path");

const fm = require("front-matter");

const {
  CONTENT_ROOT,
  CONTENT_TRANSLATED_ROOT,
  VALID_LOCALES,
} = require("../content");
const { SearchIndex } = require("../build");

function* walker(root) {
  const dirents = fs.readdirSync(root, { withFileTypes: true });
  for (const dirent of dirents) {
    // Doing it this way is faster than doing `path.join(root, dirent.name)`
    const filepath = `${root}${path.sep}${dirent.name}`;
    if (dirent.isDirectory()) {
      yield* walker(filepath);
    } else {
      yield filepath;
    }
  }
}

function populateSearchIndex(searchIndex, localeLC) {
  const root = path.join(
    localeLC === "en-us" ? CONTENT_ROOT : CONTENT_TRANSLATED_ROOT,
    localeLC
  );
  const locale = VALID_LOCALES.get(localeLC);
  for (const filePath of walker(root)) {
    const basename = path.basename(filePath);
    if (!(basename === "index.html" || basename === "index.md")) {
      continue;
    }
    const rawContent = fs.readFileSync(filePath, "utf-8");
    const { attributes: metadata } = fm(rawContent);
    metadata.locale = locale;

    const url = `/${locale}/docs/${metadata.slug}`;
    const doc = { metadata, url };
    searchIndex.add(doc);
  }
}

async function searchIndexRoute(req, res) {
  // Remember, this is always in lowercase because of a middleware
  // that lowercases all incoming requests' pathname.
  const locale = req.params.locale;
  if (locale !== "en-us" && !CONTENT_TRANSLATED_ROOT) {
    res.status(500).send("CONTENT_TRANSLATE_ROOT not set\n");
    return;
  }
  if (!VALID_LOCALES.has(locale)) {
    res.status(500).send(`unrecognized locale ${locale}`);
    return;
  }

  // The advantage of creating the search index over and over on every
  // request is that it can't possible cache itself stale.
  // Imagine if a user just seconds ago created a page, and reaches for
  // the search widget and can't find what they just created.
  // Or perhaps they edited a title and expect that to turn up.

  // However, if this is causing a problem, we can easily turn on some
  // caching. Either `Cache-Control` on the response.
  // Or, we can make this `searchIndex` a module global so it's reused
  // repeatedly until the server is restarted.

  const searchIndex = new SearchIndex();

  const label = "Populate search-index";
  console.time(label);
  populateSearchIndex(searchIndex, locale);
  searchIndex.sort();
  console.timeEnd(label);
  res.json(searchIndex.getItems()[locale]);
}

module.exports = {
  searchIndexRoute,
};