Static page generation with puppeteer headless chrome for javascript web apps.

gen-static-pages.js

const os = require('os')
const crypto = require('crypto')
const path = require('path')
const fs = require('fs')
const util = require('util')
const puppeteer = require('puppeteer')
const libxml = require('libxmljs')
const dayjs = require('dayjs')

// generates static html files for server side rendering
// usage: node cd-utility/ssr $deploymentPath $locale $projectLiveURL $prevDeploymentName
// example: node cd-utility/ssr deployments/name en-us https://project.com 0.1.23

function mkdirs(arr) {
  // make drectories
  return Promise.all(arr.filter(p => typeof p == 'string').map(function(p) {
    return new Promise(function(resolve, reject) {
      fs.mkdir(p, {recursive: true}, function(err) {
        if (err) reject(err)
        else resolve(p)
      })
    })
  }))
}

function getFileLastModDate(filepath) {
  try {
    const lastmod = fs.statSync(filepath).mtimeMs
    return dayjs(lastmod).toISOString()
  } catch (e) {
    return null
  }
}

function getFileChecksum(filepath) {
  try {
    const str = fs.readFileSync(filepath, 'utf8')
    return crypto.createHash('md5').update(str).digest('hex')
  } catch (e) {
    return null
  }
}

function writeFileS(arr, str) {
  // write files with same content
  const wfp = util.promisify(fs.writeFile)
  return Promise.all(arr.filter(p => typeof p == 'string').map(p => wfp(p, str)))
}

// cmd args
const deploymentDir = path.resolve(process.argv[2])
const srcDir = path.resolve(deploymentDir, '..', '..', 'source')
const locale = process.argv[3] // en-US, fr-FR, tr-TR etc.
const appURL = process.argv[4]

// get previous deployment name (which is a version number) but may also get from cmd
let oldDeploymentName = null
const historyFilepath = './changelog/history.json'
if (fs.existsSync(historyFilepath)) {
  const history = JSON.parse( fs.readFileSync(historyFilepath, 'utf8') )
  if (history.length > 1) {
    oldDeploymentName = history[1].version
  }
}

// constants
const currentDate = dayjs(Date.now()).toISOString()
const lineBreak = os.EOL

// create an empty sitemap document
const sitemap = new libxml.Document()
const urlset = sitemap.node('urlset').attr({
  xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9'
})

// create an empty msconfig document
const msconfig = new libxml.Document()
const tile = msconfig.node('browserconfig').node('msapplication').node('tile')

function addURLToSitemap(url, lastmod = null) {
  // use this function to add a url to the sitemap
  if (!lastmod) {
    urlset
      .node('url')
      .node('loc', url).parent()
      .node('changefreq', 'daily').parent().parent()
  }
  else {
    urlset
      .node('url')
      .node('loc', url).parent()
      .node('lastmod', lastmod).parent()
      .node('changefreq', 'daily').parent().parent()
  }
}

const attachments = {
  echo: function echo(str) {console.log(str)},
  writeFile: util.promisify(fs.writeFile),
  appendFile: util.promisify(fs.appendFile),
  addNodeToMSConfigTile: function addNodeToMSConfigTile(name, src) {
    tile.node(name).attr('src', src).parent()
  },
  saveView: function saveView(defaultLocale, locale, fullpath, prodAppURL) {
    const page = this

    defaultLocale = defaultLocale.toLowerCase()
    locale = locale.toLowerCase()

    return new Promise(function(resolve, reject) {
      page.content().then(function(str) {
        const newHash = crypto.createHash('md5').update(str).digest('hex')

        const isDefaultLocale = locale == defaultLocale
        const savepaths = [deploymentDir + fullpath]
        if (isDefaultLocale) savepaths.push(deploymentDir + '/' + locale + fullpath)
        const filepaths = savepaths.map(p => p + '/index.html')

        const webPaths = [fullpath]
        if (isDefaultLocale) webPaths.push('/' + locale + fullpath)

        const oldDeploymentDir = path.resolve(deploymentDir, '..', oldDeploymentName)
        const oldSavePaths = [oldDeploymentDir + fullpath]
        if (isDefaultLocale) oldSavePaths.push(oldDeploymentDir + '/' + locale + fullpath)
        const oldFilepaths = oldSavePaths.map(p => p + '/index.html')
        const oldFileLastMod = getFileLastModDate(oldFilepaths[0])
        const oldHash = getFileChecksum(oldFilepaths[0])

        mkdirs(savepaths)
          .then(function() {
            const lastmod = oldHash !== null && oldHash == newHash
              ? oldFileLastMod
              : currentDate

            writeFileS(filepaths, str)
              .then(function() {
                const url = prodAppURL + fullpath
                addURLToSitemap(url, lastmod)
                return resolve()
              })
          })
      }).catch(function(err) {
        console.log('saveView error: ', err)
      })
    })
  }
}

function genStaticViews(context) {
  const {deploymentDir, currentDate, lineBreak} = context
  // we're inside the app, render and save each static view
  return new Promise(function(resolve, reject) {
    const {
      viewers, settings, monument, metapatcher, locale, _, isProd
    } = window[window['GOZELBOT_APPID']]
    const {productionURL, locales, path} = monument
    const {branding} = settings
    const {app} = viewers
    const viewer = app.getRouter()
    const staticViewIDs = viewer.views
      .filter(obj => obj.static && obj.locale == locale)
      .map(obj => obj.id)
    let numberOfgeneratedViews = 0
    let numberOfFailedViews = 0
    window.echo(staticViewIDs.length + ' views found in ' + locale + '. generating static files.')

    // run only once (default locale)
    if (locales[0] == locale) {
      // generate pwa manifest
      const manifest = {
        short_name: _('projectShortName'),
        name: _('projectName'),
        start_url: '/#ref=pwam',
        display: 'standalone',
        background_color: branding.backgroundColor,
        theme_color: branding.primaryColor,
        icons: metapatcher.context.androidChrome.appIcons
      }
      window.writeFile(deploymentDir + '/manifest.json', JSON.stringify(manifest, null, 2))

      // generate microsoft browser config xml file
      metapatcher.context.microsoft.appIcons.map(function(obj) {
        const n = obj.name.split('-')[1]
        window.addNodeToMSConfigTile(n, obj.src)
      })
      window.addNodeToMSConfigTile('TileColor', branding.primaryColor)

      // robots.txt
      const robotsTXTPath = deploymentDir + '/robots.txt'
      const robotsTXT = ['User-agent: *']
      if (isProd) robotsTXT.push('Allow: /' + lineBreak)
      else robotsTXT.push('Disallow: /' + lineBreak)
      for (let i = 0; i < locales.length; i++) {
        const lo = locales[i].toLowerCase()
        robotsTXT.push('Sitemap: ' + productionURL + '/' + lo + '/sitemap.xml')
      }
      window.writeFile(robotsTXTPath, robotsTXT.join(lineBreak))
    }

    let timer = null

    // will be called after each view change
    viewer.on('afterShift', function(activeView, prevView) {
      clearTimeout(timer)

      const roots = viewer.getActiveView().roots
  
      // save page as html in v.fullpath folder
      window
        .saveView(locales[0], locale, activeView.fullpath, productionURL)
        .then(function() {
          // we are done with the last id
          staticViewIDs.pop()
          numberOfgeneratedViews += 1

          // continue to shift as long as we have views
          if (staticViewIDs.length > 0) shiftView(); else return resolve()
        })
    })

    // goes to the next view
    function shiftView() {
      const nextViewID = staticViewIDs[staticViewIDs.length - 1]
      const nextView = viewer.getViewByID(nextViewID, locale)

      // will skip to the next view or cancel ssr if full render take more than timeout
      timer = setTimeout(function() {
        window.echo('failed ' + nextViewID + ' -> ' + nextView.fullpath)
        numberOfFailedViews += 1
        if (numberOfgeneratedViews === 0) {
          return reject(new Error('Rendering of "' + viewer.getActiveView().id + '" took more than 3 seconds therefore skipped to the next view.'))
        }
        else {
          clearTimeout(timer)
          staticViewIDs.pop()
          if (staticViewIDs.length > 0) shiftView(); else return resolve()
        }
      }, 3000)

      // go to the next view
      window.echo('saving ' + nextViewID + ' -> ' + viewer.getViewByID(nextViewID, locale).fullpath)
      viewer.shift(nextViewID)
    }

    // start browsing
    shiftView()
  })
}

console.log('👨‍🔧 launching chrome for ssr. (' + locale + ')')

// open browser 
puppeteer
  .launch({
    // only enable devtools for debugging, --lang doesnt work if it is true
    //devtools: false,
    //slowMo: 500,
    args: [
      '--no-sandbox',
      '--disable-setuid-sandbox',
      '--disable-dev-shm-usage'
    ]
  })
  .then(function(browser) {
    // open new tab
    browser
      .newPage()
      .then(function(page) {
        // set window size and user agent
        Promise
          .all([
            page.setUserAgent('GOZELBOT'),
            page.setViewport({width: 1440, height: 768}),
            page.evaluateOnNewDocument(function(lo) {
              Object.defineProperty(navigator, "languages", {get: function() {return [lo];}});
              Object.defineProperty(navigator, "language", {get: function() {return lo;}});
            }, locale)
          ])
          .then(function() {
            // enter site address to the address bar, but not render
            page
              .goto(appURL + '/start.html', {waitUntil: 'networkidle0'})
              .then(function() {
                // attach node function to window object before render
                Promise
                  .all([
                    page.exposeFunction('echo', attachments.echo),
                    page.exposeFunction('writeFile', attachments.writeFile),
                    page.exposeFunction('appendFile', attachments.appendFile),
                    page.exposeFunction('addNodeToMSConfigTile', attachments.addNodeToMSConfigTile),
                    page.exposeFunction('saveView', attachments.saveView.bind(page))
                  ])
                  .then(function() {
                    // render the site
                    const context = {
                      deploymentDir: deploymentDir,
                      currentDate: currentDate,
                      lineBreak: lineBreak
                    }
                    page
                      .evaluate(genStaticViews, context)
                      .then(function() {
                        // save sitemap
                        const sPath = path.join(deploymentDir, locale.toLowerCase(), 'sitemap.xml')
                        fs.writeFileSync(sPath, sitemap.toString())

                        // save msconfig xml file
                        const msPath = path.join(deploymentDir, 'msconfig.xml')
                        fs.writeFileSync(msPath, msconfig.toString())

                        // close browser
                        browser.close()
                      })
                      .catch(function(err) {
                        console.log('Page evaluation error: ', err)
                        browser.close()
                      })
                  })
                  .catch(function(err) {
                    console.log('Page expose error: ', err)
                  })
              })
          })
      })
  })