Generate sitemaps dynamically

generateSitemap.js

const pgClient = require('./pgClient')
const fs = require('fs')
const sm = require('sitemap')
const path = require('path')

const sitemapsDir = 'sitemaps'

const formatUrl = ({ accountNumber, unparsedAddress }) => {
  return `/property/${accountNumber}/${(unparsedAddress || '').replace(
    /\s+/g,
    '-',
  )}-EDMONTON-AB`
}

const setSearchPath = {
  text: `SET search_path TO "default$default";`,
}

const query = {
  text: `
    SELECT p."accountNumber", p."unparsedAddress"
    FROM "Property" AS p
  `,
}

const removeNulls = ({ accountNumber, unparsedAddress }) => {
  return !!(accountNumber && unparsedAddress)
}

const formatXml = row => ({
  url: formatUrl(row),
  changefreq: 'monthly',
})

const getUrls = rows => rows.filter(removeNulls).map(formatXml)

const handleError = err => {
  if (err) throw err
  console.log('The file has been saved!')
}

const generateSitemap = async () => {
  const start = Date.now()
  process.stdout.write('\n💻  Querying properties... ')
  try {
    await pgClient.query(setSearchPath)
    const { rows } = await pgClient.query(query)
    const chunkSize = 50000
    const numLoops = Math.ceil(rows.length / chunkSize)
    const sitemapFiles = []
    const hostname = 'https://honestdoor.com'
    // create all the sitemap files (max 50,000 urls per file)
    for (let i = 0; i < numLoops; i++) {
      const sitemap = sm.createSitemap({
        hostname,
        cacheTime: 600000, //600 sec (10 min) cache purge period
        urls: getUrls(rows.slice(i * chunkSize, (i + 1) * chunkSize)),
      })
      console.log('count', i, (i + 1) * chunkSize)
      const sitemapFilename = `sitemap-host${i + 1}.xml`
      fs.writeFileSync(
        path.join(__dirname, '../static', sitemapsDir, sitemapFilename),
        sitemap.toString(),
        handleError,
      )
      sitemapFiles.push(`${hostname}/static/${sitemapsDir}/${sitemapFilename}`)
    }
    // create the sitemap index
    const sitemapIndex = sm.buildSitemapIndex({
      urls: sitemapFiles,
    })
    fs.writeFileSync(
      path.join(__dirname, '../static', 'sitemap.xml'),
      sitemapIndex.toString(),
      handleError,
    )
    process.stdout.write('done!')
    console.log(`\n\nFinished in ${(Date.now() - start) / 1000} sec ✌️`)
    process.exit(0)
  } catch (err) {
    console.error(err)
    process.exit(1)
  }
}
generateSitemap()