Check if urls from sitemap.xml file are active

checkSitemap.js

/* eslint-disable */
const fs = require('fs')
const urlExists = require('url-exists')
const parser = require('xml2js')
const { PromisePool } = require('@supercharge/promise-pool')

async function init() {
  const sitemapXml = fs.readFileSync('./public/sitemap.xml', 'utf8')
  parser.parseString(sitemapXml, async (err, result) => {
    const urls = result.urlset.url.map(({ loc }) => loc).flat()

    const { results, errors } = await PromisePool.withConcurrency(10)
      .for(urls)
      .onTaskStarted((item, pool) => {
        console.log(`Processing ${item}`)
      })
      .process(async (url, index, pool) => {
        return new Promise((resolve) => {
          urlExists(url, (_, exists) => resolve({ url, exists }))
        })
      })

    const p = results.reduce(
      (acc, curr) => ({
        exists: [...acc.exists, ...(curr.exists ? [curr.url] : [])],
        notExists: [...acc.notExists, ...(!curr.exists ? [curr.url] : [])],
      }),
      { exists: [], notExists: [] }
    )

    console.log({
      exists: p.exists.length,
      notExists: p.notExists.length,
    })

    fs.writeFileSync('./result.json', JSON.stringify(p))
  })
}

init()