State legislature robots.txt parser

README.md

Simple script using got to check robots.txt files of state legislature websites from a manually collated dataset state_urls.csv to check if Google and accessible means of finding resources are blocked. The resulting dataset can be found in state_access.csv.

There is one state legislative body blocking public access using a robots.txt: the Missouri State Senate http://www.senate.mo.gov

index.js

import got from 'got'
import fs from 'fs'

const checkRobotsTxt = async url => {
  try {
    const res = await got(url + '/robots.txt')
    if (res.body && !res.body.includes('</html>')) {
      const hasGlobalDisallow = /^user-agent: \*(\n|\s|\r)+disallow: \/$/gim.test(
        res.body
      )
      const hasNoGoogleAllow = !/^(\s+)?user-agent: googlebot(\n|\s|\r)+allow: \/$/gim.test(
        res.body
      )
      return hasGlobalDisallow && hasNoGoogleAllow
    }
    return false
  } catch (e) {
    return false
  }
}

;(async () => {
  const siteData = (await fs.readFileSync('./state_urls.csv', 'utf8'))
    .split('\n')
    .map(x => x.split(',').map(x => x.trim()))
  // Use map to track URLs requested to prevent duplicate requests
  const requestsMade = new Map()
  const csvData = fs.createWriteStream('./state_access.csv')
  const writeToCsv = array => {
    csvData.write(`${array.join(',')}\n`)
    return true
  }

  writeToCsv(['state', 'body', 'url', 'robotsBlocked'])

  for await (const data of siteData.slice(1)) {
    const url = new URL(data[2])
    if (!requestsMade.has(url.hostname)) {
      console.log(url.hostname)
      const hasRobotsTxt = await checkRobotsTxt(url.origin)
      writeToCsv([...data, hasRobotsTxt])
      requestsMade.set(url.hostname, hasRobotsTxt)
    } else {
      writeToCsv([...data, requestsMade.get(url.hostname)])
    }
  }

  csvData.end()
})()

state_access.csv

state_urls.csv