github.com/qri-io/walk job

gov.noaa.esrl-walk.job.json

{
  "Seeds": [
    "https://esrl.noaa.gov/"
  ],
  "SeedsPath": "",
  "Crawl": true,
  "Domains": [
    "https://esrl.noaa.gov/"
  ],
  "IgnorePatterns": null,
  "DoneScanMilli": 30000,
  "DelayMilli": 0,
  "StopAfterEntries": 10,
  "StopURL": "",
  "BackoffResponseCodes": null,
  "MaxAttempts": 3,
  "Workers": [
    {
      "Parallelism": 10,
      "Type": "local",
      "DelayMilli": 500,
      "Polite": true,
      "RecordResponseHeaders": false,
      "RecordRedirects": true,
      "UserAgent": ""
    }
  ],
  "ResourceHandlers": [
    {
      "Type": "sitemap",
      "DstPath": "sitemap.json"
    }
  ]
}