crawl.js_config_experiment1

config.json

{
   "fetcher": {
    "instances": 10,
    "request": {
        "jar": true,
        "timeout": 30000,
        "followRedirect": true,
        "maxRedirects": 3,
        "headers": {
          "User-Agent": "crawl.js v0.0.1"
        }
    }
  },
  "extractor": "parser",
  "queues": {
    "local": {
      "type": "memory",
      "options": {
        "limit": 10000
      }
    },
    "remote": {
      "type": "redis",
      "options": {
        "flushInterval": 30000,
        "host": "redis1",
        "port": "6379"
      }
    }
  },
  "seen": {
    "host": "redis1",
    "port": "6379"
  },
  "dispatcher": {
    "acceptPattern": "http://(ar|af|be).wikipedia.org/articles"
  },
  "store": {
    "type": "dummy",
    "options": {}
  },
  "url": {
    "blocks": 2
  },
  "robo": {
    "limit": 100
  }
}