test_spider.py

# coding: utf-8

from __future__ import print_function
import scrapy


class TestSpider(scrapy.Spider):
    name = 'test'
    allowed_domains = ['capybala.com']
    start_urls = (
        'http://a.capybala.com/',
        'http://b.capybala.com/',
        'http://c.capybala.com/',
        'http://d.capybala.com/',
        'http://e.capybala.com/',
    )
    custom_settings = {
        'DOWNLOAD_DELAY': 1,
        'RANDOMIZE_DOWNLOAD_DELAY': False,
        'CONCURRENT_REQUESTS_PER_IP': 1,
    }

    def parse(self, response):
        print('Download slot:', response.request.meta['download_slot'])