Skip to content

Instantly share code, notes, and snippets.

@orangain
Created December 24, 2015 04:50
Show Gist options
  • Save orangain/c6293a47d4bb9756bc16 to your computer and use it in GitHub Desktop.
Save orangain/c6293a47d4bb9756bc16 to your computer and use it in GitHub Desktop.
# coding: utf-8
from __future__ import print_function
import scrapy
class TestSpider(scrapy.Spider):
name = 'test'
allowed_domains = ['capybala.com']
start_urls = (
'http://a.capybala.com/',
'http://b.capybala.com/',
'http://c.capybala.com/',
'http://d.capybala.com/',
'http://e.capybala.com/',
)
custom_settings = {
'DOWNLOAD_DELAY': 1,
'RANDOMIZE_DOWNLOAD_DELAY': False,
'CONCURRENT_REQUESTS_PER_IP': 1,
}
def parse(self, response):
print('Download slot:', response.request.meta['download_slot'])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment