-
-
Save sureshkumargondi/5d3a88525eb4e3fae84b79ec8a93d8eb to your computer and use it in GitHub Desktop.
Crawl multiple websites with one for loop, while saving the output, logs, and job status separately for each website. Resume crawling any time simply be re-running the same code
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from urllib.parse import urlsplit | |
import advertools as adv | |
sites = [ | |
'https://www.who.int', | |
'https://www.nytimes.com', | |
'https://www.washingtonpost.com', | |
] | |
for site in sites: | |
domain = urlsplit(site).netloc | |
adv.crawl(site, | |
output_file=domain + '.jl', | |
follow_links=True, | |
custom_settings={ | |
'LOG_FILE': domain + '.log', | |
# change this to any number of pages | |
'CLOSESPIDER_PAGECOUNT': 50, | |
# resume the same crawl jobs later | |
'JOBDIR': domain | |
}) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment