A scraping script that runs in multiple, parallel processes

scrape_in_parallel.py

import requests
from time import sleep
from multiprocessing import Pool

session = None

def set_global_session():
    global session
    if not session:
        session = requests.Session()

def cache_page(identifier):
    sleep(3)
    url = f'https://mycourts.in.gov/PORP/Search/Detail?ID={identifier}'
    r = session.get(url)
    html = r.content
    with open(f".cache/SearchDetail/{identifier}.html", 'wb') as file:
        file.write(html)
    return print(f'  Cached content from {url}')


if __name__ == "__main__":
    identifiers = [i for i in range(1, 60000)]
    with multiprocessing.Pool(initializer=set_global_session) as pool:
        pool.map(cache_page, identifiers)