cached_parallel_requests.py

import bz2

import orjson
from requests.adapters import HTTPAdapter, Retry
from requests_cache import CachedSession, SerializerPipeline, Stage, serializers
from requests_futures.sessions import FuturesSession

SERIALIZER = SerializerPipeline([
    serializers.preconf.orjson_preconf_stage,
    orjson,
    Stage(dumps=bz2.compress, loads=bz2.decompress),
], is_binary=True)

API_TOKEN = ''
NUM_WORKERS = 10

http_session = CachedSession('http_cache', serializer=SERIALIZER, backend='filesystem')
http_session.headers.update({
            'User-Agent':'hamster_bth/ 1.0',
            'Accept': 'application/vnd.github+json',
            'Authorization': f'Bearer {API_TOKEN}',
        })

retries = Retry(total=5,
    connect=5,
    backoff_factor=2,
    status_forcelist=[500, 501, 502, 503, 504],
    raise_on_status=False)

http_session.mount('https://', HTTPAdapter(max_retries=retries))
http_session.mount('http://', HTTPAdapter(max_retries=retries))

with FuturesSession(max_workers=NUM_WORKERS, session=http_session) as future_session:
    http_session.get('http://httpbin.org/get')