Async Parallel HTTP GET requests through Proxy

burp-async.py

#!/usr/bin/env python
from multiprocessing.pool import ThreadPool
from time import time as timer
import requests



with open('domains.txt') as f:
    content = f.readlines()
urls = [x.strip() for x in content]

def fetch_url(url):
    proxies = {
      "http": "http://127.0.0.1:8080",
      "https": "http://127.0.0.1:8080",
    }
    try:
        headers = {'User-Agent': 'Mozilla/5.0'}
        response = requests.get(url, proxies=proxies, headers=headers, timeout=5)
        return url, response.status_code, None
    except Exception as e:
        return url, None, e

start = timer()
results = ThreadPool(100).imap_unordered(fetch_url, urls)
for url, html, error in results:
    if error is None:
        print("%r fetched in %ss" % (url, timer() - start))
    else:
        print("error fetching %r: %s" % (url, error))
print("Elapsed Time: %s" % (timer() - start,))