Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Check compression in use
#!/usr/bin/env python
from multiprocessing.dummy import Pool
from collections import defaultdict
import requests
import logging
def uses_compression(domain):
logging.info(domain)
try:
r = requests.get("https://{}".format(domain), verify=False, timeout=2)
return 'Content-Encoding' in r.headers
except:
return "Unknown"
def main():
compression = defaultdict(int)
domains = [domain.rstrip() for domain in open("domains").readlines()]
p = Pool(100)
results = p.map(uses_compression, domains)
for result in results:
compression[result] += 1
print(compression)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.