Skip to content

Instantly share code, notes, and snippets.

@twirrim
Created February 6, 2019 21:21
Show Gist options
  • Save twirrim/877bcaf373aa1fec99c102b7c84ea1ce to your computer and use it in GitHub Desktop.
Save twirrim/877bcaf373aa1fec99c102b7c84ea1ce to your computer and use it in GitHub Desktop.
Check compression in use
#!/usr/bin/env python
from multiprocessing.dummy import Pool
from collections import defaultdict
import requests
import logging
def uses_compression(domain):
logging.info(domain)
try:
r = requests.get("https://{}".format(domain), verify=False, timeout=2)
return 'Content-Encoding' in r.headers
except:
return "Unknown"
def main():
compression = defaultdict(int)
domains = [domain.rstrip() for domain in open("domains").readlines()]
p = Pool(100)
results = p.map(uses_compression, domains)
for result in results:
compression[result] += 1
print(compression)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment