BS4ReadGzip

BS4ReadGzip.py

import sys
import re
import io
import gzip
from urllib.request import urlopen
from urllib.request import Request
from bs4 import BeautifulSoup

def getLinks(pageUrl):
  bs = urlopen(Request(pageUrl, headers={"User-Agent": "Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11", "Accept-Encoding": "gzip"}) ).read()
  bi = io.BytesIO(bs)
  gf = gzip.GzipFile(fileobj=bi, mode="rb")
  html = gf.read()
  bsObj = BeautifulSoup(html)
if __name__ == '__main__':
	getLinks(sys.argv[1])