Skip to content

Instantly share code, notes, and snippets.

@rmax
Created February 1, 2012 18:59
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rmax/1718659 to your computer and use it in GitHub Desktop.
Save rmax/1718659 to your computer and use it in GitHub Desktop.
--- a/scrapy/contrib/downloadermiddleware/httpcompression.py
+++ b/scrapy/contrib/downloadermiddleware/httpcompression.py
@@ -33,7 +33,11 @@ class HttpCompressionMiddleware(object):
def _decode(self, body, encoding):
if encoding == 'gzip' or encoding == 'x-gzip':
- body = gunzip(body)
+ try:
+ body = gunzip(body)
+ except IOError:
+ # leave body as is if can't decompress
+ pass
if encoding == 'deflate':
try:
@@ -44,6 +48,10 @@ class HttpCompressionMiddleware(object):
# http://carsten.codimi.de/gzip.yaws/
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
# http://www.gzip.org/zlib/zlib_faq.html#faq38
- body = zlib.decompress(body, -15)
+ try:
+ body = zlib.decompress(body, -15)
+ except zlib.error:
+ # leave body as is if can't decompress
+ pass
return body
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment