Skip to content

Instantly share code, notes, and snippets.

@barraponto
Created July 26, 2016 15:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save barraponto/295df3142597fd1e2aa594b8f7bed75a to your computer and use it in GitHub Desktop.
Save barraponto/295df3142597fd1e2aa594b8f7bed75a to your computer and use it in GitHub Desktop.
def process_response(self, request, response, spider):
if request.method == 'HEAD':
return response
if response.status == 303:
return response
if isinstance(response, Response):
content_encoding = response.headers.getlist('Content-Encoding')
if content_encoding and not is_gzipped(response):
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
respcls = responsetypes.from_args(headers=response.headers, \
url=response.url)
kwargs = dict(cls=respcls, body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs['encoding'] = None
response = response.replace(**kwargs)
if not content_encoding:
del response.headers['Content-Encoding']
return response
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment