-
-
Save barraponto/295df3142597fd1e2aa594b8f7bed75a to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def process_response(self, request, response, spider): | |
if request.method == 'HEAD': | |
return response | |
if response.status == 303: | |
return response | |
if isinstance(response, Response): | |
content_encoding = response.headers.getlist('Content-Encoding') | |
if content_encoding and not is_gzipped(response): | |
encoding = content_encoding.pop() | |
decoded_body = self._decode(response.body, encoding.lower()) | |
respcls = responsetypes.from_args(headers=response.headers, \ | |
url=response.url) | |
kwargs = dict(cls=respcls, body=decoded_body) | |
if issubclass(respcls, TextResponse): | |
# force recalculating the encoding until we make sure the | |
# responsetypes guessing is reliable | |
kwargs['encoding'] = None | |
response = response.replace(**kwargs) | |
if not content_encoding: | |
del response.headers['Content-Encoding'] | |
return response |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment