Last active
August 29, 2015 13:57
-
-
Save mejedi/9594696 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
http_read(f) - read HTTP request or response from a file or a file-like object. | |
http_parse(f) - like http_read(), returns parsed data | |
This is a stripped down httplib.py from Python standard library with small | |
changes. | |
""" | |
from array import array | |
import os | |
import socket | |
import zlib | |
from sys import py3kwarning | |
from urlparse import urlsplit | |
import warnings | |
with warnings.catch_warnings(): | |
if py3kwarning: | |
warnings.filterwarnings("ignore", ".*mimetools has been removed", | |
DeprecationWarning) | |
import mimetools | |
try: | |
from cStringIO import StringIO | |
except ImportError: | |
from StringIO import StringIO | |
__all__ = ["HTTPException", "NotConnected", "UnknownProtocol", | |
"UnknownTransferEncoding", "UnimplementedFileMode", | |
"IncompleteRead", "InvalidURL", "ImproperConnectionState", | |
"CannotSendRequest", "CannotSendHeader", "ResponseNotReady", | |
"BadStatusLine", "UnknownContentEncoding", "ContentDecodeError"] | |
HTTP_PORT = 80 | |
HTTPS_PORT = 443 | |
_UNKNOWN = 'UNKNOWN' | |
# status codes | |
# informational | |
CONTINUE = 100 | |
SWITCHING_PROTOCOLS = 101 | |
PROCESSING = 102 | |
# successful | |
OK = 200 | |
CREATED = 201 | |
ACCEPTED = 202 | |
NON_AUTHORITATIVE_INFORMATION = 203 | |
NO_CONTENT = 204 | |
RESET_CONTENT = 205 | |
PARTIAL_CONTENT = 206 | |
MULTI_STATUS = 207 | |
IM_USED = 226 | |
# redirection | |
MULTIPLE_CHOICES = 300 | |
MOVED_PERMANENTLY = 301 | |
FOUND = 302 | |
SEE_OTHER = 303 | |
NOT_MODIFIED = 304 | |
USE_PROXY = 305 | |
TEMPORARY_REDIRECT = 307 | |
# client error | |
BAD_REQUEST = 400 | |
UNAUTHORIZED = 401 | |
PAYMENT_REQUIRED = 402 | |
FORBIDDEN = 403 | |
NOT_FOUND = 404 | |
METHOD_NOT_ALLOWED = 405 | |
NOT_ACCEPTABLE = 406 | |
PROXY_AUTHENTICATION_REQUIRED = 407 | |
REQUEST_TIMEOUT = 408 | |
CONFLICT = 409 | |
GONE = 410 | |
LENGTH_REQUIRED = 411 | |
PRECONDITION_FAILED = 412 | |
REQUEST_ENTITY_TOO_LARGE = 413 | |
REQUEST_URI_TOO_LONG = 414 | |
UNSUPPORTED_MEDIA_TYPE = 415 | |
REQUESTED_RANGE_NOT_SATISFIABLE = 416 | |
EXPECTATION_FAILED = 417 | |
UNPROCESSABLE_ENTITY = 422 | |
LOCKED = 423 | |
FAILED_DEPENDENCY = 424 | |
UPGRADE_REQUIRED = 426 | |
# server error | |
INTERNAL_SERVER_ERROR = 500 | |
NOT_IMPLEMENTED = 501 | |
BAD_GATEWAY = 502 | |
SERVICE_UNAVAILABLE = 503 | |
GATEWAY_TIMEOUT = 504 | |
HTTP_VERSION_NOT_SUPPORTED = 505 | |
INSUFFICIENT_STORAGE = 507 | |
NOT_EXTENDED = 510 | |
# Mapping status codes to official W3C names | |
responses = { | |
100: 'Continue', | |
101: 'Switching Protocols', | |
200: 'OK', | |
201: 'Created', | |
202: 'Accepted', | |
203: 'Non-Authoritative Information', | |
204: 'No Content', | |
205: 'Reset Content', | |
206: 'Partial Content', | |
300: 'Multiple Choices', | |
301: 'Moved Permanently', | |
302: 'Found', | |
303: 'See Other', | |
304: 'Not Modified', | |
305: 'Use Proxy', | |
306: '(Unused)', | |
307: 'Temporary Redirect', | |
400: 'Bad Request', | |
401: 'Unauthorized', | |
402: 'Payment Required', | |
403: 'Forbidden', | |
404: 'Not Found', | |
405: 'Method Not Allowed', | |
406: 'Not Acceptable', | |
407: 'Proxy Authentication Required', | |
408: 'Request Timeout', | |
409: 'Conflict', | |
410: 'Gone', | |
411: 'Length Required', | |
412: 'Precondition Failed', | |
413: 'Request Entity Too Large', | |
414: 'Request-URI Too Long', | |
415: 'Unsupported Media Type', | |
416: 'Requested Range Not Satisfiable', | |
417: 'Expectation Failed', | |
500: 'Internal Server Error', | |
501: 'Not Implemented', | |
502: 'Bad Gateway', | |
503: 'Service Unavailable', | |
504: 'Gateway Timeout', | |
505: 'HTTP Version Not Supported', | |
} | |
# maximal amount of data to read at one time in _safe_read | |
MAXAMOUNT = 1048576 | |
# maximal line length when calling readline(). | |
_MAXLINE = 65536 | |
class HTTPMessage(mimetools.Message): | |
def addheader(self, key, value): | |
"""Add header for field key handling repeats.""" | |
prev = self.dict.get(key) | |
if prev is None: | |
self.dict[key] = value | |
else: | |
combined = ", ".join((prev, value)) | |
self.dict[key] = combined | |
def addcontinue(self, key, more): | |
"""Add more field data from a continuation line.""" | |
prev = self.dict[key] | |
self.dict[key] = prev + "\n " + more | |
def readheaders(self): | |
"""Read header lines. | |
Read header lines up to the entirely blank line that terminates them. | |
The (normally blank) line that ends the headers is skipped, but not | |
included in the returned list. If a non-header line ends the headers, | |
(which is an error), an attempt is made to backspace over it; it is | |
never included in the returned list. | |
The variable self.status is set to the empty string if all went well, | |
otherwise it is an error message. The variable self.headers is a | |
completely uninterpreted list of lines contained in the header (so | |
printing them will reproduce the header exactly as it appears in the | |
file). | |
If multiple header fields with the same name occur, they are combined | |
according to the rules in RFC 2616 sec 4.2: | |
Appending each subsequent field-value to the first, each separated | |
by a comma. The order in which header fields with the same field-name | |
are received is significant to the interpretation of the combined | |
field value. | |
""" | |
# XXX The implementation overrides the readheaders() method of | |
# rfc822.Message. The base class design isn't amenable to | |
# customized behavior here so the method here is a copy of the | |
# base class code with a few small changes. | |
self.dict = {} | |
self.unixfrom = '' | |
self.headers = hlist = [] | |
self.status = '' | |
headerseen = "" | |
firstline = 1 | |
startofline = unread = tell = None | |
if hasattr(self.fp, 'unread'): | |
unread = self.fp.unread | |
elif self.seekable: | |
tell = self.fp.tell | |
while True: | |
if tell: | |
try: | |
startofline = tell() | |
except IOError: | |
startofline = tell = None | |
self.seekable = 0 | |
line = self.fp.readline(_MAXLINE + 1) | |
if len(line) > _MAXLINE: | |
raise LineTooLong("header line") | |
if not line: | |
self.status = 'EOF in headers' | |
break | |
# Skip unix From name time lines | |
if firstline and line.startswith('From '): | |
self.unixfrom = self.unixfrom + line | |
continue | |
firstline = 0 | |
if headerseen and line[0] in ' \t': | |
# XXX Not sure if continuation lines are handled properly | |
# for http and/or for repeating headers | |
# It's a continuation line. | |
hlist.append(line) | |
self.addcontinue(headerseen, line.strip()) | |
continue | |
elif self.iscomment(line): | |
# It's a comment. Ignore it. | |
continue | |
elif self.islast(line): | |
# Note! No pushback here! The delimiter line gets eaten. | |
break | |
headerseen = self.isheader(line) | |
if headerseen: | |
# It's a legal header line, save it. | |
hlist.append(line) | |
self.addheader(headerseen, line[len(headerseen)+1:].strip()) | |
continue | |
else: | |
# It's not a header line; throw it back and stop here. | |
if not self.dict: | |
self.status = 'No headers' | |
else: | |
self.status = 'Non-header line where header expected' | |
# Try to undo the read. | |
if unread: | |
unread(line) | |
elif tell: | |
self.fp.seek(startofline) | |
else: | |
self.status = self.status + '; bad seek' | |
break | |
class HTTPParser: | |
# This one was called HTTPResponse in httplib.py | |
# strict: If true, raise BadStatusLine if the status line can't be | |
# parsed as a valid HTTP/1.0 or 1.1 status line. By default it is | |
# false because it prevents clients from talking to HTTP/0.9 | |
# servers. Note that a response with a sufficiently corrupted | |
# status line will look like an HTTP/0.9 response. | |
# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details. | |
def __init__(self, fp, debuglevel=0, strict=True, method=None): | |
self.fp = fp | |
self.debuglevel = debuglevel | |
self.strict = strict | |
self._method = method | |
self.msg = None | |
# from the Status-Line of the response | |
self.version = _UNKNOWN # HTTP-Version | |
self.status = _UNKNOWN # Status-Code | |
self.reason = _UNKNOWN # Reason-Phrase | |
self.chunked = _UNKNOWN # is "chunked" being used? | |
self.chunk_left = _UNKNOWN # bytes left to read in current chunk | |
self.length = _UNKNOWN # number of bytes left in response | |
self.will_close = _UNKNOWN # conn will close at end of response | |
def _read_status(self): | |
# Initialize with Simple-Response defaults | |
line = self.fp.readline(_MAXLINE + 1) | |
if len(line) > _MAXLINE: | |
raise LineTooLong("header line") | |
if self.debuglevel > 0: | |
print "reply:", repr(line) | |
if not line: | |
# Presumably, the server closed the connection before | |
# sending a valid response. | |
raise BadStatusLine(line) | |
try: | |
[version, status, reason] = line.split(None, 2) | |
except ValueError: | |
try: | |
[version, status] = line.split(None, 1) | |
reason = "" | |
except ValueError: | |
# empty version will cause next test to fail and status | |
# will be treated as 0.9 response. | |
version = "" | |
if not version.startswith('HTTP/'): | |
if self.strict: | |
self.close() | |
raise BadStatusLine(line) | |
else: | |
# assume it's a Simple-Response from an 0.9 server | |
self.fp = LineAndFileWrapper(line, self.fp) | |
return "HTTP/0.9", 200, "" | |
# The status code is a three-digit number | |
try: | |
status = int(status) | |
if status < 100 or status > 999: | |
raise BadStatusLine(line) | |
except ValueError: | |
raise BadStatusLine(line) | |
return version, status, reason | |
def begin(self): | |
if self.msg is not None: | |
# we've already started reading the response | |
return | |
# read until we get a non-100 response | |
while True: | |
version, status, reason = self._read_status() | |
if status != CONTINUE: | |
break | |
# skip the header from the 100 response | |
while True: | |
skip = self.fp.readline(_MAXLINE + 1) | |
if len(skip) > _MAXLINE: | |
raise LineTooLong("header line") | |
skip = skip.strip() | |
if not skip: | |
break | |
if self.debuglevel > 0: | |
print "header:", skip | |
self.status = status | |
self.reason = reason.strip() | |
if version == 'HTTP/1.0': | |
self.version = 10 | |
elif version.startswith('HTTP/1.'): | |
self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1 | |
elif version == 'HTTP/0.9': | |
self.version = 9 | |
else: | |
raise UnknownProtocol(version) | |
if self.version == 9: | |
self.length = None | |
self.chunked = 0 | |
self.will_close = 1 | |
self.msg = HTTPMessage(StringIO()) | |
return | |
self.msg = HTTPMessage(self.fp, 0) | |
if self.debuglevel > 0: | |
for hdr in self.msg.headers: | |
print "header:", hdr, | |
# don't let the msg keep an fp | |
self.msg.fp = None | |
# are we using the chunked-style of transfer encoding? | |
tr_enc = self.msg.getheader('transfer-encoding') | |
if tr_enc and tr_enc.lower() == "chunked": | |
self.chunked = 1 | |
self.chunk_left = None | |
else: | |
self.chunked = 0 | |
# will the connection close at the end of the response? | |
self.will_close = self._check_close() | |
# do we have a Content-Length? | |
# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" | |
length = self.msg.getheader('content-length') | |
if length and not self.chunked: | |
try: | |
self.length = int(length) | |
except ValueError: | |
self.length = None | |
else: | |
if self.length < 0: # ignore nonsensical negative lengths | |
self.length = None | |
else: | |
self.length = None | |
# does the body have a fixed length? (of zero) | |
if (status == NO_CONTENT or status == NOT_MODIFIED or | |
100 <= status < 200 or # 1xx codes | |
self._method == 'HEAD'): | |
self.length = 0 | |
# if the connection remains open, and we aren't using chunked, and | |
# a content-length was not provided, then assume that the connection | |
# WILL close. | |
if not self.will_close and \ | |
not self.chunked and \ | |
self.length is None: | |
self.will_close = 1 | |
def _check_close(self): | |
conn = self.msg.getheader('connection') | |
if self.version == 11: | |
# An HTTP/1.1 proxy is assumed to stay open unless | |
# explicitly closed. | |
conn = self.msg.getheader('connection') | |
if conn and "close" in conn.lower(): | |
return True | |
return False | |
# Some HTTP/1.0 implementations have support for persistent | |
# connections, using rules different than HTTP/1.1. | |
# For older HTTP, Keep-Alive indicates persistent connection. | |
if self.msg.getheader('keep-alive'): | |
return False | |
# At least Akamai returns a "Connection: Keep-Alive" header, | |
# which was supposed to be sent by the client. | |
if conn and "keep-alive" in conn.lower(): | |
return False | |
# Proxy-Connection is a netscape hack. | |
pconn = self.msg.getheader('proxy-connection') | |
if pconn and "keep-alive" in pconn.lower(): | |
return False | |
# otherwise, assume it will close | |
return True | |
def close(self): | |
self.fp = None | |
def isclosed(self): | |
# NOTE: it is possible that we will not ever call self.close(). This | |
# case occurs when will_close is TRUE, length is None, and we | |
# read up to the last byte, but NOT past it. | |
# | |
# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be | |
# called, meaning self.isclosed() is meaningful. | |
return self.fp is None | |
def read(self, amt=None): | |
if self.fp is None: | |
return '' | |
if self._has_no_body(): | |
self.close() | |
return '' | |
if self.chunked: | |
return self._read_chunked(amt) | |
if amt is None: | |
# unbounded read | |
if self.length is None: | |
s = self.fp.read() | |
else: | |
try: | |
s = self._safe_read(self.length) | |
except IncompleteRead: | |
self.close() | |
raise | |
self.length = 0 | |
self.close() # we read everything | |
return s | |
if self.length is not None: | |
if amt > self.length: | |
# clip the read to the "end of response" | |
amt = self.length | |
# we do not use _safe_read() here because this may be a .will_close | |
# connection, and the user is reading more bytes than will be provided | |
# (for example, reading in 1k chunks) | |
s = self.fp.read(amt) | |
if not s and amt: | |
# Ideally, we would raise IncompleteRead if the content-length | |
# wasn't satisfied, but it might break compatibility. | |
self.close() | |
if self.length is not None: | |
self.length -= len(s) | |
if not self.length: | |
self.close() | |
return s | |
def _has_no_body(self): | |
return self._method == 'HEAD' or 100<=self.status<=199 or self.status in (204, 304) | |
def _read_chunked(self, amt): | |
assert self.chunked != _UNKNOWN | |
chunk_left = self.chunk_left | |
value = [] | |
while True: | |
if chunk_left is None: | |
line = self.fp.readline(_MAXLINE + 1) | |
if len(line) > _MAXLINE: | |
raise LineTooLong("chunk size") | |
i = line.find(';') | |
if i >= 0: | |
line = line[:i] # strip chunk-extensions | |
try: | |
chunk_left = int(line, 16) | |
except ValueError: | |
# close the connection as protocol synchronisation is | |
# probably lost | |
self.close() | |
raise IncompleteRead(''.join(value)) | |
if chunk_left == 0: | |
break | |
if amt is None: | |
value.append(self._safe_read(chunk_left)) | |
elif amt < chunk_left: | |
value.append(self._safe_read(amt)) | |
self.chunk_left = chunk_left - amt | |
return ''.join(value) | |
elif amt == chunk_left: | |
value.append(self._safe_read(amt)) | |
self._safe_read(2) # toss the CRLF at the end of the chunk | |
self.chunk_left = None | |
return ''.join(value) | |
else: | |
value.append(self._safe_read(chunk_left)) | |
amt -= chunk_left | |
# we read the whole chunk, get another | |
self._safe_read(2) # toss the CRLF at the end of the chunk | |
chunk_left = None | |
# read and discard trailer up to the CRLF terminator | |
### note: we shouldn't have any trailers! | |
while True: | |
line = self.fp.readline(_MAXLINE + 1) | |
if len(line) > _MAXLINE: | |
raise LineTooLong("trailer line") | |
if not line: | |
# a vanishingly small number of sites EOF without | |
# sending the trailer | |
break | |
if line == '\r\n': | |
break | |
# we read everything; close the "file" | |
self.close() | |
return ''.join(value) | |
def _safe_read(self, amt): | |
"""Read the number of bytes requested, compensating for partial reads. | |
Normally, we have a blocking socket, but a read() can be interrupted | |
by a signal (resulting in a partial read). | |
Note that we cannot distinguish between EOF and an interrupt when zero | |
bytes have been read. IncompleteRead() will be raised in this | |
situation. | |
This function should be used when <amt> bytes "should" be present for | |
reading. If the bytes are truly not available (due to EOF), then the | |
IncompleteRead exception can be used to detect the problem. | |
""" | |
# NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never | |
# return less than x bytes unless EOF is encountered. It now handles | |
# signal interruptions (socket.error EINTR) internally. This code | |
# never caught that exception anyways. It seems largely pointless. | |
# self.fp.read(amt) will work fine. | |
s = [] | |
while amt > 0: | |
chunk = self.fp.read(min(amt, MAXAMOUNT)) | |
if not chunk: | |
raise IncompleteRead(''.join(s), amt) | |
s.append(chunk) | |
amt -= len(chunk) | |
return ''.join(s) | |
def fileno(self): | |
return self.fp.fileno() | |
def getheader(self, name, default=None): | |
if self.msg is None: | |
raise ResponseNotReady() | |
return self.msg.getheader(name, default) | |
def getheaders(self): | |
"""Return list of (header, value) tuples.""" | |
if self.msg is None: | |
raise ResponseNotReady() | |
return self.msg.items() | |
def http_parse(f, *args, **kv): | |
''' | |
Parse HTTP response | |
method='HEAD': tells the method of the corresponding request | |
(affects response parsing, Ex: response to HEAD | |
never has a body) | |
''' | |
class Res: pass | |
res = Res() | |
p = HTTPParser(f, *args, **kv) | |
p.begin() | |
res.version = p.version | |
res.status = p.status | |
res.reason = p.reason | |
res.headers = p.getheaders() | |
res.body = p.read() | |
encoding = p.getheader('content-encoding') | |
if encoding and encoding.lower() == 'gzip': | |
try: | |
res.body = zlib.decompress(res.body, 16+zlib.MAX_WBITS) | |
except zlib.error as e: | |
raise ContentDecodeError(e) | |
elif encoding and encoding.lower() == 'deflate': | |
try: | |
res.body = zlib.decompress(res.body) | |
except zlib.error: | |
try: | |
res.body = zlib.decompress(res.body, -zlib.MAX_WBITS) | |
except zlib.error as e: | |
raise ContentDecodeError(e) | |
elif encoding: | |
raise UnknownContentEncoding(encoding) | |
return res | |
def http_read(f, *args, **kv): | |
acc = [] | |
class AccWrapper: | |
def __getattr__(_, attr): | |
return getattr(f, attr) | |
def read(_, limit=-1): | |
d = f.read(limit) | |
acc.append(d) | |
return d | |
def readline(_, limit=-1): | |
ln = f.readline(limit) | |
acc.append(ln) | |
return ln | |
def readlines(_): | |
lns = f.readlines() | |
acc.extend(lns) | |
return lns | |
http_parse(AccWrapper(), *args, **kv) | |
return ''.join(acc) | |
class HTTPException(Exception): | |
# Subclasses that define an __init__ must call Exception.__init__ | |
# or define self.args. Otherwise, str() will fail. | |
pass | |
class NotConnected(HTTPException): | |
pass | |
class InvalidURL(HTTPException): | |
pass | |
class UnknownProtocol(HTTPException): | |
def __init__(self, version): | |
self.args = version, | |
self.version = version | |
class UnknownTransferEncoding(HTTPException): | |
pass | |
class UnimplementedFileMode(HTTPException): | |
pass | |
class IncompleteRead(HTTPException): | |
def __init__(self, partial, expected=None): | |
self.args = partial, | |
self.partial = partial | |
self.expected = expected | |
def __repr__(self): | |
if self.expected is not None: | |
e = ', %i more expected' % self.expected | |
else: | |
e = '' | |
return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e) | |
def __str__(self): | |
return repr(self) | |
class ImproperConnectionState(HTTPException): | |
pass | |
class CannotSendRequest(ImproperConnectionState): | |
pass | |
class CannotSendHeader(ImproperConnectionState): | |
pass | |
class ResponseNotReady(ImproperConnectionState): | |
pass | |
class BadStatusLine(HTTPException): | |
def __init__(self, line): | |
if not line: | |
line = repr(line) | |
self.args = line, | |
self.line = line | |
class LineTooLong(HTTPException): | |
def __init__(self, line_type): | |
HTTPException.__init__(self, "got more than %d bytes when reading %s" | |
% (_MAXLINE, line_type)) | |
class UnknownContentEncoding(HTTPException): | |
pass | |
class ContentDecodeError(HTTPException): | |
pass | |
class LineAndFileWrapper: | |
"""A limited file-like object for HTTP/0.9 responses.""" | |
# The status-line parsing code calls readline(), which normally | |
# get the HTTP status line. For a 0.9 response, however, this is | |
# actually the first line of the body! Clients need to get a | |
# readable file object that contains that line. | |
def __init__(self, line, file): | |
self._line = line | |
self._file = file | |
self._line_consumed = 0 | |
self._line_offset = 0 | |
self._line_left = len(line) | |
def __getattr__(self, attr): | |
return getattr(self._file, attr) | |
def _done(self): | |
# called when the last byte is read from the line. After the | |
# call, all read methods are delegated to the underlying file | |
# object. | |
self._line_consumed = 1 | |
self.read = self._file.read | |
self.readline = self._file.readline | |
self.readlines = self._file.readlines | |
def read(self, amt=None): | |
if self._line_consumed: | |
return self._file.read(amt) | |
assert self._line_left | |
if amt is None or amt > self._line_left: | |
s = self._line[self._line_offset:] | |
self._done() | |
if amt is None: | |
return s + self._file.read() | |
else: | |
return s + self._file.read(amt - len(s)) | |
else: | |
assert amt <= self._line_left | |
i = self._line_offset | |
j = i + amt | |
s = self._line[i:j] | |
self._line_offset = j | |
self._line_left -= amt | |
if self._line_left == 0: | |
self._done() | |
return s | |
def readline(self): | |
if self._line_consumed: | |
return self._file.readline() | |
assert self._line_left | |
s = self._line[self._line_offset:] | |
self._done() | |
return s | |
def readlines(self, size=None): | |
if self._line_consumed: | |
return self._file.readlines(size) | |
assert self._line_left | |
L = [self._line[self._line_offset:]] | |
self._done() | |
if size is None: | |
return L + self._file.readlines() | |
else: | |
return L + self._file.readlines(size) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment