urllib2 vs urllib3 vs requests

request_time.py

#!/usr/bin/env python2.7

import time

_URL = 'http://localhost/tmp/derp.html'
_NUMBER = 1000


def test_urllib2():
    import urllib2
    try:
        response = urllib2.urlopen(_URL)
    except urllib2.HTTPError, e:
        response = e
    response.code
    return response.read()


def test_urllib3():
    import urllib3
    http = urllib3.PoolManager()
    response = http.request('GET', _URL)
    response.status
    return response.data


def test_requests():
    import requests
    response = requests.get(_URL)
    response.status_code
    return response.text


if __name__ == '__main__':
    from timeit import Timer
    t_urllib2 = Timer("test_urllib2()", "from __main__ import test_urllib2")
    print '{0} urllib2: {1}'.format(_NUMBER,  t_urllib2.timeit(number=_NUMBER))
    t_urllib3 = Timer("test_urllib3()", "from __main__ import test_urllib3")
    print '{0} urllib3: {1}'.format(_NUMBER,  t_urllib3.timeit(number=_NUMBER))
    t_requests = Timer("test_requests()", "from __main__ import test_requests")
    print '{0} requests: {1}'.format(_NUMBER,  t_requests.timeit(number=_NUMBER))

and what are the results? :)

This is what I got for my URL:

1000 urllib2: 6.06175804138
1000 urllib3: 6.29859209061
1000 requests: 7.85270094872

try this:

import requests
s = requests.Session()
def test_requests():
    response = s.get(_URL)
    response.status_code
    return response.text

You should be benchmarking using response.content instead of response.text.

When you receive a response, Requests makes a guess at the encoding to use for decoding the response when you access the Response.text attribute. Requests will first check for an encoding in the HTTP header, and if none is present, will use chardet to attempt to guess the encoding.

response.content however will return the response body as bytes, without spending extra time on the .text encoding.

Difference for a CDN image download:

response.text:
100 urllib2: 13.555662634
100 urllib3: 12.5924083333
100 requests: 162.567124378

response.content:
100 urllib2: 13.1599570738
100 urllib3: 12.8453221719
100 requests: 13.8536482152

How to convert the following code to make use of urllib3 instead of urllib

    def update_email(self, device, email, backup):
        return json.loads(urlopen('%s/devices/%s/email?key=%s' % (
            self.path, device, self.key), urlencode({
                'from_email': email,
                'backup_email': backup
            })).read())

_URL = 'https://baidu.com'
10 urllib2: 0.8630119720000948
10 urllib3: 22.80423276600004
10 requests: 3.069073326999842