requests/treq work

req_bench.py

from __future__ import print_function

"""
The "tests" here are intended to be used as benchmarks to help in performance
optimization.
"""

from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed

from time import time

import numpy
import requests

def requests_test():
    urls = [
        'https://derwolfe.net',
        'https://derwolfe.net/2016/08/08/cycling-my-gpg-key/',
        'https://derwolfe.net/2016/04/01/just-squash/'
        'https://derwolfe.net/2016/01/23/splitting-up-pull-requests/',
        'https://derwolfe.net/index.xml'
    ]
    fs = {}
    times = []
    with ThreadPoolExecutor(max_workers=10) as pool:
        start = time()
        for iteration in xrange(5):
            for url in urls:
                print('started', url)
                fs[pool.submit(requests.get, url, timeout=60)] = "{} x {}".format(url, iteration)

        for f in as_completed(fs, 60):
            url = fs[f]
            if f.exception() is not None:
                print(f, url)
            else:
                resp = f.result()
                print('done', len(resp.text))

        end = time()
        times.append(end - start)
        return times


def return_stats(times):
    # this had been running with other profiling code.
    ts = numpy.array(times)
    p50 = numpy.percentile(ts, 50)
    p99 = numpy.percentile(ts, 99)
    p2_5 = numpy.percentile(ts, 2.5)
    print('total runs: {}'.format(len(times)))
    print('p2.5 (secs): {}'.format(p2_5))
    print('p50 (secs): {}'.format(p50))
    print('p99 (secs): {}'.format(p99))


if __name__ == '__main__':
    times = requests_test()
    return_stats(times)

tx_bench.py

import time

import treq
from twisted.internet import reactor
from twisted.internet.defer import gatherResults


def fetch(url):
    print('fetching: ', url)
    d =  treq.get(url)

    def content(resp):
        return resp.text()

    def text(content):
        print('done - {} {}'.format(url, len(content)))
        return content

    def error(err):
        print(err)

    d.addCallback(content)
    d.addCallback(text)

    d.addErrback(error)
    return d


URLS = [
        'https://derwolfe.net',
        'https://derwolfe.net/2016/08/08/cycling-my-gpg-key/',
        'https://derwolfe.net/2016/04/01/just-squash/',
        'https://derwolfe.net/2016/01/23/splitting-up-pull-requests/',
        'https://derwolfe.net/index.xml'
        'https://derwolfe.net',
        'https://derwolfe.net/2016/08/08/cycling-my-gpg-key/',
        'https://derwolfe.net/2016/04/01/just-squash/',
        'https://derwolfe.net/2016/01/23/splitting-up-pull-requests/',
        'https://derwolfe.net/index.xml',
        'https://derwolfe.net',
        'https://derwolfe.net/2016/08/08/cycling-my-gpg-key/',
        'https://derwolfe.net/2016/04/01/just-squash/',
        'https://derwolfe.net/2016/01/23/splitting-up-pull-requests/',
        'https://derwolfe.net/index.xml',
    ]


def fetch_all():
    # get these URLs multiple times, wait on all
    d = gatherResults(
        [
            fetch(url)
            for url in URLS
        ]
    )
    return d

def run_it():
    print('started')
    t0 = time.time()

    d = fetch_all()

    def done(resps):
        t1 = time.time()
        print('done: {}'.format(t1 - t0))
        reactor.stop()

    d.addCallback(done)
    return d

if __name__ == '__main__':
    run_it()
    reactor.run()