measure how much time it takes for requests to make many localhost requests

requests_localhost.py

"""
measure how much time it takes for requests to make many localhost requests

try do so some optimizations


initial results on on an old mac:

$ python req.py http 100  
itrg              0.3003
tprg              0.1765
pprg              0.1835
itsg              0.3104
tpsg              0.1891
ppsg              0.1454

$ python req.py https 100 
itrg              1.6038
tprg              1.0321
pprg              0.8545
itsg              1.5903
tpsg              0.9304
ppsg              0.7672

 ncalls  tottime  percall  cumtime  percall filename:lineno(function)
      100    0.962    0.010    0.962    0.010 {method 'set_default_verify_paths' of '_ssl._SSLContext' objects}
      100    0.176    0.002    0.176    0.002 {method 'do_handshake' of '_ssl._SSLSocket' objects}
      100    0.171    0.002    0.176    0.002 socket.py:769(close)
      100    0.057    0.001    0.057    0.001 {built-in method _socket.getaddrinfo}
      100    0.045    0.000    0.045    0.000 {built-in method _socket.gethostbyname}
      200    0.030    0.000    0.030    0.000 {method 'read' of '_ssl._SSLSocket' objects}
      ...


after "reuse SSL context hack":

$ python req.py https 100 
itrg              1.6163
tprg              0.9732
pprg              0.8012
itsg              0.4873 <-- 
tpsg              0.2557 <-- 
ppsg              0.2515


after "socket.gethostbyname hack":

$ python req.py https 100 
itrg              1.5148
tprg              0.9661
pprg              0.7917
itsg              0.4462
tpsg              0.2374 <--
ppsg              0.2298


"""
import cProfile
import http.server
import multiprocessing.dummy
import requests
import ssl
import sys
import time
import warnings
from functools import partial

class Handler(http.server.BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200)
        self.end_headers()
    def log_message(self, *_):
        pass

class HTTPSServer(http.server.HTTPServer):
    # https://gist.github.com/lemon24/2dcb9c89ebe9c268f29572b90f53adf4
    protocol = 'https'
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.socket = ssl.wrap_socket(
            self.socket, 
            # openssl req -new -x509 -keyout server.pem -out server.pem -days 365 -nodes
            certfile='server.pem', 
            server_side=True,
        )
    
servers = {
    'http': http.server.HTTPServer,
    'https': HTTPSServer,
}

def run_server(Server, port, barrier):
    server = Server(('', port), Handler)
    barrier.wait()
    server.serve_forever()


proto = sys.argv[1]
nprocs = int(sys.argv[2])

Server = servers[proto]

warnings.filterwarnings('ignore', module='urllib3.connectionpool')
                        
def do_iter(get=requests.get):
    for i in range(nprocs):
        get(f'{proto}://localhost:{8000+i}', verify=False)
    
def do_pool(get, pool):
    get = partial(get, verify=False)
    args = (f'{proto}://localhost:{8000+i}' for i in range(nprocs))
    for _ in pool.imap_unordered(get, args):
        pass
    

session = requests.Session()


# BEGIN reuse SSL context hack

# https://github.com/psf/requests/issues/4322
# https://github.com/psf/requests/pull/5971/files

from urllib3.util.ssl_ import create_urllib3_context

SSL_CONTEXT = create_urllib3_context()
SSL_CONTEXT.check_hostname = False
SSL_CONTEXT.verify_mode = ssl.CERT_NONE

class HTTPAdapter(requests.adapters.HTTPAdapter):
    def init_poolmanager(
        self,
        connections,
        maxsize, 
        block=requests.adapters.DEFAULT_POOLBLOCK, 
        **pool_kwargs
    ):
        return super().init_poolmanager(
            connections, maxsize, block=block, ssl_context=SSL_CONTEXT, **pool_kwargs
        )

session.mount("https://", HTTPAdapter())
session.mount("http://", HTTPAdapter())

# END reuse SSL context hack


# BEGIN socket.gethostbyname hack

from functools import lru_cache
import socket

socket.gethostbyname = lru_cache(socket.gethostbyname)

# END socket.gethostbyname hack


if __name__ == '__main__':
    barrier = multiprocessing.Barrier(nprocs+1)
    
    processes = []
    for i in range(nprocs):
        process = multiprocessing.Process(
            target=run_server,
            args=(Server, 8000+i, barrier),
            daemon=True,
        )
        process.start()
        # print('.', end='')
        processes.append(process)

    barrier.wait()
    # print('\nready')
    
    tpool = multiprocessing.dummy.Pool(10)
    ppool = multiprocessing.Pool(10)

    do_iter()
    
    # cProfile.run("do_iter(session.get)", sort='tottime')
    # exit()
    
    fns = {
        'itrg': partial(do_iter, requests.get),
        'tprg': partial(do_pool, requests.get, tpool),
        'pprg': partial(do_pool, requests.get, ppool),
        'itsg': partial(do_iter, session.get),
        'tpsg': partial(do_pool, session.get, tpool),
        'ppsg': partial(do_pool, session.get, ppool),
    }
    
    for name, fn in fns.items():
        start = time.time()
        fn()
        end = time.time()
        print(f"{name:15} {end-start:8.4f}")