Multi-threaded vs multi-process vs serial https client

client_multi_process.py

#!/usr/bin/env python

import requests
import time
import copy
from multiprocessing import Process

from data import URLS

def send_request(num, url):
    res = requests.get(url)
    print("%d: %s -> %s" % (num, url, str(res.status_code)))

def main():

    results = []
    process = []

    start_time = time.time()

    num = 0
    for url in URLS:
        proc = Process(target=send_request, args=(num, url))
        num += 1
        process.append(proc)

    for proc in process:
        proc.start()

    for proc in process:
        proc.join(timeout=10)

    end_time = time.time()

    print('Total time: %s' % (end_time - start_time))

    print(results)

if __name__ == '__main__':
    main()

client_multi_thread.py

#!/usr/bin/env python

import requests
import time
from threading import Thread
from Queue import Queue

from data import URLS

def send_request(url, num, queue):
    res = requests.get(url)
    print("%d: %s -> %s" % (num, url, str(res.status_code)))
    queue.put(res)

def main():

    results = []
    threads = []
    queue = Queue(maxsize=100)

    start_time = time.time()

    num = 0
    for url in URLS:
        thread = Thread(target=send_request, args=(url, num, queue))
        num += 1
        threads.append(thread)

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join(timeout=10)

    while not queue.empty():
        res = queue.get()
        queue.task_done()
        results.append(res)
    
    queue.join()

    end_time = time.time()

    print('Total time: %s' % (end_time - start_time))

    print(results)


if __name__ == '__main__':
    main()

client_serial.py

#!/usr/bin/env python

import requests
import time

from data import URLS

results = []

start_time = time.time()

for url in URLS:
    print(url)
    res = requests.get(url)
    print(res.status_code)
    results.append(res)

end_time = time.time()

print('Total time: %s' % (end_time - start_time))

data.py

URLS = [
    'https://www.google.com',
    'https://www.seznam.cz',
    'https://www.root.cz',
    'https://www.linux.org/',
    'https://www.nic.cz/',
    'https://www.nix.cz/',
    'https://www.redhat.com',
    'https://www.kernel.org/',
    'https://getfedora.org/',
    'https://www.projectatomic.io/',
    'https://www.openshift.org/',
    'https://www.openstack.org/',
    'https://www.docker.com/',
    'https://kubernetes.io/',
    'https://nodejs.org/',
    'https://angularjs.org/',
    'https://getbootstrap.com/',
    'https://reactjs.org/',
    'https://www.gnome.org/',
    'https://www.freedesktop.org/',
    'https://www.google.com',
    'https://www.seznam.cz',
    'https://www.root.cz',
    'https://www.linux.org/',
    'https://www.nic.cz/',
    'https://www.nix.cz/',
    'https://www.redhat.com',
    'https://www.kernel.org/',
    'https://getfedora.org/',
    'https://www.projectatomic.io/',
    'https://www.openshift.org/',
    'https://www.openstack.org/',
    'https://www.docker.com/',
    'https://kubernetes.io/',
    'https://nodejs.org/',
    'https://angularjs.org/',
    'https://getbootstrap.com/',
    'https://reactjs.org/',
    'https://www.gnome.org/',
    'https://www.freedesktop.org/',
]

# URLS = [
#     'https://www.google.com',
#     'https://www.seznam.cz',
# ]

results.txt

client_serial.py .......... 29.3 s
client_multi_thread.py .... 1.7 s
client_multi_process.py ... 1.7 s