Hashlib example

hashlib_example.py

import hashlib
from timeit import timeit
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

a = b'1 2 3 4' * 1024 * 1024 * 100

funcs = [
    hashlib.sha1,
    hashlib.sha256,
    hashlib.sha384,
    hashlib.sha512,
]


def seq():
    return [
        func(a).hexdigest()
        for func in funcs
    ]


tpool = ThreadPoolExecutor(max_workers=4)
ppool = ProcessPoolExecutor(max_workers=4)


def apply(i):
    return funcs[i](a).hexdigest()


def threaded():
    return list(
        tpool.map(apply, range(len(funcs))))


def processed():
    return list(
        ppool.map(apply, range(len(funcs))))


if __name__ == "__main__":
    assert seq() == threaded()
    assert seq() == processed()

    number = 10
    print(timeit("seq()", globals=globals(), number=number))
    print(timeit("threaded()", globals=globals(), number=number))
    print(timeit("processed()", globals=globals(), number=number))

    # 13.097471375003806
    # 4.436675249999098
    # 4.756995249997999

hashlib_example2.py

import hashlib
from timeit import timeit
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

data = [
    b'1 2 3 4' * 1024 * 1024 * 100,
    b'2 2 3 4' * 1024 * 1024 * 100,
    b'3 3 3 4' * 1024 * 1024 * 100,
    b'4 4 4 4' * 1024 * 1024 * 100,
]


def seq():
    return [
        hashlib.sha512(a).hexdigest()
        for a in data
    ]


tpool = ThreadPoolExecutor(max_workers=4)
ppool = ProcessPoolExecutor(max_workers=4)


def apply(a):
    return hashlib.sha512(a).hexdigest()


def threaded():
    return list(
        tpool.map(apply, data))


def processed():
    return list(
        ppool.map(apply, data))


if __name__ == "__main__":
    assert seq() == threaded()
    assert seq() == processed()

    number = 10
    print(timeit("seq()", globals=globals(), number=number))
    print(timeit("threaded()", globals=globals(), number=number))
    print(timeit("processed()", globals=globals(), number=number))

    # 17.451598124993325
    # 4.525098208003328
    # 34.551474416999554