Skip to content

Instantly share code, notes, and snippets.

@turingDH
Created March 26, 2019 00:12
Show Gist options
  • Save turingDH/01493da335b4fe42929cf5928ae87cae to your computer and use it in GitHub Desktop.
Save turingDH/01493da335b4fe42929cf5928ae87cae to your computer and use it in GitHub Desktop.
multiprocessing Pool example
## Credit to LucidProgramming https://www.youtube.com/watch?v=u2jTn-Gj2Xw for the walkthrough. My changes weren't terribly significant.
import os # to get core count on machine
import time # to time the duration
from multiprocessing import Pool # to instantiate a Pool of workers to distribute the process across the cores in CPU
def sum_square(number):
s=0
for i in range(1, number + 1):
s += i * i
return s
def sum_square_with_mp(numbers):
"""Call sum_square using multiprocessing"""
start_time = time.time()
p = Pool(os.cpu_count() - 1) # instatiate Pool object, but use n - 1 cores
result_multi = p.map(sum_square, numbers) # map maps the function onto the cores
p.close()
p.join()
end_time = time.time() - start_time
print(f"Processing {len(numbers):,} numbers took {end_time:,.4f} seconds using multiprocessing\nfinal result [-1] = {result_multi[-1]:,}\n")
def sum_square_no_mp(numbers):
"""Call sum_square using for loop on single core"""
start_time = time.time()
result_serial = []
for i in numbers:
result_serial.append(sum_square(i))
end_time = time.time() - start_time
print(f"Processing {len(numbers):,} numbers took {end_time:,.4f} seconds using serial processing\nfinal result [-1] = {result_serial[-1]:,}")
if __name__ == '__main__':
numbers = range(7_500) ## range needs to be large enough to cross inflection point where Pool() results in time savings.
sum_square_with_mp(numbers)
sum_square_no_mp(numbers)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment