Simplest single-core multi-cpu parallel run on Datarmor

multiprocessing_eg_02datarmor.py

#!/usr/bin/env python
#
# This example shows how to launch multiple processes in parallel on a single machine with multiple cpus
# There is no communication between processes and no data are gathered in the end.
# Each process executes the same function but with different arguments.
# The script wait for all sub-processes to be done, then execute another task.
# 
# How to run on your computer:
# 	python multiprocessing_eg_02.py
# 
# How to run on datarmor:
#   qsub -q sequentiel -N test_pythonpar launch_this.sh
#
# The launcher script is: launch_this.sh with:
# ======================================================================
# #!/bin/bash
# source ~/.bashrc
# source activate obidam
# python /home1/datahome/gmaze/work/Play_Ground/python/parallel/examples/multiprocessing_eg_02datarmor.py
# exit 0
# ======================================================================
#
# Eg:
# >> qsub -q sequentiel -N test_python multiprocessing_eg_02.py
# >> qstat -u gmaze
# 
# datarmor0:
#                                                             Req'd  Req'd   Elap
# Job ID          Username Queue    Jobname    SessID NDS TSK Memory Time  S Time
# --------------- -------- -------- ---------- ------ --- --- ------ ----- - -----
# 508731.datarmor gmaze    ice_1t   test_pytho  23442   1   1  500mb 00:05 R 00:00
#    r2i2n10/4
#
# >> more test_pythonpar.o508731
# ======================================================================
# multiprocessing_eg_02.py.
# This machine has 56 cores.
# ======================================================================
# 
# 	- Process named: dummy_name_1 (running on r2i2n10) just started.
# 	It received argument:
# 	61
# 
# 	- Process named: dummy_name_2 (running on r2i2n10) just started.
# 	It received argument:
# 	73
# 
# 	- Process named: dummy_name_7 (running on r2i2n10) just started.
# 	It received argument:
# 	33
# [...]
#
# End of the main program (this text will be displayed after all processes finished)
# 

# Created: 2013-02-06.
# Copyright (c) 2013, Guillaume Maze (Ifremer, Laboratoire de Physique des Oceans).
# Revision by G. Maze on 2017-11-23: Added help for datarmor
# All rights reserved.

import sys, socket, time
import multiprocessing
import numpy as np

def worker(num):
	"""
		A worker function: defines the task to be performed by a process.
		This function is executed by each process with a 
		different argument 'num'
	"""
	# Retrieve the name of this process:
	procname = multiprocessing.current_process().name
	# Retrive the name of the machine:
	hostname = socket.gethostname()
	
	# Print out some usefull info:
	msg = "\t- Process named: %s (running on %s) just started.\n\tIt received argument:\n\t%i\n\n"
	sys.stdout.write(msg % (procname,hostname,num))

	# Dummy action, simple wait for anywhere between 4 and 5 seconds
	time.sleep(np.random.randint(4,5,1)) 


if __name__ == '__main__':

	# How many core have this machine ?
	N_CORES = multiprocessing.cpu_count()
	print ('%s\nmultiprocessing_eg_02.py.\nThis machine has %i cores.\n%s\n')%("="*70,N_CORES,"="*70)

	# Init the list which is going to hold the list of processes:
	process_list = []
	
	# Create a dummy list of parameters.
	param_list = np.random.randint(20,100,N_CORES)
	
	# Now we will start N_CORES processes.
	# All will run the 'worker' function with a dummy parameter from 'param_list'
	for i in range(N_CORES):
		# Define a process:
		p = multiprocessing.Process(target=worker, args=(param_list[i],), name=('dummy_name_%i')%(i))
		# Start the process:
		p.start() 
		# Register the process in a list
		process_list.append(p)

	# Now we tell the script to wait for all processes to finish before moving on
	for p in process_list:
		p.join()

	# This won't be executed before the end of all processes:
	print "End of the main program (this text will be displayed after all processes finished)\n"