Example of multi thread with arcpy process

multi_thread_example.py

import random
import time

from Queue import Queue
from threading import Thread

print 'Starting script!'

q = Queue()

def main():
	set_raster_env()

	make_job_list(q)
	
	process_queue()

def set_raster_env():

	# Commenting out for now because i don't have arcpy on this machine (or the data)
	print 'Setting raster environments . . .'
	
#	arcpy.env.cellSize = 0.00025
#	arcpy.env.outputCoordinateSystem = arcpy.SpatialReference("WGS 1984")
#	arcpy.env.snapRaster = r"Z:\Hansen Data Processing\lossyear 2014\loss_date_v2\00N_000E.tif"
	
def make_job_list(q):

	# Iterate over the inputs, making a list of all the tiles that we'll need to extract/build
	# For now, we'll just fake it
	
	for i in range(0, 100):
	
		# Create an emtpy dictionary that we'll fill with info about the job
		job = {}
		
		# Define everything that each worker will need to process the job
		job['type'] = 'FeatureToRaster_conversion'
		job['tile_num'] = i
		job['extent'] = [random.randint(0,10), random.randint(0,10), random.randint(0,10), random.randint(0,10)]
		job['output_raster'] = r"R:\analysis\mangroves\wgs84\cell_{}.tif".format(str(i).zfill(4))
		
		# Add the job to the queue
		q.put(job)
	

def process_queue():
    # Create multiple workers to process the queue
    num_threads = 4
    print 'Starting queue process now using {0} threads'.format(num_threads)
    for i in range(num_threads):
        worker = Thread(target=process_jobs, args=(i,))
        worker.setDaemon(True)
        worker.start()

    # Blocks the main thread of the program until the workers have finished the queue
    q.join()

    # Sleep for a second to finish logging all messages from the workers
    time.sleep(1)
    print 'Queued process complete'


def process_jobs(i):
    """This is the worker thread function.
    It processes items in the queue one after
    another.  These daemon threads go into an
    infinite loop, and only exit when
    the main thread ends.
    :param i: the ID of the worker
    """
    while True:
		job = q.get()
		time.sleep(1)
		
		worker_debug = 'Worker {0}, starting tile {1}, out file is: {2}\n'.format(i, job['tile_num'], job['output_raster'])
		
		# Really, this will look like this
		# Everything we need to parameterize/define a job,
		# should be in the job object, i.e. job['extent'], job['tile_num']
		#arcpy.FeatureToRaster_conversion (r"R:\temp\temp.gdb\mangroves", "const", out_raster, 0.00025)
		
		print worker_debug

		q.task_done()

			
if __name__ == '__main__':
    main()