Created
June 21, 2016 20:21
-
-
Save mappingvermont/1a77fc111b49271e17205971ab0cf846 to your computer and use it in GitHub Desktop.
Example of multi thread with arcpy process
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import random | |
import time | |
from Queue import Queue | |
from threading import Thread | |
print 'Starting script!' | |
q = Queue() | |
def main(): | |
set_raster_env() | |
make_job_list(q) | |
process_queue() | |
def set_raster_env(): | |
# Commenting out for now because i don't have arcpy on this machine (or the data) | |
print 'Setting raster environments . . .' | |
# arcpy.env.cellSize = 0.00025 | |
# arcpy.env.outputCoordinateSystem = arcpy.SpatialReference("WGS 1984") | |
# arcpy.env.snapRaster = r"Z:\Hansen Data Processing\lossyear 2014\loss_date_v2\00N_000E.tif" | |
def make_job_list(q): | |
# Iterate over the inputs, making a list of all the tiles that we'll need to extract/build | |
# For now, we'll just fake it | |
for i in range(0, 100): | |
# Create an emtpy dictionary that we'll fill with info about the job | |
job = {} | |
# Define everything that each worker will need to process the job | |
job['type'] = 'FeatureToRaster_conversion' | |
job['tile_num'] = i | |
job['extent'] = [random.randint(0,10), random.randint(0,10), random.randint(0,10), random.randint(0,10)] | |
job['output_raster'] = r"R:\analysis\mangroves\wgs84\cell_{}.tif".format(str(i).zfill(4)) | |
# Add the job to the queue | |
q.put(job) | |
def process_queue(): | |
# Create multiple workers to process the queue | |
num_threads = 4 | |
print 'Starting queue process now using {0} threads'.format(num_threads) | |
for i in range(num_threads): | |
worker = Thread(target=process_jobs, args=(i,)) | |
worker.setDaemon(True) | |
worker.start() | |
# Blocks the main thread of the program until the workers have finished the queue | |
q.join() | |
# Sleep for a second to finish logging all messages from the workers | |
time.sleep(1) | |
print 'Queued process complete' | |
def process_jobs(i): | |
"""This is the worker thread function. | |
It processes items in the queue one after | |
another. These daemon threads go into an | |
infinite loop, and only exit when | |
the main thread ends. | |
:param i: the ID of the worker | |
""" | |
while True: | |
job = q.get() | |
time.sleep(1) | |
worker_debug = 'Worker {0}, starting tile {1}, out file is: {2}\n'.format(i, job['tile_num'], job['output_raster']) | |
# Really, this will look like this | |
# Everything we need to parameterize/define a job, | |
# should be in the job object, i.e. job['extent'], job['tile_num'] | |
#arcpy.FeatureToRaster_conversion (r"R:\temp\temp.gdb\mangroves", "const", out_raster, 0.00025) | |
print worker_debug | |
q.task_done() | |
if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment