Testing Shotgun threading

stdout.txt

$ python threaded_dump.py 
..
2 in 1.157s (578.617ms each)

...............
17 in 3.286s (193.311ms each)

................
33 in 4.767s (144.452ms each)

done with Task
......
40 in 5.587s (139.681ms each)

threaded_dump.py

import threading
import sys
import time

from shotgun_api3 import Shotgun

import sgsession

shotgun = Shotgun('https://mikeboers.shotgunstudio.com/', login='xxx', password='yyy')
shotgun.config.timeout_secs = 5
shotgun.config.max_rpc_attempts = 1

thread_count = 16

session = sgsession.Session(shotgun)

children = {
    'Project': ('Sequence', 'project'),
    'Sequence': ('Shot', 'sg_sequence'),
    'Shot': ('Task', 'entity'),
}

def target(*args):
    entities.extend(session.find(*args))
    sys.stdout.write('.')
    sys.stdout.flush()

entities = session.find('Project', [])

start = time.time()
count = 0

while entities:
    to_query = entities[:thread_count]
    entities = entities[thread_count:]
    
    threads = []
    for entity in to_query:
        if entity['type'] not in children:
            print 'done with', entity['type']
            continue
        child_type, field = children[entity['type']]
        thread = threading.Thread(target=target, args=(child_type, [(field, 'is', entity.minimal)]))
        threads.append(thread)
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()
        
    print
    count += len(to_query)
    duration = time.time() - start

    print '%d in %.3fs (%.3fms each)' % (count, duration, 1000 * duration / count)
    print