parallel_rsync_date_dirs.py

#!/usr/bin/env python
import datetime
import Queue
from threading import Thread
from subprocess import call

start_date = datetime.date(2015,10,10)
end_date = datetime.date.today()

delta = end_date - start_date

dates = []

q = Queue.LifoQueue()

for i in range(delta.days +1):
    dates.append(start_date + datetime.timedelta(days=i))

for date in dates:
    q.put(date.strftime('%Y%m%d'))

def grab_data_from_queue():
    while not q.empty():
        date_dir = q.get()
        call(['rsync', '-rpg', '/nfs/mediastore001/%s/' % date_dir, '/storage/mediastore001/%s/' % date_dir])
        print('done with %s' % date_dir)
        q.task_done()

for i in range(5):
    t1 = Thread(target = grab_data_from_queue)
    t1.start()

q.join()