Script to copy a bunch of folders using bsub

bsub_copy.py

# Uses bsub with rsync to copy large folders
#
# Uses a walltime limit of 60 hours (should be enough for very large folders)
#
# Runs in parallel across each subfolder of the source folder, and creates logs for each as well
#
# Usage: python bsub_copy.py /source/folder /dest/folder

import os
import sys
import subprocess


# Put the base of the source folder here
source_base = sys.argv[1]

# Put the destination folder here
dest_base = sys.argv[2]

# Fill this with folders from `source_base` above
source_folders = os.listdir(source_base)


for src in source_folders:

  cmd = [
    'bsub',

    # Set a high walltime
    '-W 3600',

    '-eo',
    '{}_err.log'.format(src),

    '-oo',
    '{}_out.log'.format(src),

    'rsync -azvh {}/{} {}'.format(source_base, src, dest_base)
  ]

  subprocess.call(cmd)