Copy files from Broad to Google

copy-to-google.sh

#!/usr/bin/bash
#$ -N MY_JOB_NAME
#$ -o /path/on/broad/server/
#$ -e /path/on/broad/server/
#$ -cwd
#$ -j y
#$ -l h_rt=12:00:00
#$ -pe smp 8 -R y -binding linear:8
#$ -l h_vmem=2G
#
# copy-to-google.sh
# Kamil Slowikowski
# October 13, 2020
#
# This script requires gsutil: https://cloud.google.com/storage/docs/gsutil_install
#
# Please read the instructions below to learn how to run this script.
#
# Step 1: Connect your laptop to the login server:
#
#     ssh BROAD_USERNAME@login.broadinstitute.org
#
# Step 2: On the login server, enable UGER so we can run qsub:
#
#     use UGER
#
# Step 3: Modify this script to use your paths for -o and -e and
# name your job with the -N option:
#
#     vim copy-to-google.sh
#
# Step 4: Run this script with qsub:
#
#     qsub copy-to-google.sh
#

# Full path to gsutil
gsutil=/home/unix/MY_BROAD_USERNAME/google-cloud-sdk/bin/gsutil

# Full path to mail
mail=/usr/bin/mail

# Who should be notified after this job is completed?
# (Multiple addresses should be delimited with commas.)
recipients=MY_BROAD_USERNAME@broadinstitute.org

# Full path to the destination in a Google bucket
des=gs://fc-123c2456-abcd-efgh-ijkl-7f12f1012345/sequencing_data

# List of folders to copy to the destination
dirs=(
/seq/illumina_ext/SL-NVG/201008_SL-NVG_0000_AAAAAAAAAA
/seq/illumina_ext/SL-NVG/201008_SL-NVG_0000_BBBBBBBBBB
)

for d in ${dirs[*]}
do
  slug=${d:(-9)}
  t0=`date +%s`
  cmd="$gsutil -m cp -r -n ${d} ${des}/"
  eval $cmd
  t1=`date +%s`
  duration=$((($t1-$t0)/60))
  echo -e "Finished copying $slug from Broad to Google Cloud.\n\nElapsed time: $duration minutes\n\nCommand:\n\n$cmd" | $mail -s "Finished copying sequencing data" $recipients
done

Please find additional instructions here (requires Broad Institute account):

• https://broad.service-now.com/sp?id=kb_article&sys_id=63354be4137f0b00449fb86f3244b049