sbamin/sracmd

## sracmd
#!/bin/bash

## Wrapper to interact with NCBI SRA using aspera ascp command line interface
## v 2.5 | 04/2017
## @sbamin

# More info: http://download.asperasoft.com/download/docs/ascp/2.7/html/

# usage
show_help() {
cat << EOF

Wrapper to move data via aspera ascp command line interface

Required parameters:
    See usage below. Please avoid spaces and special characters or may quote those directories (untested)
    If needed, add required login password at the line containing ASPERA_SCP_PASS variable or export ASPERA_SCP_PASS in bash environment.
    If aspera command line client is not installed, visit http://downloads.asperasoft.com

Usage: ${0##*/} -s <Absolute path to source directory to be uploaded: One at a time>

    -h display this help and exit
    -s Required: Absolute path to source or target directory to be uploaded or downloaded in recursive manner: One at a time; path to file is untested
    -u Required: Aspera faspex server URL: foo@faspex.example.com
    -m Required: UP for upload, DOWN for download; All CAPS
    -p Optional: Only for downloading from remote host - Use path to source directory on a remote host which will be appended to faspex server url; path to file is untested
    -a Optional: Absolute path to aspera connect bin directory (Default is OS specific ascp bin dir path)
    -e Optional: etc path for linux stations where dsa key is being stored - useful for downloads from NCBI SRA (Default is OS specific ascp bin dir path)

    PS: Avoid using HPC default ascp at /usr/local/bin/ascp or other system default
    ascp -A should show version, 3.5.6.112424 or higher

Example Upload: ${0##*/} -s /localhost/user/upload_dir -u foo@faspex.example.com -m UP
Example Download: ${0##*/} -s /localhost/user/download_dir -u foo@faspex.example.com -p /remote_home/foo/source_dir -m DOWN

EOF
}

if [[ $# == 0 ]];then show_help;exit 1;fi

while getopts "s:a:u:m:p:e:h" opt; do
    case "$opt" in
        h) show_help;exit 0;;
        s) SOURCEDIR=$OPTARG;;
        a) ASCPBIN=$OPTARG;;
        e) ASCPETC=$OPTARG;;
        u) FASPURL=$OPTARG;;
        p) REMOTEPATH=$OPTARG;;
        m) ASCPMODE=$OPTARG;;
       '?')show_help >&2 exit 1 ;;
    esac
done

MYPWD="$(pwd)"

#### identify running OS ####
unamestr="$(uname)"

if [[ "$unamestr" == 'Linux' ]]; then
    ## locate ascp binary, faspex url, and create log directory to save logs
    ASCPBIN=${ASCPBIN:-"$HOME/.aspera/connect/bin"}
    ASCPETC=${ASCPETC:-"$HOME/.aspera/connect/etc"}
elif [[ "$unamestr" == 'Darwin' ]]; then
    ## For Mac OS, replace ASCPBIN with following. Make sure that path exists on your mac.
    ## For Mac OS, ASCPETC is same as ASCPBIN
    ASCPBIN=${ASCPBIN:-"$HOME/Applications/Aspera Connect.app/Contents/Resources"}
    ASCPETC=${ASCPETC:-"$HOME/Applications/Aspera Connect.app/Contents/Resources"}
else
    echo -e "\nWARN: Unable to detect operating system.\nThis may throw an error later unless ascp binary and key paths are specificed using -a and -e flags, respectively."
fi

#### Default upload/download faspex server ####
FASPURL=${FASPURL:-"anonftp@ftp.ncbi.nlm.nih.gov"}

#### default log directory ####
ASCPLOG=${HOME}/logs/ascp
mkdir -p "${ASCPLOG}"/up
mkdir -p "${ASCPLOG}"/down

#### confirm if source directory can be read and executed ####
if [[ ! -d "${SOURCEDIR}" ]] || [[ -z "${SOURCEDIR}" ]]; then
    echo -e "\nERROR: can not locate upload/download directory: ${SOURCEDIR}\nIt should be specified with -s flag and should have read-execute permission\n"
    ls -al "${SOURCEDIR}"
    show_help
    exit 1
fi

#### exit with error if source directory does not have read-exe permission ####
cd "${SOURCEDIR}"
if [[ "$?" != 0 ]]; then
    echo -e "\nERROR: can not change to upload/download directory: ${SOURCEDIR}\nIt should be specified with -s flag and should have read-execute permission\n"
    ls -al "${SOURCEDIR}"
    show_help
    exit 1
fi

#### return to pwd from where command was executed ####
cd "${MYPWD}"

## confirm for aspera ascp binary
if [[ ! -s "${ASCPBIN}"/ascp ]] || [[ ! -s "${ASCPETC}"/asperaweb_id_dsa.openssh ]]; then
    echo -e "\nERROR: Unable to locate ascp binary at ${ASCPBIN}/ascp or ascp dsa key at ${ASCPETC}/asperaweb_id_dsa.openssh\nYou may supply valid location via -a or -e <location to aspera bin or dsa key directory>\nDo not enter file name, i.e., ascp or asperaweb_id_dsa.openssh while supplying paths using -a and -e flags."
    echo -e "\nINFO: Visit http://downloads.asperasoft.com if you do not have aspera command line client installed\n################################\n"
    show_help
    exit 1
fi

## confirm for aspera faspex remote host
if [[ -z "${FASPURL}" ]]; then
    echo -e "\nERROR: Missing aspera faspex remote host address\nYou should supply valid remote address via -u <foo@faspex.example.com>\n"
    show_help
    exit 1
fi

##### Begin ascp download #####

if [[ "${ASCPMODE}" == "UP" ]]; then

        ## override system ascp if present by prepending updated ascp to PATH
        export PATH=${ASCPBIN}:${PATH}
        ## To skip password prompts, uncomment and add password
        #export ASPERA_SCP_PASS=12345

        TSTAMP=$(date +%d%b%y_%H%M%S%Z)
        ASCP_LOGFILE=$(printf "%s/up/ascp_%s_%s.log" "${ASCPLOG}" "${USER}" "${TSTAMP}")

        echo -e "\n##### Begin ascp upload on server: $(hostname) at ${TSTAMP}\nSource dir is ${SOURCEDIR}\nTarget faspex server is ${FASPURL}\nLog file at ${ASCP_LOGFILE}\n#####\n" | tee -a "${ASCP_LOGFILE}"

        ## slack api
        MSG=$(printf "\`\`\`\n#####\nBegin ascp upload on server: %s at %s\nSource dir is %s\nTarget faspex server is %s\nLog file at %s\n#####\n\`\`\`" "$(hostname)" "${TSTAMP}" "${SOURCEDIR}" "${FASPURL}" "${ASCP_LOGFILE}")

        if [[ -x "${HOME}"/bin/pingme ]]; then
            "${HOME}"/bin/pingme -m "${MSG}" -i hourglass -c helixbot > /dev/null 2>&1 &
        fi

        ## PS: Assuming SRA upload requires password and not a key based login. If key is required, make necessary changes here by adding -i or additional flags and supply ssh key.

        "${ASCPBIN}"/ascp -TQ -l 300m -m 1m -L "${ASCPLOG}" "${SOURCEDIR}" "${FASPURL}":.
        EXITSTAT="$?"

        TSTAMPEND=$(date +%d%b%y_%H%M%S%Z)
        echo -e "\n#####\nascp upload finished for ${SOURCEDIR} with exit status of ${EXITSTAT} on server: $(hostname) at ${TSTAMPEND}\nTarget faspex server was ${FASPURL}\nSee logfile at ${ASCP_LOGFILE} for transfer status\n#####\n" | tee -a "${ASCP_LOGFILE}"

        ## slack api
        MSGEND=$(printf "\`\`\`\n#####\nascp upload finished for %s with exit status of %s on server: %s at %s\nTarget faspex server was %s\nSee logfile at %s for transfer status\n#####\n\`\`\`" "${SOURCEDIR}" "${EXITSTAT}" "$(hostname)" "${TSTAMPEND}" "${FASPURL}" "${ASCP_LOGFILE}")

        if [[ -x "${HOME}"/bin/pingme ]]; then
            "${HOME}"/bin/pingme -m "${MSGEND}" -i white_check_mark -c helixbot > /dev/null 2>&1 &
        fi

elif [[ "${ASCPMODE}" == "DOWN" ]]; then

        ## override system ascp if present by prepending updated ascp to PATH
        export PATH=${ASCPBIN}:${PATH}
        ## To skip password prompts, uncomment and add password below
        #export ASPERA_SCP_PASS=12345

        TSTAMP=$(date +%d%b%y_%H%M%S%Z)
        ASCP_LOGFILE=$(printf "%s/down/ascp_%s_%s.log" "${ASCPLOG}" "${USER}" "${TSTAMP}")

        echo -e "\n##### Begin ascp download on server: $(hostname) at ${TSTAMP}\nTarget dir is ${SOURCEDIR}\nSource faspex server is ${FASPURL}:${REMOTEPATH}\nLog file at ${ASCP_LOGFILE}\n#####\n" | tee -a "${ASCP_LOGFILE}"

        ## slack api
        MSG=$(printf "\`\`\`\n#####\nBegin ascp download on server: %s at %s\nTarget dir is %s\nSource faspex server is %s:%s\nLog file at %s\n#####\n\`\`\`" "$(hostname)" "${TSTAMP}" "${SOURCEDIR}" "${FASPURL}" "${REMOTEPATH}" "${ASCP_LOGFILE}")

        if [[ -x "${HOME}"/bin/pingme ]]; then
            "${HOME}"/bin/pingme -m "${MSG}" -i hourglass -c helixbot > /dev/null 2>&1 &
        fi

        if [[ -z "${REMOTEPATH}" ]]; then
            echo -e "\n#### WARNING #####\nNo remote path specified using -p flag\ndefaults to home directory for ${FASPURL}:.\nTHIS WILL DOWNLOAD ALL FILES FROM THE REMOTE SERVER\nCTRL C to CANCEL ANYTIME DURING PROGRESS\n"

            read -erp "Are you sure to download all of  remote server's contents? Type YES at your own risk!" USERFBK

            if [[ "${USERFBK}" == "YES" ]]; then
                echo -e "\n#### WARNING ####\nDownloading all of files from ${FASPURL}\nUse CTLR C to abort or good luck!\n"
                "${ASCPBIN}"/ascp -i "${ASCPETC}"/asperaweb_id_dsa.openssh -k 1 -TQ -l 1200m -m 1m -L "${ASCPLOG}" "${FASPURL}":. "${SOURCEDIR}"
                EXITSTAT="$?"
            else
                echo -e "Thanks and you were wise! You can use -p flag to supply specific remoe directory path."
                exit 1
            fi

        else
            echo -e "Remote path was specified using -p flag\nDownloadig from ${FASPURL}:${REMOTEPATH}"
            "${ASCPBIN}"/ascp -i "${ASCPETC}"/asperaweb_id_dsa.openssh -k 1 -TQ -l 1200m -m 1m -L "${ASCPLOG}" "${FASPURL}:${REMOTEPATH}" "${SOURCEDIR}"
            EXITSTAT="$?"
        fi

        TSTAMPEND=$(date +%d%b%y_%H%M%S%Z)
        echo -e "\n#####\nascp download finished for ${SOURCEDIR} with exit status of ${EXITSTAT} on server: $(hostname) at ${TSTAMPEND}\nSource faspex server was ${FASPURL}:${REMOTEPATH}\nSee logfile at ${ASCP_LOGFILE} for transfer status\n#####\n" | tee -a "${ASCP_LOGFILE}"

        ## slack api
        MSGEND=$(printf "\`\`\`\n#####\nascp download finished for %s with exit status of %s on server: %s at %s\nSource faspex server was %s:%s\nSee logfile at %s for transfer status\n#####\n\`\`\`" "${SOURCEDIR}" "${EXITSTAT}" "$(hostname)" "${TSTAMPEND}" "${FASPURL}" "${REMOTEPATH}" "${ASCP_LOGFILE}")

        if [[ -x "${HOME}"/bin/pingme ]]; then
            "${HOME}"/bin/pingme -m "${MSGEND}" -i white_check_mark -c helixbot > /dev/null 2>&1 &
        fi

else
        echo -e "\nERROR: Incorrect ASCP mode at user supplied -m: ${ASCPMODE}\nIt should be either UP for upload or DOWN (all CAPS) for download to/from aspera faspex server\n"
        show_help
        exit 1
fi

#END
	#!/bin/bash

	## Wrapper to interact with NCBI SRA using aspera ascp command line interface
	## v 2.5 \| 04/2017
	## @sbamin

	# More info: http://download.asperasoft.com/download/docs/ascp/2.7/html/

	# usage
	show_help() {
	cat << EOF

	Wrapper to move data via aspera ascp command line interface

	Required parameters:
	See usage below. Please avoid spaces and special characters or may quote those directories (untested)
	If needed, add required login password at the line containing ASPERA_SCP_PASS variable or export ASPERA_SCP_PASS in bash environment.
	If aspera command line client is not installed, visit http://downloads.asperasoft.com

	Usage: ${0##*/} -s <Absolute path to source directory to be uploaded: One at a time>

	-h display this help and exit
	-s Required: Absolute path to source or target directory to be uploaded or downloaded in recursive manner: One at a time; path to file is untested
	-u Required: Aspera faspex server URL: foo@faspex.example.com
	-m Required: UP for upload, DOWN for download; All CAPS
	-p Optional: Only for downloading from remote host - Use path to source directory on a remote host which will be appended to faspex server url; path to file is untested
	-a Optional: Absolute path to aspera connect bin directory (Default is OS specific ascp bin dir path)
	-e Optional: etc path for linux stations where dsa key is being stored - useful for downloads from NCBI SRA (Default is OS specific ascp bin dir path)

	PS: Avoid using HPC default ascp at /usr/local/bin/ascp or other system default
	ascp -A should show version, 3.5.6.112424 or higher

	Example Upload: ${0##*/} -s /localhost/user/upload_dir -u foo@faspex.example.com -m UP
	Example Download: ${0##*/} -s /localhost/user/download_dir -u foo@faspex.example.com -p /remote_home/foo/source_dir -m DOWN

	EOF
	}

	if [[ $# == 0 ]];then show_help;exit 1;fi

	while getopts "s:a:u:m:p:e:h" opt; do
	case "$opt" in
	h) show_help;exit 0;;
	s) SOURCEDIR=$OPTARG;;
	a) ASCPBIN=$OPTARG;;
	e) ASCPETC=$OPTARG;;
	u) FASPURL=$OPTARG;;
	p) REMOTEPATH=$OPTARG;;
	m) ASCPMODE=$OPTARG;;
	'?')show_help >&2 exit 1 ;;
	esac
	done

	MYPWD="$(pwd)"

	#### identify running OS ####
	unamestr="$(uname)"

	if [[ "$unamestr" == 'Linux' ]]; then
	## locate ascp binary, faspex url, and create log directory to save logs
	ASCPBIN=${ASCPBIN:-"$HOME/.aspera/connect/bin"}
	ASCPETC=${ASCPETC:-"$HOME/.aspera/connect/etc"}
	elif [[ "$unamestr" == 'Darwin' ]]; then
	## For Mac OS, replace ASCPBIN with following. Make sure that path exists on your mac.
	## For Mac OS, ASCPETC is same as ASCPBIN
	ASCPBIN=${ASCPBIN:-"$HOME/Applications/Aspera Connect.app/Contents/Resources"}
	ASCPETC=${ASCPETC:-"$HOME/Applications/Aspera Connect.app/Contents/Resources"}
	else
	echo -e "\nWARN: Unable to detect operating system.\nThis may throw an error later unless ascp binary and key paths are specificed using -a and -e flags, respectively."
	fi

	#### Default upload/download faspex server ####
	FASPURL=${FASPURL:-"anonftp@ftp.ncbi.nlm.nih.gov"}

	#### default log directory ####
	ASCPLOG=${HOME}/logs/ascp
	mkdir -p "${ASCPLOG}"/up
	mkdir -p "${ASCPLOG}"/down

	#### confirm if source directory can be read and executed ####
	if [[ ! -d "${SOURCEDIR}" ]] \|\| [[ -z "${SOURCEDIR}" ]]; then
	echo -e "\nERROR: can not locate upload/download directory: ${SOURCEDIR}\nIt should be specified with -s flag and should have read-execute permission\n"
	ls -al "${SOURCEDIR}"
	show_help
	exit 1
	fi

	#### exit with error if source directory does not have read-exe permission ####
	cd "${SOURCEDIR}"
	if [[ "$?" != 0 ]]; then
	echo -e "\nERROR: can not change to upload/download directory: ${SOURCEDIR}\nIt should be specified with -s flag and should have read-execute permission\n"
	ls -al "${SOURCEDIR}"
	show_help
	exit 1
	fi

	#### return to pwd from where command was executed ####
	cd "${MYPWD}"

	## confirm for aspera ascp binary
	if [[ ! -s "${ASCPBIN}"/ascp ]] \|\| [[ ! -s "${ASCPETC}"/asperaweb_id_dsa.openssh ]]; then
	echo -e "\nERROR: Unable to locate ascp binary at ${ASCPBIN}/ascp or ascp dsa key at ${ASCPETC}/asperaweb_id_dsa.openssh\nYou may supply valid location via -a or -e <location to aspera bin or dsa key directory>\nDo not enter file name, i.e., ascp or asperaweb_id_dsa.openssh while supplying paths using -a and -e flags."
	echo -e "\nINFO: Visit http://downloads.asperasoft.com if you do not have aspera command line client installed\n################################\n"
	show_help
	exit 1
	fi

	## confirm for aspera faspex remote host
	if [[ -z "${FASPURL}" ]]; then
	echo -e "\nERROR: Missing aspera faspex remote host address\nYou should supply valid remote address via -u <foo@faspex.example.com>\n"
	show_help
	exit 1
	fi

	##### Begin ascp download #####

	if [[ "${ASCPMODE}" == "UP" ]]; then

	## override system ascp if present by prepending updated ascp to PATH
	export PATH=${ASCPBIN}:${PATH}
	## To skip password prompts, uncomment and add password
	#export ASPERA_SCP_PASS=12345

	TSTAMP=$(date +%d%b%y_%H%M%S%Z)
	ASCP_LOGFILE=$(printf "%s/up/ascp_%s_%s.log" "${ASCPLOG}" "${USER}" "${TSTAMP}")

	echo -e "\n##### Begin ascp upload on server: $(hostname) at ${TSTAMP}\nSource dir is ${SOURCEDIR}\nTarget faspex server is ${FASPURL}\nLog file at ${ASCP_LOGFILE}\n#####\n" \| tee -a "${ASCP_LOGFILE}"

	## slack api
	MSG=$(printf "\`\`\`\n#####\nBegin ascp upload on server: %s at %s\nSource dir is %s\nTarget faspex server is %s\nLog file at %s\n#####\n\`\`\`" "$(hostname)" "${TSTAMP}" "${SOURCEDIR}" "${FASPURL}" "${ASCP_LOGFILE}")

	if [[ -x "${HOME}"/bin/pingme ]]; then
	"${HOME}"/bin/pingme -m "${MSG}" -i hourglass -c helixbot > /dev/null 2>&1 &
	fi

	## PS: Assuming SRA upload requires password and not a key based login. If key is required, make necessary changes here by adding -i or additional flags and supply ssh key.

	"${ASCPBIN}"/ascp -TQ -l 300m -m 1m -L "${ASCPLOG}" "${SOURCEDIR}" "${FASPURL}":.
	EXITSTAT="$?"

	TSTAMPEND=$(date +%d%b%y_%H%M%S%Z)
	echo -e "\n#####\nascp upload finished for ${SOURCEDIR} with exit status of ${EXITSTAT} on server: $(hostname) at ${TSTAMPEND}\nTarget faspex server was ${FASPURL}\nSee logfile at ${ASCP_LOGFILE} for transfer status\n#####\n" \| tee -a "${ASCP_LOGFILE}"

	## slack api
	MSGEND=$(printf "\`\`\`\n#####\nascp upload finished for %s with exit status of %s on server: %s at %s\nTarget faspex server was %s\nSee logfile at %s for transfer status\n#####\n\`\`\`" "${SOURCEDIR}" "${EXITSTAT}" "$(hostname)" "${TSTAMPEND}" "${FASPURL}" "${ASCP_LOGFILE}")

	if [[ -x "${HOME}"/bin/pingme ]]; then
	"${HOME}"/bin/pingme -m "${MSGEND}" -i white_check_mark -c helixbot > /dev/null 2>&1 &
	fi

	elif [[ "${ASCPMODE}" == "DOWN" ]]; then

	## override system ascp if present by prepending updated ascp to PATH
	export PATH=${ASCPBIN}:${PATH}
	## To skip password prompts, uncomment and add password below
	#export ASPERA_SCP_PASS=12345

	TSTAMP=$(date +%d%b%y_%H%M%S%Z)
	ASCP_LOGFILE=$(printf "%s/down/ascp_%s_%s.log" "${ASCPLOG}" "${USER}" "${TSTAMP}")

	echo -e "\n##### Begin ascp download on server: $(hostname) at ${TSTAMP}\nTarget dir is ${SOURCEDIR}\nSource faspex server is ${FASPURL}:${REMOTEPATH}\nLog file at ${ASCP_LOGFILE}\n#####\n" \| tee -a "${ASCP_LOGFILE}"

	## slack api
	MSG=$(printf "\`\`\`\n#####\nBegin ascp download on server: %s at %s\nTarget dir is %s\nSource faspex server is %s:%s\nLog file at %s\n#####\n\`\`\`" "$(hostname)" "${TSTAMP}" "${SOURCEDIR}" "${FASPURL}" "${REMOTEPATH}" "${ASCP_LOGFILE}")

	if [[ -x "${HOME}"/bin/pingme ]]; then
	"${HOME}"/bin/pingme -m "${MSG}" -i hourglass -c helixbot > /dev/null 2>&1 &
	fi

	if [[ -z "${REMOTEPATH}" ]]; then
	echo -e "\n#### WARNING #####\nNo remote path specified using -p flag\ndefaults to home directory for ${FASPURL}:.\nTHIS WILL DOWNLOAD ALL FILES FROM THE REMOTE SERVER\nCTRL C to CANCEL ANYTIME DURING PROGRESS\n"

	read -erp "Are you sure to download all of remote server's contents? Type YES at your own risk!" USERFBK

	if [[ "${USERFBK}" == "YES" ]]; then
	echo -e "\n#### WARNING ####\nDownloading all of files from ${FASPURL}\nUse CTLR C to abort or good luck!\n"
	"${ASCPBIN}"/ascp -i "${ASCPETC}"/asperaweb_id_dsa.openssh -k 1 -TQ -l 1200m -m 1m -L "${ASCPLOG}" "${FASPURL}":. "${SOURCEDIR}"
	EXITSTAT="$?"
	else
	echo -e "Thanks and you were wise! You can use -p flag to supply specific remoe directory path."
	exit 1
	fi

	else
	echo -e "Remote path was specified using -p flag\nDownloadig from ${FASPURL}:${REMOTEPATH}"
	"${ASCPBIN}"/ascp -i "${ASCPETC}"/asperaweb_id_dsa.openssh -k 1 -TQ -l 1200m -m 1m -L "${ASCPLOG}" "${FASPURL}:${REMOTEPATH}" "${SOURCEDIR}"
	EXITSTAT="$?"
	fi

	TSTAMPEND=$(date +%d%b%y_%H%M%S%Z)
	echo -e "\n#####\nascp download finished for ${SOURCEDIR} with exit status of ${EXITSTAT} on server: $(hostname) at ${TSTAMPEND}\nSource faspex server was ${FASPURL}:${REMOTEPATH}\nSee logfile at ${ASCP_LOGFILE} for transfer status\n#####\n" \| tee -a "${ASCP_LOGFILE}"

	## slack api
	MSGEND=$(printf "\`\`\`\n#####\nascp download finished for %s with exit status of %s on server: %s at %s\nSource faspex server was %s:%s\nSee logfile at %s for transfer status\n#####\n\`\`\`" "${SOURCEDIR}" "${EXITSTAT}" "$(hostname)" "${TSTAMPEND}" "${FASPURL}" "${REMOTEPATH}" "${ASCP_LOGFILE}")

	if [[ -x "${HOME}"/bin/pingme ]]; then
	"${HOME}"/bin/pingme -m "${MSGEND}" -i white_check_mark -c helixbot > /dev/null 2>&1 &
	fi

	else
	echo -e "\nERROR: Incorrect ASCP mode at user supplied -m: ${ASCPMODE}\nIt should be either UP for upload or DOWN (all CAPS) for download to/from aspera faspex server\n"
	show_help
	exit 1
	fi

	#END