fetch emr logs

fetch_emr_logs.sh

#!/bin/bash -e

# args --------------------------------------------------------
## $1 - root
## $2 - jobflow id

# optional args ----------------------------------------------- 
## $3 - type:
##         'default' - stdout, stderr, syslog, controller
##         'jobs' - jobs logs; 
##         'tasks' - task attempts

# if tasks then also
## $4 - job_id
## $5 - task_attempt name
## $6 - target, default to logs/jobflow-id


# -- arg -----------------------------------
USAGE="1 - root, 2 - jobflowid, optional: 3 - type, 4 - jobid , 5 - task_attempt name, 6 - target (optional, default = logs/jobflowid)"

if [ -z "$1" ]
	then
		echo "Missing log root, usage: $USAGE"
fi

if [ -z "$2" ]
	then
		echo "Missing jobflow id, usage: $USAGE"
fi

TYPE="$3"
if [ -z "$TYPE" ]
	then 
		TYPE="default"
fi

# -- log root ---------------------------------
LOG_ROOT="$1/$2"
echo "dl-ing logs at $LOG_ROOT, type = $TYPE"


# --- path ------------------------------------
PATH="$LOG_ROOT/steps/1/*"
if [ "default" = "$TYPE" ]
	then
		PATH="$LOG_ROOT/steps/1/*"
elif [ "jobs" = "$TYPE" ]
	then
		PATH="$LOG_ROOT/jobs/*"
elif [ "task_attempts" = "$TYPE" ]
	then
		PATH="$LOG_ROOT/task-attempts/$4/$5*/*"
else
	echo "oops"
fi


# --- target ----------------------------------
TARGET="$6"
if [ -z "$TARGET" ]
	then 
		TARGET="logs/$2"
fi 

echo "make target $TARGET"
/bin/mkdir -p "$TARGET"

echo "run s3cmd get, path = $PATH, target = $TARGET"
/usr/bin/s3cmd get "$PATH" "$TARGET"