grab a file with sftp and put on hdfs

getcsv.sh

!/bin/bash

export tnow=$(date +"%Y-%m-%d")
export hnow=$(date +"%Y-%m-%d:%H")

export d_fname=ItemFulfillment-$tnow.csv
export h_fname=ItemFulfillment-$hnow.csv
export dirname=ItemFulfillment
export FTP_SERVER='IP'

grab_clean_upload () {
	fname=$1
	now=$2
	sftp -o identityfile=~/.ssh/sftp_hadoop USER@$FTP_SERVER:/incoming/NetSuite/$dirname/$fname .
	echo "numFiles="`ls -l $fname|wc -l`
	hdfs dfs -put $fname /etl/NetSuite/landing/$dirname
	awk -- 'NR>1'  $fname | hdfs dfs -put - /etl/NetSuite/cleancsv/$dirname/$now/$fname
	echo "hnow=$now"
}

grab_clean_upload $d_fname $tnow
#grab_clean_upload $h_fname

#echo "hnow=$hnow"