Skip to content

Instantly share code, notes, and snippets.

@cabateman
Created May 3, 2011 18:07
Show Gist options
  • Save cabateman/953862 to your computer and use it in GitHub Desktop.
Save cabateman/953862 to your computer and use it in GitHub Desktop.
#!/bin/bash
rootfile=/home/ldap/chris.bates/usph_2011_03_01
file=/home/ldap/chris.bates/usph_isloading
lockfile=$file'.lock'
if [ ! -e $lockfile ]; then
trap "rm -f $lockfile; exit" INT TERM EXIT
touch $lockfile
#/usr/bin/python /home/ldap/chris.bates/gsreports/reports/scripts/queryImpressionTotalsDaily.py > $file
#maxlines=`wc $rootfile | awk '{print $1}'`
maxlines=2000
offset=1
limit=1000
echo "maxlines" $maxlines
while [ $offset -le $maxlines ]
do
end=$(($offset+$limit))
echo $offset
echo $end
echo $rootfile
echo $file
sed -n '$offset,$endp' $rootfile > $file
#cat $file | awk 'BEGIN{FS="\t"; s="INSERT INTO mshark_stats.HadoopUserSongPlayHistory_2011_03_01 (dateComputed,userid,songid,totalplays,totalskips,totalcompletes) VALUES "}{s=s"("$1","$2","$3","$4"),"}END{ print s=substr(s,0,length(s)-1)" ON DUPLICATE KEY UPDATE dateComputed=VALUES(dateComputed), userid=VALUES(userid), songid=VALUES(songid), totalplays=VALUES(totalplays),totalcompletes=VALUES(totalcompletes);"}' | mysql --host=172.16.0.36 -umshark_stats -pmshark_stats
echo "offset value" $offset
offset=$(($offset+$limit))
done
rm $lockfile
trap - INT TERM EXIT
else
echo "critical-section is already running"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment