bash note

array

#http://stackoverflow.com/questions/15105135/bash-capturing-output-of-awk-into-array

lists=("http://www.youtube.com/watch?v=tKTZoB2Vjuk"
       "http://www.youtube.com/watch?v=EPYupizJYQI"
       "http://www.youtube.com/watch?v=haycL41dAhg"
       "http://www.youtube.com/watch?v=kWyoYtvJpe4"
       "http://www.youtube.com/watch?v=uKZ8GBKmeDM"
       "http://www.youtube.com/watch?v=Nn2KQmVF5Og"
       "http://www.youtube.com/watch?v=IcteAbMC1Ok");

for l in "${lists[@]}"
do
    echo "downloading $l"
    youtube-dl "$l" -f 22/35/34
done


#http://stackoverflow.com/questions/15105135/bash-capturing-output-of-awk-into-array
myarr=($(ps -u kdride | awk '{ print $1 }'))

# Now access elements of an array (change "1" to whatever you want)
echo ${myarr[1]}

# Or loop through every element in the array
for i in "${myarr[@]}"
do
   :
  echo $i
done

flume process

grep org.apache.flume.sink.hdfs.HDFSEventSink.process flume.log |awk -F'[\[\]]' '{print $1,$6,$8,$10,$12}'|awk '{hour=substr($4, 0, 3);all[hour]+=$6;tak[hour]+=$7;app[hour]+=$8;syn[hour]+=$9;count[hour]+=1;}END{for(i in all){print i,all[i]/count[i],tak[i]/count[i],app[i]/count[i],syn[i]/count[i], count[i]}}' |sort

sed in mac

sed -i'' -E "s:localhost:192.168.4.235:g" *

test memory param

#!/bin/bash

if [ $# -lt 1 ]; then
    echo "bad param, exit"
    exit 1
fi

flow=$1


read -a percents <<< "0.5 0.6 0.7 0.8 0.9"

for p in "${percents[@]}"
do
    echo "testing p is " $p
    cmd="sed -i -e 's/^.*set mapreduce.reduce.shuffle.input.buffer.percent=.*$/set mapreduce.reduce.shuffle.input.buffer.percent=$p;/' $flow"
    eval $cmd
    time sw run -t $flow -d -v --delta 1 --sd  1 > $flow.$p.out 2>&1
done