Skip to content

Instantly share code, notes, and snippets.

@sahilbadyal
Created March 5, 2019 11:32
Show Gist options
  • Save sahilbadyal/3990fe16712cd670e6b39460960e6377 to your computer and use it in GitHub Desktop.
Save sahilbadyal/3990fe16712cd670e6b39460960e6377 to your computer and use it in GitHub Desktop.
Assumes SSH access allowed between nodes of cluster
#!/bin/bash
##Author: Sahil Badyal <sbadyals@gmail.com>
usage()
{
echo "usage: sysinfo_page [[[-f file ] [-i]] | [-h]]"
}
if [ "1" == "" ]; then
usage
exit
fi
##Main
## bash run_distributed.sh -o <o path> -i <i path> -bs 512 -s 800 -e 4000 -f 477
while [ "$1" != "" ]; do
case $1 in
-o | --output ) shift
out_dir=$1
;;
-i | --input ) shift
input_dir=$1
;;
-f | --feature_lene) shift
flen=$1
;;
-bs | --batch-size ) shift
batch_size=$1
;;
-s | --save-every ) shift
save=$1
;;
-e | --eval-every ) shift
eval_=$1
;;
-pb | --prefetch-buffer ) shift
pb=$1
;;
-sb | --shuffle-buffer ) shift
sb=$1
;;
-h | --help ) usage
exit
;;
* ) usage
exit 1
esac
shift
done
chief=$(echo $TF_CONFIG | jq '.cluster' | jq '.chief' | jq '.[]' | sed 's/:2222//g' | tr -d '"')
ips=$(echo $TF_CONFIG | jq '.cluster' | jq '.worker' | jq '.[]' | sed 's/:2222//g' | tr -d '"')
tot_workers=$(echo $TF_CONFIG | jq '.cluster' | jq '.worker' | jq '.[]' | wc -l)
tot_workers=$((tot_workers+1))
ips_ps=$(echo $TF_CONFIG | jq '.cluster' | jq '.ps' | jq '.[]' | sed 's/:2222//g' | tr -d '"')
read -a host <<< $ips
read -a ps <<< $ips_ps
k=0
python <path to file> -rdb -bs $batch_size -i $input_dir -pb $pb -sb $sb -o $out_dir -fl $flen -s $save -e $eval_ -tw $tot_workers -wi $k > /dev/null 2>&1 &
for x in "${ps[@]}"
do
com="python <path to file> -rdb -bs $batch_size -i $input_dir -pb $pb -sb $sb -o $out_dir -fl $flen -s $save -e $eval_ -tw $tot_workers -wi $k > /dev/null 2>&1 &"
ssh -o "StrictHostKeyChecking no" hadoop@$x $com
echo $com
done
k=$((k+1))
for x in "${host[@]}"
do
com="python <path to file> -rdb -i $input_dir -bs $batch_size -pb $pb -sb $sb -o $out_dir -fl $flen -s $save -e $eval_ -tw $tot_workers -wi $k > /dev/null 2>&1 &"
ssh -o "StrictHostKeyChecking no" hadoop@$x $com
k=$((k+1))
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment