node_migration.sh

#!/usr/bin/env bash

SCRIPT_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
LOGS_DIR=${LOGS_DIR:-"./migration-logs"}

usage() {
  echo "node_migration.sh <command>"
  echo "commands:"
  echo "  migrate        Run the migration scripts"
  echo "  summary        Parse BSO logs for migration summary"
  exit 1
}

# validate number of command line arguments
if [[ $# < 1 ]]; then
  usage
fi

migrate() {
  for i in `seq 0 19`
  do
    GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" user_migration/venv/bin/python user_migration/migrate_node.py --dsns user_migration/dsns.lst --sort_users --deanon --fxa_file user_migration/580.csv --start_bso "${i}" --end_bso "${i}" --user_percent=1:50 > "./migration-logs/bso-${i}-1.out" &
    GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" user_migration/venv/bin/python user_migration/migrate_node.py --dsns user_migration/dsns.lst --sort_users --deanon --fxa_file user_migration/580.csv --start_bso "${i}" --end_bso "${i}" --user_percent=2:50 > "./migration-logs/bso-${i}-2.out" &
  done
}

migrate-anon() {
  mkdir -p "${SCRIPT_HOME}/migrtion-logs"
  for i in `seq 0 19`
  do
    GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" "${SCRIPT_HOME}/user_migration/venv/bin/python" user_migration/migrate_node.py --dsns user_migration/dsns.lst --start_bso "${i}" --end_bso "${i}" --user_percent=1:50 > "./migration-logs/bso-${i}-1.out" &
    GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" "${SCRIPT_HOME}/venv/bin/python" user_migration/migrate_node.py --dsns user_migration/dsns.lst --start_bso "${i}" --end_bso "${i}" --user_percent=2:50 > "./migration-logs/bso-${i}-2.out" &
  done
}

bso_summary() {
  let total_rows=0
  let total_seconds=0
  let total_procs=0

  for bso in $(seq 0 19)
  do
    let bso_rows=0
    let bso_seconds=0

    for file in $(ls ${LOGS_DIR}/bso-${bso}-*)
    do
      filename=$(basename $file)
      i=$(echo ${filename%%.out} | cut -d - -f3)
      # cat "${file}" | tail -n2 | grep "Finished BSO #" | awk '{print "BSO " $3 ": " substr($4,2) " rows; " $7 / 60 " mins; " substr($4,2) / $7 " rows/s"}'
      rows=$(cat "${file}" | tail -n2 | grep "Finished BSO #" | awk '{print substr($4,2)}')
      seconds=$(cat "${file}" | tail -n2 | grep "Finished BSO #" | awk '{print $7}')
      let minutes=$seconds/60
      let rate=$rows/$seconds
      echo "BSO ${bso} split ${i}: ${rows} rows, ${minutes} minutes, ${rate} rows/s"
      let bso_rows=bso_rows+rows
      let bso_seconds=bso_seconds+seconds

      let total_rows=total_rows+rows
      let total_seconds=total_seconds+seconds
      let total_procs=total_procs+1
    done
    let bso_avg=bso_seconds/2 #average of the two (TODO fix for more than 2)
    let bso_rate=bso_rows/bso_avg
    echo "Total BSO ${bso}: ${bso_rows} rows, ${bso_rate} rows/s"
  done

  let avg_time=total_seconds/total_procs
  let avg_min=avg_time/60
  let avg_rate=total_rows/avg_time
  echo ""
  echo "Total rows = ${total_rows}, ${avg_rate} rows/s, ${avg_min} mins (avg)"
}


while true
do
  case $1 in
    migrate)
      migrate
      exit 0
      ;;
    summary)
      bso_summary
      exit 0
      ;;
    *)
      usage
      ;;
  esac
  shift
done