Created
April 7, 2020 23:46
-
-
Save erkolson/5715590c96b8ad237da241f56c9e41be to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
SCRIPT_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" | |
LOGS_DIR=${LOGS_DIR:-"./migration-logs"} | |
usage() { | |
echo "node_migration.sh <command>" | |
echo "commands:" | |
echo " migrate Run the migration scripts" | |
echo " summary Parse BSO logs for migration summary" | |
exit 1 | |
} | |
# validate number of command line arguments | |
if [[ $# < 1 ]]; then | |
usage | |
fi | |
migrate() { | |
for i in `seq 0 19` | |
do | |
GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" user_migration/venv/bin/python user_migration/migrate_node.py --dsns user_migration/dsns.lst --sort_users --deanon --fxa_file user_migration/580.csv --start_bso "${i}" --end_bso "${i}" --user_percent=1:50 > "./migration-logs/bso-${i}-1.out" & | |
GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" user_migration/venv/bin/python user_migration/migrate_node.py --dsns user_migration/dsns.lst --sort_users --deanon --fxa_file user_migration/580.csv --start_bso "${i}" --end_bso "${i}" --user_percent=2:50 > "./migration-logs/bso-${i}-2.out" & | |
done | |
} | |
migrate-anon() { | |
mkdir -p "${SCRIPT_HOME}/migrtion-logs" | |
for i in `seq 0 19` | |
do | |
GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" "${SCRIPT_HOME}/user_migration/venv/bin/python" user_migration/migrate_node.py --dsns user_migration/dsns.lst --start_bso "${i}" --end_bso "${i}" --user_percent=1:50 > "./migration-logs/bso-${i}-1.out" & | |
GOOGLE_APPLICATION_CREDENTIALS="${SCRIPT_HOME}/user_migration/key.json" "${SCRIPT_HOME}/venv/bin/python" user_migration/migrate_node.py --dsns user_migration/dsns.lst --start_bso "${i}" --end_bso "${i}" --user_percent=2:50 > "./migration-logs/bso-${i}-2.out" & | |
done | |
} | |
bso_summary() { | |
let total_rows=0 | |
let total_seconds=0 | |
let total_procs=0 | |
for bso in $(seq 0 19) | |
do | |
let bso_rows=0 | |
let bso_seconds=0 | |
for file in $(ls ${LOGS_DIR}/bso-${bso}-*) | |
do | |
filename=$(basename $file) | |
i=$(echo ${filename%%.out} | cut -d - -f3) | |
# cat "${file}" | tail -n2 | grep "Finished BSO #" | awk '{print "BSO " $3 ": " substr($4,2) " rows; " $7 / 60 " mins; " substr($4,2) / $7 " rows/s"}' | |
rows=$(cat "${file}" | tail -n2 | grep "Finished BSO #" | awk '{print substr($4,2)}') | |
seconds=$(cat "${file}" | tail -n2 | grep "Finished BSO #" | awk '{print $7}') | |
let minutes=$seconds/60 | |
let rate=$rows/$seconds | |
echo "BSO ${bso} split ${i}: ${rows} rows, ${minutes} minutes, ${rate} rows/s" | |
let bso_rows=bso_rows+rows | |
let bso_seconds=bso_seconds+seconds | |
let total_rows=total_rows+rows | |
let total_seconds=total_seconds+seconds | |
let total_procs=total_procs+1 | |
done | |
let bso_avg=bso_seconds/2 #average of the two (TODO fix for more than 2) | |
let bso_rate=bso_rows/bso_avg | |
echo "Total BSO ${bso}: ${bso_rows} rows, ${bso_rate} rows/s" | |
done | |
let avg_time=total_seconds/total_procs | |
let avg_min=avg_time/60 | |
let avg_rate=total_rows/avg_time | |
echo "" | |
echo "Total rows = ${total_rows}, ${avg_rate} rows/s, ${avg_min} mins (avg)" | |
} | |
while true | |
do | |
case $1 in | |
migrate) | |
migrate | |
exit 0 | |
;; | |
summary) | |
bso_summary | |
exit 0 | |
;; | |
*) | |
usage | |
;; | |
esac | |
shift | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment