BigQuery Load Script

load_partitioned_files.sh

#!/usr/bin/env bash

STAGING_DIR="./bq_staging"
FILE_PREFIX="runs"
FS="_"
START_DATE="2017-04-18"
END_DATE="2017-07-10"
CURRENT_DATE=$START_DATE
while [ $CURRENT_DATE != $END_DATE ]
do
    echo "Loading file for $CURRENT_DATE\n"
    bq load --source_format=NEWLINE_DELIMITED_JSON jenkins_run_historian.runs_v1\$$(date -j -f "%Y-%m-%d" "$CURRENT_DATE" "+%Y%m%d") ${STAGING_DIR}/${FILE_PREFIX}_${CURRENT_DATE}
    CURRENT_DATE=$(date -j -f "%Y-%m-%d" -v +1d "${CURRENT_DATE}" "+%Y-%m-%d")
done