crux-extract-origins

download-origins.sh

#!/bin/zsh

##Does not work with glocud version 298-301 (should be fixed in 302)
##https://github.com/GoogleCloudPlatform/gsutil/issues/1052
set -e -x -o pipefail

#Check tools
function check() {
    which $1 >/dev/null || ( echo "$1 is needed; try brew install $1" ; exit 77 ; ); 
    if [ "$?" -eq 77 ]; then exit; fi
}

check jq
check gzcat
check gsutil

DT=$(php -r "echo date('Ym',strtotime('last month'));")
FILENAME="${DT}.json.gz"
CLOUD_URL="gs://firebase-xxxx.appspot.com/origin_summary/${FILENAME}"

#If cloud file not exists - extract
gsutil ls ${CLOUD_URL} || bq extract --project_id=firebase-xxxx \
    --destination_format=NEWLINE_DELIMITED_JSON \
    --compression=GZIP \
    --use_legacy_sql=false \
     'chrome-ux-report:materialized.origin_summary' \
    ${CLOUD_URL}

#If local file not exists - download
[ -f ${FILENAME} ] || gsutil cp ${CLOUD_URL} "./${FILENAME}"

#Show
gzcat "${FILENAME}" | jq -r ".origin" | head

#Show stored files
gsutil ls 'gs://firebase-xxxx.appspot.com/origin_summary/*.gz' | xargs -n1 gsutil stat