Skip to content

Instantly share code, notes, and snippets.

@koduki koduki/load_bq.sh
Last active Dec 9, 2019

Embed
What would you like to do?
This script to import from GCS to BigQuery
#!/bin/bash
JFR_NAME=$1
JFR_JSON_PATH="gs://jfr-storage_json/$(echo ${JFR_NAME}|sed 's/.jfr$//g')"
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_CPULoad ${JFR_JSON_PATH}/CPULoad.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_ExecutionSample ${JFR_JSON_PATH}/ExecutionSample.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_GCHeapSummary ${JFR_JSON_PATH}/GCHeapSummary.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_GCPhaseParallel ${JFR_JSON_PATH}/GCPhaseParallel.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.HttpRequestEvent ${JFR_JSON_PATH}/HttpRequestEvent.json
#!/bin/bash
JFR_NAME=$1
jfr print --json --events jdk.GCPhaseParallel ${JFR_NAME} | jq -c '.recording.events[]' > GCPhaseParallel.json
jfr print --json --events jdk.ExecutionSample ${JFR_NAME} | jq -c '.recording.events[]' > ExecutionSample.json
jfr print --json --events jdk.CPULoad ${JFR_NAME} | jq -c '.recording.events[]' > CPULoad.json
jfr print --json --events jdk.GCHeapSummary ${JFR_NAME} | jq -c '.recording.events[]' > GCHeapSummary.json
jfr print --json --events minica.util.profile.HttpRequestEvent ${JFR_NAME}| jq -c '.recording.events[]' > HttpRequestEvent.json
#!/bin/bash
JFR_NAME=$1
echo "START_JOB:JFR2BQ"
echo "START:DOWNLOAD"
gsutil -m cp gs://jfr-storage/${JFR_NAME} ./
echo "END:DOWNLOAD"
echo "START:PARSE"
./parse.sh ${JFR_NAME}
echo "END:PARSE"
echo "START:TRANSFER"
./transfer_gcs.sh ${JFR_NAME}
echo "END:TRANSFER"
echo "START:LOAD_BQ"
./load_bq.sh ${JFR_NAME}
echo "END:LOAD_BQ"
echo "END_JOB:JFR2BQ"
#!bin/bash
REPO=$1
WORKDIR=/tmp
CURRENT_REPO="$REPO/"$(ls -1 $REPO|sort|head -1)
ls -l $CURRENT_REPO|awk '/jfr/{print $9, $5}'|sort > $WORKDIR/jfr.list
diff ${WORKDIR}/jfr.list ${WORKDIR}/jfr.list.prev |grep jfr|awk '$NF > 0{print $2}' > ${WORKDIR}/jfr.diff
mv $WORKDIR/jfr.list $WORKDIR/jfr.list.prev
cat ${WORKDIR}/jfr.diff|xargs -n1 -P1 -I{} gsutil cp "${CURRENT_REPO}/"{} gs://jfr-storage/
#!/bin/bash
JFR_NAME=$1
JFR_JSON_PATH="gs://jfr-storage_json/$(echo ${JFR_NAME}|sed 's/.jfr$//g')"
gsutil mb gs://jfr-storage_json/
gsutil -m cp CPULoad.json \
ExecutionSample.json \
GCHeapSummary.json \
GCPhaseParallel.json \
HttpRequestEvent.json \
${JFR_JSON_PATH}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.