This script to import from GCS to BigQuery

load_bq.sh

#!/bin/bash

JFR_NAME=$1
JFR_JSON_PATH="gs://jfr-storage_json/$(echo ${JFR_NAME}|sed 's/.jfr$//g')"

bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_CPULoad ${JFR_JSON_PATH}/CPULoad.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_ExecutionSample ${JFR_JSON_PATH}/ExecutionSample.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_GCHeapSummary ${JFR_JSON_PATH}/GCHeapSummary.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.jdk_GCPhaseParallel ${JFR_JSON_PATH}/GCPhaseParallel.json
bq load --autodetect --source_format=NEWLINE_DELIMITED_JSON jfr_logs.HttpRequestEvent ${JFR_JSON_PATH}/HttpRequestEvent.json

parse.sh

#!/bin/bash

JFR_NAME=$1

jfr print --json --events jdk.GCPhaseParallel ${JFR_NAME} | jq -c '.recording.events[]' > GCPhaseParallel.json
jfr print --json --events jdk.ExecutionSample ${JFR_NAME} | jq -c '.recording.events[]' > ExecutionSample.json
jfr print --json --events jdk.CPULoad ${JFR_NAME} | jq -c '.recording.events[]' > CPULoad.json
jfr print --json --events jdk.GCHeapSummary ${JFR_NAME} | jq -c '.recording.events[]' > GCHeapSummary.json
jfr print --json --events minica.util.profile.HttpRequestEvent ${JFR_NAME}| jq -c '.recording.events[]' > HttpRequestEvent.json

run.sh

#!/bin/bash

JFR_NAME=$1

echo "START_JOB:JFR2BQ"

echo "START:DOWNLOAD"
gsutil -m cp gs://jfr-storage/${JFR_NAME} ./
echo "END:DOWNLOAD"

echo "START:PARSE"
./parse.sh ${JFR_NAME} 
echo "END:PARSE"

echo "START:TRANSFER"
./transfer_gcs.sh ${JFR_NAME} 
echo "END:TRANSFER"

echo "START:LOAD_BQ"
./load_bq.sh ${JFR_NAME} 
echo "END:LOAD_BQ"

echo "END_JOB:JFR2BQ"

transfer2gcs.sh

#!bin/bash

REPO=$1
WORKDIR=/tmp

CURRENT_REPO="$REPO/"$(ls -1 $REPO|sort|head -1)
ls -l $CURRENT_REPO|awk '/jfr/{print $9, $5}'|sort > $WORKDIR/jfr.list
diff ${WORKDIR}/jfr.list ${WORKDIR}/jfr.list.prev |grep jfr|awk '$NF > 0{print $2}' > ${WORKDIR}/jfr.diff
mv $WORKDIR/jfr.list $WORKDIR/jfr.list.prev

cat ${WORKDIR}/jfr.diff|xargs -n1 -P1 -I{} gsutil cp "${CURRENT_REPO}/"{} gs://jfr-storage/

transfer_gcs.sh

#!/bin/bash

JFR_NAME=$1
JFR_JSON_PATH="gs://jfr-storage_json/$(echo ${JFR_NAME}|sed 's/.jfr$//g')"

gsutil mb gs://jfr-storage_json/

gsutil -m cp CPULoad.json \
    ExecutionSample.json \
    GCHeapSummary.json \
    GCPhaseParallel.json \
    HttpRequestEvent.json \
    ${JFR_JSON_PATH}