query_logs_processing.sh

#!/bin/bash
# Log may contain different subgraphs - filter subgrafs manualy first of all
# query.20201126_44.log

GRAPHS_LIST="<PATH>/all-graphs-list.csv"
TIMESTAMP=$(date --date="yesterday" +%Y%m%d)
#NAME_BASE=$(echo $1 | cut -d'.' -f 1,2)
NAME_BASE="query.${TIMESTAMP}"
LOGS="/fluentd/log/query-node/${NAME_BASE}*"
SUMMARY_JSONL="/fluentd/log/summary/${NAME_BASE}.summary.jsonl"
OUTPUT_JSONL="/fluentd/log/parsed_raw/${NAME_BASE}.jsonl"
TREEBUF="${NAME_BASE}.treebuf"

AGORA=<PATH>/agora1/agora/target/release/agora

echo "Converting logs ${LOGS} to ${SUMMARY_JSONL}"
grep 'Query timing' $LOGS  | /usr/local/bin/qlog  process --text --graphql  $SUMMARY_JSONL --output  $OUTPUT_JSONL --samples 1000 --sample-file /fluentd/log/samples/samples.jsonl
sed -i 's/\\"/"/g' $OUTPUT_JSONL
sed -i 's/\\"/"/g' $OUTPUT_JSONL
sed -i 's/\\"/"/g' $SUMMARY_JSONL
sed -i 's/\\"/"/g' $SUMMARY_JSONL
sed -i 's/time/effort/g' $OUTPUT_JSONL

echo "Adding to daily summary"
summaries=""
  day_summary=$(jq -s 'group_by(.subgraph) |
    map({"subgraph":(.[0].subgraph), "calls":(reduce .[].calls as $calls (0; . + $calls)), "date":("'$TIMESTAMP'") }) |
    map([.subgraph, .calls, .date] | join(", ")) | join(" \r\n ")' < $SUMMARY_JSONL)
  summaries+="${day_summary//\"} \r\n"
printf "${summaries//\"}" >> /fluentd/log/summary/subgraph_daily_summary.csv


echo "Parsing per graph..."

for id in $(cat $GRAPHS_LIST);
do
  echo "Sub_graph: $id"
  cat $OUTPUT_JSONL | grep $id > /fluentd/log/parsed/$id.$TIMESTAMP.jsonl
done