upload_minio_logs.sh

#!/usr/bin/env bash

# Script to upload http-trace.log to ES Cluster
# Requires MINIO_HTTP_TRACE env var to be enabled (-e "MINIO_HTTP_TRACE=/data/http-trace.log")
#
# Usage:
#   upload_minio_logs.sh <http_log_file> <run_id>
#
# Environment variables:
#   ES_HOST, ES_PORT, ES_USERNAME, ES_PASSWORD - target ES cluster

# fail this script immediately if any command fails with a non-zero exit code
set -e
# fail on pipeline errors, e.g. when grepping
set -o pipefail
# Treat unset env variables as an error as of now
set -u

readonly INDEX_PREFIX="minio-logs"
readonly INDEX_DATE=$(date -u +"%Y-%m-%d")
readonly INDEX_NAME="${INDEX_PREFIX}-${INDEX_DATE}"

bulk_upload() {
    local BULK_FILE="$1"
    local URL="$ES_HOST:$ES_PORT/$INDEX_NAME/_bulk"
    echo "Upload to: $URL"

    RECORDS_COUNT="$(wc -l $BULK_FILE | awk '{print $1/2}')"
    echo "Uploading $RECORDS_COUNT records:"

    curl -X POST \
            $URL \
            -s \
            -u "$ES_USERNAME:$ES_PASSWORD" \
            -H "Content-Type: application/x-ndjson" \
            --data-binary @$BULK_FILE  | \
            jq '{"took": .took, "errors": .errors}'
    rm $BULK_FILE
}

BULK_UPLOAD_TMP=$(mktemp)

awk   -v run_id="$2" \
      -F '[][]| ' \
      '/REQUEST/{if (NR!=1) print ""; i=0}
      {
        i++;
        if (i == 1) {
          printf ("{ \"timestamp\":\"%sT%s%s\", ", $9, $10, $11);
          printf ("\"run_id\":\"%s\", ", run_id)
        } else if (i == 2) {
          printf ("\"method\":\"%s\", \"uri\":\"%s\", ", $1, $2)
        } else if ($1 == "User-Agent:") {
          printf ("\"ua\":\"%s\"", $2)
        } else if ($2 == "RESPONSE") {
          printf "}"
        }
      }' $1  | sed -e 's/^/{ "index": {"_type": "_doc" } } \
/' > "${BULK_UPLOAD_TMP}"

# Append newline to end of file for bulk upload to work
echo "" >> "${BULK_UPLOAD_TMP}"

echo $BULK_UPLOAD_TMP

# Upload results
bulk_upload "${BULK_UPLOAD_TMP}"