load_fluentd_out_to_elastic.sh

#!/bin/sh -x

function usage() {
echo "
USAGE
    $0 [IN_FILE]
    IN_FILE=/path/to/in_file $0
    ES_ADDRESS=elasticsearch.example.com IN_FILE=/path/to/in_file $0

    This is a loader from fluentd file into elastic-search
    The IN_FILE contains json events created by fluentd 'out_file' plugin

    ES_ADDRESS defualts to https://localhost:9200
"
exit 1
}

IN_FILE=${IN_FILE-$1}
ES_ADDRESS=${ES_ADDRESS-localhost}

[[ -n ${IN_FILE} ]] || usage


# main

# openshift only code. If you are authorizred to do this insecure then skip this and change
# the 'curl' cmd accordingly
oc get secret logging-fluentd --template='{{.data.ca}}' | base64 -d > ca
oc get secret logging-fluentd --template='{{.data.key}}' | base64 -d > key
oc get secret logging-fluentd --template='{{.data.cert}}' | base64 -d > cert
oc port-forward <es pod name> 9200:9200 &

cat ${IN_FILE} | gawk -F "\t" '{print "{\"index\": {\"_index\": \"project.ovirt-metrics-lago-engine\", \"_type\": \"metrics\"}}\n" $3}' | \
curl \
-XPOST \
-H "Content-Type: application/x-ndjson" \
${ES_ADDRESS}/_bulk \
--cacert ca --key key --cert ./cert \
--data-binary \
@- \
 -v

line 20 can be parameterized so the index and type can be anything and not ovirt specific