Jo-Hue

#http.port: 9200

Jo-Hue

# curl -i localhost:9200
HTTP/1.1 200 OK
Server: nginx/1.14.0 (Ubuntu)
Date: Tue, 27 Aug 2019 14:09:48 GMT
Content-Type: application/json; charset=UTF-8
Content-Length: 493
Connection: keep-alive

{
  "name" : "gametime-dev",

Jo-Hue

server {
  listen 9200;

  location / {
    proxy_pass http://localhost:9201;
  }
}

Jo-Hue

127.0.0.1 - - [27/Aug/2019:14:23:28 +0000] "GET /_nodes/_local?filter_path=nodes.*.settings.tribe HTTP/1.1" 200 2 "-" "-"
127.0.0.1 - - [27/Aug/2019:14:23:30 +0000] "HEAD / HTTP/1.1" 200 0 "-" "-"
127.0.0.1 - - [27/Aug/2019:14:23:30 +0000] "GET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip HTTP/1.1" 200 117 "-" "-"

Jo-Hue

log_format upstream_time '$remote_addr - $remote_user [$time_local] '
                           '"$request" $status $body_bytes_sent '
                           '"$http_referer" "$http_user_agent"'
                           ' rt=$request_time uct=$upstream_connect_time uht=$upstream_header_time urt=$upstream_response_time';

Jo-Hue

server {
  listen 9200;

  access_log /var/log/nginx/elasticsearch.access.upstream_time.log upstream_time;

  location / {
    proxy_pass http://localhost:9201;
  }
}

Jo-Hue

curl -s localhost:9200 -o /dev/null
tail -1 /var/log/nginx/elasticsearch.access.upstream_time.log
#=>
127.0.0.1 - - [27/Aug/2019:14:31:00 +0000] "GET / HTTP/1.1" 200 493 "-" "curl/7.58.0" rt=0.002 uct=0.000 uht=0.004 urt=0.004

Jo-Hue

curl -s 'localhost:9200/_ingest/pipeline/*' | jq ' . | keys'
#=>
[
"filebeat-6.3.2-nginx-access-default",
"filebeat-6.3.2-nginx-error-pipeline"
]

Jo-Hue

cat << EOF > pipeline.json
{
  "description": "pipeline for parsing nginx response times",
  "processors": [
    {
      "set": {
        "field": "log_line",
        "value": "{{message}}"
      }
    },

Jo-Hue

- type: log
  enabled: true
  paths:
  - /var/log/nginx/*upstream_time.log*
  exclude_files: ['.gz$']
  pipeline: nginx-response-time