es_watcher_latency.json

es_watcher_latency.json

{
  "trigger": {
    "schedule": {
      "interval": "5m"
    }
  },
  "input": {
    "http": {
      "request": {
        "scheme": "https",
        "host": "myhost.com",
        "port": 9200,
        "path": "metrics/logs/_search",
        "auth": {
          "basic": {
            "username": "XXX",
            "password": "YYY"
          }
        },
        "body": """{
           "size": 0,
           "query":{
              "bool":{
               "must": [
                 {
                   "query_string": {
                     "query": "role:idb-syslog-to-elk && direction:out",
                     "analyze_wildcard": true
                   }
                 }
               ],
               "filter": [
                 {
                   "range": {
                      "@timestamp":{
                         "from": "now-5m",
                         "to": "now",
                         "include_lower": true,
                         "include_upper": true,
                         "boost": 1
                      }
                   }
                 }
              ],
              "disable_coord": false,
              "adjust_pure_negative": true,
              "boost": 1
              }
          },
          "aggregations":{
             "minutes":{
               "date_histogram":{
                  "field": "@timestamp",
                  "interval": "minute",
                  "offset": 0,
                  "order":{
                     "_key": "asc"
                  },
                  "keyed": false,
                  "min_doc_count": 0
               },
               "aggregations":{
                   "nodes":{
                      "terms":{
                        "field": "hostname.keyword",
                        "size": 10,
                        "min_doc_count": 1,
                        "shard_min_doc_count": 0,
                        "show_term_doc_count_error": false,
                        "order": [
                           {
                             "latencyP95": "desc"
                           },
                           {
                             "_term": "asc"
                           }
                        ]
                      },
                      "aggregations":{
                         "latencyP95":{
                            "avg":{
                               "field": "latency.p95"
                            }
                         }
                      }
                   }
               }
             }
          },
          "ext":{}
        }"""
      }
    }
  },
  "condition": {
    "script": {
       "inline": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node?.latencyP95?.value >= 5000;"
    }
  },
  "transform" : {
    "script": {
      "inline": "def payload = ctx.payload;payload.myTimeNanos = ctx.trigger.scheduled_time.getMillis() * 1000000;return payload"
    }
  },
  "actions": {
     "forward_to_notification_service": {
     "transform": {
       "script": {
         "lang": "painless",
         "inline": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.stream().filter(item -> item.latencyP95.value >= 5000).collect(Collectors.toList());"
       }
     },
     "throttle_period": "30s",
      "webhook": {
        "scheme": "http",
        "method": "POST",
        "host": "desinationHOST.com",
        "port": 443,
        "connection_timeout": "15s",
        "path": "/{{ ctx.watch_id }}",
        "body": {
          "inline": """{
            "state": "fault",
            "name": "es_latency_event",
            "timestamp": "{{ ctx.payload.myTimeNanos }}",
            "message": "HIGH LATENCY (above 5000 p/sec): {{ ctx.payload._value.size }} nodes"
            "source": "myHost.com",
            "meta": {
                    "app_name": "es2noti_latency",
                    "cluster_name": "{{ ctx.payload.cluster_name }}",
                    "data": "{{#ctx.payload._value}}\"{{key}}\":{{latencyP95.value}},{{/ctx.payload._value}}"
                    #"total_nodes": "{{ ctx.payload._nodes.total}}",
                    "hostname": "myHost.com",
                    "hostenv": "dev",
                    "thresholdType": "threshold.breach",
                    "tags": ["es_watcher_latency"]
            }
          }""",
        "lang": "mustache"
        }
      }
    }
  }
}