Nodes_JVM_Heap_Pct_Debug.json

PUT _watcher/watch/memory_watch
{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
         "indices": [ ".monitoring-es-7*" ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "filter": [
                {
                  "range": {
                    "timestamp": {
                      "gte": "now-2m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          },
          "aggs": {
            "minutes": {
              "date_histogram": {
                "field": "timestamp",
                "calendar_interval": "1h"
              },
              "aggs": {
                "nodes": {
                  "terms": {
                    "field": "source_node.name"
                  },
                  "aggs": {
                    "memory": {
                      "max": {
                        "field": "node_stats.jvm.mem.heap_used_percent"
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "return ctx.payload.aggregations.minutes.buckets.stream().anyMatch(b -> b.nodes.buckets.stream().anyMatch(x -> x.memory.value > 5))",
      "lang": "painless"
    }
  },
  "actions": {
    "dt_devops_log": {
      "logging": {
        "level": "info",
        "text": "{{ ctx.payload.aggregations.minutes.buckets }}"
      }
    }
  },
  "throttle_period_in_millis": 60000
}

Nodes_JVM_Heap_Pct_Output.json

{
  "_index": ".watcher-history-10-2019.12.17",
  "_type": "_doc",
  "_id": "memory_watch_6f6a9eb1-7270-4d55-a300-65b1d5b11c69-2019-12-17T20:30:26.830179Z",
  "_version": 1,
  "_score": null,
  "_source": {
    "watch_id": "memory_watch",
    "node": "U_FMsrFQSzOxDO42uvblkA",
    "state": "executed",
    "status": {
      "state": {
        "active": true,
        "timestamp": "2019-12-17T20:22:43.655Z"
      },
      "last_checked": "2019-12-17T20:30:26.830Z",
      "last_met_condition": "2019-12-17T20:30:26.830Z",
      "actions": {
        "dt_devops_log": {
          "ack": {
            "timestamp": "2019-12-17T20:23:27.179Z",
            "state": "ackable"
          },
          "last_execution": {
            "timestamp": "2019-12-17T20:30:26.830Z",
            "successful": true
          },
          "last_successful_execution": {
            "timestamp": "2019-12-17T20:30:26.830Z",
            "successful": true
          },
          "last_throttle": {
            "timestamp": "2019-12-17T20:25:26.733Z",
            "reason": "throttling interval is set to [1m] but time elapsed since last execution is [59.6s]"
          }
        }
      },
      "execution_state": "executed",
      "version": -1
    },
    "trigger_event": {
      "type": "schedule",
      "triggered_time": "2019-12-17T20:30:26.830Z",
      "schedule": {
        "scheduled_time": "2019-12-17T20:30:26.717Z"
      }
    },
    "input": {
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            ".monitoring-es-7*"
          ],
          "rest_total_hits_as_int": true,
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "filter": [
                  {
                    "range": {
                      "timestamp": {
                        "gte": "now-2m",
                        "lte": "now"
                      }
                    }
                  }
                ]
              }
            },
            "aggs": {
              "minutes": {
                "date_histogram": {
                  "field": "timestamp",
                  "calendar_interval": "1h"
                },
                "aggs": {
                  "nodes": {
                    "terms": {
                      "field": "source_node.name"
                    },
                    "aggs": {
                      "memory": {
                        "max": {
                          "field": "node_stats.jvm.mem.heap_used_percent"
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "script": {
        "source": "return ctx.payload.aggregations.minutes.buckets.stream().anyMatch(b -> b.nodes.buckets.stream().anyMatch(x -> x.memory.value > 5))",
        "lang": "painless"
      }
    },
    "result": {
      "execution_time": "2019-12-17T20:30:26.830Z",
      "execution_duration": 250,
      "input": {
        "type": "search",
        "status": "success",
        "payload": {
          "_shards": {
            "total": 5,
            "failed": 0,
            "successful": 5,
            "skipped": 0
          },
          "hits": {
            "hits": [],
            "total": 604,
            "max_score": null
          },
          "took": 249,
          "timed_out": false,
          "aggregations": {
            "minutes": {
              "buckets": [
                {
                  "key_as_string": "2019-12-17T20:00:00.000Z",
                  "doc_count": 604,
                  "nodes": {
                    "doc_count_error_upper_bound": 0,
                    "sum_other_doc_count": 0,
                    "buckets": [
                      {
                        "doc_count": 529,
                        "memory": {
                          "value": 30
                        },
                        "key": "elasticsearch-master-2"
                      },
                      {
                        "doc_count": 38,
                        "memory": {
                          "value": 25
                        },
                        "key": "elasticsearch-master-0"
                      },
                      {
                        "doc_count": 37,
                        "memory": {
                          "value": 28
                        },
                        "key": "elasticsearch-master-1"
                      }
                    ]
                  },
                  "key": 1576612800000
                }
              ]
            }
          }
        },
        "search": {
          "request": {
            "search_type": "query_then_fetch",
            "indices": [
              ".monitoring-es-7*"
            ],
            "rest_total_hits_as_int": true,
            "body": {
              "size": 0,
              "query": {
                "bool": {
                  "filter": [
                    {
                      "range": {
                        "timestamp": {
                          "gte": "now-2m",
                          "lte": "now"
                        }
                      }
                    }
                  ]
                }
              },
              "aggs": {
                "minutes": {
                  "date_histogram": {
                    "field": "timestamp",
                    "calendar_interval": "1h"
                  },
                  "aggs": {
                    "nodes": {
                      "terms": {
                        "field": "source_node.name"
                      },
                      "aggs": {
                        "memory": {
                          "max": {
                            "field": "node_stats.jvm.mem.heap_used_percent"
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      },
      "condition": {
        "type": "script",
        "status": "success",
        "met": true
      },
      "actions": [
        {
          "id": "dt_devops_log",
          "type": "logging",
          "status": "success",
          "logging": {
            "logged_text": "{0={key_as_string=2019-12-17T20:00:00.000Z, doc_count=604, nodes={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{doc_count=529, memory={value=30.0}, key=elasticsearch-master-2}, {doc_count=38, memory={value=25.0}, key=elasticsearch-master-0}, {doc_count=37, memory={value=28.0}, key=elasticsearch-master-1}]}, key=1576612800000}}"
          }
        }
      ]
    },
    "messages": []
  },
  "fields": {
    "result.execution_time": [
      "2019-12-17T20:30:26.830Z"
    ],
    "trigger_event.schedule.scheduled_time": [
      "2019-12-17T20:30:26.717Z"
    ],
    "trigger_event.triggered_time": [
      "2019-12-17T20:30:26.830Z"
    ]
  },
  "sort": [
    1576614626830
  ]
}