watcher-bad-pod-example

watcher-bad-pod-example

{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "<*-riot-metrics-{now/d}>"
        ],
        "types": [],
        "body": {
          "query": {
            "bool": {
              "must_not": [
                {
                  "match_phrase": {
                    "kubernetes.container.status.reason": "ContainerCreating"
                  }
                },
                {
                  "match_phrase": {
                    "kubernetes.container.status.reason": "Completed"
                  }
                }
              ],
              "minimum_should_match": 0,
              "filter": [
                {
                  "exists": {
                    "field": "kubernetes.container.status.reason"
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-5m"
                    }
                  }
                }
              ]
            }
          },
          "aggs": {
            "Pod_Name": {
              "terms": {
                "field": "kubernetes.container.name"
              }
            },
            "loc_index": {
              "terms": {
                "field": "_index"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gte": 1
      }
    }
  },
  "actions": {
    "OpsGenie": {
      "webhook": {
        "scheme": "https",
        "host": "api.opsgenie.com",
        "port": 443,
        "method": "post",
        "path": "/v2/alerts",
        "params": {},
        "headers": {
          "Authorization": "GenieKey COWS_RULE",
          "Content-Type": "application/json"
        },
        "body": "{\"message\":\" (Dev) Bad Pod State\",\"alias\":\"dev Pod in Bad State\",\"description\":\"Pods in cluster are in a bad state.\\n Search for kubernetes.container.status.reason != Completed or CreatingContainer\\nat #?_g=()\"}"
      }
    }
  }
}