Skip to content

Instantly share code, notes, and snippets.

@ChrsMark

ChrsMark/es_queries_k8s_dashboard.md

Last active March 23, 2022 15:34
Show Gist options
  • Save ChrsMark/f4292c388879eeb5368218068d09d40c to your computer and use it in GitHub Desktop.
Save ChrsMark/f4292c388879eeb5368218068d09d40c to your computer and use it in GitHub Desktop.
Download ZIP
Elasticsearch Queries of the Metricbeat Kubernetes Dashboard
Raw
es_queries_k8s_dashboard.md

1 Available Pods

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:state_deployment",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "64456840-30df-11e7-8df8-6d3604a72912": {
      "terms": {
        "field": "kubernetes.deployment.name",
        "size": "10000",
        "order": {
          "_count": "desc"
        }
      },
      "aggs": {
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "64456841-30df-11e7-8df8-6d3604a72912": {
              "avg": {
                "field": "kubernetes.deployment.replicas.available"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "117fadf0-30df-11e7-8df8-6d3604a72912",
        "seriesId": "64456840-30df-11e7-8df8-6d3604a72912",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

2 Desired Pods

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:state_deployment",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "timeseries": {
      "auto_date_histogram": {
        "field": "@timestamp"
      },
      "aggs": {
        "54cf79a0-30d5-11e7-8df8-6d3604a72912": {
          "sum": {
            "field": "kubernetes.deployment.replicas.desired"
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "2fe9d3b0-30d5-11e7-8df8-6d3604a72912",
        "seriesId": "2fe9d3b1-30d5-11e7-8df8-6d3604a72912",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*",
        "normalized": true
      }
    }
  }
}

3 Unavailable Pods

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:state_deployment",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "64456840-30df-11e7-8df8-6d3604a72912": {
      "terms": {
        "field": "kubernetes.deployment.name",
        "size": "10000",
        "order": {
          "_count": "desc"
        }
      },
      "aggs": {
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "64456841-30df-11e7-8df8-6d3604a72912": {
              "avg": {
                "field": "kubernetes.deployment.replicas.unavailable"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "117fadf0-30df-11e7-8df8-6d3604a72912",
        "seriesId": "64456840-30df-11e7-8df8-6d3604a72912",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

4 Number of Deployments

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:state_deployment",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "timeseries": {
      "auto_date_histogram": {
        "field": "@timestamp"
      },
      "aggs": {
        "4c4690b2-30e0-11e7-8df8-6d3604a72912": {
          "cardinality": {
            "field": "kubernetes.deployment.name"
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "4c4690b0-30e0-11e7-8df8-6d3604a72912",
        "seriesId": "4c4690b1-30e0-11e7-8df8-6d3604a72912",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*",
        "normalized": true
      }
    }
  }
}

5 Number of Nodes

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:state_node",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "timeseries": {
      "auto_date_histogram": {
        "field": "@timestamp"
      },
      "aggs": {
        "4c4690b2-30e0-11e7-8df8-6d3604a72912": {
          "cardinality": {
            "field": "kubernetes.node.name"
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "4c4690b0-30e0-11e7-8df8-6d3604a72912",
        "seriesId": "4c4690b1-30e0-11e7-8df8-6d3604a72912",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*",
        "normalized": true
      }
    }
  }
}

6 Unavailable Per Deployment

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:state_deployment",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "64456840-30df-11e7-8df8-6d3604a72912": {
      "terms": {
        "field": "kubernetes.deployment.name",
        "size": "10000",
        "order": {
          "_count": "desc"
        }
      },
      "aggs": {
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "64456841-30df-11e7-8df8-6d3604a72912": {
              "avg": {
                "field": "kubernetes.deployment.replicas.unavailable"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "117fadf0-30df-11e7-8df8-6d3604a72912",
        "seriesId": "64456840-30df-11e7-8df8-6d3604a72912",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

7 Available Per Depployment

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:state_deployment",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "64456840-30df-11e7-8df8-6d3604a72912": {
      "terms": {
        "field": "kubernetes.deployment.name",
        "size": "10000",
        "order": {
          "_count": "desc"
        }
      },
      "aggs": {
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "64456841-30df-11e7-8df8-6d3604a72912": {
              "avg": {
                "field": "kubernetes.deployment.replicas.available"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "117fadf0-30df-11e7-8df8-6d3604a72912",
        "seriesId": "64456840-30df-11e7-8df8-6d3604a72912",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

8 CPU Usage by Node

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND (metricset.name:container OR metricset.name:state_node)",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "0d5c9220-2bf2-11e7-859b-f78b612cde28": {
      "terms": {
        "field": "kubernetes.node.name",
        "size": "10000",
        "order": {
          "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": "desc"
        }
      },
      "aggs": {
        "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": {
          "sum": {
            "field": "kubernetes.container.cpu.usage.nanocores"
          }
        },
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "0d5c9221-2bf2-11e7-859b-f78b612cde28": {
              "sum": {
                "field": "kubernetes.container.cpu.usage.nanocores"
              }
            },
            "8b346300-bf95-11ea-a07c-851701f0d645": {
              "avg": {
                "field": "metricset.period"
              }
            },
            "25ae6580-bf95-11ea-a07c-851701f0d645": {
              "bucket_script": {
                "buckets_path": {
                  "sum_nanocores": "0d5c9221-2bf2-11e7-859b-f78b612cde28",
                  "avg_period": "8b346300-bf95-11ea-a07c-851701f0d645"
                },
                "script": {
                  "source": "params.sum_nanocores / (params._interval / params.avg_period)",
                  "lang": "painless",
                  "params": {
                    "_interval": 10000
                  }
                },
                "gap_policy": "skip"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "0d5c6b10-2bf2-11e7-859b-f78b612cde28",
        "seriesId": "0d5c9220-2bf2-11e7-859b-f78b612cde28",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

9 Memory Usage by Node

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND (metricset.name:container OR metricset.name:state_node)",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "0d5c9220-2bf2-11e7-859b-f78b612cde28": {
      "terms": {
        "field": "kubernetes.node.name",
        "size": "10000",
        "order": {
          "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": "desc"
        }
      },
      "aggs": {
        "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": {
          "sum": {
            "field": "kubernetes.container.memory.usage.bytes"
          }
        },
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "0d5c9221-2bf2-11e7-859b-f78b612cde28": {
              "sum": {
                "field": "kubernetes.container.memory.usage.bytes"
              }
            },
            "9f0cf900-1ffb-11e8-81f2-43be86397500": {
              "cumulative_sum": {
                "buckets_path": "0d5c9221-2bf2-11e7-859b-f78b612cde28"
              }
            },
            "a926e130-1ffb-11e8-81f2-43be86397500": {
              "derivative": {
                "buckets_path": "9f0cf900-1ffb-11e8-81f2-43be86397500",
                "gap_policy": "skip",
                "unit": "10s"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "0d5c6b10-2bf2-11e7-859b-f78b612cde28",
        "seriesId": "0d5c9220-2bf2-11e7-859b-f78b612cde28",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

10 Network out By Node

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:pod",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "0d5c9220-2bf2-11e7-859b-f78b612cde28": {
      "terms": {
        "field": "kubernetes.node.name",
        "size": "10000",
        "order": {
          "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": "desc"
        }
      },
      "aggs": {
        "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": {
          "max": {
            "field": "kubernetes.pod.network.tx.bytes"
          }
        },
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "0d5c9221-2bf2-11e7-859b-f78b612cde28": {
              "max": {
                "field": "kubernetes.pod.network.tx.bytes"
              }
            },
            "494fc310-2bf7-11e7-859b-f78b612cde28": {
              "derivative": {
                "buckets_path": "0d5c9221-2bf2-11e7-859b-f78b612cde28",
                "gap_policy": "skip",
                "unit": "10s"
              }
            },
            "244c70e0-3598-11e7-aa4a-8313a0c92a88": {
              "bucket_script": {
                "buckets_path": {
                  "value": "494fc310-2bf7-11e7-859b-f78b612cde28[normalized_value]"
                },
                "script": {
                  "source": "params.value > 0.0 ? params.value : 0.0",
                  "lang": "painless"
                },
                "gap_policy": "skip"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "0d5c6b10-2bf2-11e7-859b-f78b612cde28",
        "seriesId": "0d5c9220-2bf2-11e7-859b-f78b612cde28",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

11 Network in By Node

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:pod",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "0d5c9220-2bf2-11e7-859b-f78b612cde28": {
      "terms": {
        "field": "kubernetes.node.name",
        "size": "100000",
        "order": {
          "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": "desc"
        }
      },
      "aggs": {
        "0d5c9221-2bf2-11e7-859b-f78b612cde28-SORT": {
          "max": {
            "field": "kubernetes.pod.network.rx.bytes"
          }
        },
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "0d5c9221-2bf2-11e7-859b-f78b612cde28": {
              "max": {
                "field": "kubernetes.pod.network.rx.bytes"
              }
            },
            "494fc310-2bf7-11e7-859b-f78b612cde28": {
              "derivative": {
                "buckets_path": "0d5c9221-2bf2-11e7-859b-f78b612cde28",
                "gap_policy": "skip",
                "unit": "10s"
              }
            },
            "37c72a70-3598-11e7-aa4a-8313a0c92a88": {
              "bucket_script": {
                "buckets_path": {
                  "value": "494fc310-2bf7-11e7-859b-f78b612cde28[normalized_value]"
                },
                "script": {
                  "source": "params.value > 0.0 ? params.value : 0.0",
                  "lang": "painless"
                },
                "gap_policy": "skip"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "0d5c6b10-2bf2-11e7-859b-f78b612cde28",
        "seriesId": "0d5c9220-2bf2-11e7-859b-f78b612cde28",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

12 Top CPU Intensive Pods

GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:container",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "5d3692a1-2bfc-11e7-859b-f78b612cde28": {
      "terms": {
        "field": "kubernetes.pod.name",
        "order": {
          "5d3692a2-2bfc-11e7-859b-f78b612cde28-SORT": "desc"
        }
      },
      "aggs": {
        "5d3692a2-2bfc-11e7-859b-f78b612cde28-SORT": {
          "max": {
            "field": "kubernetes.container.cpu.usage.core.ns"
          }
        },
        "timeseries": {
          "auto_date_histogram": {
            "field": "@timestamp"
          },
          "aggs": {
            "5d3692a2-2bfc-11e7-859b-f78b612cde28": {
              "max": {
                "field": "kubernetes.container.cpu.usage.core.ns"
              }
            },
            "6c905240-2bfc-11e7-859b-f78b612cde28": {
              "derivative": {
                "buckets_path": "5d3692a2-2bfc-11e7-859b-f78b612cde28",
                "gap_policy": "skip",
                "unit": "1s"
              }
            },
            "9a51f710-359d-11e7-aa4a-8313a0c92a88": {
              "bucket_script": {
                "buckets_path": {
                  "value": "6c905240-2bfc-11e7-859b-f78b612cde28[normalized_value]"
                },
                "script": {
                  "source": "params.value > 0.0 ? params.value : 0.0",
                  "lang": "painless"
                },
                "gap_policy": "skip"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "5d3692a0-2bfc-11e7-859b-f78b612cde28",
        "seriesId": "5d3692a1-2bfc-11e7-859b-f78b612cde28",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}

#13 Top Memory Intensive Pods (14 secs) GET metricbeat-*/_search

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "now-24h",
              "lte": "now",
              "format": "strict_date_optional_time"
            }
          }
        },
        {
          "bool": {
            "must": [
              {
                "query_string": {
                  "query": "event.module:kubernetes AND metricset.name:container",
                  "analyze_wildcard": true
                }
              }
            ],
            "filter": [],
            "should": [],
            "must_not": []
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": []
    }
  },
  "aggs": {
    "5d3692a1-2bfc-11e7-859b-f78b612cde28": {
      "terms": {
        "field": "kubernetes.pod.name",
        "size": "10",
        "order": {
          "5d3692a2-2bfc-11e7-859b-f78b612cde28-SORT": "desc"
        }
      },
      "aggs": {
        "5d3692a2-2bfc-11e7-859b-f78b612cde28-SORT": {
          "sum": {
            "field": "kubernetes.container.memory.usage.bytes"
          }
        },
        "timeseries": {
          "date_histogram": {
            "field": "@timestamp",
            "min_doc_count": 0,
            "time_zone": "Europe/Lisbon",
            "extended_bounds": {
              "min": 1647964078023,
              "max": 1647964978023
            },
            "fixed_interval": "10m"
          },
          "aggs": {
            "5d3692a2-2bfc-11e7-859b-f78b612cde28": {
              "sum": {
                "field": "kubernetes.container.memory.usage.bytes"
              }
            },
            "3972e9f0-256f-11e8-84e6-87221f87ae3b": {
              "cumulative_sum": {
                "buckets_path": "5d3692a2-2bfc-11e7-859b-f78b612cde28"
              }
            },
            "3e9fd5a0-256f-11e8-84e6-87221f87ae3b": {
              "derivative": {
                "buckets_path": "3972e9f0-256f-11e8-84e6-87221f87ae3b",
                "gap_policy": "skip",
                "unit": "10s"
              }
            }
          }
        }
      },
      "meta": {
        "timeField": "@timestamp",
        "panelId": "5d3692a0-2bfc-11e7-859b-f78b612cde28",
        "seriesId": "5d3692a1-2bfc-11e7-859b-f78b612cde28",
        "intervalString": "10s",
        "indexPatternString": "metricbeat-*"
      }
    }
  }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment