ZMON Kubernetes Pod Container Metrics

zmon-kubernetes-pod-container-metrics.py

def check():
    prometheus_url = "http://prometheus.kube-system.svc.cluster.local./api/v1/query"

    metrics = {"_use_scheduled_time": True}

    try:
        container_selector = """{{container_name="{container_name}", pod_name="{pod_name}", namespace="{pod_namespace}"}}""".format(**entity)

        def request_prometheus_metric(query):
            params={"query": query.format(container_selector=container_selector)}
            result = http(prometheus_url, params=params).json()
            if result.get("status") == "success":
                _, value = result["data"]["result"]
                return float(value)
            else:
                return float('nan')

        def usage_ratio(usage, max):
            if max == 0:
                return 0
            else:
                return float(usage) / float(max)


        # CPU
        cpu_request = request_prometheus_metric("""1000 * scalar(kube_pod_container_resource_requests_cpu_cores{container_selector})""")
        cpu_limit = request_prometheus_metric("""1000 * scalar(kube_pod_container_resource_limits_cpu_cores{container_selector})""")
        cpu_usage = request_prometheus_metric("""1000 * scalar(sum(rate(container_cpu_usage_seconds_total{container_selector}[1m])))""")
        cpu_throttled = request_prometheus_metric("""scalar(rate(container_cpu_cfs_throttled_seconds_total{container_selector}[1m]))""")
        metrics["cpu"] = {
            "usage": cpu_usage,
            "request": cpu_request,
            "limit": cpu_limit,
            "throttle_per_second": cpu_throttled,
            "usage_ratio": usage_ratio(cpu_usage, cpu_limit),
            "usage_request_ratio": usage_ratio(cpu_usage, cpu_request)
        }

        # Memory
        memory_request = request_prometheus_metric("""scalar(kube_pod_container_resource_requests_memory_bytes{container_selector})""")
        memory_limit = request_prometheus_metric("""scalar(kube_pod_container_resource_limits_memory_bytes{container_selector})""")
        memory_usage = request_prometheus_metric("""scalar(container_memory_working_set_bytes{container_selector})""")
        metrics["memory"] = {
            "usage": memory_usage,
            "request": memory_request,
            "limit": memory_limit,
            "usage_ratio": usage_ratio(memory_usage, memory_limit),
            "usage_request_ratio": usage_ratio(memory_usage, memory_request),
            "limit_request_ratio": usage_ratio(memory_limit, memory_request),
        }
    except:
        pass

    return metrics