kube-proxy-health.py

#!/usr/bin/env python3

import io
import re
import subprocess
import sys
import uuid

from contextlib import contextmanager

stub_service = """
apiVersion: v1
kind: Service
metadata:
  name: {0}
  labels:
    app: kube-proxy-health
spec:
  ports:
  - name: http
    port: 8080
    protocol: TCP
    targetPort: 8080
  selector:
    app: {0}
"""

@contextmanager
def add_service():
    svc = stub_service.format('a' + str(uuid.uuid4()).replace('-', ''))
    opts = {
        'shell': True,
        'check': True,
        'input': svc,
        'encoding': 'ascii',
        'capture_output': True,
    }
    output('adding stub service')
    subprocess.run(
        'kubectl apply -f -',
        **opts,
    )
    yield
    output('cleaning stub service')
    subprocess.run(
        'kubectl delete -f -',
        **opts,
    )

def pod_names():
    pods = subprocess.run(
        'kubectl -n kube-system get po -l component=kube-proxy -o name',
        shell=True,
        check=True,
        text=True,
        capture_output=True).stdout.split('\n')[:-1]
    return [x.split('/')[1] for x in pods]

def get_count(name):
    count = 0

    with subprocess.Popen(
        'kubectl -n kube-system port-forward {} 10249'.format(name),
        shell=True,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    ) as forward:
        out = forward.stdout.readline()
        if 'Forwarding' not in out:
            print(forward.stderr.read())
            sys.exit(1)

        metrics = subprocess.run(
            'curl localhost:10249/metrics 2> /dev/null',
            shell=True,
            check=True,
            text=True,
            capture_output=True).stdout

        count = int(re.search(
            'service_changes_total ([0-9]+)', metrics).group(1))

        forward.terminate()

    return count

def output(txt):
    print('---------- {}'.format(txt))

if __name__ == '__main__':
    output('fetching kube-proxy pod names')
    pods = { x: {} for x in pod_names() }

    output('finding current service update counts')
    for pod, data in pods.items():
        data['before'] = get_count(pod)

    with add_service():
        output('finding updated service counts')
        for pod, data in pods.items():
            data['after'] = get_count(pod)

    bad_pods = [x for x,y in pods.items() if y['before'] < y['after']]

    if len(bad_pods) == 0:
        print('Everything looks okay!')
        sys.exit(0)

    print('There appears to be issues with: \n\t{}\n'.format('\n\t'.join(bad_pods)))
    print('To fix this, run: \n\tkubectl -n kube-system delete po {}'.format(
        ' '.join(bad_pods)
    ))

    sys.exit(1)