audit.py

#!/usr/bin/env python
import json
import subprocess
import pprint

###
### Compare the EC2 state with that state of pods in your kube cluster
### Loads from EC2 with the awscli and loads from kube with kubectl
###

nodes = subprocess.check_output(["/usr/local/bin/kubectl", "get", "nodes", "-o", "json"])
nodes = json.loads(nodes)['items']

pods = subprocess.check_output(["/usr/local/bin/kubectl", "get", "pods", "-o", "json", "--all-namespaces"])
pods = json.loads(pods)['items']

us_east_1c_nodes = [node for node in nodes if node['metadata']['labels']['failure-domain.beta.kubernetes.io/zone'] == 'us-east-1c']

# genpop_nodes = [node for node in us_east_1c_nodes if node['metadata']['labels']['kops.k8s.io/instancegroup'] == 'nodes-r5']
# es_nodes = [node for node in us_east_1c_nodes if node['metadata']['labels']['kops.k8s.io/instancegroup'] == 'elasticsearch']
# print("genpop: %d, es_nodes: %d" % (len(genpop_nodes), len(es_nodes)))

ec2s = subprocess.check_output(['aws', '--region=us-east-1', '--profile=saml', 'ec2', 'describe-instances'])#, '--filters', 'Name=availability-zone,Values=us-east-1c'])
ec2s = json.loads(ec2s)['Reservations']
ec2sprivatedns = {}
for ec2 in ec2s:
  for instance in ec2['Instances']:
    privatedns = instance['PrivateDnsName']
    ec2sprivatedns[privatedns] = instance

name_node = {}
node_private_ips = {}
private_ip_node = {}
private_ip_eni = {}
for node in us_east_1c_nodes:
  node_name = node['metadata']['name']
  name_node[node_name] = node

  if node_name in ec2sprivatedns:
    node_private_ips[node_name] = []

    instance = ec2sprivatedns[node_name]
    for interface in instance['NetworkInterfaces']:
      eni = interface['NetworkInterfaceId']
      private_ip_eni[eni] = {}
      for private_ip in interface['PrivateIpAddresses']:
        if private_ip['Primary'] == False:
          node_private_ips[node_name].append(private_ip['PrivateIpAddress'])
          private_ip_eni[private_ip['PrivateIpAddress']] = eni
          private_ip_node[private_ip['PrivateIpAddress']] = node_name
  else:
    pass
    # import pdb
    # pdb.set_trace()

pod_ips = {}
for pod in pods:
  if 'podIP' in pod['status']: # pods who can't get IPs should just be ignored
    pod_ips[pod['metadata']['name']] = pod['status']['podIP']

unused = 0
unused_by_node = {}
unused_by_eni = {}

for private_ip in private_ip_node:
  if private_ip not in pod_ips:
    unused += 1

    node_name = private_ip_node[private_ip]
    ec2instance = ec2sprivatedns[node_name]
    node = name_node[node_name]
    kops_group = node['metadata']['labels']['kops.k8s.io/instancegroup']
    # import pdb; pdb.set_trace()

    if node_name in unused_by_node:
      unused_by_node[node_name] += 1
    else:
      unused_by_node[node_name] = 1

    eni = private_ip_eni[private_ip]
    node_zone = node['metadata']['labels']['failure-domain.beta.kubernetes.io/zone']
    human_nice_eni = "%s (%s) %s %s" % (node_name,node_zone,kops_group,eni)
    if human_nice_eni not in unused_by_eni:
      unused_by_eni[human_nice_eni] = 0
    unused_by_eni[human_nice_eni] += 1

pp = pprint.PrettyPrinter(indent=4)

print("found %d unused IPs" % unused)
# pp.pprint(unused_by_node)
pp.pprint(unused_by_eni)