debugging dead k8s pod ( copy k8s pods and inject busybox binary and change command to sleep )

debugdead.py

#!/usr/bin/env python

import argparse
import os
import subprocess
import sys
import uuid
from tempfile import NamedTemporaryFile

import yaml


def main():
    parser = argparse.ArgumentParser(description='Dead container rising.')
    parser.add_argument('pod', type=str,
                        help='target pod name')
    parser.add_argument('-n', '--namespace', type=str,
                        default='default',
                        help='target namespace (default: default)')
    parser.add_argument('-c', '--container', type=str,
                        default='',
                        help='target container name')
    parser.add_argument('-k', '--keep', action="store_true",
                        help='keep yaml')

    args = parser.parse_args()
    container_name = args.container

    kwargs = {}
    kwargs.setdefault('stdout', subprocess.PIPE)
    kwargs.setdefault('stderr', subprocess.STDOUT)
    kwargs.setdefault('encoding', 'utf8')
    kwargs.setdefault('universal_newlines', True)
    kwargs['env'] = os.environ.copy()

    command = ['kubectl', 'get', 'pods', '-n', args.namespace,
               args.pod, "--export=true", "-o", "yaml"]
    pipe = subprocess.Popen(command, **kwargs)
    output, _ = pipe.communicate()

    try:
        data = yaml.load(output)
    except yaml.YAMLError as exc:
        print(exc)

    if 'metadata' not in data:
        print("There is no pod")
        return

    data['metadata'].pop('ownerReferences', None)
    data['metadata'].pop('selfLink', None)
    name = "debug-"
    if 'generateName' in data['metadata']:
        name += data['metadata']['generateName']
        name += str(uuid.uuid4())[:5]
    else:
        name += data['metadata']['name']
        name += str(uuid.uuid4())[:5]
    data['metadata']['name'] = name
    data['metadata'].pop('generateName', None)
    if data['metadata'].get('labels'):
        data['metadata']['labels'].pop('pod-template-hash', None)

    if not data['spec'].get('initContainers'):
        data['spec']['initContainers'] = []

    data['spec'].pop('nodeName', None)

    initcontainer = {}
    initcontainer['name'] = 'init-debugger'
    initcontainer['image'] = 'busybox'
    initcontainer['command'] = ['cp', '/bin/busybox', '/tmp/mydebug/busybox']
    initcontainer['volumeMounts'] = [
        {'name': 'mydebug', 'mountPath': '/tmp/mydebug/'}]

    data['spec']['initContainers'].append(initcontainer)

    real_command = ''
    for c in data['spec']['containers']:
        if container_name == '' or c['name'] == container_name:
            if 'command' in c:
                real_command = " ".join(c['command'])

            c['command'] = [
                '/bin/busybox',
                'sh', '-c',
                '/bin/busybox --install -s /tmp/mydebug/ && '
                '/tmp/mydebug/sleep 86400']
            if 'volumeMounts' not in c:
                c['volumeMounts'] = []
            c['volumeMounts'].append(
                {'name': 'insert-busybox', 'mountPath': '/tmp/mydebug/'})
            c['volumeMounts'].append(
                {'name': 'mydebug-busybox', 'mountPath': '/bin/busybox'})

            c.pop('livenessProbe', None)
            c.pop('readinessProbe', None)
            if container_name == '':
                container_name = c['name']
            break
    else:
        print("Error: can not found %s" % container_name)

    if 'volumes' not in data['spec']:
        data['spec']['volumes'] = []
    for v in data['spec']['volumes']:
        if v.get('persistentVolumeClaim', None):
            v.pop('persistentVolumeClaim', None)
            v['emptyDir'] = {}

    data['spec']['volumes'].append(
        {'name': 'insert-busybox', 'emptyDir': {}})
    data['spec']['volumes'].append(
        {'name': 'mydebug', 'hostPath': {'path': '/tmp/mydebug/'}})
    data['spec']['volumes'].append(
        {'name': 'mydebug-busybox',
         'hostPath': {'path': '/tmp/mydebug/busybox'}})

    data.pop('status', None)

    file = NamedTemporaryFile(suffix='.yaml', delete=False)

    yamlfile = yaml.dump(data, None, canonical=True)
    file.write(yamlfile.encode('utf8'))
    file.close()

    command2 = ['kubectl', 'create', '-n', args.namespace, '-f', file.name]
    pipe = subprocess.Popen(command2, **kwargs)
    output, _ = pipe.communicate()
    if args.keep:
        print("Check this changed pod yaml:")
        print(file.name)
    else:
        os.unlink(file.name)

    print("To debug %s, wait some second and run:" % args.pod)
    print("    kubectl exec -ti %s -n %s "
          "-c %s -- /tmp/mydebug/sh "
          "-c \"PATH=\$PATH:/tmp/mydebug/ sh\"" %
          (name, args.namespace, container_name))

    print("")
    print("If you want to run %s container's command"
          " then check below command" % args.pod)
    print("    " + real_command)
    print("")
    print("After finish debugging please delete debugging container:")
    print("    kubectl delete pods %s -n %s " %
          (name, args.namespace))


if __name__ == '__main__':
    sys.exit(main())