Skip to content

Instantly share code, notes, and snippets.

@leoh0
Last active January 23, 2024 01:58
Show Gist options
  • Save leoh0/0af21e5839a1ee7a799e8b2b40603ecd to your computer and use it in GitHub Desktop.
Save leoh0/0af21e5839a1ee7a799e8b2b40603ecd to your computer and use it in GitHub Desktop.
debugging dead k8s pod ( copy k8s pods and inject busybox binary and change command to sleep )
#!/usr/bin/env python
import argparse
import os
import subprocess
import sys
import uuid
from tempfile import NamedTemporaryFile
import yaml
def main():
parser = argparse.ArgumentParser(description='Dead container rising.')
parser.add_argument('pod', type=str,
help='target pod name')
parser.add_argument('-n', '--namespace', type=str,
default='default',
help='target namespace (default: default)')
parser.add_argument('-c', '--container', type=str,
default='',
help='target container name')
parser.add_argument('-k', '--keep', action="store_true",
help='keep yaml')
args = parser.parse_args()
container_name = args.container
kwargs = {}
kwargs.setdefault('stdout', subprocess.PIPE)
kwargs.setdefault('stderr', subprocess.STDOUT)
kwargs.setdefault('encoding', 'utf8')
kwargs.setdefault('universal_newlines', True)
kwargs['env'] = os.environ.copy()
command = ['kubectl', 'get', 'pods', '-n', args.namespace,
args.pod, "--export=true", "-o", "yaml"]
pipe = subprocess.Popen(command, **kwargs)
output, _ = pipe.communicate()
try:
data = yaml.load(output)
except yaml.YAMLError as exc:
print(exc)
if 'metadata' not in data:
print("There is no pod")
return
data['metadata'].pop('ownerReferences', None)
data['metadata'].pop('selfLink', None)
name = "debug-"
if 'generateName' in data['metadata']:
name += data['metadata']['generateName']
name += str(uuid.uuid4())[:5]
else:
name += data['metadata']['name']
name += str(uuid.uuid4())[:5]
data['metadata']['name'] = name
data['metadata'].pop('generateName', None)
if data['metadata'].get('labels'):
data['metadata']['labels'].pop('pod-template-hash', None)
if not data['spec'].get('initContainers'):
data['spec']['initContainers'] = []
data['spec'].pop('nodeName', None)
initcontainer = {}
initcontainer['name'] = 'init-debugger'
initcontainer['image'] = 'busybox'
initcontainer['command'] = ['cp', '/bin/busybox', '/tmp/mydebug/busybox']
initcontainer['volumeMounts'] = [
{'name': 'mydebug', 'mountPath': '/tmp/mydebug/'}]
data['spec']['initContainers'].append(initcontainer)
real_command = ''
for c in data['spec']['containers']:
if container_name == '' or c['name'] == container_name:
if 'command' in c:
real_command = " ".join(c['command'])
c['command'] = [
'/bin/busybox',
'sh', '-c',
'/bin/busybox --install -s /tmp/mydebug/ && '
'/tmp/mydebug/sleep 86400']
if 'volumeMounts' not in c:
c['volumeMounts'] = []
c['volumeMounts'].append(
{'name': 'insert-busybox', 'mountPath': '/tmp/mydebug/'})
c['volumeMounts'].append(
{'name': 'mydebug-busybox', 'mountPath': '/bin/busybox'})
c.pop('livenessProbe', None)
c.pop('readinessProbe', None)
if container_name == '':
container_name = c['name']
break
else:
print("Error: can not found %s" % container_name)
if 'volumes' not in data['spec']:
data['spec']['volumes'] = []
for v in data['spec']['volumes']:
if v.get('persistentVolumeClaim', None):
v.pop('persistentVolumeClaim', None)
v['emptyDir'] = {}
data['spec']['volumes'].append(
{'name': 'insert-busybox', 'emptyDir': {}})
data['spec']['volumes'].append(
{'name': 'mydebug', 'hostPath': {'path': '/tmp/mydebug/'}})
data['spec']['volumes'].append(
{'name': 'mydebug-busybox',
'hostPath': {'path': '/tmp/mydebug/busybox'}})
data.pop('status', None)
file = NamedTemporaryFile(suffix='.yaml', delete=False)
yamlfile = yaml.dump(data, None, canonical=True)
file.write(yamlfile.encode('utf8'))
file.close()
command2 = ['kubectl', 'create', '-n', args.namespace, '-f', file.name]
pipe = subprocess.Popen(command2, **kwargs)
output, _ = pipe.communicate()
if args.keep:
print("Check this changed pod yaml:")
print(file.name)
else:
os.unlink(file.name)
print("To debug %s, wait some second and run:" % args.pod)
print(" kubectl exec -ti %s -n %s "
"-c %s -- /tmp/mydebug/sh "
"-c \"PATH=\$PATH:/tmp/mydebug/ sh\"" %
(name, args.namespace, container_name))
print("")
print("If you want to run %s container's command"
" then check below command" % args.pod)
print(" " + real_command)
print("")
print("After finish debugging please delete debugging container:")
print(" kubectl delete pods %s -n %s " %
(name, args.namespace))
if __name__ == '__main__':
sys.exit(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment