find-jenkins-zombies.py

#!/usr/bin/env python
#
#/**
# * Copyright 2018 The Apache Software Foundation
# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements.  See the NOTICE file
# * distributed with this work for additional information
# * regarding copyright ownership.  The ASF licenses this file
# * to you under the Apache License, Version 2.0 (the
# * "License"); you may not use this file except in compliance
# * with the License.  You may obtain a copy of the License at
# *
# *     http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */

import subprocess, sys, getopt

def usage():
    print >> sys.stderr, "\n%s -u <user> -c <executable> -s <searchstring>" % (sys.argv[0])
    print >> sys.stderr, "\nFind all processes which belong to <user>,"
    print >> sys.stderr, "have been alive for more than a day,"
    print >> sys.stderr, "and are either children, or unrelated (don't share common ancestor)"
    print >> sys.stderr, "of a process which uses <executable>, contains <searchstring>,"

try:
    opts, args = getopt.getopt(sys.argv[1:], "u:c:s:h")
except getopt.GetoptError as err:
    print >> sys.stderr, str(err)
    usage()
    sys.exit(2)

targetuser = None
targetcommand = None
targetsearchstring = None

for o, a in opts:
    if o in ("-h"):
        usage()
        sys.exit()
    elif o in ("-u"):
        targetuser = a
    elif o in ("-c"):
        targetcommand = a
    elif o in ("-s"):
        targetsearchstring=a
    else:
        assert False, "unhandled option"

if not (targetuser and targetcommand and targetsearchstring):
    print >> sys.stderr, "All options must be specified"
    usage()
    sys.exit(3)

class Process:
    def __init__(self, user, pid, parent, running, command, fullcommand):
        self.user=user
        self.pid=pid
        self.parent=parent
        self.running=running
        self.command=command
        self.fullcommand=fullcommand

    def add_child(self, node):
        pass

    def __str__(self):
        return "proc(pid=%s,ppid=%s,user=%s,command=%s,running=%s)" \
            % (self.pid, self.parent, self.user, self.command, self.running)

# build a dict of processes for target user
by_pid = dict()
for l in subprocess.check_output(['ps', '--no-headers',
                                  '-eo', '%u||%p||%P||',
                                  '-o', 'etimes',
                                  '-o', '||%c||',
                                  '-o', 'cmd']).split('\n'):
    parts = l.split('||')
    if len(parts) == 6:
        process = Process(parts[0].strip(), parts[1].strip(), parts[2].strip(),
                          int(parts[3].strip()), parts[4].strip(), parts[5].strip())
        if process.user == targetuser:
            by_pid[process.pid] = process

# find target process
target = None
for pid in by_pid:
    process = by_pid[pid]
    if process.command == targetcommand \
       and process.fullcommand.find(targetsearchstring) >= 0:
        target = process
    print

if not target:
    print >> sys.stderr, "Target process not found"
    sys.exit(1)

# find highest parent in process tree for target process
root_pid = target.pid
current = target
while by_pid.has_key(current.parent):
    root_pid = current.parent
    current = by_pid[current.parent]

# find processes which are not child processes of the
# target process or do not share the top most ancestor
candidates = []
for pid in by_pid:
    candidate_process = by_pid[pid]

    if candidate_process.pid == target.pid:
        continue

    candidate_root_pid = candidate_process.pid
    current = candidate_process
    while by_pid.has_key(current.parent):
        if current.parent == target.pid:
            candidates.append(candidate_process)
        candidate_root_pid = current.parent
        current = by_pid[current.parent]
    if candidate_root_pid != root_pid:
        candidates.append(candidate_process)

# Trim down candidates to those which have been around more than a day
for c in candidates:
    if c.running > 24*60*60:
        print c.pid

kill-old-docker.sh

THRESHOLD=$(date --date "1 day ago" +%s)

docker ps --format "{{.ID}} {{.CreatedAt}}" | while read LINE; do
    ID=$(echo $LINE | awk '{print $1}');
    DATE=$(echo $LINE | awk '{print $2" "$3" "$4}');
    AGE=$(date --date "$DATE" +%s);
    if [ $AGE -lt $THRESHOLD ]; then
        docker rm --force $ID
    fi;
done

kill-old-processes.sh

ps --no-headers -eo euser,pid,etimes,cmd | grep ^jenkins | \ 
grep -v -e "jenkins@notty" -e slave.jar -e systemd -e sd-pam | \
awk '{ if ($3 > 24*60*60) { print $2 } }' | xargs kill -9