topology-manager-kubelet-restart-test.sh

#!/usr/bin/env bash

# Declare some helper functions
function get_kubernetes_hostname() {
local json="$(kubectl get nodes -o json)"
echo ${json} | python -c "$(cat << EOF
import sys
import json

nodes = json.load(sys.stdin)

for node in nodes["items"]:
	for status in node["status"]["conditions"]:
		if status["type"] == "Ready":
			if status["status"] == "True":
				for key, label in node["metadata"]["labels"].items():
					if key == "kubernetes.io/hostname":
						print(label)
						sys.exit(0)
EOF
)"
}

function __wait-for-namespace() {
  local namespace="${1}"
  while [ "${active}" != "Active" ]; do
    active=$(kubectl get namespace "${namespace}" -o jsonpath="{.status.phase}")
    sleep 1
  done
}
function wait-for-namespace() {
	{ __wait-for-namespace ${@}; } 2>&-
}

function __wait-for-container() {
  local pod_name="${1}"
  local container_name="${2}"
  local state="${3:-running}"
  local namespace="${4:-${TEST_NAMESPACE}}"
  local status=""
  while [ -z "${status}" ]; do
    status=$(kubectl get pod "${pod_name}" --namespace=${namespace} -o jsonpath="{.status.containerStatuses[?(@.name==\""${container_name}\"")].state.${state}}")
    sleep 1
  done
}
function wait-for-container() {
	{ __wait-for-container ${@}; } 2>&-
}

function start_exclusive_cpu_container() {
  cat << EOF | kubectl create -f -
  apiVersion: v1
  kind: Pod
  metadata:
    name: exclusive-cpu-pod-${1}
    namespace: ${TEST_NAMESPACE}
  spec:
    restartPolicy: Never
    nodeSelector:
      kubernetes.io/hostname: ${KUBERNETES_HOSTNAME}
    containers:
      - name: exclusive-cpu-container-${1}
        image: ${TEST_IMAGE}
        command: ["/bin/bash"]
        args: ["-c", "sleep infinity"]
        resources:
          limits:
            cpu: ${CPUS_PER_CONTAINER}
            memory: 100Mi
EOF
}

function start_kill_kubelet_container() {
  cat << EOF | kubectl create -f -
      apiVersion: v1
      kind: Pod
      metadata:
        name: kubelet-restart-pod
        namespace: ${TEST_NAMESPACE}
      spec:
        hostPID: true
        restartPolicy: Never
        nodeSelector:
          kubernetes.io/hostname: ${KUBERNETES_HOSTNAME}
        containers:
          - name: kubelet-restart-container
            image: ${TEST_IMAGE}
            command: ["/bin/bash"]
            args: ["-c", "pkill -e kubelet -STOP; sleep 20; pkill -e kubelet -9"]
            securityContext:
              privileged: true
EOF
}

# Declare some global variables
: ${TEST_NAMESPACE:="cpu-manager-test"}
: ${KUBERNETES_HOSTNAME:="$(get_kubernetes_hostname)"}
: ${TEST_IMAGE:="ubuntu:16.04"}
: ${CPUS_PER_CONTAINER:=7}
: ${NUM_TOTAL_CONTAINERS:=10}
: ${NUM_CONTAINERS_TO_KILL:=2}

cat << EOF
#################################################
Running test: ${0}
#################################################

EOF

echo "*** Clean up the TEST_NAMESPACE ***"
kubectl delete --all pods --namespace=${TEST_NAMESPACE}
kubectl get namespace ${TEST_NAMESPACE} > /dev/null 2>&1
if [ "${?}" != "0" ]; then
	kubectl create namespace ${TEST_NAMESPACE}
fi
wait-for-namespace ${TEST_NAMESPACE}
echo ""

echo "*** Launch some containers asking for exclusive CPUs ***"
for i in $(seq ${NUM_TOTAL_CONTAINERS}); do
  start_exclusive_cpu_container $i
  wait-for-container exclusive-cpu-pod-$i exclusive-cpu-container-$i running
  kubectl --namespace=${TEST_NAMESPACE} exec exclusive-cpu-pod-$i -c exclusive-cpu-container-$i -- bash -c "taskset -cp 1"
  echo ""
done

echo "*** Launching container to pause the kubelet and duty-cycle it after 20 seconds ***"
start_kill_kubelet_container
wait-for-container kubelet-restart-pod kubelet-restart-container waiting
echo ""

echo "*** Wait for 10 seconds (i.e. half the time before the kubelet gets restarted) ***"
sleep 10
echo ""

echo "*** Kill a few containers with exclusive CPUs via docker directly ***"
for i in $(seq ${NUM_CONTAINERS_TO_KILL}); do
  docker kill $(docker ps | grep exclusive-cpu-container-${i}_ | cut -d" " -f1)
done
for i in $(seq ${NUM_CONTAINERS_TO_KILL}); do
  wait-for-container exclusive-cpu-pod-$i exclusive-cpu-container-$i terminated
  kubectl get pod exclusive-cpu-pod-$i --namespace=${TEST_NAMESPACE}
  echo ""
done

echo "*** Wait for the container duty-cycling the kubelet to complete ***"
wait-for-container kubelet-restart-pod kubelet-restart-container terminated
kubectl get pod kubelet-restart-pod --namespace=${TEST_NAMESPACE}
echo ""

echo "*** Launch a few new containers asking for exclusive CPUs ***"
for i in $(seq $(expr ${NUM_TOTAL_CONTAINERS} + 1) $(expr ${NUM_TOTAL_CONTAINERS} + ${NUM_CONTAINERS_TO_KILL})); do
  start_exclusive_cpu_container $i
  wait-for-container exclusive-cpu-pod-$i exclusive-cpu-container-$i running
  kubectl --namespace=${TEST_NAMESPACE} exec exclusive-cpu-pod-$i -c exclusive-cpu-container-$i -- bash -c "taskset -cp 1"
  echo ""
done

echo "*** Print all pods on the node ***"
kubectl get pod --field-selector spec.nodeName=${KUBERNETES_HOSTNAME} --all-namespaces