Recover from the upgrade patch conflict

commands.md

1. Remove the selector section in the pod-checkpointer spec:

cat<<EOF | kubectl replace -f -
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  labels:
    k8s-app: pod-checkpointer
    tier: control-plane
  name: pod-checkpointer
  namespace: kube-system
spec:
  template:
    metadata:
      annotations:
        checkpointer.alpha.coreos.com/checkpoint: "true"
      labels:
        k8s-app: pod-checkpointer
        tier: control-plane
    spec:
      containers:
      - command:
        - /checkpoint
        - --v=4
        - --lock-file=/var/run/lock/pod-checkpointer.lock
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        image: quay.io/coreos/pod-checkpointer:3517908b1a1837e78cfd041a0e51e61c7835d85f
        imagePullPolicy: Always
        name: pod-checkpointer
        volumeMounts:
        - mountPath: /etc/kubernetes
          name: etc-kubernetes
        - mountPath: /var/run
          name: var-run
      hostNetwork: true
      nodeSelector:
        node-role.kubernetes.io/master: ""
      restartPolicy: Always
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
        operator: Exists
      volumes:
      - hostPath:
          path: /etc/kubernetes
        name: etc-kubernetes
      - hostPath:
          path: /var/run
        name: var-run
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
EOF

2. Clean up the taskStatuses field in the AppVersion record:

cat<<EOF | kubectl replace -f -
apiVersion: tco.coreos.com/v1 
kind: AppVersion 
metadata: 
labels: 
managed-by-channel-operator: "true" 
name: kubernetes 
namespace: tectonic-system 
spec: 
desiredVersion: 1.8.4+tectonic.2 
paused: true 
status: 
currentVersion: 1.7.14+tectonic.2 
paused: true 
targetVersion: 1.8.4+tectonic.2 
EOF

3. Unpause the upgrade, if kvo doesn't process, kill the kvo pod and let it restart.