Debug notes for https://issues.redhat.com/browse/OCPBUGS-29690

OCPBUGS-29690

# set reload interval to default (5s)
# oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"tuningOptions":{"reloadInterval":"5s"}}}'

# enable router access logs.
oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"logging":{"access":{"destination":{"type":"Container"}}}}}'

# let's concentrate on one router pod only.
oc scale -n openshift-ingress-operator ingresscontroller/default --replicas 1

# We're going to replace the router image and change the configuration
# so that the router pods runs as user=ROOT. We need to disable CVO
# and the ingress-operator to so that our changes don't get
# overwritten.
oc scale -n openshift-cluster-version deployments/cluster-version-operator --replicas 0
oc scale -n openshift-ingress-operator deployments ingress-operator --replicas 0

# Change cluster configuration to run router pods as ROOT.
oc patch clusterversions/version --type=json --patch='[{"op":"add","path":"/spec/overrides","value":[{"kind":"Deployment","group":"apps/v1","name":"ingress-operator","namespace":"openshift-ingress-operator","unmanaged":true}]}]'
oc patch clusterrole/openshift-ingress-router --type=strategic --patch='{"rules":[{"apiGroups":[""],"resources":["endpoints","namespaces","services"],"verbs":["list","watch"]},{"apiGroups":["authentication.k8s.io"],"resources":["tokenreviews"],"verbs":["create"]},{"apiGroups":["authorization.k8s.io"],"resources":["subjectaccessreviews"],"verbs":["create"]},{"apiGroups":["route.openshift.io"],"resources":["routes"],"verbs":["list","watch"]},{"apiGroups":["route.openshift.io"],"resources":["routes/status"],"verbs":["update"]},{"apiGroups":["security.openshift.io"],"resourceNames":["privileged"],"resources":["securitycontextconstraints"],"verbs":["use"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["list","watch"]}]}'
oc patch -n openshift-ingress deployment/router-default --patch='{"spec":{"template":{"spec":{"securityContext":{"runAsUser":0}}}}}'

# replace router image.
oc -n openshift-ingress patch deployment router-default -p '{"spec":{"template":{"spec":{"$setElementOrder/containers":[{"name":"router"}],"containers":[{"imagePullPolicy":"Always","name":"router"}]}}}}'
oc -n openshift-ingress set image deployment/router-default router=quay.io/amcdermo/openshift-router-ocpbugs29690-10april24:latest

# let's concentrate on one router pod only.
oc scale -n openshift-ingress deployment/router-default --replicas 1

# verify new haproxy version
oc -n openshift-ingress rsh -c router deployment/router-default sh -c 'haproxy -v'

# oc rsh into a new pod and verify haproxy version; should look like:
# sh-4.4# haproxy -v
# [60] contrib/interposer/accept.c:289:setup(): ACCEPT INTERPOSER initialised
# HA-Proxy version 2.2.24-accept-interposer 2024-03-14T09-24-00-GMT - https://haproxy.org/
# Status: long-term supported branch - will stop receiving fixes around Q2 2025.
# Known bugs: http://www.haproxy.org/bugs/bugs-2.2.24.html
# Running on: Linux 4.18.0-372.82.1.el8_6.x86_64 #1 SMP Fri Nov 17 17:58:32 EST 2023 x86_64

# Cleanup

## AND THIS IS HOW TO REVERT EVERYTHING...
## AND THIS IS HOW TO REVERT EVERYTHING...
# And once we've finished with the interactive debug session we need
# to restore everything we have changed.
## AND THIS IS HOW TO REVERT EVERYTHING...
## AND THIS IS HOW TO REVERT EVERYTHING...

oc patch clusterversions/version --type=json --patch='[{"op":"add","path":"/spec/overrides","value":[{"kind":"Deployment","group":"apps/v1","name":"ingress-operator","namespace":"openshift-ingress-operator","unmanaged":false}]}]'
oc patch clusterrole/openshift-ingress-router --type=strategic --patch='{"rules":[{"apiGroups":[""],"resources":["endpoints","namespaces","services"],"verbs":["list","watch"]},{"apiGroups":["authentication.k8s.io"],"resources":["tokenreviews"],"verbs":["create"]},{"apiGroups":["authorization.k8s.io"],"resources":["subjectaccessreviews"],"verbs":["create"]},{"apiGroups":["route.openshift.io"],"resources":["routes"],"verbs":["list","watch"]},{"apiGroups":["route.openshift.io"],"resources":["routes/status"],"verbs":["update"]},{"apiGroups":["security.openshift.io"],"resourceNames":["hostnetwork"],"resources":["securitycontextconstraints"],"verbs":["use"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["list","watch"]}]}'
oc patch -n openshift-ingress deployment/router-default --patch='{"spec":{"template":{"spec":{"securityContext":{"runAsUser":null}}}}}'

oc scale --replicas 1 -n openshift-cluster-version deployments/cluster-version-operator
oc scale --replicas 1 -n openshift-ingress-operator deployments ingress-operator
oc scale --replicas 2 -n openshift-ingress-operator ingresscontroller/default

oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"logging":null}}'
oc patch -n openshift-ingress-operator ingresscontroller/default --type=merge --patch='{"spec":{"tuningOptions":{"reloadInterval":"5s"}}}'

## Diagnostics / log capture

# Getting access to the local stats port; port forward in a shell.
oc port-forward -n openshift-ingress deployment/router-default --address localhost 1942:1942

## Run openshift-router to log events related to reloads.
oc -n openshift-ingress rsh -c router deploy/router-default sh -c 'rm -rf /tmp/haproxy && mkdir -p /tmp/haproxy/\{router/{certs,cacerts,whitelists},\{conf/.tmp,run,bin,log}} && /bin/openshift-router --v=5 --template=/var/lib/haproxy/conf/haproxy-config.template --reload=/bin/true --update-status=false --stats-port=1937 --metrics-type=haproxy --working-dir=/tmp/haproxy' | tee -a openshift-router-"$(date --iso-8601=seconds)".txt

# In another shell gather stats.
while :; do curl -s http://localhost:1942 > router-stats-"$(date --iso-8601=seconds)".html; date; sleep 10; done