test-route-tls-cert.sh

# Script to test for bad cert and report timestamp & subject line upon receiving a bad cert
while true; do
    RES=$(echo "Q" | openssl s_client -servername api.openshift.io -connect api.openshift.io:443 2>&1 | egrep ^subject)
    echo $RES | egrep -q '^subject.*openshift.io'
    if [[ $? -ne 0 ]]; then
        echo "$(date --utc) $RES"
    fi
    sleep .1
done

# I've ran this script both on my fedora27 host and under a centos7 container running on that same host

# fedora27 (host): openssl-1.1.0h-3.fc27.x86_64
# fedora27 (container): openssl-1.1.0h-3.fc27.x86_64
# fedora21 (container): openssl-1.0.1k-12.fc21.x86_64
# centos7 (container): openssl-1.0.2k-8.el7.x86_64
# centos6 (container): openssl-1.0.1e-57.el6.x86_64
# alpine (container): openssl 1.0.2o-r0

# I have observed a LOT of bad cert errors within a 1h timespam under the centos7, centos6 and fedora21 containers
# ... and NONE on other others (fedora27, alpine...)
Wed May  2 14:56:27 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 14:58:50 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 14:59:46 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:02:27 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:04:13 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:05:53 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:10:24 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:20:11 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:21:05 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:22:32 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:29:58 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:31:32 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:32:17 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:36:53 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:43:30 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:49:08 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:49:26 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:49:46 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:50:09 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:50:40 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:53:50 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:54:17 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:58:02 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 15:59:46 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com
Wed May  2 16:00:21 UTC 2018 subject=/C=US/ST=North Carolina/L=Raleigh/O=Red Hat Inc./CN=*.09b5.dsaas.openshiftapps.com

### Running this (create/delete route every 1 sec) dit NOT cause the problem to occur more often
while true; do
    oc create -f route.yml
    sleep 1
    oc delete -f route.yml
    sleep 1
done

### I did NOT observe route/service/endpoints/pods changes around when openssl returned the wrong cert
oc get routes --all-namespaces --watch
oc get services --all-namespaces --watch
oc get endpoints --all-namespaces --watch