-
-
Save pacphi/05e6bd49b312bb92b2db1d70beb5c69c to your computer and use it in GitHub Desktop.
#!/usr/bin/env bash | |
indent4() { sed 's/^/ /'; } | |
# Install the necessary resources to support cert-manager deployed on OKE | |
# vending valid certificate via a Let's Encrypt ClusterIssuer | |
# Set environemnt variables (these are sample values, please replace with your own) | |
export DOMAIN=foo.me | |
export EMAIL_ADDRESS=any@valid.email | |
export COMPARTMENT_OCID=ocid1.compartment.oc1..aaaaaaaa_ | |
export TENANCY_OCID=ocid1.tenancy.oc1..aaaaaaaa_ | |
export USER_OCID=ocid1.user.oc1..aaaaaaaa_ | |
export REGION=us-phoenix-1 | |
export FINGERPRINT=47:5f:c7:0d:a3:a5:ac:d6:53:41:d2:23:c6:c9:24:a2 | |
# Oracle Cloud credentials | |
export OCI_CONFIG_HOME=$HOME/.oci | |
export OCI_PEM_PRIVATE_KEY_FILE_PATH=$OCI_CONFIG_HOME/oci_api_key.pem | |
# Google Cloud credentials (where I've decided to host updated webhook container image) | |
export GOOGLE_PROJECT_ID=fe-cphillipson | |
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.ssh/terraform@${GOOGLE_PROJECT_ID}.iam.gserviceaccount.com.json | |
# Convert PEM private key to RSA | |
openssl rsa -in $OCI_PEM_PRIVATE_KEY_FILE_PATH -out $OCI_CONFIG_HOME/oci_api_rsa_key | |
export RSA_PRIVATE_KEY=$(cat $OCI_CONFIG_HOME/oci_api_rsa_key | indent4) | |
# Install Contour ingress | |
kubectl apply -f https://projectcontour.io/quickstart/contour.yaml | |
# Install cert-manager | |
helm repo add jetstack https://charts.jetstack.io | |
helm repo update | |
helm install \ | |
cert-manager jetstack/cert-manager \ | |
--namespace cert-manager \ | |
--create-namespace \ | |
--version v1.10.1 \ | |
--set installCRDs=true \ | |
--set prometheus.enabled=false \ | |
--set webhook.timeoutSeconds=30 | |
# Install cert-manager OCI webhook | |
# This is from a fork of https://gitlab.com/dn13/cert-manager-webhook-oci | |
# @see https://gitlab.com/jcotton/cert-manager-webhook-oci.git | |
git clone https://gitlab.com/jcotton/cert-manager-webhook-oci.git | |
cd cert-manager-webhook-oci | |
git checkout fix_and_update | |
helm install --namespace cert-manager cert-manager-webhook-oci ./deploy/cert-manager-webhook-oci \ | |
--set image.repository=us.gcr.io/fe-cphillipson/cert-manager-webhook-oci | |
# Create image pull secret | |
kubectl create secret docker-registry regcred \ | |
--docker-server=us.gcr.io \ | |
--docker-username=_json_key \ | |
--docker-password="$(cat $GOOGLE_APPLICATION_CREDENTIALS)" \ | |
--docker-email=${EMAIL_ADDRESS} \ | |
--namespace cert-manager | |
# Create namespace to store secret | |
kubectl create ns contour-tls | |
mkdir -p /tmp/oci | |
cd /tmp/oci | |
# Define ClusterIssuer | |
cat << EOF > cluster-issuer-oci.yml | |
apiVersion: cert-manager.io/v1 | |
kind: ClusterIssuer | |
metadata: | |
name: letsencrypt-prod | |
spec: | |
acme: | |
# The ACME server URL | |
server: https://acme-v02.api.letsencrypt.org/directory | |
# Email address used for ACME registration | |
email: $EMAIL_ADDRESS | |
# Name of a secret used to store the ACME account private key | |
privateKeySecretRef: | |
name: letsencrypt-prod | |
solvers: | |
- dns01: | |
webhook: | |
groupName: acme.d-n.be | |
solverName: oci | |
config: | |
ociProfileSecretName: oci-profile | |
compartmentOCID: $COMPARTMENT_OCID | |
EOF | |
# Define Secret with OCI credentials | |
cat << EOF > secret-oci.yml | |
apiVersion: v1 | |
kind: Secret | |
metadata: | |
name: oci-profile | |
namespace: cert-manager | |
type: Opaque | |
stringData: | |
tenancy: "$TENANCY_OCID" | |
user: "$USER_OCID" | |
region: "$REGION" | |
fingerprint: "$FINGERPRINT" | |
privateKey: | | |
$RSA_PRIVATE_KEY | |
privateKeyPassphrase: "" | |
EOF | |
# Define Certificate | |
cat << EOF > certificate-oci.yml | |
apiVersion: cert-manager.io/v1 | |
kind: Certificate | |
metadata: | |
name: tls | |
namespace: contour-tls | |
spec: | |
commonName: $DOMAIN | |
dnsNames: | |
- $DOMAIN | |
issuerRef: | |
kind: ClusterIssuer | |
name: letsencrypt-prod | |
secretName: tls | |
EOF | |
cd .. | |
# Let it rip! | |
kubectl apply -f oci/ |
#!/usr/bin/env bash | |
# Build, tag and push image to GCR | |
export IMAGE_PREFIX=pacphi | |
export GOOGLE_PROJECT_ID=fe-cphillipson | |
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.ssh/terraform@${GOOGLE_PROJECT_ID}.iam.gserviceaccount.com.json | |
## Authenticate to container registry | |
cat $GOOGLE_APPLICATION_CREDENTIALS | docker login -u _json_key --password-stdin https://us.gcr.io | |
## Clone | |
cd /tmp | |
git clone https://gitlab.com/jcotton/cert-manager-webhook-oci.git | |
cd cert-manager-webhook-oci | |
git checkout fix_and_update | |
## Build image | |
docker build -t ${IMAGE_PREFIX}/cert-manager-webhook-oci . | |
## Tag image | |
docker tag ${IMAGE_PREFIX}/cert-manager-webhook-oci us.gcr.io/${GOOGLE_PROJECT_ID}/cert-manager-webhook-oci:latest | |
## Push image | |
docker push us.gcr.io/${GOOGLE_PROJECT_ID}/cert-manager-webhook-oci:latest | |
## Cleanup | |
rm -Rf /tmp/cert-manager-webhook-oci |
#!/usr/bin/env bash | |
# Uninstall the resources supporting cert-manager deployed on OKE | |
# vending valid certificate via a Let's Encrypt ClusterIssuer | |
cd /tmp | |
kubectl delete -f oci/ | |
rm -Rf /tmp/oci | |
rm -Rf /tmp/cert-manager-webhook-oci | |
# Delete namespace used to store secret | |
kubectl delete secret --all -n contour-tls | |
kubectl delete ns contour-tls | |
# Uninstall cert-manager OCI webhook | |
helm uninstall --namespace cert-manager cert-manager-webhook-oci | |
#helm repo remove cert-manager-webhook-oci | |
# Uninstall cert-manager | |
helm uninstall --namespace cert-manager cert-manager | |
helm repo remove jetstack | |
kubectl delete secret --all -n cert-manager | |
kubectl delete namespace cert-manager | |
# Uninstall Contour ingress | |
kubectl delete -f https://projectcontour.io/quickstart/contour.yaml |
Some output...
❯ k get cert -A
NAMESPACE NAME READY SECRET AGE
cert-manager cert-manager-webhook-oci-ca True cert-manager-webhook-oci-ca 27s
cert-manager cert-manager-webhook-oci-webhook-tls True cert-manager-webhook-oci-webhook-tls 27s
contour-tls tls False tls 23s
❯ k describe challenge -A
Name: tls-k84xg-1221875851-2183180315
Namespace: contour-tls
Labels: <none>
Annotations: <none>
API Version: acme.cert-manager.io/v1
Kind: Challenge
Metadata:
Creation Timestamp: 2023-01-01T00:11:14Z
Finalizers:
finalizer.acme.cert-manager.io
Generation: 1
Managed Fields:
API Version: acme.cert-manager.io/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:ownerReferences:
.:
k:{"uid":"4ca2d898-10b9-4cc9-90bc-1f9bd23a37e3"}:
f:spec:
.:
f:authorizationURL:
f:dnsName:
f:issuerRef:
.:
f:kind:
f:name:
f:key:
f:solver:
.:
f:dns01:
.:
f:webhook:
.:
f:config:
.:
f:compartmentOCID:
f:ociProfileSecretName:
f:groupName:
f:solverName:
f:token:
f:type:
f:url:
f:wildcard:
Manager: cert-manager-orders
Operation: Update
Time: 2023-01-01T00:11:14Z
API Version: acme.cert-manager.io/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:finalizers:
.:
v:"finalizer.acme.cert-manager.io":
Manager: cert-manager-challenges
Operation: Update
Time: 2023-01-01T00:11:15Z
API Version: acme.cert-manager.io/v1
Fields Type: FieldsV1
fieldsV1:
f:status:
.:
f:presented:
f:processing:
f:reason:
f:state:
Manager: cert-manager-challenges
Operation: Update
Subresource: status
Time: 2023-01-01T00:11:15Z
Owner References:
API Version: acme.cert-manager.io/v1
Block Owner Deletion: true
Controller: true
Kind: Order
Name: tls-k84xg-1221875851
UID: 4ca2d898-10b9-4cc9-90bc-1f9bd23a37e3
Resource Version: 75477
UID: 91888ab2-55f3-4f9e-ad73-e2c04ad6c5a9
Spec:
Authorization URL: https://acme-v02.api.letsencrypt.org/acme/authz-v3/191534689337
Dns Name: foo.cloudchief.me
Issuer Ref:
Kind: ClusterIssuer
Name: letsencrypt-prod
Key: lnrl9Vl6ddnEmTAH1fwvm8HK9EoR1QM4PMbfP9RW_uE
Solver:
dns01:
Webhook:
Config:
Compartment OCID: ocid1.compartment.oc1..aaaaaaaallulk3a4ivz6vqmriozeow2ms3ewbgc636kdyn4eqzrjpqotsiua
Oci Profile Secret Name: oci-profile
Group Name: acme.d-n.be
Solver Name: oci
Token: iSEVwJDIF4PQOR_Ti-VNVfqAdYDJB-qQtnUbkjc85Iw
Type: DNS-01
URL: https://acme-v02.api.letsencrypt.org/acme/chall-v3/191534689337/s1U4tg
Wildcard: false
Status:
Presented: false
Processing: true
Reason: the server could not find the requested resource (post oci.acme.d-n.be)
State: pending
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Started 30s cert-manager-challenges Challenge scheduled for processing
Warning PresentError 5s (x4 over 30s) cert-manager-challenges Error presenting challenge: the server could not find the requested resource (post oci.acme.d-n.be)
Pods in cert-manager
namespace...
❯ k get po -n cert-manager
NAME READY STATUS RESTARTS AGE
cert-manager-585ddf744d-kldkg 1/1 Running 0 9m59s
cert-manager-cainjector-7c44879bc4-h55fg 1/1 Running 0 9m59s
cert-manager-webhook-5db84854c8-8xzgb 1/1 Running 0 9m59s
cert-manager-webhook-oci-57c4b6556-vd48m 1/1 Running 0 9m55s
Log output from webhook
❯ k logs -n cert-manager cert-manager-webhook-oci-57c4b6556-vd48m
W0101 00:11:21.447131 1 configmap_cafile_content.go:102] unable to load initial CA bundle for: "client-ca::kube-system::extension-apiserver-authentication::client-ca-file" due to: configmap "extension-apiserver-authentication" not found
W0101 00:11:21.447367 1 configmap_cafile_content.go:102] unable to load initial CA bundle for: "client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" due to: configmap "extension-apiserver-authentication" not found
I0101 00:11:21.476743 1 configmap_cafile_content.go:205] Starting client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
I0101 00:11:21.476765 1 configmap_cafile_content.go:205] Starting client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0101 00:11:21.476774 1 shared_informer.go:197] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
I0101 00:11:21.476777 1 shared_informer.go:197] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0101 00:11:21.476936 1 dynamic_serving_content.go:129] Starting serving-cert::/tls/tls.crt::/tls/tls.key
I0101 00:11:21.477154 1 secure_serving.go:178] Serving securely on [::]:443
I0101 00:11:21.477169 1 tlsconfig.go:219] Starting DynamicServingCertificateController
I0101 00:11:21.576896 1 shared_informer.go:204] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
I0101 00:11:21.576916 1 shared_informer.go:204] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0101 00:11:22.799754 1 log.go:181] http: TLS handshake error from 10.1.1.0:44184: EOF
I0101 00:12:22.807952 1 log.go:181] http: TLS handshake error from 10.1.0.0:21373: EOF
I0101 00:12:22.815669 1 log.go:181] http: TLS handshake error from 10.1.0.0:31164: EOF
I0101 00:12:22.818462 1 log.go:181] http: TLS handshake error from 10.1.0.0:14338: EOF
Log output from controller...
❯ k logs -n cert-manager cert-manager-585ddf744d-kldkg
I0101 00:11:10.817871 1 start.go:75] cert-manager "msg"="starting controller" "git-commit"="a96bae172ddb1fcd4b57f1859ab9d1a9e94f7451" "version"="v1.10.1"
I0101 00:11:10.817936 1 controller.go:242] cert-manager/controller/build-context "msg"="configured acme dns01 nameservers" "nameservers"=["10.2.5.5:53"]
W0101 00:11:10.817985 1 client_config.go:617] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.
I0101 00:11:10.818673 1 controller.go:70] cert-manager/controller "msg"="enabled controllers: [certificaterequests-approver certificaterequests-issuer-acme certificaterequests-issuer-ca certificaterequests-issuer-selfsigned certificaterequests-issuer-vault certificaterequests-issuer-venafi certificates-issuing certificates-key-manager certificates-metrics certificates-readiness certificates-request-manager certificates-revision-manager certificates-trigger challenges clusterissuers ingress-shim issuers orders]"
I0101 00:11:10.819067 1 controller.go:134] cert-manager/controller "msg"="starting leader election"
I0101 00:11:10.819188 1 controller.go:91] cert-manager/controller "msg"="starting metrics server" "address"={"IP":"::","Port":9402,"Zone":""}
I0101 00:11:10.819318 1 leaderelection.go:248] attempting to acquire leader lease kube-system/cert-manager-controller...
I0101 00:11:10.850031 1 leaderelection.go:258] successfully acquired lease kube-system/cert-manager-controller
I0101 00:11:10.851814 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificates-key-manager"
I0101 00:11:10.852025 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificates-metrics"
I0101 00:11:10.852295 1 controller.go:182] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-acme"
I0101 00:11:10.852418 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificates-trigger"
I0101 00:11:10.852674 1 controller.go:182] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="gateway-shim"
I0101 00:11:10.852720 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="orders"
I0101 00:11:10.853061 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-selfsigned"
I0101 00:11:10.853386 1 controller.go:182] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-selfsigned"
I0101 00:11:10.853434 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificates-issuing"
I0101 00:11:10.853698 1 controller.go:182] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-ca"
I0101 00:11:10.853724 1 controller.go:182] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-venafi"
I0101 00:11:10.853758 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="clusterissuers"
I0101 00:11:10.854163 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="issuers"
I0101 00:11:10.854631 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-approver"
I0101 00:11:10.854859 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-venafi"
I0101 00:11:10.855218 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificates-readiness"
I0101 00:11:10.855492 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificates-revision-manager"
I0101 00:11:10.955914 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="challenges"
I0101 00:11:10.956728 1 controller.go:182] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-vault"
I0101 00:11:10.956793 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="ingress-shim"
I0101 00:11:10.957980 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-acme"
I0101 00:11:10.958212 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-ca"
I0101 00:11:10.958516 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-vault"
I0101 00:11:10.958972 1 controller.go:205] cert-manager/controller "msg"="starting controller" "controller"="certificates-request-manager"
I0101 00:11:11.054461 1 conditions.go:96] Setting lastTransitionTime for Issuer "cert-manager-webhook-oci-selfsign" condition "Ready" to 2023-01-01 00:11:11.05445153 +0000 UTC m=+0.259740759
E0101 00:11:11.054514 1 setup.go:48] cert-manager/issuers/setup "msg"="error getting signing CA TLS certificate" "error"="secret \"cert-manager-webhook-oci-ca\" not found" "resource_kind"="Issuer" "resource_name"="cert-manager-webhook-oci-ca" "resource_namespace"="cert-manager" "resource_version"="v1"
I0101 00:11:11.054554 1 conditions.go:96] Setting lastTransitionTime for Issuer "cert-manager-webhook-oci-ca" condition "Ready" to 2023-01-01 00:11:11.054549126 +0000 UTC m=+0.259838354
I0101 00:11:11.054588 1 sync.go:62] cert-manager/issuers "msg"="Error initializing issuer: secret \"cert-manager-webhook-oci-ca\" not found" "resource_kind"="Issuer" "resource_name"="cert-manager-webhook-oci-ca" "resource_namespace"="cert-manager" "resource_version"="v1"
E0101 00:11:11.130744 1 controller.go:167] cert-manager/issuers "msg"="re-queuing item due to error processing" "error"="secret \"cert-manager-webhook-oci-ca\" not found" "key"="cert-manager/cert-manager-webhook-oci-ca"
E0101 00:11:11.133375 1 setup.go:48] cert-manager/issuers/setup "msg"="error getting signing CA TLS certificate" "error"="secret \"cert-manager-webhook-oci-ca\" not found" "resource_kind"="Issuer" "resource_name"="cert-manager-webhook-oci-ca" "resource_namespace"="cert-manager" "resource_version"="v1"
I0101 00:11:11.133437 1 sync.go:62] cert-manager/issuers "msg"="Error initializing issuer: secret \"cert-manager-webhook-oci-ca\" not found" "resource_kind"="Issuer" "resource_name"="cert-manager-webhook-oci-ca" "resource_namespace"="cert-manager" "resource_version"="v1"
E0101 00:11:11.133492 1 controller.go:167] cert-manager/issuers "msg"="re-queuing item due to error processing" "error"="secret \"cert-manager-webhook-oci-ca\" not found" "key"="cert-manager/cert-manager-webhook-oci-ca"
I0101 00:11:11.352729 1 trigger_controller.go:200] cert-manager/certificates-trigger "msg"="Certificate must be re-issued" "key"="cert-manager/cert-manager-webhook-oci-ca" "message"="Issuing certificate as Secret does not exist" "reason"="DoesNotExist"
I0101 00:11:11.352755 1 conditions.go:203] Setting lastTransitionTime for Certificate "cert-manager-webhook-oci-ca" condition "Issuing" to 2023-01-01 00:11:11.352750847 +0000 UTC m=+0.558040116
I0101 00:11:11.352729 1 trigger_controller.go:200] cert-manager/certificates-trigger "msg"="Certificate must be re-issued" "key"="cert-manager/cert-manager-webhook-oci-webhook-tls" "message"="Issuing certificate as Secret does not exist" "reason"="DoesNotExist"
I0101 00:11:11.352869 1 conditions.go:203] Setting lastTransitionTime for Certificate "cert-manager-webhook-oci-webhook-tls" condition "Issuing" to 2023-01-01 00:11:11.352858409 +0000 UTC m=+0.558147639
I0101 00:11:11.356201 1 conditions.go:203] Setting lastTransitionTime for Certificate "cert-manager-webhook-oci-webhook-tls" condition "Ready" to 2023-01-01 00:11:11.356196133 +0000 UTC m=+0.561485363
I0101 00:11:11.356293 1 conditions.go:203] Setting lastTransitionTime for Certificate "cert-manager-webhook-oci-ca" condition "Ready" to 2023-01-01 00:11:11.356289279 +0000 UTC m=+0.561578508
I0101 00:11:11.398323 1 controller.go:162] cert-manager/certificates-readiness "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-ca\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-ca"
I0101 00:11:11.398362 1 conditions.go:203] Setting lastTransitionTime for Certificate "cert-manager-webhook-oci-ca" condition "Ready" to 2023-01-01 00:11:11.398359859 +0000 UTC m=+0.603649088
I0101 00:11:11.409821 1 controller.go:162] cert-manager/certificates-trigger "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-webhook-tls\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-webhook-tls"
I0101 00:11:11.409891 1 trigger_controller.go:200] cert-manager/certificates-trigger "msg"="Certificate must be re-issued" "key"="cert-manager/cert-manager-webhook-oci-webhook-tls" "message"="Issuing certificate as Secret does not exist" "reason"="DoesNotExist"
I0101 00:11:11.409914 1 conditions.go:203] Setting lastTransitionTime for Certificate "cert-manager-webhook-oci-webhook-tls" condition "Issuing" to 2023-01-01 00:11:11.409910518 +0000 UTC m=+0.615199747
I0101 00:11:11.624834 1 controller.go:162] cert-manager/certificates-key-manager "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-webhook-tls\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-webhook-tls"
I0101 00:11:11.626919 1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "cert-manager-webhook-oci-webhook-tls-k2hb7" condition "Approved" to 2023-01-01 00:11:11.626913415 +0000 UTC m=+0.832202674
I0101 00:11:11.643513 1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "cert-manager-webhook-oci-webhook-tls-k2hb7" condition "Ready" to 2023-01-01 00:11:11.643497074 +0000 UTC m=+0.848786303
I0101 00:11:11.673313 1 controller.go:162] cert-manager/certificates-key-manager "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-ca\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-ca"
I0101 00:11:11.704790 1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "cert-manager-webhook-oci-ca-5gppj" condition "Approved" to 2023-01-01 00:11:11.704785486 +0000 UTC m=+0.910074706
I0101 00:11:11.725774 1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "cert-manager-webhook-oci-ca-5gppj" condition "Ready" to 2023-01-01 00:11:11.725766894 +0000 UTC m=+0.931056123
I0101 00:11:11.764760 1 conditions.go:192] Found status change for Certificate "cert-manager-webhook-oci-ca" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-01-01 00:11:11.764754422 +0000 UTC m=+0.970043651
I0101 00:11:11.786503 1 controller.go:162] cert-manager/certificates-readiness "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-ca\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-ca"
I0101 00:11:11.786691 1 conditions.go:192] Found status change for Certificate "cert-manager-webhook-oci-ca" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-01-01 00:11:11.786687609 +0000 UTC m=+0.991976839
I0101 00:11:11.811444 1 controller.go:162] cert-manager/certificates-key-manager "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-ca\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-ca"
I0101 00:11:13.519998 1 conditions.go:203] Setting lastTransitionTime for Certificate "tls" condition "Ready" to 2023-01-01 00:11:13.519985801 +0000 UTC m=+2.725275029
I0101 00:11:13.520077 1 trigger_controller.go:200] cert-manager/certificates-trigger "msg"="Certificate must be re-issued" "key"="contour-tls/tls" "message"="Issuing certificate as Secret does not exist" "reason"="DoesNotExist"
I0101 00:11:13.520110 1 conditions.go:203] Setting lastTransitionTime for Certificate "tls" condition "Issuing" to 2023-01-01 00:11:13.520103744 +0000 UTC m=+2.725392972
I0101 00:11:13.545076 1 controller.go:162] cert-manager/certificates-readiness "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"tls\": the object has been modified; please apply your changes to the latest version and try again" "key"="contour-tls/tls"
I0101 00:11:13.545128 1 conditions.go:203] Setting lastTransitionTime for Certificate "tls" condition "Ready" to 2023-01-01 00:11:13.54512279 +0000 UTC m=+2.750412029
I0101 00:11:13.690221 1 setup.go:111] cert-manager/clusterissuers "msg"="generating acme account private key" "related_resource_kind"="Secret" "related_resource_name"="letsencrypt-prod" "related_resource_namespace"="cert-manager" "resource_kind"="ClusterIssuer" "resource_name"="letsencrypt-prod" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:13.813726 1 controller.go:162] cert-manager/certificates-key-manager "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"tls\": the object has been modified; please apply your changes to the latest version and try again" "key"="contour-tls/tls"
I0101 00:11:13.839489 1 setup.go:219] cert-manager/clusterissuers "msg"="ACME server URL host and ACME private key registration host differ. Re-checking ACME account registration" "related_resource_kind"="Secret" "related_resource_name"="letsencrypt-prod" "related_resource_namespace"="cert-manager" "resource_kind"="ClusterIssuer" "resource_name"="letsencrypt-prod" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:13.847662 1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "tls-k84xg" condition "Approved" to 2023-01-01 00:11:13.847656384 +0000 UTC m=+3.052945613
I0101 00:11:13.862626 1 conditions.go:263] Setting lastTransitionTime for CertificateRequest "tls-k84xg" condition "Ready" to 2023-01-01 00:11:13.86260456 +0000 UTC m=+3.067893809
I0101 00:11:14.013598 1 setup.go:309] cert-manager/clusterissuers "msg"="verified existing registration with ACME server" "related_resource_kind"="Secret" "related_resource_name"="letsencrypt-prod" "related_resource_namespace"="cert-manager" "resource_kind"="ClusterIssuer" "resource_name"="letsencrypt-prod" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:14.013633 1 conditions.go:96] Setting lastTransitionTime for Issuer "letsencrypt-prod" condition "Ready" to 2023-01-01 00:11:14.013629164 +0000 UTC m=+3.218918392
I0101 00:11:14.027841 1 setup.go:202] cert-manager/clusterissuers "msg"="skipping re-verifying ACME account as cached registration details look sufficient" "related_resource_kind"="Secret" "related_resource_name"="letsencrypt-prod" "related_resource_namespace"="cert-manager" "resource_kind"="ClusterIssuer" "resource_name"="letsencrypt-prod" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:15.875555 1 dns.go:88] cert-manager/challenges/Present "msg"="presenting DNS01 challenge for domain" "dnsName"="foo.cloudchief.me" "domain"="foo.cloudchief.me" "resource_kind"="Challenge" "resource_name"="tls-k84xg-1221875851-2183180315" "resource_namespace"="contour-tls" "resource_version"="v1" "type"="DNS-01"
E0101 00:11:15.968527 1 controller.go:167] cert-manager/challenges "msg"="re-queuing item due to error processing" "error"="the server could not find the requested resource (post oci.acme.d-n.be)" "key"="contour-tls/tls-k84xg-1221875851-2183180315"
I0101 00:11:15.968588 1 dns.go:88] cert-manager/challenges/Present "msg"="presenting DNS01 challenge for domain" "dnsName"="foo.cloudchief.me" "domain"="foo.cloudchief.me" "resource_kind"="Challenge" "resource_name"="tls-k84xg-1221875851-2183180315" "resource_namespace"="contour-tls" "resource_version"="v1" "type"="DNS-01"
E0101 00:11:15.972041 1 controller.go:167] cert-manager/challenges "msg"="re-queuing item due to error processing" "error"="the server could not find the requested resource (post oci.acme.d-n.be)" "key"="contour-tls/tls-k84xg-1221875851-2183180315"
I0101 00:11:16.131299 1 conditions.go:85] Found status change for Issuer "cert-manager-webhook-oci-ca" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-01-01 00:11:16.131292138 +0000 UTC m=+5.336581368
I0101 00:11:16.156517 1 conditions.go:252] Found status change for CertificateRequest "cert-manager-webhook-oci-webhook-tls-k2hb7" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-01-01 00:11:16.156508367 +0000 UTC m=+5.361797596
I0101 00:11:16.245324 1 conditions.go:192] Found status change for Certificate "cert-manager-webhook-oci-webhook-tls" condition "Ready": "False" -> "True"; setting lastTransitionTime to 2023-01-01 00:11:16.245314882 +0000 UTC m=+5.450604110
I0101 00:11:16.275722 1 controller.go:162] cert-manager/certificates-issuing "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-webhook-tls\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-webhook-tls"
I0101 00:11:16.296805 1 controller.go:162] cert-manager/certificates-issuing "msg"="re-queuing item due to optimistic locking on resource" "error"="Operation cannot be fulfilled on certificates.cert-manager.io \"cert-manager-webhook-oci-webhook-tls\": the object has been modified; please apply your changes to the latest version and try again" "key"="cert-manager/cert-manager-webhook-oci-webhook-tls"
I0101 00:11:18.839784 1 setup.go:202] cert-manager/clusterissuers "msg"="skipping re-verifying ACME account as cached registration details look sufficient" "related_resource_kind"="Secret" "related_resource_name"="letsencrypt-prod" "related_resource_namespace"="cert-manager" "resource_kind"="ClusterIssuer" "resource_name"="letsencrypt-prod" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:20.969313 1 dns.go:88] cert-manager/challenges/Present "msg"="presenting DNS01 challenge for domain" "dnsName"="foo.cloudchief.me" "domain"="foo.cloudchief.me" "resource_kind"="Challenge" "resource_name"="tls-k84xg-1221875851-2183180315" "resource_namespace"="contour-tls" "resource_version"="v1" "type"="DNS-01"
E0101 00:11:20.972987 1 controller.go:167] cert-manager/challenges "msg"="re-queuing item due to error processing" "error"="the server could not find the requested resource (post oci.acme.d-n.be)" "key"="contour-tls/tls-k84xg-1221875851-2183180315"
Log output from cainjector
❯ k logs -n cert-manager cert-manager-cainjector-7c44879bc4-h55fg
I0101 00:11:10.495932 1 start.go:126] "starting" version="v1.10.1" revision="a96bae172ddb1fcd4b57f1859ab9d1a9e94f7451"
I0101 00:11:11.997202 1 request.go:682] Waited for 1.048324503s due to client-side throttling, not priority and fairness, request: GET:https://10.2.0.1:443/apis/autoscaling/v2beta1?timeout=32s
I0101 00:11:12.699125 1 leaderelection.go:248] attempting to acquire leader lease kube-system/cert-manager-cainjector-leader-election...
I0101 00:11:12.725686 1 leaderelection.go:258] successfully acquired lease kube-system/cert-manager-cainjector-leader-election
I0101 00:11:12.725836 1 recorder.go:103] cert-manager/events "msg"="cert-manager-cainjector-7c44879bc4-h55fg_6c0d39d8-cfe7-46bb-9614-d1cdc1906d06 became leader" "object"={"kind":"Lease","namespace":"kube-system","name":"cert-manager-cainjector-leader-election","uid":"997c7fc2-89de-4fb9-a4c9-9afa8407397e","apiVersion":"coordination.k8s.io/v1","resourceVersion":"75411"} "reason"="LeaderElection" "type"="Normal"
I0101 00:11:12.927004 1 controller.go:185] cert-manager/certificate/customresourcedefinition "msg"="Starting EventSource" "source"="&{{%!s(*v1.CustomResourceDefinition=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} { { [] []} [] <nil> false} {[] { [] []} []}}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927047 1 controller.go:185] cert-manager/certificate/customresourcedefinition "msg"="Starting EventSource" "source"="&{{%!s(*v1.Certificate=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} {<nil> <nil> <nil> [] [] [] [] <nil> <nil> { } false [] <nil> <nil> <nil> []} {[] <nil> <nil> <nil> <nil> <nil> <nil> <nil>}}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927049 1 controller.go:185] cert-manager/certificate/mutatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.MutatingWebhookConfiguration=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} []}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927065 1 controller.go:185] cert-manager/certificate/customresourcedefinition "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927075 1 controller.go:193] cert-manager/certificate/customresourcedefinition "msg"="Starting Controller"
I0101 00:11:12.927079 1 controller.go:185] cert-manager/certificate/validatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.ValidatingWebhookConfiguration=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} []}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927080 1 controller.go:185] cert-manager/certificate/mutatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.Certificate=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} {<nil> <nil> <nil> [] [] [] [] <nil> <nil> { } false [] <nil> <nil> <nil> []} {[] <nil> <nil> <nil> <nil> <nil> <nil> <nil>}}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927153 1 controller.go:185] cert-manager/certificate/mutatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927108 1 controller.go:185] cert-manager/certificate/validatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.Certificate=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} {<nil> <nil> <nil> [] [] [] [] <nil> <nil> { } false [] <nil> <nil> <nil> []} {[] <nil> <nil> <nil> <nil> <nil> <nil> <nil>}}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927163 1 controller.go:185] cert-manager/certificate/apiservice "msg"="Starting EventSource" "source"="&{{%!s(*v1.APIService=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} {<nil> false [] 0 0} {[]}}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927165 1 controller.go:193] cert-manager/certificate/mutatingwebhookconfiguration "msg"="Starting Controller"
I0101 00:11:12.927186 1 controller.go:185] cert-manager/certificate/validatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927197 1 controller.go:193] cert-manager/certificate/validatingwebhookconfiguration "msg"="Starting Controller"
I0101 00:11:12.927198 1 controller.go:185] cert-manager/certificate/apiservice "msg"="Starting EventSource" "source"="&{{%!s(*v1.Certificate=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} {<nil> <nil> <nil> [] [] [] [] <nil> <nil> { } false [] <nil> <nil> <nil> []} {[] <nil> <nil> <nil> <nil> <nil> <nil> <nil>}}) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927353 1 controller.go:185] cert-manager/certificate/apiservice "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc0000b04c0}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927462 1 controller.go:193] cert-manager/certificate/apiservice "msg"="Starting Controller"
I0101 00:11:12.927897 1 controller.go:185] cert-manager/secret/customresourcedefinition "msg"="Starting EventSource" "source"="&{{%!s(*v1.CustomResourceDefinition=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} { { [] []} [] <nil> false} {[] { [] []} []}}) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927914 1 controller.go:185] cert-manager/secret/validatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.ValidatingWebhookConfiguration=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} []}) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927930 1 controller.go:185] cert-manager/secret/customresourcedefinition "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927943 1 controller.go:185] cert-manager/secret/validatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.927942 1 controller.go:193] cert-manager/secret/customresourcedefinition "msg"="Starting Controller"
I0101 00:11:12.927957 1 controller.go:193] cert-manager/secret/validatingwebhookconfiguration "msg"="Starting Controller"
I0101 00:11:12.928033 1 controller.go:185] cert-manager/secret/apiservice "msg"="Starting EventSource" "source"="&{{%!s(*v1.APIService=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} {<nil> false [] 0 0} {[]}}) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.928044 1 controller.go:185] cert-manager/secret/mutatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.MutatingWebhookConfiguration=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} []}) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.928068 1 controller.go:185] cert-manager/secret/mutatingwebhookconfiguration "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.928074 1 controller.go:185] cert-manager/secret/apiservice "msg"="Starting EventSource" "source"="&{{%!s(*v1.Secret=&{{ } { 0 {{0 0 <nil>}} <nil> <nil> map[] map[] [] [] []} <nil> map[] map[] }) %!s(*cache.informerCache=&{0xc000682780}) %!s(chan error=<nil>) %!s(func()=<nil>)}}"
I0101 00:11:12.928079 1 controller.go:193] cert-manager/secret/mutatingwebhookconfiguration "msg"="Starting Controller"
I0101 00:11:12.928097 1 controller.go:193] cert-manager/secret/apiservice "msg"="Starting Controller"
I0101 00:11:13.028592 1 controller.go:227] cert-manager/secret/mutatingwebhookconfiguration "msg"="Starting workers" "worker count"=1
I0101 00:11:13.028651 1 controller.go:227] cert-manager/secret/customresourcedefinition "msg"="Starting workers" "worker count"=1
I0101 00:11:13.028674 1 controller.go:227] cert-manager/certificate/apiservice "msg"="Starting workers" "worker count"=1
I0101 00:11:13.028707 1 controller.go:227] cert-manager/certificate/mutatingwebhookconfiguration "msg"="Starting workers" "worker count"=1
I0101 00:11:13.028709 1 controller.go:227] cert-manager/secret/validatingwebhookconfiguration "msg"="Starting workers" "worker count"=1
I0101 00:11:13.028790 1 controller.go:227] cert-manager/certificate/customresourcedefinition "msg"="Starting workers" "worker count"=1
I0101 00:11:13.028874 1 controller.go:227] cert-manager/certificate/validatingwebhookconfiguration "msg"="Starting workers" "worker count"=1
I0101 00:11:13.028800 1 controller.go:227] cert-manager/secret/apiservice "msg"="Starting workers" "worker count"=1
E0101 00:11:13.029114 1 sources.go:124] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="unable to fetch associated secret" "error"="Secret \"cert-manager-webhook-oci-webhook-tls\" not found" "certificate"={"Namespace":"cert-manager","Name":"cert-manager-webhook-oci-webhook-tls"} "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1" "secret"={"Namespace":"cert-manager","Name":"cert-manager-webhook-oci-webhook-tls"}
I0101 00:11:13.029140 1 controller.go:166] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="could not find any ca data in data source for target" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:16.258367 1 controller.go:178] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="updated object" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:16.265670 1 controller.go:178] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="updated object" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:16.275748 1 controller.go:178] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="updated object" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:16.338515 1 controller.go:178] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="updated object" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:16.373836 1 controller.go:178] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="updated object" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:22.771836 1 controller.go:178] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="updated object" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1"
I0101 00:11:22.818713 1 controller.go:178] cert-manager/certificate/apiservice/generic-inject-reconciler "msg"="updated object" "resource_kind"="APIService" "resource_name"="v1.acme.d-n.be" "resource_namespace"="" "resource_version"="v1
This line in controller log output is telling...
E0101 00:11:20.972987 1 controller.go:167] cert-manager/challenges "msg"="re-queuing item due to error processing" "error"="the server could not find the requested resource (post oci.acme.d-n.be)" "key"="contour-tls/tls-k84xg-1221875851-2183180315"
I need to build and host our own copy of Docker image with updated webhook implementation.
Why? Because we're employing a fork and the fork's (main.go) impl of webhook is what we need to use.
So what do we need to do?
Execute a prepare-cert-manager-oci.sh
script before invoking install-cert-manager-oci.sh
. The purpose of the prepare script will be to build, tag, and push the Docker image to a publicly-accessible container image repository (e.g., docker.io, gcr.io, registry.gitlab.com).
I could:
- fork the fork, and then host the image on registry.gitlab.com/<my_account>
- clone the fork, build,tag, push locally to a container image repo of my choice
then
Update the helm install
of the cert-manager-webhook-oci
webhook by adding --set image.repository=<fqn_to_new_image>
.
So having updated helm install
to use the new webhook image, still seeing integration issues.
From webhook-oci
pod log output...
E0102 17:37:48.629879 1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.23.4/tools/cache/reflector. │
│ go:167: Failed to watch *v1beta2.FlowSchema: failed to list *v1beta2.FlowSchema: flowschemas.flowcontro │
│ l.apiserver.k8s.io is forbidden: User "system:serviceaccount:cert-manager:cert-manager-webhook-oci" can │
│ not list resource "flowschemas" in API group "flowcontrol.apiserver.k8s.io" at the cluster scope │
│ W0102 17:38:07.657555 1 reflector.go:324] pkg/mod/k8s.io/client-go@v0.23.4/tools/cache/reflector. │
│ go:167: failed to list *v1beta2.PriorityLevelConfiguration: prioritylevelconfigurations.flowcontrol.api │
│ server.k8s.io is forbidden: User "system:serviceaccount:cert-manager:cert-manager-webhook-oci" cannot l │
│ ist resource "prioritylevelconfigurations" in API group "flowcontrol.apiserver.k8s.io" at the cluster s │
│ cope │
│ E0102 17:38:07.657586 1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.23.4/tools/cache/reflector. │
│ go:167: Failed to watch *v1beta2.PriorityLevelConfiguration: failed to list *v1beta2.PriorityLevelConfi │
│ guration: prioritylevelconfigurations.flowcontrol.apiserver.k8s.io is forbidden: User "system:serviceac │
│ count:cert-manager:cert-manager-webhook-oci" cannot list resource "prioritylevelconfigurations" in API │
│ group "flowcontrol.apiserver.k8s.io" at the cluster scope
Need to look into adjusting RBAC.
Added a file named rbac-ext.yaml
to deploy/cert-manager-oci/templates
directory with this content to address User "system:serviceaccount:cert-manager:cert-manager-webhook-oci" cannot list resource
issue
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ include "cert-manager-webhook-oci.fullname" . }}:flowcontrol-solver
labels:
app: {{ include "cert-manager-webhook-oci.name" . }}
chart: {{ include "cert-manager-webhook-oci.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups:
- "flowcontrol.apiserver.k8s.io"
resources:
- 'prioritylevelconfigurations'
- 'flowschemas'
verbs:
- 'list'
- 'watch'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "cert-manager-webhook-oci.fullname" . }}:flowcontrol-solver
labels:
app: {{ include "cert-manager-webhook-oci.name" . }}
chart: {{ include "cert-manager-webhook-oci.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ include "cert-manager-webhook-oci.fullname" . }}:flowcontrol-solver
subjects:
- apiGroup: ""
kind: ServiceAccount
name: {{ include "cert-manager-webhook-oci.fullname" . }}
namespace: {{ .Release.Namespace | quote }}
But, still fighting this issue:
E0102 20:52:43.400420 1 controller.go:167] cert-manager/challenges "msg"="re-queuing item due to error processing" "error"="the server could not find the requested resource (post oci.acme.d-n.be)" "key"="contour-tls/tls-whlsj-1221875851-249893563"
Some details related to the failure
❯ k get apiservice -A
NAME SERVICE AVAILABLE AGE
v1. Local True 5h20m
v1.acme.cert-manager.io Local True 35m
v1.acme.d-n.be cert-manager/cert-manager-webhook-oci False (FailedDiscoveryCheck) 33m
v1.admissionregistration.k8s.io Local True 5h20m
v1.apiextensions.k8s.io Local True 5h20m
v1.apps Local True 5h20m
v1.authentication.k8s.io Local True 5h20m
v1.authorization.k8s.io Local True 5h20m
v1.autoscaling Local True 5h20m
v1.batch Local True 5h20m
v1.cert-manager.io Local True 35m
v1.certificates.k8s.io Local True 5h20m
v1.coordination.k8s.io Local True 5h20m
v1.discovery.k8s.io Local True 5h20m
v1.events.k8s.io Local True 5h20m
v1.networking.k8s.io Local True 5h20m
v1.node.k8s.io Local True 5h20m
v1.policy Local True 5h20m
v1.projectcontour.io Local True 5h9m
v1.rbac.authorization.k8s.io Local True 5h20m
v1.scheduling.k8s.io Local True 5h20m
v1.storage.k8s.io Local True 5h20m
v1alpha1.projectcontour.io Local True 5h9m
v1beta1.batch Local True 5h20m
v1beta1.discovery.k8s.io Local True 5h20m
v1beta1.events.k8s.io Local True 5h20m
v1beta1.flowcontrol.apiserver.k8s.io Local True 5h20m
v1beta1.node.k8s.io Local True 5h20m
v1beta1.policy Local True 5h20m
v1beta1.storage.k8s.io Local True 5h20m
v1beta2.flowcontrol.apiserver.k8s.io Local True 5h20m
v2.autoscaling Local True 5h20m
v2beta1.autoscaling Local True 5h20m
v2beta2.autoscaling Local True 5h20m
❯ k describe apiservice v1.acme.d-n.be
Name: v1.acme.d-n.be
Namespace:
Labels: app=cert-manager-webhook-oci
app.kubernetes.io/managed-by=Helm
chart=cert-manager-webhook-oci-1.0.0
heritage=Helm
release=cert-manager-webhook-oci
Annotations: cert-manager.io/inject-ca-from: cert-manager/cert-manager-webhook-oci-webhook-tls
meta.helm.sh/release-name: cert-manager-webhook-oci
meta.helm.sh/release-namespace: cert-manager
API Version: apiregistration.k8s.io/v1
Kind: APIService
Metadata:
Creation Timestamp: 2023-01-02T21:26:09Z
Managed Fields:
API Version: apiregistration.k8s.io/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.:
f:cert-manager.io/inject-ca-from:
f:meta.helm.sh/release-name:
f:meta.helm.sh/release-namespace:
f:labels:
.:
f:app:
f:app.kubernetes.io/managed-by:
f:chart:
f:heritage:
f:release:
f:spec:
f:group:
f:groupPriorityMinimum:
f:service:
.:
f:name:
f:namespace:
f:port:
f:version:
f:versionPriority:
Manager: helm
Operation: Update
Time: 2023-01-02T21:26:09Z
API Version: apiregistration.k8s.io/v1
Fields Type: FieldsV1
fieldsV1:
f:status:
f:conditions:
.:
k:{"type":"Available"}:
.:
f:lastTransitionTime:
f:message:
f:reason:
f:status:
f:type:
Manager: kube-apiserver
Operation: Update
Subresource: status
Time: 2023-01-02T21:26:09Z
API Version: apiregistration.k8s.io/v1
Fields Type: FieldsV1
fieldsV1:
f:spec:
f:caBundle:
Manager: cainjector
Operation: Update
Time: 2023-01-02T21:26:15Z
Resource Version: 77529
UID: 73003593-03e0-435d-aed6-1f9b28952b1b
Spec:
Ca Bundle: <REDACTED>
Group: acme.d-n.be
Group Priority Minimum: 1000
Service:
Name: cert-manager-webhook-oci
Namespace: cert-manager
Port: 443
Version: v1
Version Priority: 15
Status:
Conditions:
Last Transition Time: 2023-01-02T21:26:09Z
Message: failing or missing response from https://10.2.3.52:443/apis/acme.d-n.be/v1: bad status from https://10.2.3.52:443/apis/acme.d-n.be/v1: 404
Reason: FailedDiscoveryCheck
Status: False
Type: Available
Events: <none>
❯ k get endpoints -A
NAMESPACE NAME ENDPOINTS AGE
cert-manager cert-manager 10.1.0.134:9402 31m
cert-manager cert-manager-webhook 10.1.1.10:10250 31m
cert-manager cert-manager-webhook-oci 10.1.1.11:443 30m
default kubernetes 192.168.3.71:6443,192.168.3.71:12250 5h16m
default oracle.com-oci <none> 5h16m
kube-system kube-dns 10.1.0.130:53,10.1.0.2:53,10.1.1.2:53 + 6 more... 5h16m
projectcontour contour 10.1.0.131:8001,10.1.0.4:8001 5h5m
projectcontour envoy 10.1.0.132:8443,10.1.0.5:8443,10.1.1.4:8443 + 3 more... 5h5m
❯ k describe endpoints oracle.com-oci
Name: oracle.com-oci
Namespace: default
Labels: <none>
Annotations: control-plane.alpha.kubernetes.io/leader:
{"holderIdentity":"control-plane-host-10-64-226-92_79ec18ad-2e5e-4783-a0dc-15c7e1c4be5e","leaseDurationSeconds":15,"acquireTime":"2023-01-...
Subsets:
Events: <none>
Update 2023-01-02: I have tested the above on a 3-node Oracle Cloud OKE cluster hosting K8s v1.24.1 and on a single-node kind
cluster hosting K8s 1.25.3. Both installations fail in the exact same fashion. Need to determine why the API Service is not functioning.
This is a simple test, but more troubleshooting necessary...
Open two terminals shells.
In terminal 1:
Execute ❯ kubectl -n cert-manager port-forward deploy/cert-manager-webhook 10250
In terminal 2:
Execute
❯ curl -vsS --resolve cert-manager-webhook.cert-manager.svc:10250:127.0.0.1 \
--service-name cert-manager-webhook-ca \
--cacert <(kubectl get validatingwebhookconfigurations cert-manager-webhook -ojson | jq '.webhooks[].clientConfig.caBundle' -r | base64 -d) \
https://cert-manager-webhook.cert-manager.svc:10250/validate 2>&1 -d@- <<'EOF' | sed '/^* /d; /bytes data]$/d; s/> //; s/< //'
{"kind":"AdmissionReview","apiVersion":"admission.k8s.io/v1","request":{"requestKind":{"group":"cert-manager.io","version":"v1","kind":"Certificate"},"requestResource":{"group":"cert-manager.io","version":"v1","resource":"certificates"},"name":"foo","namespace":"default","operation":"CREATE","object":{"apiVersion":"cert-manager.io/v1","kind":"Certificate","spec":{"dnsNames":["foo"],"issuerRef":{"group":"cert-manager.io","kind":"Issuer","name":"letsencrypt"},"secretName":"foo","usages":["digital signature"]}}}}
EOF
and get a successful response
Update 2023-01-03: I can fix the API service not being available by reverting changes made to the deploy/cert-manager-webhook-oci/templates/apiservice.yaml
.
This is what I changed it back to
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
name: v1alpha1.{{ .Values.groupName }}
labels:
app: {{ include "cert-manager-webhook-oci.name" . }}
chart: {{ include "cert-manager-webhook-oci.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/{{ include "cert-manager-webhook-oci.servingCertificate" . }}"
spec:
group: {{ .Values.groupName }}
groupPriorityMinimum: 1000
versionPriority: 15
service:
name: {{ include "cert-manager-webhook-oci.fullname" . }}
namespace: {{ .Release.Namespace }}
version: v1alpha1
And that gives us...
❯ k get apiservice v1alpha1.acme.d-n.be
NAME SERVICE AVAILABLE AGE
v1alpha1.acme.d-n.be cert-manager/cert-manager-webhook-oci True 8m24s
But we still don't have a crd we're expecting. And so we still have an unresolved challenge and no valid cert.
Going to take a crack at updating go.mod
and go.sum
with more recent versions of libraries w/
go get -u
go mod tidy
UPDATE #2 2023-01-03:
Updated library dependencies, rebuilt, tagged, and pushed image, then helm uninstalled and helm installed the webhook. Success!
❯ k get cert -A
NAMESPACE NAME READY SECRET AGE
cert-manager cert-manager-webhook-oci-ca True cert-manager-webhook-oci-ca 71s
cert-manager cert-manager-webhook-oci-webhook-tls True cert-manager-webhook-oci-webhook-tls 71s
contour-tls tls True tls 102
Next up: create a Github repo with all changes required.
UPDATE #3 2023-01-03:
Here's the Github repo: https://github.com/pacphi/cert-manager-webhook-oci.
Update 2022-12-30: The above impl is still not working. Webhook fails to create valid certificate. Need to do more research and troubleshoot why this is not working. Attempted install on Oracle OKE cluster with K8s v1.24.1 installed.