-
-
Save pacphi/05e6bd49b312bb92b2db1d70beb5c69c to your computer and use it in GitHub Desktop.
#!/usr/bin/env bash | |
indent4() { sed 's/^/ /'; } | |
# Install the necessary resources to support cert-manager deployed on OKE | |
# vending valid certificate via a Let's Encrypt ClusterIssuer | |
# Set environemnt variables (these are sample values, please replace with your own) | |
export DOMAIN=foo.me | |
export EMAIL_ADDRESS=any@valid.email | |
export COMPARTMENT_OCID=ocid1.compartment.oc1..aaaaaaaa_ | |
export TENANCY_OCID=ocid1.tenancy.oc1..aaaaaaaa_ | |
export USER_OCID=ocid1.user.oc1..aaaaaaaa_ | |
export REGION=us-phoenix-1 | |
export FINGERPRINT=47:5f:c7:0d:a3:a5:ac:d6:53:41:d2:23:c6:c9:24:a2 | |
# Oracle Cloud credentials | |
export OCI_CONFIG_HOME=$HOME/.oci | |
export OCI_PEM_PRIVATE_KEY_FILE_PATH=$OCI_CONFIG_HOME/oci_api_key.pem | |
# Google Cloud credentials (where I've decided to host updated webhook container image) | |
export GOOGLE_PROJECT_ID=fe-cphillipson | |
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.ssh/terraform@${GOOGLE_PROJECT_ID}.iam.gserviceaccount.com.json | |
# Convert PEM private key to RSA | |
openssl rsa -in $OCI_PEM_PRIVATE_KEY_FILE_PATH -out $OCI_CONFIG_HOME/oci_api_rsa_key | |
export RSA_PRIVATE_KEY=$(cat $OCI_CONFIG_HOME/oci_api_rsa_key | indent4) | |
# Install Contour ingress | |
kubectl apply -f https://projectcontour.io/quickstart/contour.yaml | |
# Install cert-manager | |
helm repo add jetstack https://charts.jetstack.io | |
helm repo update | |
helm install \ | |
cert-manager jetstack/cert-manager \ | |
--namespace cert-manager \ | |
--create-namespace \ | |
--version v1.10.1 \ | |
--set installCRDs=true \ | |
--set prometheus.enabled=false \ | |
--set webhook.timeoutSeconds=30 | |
# Install cert-manager OCI webhook | |
# This is from a fork of https://gitlab.com/dn13/cert-manager-webhook-oci | |
# @see https://gitlab.com/jcotton/cert-manager-webhook-oci.git | |
git clone https://gitlab.com/jcotton/cert-manager-webhook-oci.git | |
cd cert-manager-webhook-oci | |
git checkout fix_and_update | |
helm install --namespace cert-manager cert-manager-webhook-oci ./deploy/cert-manager-webhook-oci \ | |
--set image.repository=us.gcr.io/fe-cphillipson/cert-manager-webhook-oci | |
# Create image pull secret | |
kubectl create secret docker-registry regcred \ | |
--docker-server=us.gcr.io \ | |
--docker-username=_json_key \ | |
--docker-password="$(cat $GOOGLE_APPLICATION_CREDENTIALS)" \ | |
--docker-email=${EMAIL_ADDRESS} \ | |
--namespace cert-manager | |
# Create namespace to store secret | |
kubectl create ns contour-tls | |
mkdir -p /tmp/oci | |
cd /tmp/oci | |
# Define ClusterIssuer | |
cat << EOF > cluster-issuer-oci.yml | |
apiVersion: cert-manager.io/v1 | |
kind: ClusterIssuer | |
metadata: | |
name: letsencrypt-prod | |
spec: | |
acme: | |
# The ACME server URL | |
server: https://acme-v02.api.letsencrypt.org/directory | |
# Email address used for ACME registration | |
email: $EMAIL_ADDRESS | |
# Name of a secret used to store the ACME account private key | |
privateKeySecretRef: | |
name: letsencrypt-prod | |
solvers: | |
- dns01: | |
webhook: | |
groupName: acme.d-n.be | |
solverName: oci | |
config: | |
ociProfileSecretName: oci-profile | |
compartmentOCID: $COMPARTMENT_OCID | |
EOF | |
# Define Secret with OCI credentials | |
cat << EOF > secret-oci.yml | |
apiVersion: v1 | |
kind: Secret | |
metadata: | |
name: oci-profile | |
namespace: cert-manager | |
type: Opaque | |
stringData: | |
tenancy: "$TENANCY_OCID" | |
user: "$USER_OCID" | |
region: "$REGION" | |
fingerprint: "$FINGERPRINT" | |
privateKey: | | |
$RSA_PRIVATE_KEY | |
privateKeyPassphrase: "" | |
EOF | |
# Define Certificate | |
cat << EOF > certificate-oci.yml | |
apiVersion: cert-manager.io/v1 | |
kind: Certificate | |
metadata: | |
name: tls | |
namespace: contour-tls | |
spec: | |
commonName: $DOMAIN | |
dnsNames: | |
- $DOMAIN | |
issuerRef: | |
kind: ClusterIssuer | |
name: letsencrypt-prod | |
secretName: tls | |
EOF | |
cd .. | |
# Let it rip! | |
kubectl apply -f oci/ |
#!/usr/bin/env bash | |
# Build, tag and push image to GCR | |
export IMAGE_PREFIX=pacphi | |
export GOOGLE_PROJECT_ID=fe-cphillipson | |
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.ssh/terraform@${GOOGLE_PROJECT_ID}.iam.gserviceaccount.com.json | |
## Authenticate to container registry | |
cat $GOOGLE_APPLICATION_CREDENTIALS | docker login -u _json_key --password-stdin https://us.gcr.io | |
## Clone | |
cd /tmp | |
git clone https://gitlab.com/jcotton/cert-manager-webhook-oci.git | |
cd cert-manager-webhook-oci | |
git checkout fix_and_update | |
## Build image | |
docker build -t ${IMAGE_PREFIX}/cert-manager-webhook-oci . | |
## Tag image | |
docker tag ${IMAGE_PREFIX}/cert-manager-webhook-oci us.gcr.io/${GOOGLE_PROJECT_ID}/cert-manager-webhook-oci:latest | |
## Push image | |
docker push us.gcr.io/${GOOGLE_PROJECT_ID}/cert-manager-webhook-oci:latest | |
## Cleanup | |
rm -Rf /tmp/cert-manager-webhook-oci |
#!/usr/bin/env bash | |
# Uninstall the resources supporting cert-manager deployed on OKE | |
# vending valid certificate via a Let's Encrypt ClusterIssuer | |
cd /tmp | |
kubectl delete -f oci/ | |
rm -Rf /tmp/oci | |
rm -Rf /tmp/cert-manager-webhook-oci | |
# Delete namespace used to store secret | |
kubectl delete secret --all -n contour-tls | |
kubectl delete ns contour-tls | |
# Uninstall cert-manager OCI webhook | |
helm uninstall --namespace cert-manager cert-manager-webhook-oci | |
#helm repo remove cert-manager-webhook-oci | |
# Uninstall cert-manager | |
helm uninstall --namespace cert-manager cert-manager | |
helm repo remove jetstack | |
kubectl delete secret --all -n cert-manager | |
kubectl delete namespace cert-manager | |
# Uninstall Contour ingress | |
kubectl delete -f https://projectcontour.io/quickstart/contour.yaml |
But, still fighting this issue:
E0102 20:52:43.400420 1 controller.go:167] cert-manager/challenges "msg"="re-queuing item due to error processing" "error"="the server could not find the requested resource (post oci.acme.d-n.be)" "key"="contour-tls/tls-whlsj-1221875851-249893563"
Some details related to the failure
❯ k get apiservice -A
NAME SERVICE AVAILABLE AGE
v1. Local True 5h20m
v1.acme.cert-manager.io Local True 35m
v1.acme.d-n.be cert-manager/cert-manager-webhook-oci False (FailedDiscoveryCheck) 33m
v1.admissionregistration.k8s.io Local True 5h20m
v1.apiextensions.k8s.io Local True 5h20m
v1.apps Local True 5h20m
v1.authentication.k8s.io Local True 5h20m
v1.authorization.k8s.io Local True 5h20m
v1.autoscaling Local True 5h20m
v1.batch Local True 5h20m
v1.cert-manager.io Local True 35m
v1.certificates.k8s.io Local True 5h20m
v1.coordination.k8s.io Local True 5h20m
v1.discovery.k8s.io Local True 5h20m
v1.events.k8s.io Local True 5h20m
v1.networking.k8s.io Local True 5h20m
v1.node.k8s.io Local True 5h20m
v1.policy Local True 5h20m
v1.projectcontour.io Local True 5h9m
v1.rbac.authorization.k8s.io Local True 5h20m
v1.scheduling.k8s.io Local True 5h20m
v1.storage.k8s.io Local True 5h20m
v1alpha1.projectcontour.io Local True 5h9m
v1beta1.batch Local True 5h20m
v1beta1.discovery.k8s.io Local True 5h20m
v1beta1.events.k8s.io Local True 5h20m
v1beta1.flowcontrol.apiserver.k8s.io Local True 5h20m
v1beta1.node.k8s.io Local True 5h20m
v1beta1.policy Local True 5h20m
v1beta1.storage.k8s.io Local True 5h20m
v1beta2.flowcontrol.apiserver.k8s.io Local True 5h20m
v2.autoscaling Local True 5h20m
v2beta1.autoscaling Local True 5h20m
v2beta2.autoscaling Local True 5h20m
❯ k describe apiservice v1.acme.d-n.be
Name: v1.acme.d-n.be
Namespace:
Labels: app=cert-manager-webhook-oci
app.kubernetes.io/managed-by=Helm
chart=cert-manager-webhook-oci-1.0.0
heritage=Helm
release=cert-manager-webhook-oci
Annotations: cert-manager.io/inject-ca-from: cert-manager/cert-manager-webhook-oci-webhook-tls
meta.helm.sh/release-name: cert-manager-webhook-oci
meta.helm.sh/release-namespace: cert-manager
API Version: apiregistration.k8s.io/v1
Kind: APIService
Metadata:
Creation Timestamp: 2023-01-02T21:26:09Z
Managed Fields:
API Version: apiregistration.k8s.io/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.:
f:cert-manager.io/inject-ca-from:
f:meta.helm.sh/release-name:
f:meta.helm.sh/release-namespace:
f:labels:
.:
f:app:
f:app.kubernetes.io/managed-by:
f:chart:
f:heritage:
f:release:
f:spec:
f:group:
f:groupPriorityMinimum:
f:service:
.:
f:name:
f:namespace:
f:port:
f:version:
f:versionPriority:
Manager: helm
Operation: Update
Time: 2023-01-02T21:26:09Z
API Version: apiregistration.k8s.io/v1
Fields Type: FieldsV1
fieldsV1:
f:status:
f:conditions:
.:
k:{"type":"Available"}:
.:
f:lastTransitionTime:
f:message:
f:reason:
f:status:
f:type:
Manager: kube-apiserver
Operation: Update
Subresource: status
Time: 2023-01-02T21:26:09Z
API Version: apiregistration.k8s.io/v1
Fields Type: FieldsV1
fieldsV1:
f:spec:
f:caBundle:
Manager: cainjector
Operation: Update
Time: 2023-01-02T21:26:15Z
Resource Version: 77529
UID: 73003593-03e0-435d-aed6-1f9b28952b1b
Spec:
Ca Bundle: <REDACTED>
Group: acme.d-n.be
Group Priority Minimum: 1000
Service:
Name: cert-manager-webhook-oci
Namespace: cert-manager
Port: 443
Version: v1
Version Priority: 15
Status:
Conditions:
Last Transition Time: 2023-01-02T21:26:09Z
Message: failing or missing response from https://10.2.3.52:443/apis/acme.d-n.be/v1: bad status from https://10.2.3.52:443/apis/acme.d-n.be/v1: 404
Reason: FailedDiscoveryCheck
Status: False
Type: Available
Events: <none>
❯ k get endpoints -A
NAMESPACE NAME ENDPOINTS AGE
cert-manager cert-manager 10.1.0.134:9402 31m
cert-manager cert-manager-webhook 10.1.1.10:10250 31m
cert-manager cert-manager-webhook-oci 10.1.1.11:443 30m
default kubernetes 192.168.3.71:6443,192.168.3.71:12250 5h16m
default oracle.com-oci <none> 5h16m
kube-system kube-dns 10.1.0.130:53,10.1.0.2:53,10.1.1.2:53 + 6 more... 5h16m
projectcontour contour 10.1.0.131:8001,10.1.0.4:8001 5h5m
projectcontour envoy 10.1.0.132:8443,10.1.0.5:8443,10.1.1.4:8443 + 3 more... 5h5m
❯ k describe endpoints oracle.com-oci
Name: oracle.com-oci
Namespace: default
Labels: <none>
Annotations: control-plane.alpha.kubernetes.io/leader:
{"holderIdentity":"control-plane-host-10-64-226-92_79ec18ad-2e5e-4783-a0dc-15c7e1c4be5e","leaseDurationSeconds":15,"acquireTime":"2023-01-...
Subsets:
Events: <none>
Update 2023-01-02: I have tested the above on a 3-node Oracle Cloud OKE cluster hosting K8s v1.24.1 and on a single-node kind
cluster hosting K8s 1.25.3. Both installations fail in the exact same fashion. Need to determine why the API Service is not functioning.
This is a simple test, but more troubleshooting necessary...
Open two terminals shells.
In terminal 1:
Execute ❯ kubectl -n cert-manager port-forward deploy/cert-manager-webhook 10250
In terminal 2:
Execute
❯ curl -vsS --resolve cert-manager-webhook.cert-manager.svc:10250:127.0.0.1 \
--service-name cert-manager-webhook-ca \
--cacert <(kubectl get validatingwebhookconfigurations cert-manager-webhook -ojson | jq '.webhooks[].clientConfig.caBundle' -r | base64 -d) \
https://cert-manager-webhook.cert-manager.svc:10250/validate 2>&1 -d@- <<'EOF' | sed '/^* /d; /bytes data]$/d; s/> //; s/< //'
{"kind":"AdmissionReview","apiVersion":"admission.k8s.io/v1","request":{"requestKind":{"group":"cert-manager.io","version":"v1","kind":"Certificate"},"requestResource":{"group":"cert-manager.io","version":"v1","resource":"certificates"},"name":"foo","namespace":"default","operation":"CREATE","object":{"apiVersion":"cert-manager.io/v1","kind":"Certificate","spec":{"dnsNames":["foo"],"issuerRef":{"group":"cert-manager.io","kind":"Issuer","name":"letsencrypt"},"secretName":"foo","usages":["digital signature"]}}}}
EOF
and get a successful response
Update 2023-01-03: I can fix the API service not being available by reverting changes made to the deploy/cert-manager-webhook-oci/templates/apiservice.yaml
.
This is what I changed it back to
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
name: v1alpha1.{{ .Values.groupName }}
labels:
app: {{ include "cert-manager-webhook-oci.name" . }}
chart: {{ include "cert-manager-webhook-oci.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/{{ include "cert-manager-webhook-oci.servingCertificate" . }}"
spec:
group: {{ .Values.groupName }}
groupPriorityMinimum: 1000
versionPriority: 15
service:
name: {{ include "cert-manager-webhook-oci.fullname" . }}
namespace: {{ .Release.Namespace }}
version: v1alpha1
And that gives us...
❯ k get apiservice v1alpha1.acme.d-n.be
NAME SERVICE AVAILABLE AGE
v1alpha1.acme.d-n.be cert-manager/cert-manager-webhook-oci True 8m24s
But we still don't have a crd we're expecting. And so we still have an unresolved challenge and no valid cert.
Going to take a crack at updating go.mod
and go.sum
with more recent versions of libraries w/
go get -u
go mod tidy
UPDATE #2 2023-01-03:
Updated library dependencies, rebuilt, tagged, and pushed image, then helm uninstalled and helm installed the webhook. Success!
❯ k get cert -A
NAMESPACE NAME READY SECRET AGE
cert-manager cert-manager-webhook-oci-ca True cert-manager-webhook-oci-ca 71s
cert-manager cert-manager-webhook-oci-webhook-tls True cert-manager-webhook-oci-webhook-tls 71s
contour-tls tls True tls 102
Next up: create a Github repo with all changes required.
UPDATE #3 2023-01-03:
Here's the Github repo: https://github.com/pacphi/cert-manager-webhook-oci.
Added a file named
rbac-ext.yaml
todeploy/cert-manager-oci/templates
directory with this content to addressUser "system:serviceaccount:cert-manager:cert-manager-webhook-oci" cannot list resource
issue