gistfile1.txt

root@ubuntu:~/sylva/sylva-core# ./apply-workload-cluster.sh environment-values/workload-clusters/okd-capm3
🔎 Validate input files

🔎 Validate sylva-units values for workload cluster
namespace/sylva-units-preview created
configmap/sylva-units-values created
secret/sylva-units-secrets created
helmrelease.helm.toolkit.fluxcd.io/sylva-units created
gitrepository.source.toolkit.fluxcd.io/sylva-core created
force reconciliation of helmrelease sylva-units
  helmrelease.helm.toolkit.fluxcd.io/sylva-units annotated
helmrelease.helm.toolkit.fluxcd.io/sylva-units condition met
Wait for Helm release to be ready
 ✓ GitRepository/sylva-core - Resource is ready
 ✓ HelmChart/sylva-units-preview-sylva-units - Resource is ready
⢎⡰ HelmRelease/sylva-units - Progressing - Running 'install' action with timeout of 5m0s
 ✓ All Done: resource HelmRelease/sylva-units-preview/sylva-units is ready!

🗑 Delete preview chart and namespace
helmrelease.helm.toolkit.fluxcd.io/sylva-units
gitrepository.source.toolkit.fluxcd.io/sylva-core
namespace "sylva-units-preview" deleted

📜 Install a sylva-units Helm release for workload cluster okd-capm3
namespace/okd-capm3 created
configmap/sylva-units-values created
secret/sylva-units-secrets created
helmrelease.helm.toolkit.fluxcd.io/sylva-units created
gitrepository.source.toolkit.fluxcd.io/sylva-core created

🎯 Trigger reconciliation of units
force reconciliation of helmrelease sylva-units
  helmrelease.helm.toolkit.fluxcd.io/sylva-units annotated
helmrelease.helm.toolkit.fluxcd.io/sylva-units condition met

⏳ Wait for units to be ready
 ✓ GitRepository/sylva-capi-cluster - Resource is ready
 ✓ GitRepository/sylva-core - Resource is ready
 ✓ HelmChart/okd-capm3-cluster - Resource is ready
 ✓ HelmChart/okd-capm3-cluster-bmh - Resource is ready
 ✓ HelmChart/okd-capm3-sylva-units - Resource is ready
 ✓ HelmRelease/cluster-bmh - Resource is ready
 ✓ HelmRelease/sylva-units - Resource is ready
 ✓ HelmRepository/unit-ingress-nginx - Resource is ready
 ✓ HelmRepository/unit-kyverno - Resource is ready
 ✓ Kustomization/cluster-bmh - Resource is ready
 ✓ Kustomization/mgmt-cluster-ready - Resource is ready
 ✗ HelmRelease/cluster - Stalled - RetriesExceeded: Failed to install after 3 attempt(s)
⢄⡱ Kustomization/cluster - Progressing - Reconciliation in progress
We have the following stalled resources:
IDENTIFIER                        STATUS     REASON          MESSAGE
Kustomization/okd-capm3/cluster   InProgress                 Kustomization generation is 1, but latest observed generation is -1
╰┄╴HelmRelease/okd-capm3/cluster  Failed                     Failed to install after 3 attempt(s)
   ├┄╴┬┄┄[Conditions]
   ┆  ├┄╴Stalled                  True       RetriesExceeded Failed to install after 3 attempt(s)
   ┆  ├┄╴Ready                    False      InstallFailed   Helm install failed for release okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+7f4ec533fca7: failed to create resource: admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
   ┆  ╰┄╴Released                 False      InstallFailed   Helm install failed for release okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+7f4ec533fca7: failed to create resource: admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
   ╰┄╴┬┄┄[Events]
      ├┄╴cluster.17d9ca38da3eb3e8            InstallFailed   Helm install failed for release okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+7f4ec533fca7: 1 error occurred:
      ┆                           * admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
      ┆
      ┆  Last Helm logs:
      ┆
      ┆  2024-06-17T12:24:37.921305896Z: creating 9 resource(s)
      ├┄╴cluster.17d9ca3ff961b06f                                InstallFailed Helm install failed for release okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+7f4ec533fca7: failed to create resource: admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
      ┆
      ┆  Last Helm logs:
      ┆
      ┆  2024-06-17T12:25:08.566373829Z: checking 9 resources for changes
      ┆  2024-06-17T12:25:08.580461782Z: Created a new ServiceAccount called "cluster-pre-delete-hook-sa" in okd-capm3
      ┆
      ┆  2024-06-17T12:25:08.609207999Z: Created a new Secret called "pull-secret" in okd-capm3
      ┆
      ┆  2024-06-17T12:25:08.612397539Z: Created a new Role called "cluster-pre-delete-hook-cr" in okd-capm3
      ┆
      ┆  2024-06-17T12:25:08.616001978Z: Created a new RoleBinding called "cluster-pre-delete-hook-crb" in okd-capm3
      ├┄╴cluster.17d9ca3d6037289e                                                                                     UninstallSucceeded Helm uninstall remediation for release okd-capm3/cluster.v1 with chart sylva-capi-cluster@0.0.0+7f4ec533fca7 succeeded
      ├┄╴cluster.17d9ca446d2e9420                                                                                     InstallFailed      Helm install failed for release okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+7f4ec533fca7: failed to create resource: admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
      ┆
      ┆  Last Helm logs:
      ┆
      ┆  2024-06-17T12:25:27.668689802Z: checking 9 resources for changes
      ┆  2024-06-17T12:25:27.676614379Z: Created a new ServiceAccount called "cluster-pre-delete-hook-sa" in okd-capm3
      ┆
      ┆  2024-06-17T12:25:27.699544025Z: Created a new Secret called "pull-secret" in okd-capm3
      ┆
      ┆  2024-06-17T12:25:27.70488887Z: Created a new Role called "cluster-pre-delete-hook-cr" in okd-capm3
      ┆
      ┆  2024-06-17T12:25:27.710799051Z: Created a new RoleBinding called "cluster-pre-delete-hook-crb" in okd-capm3
      ╰┄╴cluster.17d9ca38882ec1db                                                                                     HelmChartCreated Created HelmChart/okd-capm3/okd-capm3-cluster with SourceRef 'GitRepository/okd-capm3/sylva-capi-cluster'

root@ubuntu:~/sylva/sylva-core# helm list -n okd-capm3
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/sylva/sylva-core/management-cluster-kubeconfig
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/sylva/sylva-core/management-cluster-kubeconfig
NAME       	NAMESPACE	REVISION	UPDATED                                	STATUS  	CHART                                	APP VERSION
cluster    	okd-capm3	1       	2024-06-17 12:25:26.917581468 +0000 UTC	failed  	sylva-capi-cluster-0.0.0+7f4ec533fca7	0.0.0      
cluster-bmh	okd-capm3	1       	2024-06-17 12:24:32.027128929 +0000 UTC	deployed	sylva-capi-cluster-0.0.0+7f4ec533fca7	0.0.0      
sylva-units	okd-capm3	1       	2024-06-17 12:24:22.735793875 +0000 UTC	deployed	sylva-units-0.0.0-git+40d8c0675a04.1 	0.0.0      

root@ubuntu:~/sylva/sylva-core# kubectl get helmrelease -n okd-capm3
NAME          AGE   READY   STATUS
cluster       28m   False   Helm install failed for release okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+7f4ec533fca7: failed to create resource: admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
cluster-bmh   28m   True    Helm install succeeded for release okd-capm3/cluster-bmh.v1 with chart sylva-capi-cluster@0.0.0+7f4ec533fca7
sylva-units   29m   True    Helm install succeeded for release okd-capm3/sylva-units.v1 with chart sylva-units@0.0.0-git+40d8c0675a04.1

######### re-try from flux and observed the same
root@ubuntu:~/sylva/sylva-core# flux reconcile helmrelease cluster  -n okd-capm3
► annotating HelmRelease cluster in okd-capm3 namespace
✔ HelmRelease annotated
◎ waiting for HelmRelease reconciliation
✗ context deadline exceeded
root@ubuntu:~/sylva/sylva-core# kubectl get helmrelease -n okd-capm3
NAME          AGE   READY   STATUS
cluster       35m   False   Helm install failed for release okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+7f4ec533fca7: failed to create resource: admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
cluster-bmh   35m   True    Helm install succeeded for release okd-capm3/cluster-bmh.v1 with chart sylva-capi-cluster@0.0.0+7f4ec533fca7
sylva-units   36m   True    Helm install succeeded for release okd-capm3/sylva-units.v1 with chart sylva-units@0.0.0-git+40d8c0675a04.1

##### directly use helm manifest to deploy and it does work
root@ubuntu:~/sylva/sylva-capi-cluster-jz# helm get manifest -n okd-capm3 cluster  | kubectl apply -f -
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/sylva/sylva-core/management-cluster-kubeconfig
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/sylva/sylva-core/management-cluster-kubeconfig
Warning: resource serviceaccounts/cluster-pre-delete-hook-sa is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
serviceaccount/cluster-pre-delete-hook-sa configured
Warning: resource secrets/pull-secret is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
secret/pull-secret configured
Warning: resource roles/cluster-pre-delete-hook-cr is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
role.rbac.authorization.k8s.io/cluster-pre-delete-hook-cr configured
Warning: resource rolebindings/cluster-pre-delete-hook-crb is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
rolebinding.rbac.authorization.k8s.io/cluster-pre-delete-hook-crb configured
agentcontrolplane.controlplane.cluster.x-k8s.io/okd-sno-control-plane created
cluster.cluster.x-k8s.io/okd-sno created
metal3cluster.infrastructure.cluster.x-k8s.io/okd-sno created
metal3datatemplate.infrastructure.cluster.x-k8s.io/okd-sno-cp-metadata-fabf5e7d29 created
metal3machinetemplate.infrastructure.cluster.x-k8s.io/okd-sno-cp-eac12a3e15 created

root@ubuntu:~/sylva/sylva-capi-cluster-jz# kk get cluster -n okd-capm3
NAME      CLUSTERCLASS   PHASE         AGE   VERSION
okd-sno                  Provisioned   66s   
root@ubuntu:~/sylva/sylva-capi-cluster-jz# kk get aci -n okd-capm3
NAME                    CLUSTER                 STATE
okd-sno-control-plane   okd-sno-control-plane   pending-for-input


### or if delete the helmrelease and try the workload again it works
root@ubuntu:~/sylva/sylva-capi-cluster-jz# kk get helmrelease -n my-okd-capm3
NAME          AGE     READY   STATUS
cluster       9m8s    False   Helm install failed for release my-okd-capm3/cluster with chart sylva-capi-cluster@0.0.0+f1a5ea790feb: failed to create resource: admission webhook "validate.kyverno.svc-fail" denied the request: resource agentcontrolplanes not found in group controlplane.cluster.x-k8s.io/v1alpha1
cluster-bmh   9m38s   True    Helm install succeeded for release my-okd-capm3/cluster-bmh.v1 with chart sylva-capi-cluster@0.0.0+f1a5ea790feb
sylva-units   9m59s   True    Helm install succeeded for release my-okd-capm3/sylva-units.v1 with chart sylva-units@0.0.0-git+06cdc4d8155f.1

root@ubuntu:~/sylva/sylva-capi-cluster-jz# kk delete helmrelease cluster -n my-okd-capm3
helmrelease.helm.toolkit.fluxcd.io "cluster" deleted

### deploy the workload again
root@ubuntu:~/sylva/sylva-capi-cluster-jz# kk get helmrelease -n my-okd-capm3
NAME          AGE   READY   STATUS
cluster       18s   True    Helm install succeeded for release my-okd-capm3/cluster.v1 with chart sylva-capi-cluster@0.0.0+f1a5ea790feb
cluster-bmh   11m   True    Helm install succeeded for release my-okd-capm3/cluster-bmh.v1 with chart sylva-capi-cluster@0.0.0+f1a5ea790feb
sylva-units   11m   True    Helm install succeeded for release my-okd-capm3/sylva-units.v1 with chart sylva-units@0.0.0-git+06cdc4d8155f.1