Skip to content

Instantly share code, notes, and snippets.

@eumel8

eumel8/00_vcluster_crossplane.md

Last active March 14, 2024 21:52
Show Gist options
  • Save eumel8/c08a17fd259c98f6de832bdcdf87a263 to your computer and use it in GitHub Desktop.
Save eumel8/c08a17fd259c98f6de832bdcdf87a263 to your computer and use it in GitHub Desktop.
Download ZIP
vcluster crossplane
Raw
00_vcluster_crossplane.md

Install kubectl crossplane plugin

curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh
sudo mv kubectl-crossplane /usr/local/bin
kubectl crossplane --help

Install Crossplane

helm install crossplane --namespace crossplane-system crossplane --repo https://charts.crossplane.io/stable --create-namespace

Install Crossplane Helm Provider

kubectl crossplane install provider crossplane/provider-helm:v0.10.0
Raw
01_composition.yaml
apiVersion: apiextensions.crossplane.io/v1
kind: Composition
metadata:
name: vcluster.caas.telekom.de
spec:
writeConnectionSecretsToNamespace: crossplane-system
compositeTypeRef:
apiVersion: caas.telekom.de/v1alpha1
kind: Vcluster
resources:
- name: vcluster-helm-release
base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
metadata:
annotations:
crossplane.io/external-name: # patched
spec:
rollbackLimit: 3
forProvider:
namespace: # patched
chart:
name: vcluster
repository: https://charts.loft.sh
version: "0.13.0"
values:
syncer:
extraArgs: [] # patched
# - --out-kube-config-server=https://cluster-1.cluster-1.svc
providerConfigRef:
name: default
patches:
- fromFieldPath: metadata.name
toFieldPath: spec.forProvider.namespace
policy:
fromFieldPath: Required
- fromFieldPath: metadata.name
toFieldPath: metadata.annotations[crossplane.io/external-name]
policy:
fromFieldPath: Required
transforms:
- type: string
string:
fmt: "%s-vcluster"
- name: vcluster-rancher-register
base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
metadata:
annotations:
crossplane.io/external-name: # patched
spec:
rollbackLimit: 3
forProvider:
namespace: # patched
chart:
name: rancher-cluster
repository: https://mcsps.github.io/helm-charts
version: "0.0.12"
values:
rancher:
url: https://k3s3.otc.mcsps.de
apitoken: token-7hbj7:frlfjgh85fsj949zztqsjs6f2jcnscbxhd746hz4dfkg7npkwqfw4w
clustername: # patched
providerConfigRef:
name: default
patches:
- fromFieldPath: metadata.name
toFieldPath: spec.forProvider.values.clustername
policy:
fromFieldPath: Required
- fromFieldPath: metadata.name
toFieldPath: spec.forProvider.namespace
policy:
fromFieldPath: Required
- fromFieldPath: metadata.name
toFieldPath: metadata.annotations[crossplane.io/external-name]
policy:
fromFieldPath: Required
transforms:
- type: string
string:
fmt: "%s-register-rancher"
Raw
02_compositeresourcedefinition.yaml
apiVersion: apiextensions.crossplane.io/v1
kind: CompositeResourceDefinition
metadata:
name: vclusters.caas.telekom.de
spec:
group: caas.telekom.de
names:
kind: Vcluster
plural: vclusters
versions:
- name: v1alpha1
served: true
referenceable: true
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties: {}
Raw
03_vcluster.yaml
apiVersion: caas.telekom.de/v1alpha1
kind: Vcluster
metadata:
name: kunde1
spec: {}
Raw
05_register_rancher.md 
export APITOKEN=token-7hb...
export CLUSTERNAME=kunde1
export RANCHER=https://k3s.otc.mcsps.de
CLUSTERRESPONSE=`curl -s "$RANCHER/v3/cluster" -H 'content-type: application/json' -H "Authorization: Bearer $APITOKEN" --data-binary '{"type":"cluster","name":$CLUSTERNAME,"import":true}'`
CLUSTERID=`echo $CLUSTERRESPONSE | jq -r .id`
ID=`curl -s "$RANCHER/v3/clusters/${CLUSTERID}/clusterregistrationtoken" -H 'content-type: application/json' -H "Authorization: Bearer $APITOKEN" --data-binary '{"type":"clusterRegistrationToken","clusterId":"'$CLUSTERID'"}'  |jq -r .id`
AGENTCOMMAND=`curl -s "$RANCHER/v3/clusters/${CLUSTERID}/clusterregistrationtoken/$ID" -H 'content-type: application/json' -H "Authorization: Bearer $APITOKEN" | jq -r .insecureCommand`
echo "${AGENTCOMMAND}"
Raw
06_composition_ha.yaml
apiVersion: apiextensions.crossplane.io/v1
kind: Composition
metadata:
name: vcluster.caas.telekom.de
spec:
compositeTypeRef:
apiVersion: caas.telekom.de/v1alpha1
kind: Vcluster
publishConnectionDetailsWithStoreConfigRef:
name: default
resources:
- base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
metadata: {}
spec:
forProvider:
chart:
name: vcluster
pullSecretRef:
name: mtrchartrepo
namespace: crossplane-system
repository: oci://mtr.devops.telekom.de/caas/charts
url: oci://mtr.devops.telekom.de/caas/charts/vcluster:0.15.2
version: 0.15.2
values:
coredns:
image: mtr.devops.telekom.de/rancher/mirrored-coredns-coredns:1.10.1
defaultImageRegistry: mtr.devops.telekom.de
enableHA: false
replicas: 3
storage:
classname: local-path
persistence: false
syncer:
image: /caas/vcluster:0.15.2
livenessProbe:
enabled: false
readinessProbe:
enabled: false
vcluster:
env:
- name: K3S_DATASTORE_ENDPOINT
extraArgs:
- --system-default-registry=mtr.devops.telekom.de
- --token=a12345678
- --debug
- -v 1
k3s:
workloadKind: StatefulSet
resources:
limits:
cpu: 200m
memory: 1Gi
requests:
cpu: 50m
memory: 16Mi
storage:
persistence: false
providerConfigRef:
name: crossplanecontrib-provider-helm
rollbackLimit: 3
name: vcluster-master-helm-release
patches:
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: spec.forProvider.namespace
type: FromCompositeFieldPath
- combine:
strategy: string
string:
fmt: http://%s-etcd-0.%s-etcd-headless.%s.svc.cluster.local:2379,http://%s-etcd-1.%s-etcd-headless.%s.svc.cluster.local:2379,http://%s-etcd-2.%s-etcd-headless.%s.svc.cluster.local:2379
variables:
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: spec.forProvider.values.vcluster.env[0].value
type: CombineFromComposite
- fromFieldPath: spec.k8sversion
policy:
fromFieldPath: Required
toFieldPath: spec.forProvider.values.vcluster.image
transforms:
- string:
fmt: /rancher/k3s:%s
type: Format
type: string
type: FromCompositeFieldPath
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: metadata.annotations[crossplane.io/external-name]
transforms:
- string:
fmt: '%s-vcluster'
type: Format
type: string
type: FromCompositeFieldPath
readinessChecks:
- matchCondition:
status: "True"
type: Ready
type: MatchCondition
- base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
metadata: {}
spec:
forProvider:
chart:
name: etcd
pullSecretRef:
name: mtrchartrepo
namespace: crossplane-system
repository: oci://mtr.devops.telekom.de/caas/charts
url: oci://mtr.devops.telekom.de/caas/charts/etcd:9.5.0
version: 9.5.0
values:
auth:
client:
enableAuthentication: false
secureTransport: false
useAutoTLS: true
rbac:
allowNoneAuthentication: true
create: false
rootPassword: a12345678
token:
enabled: false
image:
debug: true
registry: mtr.devops.telekom.de
repository: caas/etcd
tag: 3.5.9-debian-11-r108
loglevel: debug
replicaCount: 3
resources:
limits:
cpu: 200m
memory: 1Gi
requests:
cpu: 50m
memory: 16Mi
providerConfigRef:
name: crossplanecontrib-provider-helm
rollbackLimit: 3
name: etcd-helm-release
patches:
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: spec.forProvider.namespace
type: FromCompositeFieldPath
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: metadata.annotations[crossplane.io/external-name]
transforms:
- string:
fmt: '%s-etcd'
type: Format
type: string
type: FromCompositeFieldPath
readinessChecks:
- matchCondition:
status: "True"
type: Ready
type: MatchCondition
- base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
metadata: {}
spec:
forProvider:
chart:
name: rancher-cluster
pullSecretRef:
name: mtrchartrepo
namespace: crossplane-system
repository: oci://mtr.devops.telekom.de/caas/charts
url: oci://mtr.devops.telekom.de/caas/charts/rancher-cluster:0.0.29
version: 0.0.29
values:
defaultImageRegistry: mtr.devops.telekom.de
image: mtr.devops.telekom.de/mcsps/utils:latest
rancher:
url: https://k3s.otc.mcsps.de
providerConfigRef:
name: crossplanecontrib-provider-helm
rollbackLimit: 3
name: vcluster-rancher-register
patches:
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: spec.forProvider.values.clustername
type: FromCompositeFieldPath
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: spec.forProvider.namespace
type: FromCompositeFieldPath
- fromFieldPath: metadata.name
policy:
fromFieldPath: Required
toFieldPath: metadata.annotations[crossplane.io/external-name]
transforms:
- string:
fmt: '%s-register-rancher'
type: Format
type: string
type: FromCompositeFieldPath
readinessChecks:
- matchCondition:
status: "True"
type: Ready
type: MatchCondition
writeConnectionSecretsToNamespace: crossplane-system
@eumel8
Copy link
Author

eumel8 commented Dec 14, 2022

The last step is done within a Helm chart including a batch job: https://github.com/mcsps/helm-charts/tree/master/charts/rancher-cluster

@eumel8
Copy link
Author

eumel8 commented Sep 24, 2023

some RBAC may required

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: provider-helm
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: crossplanecontrib-provider-helm-503c3591121b
  namespace: crossplane-system

@eumel8
Copy link
Author

eumel8 commented Sep 25, 2023

output ha installation with external etcd and rancher cluster-agent:

$ kubectl -n vc2 get pods
NAME                                                              READY   STATUS      RESTARTS      AGE
vc2-register-rancher-kvrfq                                        0/1     Completed   0             43m
coredns-6cb5f66f64-vbjhc-x-kube-system-x-vc2-vcluster             1/1     Running     0             61m
cattle-cluster-agent-7fbcc94c47-jrn6s-x-cattle-syste-fde94bf3c6   1/1     Running     2 (43m ago)   43m
cattle-cluster-agent-7fbcc94c47-pgkgh-x-cattle-syste-0bdddfdb6e   1/1     Running     1 (42m ago)   42m
rancher-webhook-774444f6f9-7z7hs-x-cattle-system-x-vc2-vcluster   1/1     Running     0             42m
helm-operation-5xmzl-x-cattle-system-x-vc2-vcluster               0/2     Completed   0             42m
vc2-vcluster-2                                                    2/2     Running     0             35m
vc2-vcluster-1                                                    2/2     Running     0             35m
vc2-vcluster-0                                                    2/2     Running     0             35m
vc2-etcd-2                                                        1/1     Running     0             35m
vc2-etcd-1                                                        1/1     Running     0             35m
vc2-etcd-0                                                        1/1     Running     0             35m

@eumel8
Copy link
Author

eumel8 commented Mar 12, 2024 •
edited
Loading

composition-s3-backup.yaml

apiVersion: apiextensions.crossplane.io/v1
kind: Composition
metadata:
  annotations:
  name: vcluster.caas.telekom.de
spec:
  compositeTypeRef:
    apiVersion: caas.telekom.de/v1alpha1
    kind: Vcluster
  mode: Resources
  publishConnectionDetailsWithStoreConfigRef:
    name: default
  resources:
  - base:
      apiVersion: helm.crossplane.io/v1beta1
      kind: Release
      metadata: {}
      spec:
        forProvider:
          chart:
            name: s3-register
            repository: oci://mtr.devops.telekom.de/caas/charts
            url: oci://mtr.devops.telekom.de/caas/charts/s3-register:0.0.1
            version: 0.0.1
            pullSecretRef:
              name: "mtr-pull-secret"
              namespace: "crossplane-system"
          values:
            defaultImageRegistry: mtr.devops.telekom.de
            image: mtr.devops.telekom.de/mcsps/utils:latest
            s3:
              adminpassword: xxxxxx
              adminuser: xxxxx
              url: http://minio.minio:9000
        providerConfigRef:
          name: crossplane-provider-helm
        rollbackLimit: 3
    name: s3-register
    patches:
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.values.clustername
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.namespace
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: metadata.annotations[crossplane.io/external-name]
      transforms:
      - string:
          fmt: '%s-s3-register'
          type: Format
        type: string
      type: FromCompositeFieldPath
    readinessChecks:
    - matchCondition:
        status: "True"
        type: Ready
      type: MatchCondition
  - base:
      apiVersion: helm.crossplane.io/v1beta1
      kind: Release
      metadata: {}
      spec:
        forProvider:
          chart:
            name: vcluster
            repository: oci://mtr.devops.telekom.de/caas/charts
            url: oci://mtr.devops.telekom.de/caas/charts/vcluster:v0.19.3
            version: v0.19.3
            pullSecretRef:
              name: "mtr-pull-secret"
              namespace: "crossplane-system"
          values:
            coredns:
              image: /rancher/mirrored-coredns-coredns:1.10.1
            defaultImageRegistry: mtr.devops.telekom.de
            sidecar:
            - env:
              - name: ENDPOINT
                value: minio.minio:9000
              - name: ACCESS_KEY
                value: # patchedvc1
              - name: BUCKET_NAME
                value: # patchedvc1
              - name: CLUSTERNAME
                value: # patchedvc1
              - name: ENC_KEY
                value: "12345"
              - name: TRACE
                value: ""
              - name: INSECURE
                value: "1"
              - name: BACKUP_INTERVAL
                value: 1
              - name: SECRET_KEY
                valueFrom:
                  secretKeyRef:
                    key: s3secretkey
              image: mtr.devops.telekom.de/caas/vcluster-backup:0.0.5
              imagePullPolicy: Always
              name: backup
              resources:
                limits:
                  cpu: "1"
                  memory: 512Mi
                requests:
                  cpu: 20m
                  memory: 64Mi
              securityContext:
                allowPrivilegeEscalation: false
                capabilities:
                  drop:
                  - all
                readOnlyRootFilesystem: true
                runAsGroup: 1000
                runAsNonRoot: true
                runAsUser: 1000
              volumeMounts:
              - mountPath: /tmp
                name: tmp
              - mountPath: /data
                name: data
            storage:
              className: sas
            sync:
              hoststorageclasses:
                enabled: true
              ingresses:
                enabled: true
            syncer:
              image: /caas/vcluster:0.19.3
            vcluster:
              extraArgs:
              - --system-default-registry=mtr.devops.telekom.de
              resources:
                limits:
                  cpu: 200m
                  memory: 2Gi
                requests:
                  cpu: 200m
                  memory: 256Mi
        providerConfigRef:
          name: crossplane-provider-helm
        rollbackLimit: 3
    name: vcluster-helm-release
    patches:
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.values.sidecar[0].env[8].valueFrom.secretKeyRef.name
      transforms:
      - string:
          fmt: '%s-s3-register'
          type: Format
        type: string
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.values.sidecar[0].env[3].value
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.values.sidecar[0].env[2].value
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.values.sidecar[0].env[1].value
      type: FromCompositeFieldPath
    - fromFieldPath: spec.k8sversion
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.values.vcluster.image
      transforms:
      - string:
          fmt: /rancher/k3s:%s
          type: Format
        type: string
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.namespace
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: metadata.annotations[crossplane.io/external-name]
      transforms:
      - string:
          fmt: '%s-vcluster'
          type: Format
        type: string
      type: FromCompositeFieldPath
    readinessChecks:
    - matchCondition:
        status: "True"
        type: Ready
      type: MatchCondition
  - base:
      apiVersion: helm.crossplane.io/v1beta1
      kind: Release
      metadata: {}
      spec:
        forProvider:
          chart:
            name: rancher-cluster
            repository: oci://mtr.devops.telekom.de/caas/charts
            url: oci://mtr.devops.telekom.de/caas/charts/rancher-cluster:0.0.40
            version: 0.0.40
            pullSecretRef:
              name: "mtr-pull-secret"
              namespace: "crossplane-system"
          values:
            defaultImageRegistry: mtr.devops.telekom.de
            image: mtr.devops.telekom.de/mcsps/utils:latest
            rancher:
              url: https://raseed-test.external.otc.telekomcloud.com
        providerConfigRef:
          name: crossplane-provider-helm
        rollbackLimit: 3
    name: vcluster-rancher-register
    patches:
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.values.clustername
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: spec.forProvider.namespace
      type: FromCompositeFieldPath
    - fromFieldPath: metadata.name
      policy:
        fromFieldPath: Required
      toFieldPath: metadata.annotations[crossplane.io/external-name]
      transforms:
      - string:
          fmt: '%s-register-rancher'
          type: Format
        type: string
      type: FromCompositeFieldPath
    readinessChecks:
    - matchCondition:
        status: "True"
        type: Ready
      type: MatchCondition
  writeConnectionSecretsToNamespace: crossplane-system

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment