lxd provider cluster-api

cluster.yaml

apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  name: test
  namespace: default
spec:
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    kind: MicroK8sControlPlane
    name: test-control-plane
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
    kind: LXDCluster
    name: test
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LXDCluster
metadata:
  name: test
  namespace: default
spec: {}
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: MicroK8sControlPlane
metadata:
  name: test-control-plane
  namespace: default
spec:
  controlPlaneConfig:
    clusterConfiguration:
      portCompatibilityRemap: true
    initConfiguration:
      IPinIP: true
      addons:
      - dns
      - ingress
      joinTokenTTLInSecs: 9000
  machineTemplate:
    infrastructureTemplate:
      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
      kind: LXDMachineTemplate
      name: test-control-plane
  replicas: 1
  version: v1.25.0
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LXDMachineTemplate
metadata:
  name: test-control-plane
  namespace: default
spec:
  template:
    spec:
      imageAlias: u22
      instanceType: container
      profiles:
      - default
      - microk8s
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
  name: test-md-0
  namespace: default
spec:
  clusterName: test
  replicas: 0
  selector:
    matchLabels: null
  template:
    spec:
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
          kind: MicroK8sConfigTemplate
          name: test-md-0
      clusterName: test
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
        kind: LXDMachineTemplate
        name: test-md-0
      version: 1.25.0
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LXDMachineTemplate
metadata:
  name: test-md-0
  namespace: default
spec:
  template:
    spec:
      imageAlias: u22
      instanceType: container
      profiles:
      - default
      - microk8s
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: MicroK8sConfigTemplate
metadata:
  name: test-md-0
  namespace: default
spec:
  template:
    spec: {}

init

# configure LXD, and allow access over HTTPS.
# note the IP address (in this case, 10.0.3.181) and the trust password.
$ sudo lxd init
Would you like to use LXD clustering? (yes/no) [default=no]: yes
What IP address or DNS name should be used to reach this node? [default=10.0.3.181]: 
Are you joining an existing cluster? (yes/no) [default=no]: 
What name should be used to identify this node in the cluster? [default=test-ovn]: 
Setup password authentication on the cluster? (yes/no) [default=no]: yes
Trust password for new clients: 
Again: 
....

# create image alias for ubuntu 22.04 
$ sudo lxc launch ubuntu:22.04 t1
$ sudo lxc image alias create u22 ubuntu:22.04
$ sudo lxc rm t1 --force

# create profile
$ lxc profile create microk8s
$ curl https://raw.githubusercontent.com/ubuntu/microk8s/master/tests/lxc/microk8s.profile | lxc profile edit microk8s

# deploy LXD provider and configure access to server (replace IP and password)
$ microk8s kubectl apply -f provider.yaml
$ microk8s kubectl create configmap -n capl-system lxd-socket --from-literal=LXD_SERVER=https://10.0.3.181:8443 --from-literal=LXD_PASSWORD=password

# deploy cluster.yaml. initially, it has 1 control plane node and 0 workers
$ microk8s kubectl apply -f cluster.yaml

# wait for init node to come up, check with 'lxc list' and note its IP address
$ lxc list
+-------------------------------+---------+-----------------------------+------+-----------+-----------+
|             NAME              |  STATE  |            IPV4             | IPV6 |   TYPE    | SNAPSHOTS |
+-------------------------------+---------+-----------------------------+------+-----------+-----------+
| test-test-control-plane-pkstp | RUNNING | 10.0.0.187 (eth0)           |      | CONTAINER | 0         |
+-------------------------------+---------+-----------------------------+------+-----------+-----------+

# after deployment, fix `test-kubeconfig` secret to unblock the control plane provider
$ microk8s kubectl edit cluster test                # change control api endpoint to 10.0.0.187:6443
$ microk8s kubectl edit lxdcluster test             # change control api endpoint to 10.0.0.187:6443
$ clusterctl get kubeconfig test > kubeconfig
$ vim kubeconfig                                    # change 'https://TODO:12345' to 'https://10.0.0.187:6443' 
$ cat kubeconfig | base64 -w0
$ microk8s kubectl edit secret test-kubeconfig      # change value to the new base64 string

# cluster is now ready to scale. edit 'cluster.yaml', change 'replicas: 0' and re-apply to deploy worker nodes

provider.yaml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    control-plane: controller-manager
  name: capl-system
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdclusters.infrastructure.cluster.x-k8s.io
spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    categories:
    - cluster-api
    kind: LXDCluster
    listKind: LXDClusterList
    plural: lxdclusters
    shortNames:
    - lc
    singular: lxdcluster
  scope: Namespaced
  versions:
  - additionalPrinterColumns:
    - description: Cluster to which this LXDCluster belongs
      jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name
      name: Cluster
      type: string
    - description: Cluster infrastructure is ready for LXD instances
      jsonPath: .status.ready
      name: Ready
      type: string
    - description: Time duration since creation of LXDCluster
      jsonPath: .metadata.creationTimestamp
      name: Age
      type: date
    name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDCluster is the Schema for the lxdclusters API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDClusterSpec defines the desired state of LXDCluster
            properties:
              controlPlaneEndpoint:
                description: ControlPlaneEndpoint represents the endpoint to communicate with the control plane.
                properties:
                  host:
                    description: The hostname on which the API server is serving.
                    type: string
                  port:
                    description: The port on which the API server is serving.
                    format: int32
                    type: integer
                required:
                - host
                - port
                type: object
            type: object
          status:
            description: LXDClusterStatus defines the observed state of LXDCluster
            properties:
              ready:
                description: Ready denotes that the LXD cluster (infrastructure) is ready.
                type: boolean
            required:
            - ready
            type: object
        type: object
    served: true
    storage: true
    subresources:
      status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdclustertemplates.infrastructure.cluster.x-k8s.io
spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    kind: LXDClusterTemplate
    listKind: LXDClusterTemplateList
    plural: lxdclustertemplates
    singular: lxdclustertemplate
  scope: Namespaced
  versions:
  - name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDClusterTemplate is the Schema for the lxdclustertemplates API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDClusterTemplateSpec defines the desired state of LXDClusterTemplate
            properties:
              template:
                properties:
                  metadata:
                    description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
                    properties:
                      annotations:
                        additionalProperties:
                          type: string
                        description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
                        type: object
                      labels:
                        additionalProperties:
                          type: string
                        description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
                        type: object
                    type: object
                  spec:
                    description: Spec is the specification of the desired behaviour of the cluster.
                    properties:
                      controlPlaneEndpoint:
                        description: ControlPlaneEndpoint represents the endpoint to communicate with the control plane.
                        properties:
                          host:
                            description: The hostname on which the API server is serving.
                            type: string
                          port:
                            description: The port on which the API server is serving.
                            format: int32
                            type: integer
                        required:
                        - host
                        - port
                        type: object
                    type: object
                required:
                - spec
                type: object
            required:
            - template
            type: object
        type: object
    served: true
    storage: true
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdmachines.infrastructure.cluster.x-k8s.io
spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    categories:
    - cluster-api
    kind: LXDMachine
    listKind: LXDMachineList
    plural: lxdmachines
    shortNames:
    - lm
    singular: lxdmachine
  scope: Namespaced
  versions:
  - additionalPrinterColumns:
    - description: Cluster to which this LXDMachine belongs
      jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name
      name: Cluster
      type: string
    - description: LXD instance state
      jsonPath: .status.state
      name: State
      type: string
    - description: LXD instance ID
      jsonPath: .spec.providerID
      name: ProviderID
      type: string
    - description: Machine ready status
      jsonPath: .status.ready
      name: Ready
      type: string
    - description: Machine object which owns with this LXDMachine
      jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name
      name: Machine
      type: string
    - description: Time duration since creation of LXDMachine
      jsonPath: .metadata.creationTimestamp
      name: Age
      type: date
    name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDMachine is the Schema for the lxdmachines API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDMachineSpec defines the desired state of LXDMachine
            properties:
              imageAlias:
                description: Image is the image alias name to use.
                type: string
              instanceType:
                description: InstanceType is the instance type to create.
                enum:
                - container
                - virtual-machine
                type: string
              profiles:
                description: Profiles is a list of profiles to attach to the instance.
                items:
                  type: string
                type: array
              providerID:
                description: ProviderID is the container name in ProviderID format (lxd:///<containername>)
                type: string
            type: object
          status:
            description: LXDMachineStatus defines the observed state of LXDMachine
            properties:
              addresses:
                items:
                  description: NodeAddress contains information for the node's address.
                  properties:
                    address:
                      description: The node address.
                      type: string
                    type:
                      description: Node address type, one of Hostname, ExternalIP or InternalIP.
                      type: string
                  required:
                  - address
                  - type
                  type: object
                type: array
              ready:
                type: boolean
              state:
                type: string
            required:
            - addresses
            type: object
        type: object
    served: true
    storage: true
    subresources:
      status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdmachinetemplates.infrastructure.cluster.x-k8s.io
spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    kind: LXDMachineTemplate
    listKind: LXDMachineTemplateList
    plural: lxdmachinetemplates
    singular: lxdmachinetemplate
  scope: Namespaced
  versions:
  - name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDMachineTemplate is the Schema for the lxdmachinetemplates API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDMachineTemplateSpec defines the desired state of LXDMachineTemplate
            properties:
              template:
                properties:
                  metadata:
                    description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
                    properties:
                      annotations:
                        additionalProperties:
                          type: string
                        description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
                        type: object
                      labels:
                        additionalProperties:
                          type: string
                        description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
                        type: object
                    type: object
                  spec:
                    description: Spec is the specification of the desired behaviour of the machine.
                    properties:
                      imageAlias:
                        description: Image is the image alias name to use.
                        type: string
                      instanceType:
                        description: InstanceType is the instance type to create.
                        enum:
                        - container
                        - virtual-machine
                        type: string
                      profiles:
                        description: Profiles is a list of profiles to attach to the instance.
                        items:
                          type: string
                        type: array
                      providerID:
                        description: ProviderID is the container name in ProviderID format (lxd:///<containername>)
                        type: string
                    type: object
                required:
                - spec
                type: object
            required:
            - template
            type: object
        type: object
    served: true
    storage: true
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: capl-controller-manager
  namespace: capl-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: capl-leader-election-role
  namespace: capl-system
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - ""
  resources:
  - events
  verbs:
  - create
  - patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  creationTimestamp: null
  name: capl-manager-role
rules:
- apiGroups:
  - ""
  resources:
  - secrets
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - cluster.x-k8s.io
  resources:
  - clusters
  - clusters/status
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - cluster.x-k8s.io
  resources:
  - machines
  - machines/status
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdclusters
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdclusters/finalizers
  verbs:
  - update
- apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdclusters/status
  verbs:
  - get
  - patch
  - update
- apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdmachines
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdmachines/finalizers
  verbs:
  - update
- apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdmachines/status
  verbs:
  - get
  - patch
  - update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: capl-metrics-reader
rules:
- nonResourceURLs:
  - /metrics
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: capl-proxy-role
rules:
- apiGroups:
  - authentication.k8s.io
  resources:
  - tokenreviews
  verbs:
  - create
- apiGroups:
  - authorization.k8s.io
  resources:
  - subjectaccessreviews
  verbs:
  - create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: capl-leader-election-rolebinding
  namespace: capl-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: capl-leader-election-role
subjects:
- kind: ServiceAccount
  name: capl-controller-manager
  namespace: capl-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: capl-manager-rolebinding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: capl-manager-role
subjects:
- kind: ServiceAccount
  name: capl-controller-manager
  namespace: capl-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: capl-proxy-rolebinding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: capl-proxy-role
subjects:
- kind: ServiceAccount
  name: capl-controller-manager
  namespace: capl-system
---
apiVersion: v1
data:
  controller_manager_config.yaml: "apiVersion: controller-runtime.sigs.k8s.io/v1alpha1\nkind: ControllerManagerConfig\nhealth:\n  healthProbeBindAddress: :8081\nmetrics:\n  bindAddress: 127.0.0.1:8080\nwebhook:\n  port: 9443\nleaderElection:\n  leaderElect: true\n  resourceName: 349154e5.cluster.x-k8s.io\n#   leaderElectionReleaseOnCancel defines if the leader should step down volume \n#   when the Manager ends. This requires the binary to immediately end when the\n#   Manager is stopped, otherwise, this setting is unsafe. Setting this significantly\n#   speeds up voluntary leader transitions as the new leader don't have to wait\n#   LeaseDuration time first.\n#   In the default scaffold provided, the program ends immediately after \n#   the manager stops, so would be fine to enable this option. However, \n#   if you are doing or is intended to do any operation such as perform cleanups \n#   after the manager stops then its usage might be unsafe.\n#   leaderElectionReleaseOnCancel: true\n"
kind: ConfigMap
metadata:
  name: capl-manager-config
  namespace: capl-system
---
apiVersion: v1
kind: Service
metadata:
  labels:
    control-plane: controller-manager
  name: capl-controller-manager-metrics-service
  namespace: capl-system
spec:
  ports:
  - name: https
    port: 8443
    protocol: TCP
    targetPort: https
  selector:
    control-plane: controller-manager
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    control-plane: controller-manager
  name: capl-controller-manager
  namespace: capl-system
spec:
  replicas: 1
  selector:
    matchLabels:
      control-plane: controller-manager
  template:
    metadata:
      annotations:
        kubectl.kubernetes.io/default-container: manager
      labels:
        control-plane: controller-manager
    spec:
      containers:
      - args:
        - --secure-listen-address=0.0.0.0:8443
        - --upstream=http://127.0.0.1:8080/
        - --logtostderr=true
        - --v=0
        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0
        name: kube-rbac-proxy
        ports:
        - containerPort: 8443
          name: https
          protocol: TCP
        resources:
          limits:
            cpu: 500m
            memory: 128Mi
          requests:
            cpu: 5m
            memory: 64Mi
        securityContext:
          allowPrivilegeEscalation: false
      - args:
        - --health-probe-bind-address=:8081
        - --metrics-bind-address=127.0.0.1:8080
        - --leader-elect
        command:
        - /manager
        envFrom:
        - configMapRef:
            name: lxd-socket
        image: neoaggelos/capi-lxd:dev1
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8081
          initialDelaySeconds: 15
          periodSeconds: 20
        name: manager
        readinessProbe:
          httpGet:
            path: /readyz
            port: 8081
          initialDelaySeconds: 5
          periodSeconds: 10
        resources:
          limits:
            cpu: 500m
            memory: 128Mi
          requests:
            cpu: 10m
            memory: 64Mi
        securityContext:
          allowPrivilegeEscalation: false
      securityContext:
        runAsNonRoot: true
      serviceAccountName: capl-controller-manager
      terminationGracePeriodSeconds: 10

hi @wirwolf, this was hacked together for a weekend project, and contained lots of hard-coded things and assumptions. i have been thinking of returning to this at some point to clean this up and make it shareable

But I'm excited to see more interest in something like this