Talos + Kubevirt Bare Metal & Nested Tenant Cluster

README.md

Scratch nodes WIP

talosctl gen config talos-kubevirt https://talos-kubevirt.home.arpa:6443 --additional-sans 192.168.1.70,talos-kubevirt --install-disk /dev/vda --output-dir $(pwd)/talos
❯ cat deploy.sh
#kubectl --kubeconfig $HOME/.kube/poweredge delete -f ./kubevirt/
#sleep 12
#kubectl --kubeconfig $HOME/.kube/poweredge apply -f ./kubevirt/

#sleep 120

talosctl apply-config --insecure --nodes 192.168.1.71 -e 192.168.1.71 --file ./talos/cp01.yml --talosconfig ./talos/talosconfig
talosctl apply-config --insecure --nodes 192.168.1.72 -e 192.168.1.72 --file ./talos/cp02.yml --talosconfig ./talos/talosconfig
talosctl apply-config --insecure --nodes 192.168.1.73 -e 192.168.1.73 --file ./talos/cp03.yml --talosconfig ./talos/talosconfig

# sleep 60
# talosctl bootstrap --nodes 192.168.1.71 -e 192.168.1.71 --talosconfig ./talos/talosconfig

# sleep 6
# talosctl kubeconfig --force --context talos-kubevirt --force-context-name talos-kubevirt --merge --nodes 192.168.1.71 --endpoints 192.168.1.70 --talosconfig ./talos/talosconfig

# kubectl taint --all nodes node-role.kubernetes.io/master-

# helm install --create-namespace --namespace rook-ceph rook-ceph rook-release/rook-ceph
# helm install --create-namespace --namespace rook-ceph rook-ceph-cluster --set operatorNamespace=rook-ceph rook-release/rook-ceph-cluster --values rook-ceph/cluster.values.yaml

# talosctl dashboard --nodes 192.168.1.71,192.168.1.73,192.168.1.75 -e 192.168.1.70 --talosconfig ./talos/talosconfig

# export RELEASE=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt)
# kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/$RELEASE/kubevirt-operator.yaml
# kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/$RELEASE/kubevirt-cr.yaml
# kubectl -n kubevirt wait kv kubevirt --for condition=Available
# kubectl get po -n kubevirt

control-plane-n01.yaml

apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: talos-cp01
  labels:
    app: talos
spec:
  running: true
  template:
    spec:
      terminationGracePeriodSeconds: 0
      domain:
        clock:
          timer: {}
          utc: {}
        cpu:
          model: host-passthrough
          threads: 8
        machine:
          type: q35
        resources:
          requests:
            devices.kubevirt.io/kvm: "1"
            memory: 16G
        devices:
          rng: {}
          autoattachSerialConsole: true
          autoattachGraphicsDevice: true
          autoattachPodInterface: false
          interfaces:
          - bridge: {}
            model: virtio
            name: eth0
            macAddress: de:00:00:00:01:de
          disks:
          - name: talos-cp01-disk-vda-root
            bootOrder: 1
            disk:
              bus: virtio
          - name: talos-cp01-disk-vdb-data
            disk:
              bus: virtio
      networks:
      - name: eth0
        multus:
          networkName: nadbr0
      volumes:
      - name: talos-cp01-disk-vda-root
        dataVolume:
          name: talos-cp01-volume-vda-root
      - name: talos-cp01-disk-vdb-data
        dataVolume:
          name: talos-cp01-volume-vdb-data
  dataVolumeTemplates:
  - metadata:
      name: talos-cp01-volume-vda-root
    spec:
      pvc:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 32G
      source:
        registry:
          url: docker://docker.io/containercraft/talos:v1.2.0-alpha.0
          imagePullPolicy: Always
  - metadata:
      name: talos-cp01-volume-vdb-data
    spec:
      pvc:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 64G
      source:
        blank: {}

network-addons-operator-config.yaml

apiVersion: networkaddonsoperator.network.kubevirt.io/v1
kind: NetworkAddonsConfig
metadata:
  name: cluster
spec:
  imagePullPolicy: IfNotPresent
  multus: {}
# ovs: {}
# macvtap: {}
# kubeMacPool: {}
# linuxBridge: {}

network-attachment-definition.yaml

---
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
  name: nadbr0
spec:
  config: >
    {
        "cniVersion": "0.3.1",
        "name": "br0",
        "plugins": [
            {
                "type": "bridge",
                "bridge": "br0",
                "ipam": {}
            },
            {
                "type": "tuning"
            }
        ]
    }

reset.sh

talosctl reset \
  --talosconfig ./talos/talosconfig \
  --system-labels-to-wipe=EPHEMERAL \
  --system-labels-to-wipe=STATE \
  --reboot --graceful=false \
  --nodes 192.168.1.71 -e 192.168.1.71
talosctl reset \
  --talosconfig ./talos/talosconfig \
  --system-labels-to-wipe=EPHEMERAL \
  --system-labels-to-wipe=STATE \
  --reboot --graceful=false \
  --nodes 192.168.1.72 -e 192.168.1.72
talosctl reset \
  --talosconfig ./talos/talosconfig \
  --system-labels-to-wipe=EPHEMERAL \
  --system-labels-to-wipe=STATE \
  --reboot --graceful=false \
  --nodes 192.168.1.73 -e 192.168.1.73
talosctl reset \
  --talosconfig ./talos/talosconfig \
  --system-labels-to-wipe=EPHEMERAL \
  --system-labels-to-wipe=STATE \
  --reboot --graceful=false \
  --nodes 192.168.1.74 -e 192.168.1.74
talosctl reset \
  --talosconfig ./talos/talosconfig \
  --system-labels-to-wipe=EPHEMERAL \
  --system-labels-to-wipe=STATE \
  --reboot --graceful=false \
  --nodes 192.168.1.75 -e 192.168.1.75
talosctl reset \
  --talosconfig ./talos/talosconfig \
  --system-labels-to-wipe=EPHEMERAL \
  --system-labels-to-wipe=STATE \
  --reboot --graceful=false \
  --nodes 192.168.1.76 -e 192.168.1.76

talos-controlplane-n01.yml

version: v1alpha1
debug: true
persist: true
machine:
    type: controlplane
    token: j9oiyf.ek11sl98qfe36w9s
    ca:
        crt: LS0tLS1CRUdJTiBDRVJ....truncated....USUZJkNCOHFBREFnRUN
        key: LS0tLS1CRUdJTiBFRDI....truncated....1NTE5IFBSSVZBVEUgS0
    certSANs:
        - 192.168.1.70
        - talos-kubevirt
        - talos-kubevirt.home.arpa
    kubelet:
        image: ghcr.io/siderolabs/kubelet:v1.24.2
        extraConfig:
            serverTLSBootstrap: true
    network:
      interfaces:
          - interface: br0
            bridge:
              stp:
                enabled: true
              interfaces:
                - eth0
            addresses:
              - 192.168.1.71/24
            routes:
              - network: 0.0.0.0/0
                gateway: 192.168.1.1
                metric: 1024
            mtu: 1500
            vip:
                ip: 192.168.1.70
      nameservers:
          - 192.168.1.1
      extraHostEntries:
          - ip: 192.168.1.70
            aliases:
              - talos-kubevirt
              - talos-kubevirt.home.arpa
          - ip: 192.168.1.71
            aliases:
              - talos-cp01
              - talos-cp01.home.arpa
          - ip: 192.168.1.72
            aliases:
              - talos-cp02
              - talos-cp02.home.arpa
          - ip: 192.168.1.73
            aliases:
              - talos-cp03
              - talos-cp03.home.arpa
      kubespan:
          enabled: false
    install:
        disk: /dev/vda
        image: ghcr.io/siderolabs/installer:v1.2.0-alpha.0
        bootloader: true
        wipe: true
    features:
        rbac: true
    controlPlane:
        scheduler:
            disabled: false
    sysctls:
        kernel.domainname: home.arpa
    sysfs:
        devices.system.cpu.cpu0.cpufreq.scaling_governor: performance
cluster:
    id: m3WfmbKxtTx...truncated...j_D-sX9z_GgCMjVt
    secret: nCrlcmhQWeX...truncated...uVMT6qGtSQ80RPc
    controlPlane:
        endpoint: https://talos-kubevirt.home.arpa:6443
    clusterName: talos-kubevirt
    network:
        dnsDomain: cluster.local
        podSubnets:
            - 10.244.0.0/16
        serviceSubnets:
            - 10.96.0.0/12
    token: w22k05.fsm...truncated....pi8jy
    aescbcEncryptionSecret: sMh+5EH....truncated....w8buSfnoy54Y8YU
    ca:
        crt: LS0tLS1CRUdJTiBDRV....truncated....JUSUZJQ0FURS0tLS0tCk1JSUJpa
        key: LS0tLS1CRUdJTiBFQy....truncated....BQUklWQVRFIEtFWS0tLS0tCk1IY
    aggregatorCA:
        crt: LS0tLS1CRUdJTiBDRV....truncated....JUSUZJQ0FURS0tLS0tCk1JSUJYa
        key: LS0tLS1CRUdJTiBFQy....truncated....BQUklWQVRFIEtFWS0tLS0tCk1IY
    serviceAccount:
        key: LS0tLS1CRUdJTiBFQy....truncated....BQUklWQVRFIEtFWS0tLS0tCk1IY
    apiServer:
        image: k8s.gcr.io/kube-apiserver:v1.24.2
        certSANs:
            - 192.168.1.70
            - talos-kubevirt
            - talos-kubevirt.home.arpa
        disablePodSecurityPolicy: true
    controllerManager:
        image: k8s.gcr.io/kube-controller-manager:v1.24.2
    proxy:
        image: k8s.gcr.io/kube-proxy:v1.24.2
    scheduler:
        image: k8s.gcr.io/kube-scheduler:v1.24.2
    discovery:
        enabled: true
        registries:
            kubernetes: {}
            service: {}
    etcd:
        ca:
            crt: LS0tLS1CRUdJT....truncated....iBDRVJUSUZJQ0FURS0tLS0tCk
            key: LS0tLS1CRUdJT....truncated....iBFQyBQUklWQVRFIEtFWS0tLS
    extraManifests: []
    inlineManifests: []

vm.yaml

apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: ubuntu
  labels:
    app: kc2
    kc2: ubuntu
    ubuntu: bastion
spec:
  running: true
  template:
    spec:
      domain:
        cpu:
          cores: 1
          model: host-model
          sockets: 1
          threads: 4
        devices:
          autoattachGraphicsDevice: true
          autoattachPodInterface: false
          autoattachSerialConsole: true
          disks:
          - bootOrder: 1
            disk:
              bus: virtio
            name: root-disk
          - disk:
              bus: virtio
            name: cloudinitdisk
          interfaces:
          - bootOrder: 2
            bridge: {}
            model: virtio
            name: enp1s0
          networkInterfaceMultiqueue: true
          rng: {}
        machine:
          type: q35
        resources:
          requests:
            memory: 6G
      hostname: ubuntu
      networks:
      - name: enp1s0
        multus:
          networkName: nadbr0
      terminationGracePeriodSeconds: 0
      volumes:
      - name: root-disk
        containerDisk:
          image: docker.io/containercraft/ubuntu:22.04
          imagePullPolicy: IfNotPresent
      - name: cloudinitdisk
        cloudInitNoCloud:
          networkData: |
            version: 2
            ethernets:
              enp1s0:
                dhcp4: true
                dhcp6: false
          userData: |
            #cloud-config
            hostname: ubuntu
            ssh_pwauth: true
            chpasswd:
              list: |
                 kc2user:kc2user
              expire: False
            users:
              - name: kc2user
                sudo: ['ALL=(ALL) NOPASSWD:ALL']
                groups: sudo
                shell: /bin/bash
                ssh-authorized-keys:
                  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfKxpwT/k6ZI/LKMZYLEnTSTlco74kJsP9sfrilR2JP39y8LYmqqbiTZ2m6r4lrbyWjFvch217cOP8UaA3DpSh4BoV0Qs8W5zPcoFt5/hGfIVVCoGkyjDWbYoyjsGFTXgUpRmIl+68ph4BX3ds5wYE+SfNGUupgHxk+vjGbCPMWClBrJ0AmWP2U/uJWN/pv6OdaC8AsTkF5LkIhq2pKH/X4a8gMmXihUl1hgrWeiCmS3B49wQNn8Bc3DORKx4XZRinQ4LYklOhQYfH+P5Trz17RyEzb/ttKG3nkvHWTM4MA6/RcxGTjorYbR0lb9KtHNqaClV61voBVGbUz/Ufn4gDCZzfFMzm5UwpwY6eOYKDU8RQbNIB/+GGW4eC9XGVs8JZys0lovCLdrbeNPpJGVZ991nncvY0ouXoYZZX5atVEJW5zJnk2/BqEek/kSIZM57rj49AiZ9pFNnE+PoYxHlbXy4nQrHsCkZkMkrN4CtDumdds05TvOZKIlGXOc3/kjU= usrbinkat@honeypot.home.arpa
            package_upgrade: true
            packages:
              - ubuntu-desktop-minimal
              - docker.io
              - xrdp
              - gnome-tweaks
            runcmd:
              - "snap remove lxd"
              - "mkdir -p /home/kc2user/.kube"
              - "chown -R kc2user:kc2user /home/kc2user"
              - "usermod -a -G xrdp kc2user"
              - "usermod -a -G ssl-cert kc2user"
              - "usermod -a -G docker kc2user"
              - "apt-get -y upgrade"

vyos.config.sh

set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp01 mac-address de:00:00:00:01:de
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp01 ip-address 192.168.1.71
set system static-host-mapping host-name talos-cp01.home.arpa inet 192.168.1.71


set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp02 mac-address de:00:00:00:02:de
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp02 ip-address 192.168.1.72
set system static-host-mapping host-name talos-cp02.home.arpa inet 192.168.1.72

set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp03 mac-address de:00:00:00:03:de
set service dhcp-server shared-network-name LAN subnet 192.168.1.0/24 static-mapping talos-cp03 ip-address 192.168.1.73
set system static-host-mapping host-name talos-cp03.home.arpa inet 192.168.1.73

set system static-host-mapping host-name talos-kubevirt.home.arpa inet 192.168.1.70