~ # for i in 0 1 2; do cbsd bcreate jconf=/root/instance.jconf jname="controller-$i" \ ci_ip4_addr="10.240.0.1${i}/8" ci_jname="controller-$i" \ ci_fqdn="controller-${i}.hardk8s.local" ip_addr="10.240.0.1${i}" \ imgsize="30g" vm_cpus="2" vm_ram="8g" done Global VM ZFS guid: 18240384265212679365 To edit VM properties use: cbsd bconfig jname=controller-0 To start VM use: cbsd bstart controller-0 To stop VM use: cbsd bstop controller-0 To remove VM use: cbsd bremove controller-0 For attach VM console use: cbsd blogin controller-0 Creating controller-0 complete: Enjoy! auto-generate cloud-init settings: /usr/cbsd/jails-system/controller-0/cloud-init [ similar output for controller-1 and controller-2 ] ~ # for i in 0 1 2; do cbsd bstart jname="controller-$i"; done cloud-init: enabled Looks like /usr/cbsd/vm/controller-0/dsk1.vhd is empty. May be you want to boot from CD? [yes(1) or no(0)] 1 Temporary boot device: cd vm_iso_path: 0 No such media: /usr/cbsd/src/iso/cbsd-cloud-cloud-Ubuntu-x86-20.04.1.raw in /usr/cbsd/src/iso Shall i download it from: https://mirror.bsdstore.ru/cloud/? [yes(1) or no(0)] 1 Download to: /usr/cbsd/src/iso/cbsd-cloud-cloud-Ubuntu-x86-20.04.1.raw [ download output skipped ] Eject cloud source: media mode=detach name=cloud-ubuntu-x86-20.04.1 path=/usr/cbsd/src/iso/cbsd-cloud-cloud-Ubuntu-x86-20.04.1.raw type=iso jname=controller-0 DELETE FROM media WHERE name="cloud-ubuntu-x86-20.04.1" AND path="/usr/cbsd/src/iso/cbsd-cloud-cloud-Ubuntu-x86-20.04.1.raw" AND jname="controller-0" vm_iso_path: changed Detach to: controller-0 All CD/ISO ejected: controller-0 VRDP is enabled. VNC bind/port: 127.0.0.1:5900 For attach VM console, use: vncviewer 127.0.0.1:5900 Resolution: 1024x768. bhyve renice: 1 Waiting for PID. PID: 76286 CBSD setup: bhyve ipfw counters num: 99/100 [ similar output for controller-1 and controller-2 ] ~ # for i in 0 1 2; do cbsd bcreate jconf=/root/instance.jconf jname="worker-$i" \ ci_ip4_addr="10.240.0.2${i}/24" ci_jname="worker-$i" \ ci_fqdn="worker-${i}.hardk8s.local" ip_addr="10.240.0.2${i}" \ ci_pod_cidr="10.200.${i}.0/24" done Global VM ZFS guid: 11960195993976622261 To edit VM properties use: cbsd bconfig jname=worker-0 To start VM use: cbsd bstart worker-0 To stop VM use: cbsd bstop worker-0 To remove VM use: cbsd bremove worker-0 For attach VM console use: cbsd blogin worker-0 Creating worker-0 complete: Enjoy! auto-generate cloud-init settings: /usr/cbsd/jails-system/worker-0/cloud-init [ similar output for worker-1 and worker-2 ] ~ # for i in 0 1 2; do cbsd bstart jname="worker-$i"; done [ similar output to controllers above ] ~ # cbsd bls JNAME JID VM_RAM VM_CURMEM VM_CPUS PCPU VM_OS_TYPE IP4_ADDR STATUS VNC controller-0 28798 8192 511 2 10 linux 10.240.0.10 On 127.0.0.1:5900 controller-1 30446 8192 506 2 14 linux 10.240.0.11 On 127.0.0.1:5901 controller-2 32153 8192 511 2 28 linux 10.240.0.12 On 127.0.0.1:5902 worker-0 8967 4096 868 1 0 linux 10.240.0.20 On 127.0.0.1:5903 worker-1 10657 4096 997 1 27 linux 10.240.0.21 On 127.0.0.1:5904 worker-2 12555 4096 975 1 28 linux 10.240.0.22 On 127.0.0.1:5905