Terminal log for LXD cluster video

gistfile1.txt

# Ceph setup
python3 -m venv ceph
cd ceph
. bin/activate
pip3 install git+https://github.com/ceph/ceph-deploy.git

ceph-deploy --username ubuntu new r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas
ceph-deploy --username ubuntu install r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas --release pacific --mon --mgr --mds --osd
ceph-deploy --username ubuntu install r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas r02-amd64-04.cloud02.maas r02-amd64-05.cloud02.maas r02-amd64-06.cloud02.maas r02-arm64-01.cloud02.maas r02-arm64-02.cloud02.maas --release pacific --osd
ceph-deploy --username ubuntu mon create r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas
ceph-deploy --username ubuntu gatherkeys r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas
ceph-deploy --username ubuntu admin r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas r02-amd64-04.cloud02.maas r02-amd64-05.cloud02.maas r02-amd64-06.cloud02.maas r02-arm64-01.cloud02.maas r02-arm64-02.cloud02.maas
ceph-deploy --username ubuntu mgr create r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas
ceph-deploy --username ubuntu mds create r02-amd64-01.cloud02.maas r02-amd64-02.cloud02.maas r02-amd64-03.cloud02.maas

for i in r02-amd64-01 r02-amd64-02 r02-amd64-03 r02-amd64-04 r02-amd64-05 r02-amd64-06 r02-arm64-01 r02-arm64-02; do ceph-deploy --username ubuntu osd create --data /dev/sdb $i.cloud02.maas; ceph-deploy --username ubuntu osd create --data /dev/sdc $i.cloud02.maas; done

ssh ubuntu@r02-amd64-01.cloud02.maas -t sudo -i
    ceph osd pool create cephfs_data 8
    ceph osd pool create cephfs_metadata 8
    ceph fs new cephfs cephfs_metadata cephfs_data
    ceph fs set cephfs allow_new_snaps true

# OVN
On first 3 servers (r02-amd64-01, r02-amd64-02, r03-amd64-03):
    apt install ovn-central ovn-host
    systemctl stop ovn-central
    <Edit /etc/default/ovn-central and fill in the following template:>
        OVN_CTL_OPTS="--db-nb-addr=SERVER1 --db-nb-create-insecure-remote=yes --db-sb-addr=SERVER1 --db-sb-create-insecure-remote=yes --db-nb-cluster-local-addr=LOCAL --db-sb-cluster-local-addr=LOCAL --ovn-northd-nb-db=tcp:SERVER1:6641,tcp:SERVER2:6641,tcp:SERVER3:6641 --ovn-northd-sb-db=tcp:SERVER1:6642,tcp:SERVER2:6642,tcp:SERVER3:6642"
    systemctl start ovn-central

    ovs-vsctl set open_vswitch . external_ids:ovn-remote=tcp:SERVER1:6642,tcp:SERVER2:6642,tcp:SERVER3:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=LOCAL  (replace LOCAL)

On the remaining servers:
    apt install ovn-host
    ovs-vsctl set open_vswitch . external_ids:ovn-remote=tcp:SERVER1:6642,tcp:SERVER2:6642,tcp:SERVER3:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=LOCAL  (replace LOCAL)
    
# LXD setup
lxd init
lxc storage list
lxc storage edit remote
lxc storage create remote-fs cephfs source=cephfs

lxc config set network.ovn.northbound_connection tcp:172.17.32.8:6641,tcp:172.17.32.9:6641,tcp:172.17.32.5:6641
lxc network list
lxc network create UPLINK parent=br0 dns.nameservers=1.1.1.1 ipv4.gateway=172.17.32.1/24 ipv6.gateway=2602:fc62:b:3002::1/64 ipv4.ovn.ranges=172.17.32.200-172.17.32.254 ipv6.ovn.ranges=2602:fc62:b:3002:2::200-2602:fc62:b:3002:2::254 --type physical

# Initial network and instance
lxc network create default --type ovn
lxc profile device add default eth0 nic network=default name=eth0
lxc launch images:alpine/edge c1
lxc list
lxc exec c1 sh
  ping 8.8.8.8

# Growing the cluster
lxc cluster list
lxc cluster add r02-amd64-02
lxc cluster add r02-amd64-03
lxc cluster add r02-amd64-04
lxc cluster add r02-amd64-05
lxc cluster add r02-amd64-06
lxc cluster add r02-arm64-01
lxc cluster add r02-arm64-02

# Launching instances across the entire cluster
lxc launch images:alpine/edge c2
lxc launch images:alpine/edge c3
lxc launch images:alpine/edge c4
lxc launch images:alpine/edge c5
lxc launch images:alpine/edge c6
lxc launch images:alpine/edge c7
lxc launch images:alpine/edge c8
lxc list
lxc exec c8 sh
  ping <ip of c1>

# Setting default architecture
lxc config set images.default_architecture=x86_64
lxc launch images:alpine/edge/arm64 c9
lxc launch images:alpine/edge/arm64 c10
lxc list

# Setting cluster member descriptions and failure domains
lxc cluster list
lxc cluster edit r02-amd64-01
lxc cluster edit r02-amd64-02
lxc cluster edit r02-amd64-03
lxc cluster list
lxc cluster edit r02-amd64-01
lxc cluster edit r02-amd64-02
lxc cluster edit r02-amd64-03
lxc cluster edit r02-amd64-04
lxc cluster edit r02-amd64-05
lxc cluster edit r02-amd64-06
lxc cluster list

# Adding client to restricted project
vi client.crt
lxc config trust add client.crt 
lxc project list
lxc project create foo
lxc project list
lxc project edit foo
lxc config trust list
lxc config trust edit 390fdd27ed5d

# LXD from client
lxc remote add cluster 172.17.32.8
lxc remote switch cluster

lxc list
lxc project list
lxc config show
lxc storage list
lxc storage delete remote
lxc project list
lxc project switch foo
lxc project edit foo
lxc project show foo

lxc list
lxc profile show default
lxc profile device add default root disk pool=local path=/
lxc network list
lxc network create default
lxc network list
lxc profile device add default eth0 nic network=default name=eth0
lxc network list
lxc launch images:alpine/edge c1
lxc list
lxc launch images:alpine/edge c2
lxc launch images:alpine/edge c3 -c security.privileged=true
lxc launch images:alpine/edge c3 -c security.nesting=true
lxc launch images:alpine/edge c3 --storage remote
lxc launch images:alpine/edge c4 --storage remote
lxc launch images:alpine/edge/arm64 c5 --storage remote
lxc launch images:alpine/edge/arm64 c6 --storage remote
lxc list

lxc storage list
lxc storage volume create remote-fs blah
lxc profile create blah
lxc profile device add blah blah disk pool=remote-fs source=blah path=/blah
lxc profile add c1 blah
lxc profile add c4 blah
lxc profile add c6 blah
lxc exec c1 sh
  echo foo > /blah/bar
lxc exec c4 sh
  cat /blah/bar
lxc exec c6 sh
  cat /blah/bar
lxc launch images:ubuntu/20.04/cloud u1 --vm -p default -p blah
lxc exec u1 bash
  cat /blah/bar

lxc network acl create block-smtp
lxc network acl edit block-smtp
lxc list
lxc exec u1 bash
  nc -v smtp.gmail.com 25
lxc config device override u1 eth0 security.acls=block-smtp
lxc exec u1 bash
  nc -v smtp.gmail.com 25
lxc network acl create default
lxc network acl edit default
lxc config device set u1 eth0 security.acls=block-smtp,default
lxc exec u1 bash
  ping 8.8.8.8
  nc -v smtp.gmail.com 25
lxc exec c6 sh
  ping 8.8.8.8
  nc -v smtp.gmail.com 25
lxc network set default security.acls=default,block-smtp
lxc exec c6 sh
  ping 8.8.8.8
  nc -v smtp.gmail.com 25

lxc exec u1 bash
  ping <instance c1 in project default>
lxc network peer create default default-project default/default
lxc network peer list default
lxc exec u1 bash
  ping <instance c1 in project default>

# Setting up network peering
lxc network peer create default foo-project foo/default

# Additional project
lxc project create bar -c features.networks=true
lxc project switch bar
lxc network create default network=UPLINK --type=ovn
lxc network list
lxc profile device add default root disk pool=remote path=/
lxc profile device add default eth0 nic network=default name=eth0
lxc launch images:alpine/edge c1
lxc launch images:alpine/edge c2
lxc profile set default snapshots.schedule=@startup snapshots.expiry=7d
lxc list
lxc restart c1
lxc list
lxc launch images:alpine/edge c3
lxc list

# Cluster evacuation
lxc cluster list
lxc cluster evacuate r02-amd64-05
lxc cluster list
lxc cluster restore r02-amd64-05
lxc cluster list