LXD cluster with CEPH, OVN and Grafana on Google Compute

commands

# Deploy the LXD cluster
juju bootstrap google/northamerica-northeast1 gce
juju deploy ./lxd-gce.yaml
juju add-storage ceph-osd/0 osd-devices=gce,100G,1
juju add-storage ceph-osd/1 osd-devices=gce,100G,1
juju add-storage ceph-osd/2 osd-devices=gce,100G,1
juju add-storage ceph-osd/3 osd-devices=gce,100G,1
juju add-storage ceph-osd/4 osd-devices=gce,100G,1

# Expose the services
juju expose grafana
juju expose lxd

# Add the cluster
juju run-action lxd/leader add-trusted-client cert="$(cat ~/snap/lxd/common/config/client.crt)"
lxc remote add gce
lxc remote switch gce

# Setup LXD for CEPH distributed storage
juju run-action --wait ceph-mon/leader create-pool name=lxd app-name=lxd
lxc storage create remote ceph source=lxd --target juju-1c093c-1
lxc storage create remote ceph source=lxd --target juju-1c093c-2
lxc storage create remote ceph source=lxd --target juju-1c093c-3
lxc storage create remote ceph source=lxd --target juju-1c093c-4
lxc storage create remote ceph source=lxd --target juju-1c093c-5
lxc storage create remote ceph ceph.user.name=lxd
lxc storage list

# Setup LXD with an OVN UPLINK network
lxc network create UPLINK --type bridge --target juju-1c093c-1
lxc network create UPLINK --type bridge --target juju-1c093c-2
lxc network create UPLINK --type bridge --target juju-1c093c-3
lxc network create UPLINK --type bridge --target juju-1c093c-4
lxc network create UPLINK --type bridge --target juju-1c093c-5
lxc network create UPLINK --type bridge ipv4.address=192.168.0.1/24 ipv4.nat=true ipv6.address=none ipv4.dhcp.ranges=192.168.0.10-192.168.0.99 ipv4.ovn.ranges=192.168.0.100-192.168.0.254

# Create some OVN networks
lxc network create my-ovn1 network=UPLINK --type=ovn
lxc network create my-ovn2 network=UPLINK --type=ovn
lxc network list

# Launch some instances on the cluster
lxc launch images:alpine/edge a1 --network my-ovn1 --storage remote
lxc launch images:alpine/edge a2 --network my-ovn1 --storage remote
lxc launch images:alpine/edge a3 --network my-ovn2 --storage remote
lxc launch images:alpine/edge a4 --network my-ovn2 --storage remote
lxc launch images:ubuntu/20.04 u1 --network my-ovn1
lxc launch images:ubuntu/20.04 u2
lxc list

# Get Grafana credentials
juju run-action --wait grafana/0 get-admin-password
juju status
http://<grafana IP>:3000 (login is admin)

# Cleanup everything
juju destroy-model default --destroy-storage --force
juju destroy-controller gce

lxd-gce.yaml

series: focal
variables:
  osd-devices: &osd-devices /dev/sdb
  expected-osd-count: &expected-osd-count 5
  expected-mon-count: &expected-mon-count 3
machines:
  "0":
    # Infrastructure services
    constraints: instance-type=n2d-standard-4 root-disk=200G
    series: focal
  "1":
    # Compute
    constraints: instance-type=n2-standard-2 root-disk=100G
    series: focal
  "2":
    # Compute
    constraints: instance-type=n2-standard-2 root-disk=100G
    series: focal
  "3":
    # Compute
    constraints: instance-type=n2-standard-2 root-disk=100G
    series: focal
  "4":
    # Compute
    constraints: instance-type=n2-standard-2 root-disk=100G
    series: focal
  "5":
    # Compute
    constraints: instance-type=n2-standard-2 root-disk=100G
    series: focal
  "6":
    # Grafana
    constraints: instance-type=n2d-highcpu-2
    series: focal
applications:
  ceph-mon:
    charm: ceph-mon
    options:
      expected-osd-count: *expected-osd-count
      monitor-count: *expected-mon-count
    num_units: 3
    to:
    - lxd:0
    - lxd:0
    - lxd:0
  ceph-osd:
    charm: ceph-osd
    options:
      osd-devices: *osd-devices
    num_units: 5
    to:
    - 1
    - 2
    - 3
    - 4
    - 5
  postgresql:
    charm: postgresql
    channel: stable
    num_units: 1
    to:
    - lxd:0
  vault:
    charm: vault
    channel: stable
    num_units: 1
    to:
    - lxd:0
    options:
      auto-generate-root-ca-cert: true
      # XXX: insecure, only for testing
      totally-unsecure-auto-unlock: true
  prometheus2:
    charm: prometheus2
    # requires revision 28 or later hence edge and focal
    channel: edge
    series: focal
    num_units: 1
    to:
    - lxd:0
  grafana:
    charm: grafana
    num_units: 1
    to:
    - 6
  ovn-central:
    charm: ovn-central
    channel: stable
    num_units: 3
    to:
    - 2
    - 3
    - 4
  ovn-dedicated-chassis:
    charm: ovn-dedicated-chassis
    num_units: 5
    to:
    - 1
    - 2
    - 3
    - 4
    - 5
  lxd:
    charm: lxd
    options:
      lxd-listen-https: true
      mode: cluster
    num_units: 5
    to:
    - 1
    - 2
    - 3
    - 4
    - 5
relations:
- - ceph-osd:mon
  - ceph-mon:osd
- - lxd:ceph
  - ceph-mon:client
- - vault:db
  - postgresql:db
- - ovn-central:certificates
  - vault:certificates
- - ovn-dedicated-chassis:certificates
  - vault:certificates
- - ovn-dedicated-chassis:ovsdb
  - ovn-central:ovsdb
- - lxd:certificates
  - vault:certificates
- - lxd:ovsdb-cms
  - ovn-central:ovsdb-cms
- - prometheus2:grafana-source
  - grafana:grafana-source
- - prometheus2:manual-jobs
  - lxd:prometheus-manual
- - lxd:grafana-dashboard
  - grafana:dashboards