Skip to content

Instantly share code, notes, and snippets.

@stgraber
Created Mar 22, 2022
Embed
What would you like to do?
LXD cluster with CEPH, OVN and Grafana on Google Compute
# Deploy the LXD cluster
juju bootstrap google/northamerica-northeast1 gce
juju deploy ./lxd-gce.yaml
juju add-storage ceph-osd/0 osd-devices=gce,100G,1
juju add-storage ceph-osd/1 osd-devices=gce,100G,1
juju add-storage ceph-osd/2 osd-devices=gce,100G,1
juju add-storage ceph-osd/3 osd-devices=gce,100G,1
juju add-storage ceph-osd/4 osd-devices=gce,100G,1
# Expose the services
juju expose grafana
juju expose lxd
# Add the cluster
juju run-action lxd/leader add-trusted-client cert="$(cat ~/snap/lxd/common/config/client.crt)"
lxc remote add gce
lxc remote switch gce
# Setup LXD for CEPH distributed storage
juju run-action --wait ceph-mon/leader create-pool name=lxd app-name=lxd
lxc storage create remote ceph source=lxd --target juju-1c093c-1
lxc storage create remote ceph source=lxd --target juju-1c093c-2
lxc storage create remote ceph source=lxd --target juju-1c093c-3
lxc storage create remote ceph source=lxd --target juju-1c093c-4
lxc storage create remote ceph source=lxd --target juju-1c093c-5
lxc storage create remote ceph ceph.user.name=lxd
lxc storage list
# Setup LXD with an OVN UPLINK network
lxc network create UPLINK --type bridge --target juju-1c093c-1
lxc network create UPLINK --type bridge --target juju-1c093c-2
lxc network create UPLINK --type bridge --target juju-1c093c-3
lxc network create UPLINK --type bridge --target juju-1c093c-4
lxc network create UPLINK --type bridge --target juju-1c093c-5
lxc network create UPLINK --type bridge ipv4.address=192.168.0.1/24 ipv4.nat=true ipv6.address=none ipv4.dhcp.ranges=192.168.0.10-192.168.0.99 ipv4.ovn.ranges=192.168.0.100-192.168.0.254
# Create some OVN networks
lxc network create my-ovn1 network=UPLINK --type=ovn
lxc network create my-ovn2 network=UPLINK --type=ovn
lxc network list
# Launch some instances on the cluster
lxc launch images:alpine/edge a1 --network my-ovn1 --storage remote
lxc launch images:alpine/edge a2 --network my-ovn1 --storage remote
lxc launch images:alpine/edge a3 --network my-ovn2 --storage remote
lxc launch images:alpine/edge a4 --network my-ovn2 --storage remote
lxc launch images:ubuntu/20.04 u1 --network my-ovn1
lxc launch images:ubuntu/20.04 u2
lxc list
# Get Grafana credentials
juju run-action --wait grafana/0 get-admin-password
juju status
http://<grafana IP>:3000 (login is admin)
# Cleanup everything
juju destroy-model default --destroy-storage --force
juju destroy-controller gce
series: focal
variables:
osd-devices: &osd-devices /dev/sdb
expected-osd-count: &expected-osd-count 5
expected-mon-count: &expected-mon-count 3
machines:
"0":
# Infrastructure services
constraints: instance-type=n2d-standard-4 root-disk=200G
series: focal
"1":
# Compute
constraints: instance-type=n2-standard-2 root-disk=100G
series: focal
"2":
# Compute
constraints: instance-type=n2-standard-2 root-disk=100G
series: focal
"3":
# Compute
constraints: instance-type=n2-standard-2 root-disk=100G
series: focal
"4":
# Compute
constraints: instance-type=n2-standard-2 root-disk=100G
series: focal
"5":
# Compute
constraints: instance-type=n2-standard-2 root-disk=100G
series: focal
"6":
# Grafana
constraints: instance-type=n2d-highcpu-2
series: focal
applications:
ceph-mon:
charm: ceph-mon
options:
expected-osd-count: *expected-osd-count
monitor-count: *expected-mon-count
num_units: 3
to:
- lxd:0
- lxd:0
- lxd:0
ceph-osd:
charm: ceph-osd
options:
osd-devices: *osd-devices
num_units: 5
to:
- 1
- 2
- 3
- 4
- 5
postgresql:
charm: postgresql
channel: stable
num_units: 1
to:
- lxd:0
vault:
charm: vault
channel: stable
num_units: 1
to:
- lxd:0
options:
auto-generate-root-ca-cert: true
# XXX: insecure, only for testing
totally-unsecure-auto-unlock: true
prometheus2:
charm: prometheus2
# requires revision 28 or later hence edge and focal
channel: edge
series: focal
num_units: 1
to:
- lxd:0
grafana:
charm: grafana
num_units: 1
to:
- 6
ovn-central:
charm: ovn-central
channel: stable
num_units: 3
to:
- 2
- 3
- 4
ovn-dedicated-chassis:
charm: ovn-dedicated-chassis
num_units: 5
to:
- 1
- 2
- 3
- 4
- 5
lxd:
charm: lxd
options:
lxd-listen-https: true
mode: cluster
num_units: 5
to:
- 1
- 2
- 3
- 4
- 5
relations:
- - ceph-osd:mon
- ceph-mon:osd
- - lxd:ceph
- ceph-mon:client
- - vault:db
- postgresql:db
- - ovn-central:certificates
- vault:certificates
- - ovn-dedicated-chassis:certificates
- vault:certificates
- - ovn-dedicated-chassis:ovsdb
- ovn-central:ovsdb
- - lxd:certificates
- vault:certificates
- - lxd:ovsdb-cms
- ovn-central:ovsdb-cms
- - prometheus2:grafana-source
- grafana:grafana-source
- - prometheus2:manual-jobs
- lxd:prometheus-manual
- - lxd:grafana-dashboard
- grafana:dashboards
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment