How We Built a Caktus Cloud - Accompanying Files

deploy-ceph.yaml

# Adapted from:
# https://docs.ceph.com/projects/cephadm-ansible/en/latest/index.html#samples
- name: Bootstrap cluster
  hosts: ceph_mon1
  tags: cluster
  become: true
  gather_facts: false
  roles:
    - role: "nickjj.acme_sh"
      tags: ["acme_sh"]
  tasks:
    # Docker might not have started yet if it was just installed
    - name: Start service docker, if not started
      ansible.builtin.service:
        name: docker
        state: started
    - name: Check if ceph.conf exists
      stat:
        path: /etc/ceph/ceph.conf
      register: ceph_conf
    # https://docs.ceph.com/en/quincy/cephadm/install/#bootstrap-a-new-cluster
    # https://docs.ceph.com/projects/cephadm-ansible/en/latest/index.html#cephadm-bootstrap
    - name: Bootstrap initial cluster (this can take a while)
      cephadm_bootstrap:
        mon_ip: "{{ ceph_monitor_address }}"
        cluster_network: "{{ ceph_cluster_network | default(omit) }}"
        dashboard_user: "{{ ceph_dashboard_user }}"
        dashboard_password: "{{ ceph_dashboard_password }}"
      when: not ceph_conf.stat.exists
    - name: Configure SSL certificates (if bootstrapping cluster)
      shell: "{{ acme_sh_default_install_cert_reloadcmd }}"
      when: not ceph_conf.stat.exists
    # Allow for safely rebooting storage nodes nightly:
    # https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/8/html/director_installation_and_usage/sect-rebooting-ceph
    - name: Install cronjob to disable cluster rebalancing
      ansible.builtin.cron:
        name: "disable rebalance"
        # 1:55 AM UTC, 5 minutes before the first unattended upgrade reboot
        minute: "55"
        hour: "1"
        job: ceph osd set noout >/dev/null 2>&1 && ceph osd set norebalance >/dev/null 2>&1
    - name: Install cronjob to re-enable cluster rebalancing
      ansible.builtin.cron:
        name: "enable rebalance"
        # 3:00 AM UTC, 20 minutes after the last unattended upgrade reboot
        minute: "0"
        hour: "3"
        job: ceph osd unset noout >/dev/null 2>&1 && ceph osd unset norebalance >/dev/null 2>&1

- name: Add or configure hosts
  hosts: ceph
  tags: hosts
  become: true
  gather_facts: true
  tasks:
    - name: Find out what the remote machine's mounts are
      ansible.builtin.slurp:
        src: /etc/ceph/ceph.pub
      register: ceph_pub_key
      delegate_to: loyal_mouse  # ceph_mon1
    - name: Add ceph.pub to root's authorized_keys
      ansible.posix.authorized_key:
        user: root
        state: present
        key: "{{ ceph_pub_key.content | b64decode }}"
    # https://docs.ceph.com/en/quincy/cephadm/host-management/#adding-hosts
    - name: Add hosts to the cluster
      ceph_orch_host:
        name: "{{ ansible_facts['hostname'] }}"
        address: "{{ ansible_default_ipv4.address }}"
        labels: "{{ ceph_labels }}"
      delegate_to: loyal_mouse  # ceph_mon1

- name: Deploy osd service
  hosts: loyal_mouse  # ceph_mon1
  tags: osd
  become: true
  gather_facts: false
  tasks:
    # https://docs.ceph.com/en/quincy/cephadm/services/osd/#deploy-osds
    - name: Apply osd spec
      ceph_orch_apply:
        spec: |
          service_type: osd
          service_id: osd
          placement:
            host_pattern: '*'
            label: osd
          spec:
            data_devices:
              all: true

- name: Deploy rgw service
  hosts: loyal_mouse  # ceph_mon1
  tags: rgw
  become: true
  gather_facts: false
  tasks:
    # https://docs.ceph.com/en/quincy/cephadm/services/rgw/#deploy-rgws
    - name: Apply rgw spec
      ceph_orch_apply:
        spec: |
          service_type: rgw
          service_id: metalrgw
          placement:
            label: rgw
            count_per_host: 2

- name: Change osd_default_notify_timeout option
  hosts: loyal_mouse  # ceph_mon1
  tags: config
  become: true
  gather_facts: false
  tasks:
    - name: Decrease the value of osd_default_notify_timeout option
      ceph_config:
        action: set
        who: osd
        option: osd_default_notify_timeout
        value: 20

fetch-ceph-keyrings.yaml

# This playbook automates the key generation and Kolla-ansible configuration from:
# - https://docs.ceph.com/en/latest/rbd/rbd-openstack/
# - https://docs.openstack.org/kolla-ansible/latest/reference/storage/external-ceph-guide.html
- name: Connect OpenStack with Ceph
  hosts: loyal_mouse  # ceph_mon1
  become: true
  gather_facts: true
  tasks:
    # https://docs.ceph.com/en/latest/rbd/rbd-openstack/#configure-openstack-ceph-clients
    - name: Load ceph.conf
      ansible.builtin.slurp:
        src: /etc/ceph/ceph.conf
      register: ceph_conf
    - name: Distribute ceph.conf to localhost
      copy:
        dest: "{{ lookup('ansible.builtin.env', 'CONFIG') }}/config/{{ item }}/ceph.conf"
        # Tabs in this file throw off Kolla-Ansible; remove them.
        content: "{{ ceph_conf.content | b64decode | replace('\t', '') }}"
      loop:
        - glance
        - cinder
        - nova
      delegate_to: localhost
    # https://docs.ceph.com/en/latest/rbd/rbd-openstack/#setup-ceph-client-authentication
    - name: Get or create glance keyring
      command: ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images' mgr 'profile rbd pool=images'
      register: glance_keyring
    - name: Get or create cinder keyring
      command: ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms'
      register: cinder_keyring
    - name: Get or create cinder-backup keyring
      command: ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups' mgr 'profile rbd pool=backups'
      register: cinder_backup_keyring
    - name: Ensure keys are under cephadm management
      shell: |
        ceph orch client-keyring set client.glance '*' --mode 640
        ceph orch client-keyring set client.cinder '*' --mode 640
        ceph orch client-keyring set client.cinder-backup '*' --mode 640
    - name: Distribute keyrings to localhost
      copy:
        dest: "{{ lookup('ansible.builtin.env', 'CONFIG') }}/config/{{ item.dir }}/{{ item.name }}"
        # This file MUST maintain the ending newline, or the ceph client won't be able to
        # parse it (!).
        content: "{{ item.content + '\n' }}"
      loop:
        - dir: glance
          name: ceph.client.glance.keyring
          content: "{{ glance_keyring.stdout }}"
        - dir: cinder/cinder-volume
          name: ceph.client.cinder.keyring
          content: "{{ cinder_keyring.stdout }}"
        - dir: cinder/cinder-backup
          name: ceph.client.cinder.keyring
          content: "{{ cinder_keyring.stdout }}"
        - dir: cinder/cinder-backup
          name: ceph.client.cinder-backup.keyring
          content: "{{ cinder_backup_keyring.stdout }}"
        - dir: nova
          name: ceph.client.cinder.keyring
          content: "{{ cinder_keyring.stdout }}"
      delegate_to: localhost