Examples of parallelism when running OSP 13->16.1 control plane upgrades

1-system_upgrade-batch_1.sh

#!/bin/env bash
#
# Run overcloud Operating System upgrade on overcloud node ceph-0,controller-0,database-0,messaging-0,networker-0
#
set -euo pipefail

source /home/stack/stackrc

if [ ! -f .ceph_ran_ceph-0,controller-0,database-0,messaging-0,networker-0 ]; then

echo "[$(date)] Started ceph systemd units migration run for ceph-0,controller-0,database-0,messaging-0,networker-0"

openstack overcloud external-upgrade run --yes \
        --stack qe-Cloud-0 \
        --tags ceph_systemd \
        -e ceph_ansible_limit=ceph-0,controller-0,database-0,messaging-0,networker-0 2>&1 && touch .ceph_ran_ceph-0,controller-0,database-0,messaging-0,networker-0

echo "[$(date)] Finished ceph systemd units migration run for ceph-0,controller-0,database-0,messaging-0,networker-0"
fi

echo "[$(date)] Started system upgrade step for ceph-0,controller-0,database-0,messaging-0,networker-0"

tripleo-ansible-inventory --stack qe-Cloud-0 --static-yaml-inventory upgrade_inventory.yaml --undercloud-connection ssh --undercloud-key-file /var/lib/mistral/.ssh/tripleo-admin-rsa --ansible_ssh_user tripleo-admin

bkg_pids=""
for host in $(echo "ceph-0,controller-0,database-0,messaging-0,networker-0" | sed "s/,/ /g")
do
  openstack overcloud upgrade run --yes \
          --stack qe-Cloud-0 \
          --static-inventory upgrade_inventory.yaml \
          --tags system_upgrade \
          --limit "${host}" 2>&1 | tee -a "RHEL_upgrade_${host}" &
  bkg_pids+=" $! "
done

status=0
for p in $bkg_pids; do
    if ! wait $p; then
        status=1
    fi
done

if [[ $status == 0 ]]; then
    echo "[$(date)] Finished system upgrade step for ceph-0,controller-0,database-0,messaging-0,networker-0"
else
    echo "[$(date)] Failed in system upgrade step for ceph-0,controller-0,database-0,messaging-0,networker-0"
    exit 1
fi

2-system_upgrade-batch_2.sh

#!/bin/env bash
#
# Run overcloud Operating System upgrade on overcloud node ceph-1,controller-1,database-1,messaging-1,networker-1
#
set -euo pipefail

source /home/stack/stackrc

if [ ! -f .ceph_ran_ceph-1,controller-1,database-1,messaging-1,networker-1 ]; then

echo "[$(date)] Started ceph systemd units migration run for ceph-1,controller-1,database-1,messaging-1,networker-1"

openstack overcloud external-upgrade run --yes \
        --stack qe-Cloud-0 \
        --tags ceph_systemd \
        -e ceph_ansible_limit=ceph-1,controller-1,database-1,messaging-1,networker-1 2>&1 && touch .ceph_ran_ceph-1,controller-1,database-1,messaging-1,networker-1

echo "[$(date)] Finished ceph systemd units migration run for ceph-1,controller-1,database-1,messaging-1,networker-1"
fi

echo "[$(date)] Started system upgrade step for ceph-1,controller-1,database-1,messaging-1,networker-1"

tripleo-ansible-inventory --stack qe-Cloud-0 --static-yaml-inventory upgrade_inventory.yaml --undercloud-connection ssh --undercloud-key-file /var/lib/mistral/.ssh/tripleo-admin-rsa --ansible_ssh_user tripleo-admin

bkg_pids=""
for host in $(echo "ceph-1,controller-1,database-1,messaging-1,networker-1" | sed "s/,/ /g")
do
  openstack overcloud upgrade run --yes \
          --stack qe-Cloud-0 \
          --static-inventory upgrade_inventory.yaml \
          --tags system_upgrade \
          --limit "${host}" 2>&1 | tee -a "RHEL_upgrade_${host}" &
  bkg_pids+=" $! "
done

status=0
for p in $bkg_pids; do
    if ! wait $p; then
        status=1
    fi
done

if [[ $status == 0 ]]; then
    echo "[$(date)] Finished system upgrade step for ceph-1,controller-1,database-1,messaging-1,networker-1"
else
    echo "[$(date)] Failed in system upgrade step for ceph-1,controller-1,database-1,messaging-1,networker-1"
    exit 1
fi

3-system_upgrade-batch_3.sh

#!/bin/env bash
#
# Run overcloud Operating System upgrade on overcloud node ceph-2,controller-2,database-2,messaging-2
#
set -euo pipefail

source /home/stack/stackrc

if [ ! -f .ceph_ran_ceph-2,controller-2,database-2,messaging-2 ]; then

echo "[$(date)] Started ceph systemd units migration run for ceph-2,controller-2,database-2,messaging-2"

openstack overcloud external-upgrade run --yes \
        --stack qe-Cloud-0 \
        --tags ceph_systemd \
        -e ceph_ansible_limit=ceph-2,controller-2,database-2,messaging-2 2>&1 && touch .ceph_ran_ceph-2,controller-2,database-2,messaging-2

echo "[$(date)] Finished ceph systemd units migration run for ceph-2,controller-2,database-2,messaging-2"
fi
RUN_ANSWER=""
if openstack overcloud upgrade run --help | grep -qe "--yes"; then
    RUN_ANSWER="--yes"
fi

echo "[$(date)] Started system upgrade step for ceph-2,controller-2,database-2,messaging-2"

tripleo-ansible-inventory --stack qe-Cloud-0 --static-yaml-inventory upgrade_inventory.yaml --undercloud-connection ssh --undercloud-key-file /var/lib/mistral/.ssh/tripleo-admin-rsa --ansible_ssh_user tripleo-admin

bkg_pids=""
for host in $(echo "ceph-2,controller-2,database-2,messaging-2" | sed "s/,/ /g")
do
  openstack overcloud upgrade run --yes \
          --stack qe-Cloud-0 \
          --static-inventory upgrade_inventory.yaml \
          --tags system_upgrade \
          --limit "${host}" 2>&1 | tee -a "RHEL_upgrade_${host}" &
  bkg_pids+=" $! "
done

status=0
for p in $bkg_pids; do
    if ! wait $p; then
        status=1
    fi
done

if [[ $status == 0 ]]; then
    echo "[$(date)] Finished system upgrade step for ceph-2,controller-2,database-2,messaging-2"
else
    echo "[$(date)] Failed in system upgrade step for ceph-2,controller-2,database-2,messaging-2"
    exit 1
fi

4-openstack_upgrade-batch_1.sh

#!/bin/env bash
#
# Run major upgrade on overcloud nodes
#
set -euo pipefail

source /home/stack/stackrc

if [ ! -f .system_upgrade_transfer_data ]; then

echo "[$(date)] Started upgrade transfer data for ceph-0,controller-0,database-0,messaging-0,networker-0"

openstack overcloud external-upgrade run --yes \
        --stack qe-Cloud-0 \
        --tags system_upgrade_transfer_data 2>&1 && touch .system_upgrade_transfer_data

echo "[$(date)] Finished upgrade transfer data for ceph-0,controller-0,database-0,messaging-0,networker-0"

echo "[$(date)] Setting up hybrid state for computes"
openstack overcloud upgrade run --yes \
        --stack qe-Cloud-0 \
        --playbook upgrade_steps_playbook.yaml \
        --tags nova_hybrid_state --limit all 2>&1
echo "[$(date)] Finished setting up hybrid state for computes"

fi

echo "[$(date)] Running major upgrade for ceph-0,controller-0,database-0,messaging-0,networker-0 hosts"

openstack overcloud upgrade run --yes \
        --stack qe-Cloud-0 \
        --limit ceph-0,controller-0,database-0,messaging-0,networker-0 --playbook all 2>&1

echo "[$(date)] Finished major upgrade for ceph-0,controller-0,database-0,messaging-0,networker-0 hosts"

5-openstack_upgrade-batch_2.sh

#!/bin/env bash
#
# Run major upgrade on overcloud nodes
#
set -euo pipefail

source /home/stack/stackrc

if [ ! -f .system_upgrade_transfer_data ]; then

echo "[$(date)] Started upgrade transfer data for ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0"

openstack overcloud external-upgrade run --yes \
        --stack qe-Cloud-0 \
        --tags system_upgrade_transfer_data 2>&1 && touch .system_upgrade_transfer_data

echo "[$(date)] Finished upgrade transfer data for ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0"

echo "[$(date)] Setting up hybrid state for computes"
openstack overcloud upgrade run --yes \
        --stack qe-Cloud-0 \
        --playbook upgrade_steps_playbook.yaml \
        --tags nova_hybrid_state --limit all 2>&1
echo "[$(date)] Finished setting up hybrid state for computes"

fi

echo "[$(date)] Running major upgrade for ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0 hosts"

openstack overcloud upgrade run --yes \
        --stack qe-Cloud-0 \
        --limit ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0 --playbook all 2>&1

echo "[$(date)] Finished major upgrade for ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0 hosts"

6-openstack_upgrade-batch_3.sh

#!/bin/env bash
#
# Run major upgrade on overcloud nodes
#
set -euo pipefail

source /home/stack/stackrc

if [ ! -f .system_upgrade_transfer_data ]; then

echo "[$(date)] Started upgrade transfer data for ceph-2,controller-2,database-2,messaging-2,ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0"

openstack overcloud external-upgrade run --yes \
        --stack qe-Cloud-0 \
        --tags system_upgrade_transfer_data 2>&1 && touch .system_upgrade_transfer_data

echo "[$(date)] Finished upgrade transfer data for ceph-2,controller-2,database-2,messaging-2,ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0"

echo "[$(date)] Setting up hybrid state for computes"
openstack overcloud upgrade run --yes \
        --stack qe-Cloud-0 \
        --playbook upgrade_steps_playbook.yaml \
        --tags nova_hybrid_state --limit all 2>&1
echo "[$(date)] Finished setting up hybrid state for computes"

fi

echo "[$(date)] Running major upgrade for ceph-2,controller-2,database-2,messaging-2,ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0 hosts"

openstack overcloud upgrade run --yes \
        --stack qe-Cloud-0 \
        --limit ceph-2,controller-2,database-2,messaging-2,ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0 --playbook all 2>&1

echo "[$(date)] Finished major upgrade for ceph-2,controller-2,database-2,messaging-2,ceph-1,controller-1,database-1,messaging-1,networker-1,ceph-0,controller-0,database-0,messaging-0,networker-0 hosts"