kolla-ansible galera recovery script

mariadb-recovery.sh

#!/usr/bin/env bash

inventory_file=${1:-'/provisioning/kolla/multinode.ini'}
hosts=${2:-'control1,compute1,compute2'}


ansible -i ${inventory_file} -a 'docker stop mariadb' ${hosts}

ansible -i ${inventory_file} -a 'cat /var/lib/docker/volumes/mariadb/_data/grastate.dat' ${hosts}

echo "Getting GTID's for all nodes:---------------------------------------------------------------------"
ansible -i ${inventory_file} -a 'docker start mariadb' ${hosts}
ansible -i ${inventory_file} -a 'docker exec -d mariadb mysqld --wsrep-recover' ${hosts}
ansible -i ${inventory_file} -a 'docker stop mariadb' ${hosts}
export last_sdn=$(ansible -m shell -i ${inventory_file} -a 'tail -n 100 /var/lib/docker/volumes/kolla_logs/_data/mariadb/mariadb.log | grep "Recovered position:" | awk "{print $7}" | awk -F":" "{print $2}" | sed -e "s/$/@$(hostname)/"' ${hosts} | grep "Recovered position:"  | awk '{print $7}' | awk -F ':' '{print $2}' | sort -t '@' -n -r -k1 | awk 'NR==1 {print $1}' | awk -F '@' '{print $2}')
export seqno=$(ansible -m shell -i ${inventory_file} -a 'tail -n 100 /var/lib/docker/volumes/kolla_logs/_data/mariadb/mariadb.log | grep "Recovered position:" | awk "{print $7}" | awk -F":" "{print $2}" | sed -e "s/$/@$(hostname)/"' ${hosts} | grep "Recovered position:"  | awk '{print $7}' | awk -F ':' '{print $2}' | sort -t '@' -n -r -k1 | awk 'NR==1 {print $1}' | awk -F '@' '{print $1}')
echo "SDN: ${last_sdn}"
echo "---------------------------------------------------------------------------------------------------"


echo "Installing pyyaml:---------------------------------------------------------------------------------"
ansible -i ${inventory_file} -a 'pip install pyyaml' ${hosts}
echo "Done installing pyyaml-----------------------------------------------------------------------------"
echo "Content before:------------------------------------------------------------------------------------"
ansible -i ${inventory_file} -a 'cat /var/lib/docker/volumes/mariadb/_data/grastate.dat' ${hosts}
echo "---------------------------------------------------------------------------------------------------"
echo "Altering file:-------------------------------------------------------------------------------------"
ansible -i ${inventory_file} -a 'python -c "import yaml; import io; import os; config=yaml.load(io.open(\"/var/lib/docker/volumes/mariadb/_data/grastate.dat\",\"r\", encoding=\"utf8\")); config[\"safe_to_bootstrap\"]=1; config[\"seqno\"]='${seqno}';yaml.dump(config, io.open(\"/var/lib/docker/volumes/mariadb/_data/grastate.dat\",\"w\", encoding=\"utf8\"), default_flow_style=False)"' ${last_sdn}
echo "Done altering file---------------------------------------------------------------------------------"
echo "Content after:-------------------------------------------------------------------------------------"
ansible -i ${inventory_file} -a 'cat /var/lib/docker/volumes/mariadb/_data/grastate.dat' ${hosts}
echo "---------------------------------------------------------------------------------------------------"
echo "Attempting to recover MariaDB Cluster:-------------------------------------------------------------"
ansible -i ${inventory_file} -a 'docker stop mariadb'  ${hosts}
kolla-ansible --verbose  -i ${inventory_file} mariadb_recovery
echo "Done."

Great job!

Many thanks, save my life.