Kolla Ansible Automaticly Restart Agent Down

README.md

General

Automaticly restart docker container of Openstack Agent (network agent, compute service, volume service) when goes to XXX or DOWN state.

Network Agent

nano netagent.sh

#!/bin/bash
# ExportOS
export OS_PROJECT_DOMAIN_NAME=Default
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_NAME=admin
export OS_TENANT_NAME=admin
export OS_USERNAME=admin
export OS_PASSWORD=$PASSWORD
export OS_AUTH_URL=$URL
export OS_INTERFACE=internal
export OS_ENDPOINT_TYPE=internalURL
export OS_IDENTITY_API_VERSION=3
export OS_REGION_NAME=RegionOne
export OS_AUTH_PLUGIN=password
export OS_CACERT=/etc/kolla/certificates/ca/root.crt

# Mengambil daftar nama container dan host
containers=$(openstack network agent list --long | grep 'XXX' | awk '$15 ~ /^neutron/ {gsub("-","_",$15); print $15,$7} $16 ~ /^neutron/ {gsub("-","_",$16); print $16,$8}')
# Looping melalui setiap host yang ditemukan
for host in $(echo "$containers" | awk '{print $2}' | sort -u)
do
  # Memproses container pada host yang sedang diproses
  echo "Processing containers on host: $host"
  echo "$containers" | awk -v host="$host" '$2 == host {print $1}' | while read container
  do
    echo "Restarting container: $container"
    # Restart container pada host yang sedang diproses menggunakan SSH
    ssh $host "docker restart $container"
  done
done

chmod +x netagent.sh

Compute Service

nano computeagent.sh

#!/bin/bash
# ExportOS
export OS_PROJECT_DOMAIN_NAME=Default
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_NAME=admin
export OS_TENANT_NAME=admin
export OS_USERNAME=admin
export OS_PASSWORD=$PASSWORD
export OS_AUTH_URL=$URL
export OS_INTERFACE=internal
export OS_ENDPOINT_TYPE=internalURL
export OS_IDENTITY_API_VERSION=3
export OS_REGION_NAME=RegionOne
export OS_AUTH_PLUGIN=password
export OS_CACERT=/etc/kolla/certificates/ca/root.crt

# Mengambil daftar service yang state-nya down dan host
services=$(openstack compute service list --long | awk '$12 == "down" && $4 ~ /^nova/ {gsub("-","_",$4); print $4,$6}')

# Looping melalui setiap host yang ditemukan
for host in $(echo "$services" | awk '{print $2}' | sort -u)
do
  # Memproses service pada host yang sedang diproses
  echo "Processing services on host: $host"
  echo "$services" | awk -v host="$host" '$2 == host {print $1}' | while read service
  do
    echo "Restarting service: $service"
    # Restart service pada host yang sedang diproses menggunakan SSH
    ssh $host "docker restart $service"
  done
done

chmod +x computeagent.sh

Volume Service

• Change NODE1 & NODE2 to your hostname of server who have @lvm1 name to avoid failed ssh with @ hostname.

nano cinderagent.sh

#!/bin/bash
# ExportOS
export OS_PROJECT_DOMAIN_NAME=Default
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_NAME=admin
export OS_TENANT_NAME=admin
export OS_USERNAME=admin
export OS_PASSWORD=$PASSWORD
export OS_AUTH_URL=$URL
export OS_INTERFACE=internal
export OS_ENDPOINT_TYPE=internalURL
export OS_IDENTITY_API_VERSION=3
export OS_REGION_NAME=RegionOne
export OS_AUTH_PLUGIN=password
export OS_CACERT=/etc/kolla/certificates/ca/root.crt

# Mengambil daftar nama volume service dan host
services=$(openstack volume service list --long | awk '/down/ {gsub("-","_",$2); print $2,$4}' | sort -u | sed 's/\(NODE1\)@.*/\1/; s/\(NODE2\)@.*/\1/')
# Looping melalui setiap host yang ditemukan
for host in $(echo "$services" | awk '{print $2}' | sort -u)
do
  # Memproses service pada host yang sedang diproses
  echo "Processing services on host: $host"
  echo "$services" | awk -v host="$host" '$2 == host {print $4}' | while read service
  do
    echo "Restarting volume service: $service"
    # Restart volume service pada host yang sedang diproses menggunakan SSH
    ssh $host "docker restart $service"
  done
done

chmod +x cinderagent.sh

Point on Crontab

crontab -e
* * * * * /root/netagent.sh
* * * * * /root/computeagent.sh
* * * * * /root/cinderagent.sh