Skip to content

Instantly share code, notes, and snippets.

@madflojo
Last active February 2, 2024 19:42
Show Gist options
  • Save madflojo/95fe54b4e42ff01e1ba1 to your computer and use it in GitHub Desktop.
Save madflojo/95fe54b4e42ff01e1ba1 to your computer and use it in GitHub Desktop.
Self Healing Example Script
#!/usr/bin/python
import sys
import yaml
import rethinkdb as r
from rethinkdb.errors import RqlDriverError, RqlRuntimeError
import socket
import time
# Load Configuration
# ------------------------------------------------------------------
if len(sys.argv) < 2:
print("Hey, thats not how you launch this...")
print("%s <config file>") % sys.argv[0]
sys.exit(1)
# Open Config File and Parse Config Data
configfile = sys.argv[1]
cfh = open(configfile, "r")
config = yaml.safe_load(cfh)
cfh.close()
def callSaltRestart(config):
''' Call Saltstack to restart a service '''
import requests
url = config['salt_url'] + "/services/restart"
headers = {
"Accept:" : "application/json"
}
postdata = {
"tgt" : "db*",
"matcher" : "glob",
"args" : "rethinkdb",
"secretkey" : config['salt_key']
}
try:
req = requests.post(url=url, headers=headers, data=postdata, verify=False)
print("Called for help and got response code: %d") % req.status_code
if req.status_code == 200:
return True
else:
return False
except (requests.exceptions.RequestException) as e:
print("Error calling for help: %s") % e.message
return False
def callSaltHighstate(config):
''' Call Saltstack to initiate a highstate '''
import requests
url = config['salt_url'] + "/states/highstate"
headers = {
"Accept:" : "application/json"
}
postdata = {
"tgt" : "db*",
"matcher" : "glob",
"secretkey" : config['salt_key']
}
try:
req = requests.post(url=url, headers=headers, data=postdata, verify=False)
print("Called for help and got response code: %d") % req.status_code
if req.status_code == 200:
return True
else:
return False
except (requests.exceptions.RequestException) as e:
print("Error calling for help: %s") % e.message
return False
# Set initial values
connection_attempts = 0
first_connect = 0.00
last_restart = 0.00
last_highstate = 0.00
connected = False
called = None
# Retry RethinkDB Connections until successful
while connected == False:
if first_connect == 0.00:
first_connect = time.time()
# RethinkDB Server
try:
rdb_server = r.connect(
host=config['rethink_host'], port=config['rethink_port'],
auth_key=config['rethink_authkey'], db=config['rethink_db'])
connected = True
print("Connected to RethinkDB")
except (RqlDriverError, socket.error) as e:
print("Cannot connect to rethinkdb")
print("RethinkDB Error: %s") % e.message
timediff = time.time() - first_connect
if timediff > 300.00:
last_timediff = time.time() - last_restart
if last_timediff > 600.00 or last_restart == 0.00:
if timediff > 600:
callSaltRestart(config)
last_restart = time.time()
last_timediff = time.time() - last_highstate
if last_timediff > 300.00 or last_highstate == 0.00:
callSaltHighstate(config)
last_highstate = time.time()
connection_attempts = connection_attempts + 1
print("RethinkDB connection attempts: %d") % connection_attempts
time.sleep(60)
# Get user count
try:
result = r.table('users').count().run(rdb_server)
except (RqlDriverError, RqlRuntimeError) as e:
print("Got error while performing query: %s") % e.message
print("Exiting...")
sys.exit(1)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment