Troublshooting tools

create ssh config.sh

#! /usr/bin/env python

import re
from os import path, listdir
from os.path import isfile, join

config_path = path.abspath(path.expanduser('~') + '/.ssh/config')
template = 'Host {}\n\tStrictHostKeyChecking no\n\tHostName {}\n\tUser {}\n\n'
begin = '#WRW HOST VARS - START'
end = '#WRW HOST VARS - END'

def readHosts(hosts_dir) :

	hosts = []
	for filename in listdir(hosts_dir):
		if filename == '.DS_Store':
			continue
		
		file = open(join(hosts_dir, filename), 'r')
		lines = file.read()

		pattern = re.compile('^ansible_ssh_user:(.+?)$|^ansible_ssh_host:(.+?)$', flags=re.MULTILINE)
		matches = pattern.findall(lines)

		hostname = filename
		ipaddress = matches[0][1].strip()
		username = matches[1][0].strip()


		hosts.append((hostname, ipaddress, username))
		file.close()

	return hosts

def writeConfig(hosts):
	global template, config_path, begin, end
	
	config = open(config_path, 'r+')

	# Fetch existing content
	content = config.read()
	match = re.search(begin + '(.*)' + end, content, re.DOTALL)
	if match:
		 content = content.replace(match.group(0), '')

	config.close()

	# append new content
	content += begin
	content += '\n\n'

	for host in hosts:
		content += template.format(host[0], host[1], host[2])

	content += end

	# Write final content
	config = open(config_path, 'w')
	config.write(content)
	config.close()
	print 'Hosts added successfully'

hosts_dir1 = './inventory/scrapers/host_vars'
hosts1 = readHosts(hosts_dir1)


hosts_dir2 = './inventory/proxies/host_vars'
hosts2 = readHosts(hosts_dir2)

hosts = hosts1 + hosts2

writeConfig(hosts)

troubleshoot

#!/bin/bash          
printf "Which servers are we troubleshooting today?\n"
read servers

printf "\nOkay, I'm going to start dealing with them one by one\n"

ssh_timeout=5
servers_provision=()
servers_error=()
servers_check_proxies=()

join() {
    # $1 is return variable name
    # $2 is sep
    # $3... are the elements to join
    local retname=$1 sep=$2 ret=$3
    shift 3 || shift $(($#))
    printf -v "$retname" "%s" "$ret${@/#/$sep}"
}

for server in $servers; do
	status=$(tugboat info $server -a status)

	echo "Treating '$server'"

	if [[ "$status" =~ "off" ]]; then
		echo "Server is off"
	elif [[ "$status" =~ "active" ]]; then
		supervisor_info=`(ssh -o ConnectTimeout=$ssh_timeout $server 'supervisorctl status') 2>&1`;

		if [[ "$supervisor_info" =~ "unix:///var/run/supervisor.sock" ]]; then
			x=$(ssh $server 'supervisord -c /etc/supervisor/supervisord.conf')
			echo "Supervisor reloaded - sock"

		elif [[ "$supervisor_info" =~ "nucleus_scraper.py:nucleus_proc0 RUNNING" ]]; then
			echo "Nucleus is running. Check proxies. I'll provision anyways."
			servers_check_proxies+=("$server")
			servers_provision+=("$server")

		elif [[ "$supervisor_info" =~ "nucleus_scraper.py:nucleus_proc0" ]]; then
			echo "Nucleus is NOT running. I'll provision."
			servers_provision+=("$server")

		elif [[ "$supervisor_info" =~ "Connection timed out during banner exchange" ]]; then
			echo "Rebuilding this server"
			x=$(tugboat rebuild $server ubuntu-14-04-x64 --confirm -q)
			# tugboat rebuild $server ubuntu-14-04-x64 --confirm
			servers_provision+=("$server")
		else
			echo "I don't know what to do with this one - $supervisor_info"
			servers_error+=("$server")
		fi
	else
		echo "Error getting info from digitalocean"
	fi
	
	printf "\n"

done


if [[ ${#servers_provision[@]} > 0 ]]; then
	echo "I'll wait for 60s in order to allow rebuilt servers to complete before provisioning."
	sleep 60
	
	join sss "," "${servers_provision[@]}"
	echo "ansible-playbook -i inventory/scrapers/hosts nucleus-provision.yml --ask-vault-pass --private-key ~/.ssh/wrw-default.pem --limit $sss"
	ansible-playbook -i inventory/scrapers/hosts nucleus-provision.yml --ask-vault-pass --private-key ~/.ssh/wrw-default.pem --limit $sss
fi

if [[ ${#servers_error[@]} > 0 ]]; then
	join IFS "," "${servers_error[@]}"
	echo "The following servers have one issue or the other - $IFS"
fi

if [[ ${#servers_check_proxies[@]} > 0 ]]; then
	join sss "," "${servers_check_proxies[@]}"
	echo "You should check the proxies for the following servers - $sss"
fi