We used this guy to ensure that our webservers and databases were running,and if things were replicating as they should. In case of an "event", the script would email everyone and throw up a maintenance page until it was fixed.

python_pinger.py

#!/usr/bin/python
#@note python pinger::Python::python-pinger
import os
import sys
import re
import smtplib
import MySQLdb
from email.MIMEText import MIMEText

webservers = ["192.168.0.1", "192.168.0.2"]
mysqlservers = ["192.168.0.3", "192.168.0.4"]
search = re.compile(r"(\d) received")

def mailit(msg,sub):
    from_addr = 'root@localhost'
    to_addr = ['foo@bar.com']
    msg = MIMEText(msg)
    msg['Subject'] = sub
    msg['From'] = from_addr
    
    # mail it out
    smtp = smtplib.SMTP()
    smtp.connect()
    smtp.sendmail( from_addr, to_addr, msg.as_string())
    smtp.close()

for ip in webservers:
    cmd = "ping -q -c2 " +ip
    ping = os.popen(cmd)
    web_offline = "/home/foo/weboffline_" + ip
    while 1:
        resp = ping.readline()
        if not resp: break
        packets = re.findall(search,resp)
        if (packets and int(packets[0]) == 0 and os.path.isfile(web_offline) == False):
            open(web_offline,'wt')
            msg = "It appears web server %s is down - please check right away" % ip
            sub = 'URGENT: SERVER %s is down' % ip
            mailit( msg, sub)
        elif( os.path.isfile(web_offline) ):
            os.remove( web_offline )
            msg = "Web server %s is back up" % ip
            sub = "SERVER %s is back up" % ip
            mailit( msg, sub )

            
# Check MySQL database connections
for host in mysqlservers:
    my_offline = "/home/foo/myoffline_" + host
    try:
        conn = MySQLdb.connect (host, user = "root", passwd = "", db = "bar")
    if( os.path.isfile( my_offline ) ):
        os.remove( my_offline )
        msg = "MySQL SERVER %s is back up" % host
        sub = "MYSQL SERVER %s is back up" % host
        mailit( msg, sub )
    except MySQLdb.Error:
        msg = "It appears mysql server %s is down - please check right away. " % host    
        
        # Copy over the static file
    if( os.path.isfile( my_offline ) == False ):
                open( my_offline,'wt')
        if ( host == "192.168.0.1" or host == "192.168.0.2"):
                    msg += "The /home page has been replaced with a static home page.  This will be fixed when the MySQL database is fixed"
 
            # Mail Someone
            sub = 'URGENT: MYSQL SERVER %s is down' % host
            mailit( msg, sub)            

# Check to be sure MySQL replication is running
cmd = "mysql -u root -h 192.168.0.3 < check_slave_status.sql"
runit = os.popen(cmd)
search = re.compile( 'Slave_IO_Running:\s(.*)\n')
while 1:        
    resp = runit.readline()        
    if not resp: break        
    p = re.findall(search,resp)        
    if p:
        if( p[0] != 'Yes' and os.path.isfile('/home/foo/my_replication_offline')):
            msg = "It appears that mysql replication has crashed out between db1 and db2"
            open( '/root/my_replication_offline', 'wt' )
            sub = 'URGENT: MYSQL REPLICATION IS DOWN'
            mailit( msg, sub )
        elif( os.path.isfile( '/home/foo/my_replication_offline' ) != False ):
            os.remove( '/home/foo/my_replication_offline' )
            msg = "MySQL replication has been restored"
            sub = 'MySQL REPLICATION RESTORED'
            mailit( msg, sub )