Some simple Ubuntu server monitoring tools

.gitignore

*.pyc
conf.py

readme.md

Settings

Create a conf.py containing these:

SERVER_NAME = 'My Server'
EMAIL_HOST = 'localhost'
FROM_EMAIL = 'root@myserver.com'
RECIPIENTS = ['me@example.com']
LOG_DIR = '~/log'
URLS = ('mydomain.com', 'myotherdomain.com/testurl')

Optional settings:

WEB_TIMEOUT (default is 10)

Usage

Add these to your crontab:

*          *  *   *   *     free -m | /path/to/memory_monitor.py
45         *  *   *   *     df -h | /path/to/disk_monitor.py
15,35,55   *  *   *   *     /path/to/web_monitor.py

disk_monitor.py

#!/usr/bin/env python
import sys, smtplib, os
from datetime import datetime

import conf




table = sys.stdin.read()
lines = [[cell for cell in line.split(' ') if cell]
         for line in table.split("\n")]

# get percentage usage and gb free
if len(lines[1]) >= 3:
    usage = int(lines[1][4].replace('%', ''))
    free = lines[1][3]
else:
    usage = int(lines[2][3].replace('%', ''))
    free = lines[2][2]

f = open(os.path.join(conf.LOG_DIR, 'disk.log'), 'a+')
f.write('%s%% used %s free - %s\r\n' % (usage, free, datetime.now().strftime('%Y-%m-%d %H:%M')))
f.close()

if usage > getattr(conf, 'MAX_DISK_USAGE', 90):
        # ALARM! ALARM!
        msg = "Subject: LOW DISK SPACE on " + conf.SERVER_NAME + "\n"
	msg += "\n"
        msg += table

        # get process status
        for dir in getattr(conf, 'DISK_CHECK_DIRS', []):
            f = os.popen('du -s %s | sort -k 1 -r -n' % dir)
            msg += "\n\n%s usage:\n\n" % dir
            msg += f.read()
            msg += "\n\n"

	s = smtplib.SMTP(conf.EMAIL_HOST)
	s.sendmail(conf.FROM_EMAIL,
                   conf.RECIPIENTS,
                   msg)
        sys.exit(1)

sys.exit(0)

memory_monitor.py

#!/usr/bin/env python
import sys, smtplib, os
from datetime import datetime

import conf



current_free = lambda t: int(t[2][3])

table = sys.stdin.read()
lines = [[cell for cell in line.split(' ') if cell]
         for line in table.split("\n")]

free = current_free(lines)

#print lines, free

f = open(os.path.join(conf.LOG_DIR, 'memory.log'), 'a+')
f.write('%sM - %s\r\n' % (free, datetime.now().strftime('%Y-%m-%d %H:%M')))
f.close()

if free < getattr(conf, 'MINIMUM_FREE_MEM', 100):
        # ALARM! ALARM!
        msg = "Subject: LOW MEM on " + conf.SERVER_NAME + "\n"
	msg += "\n"
        msg += table

        # get process status
        f = os.popen('ps -eo pmem,pcpu,rss,vsize,args | sort -k 1 -r -n')
        msg += "\n\nProcesses:\n\n"
        msg += f.read()
        msg += "\n\n"

	s = smtplib.SMTP(conf.EMAIL_HOST)
	s.sendmail(conf.FROM_EMAIL,
                   conf.RECIPIENTS,
                   msg)
        sys.exit(1)

sys.exit(0)

web_monitor.py

#!/usr/bin/env python
from datetime import datetime
import httplib
import smtplib
import socket
import os
import time

import conf

WEB_TIMEOUT = getattr(conf, 'WEB_TIMEOUT', 10)
MAX_ATTEMPTS = 2

socket.setdefaulttimeout(WEB_TIMEOUT)

def get_status_code(host, path="/", attempts=0):
    """ This function retreives the status code of a website by requesting
        HEAD data from the host. This means that it only requests the headers.
        If the host cannot be reached or something else goes wrong, it returns
        None instead.
    """
    try:
        conn = httplib.HTTPConnection(host)
        conn.request("HEAD", path)
        status = conn.getresponse().status
        conn.close()
        if status == 405:
            conn = httplib.HTTPConnection(host)
            conn.request("GET", path)
            status = conn.getresponse().status
            conn.close()
    except socket.timeout, e:
        attempts += 1
        if attempts < MAX_ATTEMPTS:
            # pause then try again
            time.sleep(WEB_TIMEOUT)
            return get_status_code(host, path, attempts)
        else:
            return '%s (%s attempts)' % (e, attempts)
    except (StandardError, socket.gaierror, socket.error), e:
        return e
    else:
        return status

f = open(os.path.join(conf.LOG_DIR, 'web.log'), 'a+')
dead = []
for u in conf.URLS:
    domain = u.split('/')[0]
    path = '/' + '/'.join(u.split('/')[1:])
    status = get_status_code(domain, path)
    if status not in (200, '[Errno -3] Temporary failure in name resolution', '[Errno -2] Name or service not known'):
         dead.append((u, status)) 

    f.write('%s %s Status:%s\r\n' % (datetime.now().strftime('%Y-%m-%d %H:%M'), u, status))

f.close()

socket.setdefaulttimeout(30)

if len(dead):
    # ALARM! ALARM!
    msg = "Subject: WEB SERVER DOWN - %s\n" % ', '.join([d[0] for d in dead])
    msg += "\n"
    for d in dead:
        msg += "%s Status: %s\n" % d

    s = smtplib.SMTP(conf.EMAIL_HOST)
    s.sendmail(conf.FROM_EMAIL,
               conf.RECIPIENTS,
               msg)