gistfile1.txt

# NOTE: Needs it's own venv with latest psycopg2

import os
import psycopg2
from psycopg2.extras import DictCursor
import json
import time
import datetime

SLEEP_SECONDS = 10

DATABASE_DSNS = {
    'master': 'dbname=frank',
}

OUTPUT_DIR = '/Users/frank/work/revsys/clients/cmg/'

HOST = 'logvipprd1.ddtc.cmgdigital.com'
PORT = 5959

HOSTMAP = {
    'master': 'pgldbaprd3',
}


class LogstashHandler(logging.handlers.DatagramHandler):

    def makePickle(self, record):
        msg = record.getMessage()
        data = json.loads(msg)
        data['@source_host'] = data['host']

        # Get the non @ fields
        data['@fields'] = [k for k in data if not k.startswith('@')]

        return json.dumps(data)


logger = logging.getLogger(__name__)
logger.addHandler(LogstashHandler(HOST, PORT))
logger.setLevel(logging.DEBUG)


def make_filename(name):
    time_string = datetime.datetime.now().strftime("%Y-%m-%d-hour-%H")
    return os.path.join(OUTPUT_DIR, '{0}-{1}.locks.log'.format(name, time_string))


class DTEncoder(json.JSONEncoder):

    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return str(obj)

        return json.JSONEncoder.default(self, obj)

while True:
    for name, dsn in DATABASE_DSNS.items():
        conn = psycopg2.connect(dsn)
        conn.autocommit = True

        cursor = conn.cursor(cursor_factory=DictCursor)

        cursor.execute("SELECT * FROM pg_locks, pg_stat_activity WHERE pg_locks.pid = pg_stat_activity.procpid;")

        with open(make_filename(name), 'w+') as output:
            run_time = str(datetime.datetime.now())

            for r in cursor:
                data = dict(r)
                data.update({
                    'version': 1,
                    'host': name,
                    '@timestamp': datetime.datetime.utcnow().isoformat() + 'Z'
                })
                json_data = json.dumps(data, cls=DTEncoder)
                logger.info(json_data)
            cursor.close()
        conn.close()
    time.sleep(SLEEP_SECONDS)