Low tech Flask logging with UDP, JSON, and Twisted.

logscoop.py

import os
import subprocess

from sqlalchemy import Column, String, Integer, Float, MetaData, Table
from sqlalchemy import create_engine

from alchimia import TWISTED_STRATEGY

from flask import json

from twisted.internet.protocol import DatagramProtocol
from twisted.internet.defer import inlineCallbacks

from .utils import EnvironmentConfigurator

conf = EnvironmentConfigurator()\
    .add('SQLA_STATSDB_URI',
        'postgresql://statsdb:statsdb@127.0.0.1:5432/statsdb')\
    .add('UDPLOG_HOST', '127.0.0.1')\
    .add('UDPLOG_PORT', 4551)

metadata = MetaData()
log_table = Table('log', metadata,
    Column('id', String(36), primary_key=True, nullable=False),
    Column('version', Integer, nullable=False),
    Column('ts', Float, nullable=False),
    Column('kind', String, nullable=False),
    Column('event', String, nullable=False),
    Column('extra', String, nullable=False),
    Column('hostname', String, nullable=False),
    Column('env', String(20), nullable=False),
    Column('user_id', Integer),
    Column('guest_id', String(36)),
    Column('ip', String),
    Column('path', String),
    Column('method', String(8)),
    Column('args', String),
)

class LogServer(DatagramProtocol):

    def __init__(self, engine):
        self.engine = engine

    @inlineCallbacks
    def datagramReceived(self, data, (host, port)):
        data = json.loads(data)
        yield self.engine.execute(log_table.insert().values(
            id=data['id'],
            version=data['version'],
            ts=data['ts'],
            kind=data['kind'],
            event=json.dumps(data['event']),
            extra='{}',  # Empty for now
            hostname=data['hostname'],
            env=data['env'],
            user_id=data['user_id'],
            guest_id=data['guest_id'],
            ip=data['ip'],
            path=data['path'],
            method=data['method'],
            args=json.dumps(data['args']),
        ))

def main(reactor):
    serr = sout = open(os.devnull, 'w')
    cmd = lambda c: subprocess.call(c, stderr=serr, stdout=sout, shell=True)
    cmd('createuser -h 127.0.0.1 -s statsdb')
    cmd('createdb -h 127.0.0.1 -T template0 -E UTF-8 -O statsdb statsdb')
    metadata.create_all(create_engine(conf.get('SQLA_STATSDB_URI')))

    engine = create_engine(conf.get('SQLA_STATSDB_URI'), reactor=reactor,
                           strategy=TWISTED_STRATEGY)

    reactor.listenUDP(conf.get('UDPLOG_PORT'), LogServer(engine))
    reactor.run()

if __name__ == '__main__':
    from twisted.internet import reactor as default_reactor
    main(default_reactor)

udplog.py

import socket
import time
import uuid

from flask import json, session, request

from .utils import in_request

class Kind(object):
    
    def __init__(self, kind, fields=None):
        if not self.kind_valid(kind):
            raise ValueError(
                'Kind must be of the form OBJECT_ACTION (got %r)' % (kind,))
        self.kind = kind
        self.fields = fields or []

    def kind_valid(self, kind):
        split = kind.split('_')
        if len(split) != 2:
            return False
        if '_' in split[0]:
            return False
        if '_' in split[1]:
            return False
        return True

    def validate(self, event):
        for field in self.fields:
            if field not in event:
                raise ValueError('Must include %r field in event for %s' % (
                    field,
                    self.kind,
                ))


class LogKinds(object):
    PAGE_VIEW = Kind('page_view')
    CAMPAIGN_APPROVE = Kind('campaign_approve', ['campaign_id'])
    INFO_DEBUG = Kind('info_debug', ['data'])


class UDPLog(object):

    def __init__(self, app, sentry):
        self.app = app
        self.sentry = sentry
        self.hostname = socket.gethostname()
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    
    def log(self, kind, event=None, ts=None):
        if ts is None:
            ts = time.time()
        if event is None:
            event = {}
        if not isinstance(kind, Kind):
            raise ValueError('You must log only predefined kinds of logs.')
        kind.validate(event)
        data = {
            'id': str(uuid.uuid1()),
            'version': 1,
            'ts': ts,
            'kind': kind.kind,
            'event': event,
            'hostname': self.hostname,
            'env': self.app.config['UDPLOG_ENV'],
        }
        if in_request():
            data.update({
                'user_id': session.get('uid'),
                'guest_id': session.get('guest_id'),
                'ip': request.headers.get('X-Forwarded-For'),
                'path': request.path,
                'method': request.method,
                'args': request.args,
            })
        else:
            data.update({
                'user_id': None,
                'guest_id': None,
                'ip': None,
                'path': None,
                'method': None,
                'args': None,
            })
        encoded = json.dumps(data)
        try:
            self.socket.sendto(encoded, (
                self.app.config['UDPLOG_HOST'],
                self.app.config['UDPLOG_PORT'],
            ))
        except (SystemExit, KeyboardInterrupt):
            raise
        except:
            self.sentry.captureException()

usage.py

from flask import Flask, render_template, redirect, request

from raven.contrib.flask import Sentry

from .udplog import UDPLog, LogKinds
from .database import approve_campaign


app = Flask(__name__)
app.config['SENTRY_DSN'] = '...'
app.config['UDPLOG_ENV'] = 'staging'
app.config['UDPLOG_HOST'] = '127.0.0.1'
app.config['UDPLOG_PORT'] = 4551

sentry = Sentry(app)

udplog = UDPLog(app, sentry)

@app.route('/faq')
def faq():
    udplog.log(LogKinds.PAGE_VIEW)
    return render_template('faq.html')

@app.route('/campaign')
def campaign():
    udplog.log(LogKinds.PAGE_VIEW)
    if request.method == 'POST':
        campaign_id = request.form.get('campaign_id')
        approve_campaign(campaign_id)
        udplog.log(LogKinds.CAMPAIGN_APPROVE, {'campaign_id': campaign_id})
        return redirect(request.path)
    return render_template('campaign.html')