Last active
December 28, 2015 02:39
-
-
Save ericflo/7429453 to your computer and use it in GitHub Desktop.
Low tech Flask logging with UDP, JSON, and Twisted.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import subprocess | |
from sqlalchemy import Column, String, Integer, Float, MetaData, Table | |
from sqlalchemy import create_engine | |
from alchimia import TWISTED_STRATEGY | |
from flask import json | |
from twisted.internet.protocol import DatagramProtocol | |
from twisted.internet.defer import inlineCallbacks | |
from .utils import EnvironmentConfigurator | |
conf = EnvironmentConfigurator()\ | |
.add('SQLA_STATSDB_URI', | |
'postgresql://statsdb:statsdb@127.0.0.1:5432/statsdb')\ | |
.add('UDPLOG_HOST', '127.0.0.1')\ | |
.add('UDPLOG_PORT', 4551) | |
metadata = MetaData() | |
log_table = Table('log', metadata, | |
Column('id', String(36), primary_key=True, nullable=False), | |
Column('version', Integer, nullable=False), | |
Column('ts', Float, nullable=False), | |
Column('kind', String, nullable=False), | |
Column('event', String, nullable=False), | |
Column('extra', String, nullable=False), | |
Column('hostname', String, nullable=False), | |
Column('env', String(20), nullable=False), | |
Column('user_id', Integer), | |
Column('guest_id', String(36)), | |
Column('ip', String), | |
Column('path', String), | |
Column('method', String(8)), | |
Column('args', String), | |
) | |
class LogServer(DatagramProtocol): | |
def __init__(self, engine): | |
self.engine = engine | |
@inlineCallbacks | |
def datagramReceived(self, data, (host, port)): | |
data = json.loads(data) | |
yield self.engine.execute(log_table.insert().values( | |
id=data['id'], | |
version=data['version'], | |
ts=data['ts'], | |
kind=data['kind'], | |
event=json.dumps(data['event']), | |
extra='{}', # Empty for now | |
hostname=data['hostname'], | |
env=data['env'], | |
user_id=data['user_id'], | |
guest_id=data['guest_id'], | |
ip=data['ip'], | |
path=data['path'], | |
method=data['method'], | |
args=json.dumps(data['args']), | |
)) | |
def main(reactor): | |
serr = sout = open(os.devnull, 'w') | |
cmd = lambda c: subprocess.call(c, stderr=serr, stdout=sout, shell=True) | |
cmd('createuser -h 127.0.0.1 -s statsdb') | |
cmd('createdb -h 127.0.0.1 -T template0 -E UTF-8 -O statsdb statsdb') | |
metadata.create_all(create_engine(conf.get('SQLA_STATSDB_URI'))) | |
engine = create_engine(conf.get('SQLA_STATSDB_URI'), reactor=reactor, | |
strategy=TWISTED_STRATEGY) | |
reactor.listenUDP(conf.get('UDPLOG_PORT'), LogServer(engine)) | |
reactor.run() | |
if __name__ == '__main__': | |
from twisted.internet import reactor as default_reactor | |
main(default_reactor) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import socket | |
import time | |
import uuid | |
from flask import json, session, request | |
from .utils import in_request | |
class Kind(object): | |
def __init__(self, kind, fields=None): | |
if not self.kind_valid(kind): | |
raise ValueError( | |
'Kind must be of the form OBJECT_ACTION (got %r)' % (kind,)) | |
self.kind = kind | |
self.fields = fields or [] | |
def kind_valid(self, kind): | |
split = kind.split('_') | |
if len(split) != 2: | |
return False | |
if '_' in split[0]: | |
return False | |
if '_' in split[1]: | |
return False | |
return True | |
def validate(self, event): | |
for field in self.fields: | |
if field not in event: | |
raise ValueError('Must include %r field in event for %s' % ( | |
field, | |
self.kind, | |
)) | |
class LogKinds(object): | |
PAGE_VIEW = Kind('page_view') | |
CAMPAIGN_APPROVE = Kind('campaign_approve', ['campaign_id']) | |
INFO_DEBUG = Kind('info_debug', ['data']) | |
class UDPLog(object): | |
def __init__(self, app, sentry): | |
self.app = app | |
self.sentry = sentry | |
self.hostname = socket.gethostname() | |
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) | |
def log(self, kind, event=None, ts=None): | |
if ts is None: | |
ts = time.time() | |
if event is None: | |
event = {} | |
if not isinstance(kind, Kind): | |
raise ValueError('You must log only predefined kinds of logs.') | |
kind.validate(event) | |
data = { | |
'id': str(uuid.uuid1()), | |
'version': 1, | |
'ts': ts, | |
'kind': kind.kind, | |
'event': event, | |
'hostname': self.hostname, | |
'env': self.app.config['UDPLOG_ENV'], | |
} | |
if in_request(): | |
data.update({ | |
'user_id': session.get('uid'), | |
'guest_id': session.get('guest_id'), | |
'ip': request.headers.get('X-Forwarded-For'), | |
'path': request.path, | |
'method': request.method, | |
'args': request.args, | |
}) | |
else: | |
data.update({ | |
'user_id': None, | |
'guest_id': None, | |
'ip': None, | |
'path': None, | |
'method': None, | |
'args': None, | |
}) | |
encoded = json.dumps(data) | |
try: | |
self.socket.sendto(encoded, ( | |
self.app.config['UDPLOG_HOST'], | |
self.app.config['UDPLOG_PORT'], | |
)) | |
except (SystemExit, KeyboardInterrupt): | |
raise | |
except: | |
self.sentry.captureException() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from flask import Flask, render_template, redirect, request | |
from raven.contrib.flask import Sentry | |
from .udplog import UDPLog, LogKinds | |
from .database import approve_campaign | |
app = Flask(__name__) | |
app.config['SENTRY_DSN'] = '...' | |
app.config['UDPLOG_ENV'] = 'staging' | |
app.config['UDPLOG_HOST'] = '127.0.0.1' | |
app.config['UDPLOG_PORT'] = 4551 | |
sentry = Sentry(app) | |
udplog = UDPLog(app, sentry) | |
@app.route('/faq') | |
def faq(): | |
udplog.log(LogKinds.PAGE_VIEW) | |
return render_template('faq.html') | |
@app.route('/campaign') | |
def campaign(): | |
udplog.log(LogKinds.PAGE_VIEW) | |
if request.method == 'POST': | |
campaign_id = request.form.get('campaign_id') | |
approve_campaign(campaign_id) | |
udplog.log(LogKinds.CAMPAIGN_APPROVE, {'campaign_id': campaign_id}) | |
return redirect(request.path) | |
return render_template('campaign.html') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment