hercules.py

# _*_ coding:utf-8 _*_
# 1. 遍历httplogs, 读取到每一条doc
# 2. 根据doc的 url + method 判断该发那个埋点 传入到转发器
# 3. 转发器根据不同情况的埋点生成一个eventObj出来
# 4. save to db add one del one
from pymongo import MongoClient
import datetime
import time
import jwt
import IP
from bson.objectid import ObjectId

httplogDB = MongoClient('10.8.8.111', 27017)['koala_dev']
eventDB = MongoClient('10.8.8.111', 27017)['koala_dev']

todayDateStr = time.strftime("%Y%m%d", time.localtime())

httplogsCollectionName = "usefulHttpLog" + todayDateStr
httplogs = httplogDB[httplogsCollectionName]

eventsCollectionName = "backendEvents" + todayDateStr
events = eventDB[eventsCollectionName]
print("1. connected to collection %s %s")%(httplogsCollectionName, eventsCollectionName)

# ronfe's map dict
httpMapEvents = [
    {
        "origin": {
            "method": "POST",
            "route": "/signup"
        },
        "track": {
            "eventKey": "signupSuccess",
            "eventValue": { "userId": None, "channel": "req.body.channel" }
        }
    },
]

# deocde token
# @ input token
# @ output userid and user role
def parseToken(token):
    secret = "follow your heart&intuition"
    thisToken = token.split(' ')[-1]
    decode = {}
    decode = jwt.decode(thisToken, secret)
    return decode

def saveToDb(eventObj, httplogId):
    events.insert_one(eventObj)
    httplogs.remove({"_id": httplogId})

def parseEventValue(trackEvent, eventObj, httplog):
    request = httplog['request']
    response = httplog['response']

    if eventObj['eventKey'] == "signupSuccess":
        trackEvent['eventValue']['userId'] = eventObj['user']
        trackEvent['eventValue']['channel'] = None
        return trackEvent['eventValue']
    # return {}

# 初始化埋点信息
def generatorEvent(doc, trackEvent):
    eventObj = {
        "eventKey": "",
        "eventValue": {},
        "ua": "",
        "url": "",
        "category": "",
        "platform": "",
        "ip": "",
        "location": "",
        "user": "",
        "role": "",
        "eventTime": "",
        "serverTime": ""
    }
    try:
        event = trackEvent['eventKey']
        eventObj['eventKey'] = event
        eventObj['ua'] = doc['ua']                  # httplog ua filed
        eventObj['url'] = doc['url']                # httplog url filed
        eventObj['category'] = doc['apptag']        # 没法找？ parse url有可能拿到
        eventObj['platform'] = "backend"            # 一律backend还是根据不同的apptag传不同的platform
        if doc['ip'] != None:
            eventObj['ip'] = doc['ip']                  # read from httplog this.request.ip
            eventObj['location'] = IP.find(doc['ip'])# how 2?
        if doc['token'] != None:
            tokenObj = parseToken(doc['token'])
            eventObj['user'] = ObjectId(tokenObj['id'])  # read from httplog
            eventObj['role'] = tokenObj['role']          # read from httplog
        eventObj['eventTime'] = doc['eventTime']                   # http2mongo实时写入unixtimestamp
        eventObj['serverTime'] = doc['serverTime']                 # read from token

        eventValue = {}
        eventValue = parseEventValue(trackEvent, eventObj, doc)
        eventObj['eventValue'] = eventValue   # 根据不同的情况解析req/res填充eventValue/ read from httplog res/req
    except Exception as e:
        raise

    saveToDb(eventObj, doc['_id'])

# API 判别 生成event发送到eventObj生成器
def identify(doc):
    thisDoc = doc
    thisDocUrl = thisDoc['url']
    for mapDoc in httpMapEvents:
        route = mapDoc['origin']['route']
        trackEvent = mapDoc['track']
        if route in thisDocUrl:
            generatorEvent(thisDoc, trackEvent)

print("2. Service running...")
for doc in httplogs.find({}):
    identify(doc)
print("END. Service finished.")