Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
import json
from datetime import datetime, timedelta
from elasticsearch import Elasticsearch
if __name__ == '__main__':
# create ES instance
es = Elasticsearch()
# create index
es.indices.create(index='track_log', ignore=400)
with open('/tmp/tracking/track_log.log', 'r', encoding='utf-8') as f:
content = f.readlines()
for k, v in enumerate(content):
json_item = json.loads(v)
timestamp = datetime.fromtimestamp(json_item['time']) - timedelta(
hours=8)
if json_item['target'] and 'id' in json_item['target']:
json_item['target']['id'] = str(json_item['target']['id'])
if json_item['object'] and 'id' in json_item['object']:
json_item['object']['id'] = str(json_item['object']['id'])
print('======== start index ========')
es.index(
index="track_log",
doc_type="track_log",
body={"item": json_item,
"timestamp": timestamp})
print('======== end index ========')
import json
import time
from datetime import datetime, timedelta
import tail
def print_line(line):
print('======== start new line ========')
print(line)
print('======== end new line ========')
print('======== start split log to day ========')
today = datetime.now().date()
tomorrow = today + timedelta(1)
today_start = datetime.combine(today, time())
today_end = datetime.combine(tomorrow, time())
file_name = str(today.year) + '-' + str(today.month) + '-' + str(today.day)
track_log = open('/tmp/tracking/track_log_day/' + file_name, 'w')
json_item = json.loads(line)
item_datetime = datetime.fromtimestamp(json_item['time'])
if item_datetime > today_start and item_datetime < today_end:
track_log.write(line)
print('======== end split log to day ========')
if __name__ == '__main__':
t = tail.Tail('/tmp/tracking/track_log.log')
t.register_callback(print_line)
t.follow(s=5)
#!/usr/bin/env python
'''
Python-Tail - Unix tail follow implementation in Python.
python-tail can be used to monitor changes to a file.
Example:
import tail
# Create a tail instance
t = tail.Tail('file-to-be-followed')
# Register a callback function to be called when a new line is found in the followed file.
# If no callback function is registerd, new lines would be printed to standard out.
t.register_callback(callback_function)
# Follow the file with 5 seconds as sleep time between iterations.
# If sleep time is not provided 1 second is used as the default time.
t.follow(s=5) '''
# Author - Kasun Herath <kasunh01 at gmail.com>
# Source - https://github.com/kasun/python-tail
import os
import sys
import time
class Tail(object):
''' Represents a tail command. '''
def __init__(self, tailed_file):
''' Initiate a Tail instance.
Check for file validity, assigns callback function to standard out.
Arguments:
tailed_file - File to be followed. '''
self.check_file_validity(tailed_file)
self.tailed_file = tailed_file
self.callback = sys.stdout.write
def follow(self, s=1):
''' Do a tail follow. If a callback function is registered it is called with every new line.
Else printed to standard out.
Arguments:
s - Number of seconds to wait between each iteration; Defaults to 1. '''
with open(self.tailed_file) as file_:
# Go to the end of file
file_.seek(0, 2)
while True:
curr_position = file_.tell()
line = file_.readline()
if not line:
file_.seek(curr_position)
time.sleep(s)
else:
self.callback(line)
def register_callback(self, func):
''' Overrides default callback function to provided function. '''
self.callback = func
def check_file_validity(self, file_):
''' Check whether the a given file exists, readable and is a file '''
if not os.access(file_, os.F_OK):
raise TailError("File '%s' does not exist" % (file_))
if not os.access(file_, os.R_OK):
raise TailError("File '%s' not readable" % (file_))
if os.path.isdir(file_):
raise TailError("File '%s' is a directory" % (file_))
class TailError(Exception):
def __init__(self, msg):
self.message = msg
def __str__(self):
return self.message
import json
from datetime import datetime, timedelta
from elasticsearch import Elasticsearch
import tail
import redis
redis_conn = redis.Redis(host='localhost', port=6379, db=0, decode_responses=True)
def print_line(line):
print('======== start new line ========')
print(line)
print('======== end new line ========')
print('======== start index data to es ========')
json_item = json.loads(line.strip())
timestamp = datetime.fromtimestamp(json_item['time']) - timedelta(hours=8)
if json_item['target'] and 'id' in json_item['target']:
json_item['target']['id'] = str(json_item['target']['id'])
if json_item['object'] and 'id' in json_item['object']:
json_item['object']['id'] = str(json_item['object']['id'])
es.index(
index="track_log",
doc_type="track_log",
body={"item": json_item,
"timestamp": timestamp})
print('======== end index data to es ========')
if __name__ == '__main__':
# create ES instance
es = Elasticsearch()
# create index
es.indices.create(index='track_log', ignore=400)
# Create a tail instance
t = tail.Tail('/tmp/tracking/track_log.log')
t.register_callback(print_line)
t.follow(s=5)
{"target": {"type": "Paper", "title": "Deep Depth Super-Resolution : Learning Depth Super-Resolution using Deep Convolutional Neural Network", "id": "791", "owner": "MKFMIKU"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36", "accept_language": "en-US,en;q=0.8", "username": "qhl722", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507000406.305043}
{"target": {"type": "User", "id": "fcc3837f-1a61-4d2c-bdbf-0961085547a3", "owner": "gg5d"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36", "accept_language": "zh-CN,zh;q=0.8", "username": "", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/"}, "verb": "注册", "time": 1507000688.429523}
{"target": {"type": "User", "id": "5b614490-110b-4fd3-8342-d0567abe79a2", "owner": "daabin"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (Linux; Android 7.1.1; OD103 Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.49 Mobile MQQBrowser/6.2 TBS/043508 Safari/537.36 MicroMessenger/6.5.16.1120 NetType/WIFI Language/zh_CN", "accept_language": "zh-CN,en-US;q=0.8", "username": "", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/"}, "verb": "注册", "time": 1507001564.3203382}
{"target": {"type": "User", "id": "5f9f19c5-eb9d-4708-b960-fd1508989977", "owner": "sherlockmao"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:57.0) Gecko/20100101 Firefox/57.0", "accept_language": "en-GB,en;q=0.5", "username": "", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/"}, "verb": "注册", "time": 1507002515.9859521}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5930", "owner": "gascomplisp"}, "actor": null, "verb": "消息", "time": 1507002682.0841422}
{"target": {"type": "Paper", "title": "Memory Networks", "id": "813", "owner": "zh794390558"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:57.0) Gecko/20100101 Firefox/57.0", "accept_language": "en-GB,en;q=0.5", "username": "gascomplisp", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507002682.0844338}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5931", "owner": "gascomplisp"}, "actor": null, "verb": "消息", "time": 1507002684.2234745}
{"target": {"type": "Paper", "title": "Small-footprint Keyword Spotting Using Deep Neural Network and Connectionist Temporal Classifier", "id": "814", "owner": "zh794390558"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:57.0) Gecko/20100101 Firefox/57.0", "accept_language": "en-GB,en;q=0.5", "username": "gascomplisp", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507002684.2238252}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5932", "owner": "gascomplisp"}, "actor": null, "verb": "消息", "time": 1507002689.356459}
{"target": {"type": "Paper", "title": "Learning Precise Timing with LSTM Recurrent Networks", "id": "815", "owner": "zh794390558"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:57.0) Gecko/20100101 Firefox/57.0", "accept_language": "en-GB,en;q=0.5", "username": "gascomplisp", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507002689.3567564}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5933", "owner": "naturomics"}, "actor": null, "verb": "消息", "time": 1507004137.3740406}
{"target": {"type": "Paper", "title": "Memory Networks", "id": "813", "owner": "zh794390558"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", "accept_language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ja;q=0.2,zh-TW;q=0.2,ru;q=0.2", "username": "naturomics", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507004137.3743596}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5934", "owner": "naturomics"}, "actor": null, "verb": "消息", "time": 1507004150.909643}
{"target": {"type": "Paper", "title": "Addressing the Rare Word Problem in Neural Machine Translation", "id": "811", "owner": "zh794390558"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", "accept_language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ja;q=0.2,zh-TW;q=0.2,ru;q=0.2", "username": "naturomics", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507004150.909946}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5935", "owner": "naturomics"}, "actor": null, "verb": "消息", "time": 1507004153.5462253}
{"target": {"type": "Paper", "title": "Show, Attend and Tell: Neural Image Caption Generation with Visual Attention", "id": "812", "owner": "zh794390558"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", "accept_language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ja;q=0.2,zh-TW;q=0.2,ru;q=0.2", "username": "naturomics", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507004153.5465171}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5936", "owner": "naturomics"}, "actor": null, "verb": "消息", "time": 1507004169.2238753}
{"target": {"type": "Paper", "title": "Learning Precise Timing with LSTM Recurrent Networks", "id": "815", "owner": "zh794390558"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", "accept_language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ja;q=0.2,zh-TW;q=0.2,ru;q=0.2", "username": "naturomics", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/getting-started"}, "verb": "点赞", "time": 1507004169.2241929}
{"target": {"count": 1}, "object": {"type": "Message", "id": "messageid:5937", "owner": "gascomplisp"}, "actor": null, "verb": "消息", "time": 1507004835.610317}
{"target": {"type": "Paper", "title": "Training RNNs as Fast as CNNs", "id": "582", "owner": "paperweekly"}, "object": {}, "actor": {"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:57.0) Gecko/20100101 Firefox/57.0", "accept_language": "en-GB,en;q=0.5", "username": "gascomplisp", "host": "zijin.paperweekly.site", "referer": "http://www.paperweekly.site/rank"}, "verb": "关注", "time": 1507004835.61063}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.