Skip to content

Instantly share code, notes, and snippets.

@Ikke
Created March 29, 2017 20:25
Show Gist options
  • Save Ikke/e0fac439de912fdea9d2c96ef3bb3d86 to your computer and use it in GitHub Desktop.
Save Ikke/e0fac439de912fdea9d2c96ef3bb3d86 to your computer and use it in GitHub Desktop.
Write stats to redis
#!/bin/python
import redis
import sys
import datetime as DT
class Stat:
@staticmethod
def store_new_stat(r, stat):
p = r.pipeline()
hash_key = Stat.key_for_stat(stat)
p.hmset(hash_key, stat)
p.sadd("idx:hash:stat", hash_key)
p.sadd("idx:hash:stat:{destination}".format(**stat), hash_key)
p.sadd("idx:hash:stat:{destination}:{port}".format(**stat), hash_key)
p.execute()
return hash_key
@staticmethod
def key_for_stat(stat):
return \
"hash:stat:{destination}:{port}:{timestamp}" \
.format(**stat)
def read_lines(f):
while True:
line = f.readline()
if line:
yield line
else:
return
r = redis.Redis(charset='utf-8', decode_responses=True)
known_keys = r.smembers('idx:hash:stat') or set()
last = DT.datetime.now()
records_inserted = 0
for line in read_lines(sys.stdin):
destination, port = line.strip().split(" ")
now = DT.datetime.now()
timestamp = DT.datetime(
now.year, now.month, now.day, now.hour, 0, 0, 0
).strftime("%s")
stat = {
'destination': destination,
'port': port,
'timestamp': timestamp,
'count': 1
}
key = Stat.key_for_stat(stat)
if not key in known_keys:
Stat.store_new_stat(r, stat)
known_keys.add(key)
else:
r.hincrby(key, 'count')
records_inserted += 1
time_diff = (now - last).total_seconds()
if time_diff > 10:
print("{} records/s"
.format(records_inserted / time_diff), file=sys.stderr
)
last = now
records_inserted = 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment