stdin stream

socket_stream.py

import sys, socket, redis, json

# config
stream = {'host': '127.0.0.1', 'port': 6000} # streaming data host / port
r = redis.Redis(host='127.0.0.1', port=6379) # redis instance host / port

# consume data from stdin and publish to redis on localhost
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((socket.gethostbyname(stream['host']), stream['port'])) # host, port

# consume data
frame = [] # initialize the container obj that will hold all frame data
while True:
  data = client.recv(1024).decode('utf8')
  for l in data.split('\n'):
    if 'end_frame' in l:
      d = {i.split(':')[0]: i.split(':')[1] for i in frame if ':' in i}
      r.set('frame', json.dumps(d))
      frame = []
      print(' * published frame', d.get('frame_number', ''))
    elif l.rstrip('\n'):
      frame.append(l.rstrip('\n'))

stdin_stream.py

import sys

frame = []
frames = []

while True:
  l = sys.stdin.readline()
  if 'end_frame' in l:
    # add to the frames and trim if we have a surplus
    new_frame = {i.split(':')[0]: i.split(':')[1] for i in frame if ':' in i}
    frames = [new_frame] + frames
    if len(frames) > 50: frames = frames[:50]
    frame = []
    print(' * published frame', new_frame)
  elif l.rstrip('\n'):
    frame.append(l.rstrip('\n'))