Script that reads Graphite plaintext metrics from stdin and pushes them to carbon-cache in bulk using pickle

push_stdin_live_samples_to_graphite_using_in_bulk.py

#!/usr/bin/python
import os
import time
import socket
import pickle
import struct
import fileinput
import calendar

CARBON_CACHE_SERVER = '127.0.0.1'
CARBON_CACHE_PICKLE_PORT = 2004
METRICS_BUFFER_SIZE = 30
MAX_BUFFER_TIME = 60

if __name__ == "__main__":
    sock = socket.socket()
    sock.connect( (CARBON_CACHE_SERVER, CARBON_CACHE_PICKLE_PORT) )    
    tuples = ([])
    line_count = 0
    start_time = int(calendar.timegm(time.gmtime()))
    last_push_time = start_time
    for line in fileinput.input():
        line_count += 1;
        entries = line.split()
        tuples.append((entries[0], (entries[2], entries[1])))
        if line_count % METRICS_BUFFER_SIZE == 0 or (last_push_time > 0 and last_push_time % MAX_BUFFER_TIME == 0):  
          package = pickle.dumps(tuples, 1)
          size = struct.pack('!L', len(package))
          sock.sendall(size)
          sock.sendall(package)
          tuples = ([])
          last_push_time = int(calendar.timegm(time.gmtime())) - start_time
    package = pickle.dumps(tuples, 1)
    size = struct.pack('!L', len(package))
    sock.sendall(size)
    sock.sendall(package)