Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Script that reads Graphite plaintext metrics from stdin and pushes them to carbon-cache in bulk using pickle
#!/usr/bin/python
import os
import time
import socket
import pickle
import struct
import fileinput
import calendar
CARBON_CACHE_SERVER = '127.0.0.1'
CARBON_CACHE_PICKLE_PORT = 2004
METRICS_BUFFER_SIZE = 30
MAX_BUFFER_TIME = 60
if __name__ == "__main__":
sock = socket.socket()
sock.connect( (CARBON_CACHE_SERVER, CARBON_CACHE_PICKLE_PORT) )
tuples = ([])
line_count = 0
start_time = int(calendar.timegm(time.gmtime()))
last_push_time = start_time
for line in fileinput.input():
line_count += 1;
entries = line.split()
tuples.append((entries[0], (entries[2], entries[1])))
if line_count % METRICS_BUFFER_SIZE == 0 or (last_push_time > 0 and last_push_time % MAX_BUFFER_TIME == 0):
package = pickle.dumps(tuples, 1)
size = struct.pack('!L', len(package))
sock.sendall(size)
sock.sendall(package)
tuples = ([])
last_push_time = int(calendar.timegm(time.gmtime())) - start_time
package = pickle.dumps(tuples, 1)
size = struct.pack('!L', len(package))
sock.sendall(size)
sock.sendall(package)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment