fix sending of the remaining buffer

bitdeli-bulk-load.py

import json
import sys
import urllib

BITDELI_API = 'https://track.bitdeli.com/events'

MAX_BUFFER_SIZE = 8 * 1024 * 1024
DATA = sys.argv[1]
INPUT_ID = sys.argv[2]
AUTH = sys.argv[3]
URL = '%s/%s' % (BITDELI_API, INPUT_ID)

def send(buffer):
    events = '[%s]' % ','.join(buffer)
    ret = urllib.urlopen(URL, events)
    response = ret.read()
    if ret.getcode() != 200:
        raise Exception("Send failed (%s): %s" % (ret.getcode(), response))

# read one json object per line
def json_reader():
    for line in open(DATA):
        event = json.loads(line)
        event['$event_name'] = 'none' # change to something more meaningful
        yield {'auth': AUTH,
               'uid': event.get('uid', ''),
               'event': event}

def send_to_bitdeli(data_iter):
    buffer = []
    buffer_size = 0
    for i, event in enumerate(data_iter):
        if not i % 1000:
            print >> sys.stderr, '%d events sent' % i
        encoded = json.dumps(event)
        buffer_size += len(encoded)
        buffer.append(encoded)
        if buffer_size > MAX_BUFFER_SIZE:
            send(buffer)
            buffer = []
            buffer_size = 0
    if buffer:
        send(buffer)

send_to_bitdeli(json_reader())