Example of bulk writes to avro file

py2avro_bulk.py

import json

import avro.schema
from avro.datafile import DataFileReader, DataFileWriter
from avro.io import DatumReader, DatumWriter


# transform json into avro
name = 'test'
sfile= './%s.avsc' % name
schema = avro.schema.parse(open(sfile).read())
data = json.load(open('%s.json' % name, 'r'))
print("init data", data)

# setup writer
writer = DataFileWriter(open("%s.avro" % name, "a"), DatumWriter(), schema)
for idx in range(10):
    data["foo"] = idx
    print("write data", data)
    writer.append(data)
    writer.flush()
writer.close()

# read avro fil
reader = DataFileReader(open("%s.avro" % name, "r"), DatumReader())
for rec in reader:
    print rec
reader.close()