Skip to content

Instantly share code, notes, and snippets.

@vjache
Created March 30, 2015 11:24
Show Gist options
  • Save vjache/54f73136d56eae7842d5 to your computer and use it in GitHub Desktop.
Save vjache/54f73136d56eae7842d5 to your computer and use it in GitHub Desktop.
Batch append to Flume via Avro API.
appendBatch(SomeDir, Events) ->
ProtoFile = filename:join(SomeDir, "flume.avpr"),
{ok, P} = eavro_rpc_fsm:start_link(Host, Port, ProtoFile),
AvroEvents = encode_events_to_eavro_api_events(Events),
{ok, Result} = eavro_rpc_fsm:call(Conn, appendBatch, _Args = [ AvroEvents ]).
encode_events_to_eavro_api_events(Events) ->
[
[ [ {<<"timestamp">>,
list_to_binary(
integer_to_list(
estd_datetime:timestamp(msec, Ts))) }],
Body ] || #flume_event{ timestamp = Ts, body = Body} <- Events ].
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment