json-stream-compression.erl

-module(sc).
-define(DEBUG(S,A), io:format(S++"\n",A)).

-behaviour(gen_server).

%% API
-export([start_link/0, start/0]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
     terminate/2, code_change/3]).

-record(state, {id=0,lastmsg={}}).

start() ->
    {ok, Pid} = ?MODULE:start_link(),
    File = "oob-loader.txt",
    {ok, Bin} = file:read_file(File),
    Lines = binary:split(Bin, << 44, 10 >>, [global]),
    lists:foreach(fun(Line) -> Pid ! {line, Line} end, Lines),
    Pid ! done,
    ok.

%% API
start_link() ->
    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).

%% gen_server callbacks
init([]) ->
    ets:new(sc, [named_table, set]),
    {ok, #state{}}.

handle_call(_Request, _From, State) ->
    Reply = ok,
    {reply, Reply, State}.

handle_cast(_Msg, State) ->
    {noreply, State}.

handle_info(done, State) ->
    init:stop(),
    {stop, normal, State};

handle_info({line, Line}, State) ->
    case catch irccloud_json:decode(Line) of
        {struct, Props} ->
            {Keys, Vals} = extract_schema(Props),
            case ets:lookup(sc, Keys) of
                [] ->
                    {SchemaId, NewState} = new_schema(Keys, State),
                    send_schema(SchemaId, Keys),
                    send_msg(SchemaId, Vals, NewState),
                    {noreply, NewState#state{lastmsg={SchemaId, Vals}}};
                [{Keys, SchemaId}] ->
                    send_msg(SchemaId, Vals, State),
                    {noreply, State#state{lastmsg={SchemaId, Vals}}}
            end;
        _ ->
            {noreply, State}
    end.

terminate(_Reason, _State) ->
    ok.

code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%%% Internal functions


send_schema(SchemaId, Keys) ->
    Msg = irccloud_json:encode({struct, [{s,SchemaId},{d, Keys}]}),
    io:format("~s\n",[Msg]),
    ok.

send_msg(SchemaId, Vals, #state{lastmsg={SchemaId, OldVals}}) ->
    ValList = [ case NewVal == OldVal of true -> null ; false -> NewVal end || {NewVal, OldVal} <- lists:zip(Vals, OldVals) ],
    Msg = irccloud_json:encode(ValList),
    io:format("~s\n",[Msg]),
    ok;    

send_msg(SchemaId, Vals, _State) ->
    Msg = irccloud_json:encode({struct, [{s,SchemaId},{v, Vals}]}),
    io:format("~s\n",[Msg]),
    ok.

extract_schema(Proplist) ->
    lists:unzip(lists:keysort(1, Proplist)).

new_schema(Keys, State = #state{id=LastId}) ->
    Id = LastId + 1,
    ets:insert(sc, {Keys, Id}),
    {Id, State#state{id=Id}}.