erlang code to process a json feed : total wip, only downloads currently

get_feed_srv.erl

-module(get_feed_srv).
-behaviour(gen_server).
-define(SERVER, ?MODULE).

%% ------------------------------------------------------------------
%% API Function Exports
%% ------------------------------------------------------------------

%%-export([start_link/0, get_feed/1, process_redirect/1, process_response/1]).
-export([start_link/0, get_feed/1]).

%% ------------------------------------------------------------------
%% gen_server Function Exports
%% ------------------------------------------------------------------

-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

%% ------------------------------------------------------------------
%% API Function Definitions
%% ------------------------------------------------------------------

start_link() ->
    gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).

get_feed(Url) ->
    gen_server:call(?SERVER, {get_feed, Url}).

%% ------------------------------------------------------------------
%% gen_server Function Definitions
%% ------------------------------------------------------------------

init(Args) ->
    {ok, Args}.

handle_call(Request, From, State) ->
    case Request of
        {get_feed, Url} -> 
            {reply, do_get(Url), State};
        _ ->
            error_logger:error_msg("~p:handle_call() : Unexpected Request: Request=~p, From=~p, State=~p~n", [?MODULE, Request, From, State]),
            {reply, unexpected_request, State}
    end.

handle_cast(_Msg, State) ->
    {noreply, State}.

handle_info(_Info, State) ->
    {noreply, State}.

terminate(_Reason, _State) ->
    ok.

code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%% ------------------------------------------------------------------
%% Internal Function Definitions
%% ------------------------------------------------------------------

do_get(Url) ->
    error_logger:info_msg("~p:do_get(Url=~p)~n", [?MODULE, Url]),
    Response = process_response(ibrowse:send_req(Url, [], get)),
    error_logger:info_msg("~p:do_get(Url=~p) -->~nResponse=~p~n", [?MODULE, Url, Response]),
    Response.

process_redirect(Data) ->
    error_logger:info_msg("~p:process_redirect(Data=~p)~n", [?MODULE, Data]),
    F = fun({"Location", _}) -> true; (_) -> false end,
    {"Location", ForwardUrl} = lists:last(lists:filter(F, Data)),
    error_logger:info_msg("~p:process_redirect(Data=~p) -->~nForwardUrl=~p~n", [?MODULE, Data, ForwardUrl]),
    do_get(ForwardUrl).

process_response(Response) ->
    case Response of
        {ok, "200", Data, _Html} -> 
            Data;
        {ok, "300", Data, _Html} -> 
            process_redirect(Data);
        {ok, "301", Data, _Html} -> 
            process_redirect(Data);
        {ok, "302", Data, _Html} -> 
            process_redirect(Data);
        {ok, "303", Data, _Html} -> 
            process_redirect(Data);
        {ok, "307", Data, _Html} -> 
            process_redirect(Data);
        {Isok, Code, Data, _Html} ->
            error_logger:error_msg("~p:process_response, Isok=~p, Code=~p, Data=~p~n", [?MODULE, Isok, Code, Data]),
            unexpected_response;
        _ ->
            error_logger:error_msg("~p:process_response, Response=~p~n", [?MODULE, Response]),
            no_idea_what_is_going_on
    end.

%% ------------------------------------------------------------------
%% Sample Run
%% ------------------------------------------------------------------
%%
%%$ erl -pa ebin/ -pa deps/ibrowse/ebin/
%%Erlang R15B02 (erts-5.9.2) [source] [64-bit] [smp:6:6] [async-threads:0] [hipe] [kernel-poll:false]
%%
%%Eshell V5.9.2  (abort with ^G)
%%1> get_feed_srv:get_feed("https://raw.github.com/ToddG/experimental/master/feeds/catalog.json").                                                             
%%=INFO REPORT==== 16-Sep-2012::12:49:18 ===
%%get_feed_srv:do_get(Url="https://raw.github.com/ToddG/experimental/master/feeds/catalog.json")
%%
%%=INFO REPORT==== 16-Sep-2012::12:49:19 ===
%%get_feed_srv:do_get(Url="https://raw.github.com/ToddG/experimental/master/feeds/catalog.json") -->
%%Response=[{"Server","nginx"},
%%          {"Date","Sun, 16 Sep 2012 16:49:19 GMT"},
%%          {"Content-Type","text/plain; charset=utf-8"},
%%          {"Connection","keep-alive"},
%%          {"Status","200 OK"},
%%          {"X-RateLimit-Limit","100"},
%%          {"X-Runtime","12"},
%%          {"X-Frame-Options","deny"},
%%          {"Cache-Control","private"},
%%          {"Content-Transfer-Encoding","binary"},
%%          {"Content-Disposition","inline"},
%%          {"X-RateLimit-Remaining","100"},
%%          {"ETag","\"7b402e279bfa0590add9b916984e8721\""},
%%          {"Content-Length","293"},
%%          {"X-Content-Type-Options","nosniff"}]
%%[{"Server","nginx"},
%% {"Date","Sun, 16 Sep 2012 16:49:19 GMT"},
%% {"Content-Type","text/plain; charset=utf-8"},
%% {"Connection","keep-alive"},
%% {"Status","200 OK"},
%% {"X-RateLimit-Limit","100"},
%% {"X-Runtime","12"},
%% {"X-Frame-Options","deny"},
%% {"Cache-Control","private"},
%% {"Content-Transfer-Encoding","binary"},
%% {"Content-Disposition","inline"},
%% {"X-RateLimit-Remaining","100"},
%% {"ETag","\"7b402e279bfa0590add9b916984e8721\""},
%% {"Content-Length","293"},
%% {"X-Content-Type-Options","nosniff"}]
%%2> get_feed_srv:get_feed("http://raw.github.com/ToddG/experimental/master/feeds/catalog.json"). 
%%
%%=INFO REPORT==== 16-Sep-2012::12:49:29 ===
%%get_feed_srv:do_get(Url="http://raw.github.com/ToddG/experimental/master/feeds/catalog.json")
%%
%%=INFO REPORT==== 16-Sep-2012::12:49:29 ===
%%get_feed_srv:process_redirect(Data=[{"Server","nginx"},
%%                                    {"Date","Sun, 16 Sep 2012 16:49:29 GMT"},
%%                                    {"Content-Type","text/html"},
%%                                    {"Content-Length","178"},
%%                                    {"Connection","close"},
%%                                    {"Location",
%%                                     "https://raw.github.com/ToddG/experimental/master/feeds/catalog.json"}])
%%
%%=INFO REPORT==== 16-Sep-2012::12:49:29 ===
%%get_feed_srv:process_redirect(Data=[{"Server","nginx"},
%%                                    {"Date","Sun, 16 Sep 2012 16:49:29 GMT"},
%%                                    {"Content-Type","text/html"},
%%                                    {"Content-Length","178"},
%%                                    {"Connection","close"},
%%                                    {"Location",
%%                                     "https://raw.github.com/ToddG/experimental/master/feeds/catalog.json"}]) -->
%%ForwardUrl="https://raw.github.com/ToddG/experimental/master/feeds/catalog.json"
%%
%%=INFO REPORT==== 16-Sep-2012::12:49:29 ===
%%get_feed_srv:do_get(Url="https://raw.github.com/ToddG/experimental/master/feeds/catalog.json")
%%
%%=INFO REPORT==== 16-Sep-2012::12:49:29 ===
%%get_feed_srv:do_get(Url="https://raw.github.com/ToddG/experimental/master/feeds/catalog.json") -->
%%Response=[{"Server","nginx"},
%%          {"Date","Sun, 16 Sep 2012 16:49:29 GMT"},
%%          {"Content-Type","text/plain; charset=utf-8"},
%%          {"Connection","keep-alive"},
%%          {"Status","200 OK"},
%%          {"X-Content-Type-Options","nosniff"},
%%          {"Content-Length","293"},
%%          {"Cache-Control","private"},
%%          {"X-Frame-Options","deny"},
%%          {"X-RateLimit-Remaining","100"},
%%          {"Content-Transfer-Encoding","binary"},
%%          {"X-Runtime","15"},
%%          {"Content-Disposition","inline"},
%%          {"ETag","\"7b402e279bfa0590add9b916984e8721\""},
%%          {"X-RateLimit-Limit","100"}]
%%
%%=INFO REPORT==== 16-Sep-2012::12:49:29 ===
%%get_feed_srv:do_get(Url="http://raw.github.com/ToddG/experimental/master/feeds/catalog.json") -->
%%Response=[{"Server","nginx"},
%%          {"Date","Sun, 16 Sep 2012 16:49:29 GMT"},
%%          {"Content-Type","text/plain; charset=utf-8"},
%%          {"Connection","keep-alive"},
%%          {"Status","200 OK"},
%%          {"X-Content-Type-Options","nosniff"},
%%          {"Content-Length","293"},
%%          {"Cache-Control","private"},
%%          {"X-Frame-Options","deny"},
%%          {"X-RateLimit-Remaining","100"},
%%          {"Content-Transfer-Encoding","binary"},
%%          {"X-Runtime","15"},
%%          {"Content-Disposition","inline"},
%%          {"ETag","\"7b402e279bfa0590add9b916984e8721\""},
%%          {"X-RateLimit-Limit","100"}]
%%[{"Server","nginx"},
%% {"Date","Sun, 16 Sep 2012 16:49:29 GMT"},
%% {"Content-Type","text/plain; charset=utf-8"},
%% {"Connection","keep-alive"},
%% {"Status","200 OK"},
%% {"X-Content-Type-Options","nosniff"},
%% {"Content-Length","293"},
%% {"Cache-Control","private"},
%% {"X-Frame-Options","deny"},
%% {"X-RateLimit-Remaining","100"},
%% {"Content-Transfer-Encoding","binary"},
%% {"X-Runtime","15"},
%% {"Content-Disposition","inline"},
%% {"ETag","\"7b402e279bfa0590add9b916984e8721\""},
%% {"X-RateLimit-Limit","100"}]
%%