Quick rebar plugin to support downloading and unpacking non-vcs projects.

http_deps_plugin.erl

-module(http_deps_pluggin).
-export([ %pre_compile/2
          'pre_get-deps'/2
        ]).

-include_lib("kernel/include/file.hrl").

-define(A_CONFIG(C, K), ulist(lists:umerge(rebar_config:get_all(C, K)))).
-define(X_CONFIG(C, K, D), rebar_config:get_xconf(C, K, D)).
-define(G_CONFIG(K, D), rebar_config:get_global(K, D)).
-define(CONFIG(C, K, D), rebar_config:get_local(C, K, D)).
-define(DEBUG(Msg, Args), ?LOG(debug, Msg, Args)).
-define(WARN(Msg, Args), ?LOG(warn, Msg, Args)).
-define(LOG(Lvl, Msg, Args), rebar_log:log(Lvl, Msg, Args)).
-define(CONSOLE(Str, Args), io:format(Str, Args)).

'pre_get-deps'(Config, _) ->
    Deps = ?CONFIG(Config, deps, []),
    fetchall(Config, Deps),
    ok.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% internal interfaces
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

fetchall(Config, []) -> ok;
fetchall(Config, [{Target, _Ver, TConfig} | Deps]) ->
    {Type, Uri, Opts} = TConfig,

    BaseDir = ?G_CONFIG(base_dir, []),
    DepsDir = ?G_CONFIG(deps_dir, ""),
    TDir = filename:join([BaseDir, DepsDir, Target]),

    case filelib:is_dir(TDir) of
        false ->
            {ok, File} = case Type of
                http ->
                    ?CONSOLE("Pulling ~p from ~p~n", [Target, Uri]),
                    Tmp = get_tmp_filename(),
                    fetch(Type, Uri, Tmp, Config);
                _ ->
                    ?DEBUG("Ignoring: ~s~n", [Target]),
                    {ok, undefined}
            end,

            % unpack the file
            case File of
                undefined ->
                    ok;
                _ ->
                    case verify(File, Opts) of
                        {ok, true} ->
                            ?CONSOLE("Verified: true~n", []),
                            unpack(File, TDir, Opts),
                            ok;
                        {ok, false} ->
                            ?CONSOLE("Verified: false~n", []),
                            ok;
                        {error, Reason} ->
                            ?CONSOLE("Could not verify ~p: ~p", [Target, Reason]),
                            ok
                    end,
                    file:delete(File)
            end;
        true ->
            ok
    end,

    fetchall(Config, Deps).

verify(File, Config) ->
    Sha = proplists:get_value(sha1, Config, ""),
    case file:open(File, [read, raw, binary]) of
        {ok, Fd} ->
            C = crypto:sha_init(),
            {ok, NSha} = do_hash(C, Fd, file:read(Fd, 1024)),
            file:close(Fd),
            {ok, NSha == Sha};
        {error, Reason} ->
            {error, Reason}
    end.

do_hash(C, Fd, {ok, Data}) ->
    Nc = crypto:sha_update(C, Data),
    do_hash(Nc, Fd, file:read(Fd, 1024));

do_hash(C, _Fd, eof) ->
    <<Mac:160/integer>> = crypto:sha_final(C),
    {ok, lists:flatten(io_lib:format("~40.16.0b", [Mac]))};

do_hash(_C, _Fd, {error, Reason}) ->
    {error, Reason}.


unpack(File, TDir, TConfig) ->
    Strip = proplists:is_defined(stripdir, TConfig),
    case proplists:lookup(unpack, TConfig) of
        {unpack, zip} ->
            ?CONSOLE("Unpacking into ~p~n", [TDir]),
            zip:foldl(fun(N, I, B, _Acc) ->
                        FN = strip_leading_dir(Strip, N),
                        FI = I(),
                        Filename = filename:join([TDir, FN]),
                        case FI#file_info.type of
                            regular ->
                                rebar_utils:ensure_dir(Filename),
                                file:write_file(Filename, B()),
                                ok;
                            _ ->
                                ok
                        end
                      end, [], File),
            ok;
        none ->
            ?DEBUG("no unpacker defined~n", []),
            ok;
        {ok, P} ->
            ?DEBUG("unknown unpacker ~p~n", [P]),
            ok
    end.

fetch(http, Url, Target, Config) ->
    case get({?MODULE, httpc}) of
        started ->
            ok;
        _ ->
            inets:start(),
            application:load(sasl),
            application:set_env(sasl, sasl_error_logger, false),
            lists:map(fun application:start/1, [sasl, crypto, public_key, ssl]),
            Timeout = ?CONFIG(Config, remote_net_timeout, 6000),
            case ?CONFIG(Config, remote_proxy_host, undefined) of
                undefined ->
                    httpc:set_options([{timeout, Timeout},
                                       {connect_timeout, Timeout}]);
                Host ->
                    Port = ?CONFIG(Config, remote_proxy_port, "8080"),
                    httpc:set_options([{proxy, {{Host, Port}, ["localhost"]}},
                                       {timeout, Timeout},
                                       {connect_timeout, Timeout}])
            end,
            put({?MODULE, httpc}, started)
    end,

    rebar_utils:ensure_dir(Target),
    Request = {Url, [{"User-Agent", "Rebar-Remote-Dependency-Loader"}]},
    ?DEBUG("Attempting to fetch ~s into ~s~n", [Url, Target]),

    case httpc:request(get, Request, [{relaxed, true}],
                                     [{stream, Target}, {full_result, true}]) of
        {ok, saved_to_file} ->
            ?DEBUG("Successfully stored remote dependency!~n", []),
            {ok, Target};
        {ok, {{_, 404, _}, _, _}} ->
            {error, not_found};
        Error ->
            ?WARN("Error trying to store remote dependency: ~p~n", [Error])
    end;
fetch(_, Url, _Target, _Config) ->
    ?DEBUG("Skipping Uri: ~s: Unknown Type", [Url]),
    ok.

get_tmp_filename() ->
    % now is guarnteed to give a new number every time
    {A,B,C}=now(),
    N=node(),
    lists:flatten(io_lib:format("~p-~p.~p.~p",[N,A,B,C])).

strip_leading_dir(true, Name) ->
    [_ | FN ] = filename:split(Name),
    case FN of
        [] ->
            "";
        _ ->
            filename:join(FN)
    end;
strip_leading_dir(false, Name) ->
    Name.

rebar.config

{deps,
    [  {'amqp_client', "3.0.2",
            {http, "https://www.rabbitmq.com/releases/rabbitmq-erlang-client/v3.0.2/amqp_client-3.0.2.ez",
             [{sha1, "466b204612fff7a955dff7323a4ea890b9628d10"},
               stripdir,
               {unpack, zip}]}}
        , {'rabbit_common', "3.0.2",
            {http, "https://www.rabbitmq.com/releases/rabbitmq-erlang-client/v3.0.2/rabbit_common-3.0.2.ez",
             [{sha1, "61e76a6b6c8c180da42b62ee99e5b7fb3219569a"},
               stripdir,
               {unpack, zip}]}}
    ]
}.