useful examples using hovercraft

compact_data.erl

-module(compact_data).
-author('bob@cloudant.com').

-export([test_id_btree/1, test_db/4, test_view/1, test_view/4]).

-include("couch_db.hrl").

-define(ADMIN_USER_CTX, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}).

test_id_btree(N) ->
    io:format("chunk_size  data_size disk_size disk_size(Compacted) ratio ~n",[]),
    io:format("random ids ~n",[]),
    test_db("foo",1024,random,N),
    test_db("foo",2048,random,N),
    test_db("foo",4096,random,N),
    test_db("foo",8192,random,N),
    test_db("foo",16384,random,N),
    io:format("sequential ids ~n",[]),
    test_db("foo",1024,sequential,N),
    test_db("foo",2048,sequential,N),
    test_db("foo",4096,sequential,N),
    test_db("foo",8192,sequential,N),
    test_db("foo",16384,sequential,N).

test_view(N) ->
    io:format("chunk_size  data_size disk_size disk_size(Compacted) ratio ~n",[]),
    io:format("small reductions ~n",[]),
    test_view("foo",1024,N,small),
    test_view("foo",2048,N,small),
    test_view("foo",4096,N,small),
    test_view("foo",8192,N,small),
    io:format("large reductions ~n",[]),
    test_view("foo",1024,N,large),
    test_view("foo",2048,N,large),
    test_view("foo",4096,N,large),
    test_view("foo",8192,N,large).

test_db(Name,ChunkSize,RandomOrSequential,NoDocs) ->
    DbName = ?l2b(Name),
    couch_config:set("couchdb","btree_chunk_size",integer_to_list(ChunkSize)),
    io:format("~10p  ",[list_to_integer(couch_config:get("couchdb","btree_chunk_size"))]),
    {ok, Db} = couch_db:create(DbName,[]),
    generate_and_add_docs(Db,RandomOrSequential,NoDocs),
    DataSize = get_data_size(DbName),
    BDiskSize = get_disk_size(DbName),
    io:format("~10p  ",[DataSize]),
    io:format("~10p  ",[BDiskSize]),
    couch_db:start_compact(Db),
    % give it a sec to crank up
    timer:sleep(1000),
    while_compaction_running(DbName),
    timer:sleep(10000),
    ADiskSize = get_disk_size(DbName),
    io:format("~15p  ",[ADiskSize]),
    io:format("~3p % ~n",[(ADiskSize * 100) div BDiskSize]),
    couch_server:delete(DbName,[]).

test_view(Name,ChunkSize,NoDocs,ReductionSize) ->
    DbName = ?l2b(Name),
    couch_config:set("couchdb","btree_chunk_size",integer_to_list(ChunkSize)),
    io:format("~p  ",[list_to_integer(couch_config:get("couchdb","btree_chunk_size"))]),
    {ok, _Db} = couch_db:create(DbName,[]),
    gen_view_and_query(DbName,NoDocs,ReductionSize),
    ViewDiskSize = get_view_disk_size(DbName,<<"_design/btree">>),
    ViewDataSize = get_view_data_size(DbName,<<"_design/btree">>),
    io:format("~p  ",[ViewDataSize]),
    io:format("~p  ",[ViewDiskSize]),
    couch_view_compactor:start_compact(DbName,<<"btree">>),
    timer:sleep(1000),
    while_view_compaction_running(DbName,<<"_design/btree">>),
    timer:sleep(5000),
    CViewDiskSize = get_view_disk_size(DbName,<<"_design/btree">>),
    io:format("~p  ",[CViewDiskSize]),
    io:format("~3p % ~n",[(CViewDiskSize * 100) div ViewDiskSize]),
    %couch_server:delete(DbName,[]),
    ok.

generate_and_add_docs(Db,IdType,NoDocs) ->
    Docs = gen_docs(IdType,NoDocs),
    lists:map(fun(Doc) ->
                  couch_db:update_doc(Db,Doc,[])
              end, Docs).

gen_docs(IdType,N) ->
    Ids = gen_ids(IdType,N,[]),
    lists:foldl(fun(Id, Acc) ->
                    [gen_doc(Id) | Acc]
                end,[],Ids).

gen_ids(_Type,0,Acc) ->
    lists:reverse(Acc);
gen_ids(Type,N,Acc) ->
    Id =
        case Type of
        random ->
            couch_uuids:random();
        sequential ->
            ?l2b(integer_to_list(N))
        end,
    gen_ids(Type,N-1,[Id | Acc]).

gen_doc(Id) ->
    ejson_to_couch_doc(
    {[{<<"_id">>, Id},
      {<<"foo">>, <<"bar">>}]}).

ejson_to_couch_doc({DocProps}) ->
    Doc = case proplists:get_value(<<"_id">>, DocProps) of
        undefined ->
            DocId = couch_util:new_uuid(),
            {[{<<"_id">>, DocId}|DocProps]};
        _DocId ->
            {DocProps}
    end,
    couch_doc:from_json_obj(Doc).


get_data_size(DbName) ->
    {ok, Db} = couch_db:open(DbName,[]),
    try
        {ok, InfoProps} = couch_db:get_db_info(Db),
        proplists:get_value(data_size,InfoProps)
    after
        couch_db:close(Db)
    end.

get_view_disk_size(DbName, DDoc) ->
    {ok, Db} = couch_db:open(DbName, []),
    try
        {ok, InfoProps} = couch_view:get_group_info(Db,DDoc),
        proplists:get_value(disk_size,InfoProps)
    after
        couch_db:close(Db)
    end.

get_view_data_size(DbName, DDoc) ->
    {ok, Db} = couch_db:open(DbName, []),
    try
        {ok, InfoProps} = couch_view:get_group_info(Db,DDoc),
        proplists:get_value(data_size,InfoProps)
    after
        couch_db:close(Db)
    end.

get_disk_size(DbName) ->
    {ok, Db} = couch_db:open(DbName,[]),
    try
        {ok, InfoProps} = couch_db:get_db_info(Db),
        proplists:get_value(disk_size,InfoProps)
    after
        couch_db:close(Db)
    end.


while_compaction_running(DbName) ->
    {ok, Db} = couch_db:open(DbName,[]),
    try
        {ok, InfoProps} = couch_db:get_db_info(Db),
        Still = proplists:get_value(compaction_running,InfoProps),
        if Still ->
            couch_db:close(Db),
            while_compaction_running(DbName);
           true ->
            ok
        end
    after
        couch_db:close(Db)
    end.

while_view_compaction_running(DbName,DDoc) ->
    {ok, Db} = couch_db:open(DbName,[]),
    try
        {ok, InfoProps} = couch_view:get_group_info(Db,DDoc),
        Still = proplists:get_value(compact_running,InfoProps),
        if Still ->
            couch_db:close(Db),
            while_view_compaction_running(DbName,DDoc);
           true ->
            ok
        end
    after
        couch_db:close(Db)
    end.

gen_view_and_query(DbName,NoDocs,ReductionSize) ->
    DDocName = <<"btree">>,
    {ok, {_Resp}} = hovercraft:save_doc(DbName, make_test_ddoc(DDocName)),
    {ok, _RevInfos} = make_test_docs(DbName, {[{<<"reduction">>, <<"value">>}]}, NoDocs),
    {ok, _Result} =
        case ReductionSize of
        small ->
            hovercraft:query_view(DbName, DDocName, <<"small">>);
        large ->
            hovercraft:query_view(DbName, DDocName, <<"large">>)
        end,
    %io:format("the results is ~p ~n",[Result]),
    ok.

make_test_ddoc(DesignName) ->
    {[
        {<<"_id">>, <<"_design/", DesignName/binary>>},
        {<<"views">>, {[
            {<<"small">>,
                {[
                    {<<"map">>,
<<"function(doc) { if(doc.reduction) emit(doc._id, 1) }">>},
                  {<<"reduce">>,
                    <<"function(ks,vs,co){ return sum(vs)}">>}
                ]}
            },{<<"large">>,
                {[
                    {<<"map">>,
<<"function(doc) { if(doc.reduction) emit(doc._id, doc.reduction) }">>},
                    {<<"reduce">>,
                    <<"function(ks,vs,co){ if (!co) {
                                              return vs;}
                                           else {
                                              var res = [];
                                              for each (var item in vs) {
                                                 var cnt = 0;
                                                 for each (var val in item) {
                                                    cnt = cnt + 1;
                                                  }
                                                  res = res.concat([cnt]);
                                              }
                                              return res;
                                                }}">>}
                ]}
            }
        ]}}
    ]}.


make_test_docs(DbName, Doc, Count) ->
    Docs = [Doc || _Seq <- lists:seq(1, Count)],
    hovercraft:save_bulk(DbName, Docs).