file performance testing

dynamic_file.erl

-module(dynamic_file).

%% notes:
%% http://www.matisse.net/bitcalc/?input_amount=17179869184&input_units=bits&notation=legacy
%% http://20bits.com/articles/network-programming-in-erlang/

%% ------------------------------------------------------------------
%% API Function Exports
%% ------------------------------------------------------------------

%-export([read/1, write/2, configure/0]).
-export([test/0, create_file_slow/3, create_file/3, clear/0]).
-define(MIN_TEST_FILE_SIZE, 1024).          %% 1k
-define(MAX_TEST_FILE_SIZE, 32768).   
%-define(MAX_TEST_FILE_SIZE, 17179869184).   %% 2 gigabytes

%% ------------------------------------------------------------------
%% Function Definitions
%% ------------------------------------------------------------------

%% dynamicaly determine the correct file size for reading and writing
%% files on the node's host os
%configure() ->
%    configure(?MIN_TEST_FILE_SIZE).
%
%configure(FileSize) when FileSize =< ?MAX_TEST_FILE_SIZE -> 
%    runtest(FileSize),
%    configure(FileSize * 2);
%configure(FileSize) when FileSize > ?MAX_TEST_FILE_SIZE ->
%    reconcile_results().



% read(FileName) ->

%write(FileName, Blob) ->
%    case file:open(FileName, [write, raw, binary] of
%        {ok, Fd} 

clear() ->
    ets:delete(file_test).

test() ->
    ets:new(file_test, [bag, named_table]),
    test(?MIN_TEST_FILE_SIZE).

test(Size) when Size =< ?MAX_TEST_FILE_SIZE ->
    ok = do_test(Size),
    test(Size * 2);
test(Size) when Size > ?MAX_TEST_FILE_SIZE ->
    io:format("-----------------------------------------~n", []),
    io:format("create_file_slow~n", []),
    io:format("-----------------------------------------~n", []),
    lists:map(fun({TestName, TestTime, TestSize, BytesPerSec, BlockSize}) -> 
                io:format("~w,~w,~w,~w,~w,~n", [TestName, TestTime, TestSize, BytesPerSec, BlockSize]) end, ets:lookup(file_test, create_file_slow)),
    io:format("-----------------------------------------~n", []),
    io:format("create_file (fast)~n", []),
    io:format("-----------------------------------------~n", []),
    lists:map(fun({TestName, TestTime, TestSize, BytesPerSec, BlockSize}) -> 
                io:format("~w,~w,~w,~w,~w,~n", [TestName, TestTime, TestSize, BytesPerSec, BlockSize]) end, ets:lookup(file_test, create_file)),
    ok.

do_test(Size) ->
    Dir = "/home/user/temp",
    File = Dir ++ "/test-file",
    ok = filelib:ensure_dir(Dir),
    ok = time_test(create_file_slow, File, Size, 1),
    ok = do_fast_test(File, Size, 1),
    ok.

do_fast_test(File, Size, BlockSize) when BlockSize =< 4096 ->
    ok = time_test(create_file, File, Size, BlockSize),
    do_fast_test(File, Size, BlockSize * 2),
    ok;
do_fast_test(_File, _Size, _BlockSize) ->
    ok.

time_test(Test, File, Size, BlockSize) ->
    file:delete(File),
    {Time, ok} = timer:tc(?MODULE, Test, [File, Size, BlockSize]),
    BytesPerSec = Size/Time,
    Result = {Test, Time, Size, BytesPerSec, BlockSize},
    ets:insert(file_test, Result),
    ok.

create_file_slow(Name, N, BlockSize) when is_integer(N), N >= 0 ->
    {ok, FD} = file:open(Name, [raw, write, delayed_write, binary]),
    ok = create_file_slow(FD, 0, N, BlockSize),
    ok = file:close(FD),
    ok.
      
create_file_slow(_FD, M, M, _BlockSize) ->
    ok;
create_file_slow(FD, M, N, BlockSize) ->
    ok = file:write(FD, <<M:32/unsigned>>),
    create_file_slow(FD, M+1, N, BlockSize).

%% Name : file name
%% N : size of file
create_file(Name, N, BlockSize) when is_integer(N), N >= 0 ->
    {ok, FD} = file:open(Name, [raw, write, delayed_write, binary]),
    ok = generate_file_blocks(FD, 0, N, BlockSize),
    ok = file:close(FD),
    ok;
create_file(_Name, _N, _BlockSize) ->
    ok.
     
%% FD : file descriptor
%% M : current size
%% N : final size

% current size == final size
generate_file_blocks(_FD,_M, _M, _BlockSize) ->
    ok;
% current size + block =< final size
generate_file_blocks(FD, M, N, BlockSize) when M + BlockSize =< N ->
    create_binary(FD, M, M + BlockSize, []),
    generate_file_blocks(FD, M + BlockSize, N, BlockSize);
generate_file_blocks(FD, M, N, _BlockSize) ->
    create_binary(FD, M, N, []).
      
% current size == final size, so write the blob to the file
create_binary(FD, M, M, R) ->
    ok = file:write(FD, R);
create_binary(FD, M, N0, R) when M + 8 =< N0 ->
    N1  = N0-1,  N2  = N0-2,  N3  = N0-3,  N4  = N0-4, 
    N5  = N0-5,  N6  = N0-6,  N7  = N0-7,  N8  = N0-8, 
    create_binary(FD, M, N8, 
                [<<N8:32/unsigned,  N7:32/unsigned, 
                   N6:32/unsigned,  N5:32/unsigned, 
                   N4:32/unsigned,  N3:32/unsigned, 
                   N2:32/unsigned,  N1:32/unsigned>> | R]);
create_binary(FD, M, N0, R) ->
    N1 = N0-1,
    create_binary(FD, M, N1, [<<N1:32/unsigned>> | R]).