Simple Erlang message passing benchmark

bench.erl

-module(bench).

-export([run_suite/3, run/1]).

run_suite(Name, Passes, Messages) ->
    Results = run_suite1(Passes, Messages, []),
    Avg = lists:sum(Results) / Passes,
    StdDev = math:sqrt(lists:sum([math:pow(X - Avg, 2) || X <- Results]) / Passes),
    Summary = [{avg, Avg}, {stddev, StdDev}, {min, lists:min(Results)},
               {max, lists:max(Results)}],
    file:write_file("/Users/ksmith/results.txt", io_lib:format("[{name, ~p}, {summary, ~p}, {details, ~p}].~n",
                                                               [Name, Summary, Results]), [append]).

run_suite1(0, _Messages, Results) ->
    lists:reverse(Results);
run_suite1(Passes, Messages, Results) ->
    erlang:garbage_collect(),
    run_suite1(Passes - 1, Messages, [run(Messages) | Results]).

run(Count) ->
    Me = self(),
    Start = erlang:now(),
    F = spawn(fun() -> worker(Me, Count) end),
    spawn(fun() -> worker(Me, F, Count) end),
    wait_for_done(2),
    timer:now_diff(erlang:now(), Start).

wait_for_done(0) ->
    ok;
wait_for_done(Count) ->
    receive
        done ->
            wait_for_done(Count - 1)
    end.

worker(Owner, 0) ->
    Owner ! done;
worker(Owner, Count) ->
    receive
        {From, ping} ->
            From ! pong,
            worker(Owner, Count - 1)
    end.

worker(Owner, _Target, 0) ->
    Owner ! done;
worker(Owner, Target, Count) ->
    Target ! {self(), ping},
    receive
        pong ->
            worker(Owner, Target, Count - 1)
    end.

results.md

[{name, "R16B async-threads:0"}, {summary, [{avg,3540307.4},
                                            {stddev,57870.82022781429},
                                            {min,3485352},
                                            {max,3620480}]}, {details, [3620480,
                                                                        3510523,
                                                                        3485352,
                                                                        3486023,
                                                                        3599159]}].
[{name, "R16B async-threads:4"}, {summary, [{avg,3554938.6},
                                            {stddev,27788.99938176976},
                                            {min,3525254},
                                            {max,3598165}]}, {details, [3598165,
                                                                        3571298,
                                                                        3525903,
                                                                        3554073,
                                                                        3525254]}].
[{name, "R15B01 async-threads:0"}, {summary, [{avg,988024.0},
                                              {stddev,5657.500826336661},
                                              {min,982552},
                                              {max,996927}]}, {details, [996927,
                                                                         985344,
                                                                         982552,
                                                                         982992,
                                                                         992305]}].
[{name, "R15B01 async-threads:4"}, {summary, [{avg,983087.2},
                                              {stddev,4695.029857200058},
                                              {min,976180},
                                              {max,988886}]}, {details, [987625,
                                                                         988886,
                                                                         976180,
                                                                         980275,
                                                                         982470]}].
[{name, "R14B03 async-threads:0"}, {summary, [{avg,790603.8},
                                              {stddev,21405.434799601713},
                                              {min,755043},
                                              {max,819714}]}, {details, [755043,
                                                                         791333,
                                                                         819714,
                                                                         802467,
                                                                         784462]}].
[{name, "R14B03 async-threads:0"}, {summary, [{avg,741706.2},
                                              {stddev,8959.760474476981},
                                              {min,735944},
                                              {max,759469}]}, {details, [759469,
                                                                         735944,
                                                                         735947,
                                                                         738484,
                                                                         738687]}].

NOTE: Informal benchmark written in a few spare minutes.

Each tests consisted of 5 runs of 1 million messages in each run. Test machine is a 2.7ghz Core i7 2011 MBP.

I could replicate the results on my end.

I could also fix the issue by setting the async thread pool to 0 in R16 (with +A0 as an argument to erl -- The OTP team set the default to 10 in R16). There is likely some overhead to the default async thread pool that shows up in synthetic benchmarks, but proves really useful in real world scenarios.

Consider my curiosity sated. Synthetic benchmarks are only worth so much. Erlang continues to rock in the Real World for Real Problems.

[{name, "LING-0.2.1"}, {summary, [{avg,942989.6},
{stddev,18810.388338362394},
{min,925130},
{max,969332}]}, {details, [969332,930412,
962208,927866,
925130]}].

Unoptimized Ling VM (http://erlangonxen.org) on a single core of Intel Xeon E3-1245 3.3GHz

See OTP-10736 at http://www.erlang.org/download/otp_src_R16B.readme

OTP-10736 The runtime system will now by default use 10 async threads if thread support has been enabled when building the runtime system.

This will prevent long blocking file-operations from blocking scheduler threads for long periods of time, which can be harmful. Apart from file-operations, it also effects other operations scheduled on the async thread pool by user implemented drivers.

The amount of async threads can be controlled by using the +A command line argument of erl(1). When running some offline tools you might want to disable async threads, but you are advised not to in the general case. Instead, you might want to increase the amount of async threads used.

This change imply changes of the characteristics the system compared to the previous default. The responsiveness of the system as a whole will be improved. Operations scheduled on the async thread pool will get an increased latency. The throughput of these operations may increase, or decrease depending on the type of the operations and how they get scheduled. In the case of file operations, the throughput very much depends on how the Erlang application access files. Multiple concurrent accesses to different files have the potential of an increased throughput.