Erlang module for Tsung to extract data from MySQL

elasticsearch.erl

-module(elasticsearch).
-export([autocomplete/1, autocomplete/0,
         search/1, search/0,
         start/0,
         stop/0,
         loop/1]).
-on_load(start/0).

%% Return an autocomplete request
autocomplete() ->
    autocomplete(true).
autocomplete(_) ->
    elasticsearch_autocomplete ! {next, self()},
    receive
        {ok, Json} -> Json
    end.

%% Return a search request
search() ->
    search(true).
search(_) ->
    elasticsearch_search ! {next, self()},
    receive
        {ok, Json} -> Json
    end.

%% Start/stop SQL processes
start() ->
    true = start(elasticsearch_autocomplete, "autocomplete4"),
    true = start(elasticsearch_search, "top"),
    ok.
start(Process, Table) ->
    register(Process,
             spawn(?MODULE, loop, [Table])).
stop() ->
    true = stop(elasticsearch_autocomplete) and
        stop(elasticsearch_search),
    ok.
stop(Process) ->
    Process ! {kill, self()},
    receive
        ok -> true;
        _ -> false
    end.

%% Main loop for SQL processes
loop(Table) ->
    odbc:start(),
    {ok, Ref} = odbc:connect("DSN=gatling", []),
    loop(Ref, Table, 0).

%% Start with an empty query buffer
loop(Ref, Table, Start) ->
    loop(Ref, Table, Start, []).

%% When buffer is empty, retrieve more results from DB
loop(Ref, Table, Start, []) ->
    {selected, _, Results} =
        odbc:sql_query(Ref,
                       % Force index as MySQL won't use it for such requests
                       string:join(["SELECT request FROM",
                                    Table,
                                    "FORCE INDEX FOR ORDER BY (PRIMARY)",
                                    "ORDER BY id ASC",
                                    "LIMIT 1000 OFFSET",
                                    integer_to_list(Start)
                                   ], " ")),
    % If no more results are available, just start from the beginning
    case Results of
        [_|_] ->
            loop(Ref, Table, Start, Results);
        [] ->
            loop(Ref, Table, 0, [])
    end;

%% When we still have results in memory, just use them
loop(Ref, Table, Start, [Json|Remaining]) ->
    receive
        {next, From} ->
            From ! {ok, Json},
            loop(Ref, Table, Start + 1, Remaining);
        {kill, From} ->
            From ! ok
    end.

elasticsearch.xml

<?xml version="1.0"?>
<!DOCTYPE tsung SYSTEM "/usr/share/tsung/tsung-1.0.dtd">
<tsung loglevel="notice" version="1.0">
  <clients>
    <client host="localhost" use_controller_vm="true" maxusers="50000"/>
  </clients>
  
  <servers>
    <server host="es-18.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-19.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-20.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-21.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-22.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-23.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-24.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-25.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-26.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-27.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-28.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-29.adm.dailymotion.com" port="9200" type="tcp"></server>
    <server host="es-30.adm.dailymotion.com" port="9200" type="tcp"></server>
  </servers>

  <monitoring>
    <monitor host="es-18.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-19.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-20.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-21.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-22.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-23.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-24.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-25.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-26.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-27.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-28.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-29.adm.dailymotion.com" type="snmp"></monitor>
    <monitor host="es-30.adm.dailymotion.com" type="snmp"></monitor>
  </monitoring>
  
  <load>
   <arrivalphase phase="0" duration="30" unit="second">
     <users arrivalrate="200" unit="second"></users>
   </arrivalphase>

   <arrivalphase phase="1" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="2" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="3" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="4" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="5" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="6" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="7" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="8" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="9" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
   <arrivalphase phase="10" duration="30" unit="second">
     <users maxnumber="1000" arrivalrate="100" unit="second"></users>
   </arrivalphase>
  </load>

  <options>
   <option type="ts_http" name="user_agent">
    <user_agent probability="100">Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36</user_agent>
   </option>
  </options>

 <sessions>
  <session name="autocomplete" probability="100" type="ts_http">
    <for from="1" to="10000" incr="1" var="counter">
      <transaction name="autocomplete">
        <request subst="true">
          <http url="/autocomplete/suggest/_search" method="POST" version="1.1"
                contents="%%elasticsearch:autocomplete%%"
                content_type="application/json">
          </http>
        </request>
      </transaction>
      <thinktime min="1" max="5" random="true"/>
      <transaction name="search">
        <request subst="true">
          <http url="/video/video/_search" method="POST" version="1.1"
                contents="%%elasticsearch:search%%"
                content_type="application/json">
          </http>
        </request>
      </transaction>
      <thinktime min="2" max="10" random="true"/>
    </for>
  </session>
 </sessions>
</tsung>