Book Cloud Node :test connect nodes.

cloud_server.erl

%%%-------------------------------------------------------------------
%%% @author <zhongwencool@gmail.com>
%%% @doc run :
%%%    erl -name cloud_server@127.0.0.1 -pa "../ebin/" -setcookie best -run cloud_server start_link 
%%%   erl -name server0@127.0.0.1 -pa "../ebin/" -run deal_book_server -extra server0@127.0.0.1 
%%%   erl -name server1@127.0.0.1 -pa "../ebin/" -run deal_book_server -extra server1@127.0.0.1 
%%%    ....... 
%%%   erl -name server9@127.0.0.1 -pa "../ebin/" -run deal_book_server -extra server9@127.0.0.1
%%% Created : 11 Apr 2014
%%%-------------------------------------------------------------------
-module(cloud_server).

-behaviour(gen_server).

-export([
    add/4,
    delete/1,
    modify/2,
    sum/0,
    reset/0,
    stop/0,
    query_book/1,
    svr_test/0
]).

%% API
-export([start_link/0,i/0]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
    terminate/2, code_change/3]).


%% svrs::[{svr_name,svr_pid}]
%% name_keys::[{name,key}] like [{"zhongwen",1},{zhongwen2",2}],the key is uniqueness
%% unique_key :: begin from 0  unique_key++ when add a new name.%%use for name_keys (key)
-record(state, {svrs = [],name_keys=[],unique_key = 0}).

-record(person,{name="",address="",phone_number=0,email=""}).

%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
start_link() ->
    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).

%% for debugger look state
i() ->
    call(i).

call(Msg) ->
    gen_server:call(?MODULE, Msg, infinity).

%% return ok|{error, Reason}
add(Name, Address, PhoneNumber, Email) ->
    AddPerson = #person{name = Name, address = Address, phone_number = PhoneNumber, email = Email},
    call({add,AddPerson}).

%% return #person|{error, Reason}
query_book(Name) ->
    call({query_book,Name}).

%% return ok|{error, Reason}
modify(Name,{Field, Value}) ->
    call({modify,Name,Field,Value}).

%% return ok|{error, Reason}. 
delete(Name) ->
    call({delete,Name}).

%% return the number of records. return Int.
sum() ->
    call(sum).

%% reset the server, all data & state is clear.
reset() ->
    todo.

stop() ->
    gen_server:cast(?MODULE, stop).

%%%===================================================================
%%% gen_server callbacks
%%%===================================================================

%%--------------------------------------------------------------------
init([]) ->
    %% make sure send {nodedown,servername} to cloud server when svr0~9 nodedown
    ok = net_kernel:monitor_nodes(true),
    {ok, #state{}}.

%%--------------------------------------------------------------------
handle_call({add,Person},_From,#state{svrs = Svrs,name_keys = NameKeys,unique_key=UniqueKey}=State) ->
    case do_add_person(Person,Svrs,NameKeys,UniqueKey) of
        {ok,NewNameKeys,NewUniqueKey} ->
            {reply,ok,State#state{name_keys = NewNameKeys,unique_key = NewUniqueKey}};
        {error,_Reason} = Err ->
            {reply,Err,State}
    end;

handle_call({query_book,Name},_From,State) ->
    Reply = do_query(Name,State),
    {reply,Reply,State};

handle_call({modify,Name,Field,Value},_From,State) ->
    case do_modify(Name,Field,Value,State) of
        ok ->
            {reply,ok,State};
        {error,_Reason} =Err ->
            {reply,Err,State}
    end;

handle_call({delete,Name},_From,State) ->
    case do_delete(Name,State) of
        {ok,NewState} ->
            {reply,ok,NewState};
        {error,_Reason}=Error ->
            {reply,Error,State}
    end;

handle_call(sum,_From,State = #state{name_keys =NameKeys}) ->
    {reply,erlang:length(NameKeys),State};

handle_call(i,_From,State) ->
    {reply,{ok,State},State};

handle_call(Request, _From, BookList) ->
    io:format("unknow_call_msg:Request:~p",[Request]),
    {reply, ok, BookList}.

%%--------------------------------------------------------------------
handle_cast(_Msg, State) ->
    {noreply, State}.

%%--------------------------------------------------------------------
%% book_server start syns pid
handle_info({book_server_to_cloud,SvrPID,SvrName,BackupSvrName},State=#state{svrs = Svrs}) ->
    NewState = do_change_book_server(SvrPID,SvrName,State),
    BackupData = get_backup_data(SvrName,BackupSvrName,Svrs),
    erlang:send(SvrPID,{clound_respond_server_ok,self(),BackupData}),
    {noreply,NewState};

handle_info({nodedown, NodeName},State=#state{svrs = Svrs})  ->
    NewSvrs = do_node_down(NodeName,Svrs),
    {noreply,State#state{svrs = NewSvrs}};

handle_info({svr_pid_change,SvrPID,SvrName},State) ->
    NewState = do_change_book_server(SvrPID,SvrName,State),
    erlang:send(SvrPID,{clound_respond_server_ok,self()}),
    {noreply,NewState};

handle_info(_Info, State) ->
    {noreply, State}.

%%--------------------------------------------------------------------
terminate(_Reason, _State) ->
    ok.

%%--------------------------------------------------------------------
code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%%%===================================================================
%% Internal functions
%%%===================================================================

%% update logical {svrname,pid} to cloud server
do_change_book_server(SvrPID,SvrName,State =#state{svrs = Svrs} ) ->
    NewSvrsT = lists:keydelete(SvrName,1,Svrs),
    NewSvrs = [{SvrName,SvrPID}|NewSvrsT],
    [begin erlang:send(SvrPIDT,{sync_svrs_from_cloud,NewSvrs}) end|| {_, SvrPIDT} <- NewSvrs],
    State#state{svrs = NewSvrs}.

get_backup_data(SvrName,BackupSvrName,Svrs) ->
    case lists:keyfind(BackupSvrName,1,Svrs) of
        false ->
            [];
        {_,SvrPID} ->
            catch gen_server:call(SvrPID,{get_backup_data,SvrName})
    end.

do_add_person_check(Name,Svrs,NameKeys,UniqueKey) ->
    case lists:keyfind(Name,1,NameKeys) of
        false  ->
            ok;
        _ ->
            erlang:throw({error,already_add})
    end,
    case genernate_new_uniquekey(UniqueKey,Svrs,10) of
        {error,_Reason} = Err  ->
            erlang:throw(Err);
        {ok,NewKey,SvrPID}  ->
            {ok,{Name,NewKey},SvrPID}
    end.

do_add_person(Person,Svrs,NameKeys,UniqueKey) ->
    case catch do_add_person_check(Person#person.name,Svrs,NameKeys,UniqueKey) of
        {ok,NewNameKey,SvrPID} ->
            do_add_person2(Person,NewNameKey,NameKeys,SvrPID);
        T  ->
            T
    end.

do_add_person2(Person,NewNameKey={_,NewUniqueKey},NameKeys,SvrPID) ->
    case deal_book_server:add(SvrPID,Person) of
        {error,_Reason} = Err ->
            Err;
        ok ->
            {ok,[NewNameKey|NameKeys],NewUniqueKey}
    end.

do_query(Name,#state{name_keys = NameKeys,svrs = Svrs}) ->
    case lists:keyfind(Name,1,NameKeys) of
        false ->
            {error,person_not_find};
        {_,QuiqueKey} ->
            ServerName = find_server_by_key(QuiqueKey rem 10),
            case lists:keyfind(ServerName,1,Svrs) of
                false ->
                    {error,server_no_alive_todo_backup};%% todo
                {_,SvrPID}  ->
                    deal_book_server:query_book(SvrPID,Name)
            end
    end.

do_modify(Name,Field,Value,#state{svrs = Svrs,name_keys = NameKeys}) ->
    case lists:keyfind(Name,1,NameKeys) of
        false ->
            {error, person_not_find};
        {_,QuiqueKey}  ->
            ServerName = find_server_by_key(QuiqueKey rem 10),
            case lists:keyfind(ServerName,1,Svrs) of
                false ->
                    {error,server_no_alive_todo_backup};%% todo
                {_,SvrPID} ->
                    deal_book_server:modify(SvrPID,Name,{Field,Value})
            end
    end.

do_delete(Name,State = #state{svrs = Svrs, name_keys = NameKeys}) ->
    case lists:keyfind(Name, 1, NameKeys) of
        false ->
            {error,person_not_find};
        {_,QuiqueKey} ->
            ServerName = find_server_by_key(QuiqueKey rem 10),
            case lists:keyfind(ServerName,1,Svrs) of
                false ->
                    {error,server_no_alive_todo_backup};
                {_,SvrPID} ->
                    case deal_book_server:delete(SvrPID,Name) of
                        ok ->
                            {ok,State#state{name_keys = lists:keydelete(Name,1,NameKeys)}};
                        {error,_Reason} =Err ->
                            Err
                    end
            end
    end.

genernate_new_uniquekey(_Key,w_Svrs,0)  ->
    {error,no_server_is_alive};
genernate_new_uniquekey(Key,Svrs,Counts) ->
    NewKey = Key+1,
    SvrName = find_server_by_key(NewKey rem 10),
    case lists:keyfind(SvrName,1,Svrs) of
        false ->
            genernate_new_uniquekey(NewKey,Svrs,Counts-1);
        {_,SvrPID} ->
            case rpc:call(SvrName,erlang,is_process_alive,[SvrPID]) of
                false ->
                    genernate_new_uniquekey(NewKey,Svrs,Counts-1);
                true ->
                    {ok,NewKey,SvrPID}
            end
    end.

do_node_down(NodeName,Svrs) ->
    io:format("node_down:~p~n",[NodeName]),
    NewSvrs = lists:keydelete(NodeName,1,Svrs),
    [begin erlang:send(SvrPIDT,{sync_svrs_from_cloud,NewSvrs}) end|| {_, SvrPIDT} <- NewSvrs],
    NewSvrs.

%% do not want to use erlang:list_to_atom("servername"++erlang:integer_to_list(12)).
find_server_by_key(0) ->
    'server0@127.0.0.1';
find_server_by_key(1) ->
    'server1@127.0.0.1';
find_server_by_key(2) ->
    'server2@127.0.0.1';
find_server_by_key(3) ->
    'server3@127.0.0.1';
find_server_by_key(4) ->
    'server4@127.0.0.1';
find_server_by_key(5) ->
    'server5@127.0.0.1';
find_server_by_key(6) ->
    'server6@127.0.0.1';
find_server_by_key(7) ->
    'server7@127.0.0.1';
find_server_by_key(8) ->
    'server8@127.0.0.1';
find_server_by_key(9) ->
    'server9@127.0.0.1'.

svr_test() ->
    ok = add("zhongwen","China Shanghai",138000001,"zhongwencool@gmail.com"),
    1 = sum(),
    {ok,#person{name = "zhongwen",address = "China Shanghai", phone_number = 138000001,email = "zhongwencool@gmail.com"}} = query_book("zhongwen"),
    %% add 100 record
    [begin
         Name = "zhongwen"++erlang:integer_to_list(Num),
         ok = add(Name,"China Shanghai",138000001,"zhongwencool@gmail.com") end|| Num <- lists:seq(1,100)],
    101= sum(),
    ok = delete("zhongwen3"),
    {error,person_not_find} = delete("zhongwen3"),
    {error,already_add} = add("zhongwen","test",138000001,"zhongwencool@gmail.com"),
    ok = modify("zhongwen5",{phone_number,19027132875}),
    {ok,#person{name = "zhongwen5", address = "China Shanghai", phone_number = 19027132875,email = "zhongwencool@gmail.com"}} = query_book("zhongwen5"),
    ok.

deal_book_server.erl

%%%-------------------------------------------------------------------
%%% @author <zhongwencool@gmail.com>
%%% @doc handle the actual logic  book_server
%%%
%%% Created :  9 Apr 2014
%%%-------------------------------------------------------------------
-module(deal_book_server).

-behaviour(gen_server).

%% API
-export([start_link/0,
    i/0,
    start/0,
    add/2,
    delete/2,
    modify/3,
    stop/0,
    query_book/2
]).

%% gen_server callback
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
    terminate/2, code_change/3]).

-record(person,{name="",address="",phone_number=0,email=""}).

%%cloud_state:: undefined|node_connected|{available,CloudPID}
%%booklist:: [#person{},#person{}] local svr data
%%backup_data::[{svr num+1,BackupList},{svr num+2,BackupList}]
%%changes::local svr changes(add,delete,modify) record when num>=5 run backup
%%svrs::[{svr_name,svr_pid}] for svr1 send backup data to svr2(PID)%% this data sync from the cloud svrs
-record(state,{cloud_state,booklist=[],backup_data=[],svr_name,changes=[],svrs = []}).

%%%===================================================================
%%% API
%%%===================================================================

call(PID,Msg)  ->
    gen_server:call(PID,Msg,infinity).

%% @doc for debugger :look state
i() ->
    call( get_svrname(), i).

%%--------------------------------------------------------------------
%%
start() ->
    SvrName = get_svrname(),
    gen_server:start({local,SvrName},?MODULE,[SvrName],[]).

start_link() ->
    gen_server:start_link({local, ?MODULE}, ?MODULE, [?MODULE], []).

%%--------------------------------------------------------------------
%%
%% return ok|{error, Reason}
add(SvrPID,AddPerson) ->
    call(SvrPID,{add,AddPerson}).

%% return #person|{error, Reason}
query_book(SvrPID,Name) ->
    call(SvrPID,{query_book,Name}).

%% return ok|{error, Reason}
modify(SvrPID,Name,{Field, Value}) ->
    call(SvrPID,{modify,Name,Field,Value}).

%% return ok|{error, Reason}.
delete(SvrPID,Name) ->
    call(SvrPID,{delete,Name}).

stop() ->
    ServerName = get_svrname(),
    gen_server:cast(ServerName, stop).

%%%===================================================================
%%%
%%%===================================================================
%%% gen_server callbacks
%%%===================================================================

%%--------------------------------------------------------------------
init([SvrName]) ->
    erlang:process_flag(trap_exit,true),
    erlang:send_after(1000,self(),loop_second),
    erlang:send_after(15000,self(),loop_15_second),
    [Backup1,Backup2] = find_backup_on_svr(SvrName),
    {ok, #state{svr_name=SvrName,backup_data = [{Backup1,[]},{Backup2,[]}]}}.

%%--------------------------------------------------------------------
handle_call({add,Person},_From,State = #state{booklist = BookList,changes = Changes}) ->
    case do_add_person(Person,BookList) of
        {ok,NewBookList} ->
            {reply,ok,State#state{booklist = NewBookList,changes = [{add,Person}|Changes]}};
        {error,_Reason} = Err ->
            {reply,Err,State}
    end;

handle_call({query_book,Name},_From,State=#state{booklist =BookList} ) ->
    Reply = do_query(Name,BookList),
    {reply,Reply,State};

handle_call({modify,Name,Field,Value},_From,State=#state{booklist =BookList,changes = Changes}) ->
    case do_modify(Name,Field,Value,BookList) of
        {ok,NewBookList,Person} ->
            {reply,ok,State#state{booklist =NewBookList,changes = [{modify,Person}|Changes]}};
        {error,_Reason} =Err ->
            {reply,Err,State#state{booklist =BookList}}
    end;

handle_call({delete,Name},_From,State=#state{booklist =BookList,changes = Changes}) ->
    case do_delete(Name,BookList) of
        {ok,NewBookList} ->
            {reply,ok,State#state{booklist =NewBookList,changes = [{delete,Name}|Changes]}};
        {error,_Reason}=Error ->
            {reply,Error,State}
    end;

handle_call(sum,_From,State=#state{booklist =BookList}) ->
    {reply,do_sum(BookList),State};

handle_call({backup_to_you,DataSvrName,Changes},_From,State) ->
    NewState = do_backup(DataSvrName,Changes,State),
    {reply,ok,NewState};

handle_call({get_backup_data,SvrName},_From,State=#state{backup_data=BackupData}) ->
    SvrBackupData = case lists:keyfind(SvrName,1,BackupData) of
                        false -> [];
                        {_,BackupDataT} -> BackupDataT
                    end,
    {reply,SvrBackupData,State};

handle_call(i,_From,State) ->
    {reply,{ok,State},State};

handle_call(Request, _From, BookList) ->
    io:format("unknow_call_msg:Request:~p",[Request]),
    {reply, ok, BookList}.

%%--------------------------------------------------------------------
handle_cast(stop,State) ->
    {stop, normal, State};

handle_cast(_Msg, State) ->
    {noreply, State}.

%%--------------------------------------------------------------------
handle_info(loop_second,State) ->
    erlang:send_after(1000,self(),loop_second),
    NewState = do_loop(State),
    {noreply,NewState};

handle_info(loop_15_second,State) ->
    erlang:send_after(15000,self(),loop_15_second),
    do_15_loop(State),
    {noreply,State};

handle_info(connect_center_ok,State) ->
    {noreply,State#state{cloud_state= node_connected}};

handle_info({clound_respond_server_ok,CloudPID,BackupData},State) ->
    {noreply,State#state{cloud_state={available,CloudPID},booklist = BackupData}};

handle_info({sync_svrs_from_cloud,NewSvrs},State) ->
    {noreply,State#state{svrs= NewSvrs}};

%%handle_info({nodedown, NodeName},State) ->
%%    do_node_down(NodeName),
%%    {noreply,State};

handle_info(_Info, State) ->
    io:format("unknow msg info :~p~n",[_Info]),
    {noreply,State}.
%%--------------------------------------------------------------------
terminate(_Reason, _State) ->
    ok.

%%--------------------------------------------------------------------
code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%%%===================================================================
%% Internal functions
%%%===================================================================

%% add person
do_add_person_check(Person,BookList) ->
    case lists:keymember(Person#person.name,#person.name,BookList) of
        true ->
            erlang:throw({error,already_add});
        false ->
            {ok,[Person|BookList]}
    end.


do_add_person(Person,BookList) ->
    case catch do_add_person_check(Person,BookList)  of
        {error,_Reason}=Error  ->
            Error;
        {ok,_NewBookList}=Ok ->
            Ok
    end.

%% query_person
do_query(Name,BookList) ->
    case lists:keyfind(Name,#person.name,BookList) of
        false ->
            {error,person_not_find};
        Person ->
            {ok,Person}
    end.

do_modify_check(Name,Field,Value,BookList) ->
    case lists:keytake(Name,#person.name,BookList) of
        false ->
            {error,person_not_find};
        {value,T,LeftList} ->
            KeyPos = get_key_pos(Field),
            case erlang:is_integer(KeyPos) of
                true ->
                    NewT = erlang:setelement(KeyPos,T,Value),
                    {ok,[NewT|LeftList],NewT};
                false ->
                    {error,field_error}
            end
    end.

%%modify(Name,Field,Value)
do_modify(Name,Field,Value,BookList) ->
    case do_modify_check(Name,Field,Value,BookList) of
        {ok,_NewBookList,_Person} = Ok ->
            Ok;
        {error,_Reason} =Err ->
            Err
    end.

%% -record(person,{name="",address="",phone_number=0,email=""}).
get_key_pos(name) ->
    2;
get_key_pos(address) ->
    3;
get_key_pos(phone_number) ->
    4;
get_key_pos(email) ->
    5;
get_key_pos(_) ->
    undefined.

%% delete
do_delete(Name,BookList) ->
    case lists:keymember(Name,#person.name,BookList) of
        false ->
            {error,person_not_find};
        true ->
            {ok,lists:keydelete(Name,#person.name,BookList)}
    end.

%%sum
do_sum(BookList) ->
    erlang:length(BookList).

do_backup(DataSvrName,Changes,State = #state{backup_data=BackupData}) ->
    {_,BackupList} =  lists:keyfind(DataSvrName,1,BackupData),
    NewBackupList = lists:foldl(
        fun({modify,Change},Acc) ->
            case lists:keytake(Change,#person.name,Acc)  of
                false ->
                    [Change|Acc];
                {value,_T,Left}  ->
                    [Change|Left]
            end;
            ({add,Change},Acc) ->
                [Change|Acc];
            ({delete,Name},Acc) ->
                lists:keydelete(Name,#person.name,Acc)
        end,BackupList,Changes),
    State#state{backup_data = lists:keyreplace(DataSvrName,1,BackupData,{DataSvrName,NewBackupList})}.

%% not connect cloud node yet
do_loop( State = #state{cloud_state=undefined}) ->
    io:format("connecting cloud server node~n "),
    SelfPID = erlang:self(),
    erlang:spawn(fun() -> do_connect_cloud(SelfPID) end),
    State;

%% node connected but not sure cloud server is alive
do_loop( State = #state{cloud_state= node_connected}) ->
    io:format("confriming cloud server is ok:~n"),
    connect_to_cloud_server(),
    State;

do_loop( #state{cloud_state={available,_CloudPID}}=State) ->
    backup_to_other_server(State).

%% check svr pid is change (restart by error) every 15s
do_15_loop(#state{cloud_state = {available,CloudPID},svr_name=SvrName,svrs =Svrs} ) ->
    check_svr_pid(SvrName,Svrs,CloudPID),
    ok;
do_15_loop(_) ->
    ok.

do_connect_cloud(ParentPID) ->
    true = erlang:set_cookie('cloud_server@127.0.0.1', 'best'),
    case net_kernel:connect_node('cloud_server@127.0.0.1') of
        true ->
            erlang:send(ParentPID,connect_center_ok);
        _ ->
            ignore
    end.

%% connect to cloud_server app
connect_to_cloud_server() ->
    case rpc:call('cloud_server@127.0.0.1', erlang, whereis, [cloud_server]) of
        CloudPID when erlang:is_pid(CloudPID)  ->
            SvrName = get_svrname(),
            %% get backupdata from backsvr1 when book server start normal
            [BackupSvrName1,_] = find_backup_svr(SvrName),
            RegisterMsg={book_server_to_cloud,self(),SvrName,BackupSvrName1},
            erlang:send(CloudPID,RegisterMsg);
        _ ->
            net_kernel:disconnect('cloud_server@127.0.0.1')
    end.

backup_to_other_server(#state{changes=Changes,svrs = Svrs,svr_name = SvrName} = State)
    when erlang:length(Changes) >= 5 ->
    [begin
         case lists:keyfind(BackupSvr,1,Svrs) of
             false -> error;
             {_,BackupSvrPID} ->
                 catch call(BackupSvrPID,{backup_to_you,SvrName,Changes}) %% todo   fault tolerant
         end
     end || BackupSvr <- find_backup_svr(SvrName)],
    State#state{changes=[]};
backup_to_other_server(State) ->
    State.

check_svr_pid(SvrName,Svrs,CloudPID) ->
    Self = self(),
    case lists:keyfind(SvrName,1,Svrs) of
        {_,Self} ->
            ok;
        _  ->
            erlang:send(CloudPID,{svr_pid_change,Self,SvrName})
    end.

%% @doc get svrname from shell start .i set  svrname=:=node()
get_svrname() ->
    [SvrName1] = init:get_plain_arguments(),
    erlang:list_to_atom(SvrName1).

%% do not want to use erlang:list_to_atom("servername"++erlang:integer_to_list(12)).
%% server num backup to server(num+1),server(num+2)
find_backup_svr('server0@127.0.0.1') ->
    ['server1@127.0.0.1','server2@127.0.0.1'];
find_backup_svr('server1@127.0.0.1') ->
    ['server2@127.0.0.1','server3@127.0.0.1'];
find_backup_svr('server2@127.0.0.1') ->
    ['server3@127.0.0.1','server4@127.0.0.1'];
find_backup_svr('server3@127.0.0.1') ->
    ['server4@127.0.0.1','server5@127.0.0.1'];
find_backup_svr('server4@127.0.0.1') ->
    ['server5@127.0.0.1','server6@127.0.0.1'];
find_backup_svr('server5@127.0.0.1') ->
    ['server6@127.0.0.1','server7@127.0.0.1'];
find_backup_svr('server6@127.0.0.1') ->
    ['server7@127.0.0.1','server8@127.0.0.1'];
find_backup_svr('server7@127.0.0.1') ->
    ['server8@127.0.0.1','server9@127.0.0.1'];
find_backup_svr('server8@127.0.0.1') ->
    ['server9@127.0.0.1','server0@127.0.0.1'];
find_backup_svr('server9@127.0.0.1') ->
    ['server0@127.0.0.1','server1@127.0.0.1'].

%%
find_backup_on_svr('server3@127.0.0.1') ->
    ['server1@127.0.0.1','server2@127.0.0.1'];
find_backup_on_svr('server4@127.0.0.1') ->
    ['server2@127.0.0.1','server3@127.0.0.1'];
find_backup_on_svr('server5@127.0.0.1') ->
    ['server3@127.0.0.1','server4@127.0.0.1'];
find_backup_on_svr('server6@127.0.0.1') ->
    ['server4@127.0.0.1','server5@127.0.0.1'];
find_backup_on_svr('server7@127.0.0.1') ->
    ['server5@127.0.0.1','server6@127.0.0.1'];
find_backup_on_svr('server8@127.0.0.1') ->
    ['server6@127.0.0.1','server7@127.0.0.1'];
find_backup_on_svr('server9@127.0.0.1') ->
    ['server7@127.0.0.1','server8@127.0.0.1'];
find_backup_on_svr('server0@127.0.0.1') ->
    ['server8@127.0.0.1','server9@127.0.0.1'];
find_backup_on_svr('server1@127.0.0.1') ->
    ['server9@127.0.0.1','server0@127.0.0.1'];
find_backup_on_svr('server2@127.0.0.1') ->
    ['server0@127.0.0.1','server1@127.0.0.1'].