[root@d-kvs-01 riak]# riak version
Attempting to restart script through sudo -H -u riak
riak-ee (1.2.0-92bc3fa0 2012-08-27) RedHat x86_64
[root@d-kvs-01 ~]# riak-admin cluster leave
Attempting to restart script through sudo -H -u riak
Success: staged leave request for 'kvs@d-kvs-01.tradecard.com'
[root@d-kvs-01 ~]# riak-admin cluster plan
Attempting to restart script through sudo -H -u riak
=============================== Staged Changes ================================
Action Nodes(s)
-------------------------------------------------------------------------------
leave 'kvs@d-kvs-01.tradecard.com'
-------------------------------------------------------------------------------
NOTE: Applying these changes will result in 1 cluster transition
###############################################################################
After cluster transition 1/1
###############################################################################
================================= Membership ==================================
Status Ring Pending Node
-------------------------------------------------------------------------------
leaving 25.0% 0.0% 'kvs@d-kvs-01.tradecard.com'
valid 25.0% 33.4% 'kvs@d-kvs-02.tradecard.com'
valid 25.0% 33.4% 'kvs@d-kvs-03.tradecard.com'
valid 25.0% 33.2% 'kvs@d-kvs-04.tradecard.com'
-------------------------------------------------------------------------------
Valid:3 / Leaving:1 / Exiting:0 / Joining:0 / Down:0
WARNING: Not all replicas will be on distinct nodes
Transfers resulting from cluster changes: 386
42 transfers from 'kvs@d-kvs-01.tradecard.com' to 'kvs@d-kvs-04.tradecard.com'
43 transfers from 'kvs@d-kvs-04.tradecard.com' to 'kvs@d-kvs-03.tradecard.com'
43 transfers from 'kvs@d-kvs-03.tradecard.com' to 'kvs@d-kvs-02.tradecard.com'
43 transfers from 'kvs@d-kvs-02.tradecard.com' to 'kvs@d-kvs-04.tradecard.com'
43 transfers from 'kvs@d-kvs-01.tradecard.com' to 'kvs@d-kvs-03.tradecard.com'
43 transfers from 'kvs@d-kvs-04.tradecard.com' to 'kvs@d-kvs-02.tradecard.com'
43 transfers from 'kvs@d-kvs-03.tradecard.com' to 'kvs@d-kvs-04.tradecard.com'
43 transfers from 'kvs@d-kvs-02.tradecard.com' to 'kvs@d-kvs-03.tradecard.com'
43 transfers from 'kvs@d-kvs-01.tradecard.com' to 'kvs@d-kvs-02.tradecard.com'
[root@d-kvs-01 ~]# riak-admin cluster commit
Attempting to restart script through sudo -H -u riak
Cluster changes committed
2012-08-30 22:04:01.135 [info] <0.22219.0>@riak_core_handoff_sender:start_fold:126 Starting ownership_handoff transfer of riak_kv_vnode from 'kvs@d-kvs-01.tradecard.com' 11417981541647679048466287755595961091061972992 to 'kvs@d-kvs-03.tradecard.com' 11417981541647679048466287755595961091061972992
2012-08-30 22:04:03.658 [info] <0.198.0>@riak_core_handoff_manager:handle_info:271 An outbound handoff of partition riak_kv_vnode 0 was terminated for reason: {shutdown,max_concurrency}
2012-08-30 22:04:10.079 [info] <0.22254.0>@riak_core_handoff_sender:start_fold:126 Starting ownership_handoff transfer of riak_kv_vnode from 'kvs@d-kvs-01.tradecard.com' 0 to 'kvs@d-kvs-02.tradecard.com' 0
2012-08-30 22:04:32.449 [error] <0.1729.0> gen_fsm <0.1729.0> in state active terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}}
2012-08-30 22:04:36.755 [error] <0.1729.0> CRASH REPORT Process <0.1729.0> with 1 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:36.759 [error] <0.1732.0> gen_fsm <0.1732.0> in state ready terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}}
2012-08-30 22:04:36.763 [error] <0.1714.0> gen_fsm <0.1714.0> in state active terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}
2012-08-30 22:04:36.766 [error] <0.1714.0> CRASH REPORT Process <0.1714.0> with 1 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:36.790 [error] <0.1732.0> CRASH REPORT Process <0.1732.0> with 10 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:36.793 [error] <0.1717.0> gen_fsm <0.1717.0> in state ready terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}
2012-08-30 22:04:36.797 [error] <0.1717.0> CRASH REPORT Process <0.1717.0> with 10 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:36.802 [error] <0.1718.0> Supervisor poolboy_sup had child riak_core_vnode_worker started with riak_core_vnode_worker:start_link([{worker_args,[0,[],worker_props]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,...}]) at undefined exit with reason {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}} in context shutdown_error
2012-08-30 22:04:36.806 [error] <0.1718.0> gen_server <0.1718.0> terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}
2012-08-30 22:04:36.810 [error] <0.1718.0> CRASH REPORT Process <0.1718.0> with 0 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}} in gen_server:terminate/6 line 747
2012-08-30 22:04:36.814 [error] <0.1733.0> Supervisor poolboy_sup had child riak_core_vnode_worker started with riak_core_vnode_worker:start_link([{worker_args,[11417981541647679048466287755595961091061972992,[],worker_props]},{worker_callback_mod,...},...]) at undefined exit with reason {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}} in context shutdown_error
2012-08-30 22:04:36.818 [error] <0.1733.0> gen_server <0.1733.0> terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}}
2012-08-30 22:04:36.821 [error] <0.1733.0> CRASH REPORT Process <0.1733.0> with 0 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}} in gen_server:terminate/6 line 747
2012-08-30 22:04:36.825 [error] <0.1744.0> gen_fsm <0.1744.0> in state active terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,22835963083295358096932575511191922182123945984,'kvs@d-kvs-04.tradecard.com',<0.1744.0>}]}}
2012-08-30 22:04:36.829 [error] <0.1744.0> CRASH REPORT Process <0.1744.0> with 1 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,22835963083295358096932575511191922182123945984,'kvs@d-kvs-04.tradecard.com',<0.1744.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:36.832 [error] <0.2750.0> gen_fsm <0.2750.0> in state active terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,787840726373689854344173855136121315283276136448,'kvs@d-kvs-02.tradecard.com',<0.2750.0>}]}}
2012-08-30 22:04:36.835 [error] <0.2750.0> CRASH REPORT Process <0.2750.0> with 1 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,787840726373689854344173855136121315283276136448,'kvs@d-kvs-02.tradecard.com',<0.2750.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:36.839 [error] <0.1747.0> gen_fsm <0.1747.0> in state ready terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,22835963083295358096932575511191922182123945984,'kvs@d-kvs-04.tradecard.com',<0.1744.0>}]}}
2012-08-30 22:04:36.844 [error] <0.1747.0> CRASH REPORT Process <0.1747.0> with 10 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,22835963083295358096932575511191922182123945984,'kvs@d-kvs-04.tradecard.com',<0.1744.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:34 =ERROR REPORT====
** State machine <0.1729.0> terminating
** Last event in was {trigger_handoff,'kvs@d-kvs-03.tradecard.com'} (for all states)
** When State == active
** Data == {state,11417981541647679048466287755595961091061972992,riak_kv_vnode,{state,11417981541647679048466287755595961091061972992,riak_kv_eleveldb_backend,{state,<<>>,"/database/riak/leveldb/11417981541647679048466287755595961091061972992",[{cache_size,268435456},{create_if_missing,true},{max_open_files,30},{sst_block_size,4096},{use_bloomfilter,true},{write_buffer_size,33931693}],[{add_paths,[]},{allow_strfun,false},{async_folds,true},{cache_size,268435456},{create_if_missing,true},{data_root,"/database/riak/leveldb"},{hook_js_vm_count,2},{http_url_encoding,on},{included_applications,[]},{js_max_vm_mem,8},{js_thread_stack,16},{legacy_keylisting,false},{legacy_stats,true},{listkeys_backpressure,true},{map_js_vm_count,8},{mapred_2i_pipe,true},{mapred_name,"mapred"},{mapred_system,pipe},{max_open_files,30},{reduce_js_vm_count,6},{sst_block_size,4096},{stats_urlpath,"stats"},{storage_backend,riak_kv_eleveldb_backend},{sync,false},{use_bloomfilter,true},{verify_checksums,true},{vnode_vclocks,true},{write_buffer_size,33931693},{write_buffer_size_max,62914560},{write_buffer_size_min,31457280}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}]},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<198,192,70,166,79,210,86,143>>,3000,1000,100,100,true,true},undefined,'kvs@d-kvs-03.tradecard.com',<0.1731.0>,undefined,60000}
** Reason for termination =
** {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}}
2012-08-30 22:04:36 =CRASH REPORT====
crasher:
initial call: riak_core_vnode:init/1
pid: <0.1729.0>
registered_name: []
exception exit: {{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,611}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]}
ancestors: [riak_core_vnode_sup,riak_core_sup,<0.187.0>]
messages: []
links: [<0.190.0>,<0.1731.0>]
dictionary: [{random_seed,{18454,4298,9921}}]
trap_exit: true
status: running
heap_size: 1597
stack_size: 24
reductions: 7866
neighbours:
neighbour: [{pid,<0.1731.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker_pool,init,['Argument__1']}},{current_function,{gen_fsm,loop,7}},{ancestors,[<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1729.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,377},{stack_size,10},{reductions,88}]
2012-08-30 22:04:36 =ERROR REPORT====
** State machine <0.1732.0> terminating
** Last message in was {'EXIT',<0.1731.0>,{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}}}
** When State == ready
** Data == {state,<0.1733.0>,{[<0.1743.0>,<0.1742.0>],[<0.1735.0>,<0.1736.0>,<0.1737.0>,<0.1738.0>,<0.1739.0>,<0.1740.0>,<0.1741.0>]},{[],[]},233574,10,0,0}
** Reason for termination =
** {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}}
2012-08-30 22:04:36 =ERROR REPORT====
** State machine <0.1714.0> terminating
** Last event in was {trigger_handoff,'kvs@d-kvs-02.tradecard.com'} (for all states)
** When State == active
** Data == {state,0,riak_kv_vnode,{state,0,riak_kv_eleveldb_backend,{state,<<>>,"/database/riak/leveldb/0",[{cache_size,268435456},{create_if_missing,true},{max_open_files,30},{sst_block_size,4096},{use_bloomfilter,true},{write_buffer_size,35823923}],[{add_paths,[]},{allow_strfun,false},{async_folds,true},{cache_size,268435456},{create_if_missing,true},{data_root,"/database/riak/leveldb"},{hook_js_vm_count,2},{http_url_encoding,on},{included_applications,[]},{js_max_vm_mem,8},{js_thread_stack,16},{legacy_keylisting,false},{legacy_stats,true},{listkeys_backpressure,true},{map_js_vm_count,8},{mapred_2i_pipe,true},{mapred_name,"mapred"},{mapred_system,pipe},{max_open_files,30},{reduce_js_vm_count,6},{sst_block_size,4096},{stats_urlpath,"stats"},{storage_backend,riak_kv_eleveldb_backend},{sync,false},{use_bloomfilter,true},{verify_checksums,true},{vnode_vclocks,true},{write_buffer_size,35823923},{write_buffer_size_max,62914560},{write_buffer_size_min,31457280}],[{verify_checksums,true}],[{sync,false}],[{verify_checksums,true},{fill_cache,false}]},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}},<<198,192,70,166,79,210,86,143>>,3000,1000,100,100,true,true},undefined,'kvs@d-kvs-02.tradecard.com',<0.1716.0>,#Ref<0.0.0.130766>,60000}
** Reason for termination =
** {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}
2012-08-30 22:04:36 =CRASH REPORT====
crasher:
initial call: riak_core_vnode:init/1
pid: <0.1714.0>
registered_name: []
exception exit: {{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,611}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]}
ancestors: [riak_core_vnode_sup,riak_core_sup,<0.187.0>]
messages: []
links: [<0.190.0>,<0.1716.0>]
dictionary: [{random_seed,{18454,4298,11745}}]
trap_exit: true
status: running
heap_size: 1597
stack_size: 24
reductions: 5491
neighbours:
neighbour: [{pid,<0.1716.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker_pool,init,['Argument__1']}},{current_function,{gen_fsm,loop,7}},{ancestors,[<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1714.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,377},{stack_size,10},{reductions,88}]
2012-08-30 22:04:36 =CRASH REPORT====
crasher:
initial call: poolboy:init/1
pid: <0.1732.0>
registered_name: []
exception exit: {{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,611}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]}
ancestors: [<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]
messages: [{'DOWN',#Ref<0.0.0.130717>,process,<0.1731.0>,{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,11417981541647679048466287755595961091061972992,'kvs@d-kvs-03.tradecard.com',<0.1729.0>}]}}}]
links: [<0.1739.0>,<0.1741.0>,<0.1742.0>,<0.1743.0>,<0.1740.0>,<0.1735.0>,<0.1737.0>,<0.1738.0>,<0.1736.0>,<0.1733.0>,<0.1734.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 610
stack_size: 24
reductions: 332
neighbours:
neighbour: [{pid,<0.1734.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{prim_inet,recv0,3}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1732.0>,<0.1733.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,987},{stack_size,47},{reductions,18528151}]
neighbour: [{pid,<0.1736.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1738.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1737.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1735.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1740.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1743.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1742.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1741.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1739.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1733.0>,<0.1732.0>,<0.1731.0>,<0.1729.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1733.0>,<0.1732.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
2012-08-30 22:04:36 =ERROR REPORT====
** State machine <0.1717.0> terminating
** Last message in was {'EXIT',<0.1716.0>,{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}}
** When State == ready
** Data == {state,<0.1718.0>,{[<0.1728.0>,<0.1727.0>],[<0.1720.0>,<0.1721.0>,<0.1722.0>,<0.1723.0>,<0.1724.0>,<0.1725.0>,<0.1726.0>]},{[],[]},229477,10,0,0}
** Reason for termination =
** {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}
2012-08-30 22:04:36 =CRASH REPORT====
crasher:
initial call: poolboy:init/1
pid: <0.1717.0>
registered_name: []
exception exit: {{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,611}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]}
ancestors: [<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]
messages: [{'DOWN',#Ref<0.0.0.131724>,process,<0.1716.0>,{timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}}]
links: [<0.1724.0>,<0.1726.0>,<0.1727.0>,<0.1728.0>,<0.1725.0>,<0.1720.0>,<0.1722.0>,<0.1723.0>,<0.1721.0>,<0.1718.0>,<0.1719.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 610
stack_size: 24
reductions: 353
neighbours:
neighbour: [{pid,<0.1719.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{prim_inet,recv0,3}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1717.0>,<0.1718.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,610},{stack_size,47},{reductions,16544161}]
neighbour: [{pid,<0.1721.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1723.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1722.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1720.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1725.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1728.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1727.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1726.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
neighbour: [{pid,<0.1724.0>},{registered_name,[]},{initial_call,{riak_core_vnode_worker,init,['Argument__1']}},{current_function,{gen_server,loop,6}},{ancestors,[<0.1718.0>,<0.1717.0>,<0.1716.0>,<0.1714.0>,riak_core_vnode_sup,riak_core_sup,<0.187.0>]},{messages,[]},{links,[<0.1718.0>,<0.1717.0>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,233},{stack_size,9},{reductions,25}]
2012-08-30 22:04:36 =SUPERVISOR REPORT====
Supervisor: {<0.1718.0>,poolboy_sup}
Context: shutdown_error
Reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,0,'kvs@d-kvs-02.tradecard.com',<0.1714.0>}]}}
Offender: [{nb_children,10},{name,riak_core_vnode_worker},{mfargs,{riak_core_vnode_worker,start_link,[[{worker_args,[0,[],worker_props]},{worker_callback_mod,riak_kv_worker},{size,10},{max_overflow,0}]]}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]
2012-08-30 22:04:38.288 [error] <0.2572.0> gen_fsm <0.2572.0> in state ready terminated with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,650824947873917705762578402068969782190532460544,'kvs@d-kvs-02.tradecard.com',<0.2569.0>}]}}
2012-08-30 22:04:38.309 [error] <0.22603.0>@riak_kv_vnode:init:265 Failed to start riak_kv_eleveldb_backend Reason: {db_open,"IO error: lock /database/riak/leveldb/91343852333181432387730302044767688728495783936/LOCK: Resource temporarily unavailable"}
2012-08-30 22:04:38.291 [error] <0.2572.0> CRASH REPORT Process <0.2572.0> with 10 neighbours exited with reason: {timeout,{gen_server,call,[riak_core_handoff_manager,{add_outbound,riak_kv_vnode,650824947873917705762578402068969782190532460544,'kvs@d-kvs-02.tradecard.com',<0.2569.0>}]}} in gen_fsm:terminate/7 line 611
2012-08-30 22:04:38.311 [notice] <0.22603.0>@riak:stop:46 "backend module failed to start."
