Last active
January 4, 2016 03:18
-
-
Save jwestboston/8560559 to your computer and use it in GitHub Desktop.
Federated Queues w/ Cluster Failure. See http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/2014-January/033231.html
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
=CRASH REPORT==== 22-Jan-2014::16:17:07 === | |
crasher: | |
initial call: gen:init_it/6 | |
pid: <0.768.0> | |
registered_name: [] | |
exception exit: {{badmatch,{error,not_found}}, | |
[{rabbit_federation_queue_link_sup_sup,stop_child,1,[]}, | |
{rabbit_federation_queue,shutdown,1,[]}, | |
{rabbit_amqqueue_process, | |
'-decorator_callback/3-lc$^0/1-0-',4,[]}, | |
{rabbit_amqqueue_process,terminate_shutdown,2,[]}, | |
{gen_server2,terminate,3,[]}, | |
{gen_server2,handle_msg,2,[]}, | |
{proc_lib,wake_up,3, | |
[{file,"proc_lib.erl"},{line,237}]}]} | |
in function gen_server2:handle_msg/2 | |
ancestors: [rabbit_amqqueue_sup,rabbit_sup,<0.171.0>] | |
messages: [] | |
links: [<0.239.0>,<0.769.0>,#Port<0.15738>] | |
dictionary: [{{xtype_to_module,direct},rabbit_exchange_type_direct}, | |
{{"/var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a/queues/6WU5COL6E3877LB3YM76GQP7A/journal.jif", | |
fhc_file}, | |
{file,1,true}}, | |
{{#Ref<0.0.0.16302>,fhc_handle}, | |
{handle,{file_descriptor,prim_file,{#Port<0.15738>,24}}, | |
0,false,0,infinity,[],true, | |
"/var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a/queues/6WU5COL6E3877LB3YM76GQP7A/journal.jif", | |
[write,binary,raw,read], | |
[{write_buffer,infinity}], | |
true,true, | |
{1390,403801,547470}}}, | |
{{credit_from,<0.233.0>},1998}, | |
{fhc_age_tree,{1, | |
{{1390,403801,547470}, | |
#Ref<0.0.0.16302>,nil,nil}}}, | |
{{credit_to,<0.794.0>},48}, | |
{guid,{{1405556399,1269750758,1314049339,4249162108},1}}] | |
trap_exit: true | |
status: running | |
heap_size: 4181 | |
stack_size: 24 | |
reductions: 10551 | |
neighbours: | |
neighbour: [{pid,<0.770.0>}, | |
{registered_name,[]}, | |
{initial_call,{gen,init_it, | |
['Argument__1','Argument__2', | |
'Argument__3','Argument__4', | |
'Argument__5','Argument__6']}}, | |
{current_function,{erlang,hibernate,3}}, | |
{ancestors,[<0.769.0>,<0.768.0>,rabbit_amqqueue_sup, | |
rabbit_sup,<0.171.0>]}, | |
{messages,[]}, | |
{links,[<0.769.0>]}, | |
{dictionary,[{random_seed,{1391,9801,28346}}]}, | |
{trap_exit,false}, | |
{status,waiting}, | |
{heap_size,233}, | |
{stack_size,0}, | |
{reductions,2846}] | |
neighbour: [{pid,<0.769.0>}, | |
{registered_name,[]}, | |
{initial_call,{gen,init_it, | |
['Argument__1','Argument__2', | |
'Argument__3','Argument__4', | |
'Argument__5','Argument__6']}}, | |
{current_function,{erlang,hibernate,3}}, | |
{ancestors,[<0.768.0>,rabbit_amqqueue_sup,rabbit_sup, | |
<0.171.0>]}, | |
{messages,[]}, | |
{links,[<0.768.0>,<0.770.0>]}, | |
{dictionary,[]}, | |
{trap_exit,false}, | |
{status,waiting}, | |
{heap_size,377}, | |
{stack_size,0}, | |
{reductions,212}] | |
=SUPERVISOR REPORT==== 22-Jan-2014::16:17:07 === | |
Supervisor: {local,rabbit_amqqueue_sup} | |
Context: child_terminated | |
Reason: {{badmatch,{error,not_found}}, | |
[{rabbit_federation_queue_link_sup_sup,stop_child,1,[]}, | |
{rabbit_federation_queue,shutdown,1,[]}, | |
{rabbit_amqqueue_process, | |
'-decorator_callback/3-lc$^0/1-0-',4,[]}, | |
{rabbit_amqqueue_process,terminate_shutdown,2,[]}, | |
{gen_server2,terminate,3,[]}, | |
{gen_server2,handle_msg,2,[]}, | |
{proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,237}]}]} | |
Offender: [{pid,<0.768.0>}, | |
{name,rabbit_amqqueue}, | |
{mfargs,{rabbit_amqqueue_process,start_link,undefined}}, | |
{restart_type,temporary}, | |
{shutdown,4294967295}, | |
{child_type,worker}] | |
=CRASH REPORT==== 22-Jan-2014::16:18:03 === | |
crasher: | |
initial call: gen:init_it/6 | |
pid: <0.942.0> | |
registered_name: [] | |
exception exit: {{badmatch,true}, | |
[{rabbit_queue_index,init,2,[]}, | |
{rabbit_variable_queue,init,5,[]}, | |
{rabbit_mirror_queue_master,init,3,[]}, | |
{rabbit_amqqueue_process,declare,3,[]}, | |
{gen_server2,handle_msg,2,[]}, | |
{proc_lib,init_p_do_apply,3, | |
[{file,"proc_lib.erl"},{line,227}]}]} | |
in function gen_server2:terminate/3 | |
ancestors: [rabbit_amqqueue_sup,rabbit_sup,<0.171.0>] | |
messages: [] | |
links: [<0.239.0>] | |
dictionary: [{{xtype_to_module,direct},rabbit_exchange_type_direct}] | |
trap_exit: true | |
status: running | |
heap_size: 1597 | |
stack_size: 24 | |
reductions: 1166 | |
neighbours: | |
=SUPERVISOR REPORT==== 22-Jan-2014::16:18:03 === | |
Supervisor: {local,rabbit_amqqueue_sup} | |
Context: child_terminated | |
Reason: {{badmatch,true}, | |
[{rabbit_queue_index,init,2,[]}, | |
{rabbit_variable_queue,init,5,[]}, | |
{rabbit_mirror_queue_master,init,3,[]}, | |
{rabbit_amqqueue_process,declare,3,[]}, | |
{gen_server2,handle_msg,2,[]}, | |
{proc_lib,init_p_do_apply,3, | |
[{file,"proc_lib.erl"},{line,227}]}]} | |
Offender: [{pid,<0.942.0>}, | |
{name,rabbit_amqqueue}, | |
{mfargs,{rabbit_amqqueue_process,start_link,undefined}}, | |
{restart_type,temporary}, | |
{shutdown,4294967295}, | |
{child_type,worker}] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
Starting RabbitMQ 3.2.2 on Erlang R15B01 | |
Copyright (C) 2007-2013 GoPivotal, Inc. | |
Licensed under the MPL. See http://www.rabbitmq.com/ | |
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
node : rabbit@fedcluster-pod1a | |
home dir : /var/lib/rabbitmq | |
config file(s) : /etc/rabbitmq/rabbitmq.config | |
cookie hash : RR95TRGoM9CLWBVToK3pEw== | |
log : /var/log/rabbitmq/rabbit@fedcluster-pod1a.log | |
sasl log : /var/log/rabbitmq/rabbit@fedcluster-pod1a-sasl.log | |
database dir : /var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a | |
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
Limiting to approx 924 file handles (829 sockets) | |
=WARNING REPORT==== 22-Jan-2014::16:06:41 === | |
Converting legacy 'cluster_nodes' configuration | |
['rabbit@fedcluster-pod1a','rabbit@fedcluster-pod1b'] | |
to | |
{['rabbit@fedcluster-pod1b'],disc}. | |
Please update the configuration to the new format {Nodes, NodeType}, where Nodes contains the nodes that the node will try to cluster with, and NodeType is either 'disc' or 'ram' | |
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
Memory limit set to 198MB of 496MB total. | |
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
Disk free limit set to 50MB | |
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
msg_store_transient: using rabbit_msg_store_ets_index to provide index | |
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
msg_store_persistent: using rabbit_msg_store_ets_index to provide index | |
=WARNING REPORT==== 22-Jan-2014::16:06:41 === | |
msg_store_persistent: rebuilding indices from scratch | |
=INFO REPORT==== 22-Jan-2014::16:06:41 === | |
Adding vhost '/' | |
=INFO REPORT==== 22-Jan-2014::16:06:42 === | |
Creating user 'guest' | |
=INFO REPORT==== 22-Jan-2014::16:06:42 === | |
Setting user tags for user 'guest' to [administrator] | |
=INFO REPORT==== 22-Jan-2014::16:06:42 === | |
Setting permissions for 'guest' in '/' to '.*', '.*', '.*' | |
=INFO REPORT==== 22-Jan-2014::16:06:42 === | |
started TCP Listener on [::]:5672 | |
=INFO REPORT==== 22-Jan-2014::16:06:42 === | |
started SSL Listener on [::]:5671 | |
=INFO REPORT==== 22-Jan-2014::16:06:43 === | |
Management plugin started. Port: 15672 | |
=INFO REPORT==== 22-Jan-2014::16:06:43 === | |
Statistics database started. | |
=INFO REPORT==== 22-Jan-2014::16:06:43 === | |
Server startup complete; 8 plugins started. | |
* amqp_client | |
* mochiweb | |
* rabbitmq_federation | |
* rabbitmq_federation_management | |
* rabbitmq_management | |
* rabbitmq_management_agent | |
* rabbitmq_web_dispatch | |
* webmachine | |
=ERROR REPORT==== 22-Jan-2014::16:06:43 === | |
webmachine error: path="/api/queues" | |
"Unauthorized" | |
=INFO REPORT==== 22-Jan-2014::16:06:43 === | |
Creating user 'admin' | |
=INFO REPORT==== 22-Jan-2014::16:06:44 === | |
Setting user tags for user 'admin' to [administrator] | |
=INFO REPORT==== 22-Jan-2014::16:06:44 === | |
Setting permissions for 'admin' in '/' to '.*', '.*', '.*' | |
=INFO REPORT==== 22-Jan-2014::16:06:45 === | |
Setting user tags for user 'guest' to [] | |
=INFO REPORT==== 22-Jan-2014::16:06:45 === | |
Setting permissions for 'guest' in '/' to '.*', '.*', '.*' | |
=INFO REPORT==== 22-Jan-2014::16:06:48 === | |
Stopping RabbitMQ | |
=INFO REPORT==== 22-Jan-2014::16:06:48 === | |
stopped SSL Listener on [::]:5671 | |
=INFO REPORT==== 22-Jan-2014::16:06:48 === | |
stopped TCP Listener on [::]:5672 | |
=INFO REPORT==== 22-Jan-2014::16:06:48 === | |
Halting Erlang VM | |
=INFO REPORT==== 22-Jan-2014::16:06:52 === | |
Starting RabbitMQ 3.2.2 on Erlang R15B01 | |
Copyright (C) 2007-2013 GoPivotal, Inc. | |
Licensed under the MPL. See http://www.rabbitmq.com/ | |
=INFO REPORT==== 22-Jan-2014::16:06:52 === | |
node : rabbit@fedcluster-pod1a | |
home dir : /var/lib/rabbitmq | |
config file(s) : /etc/rabbitmq/rabbitmq.config | |
cookie hash : RR95TRGoM9CLWBVToK3pEw== | |
log : /var/log/rabbitmq/rabbit@fedcluster-pod1a.log | |
sasl log : /var/log/rabbitmq/rabbit@fedcluster-pod1a-sasl.log | |
database dir : /var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
Limiting to approx 924 file handles (829 sockets) | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
Memory limit set to 198MB of 496MB total. | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
Disk free limit set to 50MB | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
msg_store_transient: using rabbit_msg_store_ets_index to provide index | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
msg_store_persistent: using rabbit_msg_store_ets_index to provide index | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
started TCP Listener on [::]:5672 | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
started SSL Listener on [::]:5671 | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
Management plugin started. Port: 15672 | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
Statistics database started. | |
=INFO REPORT==== 22-Jan-2014::16:06:53 === | |
Server startup complete; 8 plugins started. | |
* amqp_client | |
* mochiweb | |
* rabbitmq_federation | |
* rabbitmq_federation_management | |
* rabbitmq_management | |
* rabbitmq_management_agent | |
* rabbitmq_web_dispatch | |
* webmachine | |
=INFO REPORT==== 22-Jan-2014::16:07:06 === | |
rabbit on node 'rabbit@fedcluster-pod1b' up | |
=INFO REPORT==== 22-Jan-2014::16:16:18 === | |
accepting AMQP connection <0.759.0> (127.0.0.1:43227 -> 127.0.0.1:5672) | |
=INFO REPORT==== 22-Jan-2014::16:16:18 === | |
Adding mirror of queue 'one.pod1.queue.testing' in vhost '/' on node 'rabbit@fedcluster-pod1b': <2781.688.0> | |
=INFO REPORT==== 22-Jan-2014::16:16:18 === | |
Federation queue 'one.pod1.queue.testing' in vhost '/' connected to queue 'one.pod1.queue.testing' in vhost '/' on amqps://fedcluster-pod2a | |
=INFO REPORT==== 22-Jan-2014::16:16:18 === | |
Federation queue 'one.pod1.queue.testing' in vhost '/' connected to queue 'one.pod1.queue.testing' in vhost '/' on amqps://fedcluster-pod3b | |
=WARNING REPORT==== 22-Jan-2014::16:17:07 === | |
closing AMQP connection <0.759.0> (127.0.0.1:43227 -> 127.0.0.1:5672): | |
connection_closed_abruptly | |
=ERROR REPORT==== 22-Jan-2014::16:17:07 === | |
** Generic server <0.768.0> terminating | |
** Last message in was {notify_down,<0.767.0>} | |
** When Server state == {q, | |
{amqqueue, | |
{resource,<<"/">>,queue, | |
<<"one.pod1.queue.testing">>}, | |
true,true,none,[],<0.768.0>, | |
[<2781.688.0>], | |
[<2781.688.0>], | |
[{vhost,<<"/">>}, | |
{name,<<"federation-my-queues">>}, | |
{pattern,<<"^one.pod1.queue.*">>}, | |
{'apply-to',<<"queues">>}, | |
{definition, | |
[{<<"federation-upstream-set">>,<<"all">>}, | |
{<<"ha-mode">>,<<"all">>}]}, | |
{priority,10}], | |
[{<2781.689.0>,<2781.688.0>},{<0.770.0>,<0.768.0>}], | |
[rabbit_federation_queue]}, | |
none,true,rabbit_mirror_queue_master, | |
{state, | |
{resource,<<"/">>,queue, | |
<<"one.pod1.queue.testing">>}, | |
<0.770.0>,<0.769.0>,rabbit_variable_queue, | |
{vqstate, | |
{0,{[],[]}}, | |
{0,{[],[]}}, | |
{delta,undefined,0,undefined}, | |
{0,{[],[]}}, | |
{0,{[],[]}}, | |
2, | |
{0,nil}, | |
{0,nil}, | |
{qistate, | |
"/var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a/queues/6WU5COL6E3877LB3YM76GQP7A", | |
{{dict,0,16,16,8,80,48, | |
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[]}, | |
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[]}}}, | |
[]}, | |
#Ref<0.0.0.16302>,0,65536, | |
#Fun<rabbit_variable_queue.2.81334491>, | |
{0,nil}}, | |
{{client_msstate,msg_store_persistent, | |
<<231,119,47,33,174,130,7,211,182,235,202,154,179, | |
23,220,71>>, | |
{dict,0,16,16,8,80,48, | |
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[]}, | |
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[]}}}, | |
{state,245835, | |
"/var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a/msg_store_persistent"}, | |
rabbit_msg_store_ets_index, | |
"/var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a/msg_store_persistent", | |
<0.238.0>,249932,241738,254029,258126}, | |
{client_msstate,msg_store_transient, | |
<<83,199,22,175,75,174,219,230,78,82,205,59,253, | |
69,17,124>>, | |
{dict,0,16,16,8,80,48, | |
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[]}, | |
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[]}}}, | |
{state,225350, | |
"/var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a/msg_store_transient"}, | |
rabbit_msg_store_ets_index, | |
"/var/lib/rabbitmq/mnesia/rabbit@fedcluster-pod1a/msg_store_transient", | |
<0.231.0>,229447,221253,233544,237641}}, | |
true,0,0,0,infinity,0,0,0,0,0, | |
{rates, | |
{{1390,403796,755015},1}, | |
{{1390,403796,755015},1}, | |
0.09132837949937253,0.09132837949937253, | |
{1390,403801,547012}}, | |
{0,nil}, | |
{0,nil}, | |
{0,nil}, | |
{0,nil}, | |
0,0, | |
{rates, | |
{{1390,403796,755015},1}, | |
{{1390,403796,755015},1}, | |
0.09132837949937253,0.09132837949937253, | |
{1390,403801,547012}}}, | |
{dict,0,16,16,8,80,48, | |
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, | |
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[]}}}, | |
[], | |
{set,1,16,16,8,80,48, | |
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, | |
{{[],[],[],[],[], | |
[<0.794.0>], | |
[],[],[],[],[],[],[],[],[],[]}}}}, | |
{queue,[],[],0}, | |
undefined,undefined,undefined,undefined, | |
{state,fine,5000,undefined}, | |
{0,nil}, | |
undefined,undefined,undefined, | |
{state, | |
{dict,1,16,16,8,80,48, | |
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, | |
{{[],[],[],[],[], | |
[[<0.794.0>|#Ref<0.0.0.16293>]], | |
[],[],[],[],[],[],[],[],[],[]}}}, | |
delegate}, | |
undefined,undefined,undefined,3,running} | |
** Reason for termination == | |
** {{badmatch,{error,not_found}}, | |
[{rabbit_federation_queue_link_sup_sup,stop_child,1,[]}, | |
{rabbit_federation_queue,shutdown,1,[]}, | |
{rabbit_amqqueue_process,'-decorator_callback/3-lc$^0/1-0-',4,[]}, | |
{rabbit_amqqueue_process,terminate_shutdown,2,[]}, | |
{gen_server2,terminate,3,[]}, | |
{gen_server2,handle_msg,2,[]}, | |
{proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,237}]}]} | |
** In 'terminate' callback with reason == | |
** normal | |
=INFO REPORT==== 22-Jan-2014::16:17:43 === | |
accepting AMQP connection <0.907.0> (127.0.0.1:43228 -> 127.0.0.1:5672) | |
=WARNING REPORT==== 22-Jan-2014::16:17:56 === | |
closing AMQP connection <0.907.0> (127.0.0.1:43228 -> 127.0.0.1:5672): | |
connection_closed_abruptly | |
=INFO REPORT==== 22-Jan-2014::16:18:03 === | |
accepting AMQP connection <0.933.0> (127.0.0.1:43229 -> 127.0.0.1:5672) | |
=ERROR REPORT==== 22-Jan-2014::16:18:03 === | |
** Generic server <0.942.0> terminating | |
** Last message in was {init,new} | |
** When Server state == {q,{amqqueue, | |
{resource,<<"/">>,queue, | |
<<"one.pod1.queue.testing">>}, | |
true,true,none,[],<0.942.0>,[],[], | |
[{vhost,<<"/">>}, | |
{name,<<"federation-my-queues">>}, | |
{pattern,<<"^one.pod1.queue.*">>}, | |
{'apply-to',<<"queues">>}, | |
{definition, | |
[{<<"federation-upstream-set">>,<<"all">>}, | |
{<<"ha-mode">>,<<"all">>}]}, | |
{priority,10}], | |
[], | |
[rabbit_federation_queue]}, | |
none,false,undefined,undefined, | |
{queue,[],[],0}, | |
undefined,undefined,undefined,undefined, | |
{state,fine,5000,undefined}, | |
{0,nil}, | |
undefined,undefined,undefined, | |
{state, | |
{dict,0,16,16,8,80,48, | |
{[],[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[],[]}, | |
{{[],[],[],[],[],[],[],[],[],[],[],[],[], | |
[],[],[]}}}, | |
delegate}, | |
undefined,undefined,undefined,0,running} | |
** Reason for termination == | |
** {{badmatch,true}, | |
[{rabbit_queue_index,init,2,[]}, | |
{rabbit_variable_queue,init,5,[]}, | |
{rabbit_mirror_queue_master,init,3,[]}, | |
{rabbit_amqqueue_process,declare,3,[]}, | |
{gen_server2,handle_msg,2,[]}, | |
{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]} | |
=WARNING REPORT==== 22-Jan-2014::16:18:29 === | |
closing AMQP connection <0.933.0> (127.0.0.1:43229 -> 127.0.0.1:5672): | |
connection_closed_abruptly |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment