Skip to content

Instantly share code, notes, and snippets.

@grepory

grepory/gist:384410ac90186ed0ce2a

Created May 18, 2014 18:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save grepory/384410ac90186ed0ce2a to your computer and use it in GitHub Desktop.
Save grepory/384410ac90186ed0ce2a to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
** Reason for termination ==
** {{case_clause,
{{{value,
{15825,
{publish,<0.1876.1333>,
{message_properties,undefined,false},
{basic_message,
{resource,<<"/outbound-dev-rabbitmq">>,exchange,
<<"outbound.generation.exchange">>},
[<<"outbound.generation.4bcc31a4-557b-46dd-86a5-efbaecc77f7b.EMAIL">>],
{content,60,
{'P_basic',<<"application/json">>,<<"UTF-8">>,
[{<<"GZIP-Compression">>,longstr,<<"true">>},
{<<"__TypeId__">>,longstr,
<<"opower.outbound.core.pipeline.model.CommunicationContext">>}],
2,0,undefined,undefined,undefined,undefined,
undefined,undefined,undefined,undefined,
undefined},
<<248,0,16,97,112,112,108,105,99,97,116,105,111,110,
47,106,115,111,110,5,85,84,70,45,56,0,0,0,98,16,
71,90,73,80,45,67,111,109,112,114,101,115,115,105,
111,110,83,0,0,0,4,116,114,117,101,10,95,95,84,
121,112,101,73,100,95,95,83,0,0,0,56,111,112,111,
119,101,114,46,111,117,116,98,111,117,110,100,46,
99,111,114,101,46,112,105,112,101,108,105,110,101,
46,109,111,100,101,108,46,67,111,109,109,117,110,
105,99,97,116,105,111,110,67,111,110,116,101,120,
116,2,0>>,
rabbit_framing_amqp_0_9_1,
[<<31,139,8,0,0,0,0,0,0,0,117,82,93,143,218,48,16,
252,47,121,174,169,109,236,124,240,84,10,81,
133,218,187,67,37,60,148,170,138,28,219,92,221,
38,142,149,56,61,78,136,255,222,117,56,113,112,
106,165,60,68,187,179,227,157,217,57,70,178,
109,154,193,26,41,188,105,109,52,59,70,70,69,
179,232,227,167,124,129,82,196,51,193,98,90,69,
239,162,78,75,227,140,182,254,2,73,17,65,132,
164,152,43,104,215,173,20,181,134,170,182,229,
118,3,5,111,26,189,107,109,40,205,27,221,1,255,
<snip>
249,42,77,123,115,208,106,41,188,142,174,97,99,
33,184,78,207,121,200,138,49,12,175,121,56,253,
5,171,218,60,120,99,3,0,0>>]},
<<224,79,149,166,221,104,157,225,44,153,105,8,196,224,
90,52>>,
true}}}},
Several of those, and then:
8908,
{0,nil},
{0,nil},
{qistate,
"/var/lib/rabbitmq/mnesia/rabbit@dev-rabbitmq-1003/queues/5UEEHBXGHKNMK4GTZ69DX3QZQ",
{{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
[{segment,0,
"/var/lib/rabbitmq/mnesia/rabbit@dev-rabbitmq-1003/queues/5UEEHBXGHKNMK4GTZ69DX3QZQ/0.idx",
{array,16384,0,undefined,
{{{{{undefined,undefined,undefined,undefined,
undefined,undefined,undefined,undefined,
undefined,undefined},
{undefined,undefined,undefined,undefined,
undefined,undefined,undefined,undefined,
undefined,undefined},
{undefined,undefined,undefined,undefined,
undefined,undefined,undefined,undefined,
undefined,undefined},
lots of this
then:
{{<<124,171,143,13,140,195,155,171,57,176,
214,180,21,164,32,99>>,
{message_properties,undefined,false},
true},
no_del,no_ack},
{{<<22,208,179,203,125,255,201,158,105,39,
29,173,2,244,88,0>>,
{message_properties,undefined,false},
true},
no_del,no_ack}},
{{{<<223,83,93,224,46,128,171,1,108,75,86,
207,90,193,182,243>>,
{message_properties,undefined,false},
true},
no_del,no_ack},
{{<<214,16,168,25,10,240,96,176,30,78,178,
29,87,70,174,157>>,
{message_properties,undefined,false},
true},
no_del,no_ack},
{{<<231,254,229,208,238,51,173,86,106,110,
58,30,97,5,55,139>>,
{message_properties,undefined,false},
true},
no_del,no_ack},
lots of those, then:
{{<<"çVçÁËØ+ËøÁ/òHSÛo">>,
{message_properties,undefined,false},
true},
no_del,no_ack},
{{<<8,90,38,190,124,27,95,219,81,131,142,
70,193,196,73,44>>,
{message_properties,undefined,false},
true},
no_del,no_ack},
{{<<249,70,119,192,204,235,4,22,68,197,
156,222,54,94,186,53>>,
{message_properties,undefined,false},
true},
no_del,no_ack},
{{<<"˶\f2¼1ÊÀg¡7t`²ªH">>,
{message_properties,undefined,false},
true},
no_del,no_ack},
{{<<52,222,247,30,242,107,46,59,144,185,
195,182,128,150,148,208>>,
{message_properties,undefined,false},
true},
no_del,no_ack},
undefined,undefined},
10,10,10,10,10,10,10,10,10,10},
100},
1000,1000},
10000,10000,10000,10000,10000,10000,10000,
10000,10000,10000}},
290}]},
#Ref<0.0.1642.38964>,26144,65536,
#Fun<rabbit_variable_queue.2.81334491>,
{0,nil}},
{{client_msstate,msg_store_persistent,
<<184,197,138,103,159,149,107,1,148,124,30,168,
140,228,103,72>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{state,327757,
"/var/lib/rabbitmq/mnesia/rabbit@dev-rabbitmq-1003/msg_store_persistent"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit@dev-rabbitmq-1003/msg_store_persistent",
<0.346.0>,331854,323660,335951,340048},
{client_msstate,msg_store_transient,
<<201,81,136,41,194,184,253,128,218,32,132,96,86,
196,227,40>>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
{state,307272,
"/var/lib/rabbitmq/mnesia/rabbit@dev-rabbitmq-1003/msg_store_transient"},
rabbit_msg_store_ets_index,
"/var/lib/rabbitmq/mnesia/rabbit@dev-rabbitmq-1003/msg_store_transient",
<0.341.0>,311369,303170,315466,319563}},
true,0,290,290,infinity,290,135,0,300,455,
{rates,
{{1399,935015,956016},1344},
{{1399,935015,956016},1479},
134.37123111941733,147.8683413881088,
{1399,935020,957247}},
{0,nil},
{0,nil},
{0,nil},
{0,nil},
0,0,
{rates,
{{1399,935015,956016},0},
{{1399,935015,956016},0},
0.0,0.0,
{1399,935020,957247}}},
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]}}},
[],
{set,1,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],
[<0.4805.1333>],
[],[],[],[],[],[],[],[]}}}},
{queue,[],[],0},
undefined,undefined,#Ref<0.0.1644.13001>,undefined,
{state,fine,5000,#Ref<0.0.1644.13185>},
{0,nil},
undefined,undefined,undefined,
{state,
{dict,1,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
{{[],[],[],[],[],[],[],
[[<0.4805.1333>|#Ref<0.0.1644.18403>]],
[],[],[],[],[],[],[],[]}}},
delegate},
undefined,undefined,undefined,6,running}
** Reason for termination ==
** {{badmatch,{error,not_found}},
[{rabbit_mirror_queue_master,stop_all_slaves,2},
{rabbit_mirror_queue_master,delete_and_terminate,2},
{rabbit_amqqueue_process,'-terminate_delete/3-fun-1-',6},
{rabbit_amqqueue_process,terminate_shutdown,2},
{gen_server2,terminate,3},
{proc_lib,init_p_do_apply,3}]}
** In 'terminate' callback with reason ==
** {{case_clause,
{{{value,
{15825,
{publish,<0.1876.1333>,
{message_properties,undefined,false},
{basic_message,
{resource,<<"/outbound-dev-rabbitmq">>,exchange,
<<"outbound.generation.exchange">>},
[<<"outbound.generation.4bcc31a4-557b-46dd-86a5-efbaecc77f7b.EMAIL">>],
{content,60,
{'P_basic',<<"application/json">>,<<"UTF-8">>,
[{<<"GZIP-Compression">>,longstr,<<"true">>},
{<<"__TypeId__">>,longstr,
<<"opower.outbound.core.pipeline.model.CommunicationContext">>}],
2,0,undefined,undefined,undefined,undefined,
undefined,undefined,undefined,undefined,
undefined},
basically the same thing as the top...
We then upgraded to 3.3, because hey... opportunistic upgrade during pre-existing downtime.
This led to the server consistently refusing to start up:
** Reason for termination ==
** {{badmatch,{error,not_found}},
[{rabbit_mirror_queue_master,stop_all_slaves,2},
{rabbit_mirror_queue_master,delete_and_terminate,2},
{rabbit_amqqueue_process,'-terminate_delete/3-fun-1-',6},
{rabbit_amqqueue_process,terminate_shutdown,2},
{gen_server2,terminate,3},
{proc_lib,wake_up,3}]}
** In 'terminate' callback with reason ==
** {{badmatch,{error,not_found}},
[{rabbit_amqqueue_process,i,2},
{rabbit_amqqueue_process,'-infos/2-lc$^0/1-0-',2},
{rabbit_amqqueue_process,'-infos/2-lc$^0/1-0-',2},
{rabbit_amqqueue_process,emit_stats,2},
{rabbit_event,if_enabled,3},
{rabbit_amqqueue_process,handle_pre_hibernate,1},
{gen_server2,pre_hibernate,1},
{proc_lib,wake_up,3}]}
=ERROR REPORT==== 12-May-2014::18:50:52 ===
** Generic server <0.13133.1230> terminating
** Last message in was pre_hibernate
** When Server state == {q,
{amqqueue,
{resource,<<"/outbound-dev-rabbitmq">>,queue,
<<"outbound.generation.ed8e2768-4064-403a-ae7f-37a64f65d16c.IVR">>},
true,false,none,[],<0.13133.1230>,[],[],
[{vhost,<<"/outbound-dev-rabbitmq">>},
{name,<<"outbound-dev">>},
{pattern,<<"^(?!amq\\.).*">>},
{'apply-to',<<"queues">>},
{definition,
[{<<"ha-mode">>,<<"all">>},
{<<"ha-sync-mode">>,<<"automatic">>}]},
{priority,99}],
[{<0.13134.1230>,<0.13133.1230>}],
[]},
none,false,rabbit_mirror_queue_master,
{state,
{resource,<<"/outbound-dev-rabbitmq">>,queue,
<<"outbound.generation.ed8e2768-4064-403a-ae7f-37a64f65d16c.IVR">>},
<0.13134.1230>,<0.7120.1333>,rabbit_variable_queue,
{vqstate,
{0,{[],[]}},
{0,{[],[]}},
{delta,undefined,0,undefined},
{1,
<more of this, over and over again.>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment