jwadolowski/opscode-erchef_crash.log

## opscode-erchef_crash.log
2016-06-08 18:36:39 =ERROR REPORT====
** Generic server <0.1716.0> terminating
** Last message in was connect
** When Server state == {state,<0.1714.0>,{amqp_params,<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>,<<"/analytics">>,{127,0,0,1},5672,0,0,0,none,[]},undefined,undefined,undefined,undefined,undefined,undefined,<0.1715.0>,false,undefined,{{0,nil},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}},undefined,#Fun<amqp_connection_sup.0.94524864>}
** Reason for termination ==
** {socket_closed_unexpectedly,[{amqp_network_connection,handshake_recv,0,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,462}]},{amqp_network_connection,network_handshake,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,379}]},{amqp_network_connection,handshake,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,362}]},{amqp_network_connection,handle_call,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,93}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,607}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,639}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]}
2016-06-08 18:36:39 =CRASH REPORT====
  crasher:
    initial call: amqp_network_connection:init/1
    pid: <0.1716.0>
    registered_name: []
    exception exit: {socket_closed_unexpectedly,[{gen_server,terminate,7,[{file,"gen_server.erl"},{line,804}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]}
    ancestors: [<0.1714.0>,gen_bunny_mon,gen_bunny_sup,<0.1535.0>]
    messages: []
    links: [<0.1714.0>,#Port<0.6328>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 1598
    stack_size: 27
    reductions: 675
  neighbours:
2016-06-08 18:36:39 =SUPERVISOR REPORT====
     Supervisor: {<0.1714.0>,amqp_connection_sup}
     Context:    child_terminated
     Reason:     socket_closed_unexpectedly
     Offender:   [{pid,<0.1716.0>},{name,connection},{mfa,{amqp_network_connection,start_link,[{amqp_params,<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>,<<"/analytics">>,{127,0,0,1},5672,0,0,0,none,[]},<0.1715.0>,#Fun<amqp_connection_sup.0.94524864>]}},{restart_type,intrinsic},{shutdown,brutal_kill},{child_type,worker}]

2016-06-08 18:36:39 =SUPERVISOR REPORT====
     Supervisor: {<0.1714.0>,amqp_connection_sup}
     Context:    shutdown
     Reason:     reached_max_restart_intensity
     Offender:   [{pid,<0.1716.0>},{name,connection},{mfa,{amqp_network_connection,start_link,[{amqp_params,<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>,<<"/analytics">>,{127,0,0,1},5672,0,0,0,none,[]},<0.1715.0>,#Fun<amqp_connection_sup.0.94524864>]}},{restart_type,intrinsic},{shutdown,brutal_kill},{child_type,worker}]

2016-06-08 18:36:39 =ERROR REPORT====
["Could not connect, scheduling reconnect.",{error,{{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1716.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_info,2,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,134}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,593}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,659}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}}]

## opscode-erchef_current
2016-06-08_17:35:12.47349 [error] gen_server <0.1755.0> terminated with reason: socket_closed_unexpectedly in amqp_network_connection:handshake_recv/0 line 462
2016-06-08_17:35:12.47578 [error] CRASH REPORT Process <0.1755.0> with 0 neighbours exited with reason: socket_closed_unexpectedly in gen_server:terminate/7 line 804
2016-06-08_17:35:12.47824 [error] Supervisor {<0.1753.0>,amqp_connection_sup} had child at module undefined at <0.1755.0> exit with reason socket_closed_unexpectedly in context child_terminated
2016-06-08_17:35:12.48058 [error] Supervisor {<0.1753.0>,amqp_connection_sup} had child at module undefined at <0.1755.0> exit with reason reached_max_restart_intensity in context shutdown
2016-06-08_17:35:12.48280 [error] "Could not connect, scheduling reconnect.", error: {{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1755.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_info,2,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,134}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,593}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,659}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}
2016-06-08_17:35:15.44548 [error] CRASH REPORT Process <0.1706.0> with 0 neighbours exited with reason: {timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}} in gen_server:init_it/6 line 330
2016-06-08_17:35:15.44817 [error] Supervisor oc_chef_wm_sup had child oc_chef_action_queue started with bunnyc:start_link(oc_chef_action_queue, {network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e...">>},...}, {{'exchange.declare',0,<<"actions">>,<<"topic">>,false,true,false,false,false,[]}}, []) at undefined exit with reason {timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}} in context start_error
2016-06-08_17:35:15.45046 [error] CRASH REPORT Process <0.1645.0> with 0 neighbours exited with reason: {{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}}}},{oc_chef_wm_app,start,[normal,[]]}} in application_master:init/4 line 133
2016-06-08_17:35:15.45057 [info] Application oc_chef_wm exited with reason: {{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}}}},{oc_chef_wm_app,start,[normal,[]]}}
2016-06-08_17:35:17.02110 {"Kernel pid terminated",application_controller,"{application_start_failure,oc_chef_wm,{{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<\"actions\">>,<<\"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983\">>},<<\"/analytics\">>}},30000]}}}},{oc_chef_wm_app,start,[normal,[]]}}}"}
2016-06-08_17:35:17.02131 Kernel pid terminated (application_controller) ({application_start_failure,oc_chef_wm,{{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706
2016-06-08_17:35:17.02694 waiting for rabbitmq...
2016-06-08_17:35:17.23724 Waiting for rabbit@localhost ...
2016-06-08_17:35:17.23743 pid is 14115 ...
2016-06-08_17:35:17.26665 ...done.
2016-06-08_17:35:17.26937 rabbbitmq might be up...
2016-06-08_17:35:17.27886 Exec: /opt/opscode/embedded/service/opscode-erchef/erts-6.4/bin/erlexec -noshell -noinput +Bd -boot /opt/opscode/embedded/service/opscode-erchef/releases/12.6.0+20160430001037/oc_erchef -mode embedded -config /opt/opscode/embedded/service/opscode-erchef/sys.config -boot_var ERTS_LIB_DIR /opt/opscode/embedded/service/opscode-erchef/erts-6.4/../lib -args_file /opt/opscode/embedded/service/opscode-erchef/vm.args -- foreground
2016-06-08_17:35:17.27887 Root: /opt/opscode/embedded/service/opscode-erchef
2016-06-08_17:35:20.79260
2016-06-08_17:35:20.79262 =INFO REPORT==== 8-Jun-2016::18:35:20 ===
2016-06-08_17:35:20.79262 starting stats_hero_sup
2016-06-08_17:35:20.79285
2016-06-08_17:35:20.79287 =INFO REPORT==== 8-Jun-2016::18:35:20 ===
2016-06-08_17:35:20.79287 starting stats_hero_sender_sup with 350 senders
2016-06-08_17:35:20.90398
2016-06-08_17:35:20.90400 =INFO REPORT==== 8-Jun-2016::18:35:20 ===
2016-06-08_17:35:20.90400 Folsom Graphite adaptor starting.
2016-06-08_17:35:20.91480 [info] Application lager started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.91535 [info] Application ssl started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.91558 [info] Application chef_objects started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.91579 [info] Application uuid started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.91600 [info] Application mochiweb started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.91621 [info] Application epgsql started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.91641 [info] Application chef_index started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.91662 [info] Application mini_s3 started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.92136 [info] Application webmachine started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.92150 [info] Application sqerl started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.92172 [info] Application opscoderl_wm started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.92190 [info] Application chef_db started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.92566 [info] Application oc_chef_authz started on node 'erchef@127.0.0.1'
2016-06-08_17:35:20.92598 [info] Loading ibrowse configuration from /opt/opscode/embedded/service/opscode-erchef/etc/ibrowse/ibrowse.config
2016-06-08_17:35:20.96840 [info] Starting oc_chef_action
2016-06-08_17:35:21.06361 [error] Unknown response from RabbitMQ management console: {ok,"401",[{"Server","MochiWeb/1.1 WebMachine/1.10.0 (never breaks eye contact)"},{"Date","Wed, 08 Jun 2016 17:35:21 GMT"},{"Content-Length","50"}],<<"{\"error\":\"not_authorised\",\"reason\":\"Login failed\"}">>}
2016-06-08_17:35:21.06374 [info] Vhost "/analytics", queue "alaska" not at capacity or RabbitMQ Management Plugin unavailable
2016-06-08_17:35:21.06382 [info] Starting actions queue monitoring for vhost "/analytics" and queue "alaska"
2016-06-08_17:35:21.06389 [info] Chef Actions: Connecting to RabbitMQ at {127,0,0,1}:5672/analytics (exchange: <<"actions">>)
2016-06-08_17:35:24.06771 [error] gen_server <0.1709.0> terminated with reason: socket_closed_unexpectedly in amqp_network_connection:handshake_recv/0 line 462
2016-06-08_17:35:24.06992 [error] CRASH REPORT Process <0.1709.0> with 0 neighbours exited with reason: socket_closed_unexpectedly in gen_server:terminate/7 line 804
2016-06-08_17:35:24.07228 [error] Supervisor {<0.1707.0>,amqp_connection_sup} had child at module undefined at <0.1709.0> exit with reason socket_closed_unexpectedly in context child_terminated
2016-06-08_17:35:24.07467 [error] Supervisor {<0.1707.0>,amqp_connection_sup} had child at module undefined at <0.1709.0> exit with reason reached_max_restart_intensity in context shutdown
2016-06-08_17:35:24.07759 [error] "Could not connect, scheduling reconnect.", error: {{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1709.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_call,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,87}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,607}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,639}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}
2016-06-08_17:35:28.07138 [error] gen_server <0.1716.0> terminated with reason: socket_closed_unexpectedly in amqp_network_connection:handshake_recv/0 line 462
2016-06-08_17:35:28.07377 [error] CRASH REPORT Process <0.1716.0> with 0 neighbours exited with reason: socket_closed_unexpectedly in gen_server:terminate/7 line 804
2016-06-08_17:35:28.07610 [error] "Could not connect, scheduling reconnect.", error: {{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1716.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_info,2,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,134}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,593}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,659}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}
2016-06-08_17:35:28.07827 [error] Supervisor {<0.1714.0>,amqp_connection_sup} had child at module undefined at <0.1716.0> exit with reason socket_closed_unexpectedly in context child_terminated
2016-06-08_17:35:28.08054 [error] Supervisor {<0.1714.0>,amqp_connection_sup} had child at module undefined at <0.1716.0> exit with reason reached_max_restart_intensity in context shutdown
	2016-06-08 18:36:39 =ERROR REPORT====
	** Generic server <0.1716.0> terminating
	** Last message in was connect
	** When Server state == {state,<0.1714.0>,{amqp_params,<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>,<<"/analytics">>,{127,0,0,1},5672,0,0,0,none,[]},undefined,undefined,undefined,undefined,undefined,undefined,<0.1715.0>,false,undefined,{{0,nil},{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}},undefined,#Fun<amqp_connection_sup.0.94524864>}
	** Reason for termination ==
	** {socket_closed_unexpectedly,[{amqp_network_connection,handshake_recv,0,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,462}]},{amqp_network_connection,network_handshake,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,379}]},{amqp_network_connection,handshake,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,362}]},{amqp_network_connection,handle_call,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/amqp_client/src/amqp_network_connection.erl"},{line,93}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,607}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,639}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]}
	2016-06-08 18:36:39 =CRASH REPORT====
	crasher:
	initial call: amqp_network_connection:init/1
	pid: <0.1716.0>
	registered_name: []
	exception exit: {socket_closed_unexpectedly,[{gen_server,terminate,7,[{file,"gen_server.erl"},{line,804}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]}
	ancestors: [<0.1714.0>,gen_bunny_mon,gen_bunny_sup,<0.1535.0>]
	messages: []
	links: [<0.1714.0>,#Port<0.6328>]
	dictionary: []
	trap_exit: false
	status: running
	heap_size: 1598
	stack_size: 27
	reductions: 675
	neighbours:
	2016-06-08 18:36:39 =SUPERVISOR REPORT====
	Supervisor: {<0.1714.0>,amqp_connection_sup}
	Context: child_terminated
	Reason: socket_closed_unexpectedly
	Offender: [{pid,<0.1716.0>},{name,connection},{mfa,{amqp_network_connection,start_link,[{amqp_params,<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>,<<"/analytics">>,{127,0,0,1},5672,0,0,0,none,[]},<0.1715.0>,#Fun<amqp_connection_sup.0.94524864>]}},{restart_type,intrinsic},{shutdown,brutal_kill},{child_type,worker}]

	2016-06-08 18:36:39 =SUPERVISOR REPORT====
	Supervisor: {<0.1714.0>,amqp_connection_sup}
	Context: shutdown
	Reason: reached_max_restart_intensity
	Offender: [{pid,<0.1716.0>},{name,connection},{mfa,{amqp_network_connection,start_link,[{amqp_params,<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>,<<"/analytics">>,{127,0,0,1},5672,0,0,0,none,[]},<0.1715.0>,#Fun<amqp_connection_sup.0.94524864>]}},{restart_type,intrinsic},{shutdown,brutal_kill},{child_type,worker}]

	2016-06-08 18:36:39 =ERROR REPORT====
	["Could not connect, scheduling reconnect.",{error,{{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1716.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_info,2,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,134}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,593}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,659}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}}]
	2016-06-08_17:35:12.47349 [error] gen_server <0.1755.0> terminated with reason: socket_closed_unexpectedly in amqp_network_connection:handshake_recv/0 line 462
	2016-06-08_17:35:12.47578 [error] CRASH REPORT Process <0.1755.0> with 0 neighbours exited with reason: socket_closed_unexpectedly in gen_server:terminate/7 line 804
	2016-06-08_17:35:12.47824 [error] Supervisor {<0.1753.0>,amqp_connection_sup} had child at module undefined at <0.1755.0> exit with reason socket_closed_unexpectedly in context child_terminated
	2016-06-08_17:35:12.48058 [error] Supervisor {<0.1753.0>,amqp_connection_sup} had child at module undefined at <0.1755.0> exit with reason reached_max_restart_intensity in context shutdown
	2016-06-08_17:35:12.48280 [error] "Could not connect, scheduling reconnect.", error: {{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1755.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_info,2,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,134}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,593}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,659}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}
	2016-06-08_17:35:15.44548 [error] CRASH REPORT Process <0.1706.0> with 0 neighbours exited with reason: {timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}} in gen_server:init_it/6 line 330
	2016-06-08_17:35:15.44817 [error] Supervisor oc_chef_wm_sup had child oc_chef_action_queue started with bunnyc:start_link(oc_chef_action_queue, {network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e...">>},...}, {{'exchange.declare',0,<<"actions">>,<<"topic">>,false,true,false,false,false,[]}}, []) at undefined exit with reason {timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}} in context start_error
	2016-06-08_17:35:15.45046 [error] CRASH REPORT Process <0.1645.0> with 0 neighbours exited with reason: {{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}}}},{oc_chef_wm_app,start,[normal,[]]}} in application_master:init/4 line 133
	2016-06-08_17:35:15.45057 [info] Application oc_chef_wm exited with reason: {{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}},30000]}}}},{oc_chef_wm_app,start,[normal,[]]}}
	2016-06-08_17:35:17.02110 {"Kernel pid terminated",application_controller,"{application_start_failure,oc_chef_wm,{{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706.0>,{network,{127,0,0,1},5672,{<<\"actions\">>,<<\"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983\">>},<<\"/analytics\">>}},30000]}}}},{oc_chef_wm_app,start,[normal,[]]}}}"}
	2016-06-08_17:35:17.02131 Kernel pid terminated (application_controller) ({application_start_failure,oc_chef_wm,{{shutdown,{failed_to_start_child,oc_chef_action_queue,{timeout,{gen_server,call,[gen_bunny_mon,{connect,<0.1706
	2016-06-08_17:35:17.02694 waiting for rabbitmq...
	2016-06-08_17:35:17.23724 Waiting for rabbit@localhost ...
	2016-06-08_17:35:17.23743 pid is 14115 ...
	2016-06-08_17:35:17.26665 ...done.
	2016-06-08_17:35:17.26937 rabbbitmq might be up...
	2016-06-08_17:35:17.27886 Exec: /opt/opscode/embedded/service/opscode-erchef/erts-6.4/bin/erlexec -noshell -noinput +Bd -boot /opt/opscode/embedded/service/opscode-erchef/releases/12.6.0+20160430001037/oc_erchef -mode embedded -config /opt/opscode/embedded/service/opscode-erchef/sys.config -boot_var ERTS_LIB_DIR /opt/opscode/embedded/service/opscode-erchef/erts-6.4/../lib -args_file /opt/opscode/embedded/service/opscode-erchef/vm.args -- foreground
	2016-06-08_17:35:17.27887 Root: /opt/opscode/embedded/service/opscode-erchef
	2016-06-08_17:35:20.79260
	2016-06-08_17:35:20.79262 =INFO REPORT==== 8-Jun-2016::18:35:20 ===
	2016-06-08_17:35:20.79262 starting stats_hero_sup
	2016-06-08_17:35:20.79285
	2016-06-08_17:35:20.79287 =INFO REPORT==== 8-Jun-2016::18:35:20 ===
	2016-06-08_17:35:20.79287 starting stats_hero_sender_sup with 350 senders
	2016-06-08_17:35:20.90398
	2016-06-08_17:35:20.90400 =INFO REPORT==== 8-Jun-2016::18:35:20 ===
	2016-06-08_17:35:20.90400 Folsom Graphite adaptor starting.
	2016-06-08_17:35:20.91480 [info] Application lager started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.91535 [info] Application ssl started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.91558 [info] Application chef_objects started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.91579 [info] Application uuid started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.91600 [info] Application mochiweb started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.91621 [info] Application epgsql started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.91641 [info] Application chef_index started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.91662 [info] Application mini_s3 started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.92136 [info] Application webmachine started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.92150 [info] Application sqerl started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.92172 [info] Application opscoderl_wm started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.92190 [info] Application chef_db started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.92566 [info] Application oc_chef_authz started on node 'erchef@127.0.0.1'
	2016-06-08_17:35:20.92598 [info] Loading ibrowse configuration from /opt/opscode/embedded/service/opscode-erchef/etc/ibrowse/ibrowse.config
	2016-06-08_17:35:20.96840 [info] Starting oc_chef_action
	2016-06-08_17:35:21.06361 [error] Unknown response from RabbitMQ management console: {ok,"401",[{"Server","MochiWeb/1.1 WebMachine/1.10.0 (never breaks eye contact)"},{"Date","Wed, 08 Jun 2016 17:35:21 GMT"},{"Content-Length","50"}],<<"{\"error\":\"not_authorised\",\"reason\":\"Login failed\"}">>}
	2016-06-08_17:35:21.06374 [info] Vhost "/analytics", queue "alaska" not at capacity or RabbitMQ Management Plugin unavailable
	2016-06-08_17:35:21.06382 [info] Starting actions queue monitoring for vhost "/analytics" and queue "alaska"
	2016-06-08_17:35:21.06389 [info] Chef Actions: Connecting to RabbitMQ at {127,0,0,1}:5672/analytics (exchange: <<"actions">>)
	2016-06-08_17:35:24.06771 [error] gen_server <0.1709.0> terminated with reason: socket_closed_unexpectedly in amqp_network_connection:handshake_recv/0 line 462
	2016-06-08_17:35:24.06992 [error] CRASH REPORT Process <0.1709.0> with 0 neighbours exited with reason: socket_closed_unexpectedly in gen_server:terminate/7 line 804
	2016-06-08_17:35:24.07228 [error] Supervisor {<0.1707.0>,amqp_connection_sup} had child at module undefined at <0.1709.0> exit with reason socket_closed_unexpectedly in context child_terminated
	2016-06-08_17:35:24.07467 [error] Supervisor {<0.1707.0>,amqp_connection_sup} had child at module undefined at <0.1709.0> exit with reason reached_max_restart_intensity in context shutdown
	2016-06-08_17:35:24.07759 [error] "Could not connect, scheduling reconnect.", error: {{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1709.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_call,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,87}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,607}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,639}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}
	2016-06-08_17:35:28.07138 [error] gen_server <0.1716.0> terminated with reason: socket_closed_unexpectedly in amqp_network_connection:handshake_recv/0 line 462
	2016-06-08_17:35:28.07377 [error] CRASH REPORT Process <0.1716.0> with 0 neighbours exited with reason: socket_closed_unexpectedly in gen_server:terminate/7 line 804
	2016-06-08_17:35:28.07610 [error] "Could not connect, scheduling reconnect.", error: {{error,{badmatch,{error,{auth_failure_likely,{socket_closed_unexpectedly,{gen_server,call,[<0.1716.0>,connect,infinity]}}}}},[{bunny_util,connect,1,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/bunny_util.erl"},{line,191}]},{gen_bunny_mon,do_connect,3,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,192}]},{gen_bunny_mon,handle_info,2,[{file,"/var/cache/omnibus/src/oc_erchef/_build/default/lib/gen_bunny/src/gen_bunny_mon.erl"},{line,134}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,593}]},{gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,659}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,237}]}]},{connection_info,{network,{127,0,0,1},5672,{<<"actions">>,<<"dae42ef11f3a054990c2bed02e6266648d1cdc280d873a8282798e83c0e19a61d7be1835b98dd092b9154f56c66cb8f94983">>},<<"/analytics">>}}}
	2016-06-08_17:35:28.07827 [error] Supervisor {<0.1714.0>,amqp_connection_sup} had child at module undefined at <0.1716.0> exit with reason socket_closed_unexpectedly in context child_terminated
	2016-06-08_17:35:28.08054 [error] Supervisor {<0.1714.0>,amqp_connection_sup} had child at module undefined at <0.1716.0> exit with reason reached_max_restart_intensity in context shutdown