debug output for https://github.com/open-mpi/ompi/pull/1335

output.txt

[pacini061.arcetri.cisco.com:13809] [[10049,0],47] LOOPING BRKS
malloc debug: Request for 139764985974018 bytes failed (rml_oob_send.c, 175)
[pacini061:13809] *** Process received signal ***
[pacini061:13809] Signal: Segmentation fault (11)
[pacini061:13809] Signal code:  (128)
[pacini061:13809] Failing at address: (nil)
[pacini061:13809] [ 0] /usr/lib64/libpthread.so.0(+0xf100)[0x7f1d95ced100]
[pacini061:13809] [ 1] /usr/lib64/libc.so.6(+0x1496d6)[0x7f1d95a666d6]
[pacini061:13809] [ 2] /home/jsquyres/bogus/lib/openmpi/mca_rml_oob.so(+0x36ce)[0x7f1d925736ce]
[pacini061:13809] [ 3] /home/jsquyres/bogus/lib/libopen-pal.so.0(opal_libevent2022_event_base_loop+0x8fc)[0x7f1d96cbe1ec]
[pacini061:13809] [ 4] /home/jsquyres/bogus/lib/libopen-rte.so.0(orte_daemon+0x2339)[0x7f1d96fb13a6]
[pacini061:13809] [ 5] orted[0x400906]
[pacini061:13809] [ 6] /usr/lib64/libc.so.6(__libc_start_main+0xf5)[0x7f1d9593eb15]
[pacini061:13809] [ 7] orted[0x4007b9]
[pacini061:13809] *** End of error message ***
[pacini026.arcetri.cisco.com:09838] [[10049,0],12] LOOPING BRKS
[pacini069.arcetri.cisco.com:13796] [[10049,0],55] LOOPING BRKS
malloc debug: Request for 140276538379522 bytes failed (rml_oob_send.c, 175)
[pacini026:09838] *** Process received signal ***
[pacini026:09838] Signal: Segmentation fault (11)
[pacini026:09838] Signal code:  (128)
[pacini026:09838] Failing at address: (nil)
[pacini026:09838] [ 0] /lib64/libpthread.so.0(+0xf100)[0x7f94b0b51100]
[pacini026:09838] [ 1] /lib64/libc.so.6(+0x1496d6)[0x7f94b08ca6d6]
[pacini026:09838] [ 2] /home/jsquyres/bogus/lib/openmpi/mca_rml_oob.so(+0x36ce)[0x7f94ad3d76ce]
[pacini026:09838] [ 3] /home/jsquyres/bogus/lib/libopen-pal.so.0(opal_libevent2022_event_base_loop+0x8fc)[0x7f94b1b221ec]
[pacini026:09838] [ 4] /home/jsquyres/bogus/lib/libopen-rte.so.0(orte_daemon+0x2339)[0x7f94b1e153a6]
[pacini026:09838] [ 5] orted[0x400906]
[pacini026:09838] [ 6] malloc debug: Request for 140487095713026 bytes failed (rml_oob_send.c, 175)
[pacini069:13796] *** Process received signal ***
[pacini069:13796] Signal: Segmentation fault (11)
[pacini069:13796] Signal code:  (128)
[pacini069:13796] Failing at address: (nil)
/lib64/libc.so.6(__libc_start_main+0xf5)[0x7f94b07a2b15]
[pacini026:09838] [ 7] orted[0x4007b9]
[pacini026:09838] *** End of error message ***
[pacini069:13796] [ 0] /usr/lib64/libpthread.so.0(+0xf100)[0x7fc5b6e70100]
[pacini069:13796] [ 1] /usr/lib64/libc.so.6(+0x1496d6)[0x7fc5b6be96d6]
[pacini069:13796] [ 2] /home/jsquyres/bogus/lib/openmpi/mca_rml_oob.so(+0x36ce)[0x7fc5b36f66ce]
[pacini069:13796] [ 3] /home/jsquyres/bogus/lib/libopen-pal.so.0(opal_libevent2022_event_base_loop+0x8fc)[0x7fc5b7e411ec]
[pacini069:13796] [ 4] /home/jsquyres/bogus/lib/libopen-rte.so.0(orte_daemon+0x2339)[0x7fc5b81343a6]
[pacini069:13796] [ 5] orted[0x400906]
[pacini069:13796] [ 6] /usr/lib64/libc.so.6(__libc_start_main+0xf5)[0x7fc5b6ac1b15]
[pacini069:13796] [ 7] orted[0x4007b9]
[pacini069:13796] *** End of error message ***
[pacini066.arcetri.cisco.com:13808] [[10049,0],52] LOOPING BRKS
malloc debug: Request for 140664786865410 bytes failed (rml_oob_send.c, 175)
[pacini066:13808] *** Process received signal ***
[pacini066:13808] Signal: Segmentation fault (11)
[pacini066:13808] Signal code:  (128)
[pacini066:13808] Failing at address: (nil)
[pacini066:13808] [ 0] /usr/lib64/libpthread.so.0(+0xf100)[0x7fef161ed100]
[pacini066:13808] [ 1] /usr/lib64/libc.so.6(+0x1496d6)[0x7fef15f666d6]
[pacini066:13808] [ 2] /home/jsquyres/bogus/lib/openmpi/mca_rml_oob.so(+0x36ce)[0x7fef12a736ce]
[pacini066:13808] [ 3] /home/jsquyres/bogus/lib/libopen-pal.so.0(opal_libevent2022_event_base_loop+0x8fc)[0x7fef171be1ec]
[pacini066:13808] [ 4] /home/jsquyres/bogus/lib/libopen-rte.so.0(orte_daemon+0x2339)[0x7fef174b13a6]
[pacini066:13808] [ 5] orted[0x400906]
[pacini066:13808] [ 6] /usr/lib64/libc.so.6(__libc_start_main+0xf5)[0x7fef15e3eb15]
[pacini066:13808] [ 7] orted[0x4007b9]
[pacini066:13808] *** End of error message ***
[pacini060.arcetri.cisco.com:14063] [[10049,0],46] LOOPING BRKS
malloc debug: Request for 140711099489538 bytes failed (rml_oob_send.c, 175)
[pacini060:14063] *** Process received signal ***
[pacini060:14063] Signal: Segmentation fault (11)
[pacini060:14063] Signal code: Address not mapped (1)
[pacini060:14063] Failing at address: (nil)
[pacini060:14063] [ 0] /usr/lib64/libpthread.so.0(+0xf100)[0x7ff9de916100]
[pacini060:14063] [ 1] /usr/lib64/libc.so.6(+0x149544)[0x7ff9de68f544]
[pacini060:14063] [ 2] /home/jsquyres/bogus/lib/openmpi/mca_rml_oob.so(+0x36ce)[0x7ff9db19c6ce]
[pacini060:14063] [ 3] /home/jsquyres/bogus/lib/libopen-pal.so.0(opal_libevent2022_event_base_loop+0x8fc)[0x7ff9df8e71ec]
[pacini060:14063] [ 4] /home/jsquyres/bogus/lib/libopen-rte.so.0(orte_daemon+0x2339)[0x7ff9dfbda3a6]
[pacini060:14063] [ 5] orted[0x400906]
[pacini060:14063] [ 6] /usr/lib64/libc.so.6(__libc_start_main+0xf5)[0x7ff9de567b15]
[pacini060:14063] [ 7] orted[0x4007b9]
[pacini060:14063] *** End of error message ***
[pacini075.arcetri.cisco.com:13614] [[10049,0],61] LOOPING BRKS
malloc debug: Request for 140512675454210 bytes failed (rml_oob_send.c, 175)
[pacini075:13614] *** Process received signal ***
[pacini075:13614] Signal: Segmentation fault (11)
[pacini075:13614] Signal code: Address not mapped (1)
[pacini075:13614] Failing at address: (nil)
[pacini075:13614] [ 0] /usr/lib64/libpthread.so.0(+0xf100)[0x7fcbab92e100]
[pacini075:13614] [ 1] /usr/lib64/libc.so.6(+0x149544)[0x7fcbab6a7544]
[pacini075:13614] [ 2] /home/jsquyres/bogus/lib/openmpi/mca_rml_oob.so(+0x36ce)[0x7fcba81b46ce]
[pacini075:13614] [ 3] /home/jsquyres/bogus/lib/libopen-pal.so.0(opal_libevent2022_event_base_loop+0x8fc)[0x7fcbac8ff1ec]
[pacini075:13614] [ 4] /home/jsquyres/bogus/lib/libopen-rte.so.0(orte_daemon+0x2339)[0x7fcbacbf23a6]
[pacini075:13614] [ 5] orted[0x400906]
[pacini075:13614] [ 6] /usr/lib64/libc.so.6(__libc_start_main+0xf5)[0x7fcbab57fb15]
[pacini075:13614] [ 7] orted[0x4007b9]
[pacini075:13614] *** End of error message ***
--------------------------------------------------------------------------
ORTE has lost communication with its daemon located on node:

  hostname:  pacini069

This is usually due to either a failure of the TCP network
connection to the node, or possibly an internal failure of
the daemon itself. We cannot recover from this failure, and
therefore will terminate the job.
--------------------------------------------------------------------------