MPI.jl fail in MPI.Init()

45886643_1_rank.00_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1379 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436
Array at ./boot.jl:459 [inlined]
Array at ./boot.jl:468 [inlined]
zeros at ./array.jl:588 [inlined]
zeros at ./array.jl:584 [inlined]
DFSTree at ./compiler/ssair/domtree.jl:88
DomTree at ./compiler/ssair/domtree.jl:200 [inlined]
construct_domtree at ./compiler/ssair/domtree.jl:204
slot2reg at ./compiler/optimize.jl:643 [inlined]
run_passes at ./compiler/optimize.jl:536
optimize at ./compiler/optimize.jl:504 [inlined]
_typeinf at ./compiler/typeinfer.jl:257
typeinf at ./compiler/typeinfer.jl:213
typeinf_edge at ./compiler/typeinfer.jl:877
abstract_call_method at ./compiler/abstractinterpretation.jl:647
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:139
abstract_call_known at ./compiler/abstractinterpretation.jl:1716
abstract_call at ./compiler/abstractinterpretation.jl:1786
abstract_call at ./compiler/abstractinterpretation.jl:1753
abstract_eval_statement at ./compiler/abstractinterpretation.jl:1910
typeinf_local at ./compiler/abstractinterpretation.jl:2386
typeinf_nocycle at ./compiler/abstractinterpretation.jl:2482
_typeinf at ./compiler/typeinfer.jl:230
typeinf at ./compiler/typeinfer.jl:213
typeinf_ext at ./compiler/typeinfer.jl:967
typeinf_ext_toplevel at ./compiler/typeinfer.jl:1000
typeinf_ext_toplevel at ./compiler/typeinfer.jl:996
jfptr_typeinf_ext_toplevel_17873.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
jl_type_infer at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:315
jl_generate_fptr_impl at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:319
jl_compile_method_internal at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2091 [inlined]
jl_compile_method_internal at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2035
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2369 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
foreach at ./abstractarray.jl:2774
unknown function (ip: 0x7f4c805d9c43)
run_load_time_hooks at /home/lwilcox/.julia/packages/MPI/APiiL/src/MPI.jl:57 [inlined]
__init__ at /home/lwilcox/.julia/packages/MPI/APiiL/src/MPI.jl:137
unknown function (ip: 0x7f4c805d8fcf)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
jl_module_run_initializer at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:75
ijl_init_restored_modules at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/dump.c:2866
_include_from_serialized at ./loading.jl:831
_require_search_from_serialized at ./loading.jl:1039
_require at ./loading.jl:1315
_require_prelocked at ./loading.jl:1200
macro expansion at ./loading.jl:1180 [inlined]
macro expansion at ./lock.jl:223 [inlined]
require at ./loading.jl:1144
jfptr_require_29799.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
call_require at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:462 [inlined]
eval_import_path at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:499
jl_toplevel_eval_flex at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:725
jl_toplevel_eval_flex at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:850
ijl_toplevel_eval_in at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:965
eval at ./boot.jl:368 [inlined]
include_string at ./loading.jl:1428
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
_include at ./loading.jl:1488
include at ./Base.jl:419
jfptr_include_47457.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
exec_options at ./client.jl:303
_start at ./client.jl:522
jfptr__start_58474.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
true_main at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jlapi.c:575
jl_repl_entrypoint at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jlapi.c:719
main at julia (unknown line)
__libc_start_main at /lib64/libc.so.6 (unknown line)
unknown function (ip: 0x401098)
unknown function (ip: (nil))
Allocations: 710570 (Pool: 710210; Big: 360); GC: 0

45886643_1_rank.00_stdout

45886643_1_rank.01_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1379 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.01_stdout

45886643_1_rank.02_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:2
_int_malloc at /lib64/libc.so.6 (unknown line)
__libc_malloc at /lib64/libc.so.6 (unknown line)
operator new at /workspace/srcdir/gcc-12.1.0/libstdc++-v3/libsupc++/new_op.cc:50
_ZN4llvm20BumpPtrAllocatorImplINS_15MallocAllocatorELm4096ELm4096ELm128EE8AllocateEmNS_5AlignE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm15MachineFunction18CreateMachineInstrERKNS_11MCInstrDescERKNS_8DebugLocEb at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm12InstrEmitter15EmitMachineNodeEPNS_6SDNodeEbbRNS_8DenseMapINS_7SDValueENS_8RegisterENS_12DenseMapInfoIS4_EENS_6detail12DenseMapPairIS4_S5_EEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm18ScheduleDAGSDNodes12EmitScheduleERNS_26MachineInstrBundleIteratorINS_12MachineInstrELb0EEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm16SelectionDAGISel17CodeGenAndEmitDAGEv at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm16SelectionDAGISel20SelectAllBasicBlocksERKNS_8FunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm16SelectionDAGISel20runOnMachineFunctionERNS_15MachineFunctionE.part.899 at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN12_GLOBAL__N_115X86DAGToDAGISel20runOnMachineFunctionERN4llvm15MachineFunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm19MachineFunctionPass13runOnFunctionERNS_8FunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm13FPPassManager13runOnFunctionERNS_8FunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm13FPPassManager11runOnModuleERNS_6ModuleE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm6legacy15PassManagerImpl3runERNS_6ModuleE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
operator() at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:528
_ZN4llvm3orc14IRCompileLayer4emitESt10unique_ptrINS0_29MaterializationResponsibilityESt14default_deleteIS3_EENS0_16ThreadSafeModuleE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc31BasicIRLayerMaterializationUnit11materializeESt10unique_ptrINS0_29MaterializationResponsibilityESt14default_deleteIS3_EE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc19MaterializationTask3runEv at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm6detail18UniqueFunctionBaseIvJSt10unique_ptrINS_3orc4TaskESt14default_deleteIS4_EEEE8CallImplIPFvS7_EEEvPvRS7_ at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession22dispatchOutstandingMUsEv at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession17OL_completeLookupESt10unique_ptrINS0_21InProgressLookupStateESt14default_deleteIS3_EESt10shared_ptrINS0_23AsynchronousSymbolQueryEESt8functionIFvRKNS_8DenseMapIPNS0_8JITDylibENS_8DenseSetINS0_15SymbolStringPtrENS_12DenseMapInfoISF_EEEENSG_ISD_EENS_6detail12DenseMapPairISD_SI_EEEEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc25InProgressFullLookupState8completeESt10unique_ptrINS0_21InProgressLookupStateESt14default_deleteIS3_EE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession19OL_applyQueryPhase1ESt10unique_ptrINS0_21InProgressLookupStateESt14default_deleteIS3_EENS_5ErrorE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupENS0_10LookupKindERKSt6vectorISt4pairIPNS0_8JITDylibENS0_19JITDylibLookupFlagsEESaIS8_EENS0_15SymbolLookupSetENS0_11SymbolStateENS_15unique_functionIFvNS_8ExpectedINS_8DenseMapINS0_15SymbolStringPtrENS_18JITEvaluatedSymbolENS_12DenseMapInfoISI_EENS_6detail12DenseMapPairISI_SJ_EEEEEEEEESt8functionIFvRKNSH_IS6_NS_8DenseSetISI_SL_EENSK_IS6_EENSN_IS6_SV_EEEEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupERKSt6vectorISt4pairIPNS0_8JITDylibENS0_19JITDylibLookupFlagsEESaIS7_EERKNS0_15SymbolLookupSetENS0_10LookupKindENS0_11SymbolStateESt8functionIFvRKNS_8DenseMapIS5_NS_8DenseSetINS0_15SymbolStringPtrENS_12DenseMapInfoISK_EEEENSL_IS5_EENS_6detail12DenseMapPairIS5_SN_EEEEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupERKSt6vectorISt4pairIPNS0_8JITDylibENS0_19JITDylibLookupFlagsEESaIS7_EENS0_15SymbolStringPtrENS0_11SymbolStateE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupENS_8ArrayRefIPNS0_8JITDylibEEENS0_15SymbolStringPtrENS0_11SymbolStateE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupENS_8ArrayRefIPNS0_8JITDylibEEENS_9StringRefENS0_11SymbolStateE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
addModule at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:968
jl_add_to_ee at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:1261 [inlined]
jl_add_to_ee at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:1305
jl_add_to_ee at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:1327 [inlined]
_jl_compile_codeinst at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:149
jl_generate_fptr_impl at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:332

45886643_1_rank.02_stdout

45886643_1_rank.03_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.03_stdout

45886643_1_rank.04_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.04_stdout

45886643_1_rank.05_stderr

45886643_1_rank.05_stdout

45886643_1_rank.06_stderr

45886643_1_rank.06_stdout

45886643_1_rank.07_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_malloced_arrays at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1170 [inlined]
gc_sweep_other at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1530 [inlined]
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3240
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.07_stdout

45886643_1_rank.08_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.08_stdout

45886643_1_rank.09_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.09_stdout

45886643_1_rank.10_stderr

45886643_1_rank.10_stdout

45886643_1_rank.11_stderr

45886643_1_rank.11_stdout

45886643_1_rank.12_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1

45886643_1_rank.12_stdout

45886643_1_rank.13_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.13_stdout

45886643_1_rank.14_stderr

45886643_1_rank.14_stdout

45886643_1_rank.15_stderr

45886643_1_rank.15_stdout

45886643_1_rank.16_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1388 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.16_stdout

45886643_1_rank.17_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.17_stdout

45886643_1_rank.18_stderr

45886643_1_rank.18_stdout

45886643_1_rank.19_stderr

45886643_1_rank.19_stdout

45886643_1_rank.20_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

45886643_1_rank.20_stdout

45886643_1_rank.21_stderr

45886643_1_rank.21_stdout

45886643_1_rank.22_stderr

45886643_1_rank.22_stdout

45886643_1_rank.23_stderr

45886643_1_rank.23_stdout

45886643_1_rank.24_stderr

45886643_1_rank.24_stdout

45886643_1_rank.25_stderr

45886643_1_rank.25_stdout

45886643_1_rank.26_stderr

[compute-2-30.hamming.cluster:50550] OPAL ERROR: Error in file ../../opal/runtime/opal_progress_threads.c at line 134
[compute-2-30.hamming.cluster:50550] OPAL ERROR: Error in file ../../opal/runtime/opal_progress_threads.c at line 193
[compute-2-30.hamming.cluster:50550] PMIX ERROR: ERROR in file ../../../../../../../opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c at line 140
[compute-2-30.hamming.cluster:50550] PMIX ERROR: ERROR in file ../../../../../../../opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c at line 199
--------------------------------------------------------------------------
It looks like pmix_init failed for some reason; your parallel process is
likely to abort.  There are many reasons that a parallel process can
fail during pmix_init; some of which are due to configuration or
environment problems.  This failure appears to be an internal failure;
here's some additional information (which may only be relevant to an
PMIX developer):

  progress thread failed
  --> Returned value -1 instead of PMIX_SUCCESS
--------------------------------------------------------------------------
[compute-2-30.hamming.cluster:50550] PMIX ERROR: ERROR in file ../../../../../../../opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c at line 426
[compute-2-30.hamming.cluster:50550] OPAL ERROR: Error in file ../../../../../opal/mca/pmix/pmix2x/pmix2x_client.c at line 112
*** An error occurred in MPI_Init_thread
*** on a NULL communicator
*** MPI_ERRORS_ARE_FATAL (processes in this communicator will now abort,
***    and potentially your MPI job)
[compute-2-30.hamming.cluster:50550] Local abort before MPI_INIT completed completed successfully, but am not able to aggregate error messages, and not able to guarantee that all other processes were killed!

45886643_1_rank.26_stdout

45886643_1_rank.27_stderr

45886643_1_rank.27_stdout

45886643_1_rank.28_stderr

45886643_1_rank.28_stdout

45886643_1_rank.29_stderr

45886643_1_rank.29_stdout

45886643_1_rank.30_stderr

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:2
nanosleep at /lib64/libc.so.6 (unknown line)
usleep at /lib64/libc.so.6 (unknown line)

45886643_1_rank.30_stdout

45886643_1_rank.31_stderr

45886643_1_rank.31_stdout

45886643_1_rank.32_stderr

45886643_1_rank.32_stdout

45886643_1_rank.33_stderr

45886643_1_rank.33_stdout

45886643_1_rank.34_stderr

45886643_1_rank.34_stdout

45886643_1_rank.35_stderr

45886643_1_rank.35_stdout

45886643_1_rank.36_stderr

45886643_1_rank.36_stdout

45886643_1_rank.37_stderr

45886643_1_rank.37_stdout

45886643_1_rank.38_stderr

45886643_1_rank.38_stdout

45886643_1_rank.39_stderr

45886643_1_rank.39_stdout

45886643_1_rank.40_stderr

45886643_1_rank.40_stdout

45886643_1_rank.41_stderr

45886643_1_rank.41_stdout

45886643_1_rank.42_stderr

45886643_1_rank.42_stdout

45886643_1_rank.43_stderr

45886643_1_rank.43_stdout

45886643_1_rank.44_stderr

45886643_1_rank.44_stdout

45886643_1_rank.45_stderr

45886643_1_rank.45_stdout

45886643_1_rank.46_stderr

45886643_1_rank.46_stdout

45886643_1_rank.47_stderr

45886643_1_rank.47_stdout

45886643_1_rank.48_stderr

45886643_1_rank.48_stdout

45886643_1_rank.49_stderr

45886643_1_rank.49_stdout

45886643_1_rank.50_stderr

45886643_1_rank.50_stdout

45886643_1_rank.51_stderr

45886643_1_rank.51_stdout

45886643_1_rank.52_stderr

45886643_1_rank.52_stdout

45886643_1_rank.53_stderr

45886643_1_rank.53_stdout

45886643_1_rank.54_stderr

45886643_1_rank.54_stdout

45886643_1_rank.55_stderr

45886643_1_rank.55_stdout

45886643_1_rank.56_stderr

45886643_1_rank.56_stdout

45886643_1_rank.57_stderr

45886643_1_rank.57_stdout

45886643_1_rank.58_stderr

45886643_1_rank.58_stdout

45886643_1_rank.59_stderr

45886643_1_rank.59_stdout

45886643_1_rank.60_stderr

45886643_1_rank.60_stdout

45886643_1_rank.61_stderr

45886643_1_rank.61_stdout

45886643_1_rank.62_stderr

45886643_1_rank.62_stdout

45886643_1_rank.63_stderr

45886643_1_rank.63_stdout

hello.jl

using MPI
MPI.Init()

comm = MPI.COMM_WORLD
println("Hello, world, I am rank $(MPI.Comm_rank(comm)) of $(MPI.Comm_size(comm)).")

mpi-job.sh

#!/bin/bash
#SBATCH --time=0:10:00
#SBATCH --nodes=2            # node count
#SBATCH --ntasks-per-node=32 # number of tasks per node
#SBATCH --cpus-per-task=1    # cpu-cores per task
#SBATCH --mem=2G
#SBATCH --output=output-%j.txt

source /etc/profile
module load sdk/nvidia/22.7

# have the shell echo the commands that are run
set -x

export JULIA_NUM_THREADS=1
julia --project -e 'using Pkg; pkg"instantiate"'
julia --project -e 'using Pkg; pkg"precompile"'
mpiexec --mca mpi_cuda_support 0 -output-filename $SLURM_JOB_ID julia hello.jl

#mpiexec --mca mpi_cuda_support 0 -output-filename $SLURM_JOB_ID ./hello

exit 0

output-45886643.txt

+ export JULIA_NUM_THREADS=1
+ JULIA_NUM_THREADS=1
+ julia --project -e 'using Pkg; pkg"instantiate"'
┌ Warning: The Pkg REPL mode is intended for interactive use only, and should not be used from scripts. It is recommended to use the functional API instead.
└ @ Pkg.REPLMode /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/usr/share/julia/stdlib/v1.8/Pkg/src/REPLMode/REPLMode.jl:379
+ julia --project -e 'using Pkg; pkg"precompile"'
┌ Warning: The Pkg REPL mode is intended for interactive use only, and should not be used from scripts. It is recommended to use the functional API instead.
└ @ Pkg.REPLMode /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/usr/share/julia/stdlib/v1.8/Pkg/src/REPLMode/REPLMode.jl:379
+ mpiexec --mca mpi_cuda_support 0 -output-filename 45886643 julia hello.jl
--------------------------------------------------------------------------
WARNING: No preset parameters were found for the device that Open MPI
detected:

  Local host:            compute-2-30
  Device name:           mlx5_0
  Device vendor ID:      0x02c9
  Device vendor part ID: 4123

Default device parameters will be used, which may result in lower
performance.  You can edit any of the files specified by the
btl_openib_device_param_files MCA parameter to set values for your
device.

NOTE: You can turn off this warning by setting the MCA parameter
      btl_openib_warn_no_device_params_found to 0.
--------------------------------------------------------------------------
[compute-2-30.hamming.cluster:50550] OPAL ERROR: Error in file ../../opal/runtime/opal_progress_threads.c at line 134
[compute-2-30.hamming.cluster:50550] OPAL ERROR: Error in file ../../opal/runtime/opal_progress_threads.c at line 193
[compute-2-30.hamming.cluster:50550] PMIX ERROR: ERROR in file ../../../../../../../opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c at line 140
[compute-2-30.hamming.cluster:50550] PMIX ERROR: ERROR in file ../../../../../../../opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c at line 199
--------------------------------------------------------------------------
It looks like pmix_init failed for some reason; your parallel process is
likely to abort.  There are many reasons that a parallel process can
fail during pmix_init; some of which are due to configuration or
environment problems.  This failure appears to be an internal failure;
here's some additional information (which may only be relevant to an
PMIX developer):

  progress thread failed
  --> Returned value -1 instead of PMIX_SUCCESS
--------------------------------------------------------------------------
[compute-2-30.hamming.cluster:50550] PMIX ERROR: ERROR in file ../../../../../../../opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c at line 426
[compute-2-30.hamming.cluster:50550] OPAL ERROR: Error in file ../../../../../opal/mca/pmix/pmix2x/pmix2x_client.c at line 112
*** An error occurred in MPI_Init_thread
*** on a NULL communicator
*** MPI_ERRORS_ARE_FATAL (processes in this communicator will now abort,
***    and potentially your MPI job)
[compute-2-30.hamming.cluster:50550] Local abort before MPI_INIT completed completed successfully, but am not able to aggregate error messages, and not able to guarantee that all other processes were killed!
--------------------------------------------------------------------------
Primary job  terminated normally, but 1 process returned
a non-zero exit code. Per user-direction, the job has been aborted.
--------------------------------------------------------------------------

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1379 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436
Array at ./boot.jl:459 [inlined]
Array at ./boot.jl:468 [inlined]
zeros at ./array.jl:588 [inlined]
zeros at ./array.jl:584 [inlined]
DFSTree at ./compiler/ssair/domtree.jl:88
DomTree at ./compiler/ssair/domtree.jl:200 [inlined]
construct_domtree at ./compiler/ssair/domtree.jl:204
slot2reg at ./compiler/optimize.jl:643 [inlined]
run_passes at ./compiler/optimize.jl:536
optimize at ./compiler/optimize.jl:504 [inlined]
_typeinf at ./compiler/typeinfer.jl:257
typeinf at ./compiler/typeinfer.jl:213
typeinf_edge at ./compiler/typeinfer.jl:877
abstract_call_method at ./compiler/abstractinterpretation.jl:647
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:139
abstract_call_known at ./compiler/abstractinterpretation.jl:1716
abstract_call at ./compiler/abstractinterpretation.jl:1786
abstract_call at ./compiler/abstractinterpretation.jl:1753
abstract_eval_statement at ./compiler/abstractinterpretation.jl:1910
typeinf_local at ./compiler/abstractinterpretation.jl:2386
typeinf_nocycle at ./compiler/abstractinterpretation.jl:2482
_typeinf at ./compiler/typeinfer.jl:230
typeinf at ./compiler/typeinfer.jl:213
typeinf_ext at ./compiler/typeinfer.jl:967
typeinf_ext_toplevel at ./compiler/typeinfer.jl:1000
typeinf_ext_toplevel at ./compiler/typeinfer.jl:996
jfptr_typeinf_ext_toplevel_17873.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
jl_type_infer at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:315
jl_generate_fptr_impl at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:319
jl_compile_method_internal at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2091 [inlined]
jl_compile_method_internal at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2035
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2369 [inlined]
ijl_apply_generic at /cac
signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1379 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:2
_int_malloc at /lib64/libc.so.6 (unknown line)
__libc_malloc at /lib64/libc.so.6 (unknown line)
operator new at /workspace/srcdir/gcc-12.1.0/libstdc++-v3/libsupc++/new_op.cc:50
_ZN4llvm20BumpPtrAllocatorImplINS_15MallocAllocatorELm4096ELm4096ELm128EE8AllocateEmNS_5AlignE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm15MachineFunction18CreateMachineInstrERKNS_11MCInstrDescERKNS_8DebugLocEb at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm12InstrEmitter15EmitMachineNodeEPNS_6SDNodeEbbRNS_8DenseMapINS_7SDValueENS_8RegisterENS_12DenseMapInfoIS4_EENS_6detail12DenseMapPairIS4_S5_EEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm18ScheduleDAGSDNodes12EmitScheduleERNS_26MachineInstrBundleIteratorINS_12MachineInstrELb0EEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm16SelectionDAGISel17CodeGenAndEmitDAGEv at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm16SelectionDAGISel20SelectAllBasicBlocksERKNS_8FunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm16SelectionDAGISel20runOnMachineFunctionERNS_15MachineFunctionE.part.899 at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN12_GLOBAL__N_115X86DAGToDAGISel20runOnMachineFunctionERN4llvm15MachineFunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm19MachineFunctionPass13runOnFunctionERNS_8FunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm13FPPassManager13runOnFunctionERNS_8FunctionE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm13FPPassManager11runOnModuleERNS_6ModuleE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm6legacy15PassManagerImpl3runERNS_6ModuleE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
operator() at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:528
_ZN4llvm3orc14IRCompileLayer4emitESt10unique_ptrINS0_29MaterializationResponsibilityESt14default_deleteIS3_EENS0_16ThreadSafeModuleE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc31BasicIRLayerMaterializationUnit11materializeESt10unique_ptrINS0_29MaterializationResponsibilityESt14default_deleteIS3_EE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc19MaterializationTask3runEv at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm6detail18UniqueFunctionBaseIvJSt10unique_ptrINS_3orc4TaskESt14default_deleteIS4_EEEE8CallImplIPFvS7_EEEvPvRS7_ at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession22dispatchOutstandingMUsEv at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession17OL_completeLookupESt10unique_ptrINS0_21InProgressLookupStateESt14default_deleteIS3_EESt10shared_ptrINS0_23AsynchronousSymbolQueryEESt8functionIFvRKNS_8DenseMapIPNS0_8JITDylibENS_8DenseSetINS0_15SymbolStringPtrENS_12DenseMapInfoISF_EEEENSG_ISD_EENS_6detail12DenseMapPairISD_SI_EEEEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc25InProgressFullLookupState8completeESt10unique_ptrINS0_21InProgressLookupStateESt14default_deleteIS3_EE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession19OL_applyQueryPhase1ESt10unique_ptrINS0_21InProgressLookupStateESt14default_deleteIS3_EENS_5ErrorE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupENS0_10LookupKindERKSt6vectorISt4pairIPNS0_8JITDylibENS0_19JITD
signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_malloced_arrays at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1170 [inlined]
gc_sweep_other at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1530 [inlined]
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3240
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1388 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:1
reset_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1209 [inlined]
sweep_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1342 [inlined]
sweep_pool_page at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1447 [inlined]
sweep_pool_pagetable0 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1467 [inlined]
sweep_pool_pagetable1 at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1487 [inlined]
sweep_pool_pagetable at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1517 [inlined]
gc_sweep_pool at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1592
_jl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3243
ijl_gc_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3346
maybe_collect at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:903 [inlined]
jl_gc_pool_alloc_inner at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1247 [inlined]
jl_gc_pool_alloc_noinline at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:1306 [inlined]
jl_gc_alloc_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia_internal.h:368 [inlined]
jl_gc_alloc at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gc.c:3391
_new_array_ at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:134 [inlined]
_new_array at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:198 [inlined]
ijl_alloc_array_1d at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/array.c:436

signal (15): Terminated
in expression starting at /home/lwilcox/scalable/MPI/hello.jl:2
nanosleep at /lib64/libc.so.6 (unknown line)
usleep at /lib64/libc.so.6 (unknown line)
he/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
foreach at ./abstractarray.jl:2774
unknown function (ip: 0x7f4c805d9c43)
run_load_time_hooks at /home/lwilcox/.julia/packages/MPI/APiiL/src/MPI.jl:57 [inlined]
__init__ at /home/lwilcox/.julia/packages/MPI/APiiL/src/MPI.jl:137
unknown function (ip: 0x7f4c805d8fcf)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
jl_module_run_initializer at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:75
ijl_init_restored_modules at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/dump.c:2866
_include_from_serialized at ./loading.jl:831
_require_search_from_serialized at ./loading.jl:1039
_require at ./loading.jl:1315
_require_prelocked at ./loading.jl:1200
macro expansion at ./loading.jl:1180 [inlined]
macro expansion at ./lock.jl:223 [inlined]
require at ./loading.jl:1144
jfptr_require_29799.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
call_require at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:462 [inlined]
eval_import_path at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:499
jl_toplevel_eval_flex at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:725
jl_toplevel_eval_flex at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:850
ijl_toplevel_eval_in at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/toplevel.c:965
eval at ./boot.jl:368 [inlined]
include_string at ./loading.jl:1428
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
_include at ./loading.jl:1488
include at ./Base.jl:419
jfptr_include_47457.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
exec_options at ./client.jl:303
_start at ./client.jl:522
jfptr__start_58474.clone_1 at /home/lwilcox/local/julia/1.8.4/lib/julia/sys.so (unknown line)
_jl_invoke at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2377 [inlined]
ijl_apply_generic at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/gf.c:2559
jl_apply at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/julia.h:1843 [inlined]
true_main at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jlapi.c:575
jl_repl_entrypoint at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jlapi.c:719
main at julia (unknown line)
__libc_start_main at /lib64/libc.so.6 (unknown line)
unknown function (ip: 0x401098)
unknown function (ip: (nil))
Allocations: 710570 (Pool: 710210; Big: 360); GC: 0
ylibLookupFlagsEESaIS8_EENS0_15SymbolLookupSetENS0_11SymbolStateENS_15unique_functionIFvNS_8ExpectedINS_8DenseMapINS0_15SymbolStringPtrENS_18JITEvaluatedSymbolENS_12DenseMapInfoISI_EENS_6detail12DenseMapPairISI_SJ_EEEEEEEEESt8functionIFvRKNSH_IS6_NS_8DenseSetISI_SL_EENSK_IS6_EENSN_IS6_SV_EEEEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupERKSt6vectorISt4pairIPNS0_8JITDylibENS0_19JITDylibLookupFlagsEESaIS7_EERKNS0_15SymbolLookupSetENS0_10LookupKindENS0_11SymbolStateESt8functionIFvRKNS_8DenseMapIS5_NS_8DenseSetINS0_15SymbolStringPtrENS_12DenseMapInfoISK_EEEENSL_IS5_EENS_6detail12DenseMapPairIS5_SN_EEEEEE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupERKSt6vectorISt4pairIPNS0_8JITDylibENS0_19JITDylibLookupFlagsEESaIS7_EENS0_15SymbolStringPtrENS0_11SymbolStateE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupENS_8ArrayRefIPNS0_8JITDylibEEENS0_15SymbolStringPtrENS0_11SymbolStateE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
_ZN4llvm3orc16ExecutionSession6lookupENS_8ArrayRefIPNS0_8JITDylibEEENS_9StringRefENS0_11SymbolStateE at /home/lwilcox/local/julia/1.8.4/bin/../lib/julia/libLLVM-13jl.so (unknown line)
addModule at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:968
jl_add_to_ee at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:1261 [inlined]
jl_add_to_ee at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:1305
jl_add_to_ee at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:1327 [inlined]
_jl_compile_codeinst at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:149
jl_generate_fptr_impl at /cache/build/default-aws-shared0-3/julialang/julia-release-1-dot-8/src/jitlayers.cpp:332
--------------------------------------------------------------------------
mpiexec detected that one or more processes exited with non-zero status, thus causing
the job to be terminated. The first process to do so was:

  Process name: [[5555,1],26]
  Exit code:    1
--------------------------------------------------------------------------
[compute-2-30.hamming.cluster:50471] PMIX ERROR: ERROR in file ../../../../../../../../../../opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c at line 99
[compute-2-30.hamming.cluster:50471] PMIX ERROR: ERROR in file ../../../../../../../../../../opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c at line 99
[compute-2-30.hamming.cluster:50471] PMIX ERROR: ERROR in file ../../../../../../../../../../opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c at line 99
+ exit 0