Created
November 9, 2022 19:10
-
-
Save Csinclair0/48ee4f1388d4901e6958069ee272a305 to your computer and use it in GitHub Desktop.
nccl_tests_error
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| + POD_NAME=nccl-tests-worker-1 | |
| + shift | |
| + POD_NAME=nccl-tests-worker-0 | |
| + shift | |
| + /opt/kube/kubectl exec nccl-tests-worker-1 -- /bin/sh -c ' OPAL_PREFIX= ; export OPAL_PREFIX; PATH=/opt/amazon/openmpi/bin:$PATH ; export PATH ; LD_LIBRARY_PATH=/opt/amazon/openmpi/lib:${LD_LIBRARY_PATH:-} ; export LD_LIBRARY_PATH ; DYLD_LIBRARY_PATH=/opt/amazon/openmpi/lib:${DYLD_LIBRARY_PATH:-} ; export DYLD_LIBRARY_PATH ; /opt/amazon/openmpi/bin/orted -mca ess "env" -mca ess_base_jobid "390594560" -mca ess_base_vpid 2 -mca ess_base_num_procs "3" -mca orte_node_regex "nccl-tests-launcher,nccl-tests-worker-[1:0-1]@0(3)" -mca orte_hnp_uri "390594560.0;tcp://100.74.165.198:34707" --mca pml "^cm" -mca plm "rsh" --tree-spawn -mca routed "radix" -mca orte_parent_uri "390594560.0;tcp://100.74.165.198:34707" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca coll_hcoll_enable "0" -mca orte_tag_output "1" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated"' | |
| + /opt/kube/kubectl exec nccl-tests-worker-0 -- /bin/sh -c ' OPAL_PREFIX= ; export OPAL_PREFIX; PATH=/opt/amazon/openmpi/bin:$PATH ; export PATH ; LD_LIBRARY_PATH=/opt/amazon/openmpi/lib:${LD_LIBRARY_PATH:-} ; export LD_LIBRARY_PATH ; DYLD_LIBRARY_PATH=/opt/amazon/openmpi/lib:${DYLD_LIBRARY_PATH:-} ; export DYLD_LIBRARY_PATH ; /opt/amazon/openmpi/bin/orted -mca ess "env" -mca ess_base_jobid "390594560" -mca ess_base_vpid 1 -mca ess_base_num_procs "3" -mca orte_node_regex "nccl-tests-launcher,nccl-tests-worker-[1:0-1]@0(3)" -mca orte_hnp_uri "390594560.0;tcp://100.74.165.198:34707" --mca pml "^cm" -mca plm "rsh" --tree-spawn -mca routed "radix" -mca orte_parent_uri "390594560.0;tcp://100.74.165.198:34707" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca coll_hcoll_enable "0" -mca orte_tag_output "1" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated"' | |
| [1,11]<stderr>:libfabric:33:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017249::core:core:fi_param_get_():278<info> variable perf_cntr=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017249::core:core:fi_param_get_():278<info> variable hook=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,7]<stderr>:libfabric:39:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,7]<stderr>:libfabric:39:1668017250::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,7]<stderr>:libfabric:39:1668017250::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,7]<stderr>:libfabric:39:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,7]<stderr>:libfabric:39:1668017250::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,7]<stderr>:libfabric:39:1668017250::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,11]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,11]<stderr>:libfabric:33:1668017250::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,12]<stderr>:libfabric:34:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,12]<stderr>:libfabric:34:1668017250::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,12]<stderr>:libfabric:34:1668017250::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,11]<stderr>:libfabric:33:1668017250::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,11]<stderr>:libfabric:33:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,11]<stderr>:libfabric:33:1668017250::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,12]<stderr>:libfabric:34:1668017250::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,12]<stderr>:libfabric:34:1668017250::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,11]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,2]<stderr>:libfabric:32:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,2]<stderr>:libfabric:32:1668017250::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,2]<stderr>:libfabric:32:1668017250::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,2]<stderr>:libfabric:32:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,2]<stderr>:libfabric:32:1668017250::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,2]<stderr>:libfabric:32:1668017250::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,15]<stderr>:libfabric:37:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,15]<stderr>:libfabric:37:1668017250::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,15]<stderr>:libfabric:37:1668017250::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,15]<stderr>:libfabric:37:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,15]<stderr>:libfabric:37:1668017250::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,15]<stderr>:libfabric:37:1668017250::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,3]<stderr>:libfabric:33:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,3]<stderr>:libfabric:33:1668017250::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,3]<stderr>:libfabric:33:1668017250::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,3]<stderr>:libfabric:33:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,3]<stderr>:libfabric:33:1668017250::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,3]<stderr>:libfabric:33:1668017250::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,13]<stderr>:libfabric:35:1668017250:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,13]<stderr>:libfabric:35:1668017250::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,13]<stderr>:libfabric:35:1668017250::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,13]<stderr>:libfabric:35:1668017250::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,13]<stderr>:libfabric:35:1668017250::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,13]<stderr>:libfabric:35:1668017250::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,6]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,6]<stderr>:libfabric:36:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,6]<stderr>:libfabric:36:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,1]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,1]<stderr>:libfabric:31:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,1]<stderr>:libfabric:31:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,9]<stderr>:libfabric:31:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,9]<stderr>:libfabric:31:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,9]<stderr>:libfabric:31:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,9]<stderr>:libfabric:31:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,9]<stderr>:libfabric:31:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,9]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,5]<stderr>:libfabric:35:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,5]<stderr>:libfabric:35:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,5]<stderr>:libfabric:35:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,8]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,8]<stderr>:libfabric:30:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,8]<stderr>:libfabric:30:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,8]<stderr>:libfabric:30:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,8]<stderr>:libfabric:30:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,8]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,4]<stderr>:libfabric:34:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,4]<stderr>:libfabric:34:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,4]<stderr>:libfabric:34:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,14]<stderr>:libfabric:36:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,14]<stderr>:libfabric:36:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,14]<stderr>:libfabric:36:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,14]<stderr>:libfabric:36:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,14]<stderr>:libfabric:36:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,14]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,10]<stderr>:libfabric:32:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,10]<stderr>:libfabric:32:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,10]<stderr>:libfabric:32:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,10]<stderr>:libfabric:32:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,10]<stderr>:libfabric:32:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,10]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_max_count=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cache_monitor=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_cuda_cache_monitor_enabled=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_rocr_cache_monitor_enabled=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable mr_ze_cache_monitor_enabled=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:mr:ofi_default_cache_size():77<info> default cache size=4181811882 | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():287<info> read string var provider=efa | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable poll_fairness=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable provider_path=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable sar_threshold=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:fi_param_get_():278<info> variable disable_cma=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: shm (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "shm" filtered by provider include/exclude list, skipping | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable enable_passthru=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable buffer_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_tx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable msg_rx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cm_progress_interval=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable cq_eq_fairness=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable data_auto_progress=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_rndv_write=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_wait_obj=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable def_tcp_wait_obj=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_rxm (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable config=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_mrail:core:fi_param_get_():278<info> variable addr_strc=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_mrail:core:mrail_parse_env_vars():115<info> unable to read FI_OFI_MRAIL_ADDR env variable | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_mrail (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_min_credits=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_queue_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable enable_shm_transfer=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():292<info> read int var use_device_rdma=0 | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable use_zcpy_rx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable zcpy_rx_seed=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable shm_av_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable recvwin_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable readcopy_pool_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable cq_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable max_memcpy_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mr_cache_enable=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_count=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mr_max_cached_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable mtu_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable tx_iov_limit=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_iov_limit=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_unexp=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable rx_copy_ooo=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable max_timeout=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable timeout_interval=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable efa_cq_read_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable shm_cq_read_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_medium_message_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_message_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_min_read_write_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_read_segment_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable inter_max_gdrcopy_message_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable runt_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:fi_param_get_():278<info> variable fork_safe=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: efa (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: udp (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "udp" filtered by provider include/exclude list, skipping | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: sockets (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "sockets" filtered by provider include/exclude list, skipping | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable port_high_range=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable port_low_range=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable tx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable rx_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable nodelay=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable staging_sbuf_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable prefetch_rbuf_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::tcp:core:fi_param_get_():278<info> variable zerocopy_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: tcp (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():494<info> "tcp" filtered by provider include/exclude list, skipping | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_perf (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_debug (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_use_gdrcopy=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:cuda_gdrcopy_hmem_init():200<warn> gdr_open failed! | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:cuda_hmem_init():481<warn> gdrcopy initialization failed! gdrcopy will not be used. | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_cuda_enable_xfer=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ROCR not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_ZE not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:neuron_dl_init():63<info> Failed to dlopen libnrt.so | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_NEURON not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_hmem_init():241<info> Hmem iface FI_HMEM_SYNAPSEAI not supported | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable hmem_disable_p2p=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_hmem (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_dmabuf_peer_mem (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_register_provider():466<info> registering provider: ofi_hook_noop (116.0) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_rxm layering | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:ofi_layering_ok():1074<info> Skipping shm;ofi_mrail layering | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,0]<stderr>:libfabric:30:1668017251:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,0]<stderr>:libfabric:30:1668017251::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,0]<stderr>:libfabric:30:1668017251::core:mr:ofi_monitor_import():818<info> setting imported memory monitor as default | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:domain:efa_mr_cache_open():157<info> EFA MR cache enabled, max_cnt: 235929 max_size: 92771293593 | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,7]<stderr>:libfabric:39:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,7]<stderr>:libfabric:39:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 0 qkey: 1434987231 | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] (length 32) | |
| [1,7]<stderr>:libfabric:39:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] (length 64) | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,2]<stderr>:libfabric:32:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,2]<stderr>:libfabric:32:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 1 qkey: 461479371 | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] (length 32) | |
| [1,2]<stderr>:libfabric:32:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] (length 64) | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,6]<stderr>:libfabric:36:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,6]<stderr>:libfabric:36:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 2 qkey: 560562349 | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] (length 32) | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,3]<stderr>:libfabric:33:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,3]<stderr>:libfabric:33:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 3 qkey: 346438214 | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] (length 32) | |
| [1,6]<stderr>:libfabric:36:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] (length 64) | |
| [1,3]<stderr>:libfabric:33:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] (length 64) | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,1]<stderr>:libfabric:31:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,1]<stderr>:libfabric:31:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 4 qkey: 1925818745 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] (length 32) | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,5]<stderr>:libfabric:35:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,5]<stderr>:libfabric:35:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 5 qkey: 478203478 | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] (length 32) | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,4]<stderr>:libfabric:34:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,4]<stderr>:libfabric:34:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 6 qkey: 784308060 | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] (length 32) | |
| [1,1]<stderr>:libfabric:31:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] (length 64) | |
| [1,5]<stderr>:libfabric:35:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] (length 64) | |
| [1,4]<stderr>:libfabric:34:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] (length 64) | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,0]<stderr>:libfabric:30:1668017251::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:av:util_av_init():487<info> AV size 128 | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,0]<stderr>:libfabric:30:1668017251::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 7 qkey: 769300221 | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] (length 32) | |
| [1,0]<stderr>:libfabric:30:1668017251::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] (length 64) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,8]<stderr>:libfabric:30:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 0 qkey: 518782868 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] (length 32) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,10]<stderr>:libfabric:32:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,11]<stderr>:libfabric:33:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,14]<stderr>:libfabric:36:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,15]<stderr>:libfabric:37:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 1 qkey: 803514141 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] (length 32) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,12]<stderr>:libfabric:34:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 2 qkey: 1508432900 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,13]<stderr>:libfabric:35:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] (length 32) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,9]<stderr>:libfabric:31:1668017252::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:util_av_init():487<info> AV size 128 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:cq:rxr_endpoint():2551<info> FI_DELIVERY_COMPLETE unsupported | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 3 qkey: 1810769180 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] (length 32) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 4 qkey: 655201124 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] (length 32) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 5 qkey: 1860392851 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 6 qkey: 543260881 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] (length 32) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] (length 32) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 7 qkey: 993320308 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] (length 32) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] (length 64) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] (length 64) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] (length 64) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] (length 64) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] (length 64) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] (length 64) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] (length 64) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] (length 64) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,15]<stderr>:libfabric:37:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,11]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,10]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,2]<stderr>:libfabric:32:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,14]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,13]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,12]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,7]<stderr>:libfabric:39:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,6]<stderr>:libfabric:36:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,0]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,5]<stderr>:libfabric:35:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0000_1eebff94_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 0 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0007_3b34dd74_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0001_2fe4a71d_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 2 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0002_59e8dc04_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 3 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0005_6ee35793_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 4 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0007_2dda96fd_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0006_206180d1_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0006_206180d1_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 5 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0003_6bee251c_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 6 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0004_270d9364_0000 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,9]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,8]<stderr>:libfabric:30:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0004_270d9364_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 7 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 1 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 1 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0004_72c9a979_0000 to shm provider's av. efa_fiaddr: 1 shm_fiaddr = 1 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 1 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: fi_sockaddr_in://0.0.0.0:0 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 2 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 2 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0001_1b819dcb_0000 to shm provider's av. efa_fiaddr: 2 shm_fiaddr = 2 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 2 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 3 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 3 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0003_14a63a46_0000 to shm provider's av. efa_fiaddr: 3 shm_fiaddr = 3 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 3 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 4 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 4 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0006_2ebf975c_0000 to shm provider's av. efa_fiaddr: 4 shm_fiaddr = 4 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 4 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 5 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 5 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0005_1c80ce56_0000 to shm provider's av. efa_fiaddr: 5 shm_fiaddr = 5 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 5 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,1]<stderr>:libfabric:31:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0002_216980ad_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 6 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 6 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0002_216980ad_0000 to shm provider's av. efa_fiaddr: 6 shm_fiaddr = 6 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 6 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0000_55882adf_0000 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:ofi_av_insert_addr():314<info> fi_addr: 7 | |
| [1,3]<stderr>:libfabric:33:1668017252::shm:av:smr_av_insert():93<info> fi_addr: 7 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0000_55882adf_0000 to shm provider's av. efa_fiaddr: 7 shm_fiaddr = 7 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 7 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,14]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[2] QKEY[560562349] to AV. fi_addr: 8 | |
| [1,12]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[6] QKEY[784308060] to AV. fi_addr: 8 | |
| [1,11]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[3] QKEY[346438214] to AV. fi_addr: 8 | |
| [1,13]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[5] QKEY[478203478] to AV. fi_addr: 8 | |
| [1,8]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[7] QKEY[769300221] to AV. fi_addr: 8 | |
| [1,15]<stderr>:libfabric:37:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[0] QKEY[1434987231] to AV. fi_addr: 8 | |
| [1,9]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[4] QKEY[1925818745] to AV. fi_addr: 8 | |
| [1,10]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[1] QKEY[461479371] to AV. fi_addr: 8 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 8 | |
| [1,1]<stderr>:libfabric:31:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[7] QKEY[993320308] to AV. fi_addr: 8 | |
| [1,3]<stderr>:libfabric:33:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[2] QKEY[1508432900] to AV. fi_addr: 8 | |
| [1,6]<stderr>:libfabric:36:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[3] QKEY[1810769180] to AV. fi_addr: 8 | |
| [1,5]<stderr>:libfabric:35:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[6] QKEY[543260881] to AV. fi_addr: 8 | |
| [1,4]<stderr>:libfabric:34:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 8 | |
| [1,2]<stderr>:libfabric:32:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[1] QKEY[803514141] to AV. fi_addr: 8 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[0] QKEY[518782868] to AV. fi_addr: 8 | |
| [1,7]<stderr>:libfabric:39:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[4] QKEY[655201124] to AV. fi_addr: 8 | |
| [1,0]<stdout>:# nThread 1 nGpus 1 minBytes 8 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 100 agg iters: 1 validation: 1 graph: 0 | |
| [1,0]<stdout>:# | |
| [1,0]<stdout>:# Using devices | |
| [1,0]<stdout>:# Rank 0 Pid 30 on nccl-tests-worker-0 device 0 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 1 Pid 31 on nccl-tests-worker-0 device 1 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 2 Pid 32 on nccl-tests-worker-0 device 2 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 3 Pid 33 on nccl-tests-worker-0 device 3 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 4 Pid 34 on nccl-tests-worker-0 device 4 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 5 Pid 35 on nccl-tests-worker-0 device 5 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 6 Pid 36 on nccl-tests-worker-0 device 6 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 7 Pid 39 on nccl-tests-worker-0 device 7 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 8 Pid 30 on nccl-tests-worker-1 device 0 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 9 Pid 31 on nccl-tests-worker-1 device 1 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 10 Pid 32 on nccl-tests-worker-1 device 2 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 11 Pid 33 on nccl-tests-worker-1 device 3 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 12 Pid 34 on nccl-tests-worker-1 device 4 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 13 Pid 35 on nccl-tests-worker-1 device 5 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 14 Pid 36 on nccl-tests-worker-1 device 6 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:# Rank 15 Pid 37 on nccl-tests-worker-1 device 7 [0x00] Tesla V100-SXM2-32GB | |
| [1,0]<stdout>:nccl-tests-worker-0:30:30 [0] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,0]<stdout>:nccl-tests-worker-0:30:30 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,0]<stdout>:nccl-tests-worker-0:30:30 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:ofi_av_insert_addr():314<info> fi_addr: 9 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[5] QKEY[1860392851] to AV. fi_addr: 9 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:30 [0] NCCL INFO cudaDriverVersion 11040 | |
| [1,0]<stdout>:NCCL version 2.14.3+cuda11.6 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,0]<stderr>:libfabric:30:1668017252::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,0]<stderr>:libfabric:30:1668017252::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,0]<stderr>:libfabric:30:1668017252::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,0]<stderr>:libfabric:30:1668017252::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,0]<stderr>:libfabric:30:1668017252:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,0]<stderr>:libfabric:30:1668017252::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,0]<stderr>:libfabric:30:1668017252::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,0]<stderr>:libfabric:30:1668017252::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Using network AWS Libfabric | |
| [1,8]<stdout>:nccl-tests-worker-1:30:30 [0] NCCL INFO cudaDriverVersion 11040 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:30 [0] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,8]<stdout>:nccl-tests-worker-1:30:30 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,8]<stdout>:nccl-tests-worker-1:30:30 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,8]<stderr>:libfabric:30:1668017253::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,8]<stderr>:libfabric:30:1668017253::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,8]<stderr>:libfabric:30:1668017253::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,8]<stderr>:libfabric:30:1668017253::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,8]<stderr>:libfabric:30:1668017253:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,8]<stderr>:libfabric:30:1668017253::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,8]<stderr>:libfabric:30:1668017253::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,8]<stderr>:libfabric:30:1668017253::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Using network AWS Libfabric | |
| [1,2]<stdout>:nccl-tests-worker-0:32:32 [2] NCCL INFO cudaDriverVersion 11040 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:35 [5] NCCL INFO cudaDriverVersion 11040 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:36 [6] NCCL INFO cudaDriverVersion 11040 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:34 [4] NCCL INFO cudaDriverVersion 11040 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:31 [1] NCCL INFO cudaDriverVersion 11040 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:39 [7] NCCL INFO cudaDriverVersion 11040 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:33 [3] NCCL INFO cudaDriverVersion 11040 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:35 [5] NCCL INFO cudaDriverVersion 11040 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:32 [2] NCCL INFO cudaDriverVersion 11040 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:34 [4] NCCL INFO cudaDriverVersion 11040 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:31 [1] NCCL INFO cudaDriverVersion 11040 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:36 [6] NCCL INFO cudaDriverVersion 11040 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:37 [7] NCCL INFO cudaDriverVersion 11040 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:33 [3] NCCL INFO cudaDriverVersion 11040 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:32 [2] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,2]<stdout>:nccl-tests-worker-0:32:32 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,2]<stdout>:nccl-tests-worker-0:32:32 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Using network AWS Libfabric | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,2]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,2]<stderr>:libfabric:32:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,5]<stdout>:nccl-tests-worker-0:35:35 [5] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,5]<stdout>:nccl-tests-worker-0:35:35 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,5]<stdout>:nccl-tests-worker-0:35:35 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,5]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,5]<stderr>:libfabric:35:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Using network AWS Libfabric | |
| [1,4]<stdout>:nccl-tests-worker-0:34:34 [4] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,4]<stdout>:nccl-tests-worker-0:34:34 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,4]<stdout>:nccl-tests-worker-0:34:34 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,6]<stdout>:nccl-tests-worker-0:36:36 [6] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,7]<stdout>:nccl-tests-worker-0:39:39 [7] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,1]<stdout>:nccl-tests-worker-0:31:31 [1] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,6]<stdout>:nccl-tests-worker-0:36:36 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,6]<stdout>:nccl-tests-worker-0:36:36 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,7]<stdout>:nccl-tests-worker-0:39:39 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,7]<stdout>:nccl-tests-worker-0:39:39 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,1]<stdout>:nccl-tests-worker-0:31:31 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,1]<stdout>:nccl-tests-worker-0:31:31 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Using network AWS Libfabric | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,4]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,4]<stderr>:libfabric:34:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,13]<stdout>:nccl-tests-worker-1:35:35 [5] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,13]<stdout>:nccl-tests-worker-1:35:35 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,13]<stdout>:nccl-tests-worker-1:35:35 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,6]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,6]<stderr>:libfabric:36:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO Using network AWS Libfabric | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,7]<stderr>:libfabric:39:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,7]<stderr>:libfabric:39:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO Using network AWS Libfabric | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Using network AWS Libfabric | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,1]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,1]<stderr>:libfabric:31:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Using network AWS Libfabric | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,13]<stderr>:libfabric:35:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,13]<stderr>:libfabric:35:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,10]<stdout>:nccl-tests-worker-1:32:32 [2] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,10]<stdout>:nccl-tests-worker-1:32:32 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,10]<stdout>:nccl-tests-worker-1:32:32 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Using network AWS Libfabric | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,10]<stderr>:libfabric:32:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,10]<stderr>:libfabric:32:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,9]<stdout>:nccl-tests-worker-1:31:31 [1] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,9]<stdout>:nccl-tests-worker-1:31:31 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,9]<stdout>:nccl-tests-worker-1:31:31 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,9]<stderr>:libfabric:31:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,9]<stderr>:libfabric:31:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Using network AWS Libfabric | |
| [1,3]<stdout>:nccl-tests-worker-0:33:33 [3] NCCL INFO Bootstrap : Using eth0:100.76.89.31<0> | |
| [1,3]<stdout>:nccl-tests-worker-0:33:33 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,3]<stdout>:nccl-tests-worker-0:33:33 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,15]<stdout>:nccl-tests-worker-1:37:37 [7] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,15]<stdout>:nccl-tests-worker-1:37:37 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,15]<stdout>:nccl-tests-worker-1:37:37 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,14]<stdout>:nccl-tests-worker-1:36:36 [6] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:36 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,14]<stdout>:nccl-tests-worker-1:36:36 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,3]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,3]<stderr>:libfabric:33:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO Using network AWS Libfabric | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO Using network AWS Libfabric | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,15]<stderr>:libfabric:37:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,15]<stderr>:libfabric:37:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,14]<stderr>:libfabric:36:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,14]<stderr>:libfabric:36:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO Using network AWS Libfabric | |
| [1,12]<stdout>:nccl-tests-worker-1:34:34 [4] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,12]<stdout>:nccl-tests-worker-1:34:34 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,12]<stdout>:nccl-tests-worker-1:34:34 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Using network AWS Libfabric | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,12]<stderr>:libfabric:34:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,12]<stderr>:libfabric:34:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,11]<stdout>:nccl-tests-worker-1:33:33 [3] NCCL INFO Bootstrap : Using eth0:100.76.254.204<0> | |
| [1,11]<stdout>:nccl-tests-worker-1:33:33 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. | |
| [1,11]<stdout>:nccl-tests-worker-1:33:33 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.4.0aws | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI Setting FI_EFA_FORK_SAFE environment variable to 1 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:efa_user_info_alter_rxr():332<warn> FI_HMEM capability requires RDMA, which this device does not support. | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider efa returned -61 (No data available) | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1059<info> Unsupported capabilities | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_COLLECTIVE, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:ofi_check_info():1060<info> Requested: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_REMOTE_COMM, FI_HMEM | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1037<info> Provider ofi_rxm is excluded | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_rxm:core:fi_param_get_():278<info> variable use_srx=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1055<info> Skipping util;efa layering | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI Selected Provider is efa | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO Using network AWS Libfabric | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_rxm | |
| [1,11]<stderr>:libfabric:33:1668017254:ofi_rxm:core:core:ofi_layering_ok():1048<info> Need core provider, skipping ofi_mrail | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_rxm returned -61 (No data available) | |
| [1,11]<stderr>:libfabric:33:1668017254::ofi_mrail:fabric:mrail_get_core_info():288<info> OFI_MRAIL_ADDR_STRC env variable not set! | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_getinfo_():1153<info> fi_getinfo: provider ofi_mrail returned -61 (No data available) | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO Trees [0] 12/-1/-1->15->14 [1] 14/-1/-1->15->12 [2] 14/-1/-1->15->12 [3] 12/-1/-1->15->14 [4] 12/-1/-1->15->14 [5] 14/-1/-1->15->12 [6] 14/-1/-1->15->12 [7] 12/-1/-1->15->14 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 13/-1/-1->14->15 [2] 13/-1/-1->14->15 [3] 15/-1/-1->14->13 [4] 15/-1/-1->14->13 [5] 13/-1/-1->14->15 [6] 13/-1/-1->14->15 [7] 15/-1/-1->14->13 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Trees [0] 6/-1/-1->5->1 [1] -1/-1/-1->5->6 [2] 1/-1/-1->5->6 [3] 6/13/-1->5->-1 [4] 6/-1/-1->5->1 [5] -1/-1/-1->5->6 [6] 1/-1/-1->5->6 [7] 6/-1/-1->5->13 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO Trees [0] 2/-1/-1->3->0 [1] 0/-1/-1->3->2 [2] 0/-1/-1->3->2 [3] 2/-1/-1->3->0 [4] 2/-1/-1->3->0 [5] 0/-1/-1->3->2 [6] 0/-1/-1->3->2 [7] 2/-1/-1->3->0 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Trees [0] 14/-1/-1->13->9 [1] -1/-1/-1->13->14 [2] 9/-1/-1->13->14 [3] 14/-1/-1->13->5 [4] 14/-1/-1->13->9 [5] -1/-1/-1->13->14 [6] 9/-1/-1->13->14 [7] 14/5/-1->13->-1 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Trees [0] -1/-1/-1->12->15 [1] 15/-1/-1->12->8 [2] 15/-1/-1->12->4 [3] 8/-1/-1->12->15 [4] -1/-1/-1->12->15 [5] 15/-1/-1->12->8 [6] 15/4/-1->12->-1 [7] 8/-1/-1->12->15 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 5/-1/-1->6->7 [2] 5/-1/-1->6->7 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 5/-1/-1->6->7 [6] 5/-1/-1->6->7 [7] 7/-1/-1->6->5 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Trees [0] -1/-1/-1->4->7 [1] 7/-1/-1->4->0 [2] 7/12/-1->4->-1 [3] 0/-1/-1->4->7 [4] -1/-1/-1->4->7 [5] 7/-1/-1->4->0 [6] 7/-1/-1->4->12 [7] 0/-1/-1->4->7 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Trees [0] 13/-1/-1->9->10 [1] 10/-1/-1->9->1 [2] 10/-1/-1->9->13 [3] -1/-1/-1->9->10 [4] 13/-1/-1->9->10 [5] 10/1/-1->9->-1 [6] 10/-1/-1->9->13 [7] -1/-1/-1->9->10 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO Trees [0] 4/-1/-1->7->6 [1] 6/-1/-1->7->4 [2] 6/-1/-1->7->4 [3] 4/-1/-1->7->6 [4] 4/-1/-1->7->6 [5] 6/-1/-1->7->4 [6] 6/-1/-1->7->4 [7] 4/-1/-1->7->6 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Trees [0] 9/-1/-1->10->11 [1] 11/-1/-1->10->9 [2] 11/-1/-1->10->9 [3] 9/-1/-1->10->11 [4] 9/-1/-1->10->11 [5] 11/-1/-1->10->9 [6] 11/-1/-1->10->9 [7] 9/-1/-1->10->11 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Trees [0] 5/-1/-1->1->2 [1] 2/9/-1->1->-1 [2] 2/-1/-1->1->5 [3] -1/-1/-1->1->2 [4] 5/-1/-1->1->2 [5] 2/-1/-1->1->9 [6] 2/-1/-1->1->5 [7] -1/-1/-1->1->2 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Trees [0] 11/-1/-1->8->0 [1] 12/-1/-1->8->11 [2] -1/-1/-1->8->11 [3] 11/-1/-1->8->12 [4] 11/0/-1->8->-1 [5] 12/-1/-1->8->11 [6] -1/-1/-1->8->11 [7] 11/-1/-1->8->12 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Trees [0] 1/-1/-1->2->3 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 1/-1/-1->2->3 [4] 1/-1/-1->2->3 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 1/-1/-1->2->3 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO Trees [0] 10/-1/-1->11->8 [1] 8/-1/-1->11->10 [2] 8/-1/-1->11->10 [3] 10/-1/-1->11->8 [4] 10/-1/-1->11->8 [5] 8/-1/-1->11->10 [6] 8/-1/-1->11->10 [7] 10/-1/-1->11->8 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 00/08 : 0 3 2 1 5 6 7 4 8 11 10 9 13 14 15 12 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 01/08 : 0 4 7 6 5 9 10 11 8 12 15 14 13 1 2 3 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 02/08 : 0 1 5 6 10 11 15 12 8 9 13 14 2 3 7 4 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 03/08 : 0 1 2 6 5 4 7 11 8 9 10 14 13 12 15 3 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 04/08 : 0 3 2 1 5 6 7 4 8 11 10 9 13 14 15 12 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 05/08 : 0 4 7 6 5 9 10 11 8 12 15 14 13 1 2 3 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 06/08 : 0 1 5 6 10 11 15 12 8 9 13 14 2 3 7 4 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 07/08 : 0 1 2 6 5 4 7 11 8 9 10 14 13 12 15 3 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Trees [0] 3/8/-1->0->-1 [1] 4/-1/-1->0->3 [2] -1/-1/-1->0->3 [3] 3/-1/-1->0->4 [4] 3/-1/-1->0->8 [5] 4/-1/-1->0->3 [6] -1/-1/-1->0->3 [7] 3/-1/-1->0->4 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO NCCL_BUFFSIZE set by environment to 2097152. | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Channel 01/0 : 4[1a0] -> 7[1d0] via P2P/IPC | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Channel 00/0 : 13[1b0] -> 14[1c0] via P2P/IPC | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 02/0 : 8[160] -> 9[170] via P2P/IPC | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Channel 01/0 : 12[1a0] -> 15[1d0] via P2P/IPC | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Channel 00/0 : 5[1b0] -> 6[1c0] via P2P/IPC | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 02/0 : 0[160] -> 1[170] via P2P/IPC | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Channel 03/0 : 4[1a0] -> 7[1d0] via P2P/IPC | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Channel 02/0 : 13[1b0] -> 14[1c0] via P2P/IPC | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 03/0 : 8[160] -> 9[170] via P2P/IPC | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Channel 02/0 : 5[1b0] -> 6[1c0] via P2P/IPC | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Channel 03/0 : 12[1a0] -> 15[1d0] via P2P/IPC | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 03/0 : 0[160] -> 1[170] via P2P/IPC | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Channel 05/0 : 4[1a0] -> 7[1d0] via P2P/IPC | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Channel 04/0 : 13[1b0] -> 14[1c0] via P2P/IPC | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 06/0 : 8[160] -> 9[170] via P2P/IPC | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Channel 05/0 : 12[1a0] -> 15[1d0] via P2P/IPC | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Channel 04/0 : 5[1b0] -> 6[1c0] via P2P/IPC | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 06/0 : 0[160] -> 1[170] via P2P/IPC | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Channel 07/0 : 4[1a0] -> 7[1d0] via P2P/IPC | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Channel 06/0 : 13[1b0] -> 14[1c0] via P2P/IPC | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Channel 06/0 : 5[1b0] -> 6[1c0] via P2P/IPC | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 07/0 : 8[160] -> 9[170] via P2P/IPC | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 07/0 : 0[160] -> 1[170] via P2P/IPC | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Channel 07/0 : 12[1a0] -> 15[1d0] via P2P/IPC | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO Channel 00/0 : 14[1c0] -> 15[1d0] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 01/0 : 1[170] -> 2[180] via P2P/IPC | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 01/0 : 2[180] -> 3[190] via P2P/IPC | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO Channel 00/0 : 6[1c0] -> 7[1d0] via P2P/IPC | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 01/0 : 9[170] -> 10[180] via P2P/IPC | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 01/0 : 10[180] -> 11[190] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 03/0 : 1[170] -> 2[180] via P2P/IPC | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO Channel 04/0 : 6[1c0] -> 7[1d0] via P2P/IPC | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 02/0 : 2[180] -> 3[190] via P2P/IPC | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO Channel 04/0 : 14[1c0] -> 15[1d0] via P2P/IPC | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 03/0 : 9[170] -> 10[180] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 05/0 : 1[170] -> 2[180] via P2P/IPC | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 05/0 : 2[180] -> 3[190] via P2P/IPC | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 02/0 : 10[180] -> 11[190] via P2P/IPC | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 05/0 : 9[170] -> 10[180] via P2P/IPC | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 05/0 : 10[180] -> 11[190] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 07/0 : 1[170] -> 2[180] via P2P/IPC | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 06/0 : 2[180] -> 3[190] via P2P/IPC | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 07/0 : 9[170] -> 10[180] via P2P/IPC | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 06/0 : 10[180] -> 11[190] via P2P/IPC | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 00/0 : 0[160] -> 3[190] via P2P/IPC | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 00/0 : 8[160] -> 11[190] via P2P/IPC | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 04/0 : 0[160] -> 3[190] via P2P/IPC | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 04/0 : 8[160] -> 11[190] via P2P/IPC | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO Channel 02/0 : 14[1c0] -> 2[180] [send] via NET/AWS Libfabric/2 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Channel 01/0 : 13[1b0] -> 1[170] [send] via NET/AWS Libfabric/1 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:87 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:89 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:82 [6] NCCL INFO Channel 06/0 : 14[1c0] -> 2[180] [send] via NET/AWS Libfabric/2 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:78 [5] NCCL INFO Channel 05/0 : 13[1b0] -> 1[170] [send] via NET/AWS Libfabric/1 | |
| [1,14]<stdout>:nccl-tests-worker-1:36:87 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,13]<stdout>:nccl-tests-worker-1:35:89 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Channel 01/0 : 5[1b0] -> 9[170] [send] via NET/AWS Libfabric/1 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO Channel 02/0 : 6[1c0] -> 10[180] [send] via NET/AWS Libfabric/2 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:92 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:86 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:82 [6] NCCL INFO Channel 06/0 : 6[1c0] -> 10[180] [send] via NET/AWS Libfabric/2 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:80 [5] NCCL INFO Channel 05/0 : 5[1b0] -> 9[170] [send] via NET/AWS Libfabric/1 | |
| [1,6]<stdout>:nccl-tests-worker-0:36:92 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,5]<stdout>:nccl-tests-worker-0:35:86 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:88 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:85 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:86 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:88 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:89 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,2]<stdout>:nccl-tests-worker-0:32:90 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,0]<stderr>:libfabric:30:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,0]<stderr>:libfabric:30:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:91 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,11]<stderr>:libfabric:33:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,9]<stderr>:libfabric:31:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,10]<stderr>:libfabric:32:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,2]<stderr>:libfabric:32:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,3]<stderr>:libfabric:33:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,1]<stderr>:libfabric:31:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 8 qkey: 1069616970 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 9 qkey: 1537236331 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] (length 32) | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] (length 32) | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,0]<stderr>:libfabric:30:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 8 qkey: 1327339321 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 10 qkey: 1201214331 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 9 qkey: 1334957947 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] (length 32) | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] (length 32) | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] (length 32) | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 10 qkey: 788326366 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] (length 32) | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 11 qkey: 44184044 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] (length 32) | |
| [1,8]<stderr>:libfabric:30:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,8]<stderr>:libfabric:30:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:92 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO Channel 03/0 : 7[1d0] -> 11[190] [send] via NET/AWS Libfabric/3 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Channel 00/0 : 4[1a0] -> 8[160] [send] via NET/AWS Libfabric/0 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:87 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,4]<stdout>:nccl-tests-worker-0:34:93 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:83 [7] NCCL INFO Channel 07/0 : 7[1d0] -> 11[190] [send] via NET/AWS Libfabric/3 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:81 [4] NCCL INFO Channel 04/0 : 4[1a0] -> 8[160] [send] via NET/AWS Libfabric/0 | |
| [1,4]<stdout>:nccl-tests-worker-0:34:93 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,7]<stdout>:nccl-tests-worker-0:39:87 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,8]<stderr>:libfabric:30:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 11 qkey: 324471052 | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] (length 32) | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] (length 56) | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] (length 56) | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO Channel 03/0 : 15[1d0] -> 3[190] [send] via NET/AWS Libfabric/3 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Channel 00/0 : 12[1a0] -> 0[160] [send] via NET/AWS Libfabric/0 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:90 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] (length 56) | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] (length 56) | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] (length 56) | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] to AV .... | |
| [1,12]<stdout>:nccl-tests-worker-1:34:91 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,15]<stdout>:nccl-tests-worker-1:37:81 [7] NCCL INFO Channel 07/0 : 15[1d0] -> 3[190] [send] via NET/AWS Libfabric/3 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,15]<stdout>:nccl-tests-worker-1:37:90 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ | |
| [1,12]<stdout>:nccl-tests-worker-1:34:83 [4] NCCL INFO Channel 04/0 : 12[1a0] -> 0[160] [send] via NET/AWS Libfabric/0 | |
| [1,12]<stdout>:nccl-tests-worker-1:34:91 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] (length 56) | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] (length 56) | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] (length 56) | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0008_3fc10f4a_0000 | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,9]<stderr>:: (null) | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0008_3fc10f4a_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_0009_5ba05d6b_0000 | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,11]<stderr>:: (null) | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,11]<stderr>:libfabric:33:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_0009_5ba05d6b_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] to AV. fi_addr: 0 | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] to AV. fi_addr: 0 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO Channel 03/0 : 7[1d0] -> 11[190] [receive] via NET/AWS Libfabric/3 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 01/0 : 5[1b0] -> 9[170] [receive] via NET/AWS Libfabric/1 | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] (length 56) | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] to AV .... | |
| [1,11]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:88 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO Channel 07/0 : 7[1d0] -> 11[190] [receive] via NET/AWS Libfabric/3 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:85 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO Channel 02/0 : 11[190] -> 15[1d0] via P2P/IPC | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 05/0 : 5[1b0] -> 9[170] [receive] via NET/AWS Libfabric/1 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 00/0 : 9[170] -> 13[1b0] via P2P/IPC | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] (length 56) | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] to AV .... | |
| [1,9]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0009_4f91d77b_0000 | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,3]<stderr>:: (null) | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0009_4f91d77b_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] to AV. fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_0008_4f1d9739_0000 | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,2]<stderr>:: (null) | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_0008_4f1d9739_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] to AV. fi_addr: 0 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO Channel 03/0 : 15[1d0] -> 3[190] [receive] via NET/AWS Libfabric/3 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_000a_2efce7de_0000 | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,1]<stderr>:: (null) | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,1]<stderr>:libfabric:31:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 02/0 : 14[1c0] -> 2[180] [receive] via NET/AWS Libfabric/2 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_000a_2efce7de_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] to AV. fi_addr: 0 | |
| [1,11]<stdout>:nccl-tests-worker-1:33:84 [3] NCCL INFO Channel 06/0 : 11[190] -> 15[1d0] via P2P/IPC | |
| [1,3]<stdout>:nccl-tests-worker-0:33:88 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 01/0 : 13[1b0] -> 1[170] [receive] via NET/AWS Libfabric/1 | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] (length 56) | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] to AV .... | |
| [1,3]<stderr>:libfabric:33:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO Channel 07/0 : 15[1d0] -> 3[190] [receive] via NET/AWS Libfabric/3 | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO Channel 02/0 : 3[190] -> 7[1d0] via P2P/IPC | |
| [1,2]<stdout>:nccl-tests-worker-0:32:90 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] (length 56) | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] to AV .... | |
| [1,2]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 02/0 : 9[170] -> 13[1b0] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:89 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 06/0 : 14[1c0] -> 2[180] [receive] via NET/AWS Libfabric/2 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 05/0 : 13[1b0] -> 1[170] [receive] via NET/AWS Libfabric/1 | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 00/0 : 1[170] -> 5[1b0] via P2P/IPC | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] (length 56) | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] to AV .... | |
| [1,1]<stderr>:libfabric:31:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 03/0 : 2[180] -> 6[1c0] via P2P/IPC | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 04/0 : 9[170] -> 13[1b0] via P2P/IPC | |
| [1,3]<stdout>:nccl-tests-worker-0:33:85 [3] NCCL INFO Channel 06/0 : 3[190] -> 7[1d0] via P2P/IPC | |
| [1,2]<stdout>:nccl-tests-worker-0:32:79 [2] NCCL INFO Channel 07/0 : 2[180] -> 6[1c0] via P2P/IPC | |
| [1,9]<stdout>:nccl-tests-worker-1:31:80 [1] NCCL INFO Channel 06/0 : 9[170] -> 13[1b0] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 02/0 : 1[170] -> 5[1b0] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 04/0 : 1[170] -> 5[1b0] via P2P/IPC | |
| [1,1]<stdout>:nccl-tests-worker-0:31:84 [1] NCCL INFO Channel 06/0 : 1[170] -> 5[1b0] via P2P/IPC | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_000b_1357090c_0000 | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,8]<stderr>:: (null) | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,8]<stderr>:libfabric:30:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_000b_1357090c_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 00/0 : 4[1a0] -> 8[160] [receive] via NET/AWS Libfabric/0 | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] to AV. fi_addr: 0 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:92 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] (length 56) | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] to AV .... | |
| [1,8]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 04/0 : 4[1a0] -> 8[160] [receive] via NET/AWS Libfabric/0 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 01/0 : 8[160] -> 12[1a0] via P2P/IPC | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,7]<stderr>:libfabric:39:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,7]<stderr>:libfabric:39:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,15]<stderr>:libfabric:37:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,15]<stderr>:libfabric:37:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,8]<stdout>:nccl-tests-worker-1:30:77 [0] NCCL INFO Channel 05/0 : 8[160] -> 12[1a0] via P2P/IPC | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 12 qkey: 836282432 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[12] QKEY[836282432] (length 32) | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 12 qkey: 1867249410 | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[12] QKEY[1867249410] (length 32) | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,13]<stderr>:libfabric:35:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,13]<stderr>:libfabric:35:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,5]<stderr>:libfabric:35:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,5]<stderr>:libfabric:35:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 13 qkey: 2081844920 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[13] QKEY[2081844920] (length 32) | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 13 qkey: 836769862 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[13] QKEY[836769862] (length 32) | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] to AV .... | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,7]<stderr>:: (null) | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] to AV .... | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,15]<stderr>:: (null) | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] to AV .... | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,13]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:av:smr_av_insert():74<info> fe80::5d:cff:fe3e:7857_000b_02a231ec_0000 | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,0]<stderr>:: (null) | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,0]<stderr>:libfabric:30:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::5d:cff:fe3e:7857_000b_02a231ec_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] to AV. fi_addr: 0 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 00/0 : 12[1a0] -> 0[160] [receive] via NET/AWS Libfabric/0 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:91 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] (length 56) | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] to AV .... | |
| [1,0]<stderr>:libfabric:30:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 04/0 : 12[1a0] -> 0[160] [receive] via NET/AWS Libfabric/0 | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 01/0 : 0[160] -> 4[1a0] via P2P/IPC | |
| [1,0]<stdout>:nccl-tests-worker-0:30:78 [0] NCCL INFO Channel 05/0 : 0[160] -> 4[1a0] via P2P/IPC | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] to AV .... | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,5]<stderr>:: (null) | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:efa_is_local_peer():64<info> The peer is local. | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:av:smr_av_insert():74<info> fe80::87:8bff:fed6:bde3_000a_4799137b_0000 | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,10]<stderr>:: (null) | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,10]<stderr>:libfabric:32:1668017254::shm:av:smr_av_insert():93<info> fi_addr: 0 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:efa_conn_rdm_init():426<info> Successfully inserted fe80::87:8bff:fed6:bde3_000a_4799137b_0000 to shm provider's av. efa_fiaddr: 0 shm_fiaddr = 0 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] to AV. fi_addr: 0 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 02/0 : 6[1c0] -> 10[180] [receive] via NET/AWS Libfabric/2 | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:86 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] (length 56) | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] to AV .... | |
| [1,10]<stderr>:libfabric:32:1668017254::efa:av:efa_av_insert_one():714<info> Found existing AV entry pointing to this address! fi_addr: 0 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 06/0 : 6[1c0] -> 10[180] [receive] via NET/AWS Libfabric/2 | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 03/0 : 10[180] -> 14[1c0] via P2P/IPC | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,12]<stderr>:libfabric:34:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,12]<stderr>:libfabric:34:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,4]<stderr>:libfabric:34:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,4]<stderr>:libfabric:34:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 14 qkey: 1394792767 | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[14] QKEY[1394792767] (length 32) | |
| [1,10]<stdout>:nccl-tests-worker-1:32:79 [2] NCCL INFO Channel 07/0 : 10[180] -> 14[1c0] via P2P/IPC | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 14 qkey: 735876419 | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[14] QKEY[735876419] (length 32) | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: shm | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_fabric_():1352<info> Opened fabric: efa | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:domain:efa_hmem_support_status_update_cuda():73<warn> Failed to register CUDA buffer with the EFA device, FI_HMEM transfers that require peer to peer support will fail. | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:domain:efa_hmem_support_status_update_neuron():111<info> FI_HMEM_NEURON is not initialized | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_ep_type():666<info> unsupported endpoint type | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_ep_type():667<info> Supported: FI_EP_DGRAM | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:core:ofi_check_ep_type():667<info> Requested: FI_EP_RDM | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_rx_attr():804<info> Tx only caps ignored in Rx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():902<info> Rx only caps ignored in Tx caps | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():939<info> inject_size too large | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Supported: 0 | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:core:ofi_check_tx_attr():940<info> Requested: 4096 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:rxr_endpoint():2578<info> rxr_ep->use_zcpy_rx = 0 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:rxr_endpoint():2581<info> rxr_ep->handle_resource_management = 2 | |
| [1,6]<stderr>:libfabric:36:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,6]<stderr>:libfabric:36:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,14]<stderr>:libfabric:36:1668017254::core:core:fi_param_get_():278<info> variable universe_size=<not set> | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:av:util_av_init():487<info> AV size 16384 | |
| [1,14]<stderr>:libfabric:36:1668017254::shm:av:util_av_init():487<info> AV size 128 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:av:efa_av_open():1052<info> fi_av_attr:0 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 15 qkey: 857512227 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[15] QKEY[857512227] (length 32) | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:efa_ep_create_qp_ex():164<info> efa_ep_create_qp_ex(): create QP 15 qkey: 1076113575 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[15] QKEY[1076113575] (length 32) | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] to AV .... | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,4]<stderr>:: (null) | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] to AV .... | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,12]<stderr>:: (null) | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] to AV .... | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,6]<stderr>:: (null) | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:av:efa_av_insert_one():704<info> Inserting address GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] to AV .... | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:av:ofi_av_insert_addr():291<info> inserting addr | |
| [1,14]<stderr>:: (null) | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[9] QKEY[1537236331] to AV. fi_addr: 0 | |
| [1,7]<stderr>:libfabric:39:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[12] QKEY[836282432] (length 56) | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[10] QKEY[788326366] to AV. fi_addr: 0 | |
| [1,13]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[13] QKEY[2081844920] (length 56) | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[9] QKEY[1334957947] to AV. fi_addr: 0 | |
| [1,15]<stderr>:libfabric:37:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[12] QKEY[1867249410] (length 56) | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[11] QKEY[324471052] to AV. fi_addr: 0 | |
| [1,4]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[14] QKEY[735876419] (length 56) | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[8] QKEY[1069616970] to AV. fi_addr: 0 | |
| [1,5]<stderr>:libfabric:35:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[13] QKEY[836769862] (length 56) | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::87:8bff:fed6:bde3] QP[10] QKEY[1201214331] to AV. fi_addr: 0 | |
| [1,6]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::5d:cff:fe3e:7857] QP[15] QKEY[857512227] (length 56) | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[8] QKEY[1327339321] to AV. fi_addr: 0 | |
| [1,14]<stderr>:libfabric:36:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[15] QKEY[1076113575] (length 56) | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:av:ofi_av_insert_addr():314<info> fi_addr: 0 | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:av:efa_av_insert_one():727<info> Successfully inserted address GID[fe80::5d:cff:fe3e:7857] QP[11] QKEY[44184044] to AV. fi_addr: 0 | |
| [1,12]<stderr>:libfabric:34:1668017254::efa:ep_ctrl:efa_ep_getname():62<info> EP addr: GID[fe80::87:8bff:fed6:bde3] QP[14] QKEY[1394792767] (length 56) | |
| [1,7]<stderr>:libfabric:39:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) | |
| [1,7]<stdout>: | |
| [1,7]<stdout>:nccl-tests-worker-0:39:87 [7] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7fc4fc470118 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 3, size: 0, state: CREATED, direction: SEND } | |
| [1,13]<stdout>: | |
| [1,13]<stdout>:nccl-tests-worker-1:35:89 [5] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7fbb944701b8 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 1, size: 0, state: CREATED, direction: SEND } | |
| [1,13]<stderr>:libfabric:35:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) | |
| [1,4]<stderr>:libfabric:34:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) | |
| [1,5]<stderr>:libfabric:35:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) | |
| [1,4]<stdout>: | |
| [1,4]<stdout>:nccl-tests-worker-0:34:93 [4] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7f9b3446ec38 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 0, size: 0, state: CREATED, direction: SEND } | |
| [1,5]<stdout>: | |
| [1,5]<stdout>:nccl-tests-worker-0:35:86 [5] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7f1c904701b8 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 1, size: 0, state: CREATED, direction: SEND } | |
| [1,6]<stderr>:libfabric:36:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) | |
| [1,6]<stdout>: | |
| [1,6]<stdout>:nccl-tests-worker-0:36:92 [6] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7fb3a046ff38 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 2, size: 0, state: CREATED, direction: SEND } | |
| [1,14]<stdout>: | |
| [1,14]<stdout>:nccl-tests-worker-1:36:87 [6] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7f865c46ff38 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 2, size: 0, state: CREATED, direction: SEND } | |
| [1,14]<stderr>:libfabric:36:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) | |
| [1,12]<stdout>: | |
| [1,12]<stdout>:nccl-tests-worker-1:34:91 [4] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7f0cc846ec38 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 0, size: 0, state: CREATED, direction: SEND } | |
| [1,15]<stderr>:libfabric:37:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) | |
| [1,15]<stdout>: | |
| [1,15]<stdout>:nccl-tests-worker-1:37:90 [7] ofi_process_cq:1039 NCCL WARN NET/OFI Request 0x7fbdbc470118 completed with error. RC: 20. Error: unknown error. Completed length: 0, Request: { buffer_index: 255, dev: 3, size: 0, state: CREATED, direction: SEND } | |
| [1,12]<stderr>:libfabric:34:1668017255::efa:cq:rxr_cq_write_tx_error():243<warn> rxr_cq_write_tx_error: err: 20, prov_err: Not a directory (20) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment