Commands and config file needed to run daos_test to validate installation using psm2 provider.

daos_test_wolf_psm2

Build from commit 2e4fcb939e83f8d65b1cc1667f2a195f8cf1131c

Verify provider is available:

daos_m/install/bin/fi_info | grep psm
(should show some output, if not then rebuild daos_m/_build.external/ofi)

need to run client and server on different hosts to be able to reconnect without restarting server (required in daos_test)

CONFIG:

--- a/utils/config/examples/daos_server_psm2.yml
+++ b/utils/config/examples/daos_server_psm2.yml
@@ -3,7 +3,7 @@
 name: daos_server   # map to -g daos_server
 port: 10001         # mgmt port
 provider: ofi+psm2  # map to CRT_PHY_ADDR_STR=ofi+psm2
-socket_dir: /tmp/daos_psm2
+#socket_dir: /tmp/daos_psm2
 nr_hugepages: 4096
 control_log_mask: DEBUG
 control_log_file: /tmp/daos_control.log
@@ -18,7 +18,7 @@ control_log_file: /tmp/daos_control.log
 ## Transport Credentials Specifying certificates to secure communications
 ##
 #transport_config:
-#  # Specify to bypass loading certificates and use insecure communications channnels
+#  # Specify to bypass loading certificates and use insecure communications channnelsnd 20
 #  allow_insecure: false
 #  # Location where daos_server will look for Client certificates
 #  client_cert_dir: .daos/clients
@@ -32,44 +32,46 @@ control_log_file: /tmp/daos_control.log
 # single server instance per config file for now
 servers:
 -
-  targets: 8                # count of storage targets per each server
+  targets: 1                # count of storage targets per each server
   first_core: 0             # offset of the first core for service xstreams
-  nr_xs_helpers: 2          # count of offload/helper xstreams per target
+  nr_xs_helpers: 1          # count of offload/helper xstreams per target
   fabric_iface: ib0         # map to OFI_INTERFACE=ib0
   fabric_iface_port: 31416  # map to OFI_PORT=31416
-  log_mask: ERR             # map to D_LOG_MASK=ERR
+  log_mask: DEBUG             # map to D_LOG_MASK=ERR
   log_file: /tmp/server.log # map to D_LOG_FILE=/tmp/server.log

   # Environment variable values should be supplied without encapsulating quotes.
   env_vars:                 # influence DAOS IO Server behaviour by setting env variables
-  - DAOS_MD_CAP=1024
-  - CRT_CTX_SHARE_ADDR=0
-  - CRT_TIMEOUT=30
+  - DD_MASK=all
+  - CRT_TIMEOUT=3600
   - CRT_CREDIT_EP_CTX=0
   # uncomment to enable scalable endpoint
-  # - PSM2_MULTI_EP=1
-  # - CRT_CTX_SHARE_ADDR=1
-  # - CRT_CTX_NUM=8
+  - PSM2_MULTI_EP=1
+  - CRT_CTX_SHARE_ADDR=1
+  - CRT_CTX_NUM=8
   # nvme options
-  # - IO_STAT_PERIOD=10
+  #- IO_STAT_PERIOD=20
+  - DAOS_IO_MODE=1
+  - PMEMOBJ_CONF=prefault.at_open=1;prefault.at_create=1;
+  - PMEM_IS_PMEM_FORCE=1

   # Storage definitions

   # When scm_class is set to ram, tmpfs will be used to emulate SCM.
   # The size of ram is specified by scm_size in GB units.
   scm_mount: /mnt/daos # map to -s /mnt/daos
-  scm_class: ram
-  scm_size: 6
+  #scm_class: ram
+  #scm_size: 6

   # When scm_class is set to dcpm, scm_list is the list of device paths for
   # AppDirect pmem namespaces (currently only one per server supported).
-  # scm_class: dcpm
-  # scm_list: [/dev/pmem0]
+  scm_class: dcpm
+  scm_list: [/dev/pmem1]

   # If using NVMe SSD (will write /mnt/daos/daos_nvme.conf and start I/O
   # service with -n <path>)
   bdev_class: nvme
-  bdev_list: ["0000:81:00.0"]  # generate regular nvme.conf
+  bdev_list: ["0000:87:00.0"]  # generate regular nvme.conf

   # If emulating NVMe SSD with malloc devices
   # bdev_class: malloc # map to VOS_BDEV_CLASS=MALLOC

SERVER:

orterun -np 1 -H wolf-71 --allow-run-as-root --report-uri /tmp/urifile daos_server -o /home/tanabarr/projects/daos_m/utils/config/examples/daos_server_psm2.yml -i

CLIENT:

daos_agent -i &
# test we can create a pool
orterun -np 1 --ompi-server file:/tmp/urifile --allow-run-as-root -x OFI_INTERFACE=ib0 -x CRT_PHY_ADDR_STR=ofi+psm2 dmg create --group=daos_server --size=8G --svcn=1
# run the tests
orterun -np 1 --ompi-server file:/tmp/urifile --allow-run-as-root --mca mtl ^psm2,ofi -x FI_PSM2_DISCONNECT=1 -x OFI_INTERFACE=ib0 -x CRT_PHY_ADDR_STR=ofi+psm2 daos_test -p