client.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <rdma/fabric.h>
#include <rdma/fi_eq.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_errno.h>


#define ERROR(x, ...) \
do { \
  fprintf(stderr, "error: "x"\n", __VA_ARGS__);\
  exit(EXIT_FAILURE);\
} while(0)


void client(struct fi_info* fi)
{
  struct fid_fabric *fabric;
  struct fi_eq_attr eq_attr = {
    .wait_obj = FI_WAIT_UNSPEC // waiting only through fi_ calls (no epoll)
  };
  struct fid_eq *eq;
  struct fid_domain *domain;


  struct fi_eq_cm_entry entry;
  uint32_t event;
  ssize_t rd;
  int ret;
  struct fid_ep *ep;


  if(ret = fi_fabric(fi->fabric_attr, &fabric, NULL)) {
    ERROR("fi_fabric failed: %d", ret);
  }

  if(ret = fi_eq_open(fabric, &eq_attr, &eq, NULL)) {
    ERROR("fi_eq_open failed: %d", ret);
  }

  if(ret = fi_domain(fabric, fi, &domain, NULL)) {
    ERROR("fi_domain failed: %d", ret);
  }

  if(ret = fi_endpoint(domain, fi, &ep, NULL)) {
    ERROR("fi_endpoint failed: %d", ret);
  }

  if(ret = fi_ep_bind((ep), &eq->fid, 0)) {
    ERROR("fi_ep_bind failed: %d", ret);
  }

  /* Connect to server */
  if(ret = fi_connect(ep, fi->dest_addr, NULL, 0)) {
    ERROR("fi_connect failed: %d '%s'", ret, strerror(-ret));
  }

   /* Wait for the connection to be established */
  rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
  if(rd == -FI_EAVAIL)
  {
    struct fi_eq_err_entry err_entry;
    if((ret = fi_eq_readerr(eq, &err_entry, 0)) < 0)
    {
      ERROR("fi_eq_readerr failed: %d", ret);
    }

    char buf[1024];
    fprintf(stderr, "error event: %s\n", fi_eq_strerror(eq, err_entry.prov_errno, 
      err_entry.err_data, buf, 1024));
    exit(EXIT_FAILURE);
  }
  else if(rd < 0)
  {
    ERROR("fi_eq_sread failed: %d '%s'", rd, fi_strerror(-rd));
  }
  if (rd != sizeof entry) {
    ERROR("reading from event queue failed (after connect): rd=%d", rd);
  }
  printf("received event\n");

  if (event != FI_CONNECTED || entry.fid != &ep->fid) {
    ERROR("Unexpected CM event %d fid %p (ep %p)", event, entry.fid, ep);
  }

  printf("connected\n");
}


int main()
{

  printf("Getting info\n");
  struct fi_info* fi;
  struct fi_info* hints;

  hints = fi_allocinfo();
  hints->ep_attr->type  = FI_EP_MSG;
  hints->caps = FI_MSG;
  hints->mode   = FI_LOCAL_MR;

  if(0 != fi_getinfo(FI_VERSION(1, 1), "127.0.0.1", "12345", 0, hints, &fi))
    {
      printf("error: fi_getinfo\n");
      return 1;
    }

    printf("CAPS = %d\n", fi->caps);

    client(fi);

  return 0;
}

server.c

#include <stdio.h>
#include <stdlib.h>
#include <rdma/fabric.h>
#include <rdma/fi_eq.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_cm.h>

#define ERROR(x, ...) \
do { \
  fprintf(stderr, "error: "x"\n", __VA_ARGS__);\
  exit(EXIT_FAILURE);\
} while(0)

void server(struct fi_info* fi)
{
  int ret;
  struct fid_fabric *fabric;
  struct fi_eq_attr eq_attr = {
    .wait_obj = FI_WAIT_UNSPEC // waiting only through fi_ calls (no epoll)
  };
  struct fid_eq *eq;
  struct fid_pep *pep;
  struct fid_domain *domain;
  struct fi_eq_cm_entry entry;
  uint32_t event;
  struct fi_info *info = NULL;
  ssize_t rd;
  struct fid_ep *ep;


  // from ft_start_server
  if(ret = fi_fabric(fi->fabric_attr, &fabric, NULL)) {
    ERROR("fi_fabric failed: %d", ret);
  }

  if(ret = fi_eq_open(fabric, &eq_attr, &eq, NULL)) {
    ERROR("fi_eq_open failed: %d", ret);
  }

  if(ret = fi_passive_ep(fabric, fi, &pep, NULL)) {
    ERROR("fi_passive_ep failed: %d", ret);
  }

  if(ret = fi_pep_bind(pep, &eq->fid, 0)) {
    ERROR("fi_pep_bind failed: %d", ret);
  }
  
  if(ret = fi_listen(pep)) {
    ERROR("fi_listen failed: %d '%s'", ret, fi_strerror(-ret));
  }  

  // server_connect (from fabtests/msg.c example)

  /* Wait for connection request from client */
  rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
  if(rd < 0)
  {
    ERROR("fi_eq_sread failed: %d '%s'", rd, fi_strerror(-rd));
  }
  if (rd != sizeof entry) {
    ERROR("reading from event queue failed (after listen): rd=%d", rd);
  }
  printf("received event\n");

  info = entry.info;
  if (event != FI_CONNREQ) {
    ERROR("unexpected event: %d", event);
  }  

  if(ret = fi_domain(fabric, info, &domain, NULL)) {
    ERROR("fi_domain failed: %d", ret);
  }

  if(ret = fi_endpoint(domain, info, &ep, NULL)) {
    ERROR("fi_endpoint failed: %d", ret);
  }

  if(ret = fi_ep_bind((ep), &eq->fid, 0)) {
    ERROR("fi_ep_bind failed: %d", ret);
  }

  if(ret = fi_accept(ep, NULL, 0)) {
    ERROR("fi_accept failed: %d", ret);
  }

  /* Wait for the connection to be established */
  rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
  if(rd < 0)
  {
    ERROR("fi_eq_sread failed: %d '%s'", rd, fi_strerror(-rd));
  }
  if (rd != sizeof entry) {
    ERROR("reading from event queue failed (after listen): rd=%d", rd);
  }

  if (event != FI_CONNECTED || entry.fid != &ep->fid) {
    ERROR("Unexpected CM event %d fid %p (ep %p)", event, entry.fid, ep);
  }

  printf("connected\n");

  fi_freeinfo(info);
}


int main()
{

  printf("Getting info\n");
  struct fi_info* fi;
  struct fi_info* hints;

  hints = fi_allocinfo();
  hints->ep_attr->type  = FI_EP_MSG;
  hints->caps = FI_MSG;
  //hints->mode   = FI_LOCAL_MR;

  if(0 != fi_getinfo(FI_VERSION(1, 1), "127.0.0.1", "12345", FI_SOURCE, hints, &fi))
    {
      printf("error: fi_getinfo\n");
      return 1;
    }

    printf("CAPS = %d\n", fi->caps);

    server(fi);

  return 0;
}