weird behavior observed with tx/rx sizes

tx_behavior.c

#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <getopt.h>
#include <assert.h>
#include <stdbool.h>

#include <rdma/fabric.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_errno.h>
#include <rdma/fi_endpoint.h>

struct fid_domain *domain;
struct fid_ep *ep;
struct fid_cq *txcq, *rxcq;
struct fid_fabric *fabric;
struct fi_info *hints;
struct fi_cq_attr cq_attr;
int ret;


#define DIE(...)                                                               \
	do {                                                                   \
		log_(__FILE__, __func__, __LINE__, __VA_ARGS__);               \
		exit(EXIT_FAILURE);                                            \
	} while (0)

#define INFO(...) log_(__FILE__, __func__, __LINE__, __VA_ARGS__)

void log_(const char *file, const char *func, int line, const char *fmt, ...)
{
	char buf[8192] = {0};
	va_list vargs;
	int size = 0;

	size = snprintf(buf, sizeof(buf), "[%s]:%s():<%d> ", file, func, line);

	va_start(vargs, fmt);
	vsnprintf(buf + size, sizeof(buf) - size, fmt, vargs);
	va_end(vargs);

	fprintf(stderr, "%s", buf);
}

void check_size(int before, int after_getinfo, bool change)
{
	struct fi_info *fi;

	fprintf(stderr, "############\n");

	INFO("providing tx size hint: %d\n", before);
	hints->tx_attr->size = before;

	ret = fi_getinfo(FI_VERSION(1, 3), NULL, 0, 0, hints, &fi);
	if (ret == -FI_ENODATA) {
		INFO("provider rejected hint.\n");
		return;
	}

	INFO("provider returned: %d\n", fi->tx_attr->size);

	if (change) {
		INFO("changing size to %d after getinfo\n", after_getinfo);
		fi->tx_attr->size = after_getinfo;
	}

	ret = fi_fabric(fi->fabric_attr, &fabric, NULL);
	if (ret)
		DIE("fi_fabric\n");

	ret = fi_domain(fabric, fi, &domain, NULL);
	if (ret)
		DIE("fi_domain\n");

	ret = fi_cq_open(domain, &cq_attr, &txcq, &txcq);
	if (ret)
		DIE("fi_cq_open\n");

	ret = fi_cq_open(domain, &cq_attr, &rxcq, &rxcq);
	if (ret)
		DIE("fi_cq_open");

	ret = fi_endpoint(domain, fi, &ep, NULL);
	if (ret == -FI_EINVAL) {
		INFO("fi_endpoint rejected size.\n");
		goto fail;
	}

	ret = fi_ep_bind(ep, &txcq->fid, FI_TRANSMIT);
	if (ret)
		DIE("fi_ep_bind");

	ret = fi_ep_bind(ep, &rxcq->fid, FI_RECV);
	if (ret)
		DIE("fi_ep_bind");

	ret = fi_enable(ep);
	if (ret)
		DIE("fi_enable");

	ret = fi_tx_size_left(ep);
	INFO("tx size left reported: %d\n", ret);

	fi_close(&ep->fid);

fail:
	fi_close(&rxcq->fid);
	fi_close(&txcq->fid);
	fi_close(&domain->fid);
	fi_close(&fabric->fid);

	fi_freeinfo(fi);

	fprintf(stderr, "############\n");
}

int main(void)
{
	hints = fi_allocinfo();
	if (!hints)
		DIE("fi_allocinfo\n");

	hints->fabric_attr->prov_name = strdup("sockets");
	hints->mode = ~0;
	hints->caps = FI_MSG;

	cq_attr.format = FI_CQ_FORMAT_CONTEXT;
	cq_attr.wait_obj = FI_WAIT_NONE;

	check_size(0, 0, false);
	check_size(300, 0, false);
	check_size(288, 0, false);
	check_size(0, 300, true);
	check_size(0, 0, true);

	return 0;
}

tx_behavior_output

############
[src/tx_behavior.c]:check_size():<52> providing tx size hint: 0
[src/tx_behavior.c]:check_size():<61> provider returned: 256
[src/tx_behavior.c]:check_size():<103> tx size left reported: 368
############
############
[src/tx_behavior.c]:check_size():<52> providing tx size hint: 300
[src/tx_behavior.c]:check_size():<57> provider rejected hint.
############
[src/tx_behavior.c]:check_size():<52> providing tx size hint: 288
[src/tx_behavior.c]:check_size():<57> provider rejected hint.
############
[src/tx_behavior.c]:check_size():<52> providing tx size hint: 0
[src/tx_behavior.c]:check_size():<61> provider returned: 256
[src/tx_behavior.c]:check_size():<64> changing size to 300 after getinfo
[src/tx_behavior.c]:check_size():<86> fi_endpoint rejected size.
############
############
[src/tx_behavior.c]:check_size():<52> providing tx size hint: 0
[src/tx_behavior.c]:check_size():<61> provider returned: 256
[src/tx_behavior.c]:check_size():<64> changing size to 0 after getinfo
[src/tx_behavior.c]:check_size():<103> tx size left reported: 188508
############