Unit test for Open MPI Issue #2256:https://github.com/open-mpi/ompi/issues/2256

test-nbc-dt.c

/**
 * Copyright (c) 2016      IBM Corporation.  All rights reserved.
 *
 * MPI_Ibcast, MPI_Iallgather, MPI_Iallgatherv non-uniform types test case
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <mpi.h>

int test_ibcast(void);
int test_iallgather(void);
int test_iallgatherv(void);
int chkbuf(long long *buf, long count, int dt, int pattern, int from);
void fillbuf(long long *buf, long count, int dt, int pattern);

static MPI_Datatype i8space, i8i8;
#define i8 MPI_LONG_LONG
#define DTEXTENT(a) (((a)==0)?(8):(16))
#define MPIDT(a)    (((a)==0)?(i8):(((a)==1)?(i8space):(i8i8)))
#define DTBYTES(a)  (((a)==2)?(8):(4))

int mcw_rank, mcw_size;

int main(int argc, char **argv) {
	int len;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &mcw_rank);
    MPI_Comm_size(MPI_COMM_WORLD, &mcw_size);

    //-----------------------------------
    // Create some datatypes
	MPI_Type_size(MPI_LONG_LONG, &len);
	MPI_Type_create_resized(MPI_LONG_LONG, 0, 2*len, &i8space);
	MPI_Type_contiguous(2, MPI_LONG_LONG, &i8i8);
	MPI_Type_commit(&i8space);
	MPI_Type_commit(&i8i8);

    //-----------------------------------
    if( argc == 1 || (argc > 1 && argv[1][0] == '0') ) {
        printf("%2d / %2d) Running MPI_Ibcast...\n", mcw_rank, mcw_size);
        test_ibcast();
    }
    if( argc == 1 || (argc > 1 && argv[1][0] == '1') ) {
        printf("%2d / %2d) Running MPI_Iallgather...\n", mcw_rank, mcw_size);
        test_iallgather();
    }
    if( argc == 1 || (argc > 1 && argv[1][0] == '2') ) {
        printf("%2d / %2d) Running MPI_Iallgatherv...\n", mcw_rank, mcw_size);
        test_iallgatherv();
    }

    //-----------------------------------
    // Pass
    printf("%2d / %2d) Passed!\n", mcw_rank, mcw_size);

    MPI_Finalize();

    return 0;
}

int test_iallgatherv(void)
{
    int pattern = 1;
	long long i, j, byteoffset;
	int *sdt, *scount, *rdt, **rcount, *rdisps;
	long long *sbuf, *rbuf;
	MPI_Request myreq = NULL;
    MPI_Status mystatus;
	int err, nbytes, myflag;
    int tcount;

    if( mcw_size != 5 ) {
        printf("%2d / %2d) Test only works for MCW size = 4\n", mcw_rank, mcw_size);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }

    //-----------------------------------
    // Allocate buffers
    //nbytes = 721920;
    nbytes   = 332800;

    sdt    = (int* )malloc(sizeof(int ) * mcw_size );
    scount = (int* )malloc(sizeof(int ) * mcw_size );
    rdt    = (int* )malloc(sizeof(int ) * mcw_size );
    rcount = (int**)malloc(sizeof(int*) * mcw_size + sizeof(int) * mcw_size * mcw_size );
    rdisps = (int* )malloc(sizeof(int ) * mcw_size );

	for(i = 0; i < mcw_size; ++i) {
		rcount[i] = (int*) ((char*)rcount + mcw_size * sizeof(int*) + i * mcw_size * sizeof(int));
	}

    sdt[0] = 0;
    scount[0] = 56548;

    sdt[1] = 1; // Passes if set to 0
    scount[1] = 41600;

    sdt[2] = 0;//1;
    scount[2] = 43212;

    sdt[3] = 0;//1;
    scount[3] = 48776;

    sdt[4] = 0;
    scount[4] = 66160;

    rdt[0] = 2;
    rcount[0][0] = scount[0] / 2;
    rcount[0][1] = scount[1] / 2;
    rcount[0][2] = scount[2] / 2;
    rcount[0][3] = scount[3] / 2;
    rcount[0][4] = scount[4] / 2;
 
    for( i = 1; i < mcw_size; ++i ) {
        rdt[i] = rdt[0];
        for( j = 0; j < mcw_size; ++j ) {
            rcount[i][j] = rcount[0][j];
        }
    }

	tcount = 0;
	rdisps[0] = tcount;
	for(i = 0; i < mcw_size-1; ++i) {
		tcount += rcount[mcw_rank][i];
		rdisps[i+1] = tcount;
	}
	tcount += rcount[mcw_rank][mcw_size-1];

	rbuf = malloc(tcount * DTEXTENT(rdt[mcw_rank]) + 1);
	memset(rbuf, 123, tcount * DTEXTENT(rdt[mcw_rank]));

    sbuf = malloc(scount[mcw_rank] * DTEXTENT(sdt[mcw_rank]) + 1);
    fillbuf(sbuf, scount[mcw_rank], sdt[mcw_rank], pattern);


    //-----------------------------------
    // MPI_Iallgatherv
    err = MPI_Iallgatherv(sbuf, scount[mcw_rank], MPIDT(sdt[mcw_rank]),
                          rbuf, rcount[mcw_rank], rdisps, MPIDT(rdt[mcw_rank]),
                          MPI_COMM_WORLD, &myreq);
    if (err != MPI_SUCCESS) {
        printf("%2d / %2d) Failed in MPI_Iallgatherv. err = %d\n", mcw_rank, mcw_size, err);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }


    //-----------------------------------
    // Wait for completion
    do {
        err = MPI_Test(&myreq,&myflag, &mystatus);
        if (!myflag ) {
            usleep(1000);
        }
    } while (!myflag);
    if (err != MPI_SUCCESS) {
        printf("%2d / %2d) Failed in MPI_Test. err = %d\n", mcw_rank, mcw_size, err);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }


    //-----------------------------------
    // Check results
	byteoffset = 0;
	for(i = 0; i < mcw_size; ++i) {
		err = chkbuf((long long*)(((char*)rbuf) + byteoffset),
                     rcount[mcw_rank][i], rdt[mcw_rank], pattern, i);

        if (err != 0 ) {
            printf("%2d / %2d) Failed in chkbuf. err = %d\n", mcw_rank, mcw_size, err);
            MPI_Abort(MPI_COMM_WORLD, -2);
        }

		byteoffset += rcount[mcw_rank][i] * DTEXTENT(rdt[mcw_rank]);
    }


    return 0;
}

int test_iallgather(void)
{
    int pattern = 1;
	long long i, byteoffset;
	int *sdt, *scount, *rdt, *rcount;
	long long *sbuf, *rbuf;
	MPI_Request myreq = NULL;
    MPI_Status mystatus;
	int err, nbytes, myflag;

    if( mcw_size != 5 ) {
        printf("%2d / %2d) Test only works for MCW size = 4\n", mcw_rank, mcw_size);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }

    //-----------------------------------
    // Allocate buffers
    //nbytes = 1082896;
    nbytes = 1080704;

    sdt    = (int*)malloc(sizeof(int) * mcw_size);
    scount = (int*)malloc(sizeof(int) * mcw_size);
    rdt    = (int*)malloc(sizeof(int) * mcw_size);
    rcount = (int*)malloc(sizeof(int) * mcw_size);

    rdt[0] = 0;
    rcount[0] = nbytes / DTEXTENT(rdt[0]);
    sdt[0] = 1;
    scount[0] = nbytes / DTEXTENT( 0 );

    rdt[1] = 1;
    rcount[1] = nbytes / DTEXTENT( 0 );
    sdt[1] = 0;
    scount[1] = nbytes / DTEXTENT(sdt[1]);

    rdt[2] = 2;
    rcount[2] = nbytes / DTEXTENT(rdt[2]);
    sdt[2] = 2;
    scount[2] = nbytes / DTEXTENT(sdt[2]);

    rdt[3] = 0;
    rcount[3] = nbytes / DTEXTENT(rdt[3]);
    sdt[3] = 2;
    scount[3] = nbytes / DTEXTENT(sdt[3]);

    rdt[4] = 2;
    rcount[4] = nbytes / DTEXTENT(rdt[4]);
    sdt[4] = 2;
    scount[4] = nbytes / DTEXTENT(sdt[4]);
 
	rbuf = malloc(rcount[mcw_rank] * mcw_size  * DTEXTENT(rdt[mcw_rank]) + 1);
	memset(rbuf, 123, rcount[mcw_rank] * mcw_size * DTEXTENT(rdt[mcw_rank]));

    sbuf = malloc(scount[mcw_rank] * DTEXTENT(sdt[mcw_rank]) + 1);
    fillbuf(sbuf, scount[mcw_rank], sdt[mcw_rank], pattern);


    //-----------------------------------
    // MPI_Iallgather
    err = MPI_Iallgather(sbuf, scount[mcw_rank], MPIDT(sdt[mcw_rank]),
                         rbuf, rcount[mcw_rank], MPIDT(rdt[mcw_rank]),
                         MPI_COMM_WORLD, &myreq);
    if (err != MPI_SUCCESS) {
        printf("%2d / %2d) Failed in MPI_Iallgather. err = %d\n", mcw_rank, mcw_size, err);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }


    //-----------------------------------
    // Wait for completion
    do {
        err = MPI_Test(&myreq,&myflag, &mystatus);
        if (!myflag ) {
            usleep(1000);
        }
    } while (!myflag);
    if (err != MPI_SUCCESS) {
        printf("%2d / %2d) Failed in MPI_Test. err = %d\n", mcw_rank, mcw_size, err);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }


    //-----------------------------------
    // Check results
	byteoffset = 0;
	for(i = 0; i < mcw_size; ++i) {
		err = chkbuf((long long*)(((char*)rbuf) + byteoffset),
                     rcount[mcw_rank], rdt[mcw_rank], pattern, i);

        if (err != 0 ) {
            printf("%2d / %2d) Failed in chkbuf. err = %d\n", mcw_rank, mcw_size, err);
            MPI_Abort(MPI_COMM_WORLD, -2);
        }

		byteoffset += rcount[mcw_rank] * DTEXTENT(rdt[mcw_rank]);
    }


    return 0;
}

int test_ibcast(void)
{
	int root, scount, sdt, *rdt, *rcount;
	long long *sbuf;
    int pattern = 1;
	MPI_Request myreq = NULL;
    MPI_Status mystatus;
    int myflag = 0;
	int err, i, nbytes;

    //-----------------------------------
    // Allocate buffers
    root   = 1;

    //nbytes = 1925168; // Ok, with this # bytes
    nbytes = 110576;

    sdt    = 0;
    scount = nbytes / DTEXTENT(sdt);;
    rdt    = (int*)malloc(sizeof(int) * mcw_size);
    rcount = (int*)malloc(sizeof(int) * mcw_size);
    for( i = 0; i < mcw_size; ++i ) {
        if( root == i ) {
            rcount[i] = -1;
            rdt[i] = -1;
        }
#if 0
        // Fails here if mismatched datatypes, only at np >= 5
        else if( 4 == i ) {
            rcount[i] = scount;
            rdt[i]    = sdt;
        }
        else {
            rcount[i] = scount / 2;
            rdt[i]    = 2;
        }
#else
        else {
            // Passes if uniform type / count
            //rcount[i] = scount;
            //rdt[i]    = sdt;

            // Fails here if signature does not match root, only at np >= 5
            rcount[i] = scount / 2;
            rdt[i]    = 2;
        }
#endif
    }
    pattern = 1;
    if( mcw_rank == root ) {
        sbuf   = (long long*)malloc(scount * DTEXTENT(sdt) + 1);
        fillbuf(sbuf, scount, sdt, pattern);
    }
    else {
        sbuf   = (long long*)malloc(rcount[mcw_rank] * DTEXTENT(rdt[mcw_rank]) + 1);
        memset(sbuf, 123, rcount[mcw_rank] * DTEXTENT(rdt[mcw_rank]) );
    }


    //-----------------------------------
    // MPI_Ibcast
    if( mcw_rank == root ) {
        err = MPI_Ibcast(sbuf, scount, MPIDT(sdt), root, MPI_COMM_WORLD, &myreq);
    }
    else {
        err = MPI_Ibcast(sbuf, rcount[mcw_rank], MPIDT(rdt[mcw_rank]), root, MPI_COMM_WORLD, &myreq);
    }
    if (err != MPI_SUCCESS) {
        printf("%2d / %2d) Failed in MPI_Ibcast. err = %d\n", mcw_rank, mcw_size, err);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }


    //-----------------------------------
    // Wait for completion
    do {
        err = MPI_Test(&myreq,&myflag, &mystatus);
        if (!myflag ) {
            usleep(1000);
        }
    } while (!myflag);
    if (err != MPI_SUCCESS) {
        printf("%2d / %2d) Failed in MPI_Test. err = %d\n", mcw_rank, mcw_size, err);
        MPI_Abort(MPI_COMM_WORLD, -1);
    }


    //-----------------------------------
    // Check results
    if( mcw_rank == root ) {
        err = chkbuf(sbuf, scount, sdt, pattern, root);
    }
    else {
		err = chkbuf(sbuf, rcount[mcw_rank], rdt[mcw_rank], pattern, root);
    }
    if (err != 0 ) {
        printf("%2d / %2d) Failed in chkbuf. err = %d\n", mcw_rank, mcw_size, err);
        MPI_Abort(MPI_COMM_WORLD, -2);
    }

    return 0;
}

void fillbuf(long long *buf, long count, int dt, int pattern)
{
	long i;
	for (i=0; i<count; ++i) {
		if (dt == 0) {
			buf[i] = mcw_rank + (long long)pattern * i;
		}
		else if (dt == 1) {
			buf[2*i]   = mcw_rank + (long long)pattern * i;
			buf[2*i+1] = -1;
		}
		else if (dt == 2) {
			buf[2*i]   = mcw_rank + (long long)pattern * (2*i);
			buf[2*i+1] = mcw_rank + (long long)pattern * (2*i+1);
		}
	}
}

int chkbuf(long long *buf, long count, int dt, int pattern, int from)
{
	long i;
	long long l123;

	((char*)(&l123))[0] = 123;
	((char*)(&l123))[1] = 123;
	((char*)(&l123))[2] = 123;
	((char*)(&l123))[3] = 123;
	((char*)(&l123))[4] = 123;
	((char*)(&l123))[5] = 123;
	((char*)(&l123))[6] = 123;
	((char*)(&l123))[7] = 123;

	for (i=0; i<count; ++i) {
		if (dt == 0) {
			if ( buf[i] != from + (long long)pattern * i) {
				printf("%2d / %2d) buf[%ld] : actual %lld [%llx], expected %lld [%llx]. from %d\n",
                       mcw_rank, mcw_size,
                       i,
                       buf[i], buf[i],
                       (long long int)(from + (long long)pattern * i),
                       (long long int)(from + (long long)pattern * i),
                       from);
                return -1;
			}
		}
		else if (dt == 1) {
			if ( buf[2*i] != from + (long long)pattern * i) {
				printf("%2d / %2d) buf[%ld] : actual %lld [%llx], expected %lld [%llx]. from %d\n",
                       mcw_rank, mcw_size,
                       2*i,
                       buf[2*i], buf[2*i],
                       (long long int)(from + (long long)pattern * i),
                       (long long int)(from + (long long)pattern * i),
                       from);
                return -2;
			}

            if ( buf[2*i+1] != l123) {
                printf("%2d / %2d) buf[%ld] : actual %lld [%llx], expected %lld [%llx]. from %d\n",
                       mcw_rank, mcw_size,
                       2*i+1,
                       buf[2*i+1], buf[2*i+1],
                       l123, l123,
                       from);
                return -3;
            }
		}
		else if (dt == 2) {
			if ( buf[2*i] != from + (long long)pattern * 2*i) {
                printf("%2d / %2d) buf[%ld] : actual %lld [%llx], expected %lld [%llx]. from %d\n",
                       mcw_rank, mcw_size,
                       2*i,
                       buf[2*i], buf[2*i],
                       (long long int)(from + (long long)pattern * 2 * i),
                       (long long int)(from + (long long)pattern * 2 * i),
                       from);
                return -5;
			}
			if ( buf[2*i+1] != from + (long long)pattern * (2*i+1)) {
                printf("%2d / %2d) buf[%ld] : actual %lld [%llx], expected %lld [%llx]. from %d\n",
                       mcw_rank, mcw_size,
                       2*i+1,
                       buf[2*i+1], buf[2*i+1],
                       (long long int)(from + (long long)pattern * (2*i+1)),
                       (long long int)(from + (long long)pattern * (2*i+1)),
                       from);
                return -6;
			}
		}
	}

    return 0;
}