OpenMPI RMA operation.

OneSideMPI.cpp

#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>

/**
 * @brief OpenMPI RMA operation.
 * @details Create two window with MPI_Win_create_dynamic, and attach two arrays
 * with different data type to one window. Attach another array that has data overlapped
 * with other array to different window or the same window will still work.
 * Then use MPI_Get to perform RMA operation to get data from target rank.
 **/

int main(int argc, char *argv[]){
	/* Initialize MPI. */
	MPI_Init(&argc, &argv);

	/* Get my rank and size. */
	int MyRank, NRank;
	MPI_Comm_rank(MPI_COMM_WORLD, &MyRank);
	MPI_Comm_size(MPI_COMM_WORLD, &NRank);

	int length = 10;
	int *buffer = (int *) malloc(length * sizeof(int));
	int length1 = 5;
	int *buffer1 = (int *) malloc(length1 * sizeof(int));
	int length2 = 5;
	double *buffer2 = (double *) malloc(length2 * sizeof(double));
	int length3 = 3;
	double *buffer3 = &(buffer2[2]);

	/* Create window. */
	MPI_Win window[2];
	MPI_Win_create_dynamic(MPI_INFO_NULL, MPI_COMM_WORLD, &(window[0]));
	MPI_Win_create_dynamic(MPI_INFO_NULL, MPI_COMM_WORLD, &(window[1]));

	/* Open the epochs. */
	MPI_Win_fence(0, window[0]);
	MPI_Win_fence(0, window[1]);

	/* Window address. Since we are using MPI_Win_create_dynamic,
	   we need to let other rank know where the window is. */
	MPI_Aint window_address0[4];
	
	if ( MyRank == 0 ){
		for ( int i = 0; i < 10; i++ ){
			buffer[i] = i;
		}
		if( MPI_Win_attach(window[0], buffer, length * sizeof(int)) != MPI_SUCCESS ){
			printf("MPI_Win_attach failed.\n");
		}
		MPI_Get_address(buffer, &(window_address0[0]));

		/* Attach to window[1]. */
		for (int i = 0; i < length1; i++){
			buffer1[i] = i * 100;
		}
		for (int i = 0; i < length2; i++){
			buffer2[i] = i * 5.5;
		}
		if ( MPI_Win_attach(window[1], buffer1, length1 * sizeof(int)) != MPI_SUCCESS ){
			printf("MPI_Win_attach failed.\n");
		}
		if ( MPI_Win_attach(window[1], buffer2, length2 * sizeof(double)) != MPI_SUCCESS ){
			printf("MPI_Win_attach failed.\n");
		}
		if ( MPI_Win_attach(window[0], buffer3, length3 * sizeof(double)) != MPI_SUCCESS ){
			printf("MPI_Win_attach failed on buffer3.\n");
		}
		MPI_Get_address(buffer1, &(window_address0[1]));
		MPI_Get_address(buffer2, &(window_address0[2]));
		MPI_Get_address(buffer3, &(window_address0[3]));
	}

	MPI_Bcast(window_address0, 4, MPI_AINT, 0, MPI_COMM_WORLD);
	
	for(int i = 0; i < length; i ++){
		printf("Rank %d, buffer[%d] = %d\n", MyRank, i, buffer[i]);
	}
	printf("------------------------\n");
	for(int i = 0; i < length1; i++){
		printf("Rank %d, buffer1[%d] = %d\n", MyRank, i, buffer1[i]);
	}
	printf("------------------------\n");
	for(int i = 0; i < length2; i++){
		printf("Rank %d, buffer2[%d] = %lf\n", MyRank, i, buffer2[i]);
	}
	printf("------------------------\n");
	for(int i = 0; i < length3; i++){
		printf("Rank %d, buffer3[%d] = %lf\n", MyRank, i, buffer3[i]);
	}
	printf("========================\n");
	
	/* Perform MPI_Get if MyRank != 0. */
	if ( MyRank != 0 ){
		int target_count = 10;
		if ( MPI_Get(buffer, 10, MPI_INT, 0, window_address0[0], target_count, MPI_INT, window[0]) != MPI_SUCCESS){
			printf("MPI_Get failed!\n");
		}

		if ( MPI_Get(buffer1, 5, MPI_INT, 0, window_address0[1], 5, MPI_INT, window[1]) != MPI_SUCCESS ){
			printf("MPI failed!\n");
		}

		if ( MPI_Get(buffer2, 5, MPI_DOUBLE, 0, window_address0[2], 5, MPI_DOUBLE, window[1]) != MPI_SUCCESS ){
			printf("MPI failed!\n");
		}

		if ( MPI_Get(buffer3, 3, MPI_DOUBLE, 0, window_address0[3], 3, MPI_DOUBLE, window[0]) != MPI_SUCCESS ){
			printf("MPI failed!\n");
		}
	}

	/* Close the epoch. */
	MPI_Win_fence(0, window[0]);
	MPI_Win_fence(0, window[1]);


	for(int i = 0; i < length; i ++){
		printf("Rank %d, buffer[%d] = %d\n", MyRank, i, buffer[i]);
	}
	printf("------------------------\n");
	for(int i = 0; i < length1; i++){
		printf("Rank %d, buffer[%d] = %d\n", MyRank, i, buffer1[i]);
	}
	printf("------------------------\n");
	for(int i = 0; i < length2; i++){
		printf("Rank %d, buffer2[%d] = %lf\n", MyRank, i, buffer2[i]);
	}
	printf("------------------------\n");
	for(int i = 0; i < length3; i++){
		printf("Rank %d, buffer3[%d] = %lf\n", MyRank, i, buffer3[i]);
	}
	/* Detach buffer. */
	if ( MyRank == 0 ){
		MPI_Win_detach(window[0], buffer);
		MPI_Win_detach(window[1], buffer1);
		MPI_Win_detach(window[1], buffer2);
		MPI_Win_detach(window[0], buffer3);
	}

	/* Destroy the window. */
	MPI_Win_free(&(window[0]));
	MPI_Win_free(&(window[1]));

	/* Free resources. */
	if (MyRank == 0){
		free(buffer);
		free(buffer1);
		free(buffer2);
	}

	MPI_Finalize();
}

Attach another array that has data overlapped with other attached array to different window or the same window will still work.