gistfile1.c

/* file: transpose.c
 *
 * Description: Transpose Operation using MPI derived datatypes with
 * MPI collective communications.
 *
 * Author: Spenser Gilliland <spenser@gillilanding.com>
 */

#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

void print_matrix(float * matrix, int n, int wrank, int wsize) {
	int i , j;
	int wrows = n / wsize;

	MPI_Barrier(MPI_COMM_WORLD);
	if(wrank == 0) printf("Matrix = \n");
	MPI_Barrier(MPI_COMM_WORLD);

	for(i = 0; i < n; i++) {
		if(i >= wrank*wrows && i < (wrank+1)*wrows) {
			printf("%2d:", wrank);
			for(j = 0; j < n; j++) {
				printf("%6.2g", matrix[i*n + j]);
			}
			printf("\n");
		}
		usleep(10000); /* Wait for Console to write */
		MPI_Barrier(MPI_COMM_WORLD);
	}
}

void mpi_transpose(float *matrix, int n, int wrank, int wsize) {
	int i, j, k;
	int row, col;
	float temp;
	int wrows = n / wsize;
	float *tmat = malloc(sizeof(float)*n*n);

	MPI_Datatype mpi_all_unaligned_t, mpi_all_t;

	MPI_Type_vector(n, wrows, n, MPI_FLOAT, &mpi_all_unaligned_t);
	MPI_Type_create_resized(mpi_all_unaligned_t, 0, wrows*sizeof(float), &mpi_all_t);
	MPI_Type_commit(&mpi_all_t);
	MPI_Type_free(&mpi_all_unaligned_t);

	/* Local Transpose */
	row = wrank*wrows;
	for(k = 0; k < wsize; k++) {
		col = k*wrows;
		for( i = 0; i < wrows; i++) {
			tmat[(row + i)*n + col + i] = matrix[(row + i)*n + col + i];
			for(j = i + 1; j < wrows; j++) {
				tmat[(row + i)*n + col + j] = matrix[(row + j)*n + col + i];
				tmat[(row + j)*n + col + i] = matrix[(row + i)*n + col + j];
			}
		}
	}

	/* Global Transpose */
	MPI_Alltoall(&tmat[wrank*wrows*n], 1, mpi_all_t,
	             &matrix[wrank*wrows*n], 1, mpi_all_t,
	             MPI_COMM_WORLD);

	free(tmat);
	MPI_Type_free(&mpi_all_t);
}

int main(int argc, char *argv[]) {
	int wsize, wrank, wrows;
	int i, j, k;
	int row, col;
	float *matrix;

	MPI_Init(&argc, &argv);
	MPI_Comm_size(MPI_COMM_WORLD, &wsize);
	MPI_Comm_rank(MPI_COMM_WORLD, &wrank);

	if (argc != 2) {
		printf("usage: %s <matrix size>\n",argv[0]);
		MPI_Abort(MPI_COMM_WORLD, -1);
	}

	int n = strtol(argv[1], NULL, 10);

	if (wrank == 0) printf("N = %d\n", n);

	wrows = n / wsize;

	matrix = malloc(sizeof(float)*n*n);
	if (matrix == NULL) {
		printf("ERROR: Could not allocate %d bytes\n", sizeof(float)*n*n);
		fflush(stdout);
		MPI_Abort(MPI_COMM_WORLD, -1);
	}


	for(i = 0; i < n; i++) {
		/* Initialize data on the rows of the matrix owned by this rank */
		if (i >= wrank*wrows && i < (wrank+1)*wrows) {
			for(j = 0; j < n; j++) {
				matrix[i*n + j] = i*n + j;
			}
		}
	}

	print_matrix(matrix, n, wrank, wsize);

	mpi_transpose(matrix, n, wrank, wsize);

	print_matrix(matrix, n, wrank, wsize);

	free(matrix);

	MPI_Finalize();
	return 0;
}