CudaWithMPI Example

CMakeLists.txt

cmake_minimum_required(VERSION 2.8)
project(MPITest)
find_package(CUDA QUIET REQUIRED)
find_package(MPI REQUIRED)
if (MPI_FOUND)
    include_directories(SYSTEM ${MPI_INCLUDE_PATH})
else (MPI_FOUND)
    message(SEND_ERROR "This application cannot compile without MPI")
endif (MPI_FOUND)
# Specify binary name and source file to build it from
cuda_add_executable(
        MPITest
        main.cu)
target_link_libraries(MPITest ${MPI_LIBRARIES})
#call cmake -DMPI_C_COMPILER=/opt/openmpi-3.0.0/bin/mpicc -DMPI_CXX_COMPILER=/opt/openmpi-3.0.0/bin/mpicxx .
#when compile openmpi-3.0.0 don't forget specify --with-cuda tag and --prefix

main.cu

#include <mpi.h>
#include <stdio.h>
#include <cuda_runtime.h>

int main(int argc, char **argv) {
    int rank;
    int *buf;
    const int root = 0;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    cudaMalloc(&buf, sizeof(int));

    if (rank == root) {
        int val = 1234;
        cudaMemcpy(buf, &val, sizeof(int), cudaMemcpyHostToDevice);
    }

    int val;

    cudaMemcpy(&val, buf, sizeof(int), cudaMemcpyDeviceToHost);
    printf("[%d]: Before Bcast, buf is %d\n", rank, buf);

    /* everyone calls bcast, data is taken from root and ends up in everyone's buf */
    MPI_Bcast(buf, 1, MPI_INT, root, MPI_COMM_WORLD);

    cudaMemcpy(&val, buf, sizeof(int), cudaMemcpyDeviceToHost);
    printf("[%d]: After Bcast, buf is %d\n", rank, val);

    MPI_Finalize();
    return 0;
}