HDF5_MPI_issue.c

#include <mpi.h>

#include <stdlib.h>
#include <string.h>
#include <stdio.h>

#define H5S_MAX_MPI_COUNT               536870911 
static MPI_Aint bigio_count = H5S_MAX_MPI_COUNT;

#define MPI_CHECK(cmd) {int error = cmd; if(error!=MPI_SUCCESS){printf("%s %i %d\n""[MPI] Error code ",  error,__LINE__);}}

int write_MPIO(
    MPI_Comm const comm, MPI_Info const info,
    int* data, size_t len)
{
    int myrank = 0;

    MPI_File thefile; 

    MPI_File_open(MPI_COMM_WORLD, "testfile", 
        MPI_MODE_CREATE | MPI_MODE_WRONLY, 
        MPI_INFO_NULL, &thefile); 

    //**** create a new compound type ******
    size_t num_elements = len;
    MPI_Aint num_big_types = (int)(num_elements/bigio_count);
    MPI_Aint leftover = num_elements - num_big_types * (size_t)bigio_count;
    size_t remaining_bytes = leftover;

    MPI_Aint disp[2], old_extent;
    MPI_Datatype old_type = MPI_BYTE;
    MPI_Datatype  inner_type, outer_type, leftover_type, type[2];
    MPI_Type_contiguous(bigio_count, old_type,  &inner_type);
    MPI_Type_contiguous(num_big_types, inner_type, &outer_type);
    MPI_Type_contiguous (remaining_bytes, old_type, &leftover_type);
    MPI_Type_extent (old_type, &old_extent);

    int block_len[2];
    type[0] = outer_type;
    type[1] = leftover_type;
    block_len[0] = 1;
    block_len[1] = 1;
    disp[0] = 0;
    disp[1] = (old_extent)*num_big_types*(MPI_Aint)bigio_count;
    MPI_Datatype new_type;
    MPI_Type_create_struct(2, block_len, disp, type, &new_type);

    MPI_Type_commit(&new_type);

    MPI_Status mstat;
    MPI_Offset mpi_off = 0;
    
    memset(&mstat,0,sizeof(MPI_Status));
    MPI_CHECK(MPI_File_set_view(thefile,mpi_off , 
          MPI_BYTE, new_type, "native", 
                      MPI_INFO_NULL));  

    MPI_CHECK(MPI_File_write_at_all(thefile, mpi_off, data, 1, new_type, &mstat)); 

    MPI_Count bytes_written = 0;
    MPI_CHECK(MPI_Get_elements_x(&mstat, MPI_BYTE, &bytes_written));
    printf("byte %lu\n", &bytes_written);
    MPI_File_close(&thefile); 

    return 0;
}

int main(int argc, char* argv[])
{

    MPI_Comm comm = MPI_COMM_WORLD; 
    MPI_Info info = MPI_INFO_NULL;  

    MPI_Init(&argc, &argv);
    
    size_t lengths[1] = {134217727u + 5};
    for( size_t i = 0; i < 1; ++i )
    {
        size_t len = lengths[i];
        printf("Writing for len=%zu ...\n", len);
        int* data = (int*)malloc(len * sizeof(int));
        for( size_t k=0; k<len; ++k)
            data[k] = 420;
    
        write_MPIO(comm, info, data, len * sizeof(int));
        free(data);
        printf("Finished write for len=%zu ...\n", len);
    }
    
    MPI_Finalize();

    return 0;
}