write31.c

#include <stdio.h>
#include <stdlib.h>
#include "mpi.h"
#include "hdf5.h"

static void SetHDF5fapl(hid_t fapl, MPI_Comm comm)
{
    H5Pset_fapl_mpio(fapl, comm, MPI_INFO_NULL);

    // Alignment and metadata block size
    int alignment = 16 * 1024 * 1024;
    int blocksize =  4 * 1024 * 1024;
    H5Pset_alignment(fapl, alignment, alignment);
    H5Pset_meta_block_size(fapl, blocksize);

    // Collective metadata ops
    H5Pset_coll_metadata_write(fapl, true);
    H5Pset_all_coll_metadata_ops(fapl, true);

    // Defer cache flush
    H5AC_cache_config_t cache_config;
    cache_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
    H5Pget_mdc_config(fapl, &cache_config);
    cache_config.set_initial_size = 1;
    cache_config.initial_size = 16 * 1024 * 1024;
    cache_config.evictions_enabled = 0;
    cache_config.incr_mode = H5C_incr__off;
    cache_config.flash_incr_mode = H5C_flash_incr__off;
    cache_config.decr_mode = H5C_decr__off;
    H5Pset_mdc_config (fapl, &cache_config);

    return;
}

int main(int argc, char *argv[])
{
    int proc_num, my_rank, ret, i;

    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &proc_num);
    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

    hid_t fid, dset, fapl, dxpl_ind, dxpl_col, dcpl, dataspace, memdataspace;
    hsize_t offsets[31] = {0, 46080, 92160, 138240, 184320, 230400, 276480, 322560, 368640, 410112, 451584, 493056, 520704, 548352, 594432, 663552, 728064, 769536, 824832, 880128, 921600, 958464, 1013760, 1059840, 1105920, 1152000, 1198080, 1244160, 1290240, 1336320, 1382400};
    hsize_t sizes[31] = {46080, 46080, 46080, 46080, 46080, 46080, 46080, 46080, 41472, 41472, 41472, 27648, 27648, 46080, 69120, 64512, 41472, 55296, 55296, 41472, 36864, 55296, 46080, 46080, 46080, 46080, 46080, 46080, 46080, 46080, 46080};
    hsize_t total = 1428480;

    fapl = H5Pcreate (H5P_FILE_ACCESS);
    SetHDF5fapl(fapl, MPI_COMM_WORLD);

    fid = H5Fcreate("test.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);

    dcpl     = H5Pcreate(H5P_DATASET_CREATE);
    dxpl_col = H5Pcreate(H5P_DATASET_XFER);
    dxpl_ind = H5Pcreate(H5P_DATASET_XFER);
    H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);

    double *mydata = (double*)malloc(sizeof(double)*sizes[my_rank]);
    for (i = 0; i < sizes[my_rank]; i++)
        mydata[i] = my_rank * 1.0;
    
    dataspace = H5Screate_simple(1, &total, NULL);
    dset = H5Dcreate(fid, "testdset", H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);

    H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &offsets[my_rank], NULL, &sizes[my_rank], NULL);

    memdataspace = H5Screate_simple(1, &sizes[my_rank], NULL);

    fprintf(stderr, "Rank %d, offset %ld size %ld\n", my_rank, offsets[my_rank], sizes[my_rank]);
    ret = H5Dwrite(dset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl_col, mydata);
    if (ret < 0)
        fprintf(stderr, "Rank %d: Error with H5Dwrite\n", my_rank);

    H5Dclose(dset);
    H5Sclose(dataspace);
    H5Sclose(memdataspace);
    H5Pclose(fapl);
    H5Pclose(dxpl_ind);
    H5Pclose(dxpl_col);
    H5Pclose(dcpl);
    H5Fclose(fid);

    free(mydata);
    MPI_Finalize();
    return 0;
}