MPI test for mixing fetch and op with accumulate

c_accumulate_fetch_and_op.c

/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
 *                         reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

/*
 * This test ensures that per the MPI specification that we get
 * consistent results when mixing MPI_Fetch_and_op() with
 * MPI_Accumulate(). This test works by each MPI process
 * performing an MPI_Accumulate() to its left peer and
 * MPI_Fetch_and_op() on its right peer.
 */

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include "mpi.h"

#define BUFFER_COUNT 1024
#define LOOP_ITER 1000

int main (int argc, char *argv[]) {
  int32_t buffer[BUFFER_COUNT], *base, result, value;
  int size, rc, rank, target_left, target_right;
  MPI_Win win;

  MPI_Init (NULL, NULL);
  MPI_Comm_rank (MPI_COMM_WORLD, &rank);
  MPI_Comm_size (MPI_COMM_WORLD, &size);

  for (int i = 0 ; i < BUFFER_COUNT ; ++i) {
    buffer[i] = 1;
  }

  MPI_Win_allocate (sizeof (buffer), 4, MPI_INFO_NULL, MPI_COMM_WORLD, &base, &win);

  memset (base, 0, 4096);

  MPI_Barrier (MPI_COMM_WORLD);

  MPI_Win_lock_all (MPI_MODE_NOCHECK, win);

  target_left = (rank + size - 1) % size;
  target_right = (rank + 1) % size;

  value = -2;

  for (int i = 0 ; i < LOOP_ITER ; ++i) {
    /* mix up the ordering somewhat. we are trying to catch a potential race condition */
    if ((rank + i) & 1) {
      MPI_Fetch_and_op (&value, &result, MPI_INT32_T, target_left, 0, MPI_SUM, win);
      MPI_Accumulate (buffer, BUFFER_COUNT, MPI_INT32_T, target_right, 0, BUFFER_COUNT, MPI_INT32_T, MPI_SUM, win);
    } else {
      MPI_Accumulate (buffer, BUFFER_COUNT, MPI_INT32_T, target_right, 0, BUFFER_COUNT, MPI_INT32_T, MPI_SUM, win);
      MPI_Fetch_and_op (&value, &result, MPI_INT32_T, target_left, 0, MPI_SUM, win);
    }

    /* ensure everything is flushed out in case the OSC component delays communication to flush */
    MPI_Win_flush_all (win);
  }

  MPI_Barrier (MPI_COMM_WORLD);

  if (0 == (rank & 0x1)) {
    int32_t expected = LOOP_ITER * -1;

    if (base[0] != expected) {
      fprintf (stderr, "Fail @ index 0! Expected %d, got %d\n", expected, base[0]);
      rc = -1;
    }

    expected = LOOP_ITER;

    for (int i = 1 ; i < BUFFER_COUNT ; ++i) {
      if (base[i] != expected) {
        fprintf (stderr, "Fail @ index %d! Expected %d, got %d\n", i, expected, base[i]);
        rc = -1;
      }
    }
  }

  MPI_Win_unlock_all (win);

  MPI_Win_free (&win);

  MPI_Finalize ();

  return rc;
}