deadlock.c

/* This program should be run on a single Blue Gene/Q node
 * using 5 or more parallel processes and 8 threads per process.
 * Run the program with:
 *   mpirun -np 5 ./deadlock 8
 * where 8 specifies number of threads per process.
 * It has a high probability of going into a deadlocked state
 * (or at least timing out at 5~10 minutes).
 * The probability increases as LOOPS is increased. */

#include <mpi.h>
#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <stddef.h>

static int nthreads;

#define LOOPS 1200

void* thread_main(void* arg)
{
  int myproc;
  MPI_Comm_rank(MPI_COMM_WORLD,&myproc);
  int mythread = *((int*)(arg));
  //fprintf(stdout, "%d %d\n", myproc, mythread);
  for (int i=0; i < LOOPS; ++i)
  {
    if (mythread%2)
    {
      int tothread = mythread-1;
      int tag = (mythread<<10)+tothread;
      MPI_Request request;
      MPI_Issend(NULL,0,MPI_BYTE,myproc,tag,MPI_COMM_WORLD,&request);
      MPI_Wait(&request,MPI_STATUS_IGNORE);
    }
    else
    {
      int fromthread = mythread+1;
      int tag = (fromthread<<10)+mythread;
      MPI_Recv(NULL,0,MPI_BYTE,myproc,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
    }
  }
  return NULL;
}

int main(int argc, char** argv)
{
  int provided;
  MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided);
  assert(argc==2);
  assert(provided==MPI_THREAD_MULTIPLE);
  nthreads = atoi(argv[1]);
  int err;
  pthread_t global_threads[100];
  int tId[100];
  tId[0] = 0;
  for (int i=1; i < nthreads; ++i)
  {
    tId[i] = i;
    err = pthread_create(global_threads+i,NULL,thread_main,(void*)&(tId[i]));
    assert(!err);
  }
  thread_main((void*)&(tId[0]));
  for (int i=1; i < nthreads; ++i)
  {
    err = pthread_join(global_threads[i],NULL);
    assert(!err);
  }
  MPI_Finalize();
  return 0;
}