hybrid MPI/thread program that deadlocks on Blue Gene/Q

deadlock.c

/* This program should be run on a single Blue Gene/Q node
   using 5 or more parallel processes and 8 threads per process
   (threads per process is the only argument to the program)
   mpirun -np 5 ./deadlock 8
   It has a high probability of going into a deadlocked state
   (or at least timing out at 5~10 minutes).
   The probability increases as LOOPS is increased */

#include <mpi.h>
#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <stddef.h>

static int nthreads;

#define LOOPS 100

void* thread_main(void* arg)
{
  int myproc;
  MPI_Comm_rank(MPI_COMM_WORLD,&myproc);
  int mythread = (int)(ptrdiff_t)(arg);
  for (int i=0; i < LOOPS; ++i)
  {
    if (mythread%2)
    {
      int tothread = mythread-1;
      int tag = (mythread<<10)+tothread;
      MPI_Request request;
      MPI_Issend(NULL,0,MPI_BYTE,myproc,tag,MPI_COMM_WORLD,&request);
      MPI_Wait(&request,MPI_STATUS_IGNORE);
    }
    else
    {
      int fromthread = mythread+1;
      int tag = (fromthread<<10)+mythread;
      MPI_Recv(NULL,0,MPI_BYTE,myproc,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
    }
  }
  return NULL;
}

int main(int argc, char** argv)
{
  int provided;
  MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided);
  assert(argc==2);
  assert(provided==MPI_THREAD_MULTIPLE);
  nthreads = atoi(argv[1]);
  int err;
  pthread_t global_threads[100];
  for (int i=1; i < nthreads; ++i)
  {
    err = pthread_create(global_threads+i,NULL,thread_main,(void*)(ptrdiff_t)i);
    assert(!err);
  }
  thread_main((void*)0);
  for (int i=1; i < nthreads; ++i)
  {
    err = pthread_join(global_threads[i],NULL);
    assert(!err);
  }
  MPI_Finalize();
  return 0;
}