Immediate barrier not completed on all ranks when interleaved with point to point communication

repro.cpp

// STL Includes
#include <algorithm>
#include <vector>
#include <cstdint>
#include <iostream>

// Third Party Includes
#include <mpi.h>

using std::uint64_t;

int main(int argc, char **argv)
{
    MPI_Init(&argc, &argv);

    MPI_Comm world = MPI_COMM_WORLD;

    int rank;
    MPI_Comm_rank(world, &rank);

    int size;
    MPI_Comm_size(world, &size);

    if (size < 2)
    {
        std::cerr << "This program requires at least 2 ranks." << std::endl;
        return 1;
    }

    int const receiver_rank = 0;

    if (rank == receiver_rank)
    {
        // receiver process

        size_t const n = size - 1;
        std::vector<uint64_t> buf(3 * n);

        // receive first 2*n messages
        for (size_t i = 0; i < 2 * n; i++)
        {
            MPI_Status status;
            MPI_Recv(&buf.at(i), 1, MPI_UINT64_T, MPI_ANY_SOURCE, MPI_ANY_TAG, world, &status);
            std::cout << rank << ": Received " << buf.at(i) << " from " << status.MPI_SOURCE << std::endl;
        }

        // signal the waiting senders that 2 * n messages have been received
        MPI_Request breq;
        MPI_Ibarrier(world, &breq);

        // receive remaining n messages
        for (size_t i = 2 * n; i < 3 * n; i++)
        {
            MPI_Status status;
            MPI_Recv(&buf.at(i), 1, MPI_UINT64_T, MPI_ANY_SOURCE, MPI_ANY_TAG, world, &status);
            std::cout << rank << ": Received " << buf.at(i) << " from " << status.MPI_SOURCE << std::endl;
        }

        bool bad = false;

        // messages "1" and "2" may be interleaved, but all have to be contained within the first 2 * n slots of the buffer
        auto const num_one = std::count(buf.begin(), buf.begin() + 2 * n, 1);
        auto const num_two = std::count(buf.begin(), buf.begin() + 2 * n, 2);

        if (num_one != n) {
            std::cerr << "Wrong number of 1's" << std::endl;
            bad = true;
        }

        if (num_two != n) {
            std::cerr << "Wrong number of 2's" << std::endl;
            bad = true;
        }

        auto const num_three = std::count(buf.begin() + 2 * n, buf.begin() + 3 * n, 3);

        // the last n slots in the buffer may only contain message "3"
        if (num_three != n) {
            std::cerr << "Wrong number of 3's" << std::endl;
            bad = true;
        }

        // clean up the barrier request
        MPI_Wait(&breq, MPI_STATUS_IGNORE);

        if (bad) {
            return 1;
        }
    } else {
        uint64_t send;
        // sender processes
        // send message "1"
        send = 1;
        std::cout << rank << ": Sending " << send << std::endl;
        MPI_Send(&send, 1, MPI_UINT64_T, 0, 0, world);
        std::cout << rank << ": Sent " << send << std::endl;
        // join barrier, but do not block
        MPI_Request breq;
        MPI_Ibarrier(world, &breq);
        // send message "2"
        send = 2;
        std::cout << rank << ": Sending " << send << std::endl;
        MPI_Send(&send, 1, MPI_UINT64_T, 0, 0, world);
        std::cout << rank << ": Sent " << send << std::endl;
        // wait for receiver process to receive the first 2 * n messages
        MPI_Wait(&breq, MPI_STATUS_IGNORE);
        // send message "3"
        send = 3;
        std::cout << rank << ": Sending " << send << std::endl;
        MPI_Send(&send, 1, MPI_UINT64_T, 0, 0, world);
        std::cout << rank << ": Sent " << send << std::endl;
    }

    MPI_Finalize();
}