mpi_debugging_sleep.py

# See sample output at the bottom of this file
def MPI_Debugging(numSimRuns=1, iterations = 8, batch_size = 1):
    #MPI Debugging for testing purposes

    #Setup of the communications
    comm = MPI.COMM_WORLD
    size = comm.Get_size()
    rank = comm.Get_rank()
    num_agents = size / batch_size
    comm2D = comm.Create_cart([num_agents, batch_size])
    commX = comm2D.Sub(remain_dims=[False, True])
    commY = comm2D.Sub(remain_dims=[True, False])
    px, py = comm2D.Get_coords(rank)

    #Create memory windows that will hold agent data (weights, gradients of policies)
    #And create another one for iterations

    windows_iterations = []

    #iteration of the local agent
    iter_data = np.zeros(shape=(1), dtype=np.int32)
    iter_data[0] = -1;
    localiter = np.zeros(shape=(1), dtype=np.int32)
    localiter[0] = -1;

    #local variable
    iter_of_all_agents = np.zeros(shape=(int(num_agents)), dtype = np.int32)
    iter_of_all_agents[:] = -1
    num_agents = int(num_agents)

    if py == 0:
        for k in range(num_agents):
            if px==k:
                print('px,py is {} {}'.format(px,py))
                print('in window creating loop for agent {}'.format(k))
                win_iter = MPI.Win.Create(iter_data,1, info=MPI.INFO_NULL, comm=commY)
            else:
                win_iter = MPI.Win.Create(None,1, info=MPI.INFO_NULL, comm=commY)

            windows_iterations.append(win_iter)


    if py == 0: #each master agent

        print('time for this iteration for agent {} is {}'.format(px, np.round(tn2-tn1,2)))

        tn3 = time.time()
        iteration = 1
        #Now that the iteration is complete, write to the window the iteration number
        windows_iterations[px].Lock(px,lock_type=MPI.LOCK_EXCLUSIVE, assertion=0)
        np.copyto(iter_data, iteration)
        windows_iterations[px].Unlock(px)
        tn4 = time.time()
        print("time for exclusive lock {} for agent {}".format(np.round(tn4-tn3,2), px))

        # Gather results
        new_results = [] #idx of agents with new results
        #Essentially, the following loop goes through each agent and checks for new iterations from other agents
        #If the new data exists, this should then be stored

        comm2D.Barrier()
        print("forced barrier")
        if px == 0:
            time.sleep(40)
        else:
            time.sleep(2)
        for agent_id in range(num_agents):
            if agent_id != px:
                wit0 = time.time()
                #Apply shared lock, so that tha iteration data can be read
                windows_iterations[agent_id].Lock(agent_id,lock_type=MPI.LOCK_SHARED, assertion=0)
                windows_iterations[agent_id].Get([localiter, 1, MPI.INT], agent_id)
                windows_iterations[agent_id].Unlock(agent_id)
                wit1 = time.time()


                print(
                "localiter is {} agent_id is {} and current agent is {} timetaken {}".format(localiter, agent_id, px, np.round(wit1-wit0,2))
                )

                if localiter > iter_of_all_agents[agent_id]:
                    iter_of_all_agents[agent_id] = localiter
                    new_results.append(agent_id)

            new_results.append(px)

    ft = time.time()
    if py == 0:  # each master agent
        print('Took time {:.2f}s for agent {} to complete all steps in iteration {}'.format(
        ft-tn1, px, iteration
    ))
    return

"""
OUTPUT

px,py is 0 0
in window creating loop for agent 0
px,py is 1 0
in window creating loop for agent 1
px,py is 2 0
in window creating loop for agent 2
px,py is 3 0
in window creating loop for agent 3
time for this iteration for agent 0 is 0.0
time for this iteration for agent 3 is 0.0
time for this iteration for agent 2 is 0.0
time for this iteration for agent 1 is 0.0
time for exclusive lock 0.0 for agent 0
forced barrier
time for exclusive lock 0.0 for agent 2
forced barrier
time for exclusive lock 0.0 for agent 3
forced barrier
time for exclusive lock 0.0 for agent 1
forced barrier
localiter is [1] agent_id is 1 and current agent is 0 timetaken 0.0
localiter is [1] agent_id is 0 and current agent is 1 timetaken 38.04
localiter is [1] agent_id is 0 and current agent is 2 timetaken 38.04
localiter is [1] agent_id is 1 and current agent is 2 timetaken 0.0
localiter is [1] agent_id is 2 and current agent is 0 timetaken 0.0
localiter is [1] agent_id is 2 and current agent is 1 timetaken 0.0
localiter is [1] agent_id is 0 and current agent is 3 timetaken 38.04
localiter is [1] agent_id is 3 and current agent is 2 timetaken 0.0
Took time 40.04s for agent 2 to complete all steps in iteration 1
localiter is [1] agent_id is 1 and current agent is 3 timetaken 0.0
localiter is [1] agent_id is 3 and current agent is 0 timetaken 0.0
Took time 40.04s for agent 0 to complete all steps in iteration 1
localiter is [1] agent_id is 3 and current agent is 1 timetaken 0.0
Took time 40.04s for agent 1 to complete all steps in iteration 1
localiter is [1] agent_id is 2 and current agent is 3 timetaken 0.01
Took time 40.05s for agent 3 to complete all steps in iteration 1
"""