-
-
Save edbennett/1a56737e2710e101ed9eee70100955ee to your computer and use it in GitHub Desktop.
This vomits uninitialised halo data all over my nice array bulk
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module params | |
implicit none | |
save | |
! Type definitions | |
integer, parameter :: dp=kind(1.d0) | |
! Lattice parameters | |
integer, parameter :: ksize=12, ksizet=12 | |
integer, parameter :: kthird=24 | |
integer, parameter :: np_x=2, np_y=1, np_t=1 | |
integer, parameter :: ksizex_l = ksize / np_x | |
integer, parameter :: ksizey_l = ksize / np_y | |
integer, parameter :: ksizet_l = ksizet / np_t | |
end module params | |
module data | |
use params | |
implicit none | |
save | |
complex(dp) :: test_array(kthird, 0:ksizex_l+1, 0:ksizey_l+1, 0:ksizet_l+1, 4, 12) | |
end module data | |
pure function pid(ix, iy, it) result(id) | |
use params | |
implicit none | |
integer, intent(in) :: ix, iy, it | |
integer :: id | |
id = ix + iy * np_x + it * np_x * np_y | |
end function pid | |
module comms | |
use params | |
use mpi_f08 | |
implicit none | |
save | |
integer :: ip_x, ip_y, ip_t, ip_global, np_global | |
type(MPI_Comm) :: comm | |
type(MPI_Datatype) :: mpiio_type | |
contains | |
subroutine start_halo_update_6_x_lower(size5, size6, Array, tag, reqs) | |
! | |
integer, intent(in) :: size5, size6, tag | |
complex(dp), intent(inout) :: Array(kthird, 0:ksizex_l+1, 0:ksizey_l+1, & | |
& 0:ksizet_l+1, size5, size6) | |
type(MPI_Request), intent(out) :: reqs(2) | |
integer :: ip_xup, ip_xdn | |
integer :: tag_offset, size | |
tag_offset = 6 * tag | |
call MPI_Cart_Shift(comm, 0, 1, ip_xdn, ip_xup) | |
size = kthird * size5 * size6 * ksizey_l * ksizet_l | |
call MPI_Isend(Array(1:kthird, ksizex_l, 1:ksizey_l, 1:ksizet_l, 1:size5, 1:size6), size, MPI_Double_Complex, ip_xup, 0 + tag_offset, comm, reqs(1)) | |
call MPI_Irecv(Array(1:kthird, 0, 1:ksizey_l, 1:ksizet_l, 1:size5, 1:size6), size, MPI_Double_Complex, ip_xdn, 0 + tag_offset, comm, reqs(2)) | |
end subroutine start_halo_update_6_x_lower | |
subroutine complete_halo_update(reqs) | |
type(MPI_Request), intent(inout) :: reqs(2) | |
call MPI_Waitall(2, reqs, MPI_Statuses_Ignore) | |
end subroutine complete_halo_update | |
!*********************************************************************** | |
! Initialise MPI variables | |
!*********************************************************************** | |
subroutine init_MPI() | |
integer :: coords(3) | |
call MPI_init | |
! Check that we have the right number of processes | |
call MPI_comm_size(MPI_COMM_WORLD, np_global) | |
call MPI_comm_rank(MPI_COMM_WORLD, ip_global) | |
if (np_global .ne. NP_X * NP_Y * NP_T) then | |
print *,"MPI dimensionality mismatch: ", NP_X, "*", NP_Y, "*", NP_T, "!=", np_global | |
call MPI_finalize | |
call exit(2) | |
end if | |
! Set up a Cartesian communicator; periodic boundaries, allow reordering | |
call MPI_cart_create(MPI_COMM_WORLD, 3, (/ NP_X, NP_Y, NP_T /), & | |
& (/ .true., .true., .true. /), .true., comm) | |
! Know where I am | |
call MPI_cart_coords(comm, ip_global, 3, coords) | |
ip_x = coords(1) | |
ip_y = coords(2) | |
ip_t = coords(3) | |
return | |
end subroutine init_MPI | |
end module comms | |
program test_halo_6 | |
use params | |
use data | |
use comms | |
implicit none | |
integer :: ithird, ix, iy, it, i5, i6, i=0 | |
integer :: pid | |
type(MPI_Request) :: reqs(2) | |
call init_MPI | |
test_array(:,0,:,:,:,:) = cmplx(-1,-1) | |
test_array(:,:,0,:,:,:) = cmplx(-2,-1) | |
test_array(:,:,:,0,:,:) = cmplx(-3,-1) | |
test_array(:,ksizex_l+1,:,:,:,:) = cmplx(-1,-2) | |
test_array(:,:,ksizey_l+1,:,:,:) = cmplx(-2,-2) | |
test_array(:,:,:,ksizet_l+1,:,:) = cmplx(-3,-2) | |
! Set up local arrays | |
do i6=1,12 | |
do i5=1,4 | |
do it=1,ksizet_l | |
do iy=1,ksizey_l | |
do ix=1,ksizex_l | |
do ithird=1,kthird | |
i = i + 1 | |
test_array(ithird, ix, iy, it, i5, i6) = cmplx(i, pid(ip_x, ip_y, ip_t)) | |
end do | |
end do | |
end do | |
end do | |
end do | |
end do | |
! Communicate | |
call start_halo_update_6_x_lower(4, 12, test_array, 0, reqs) | |
call complete_halo_update(reqs) | |
! Check output | |
if (ip_global .eq. 0) then | |
do i6=1,12 | |
do i5=1,4 | |
do it=1,ksizet_l | |
do iy=1,ksizey_l | |
do ix=1,ksizex_l | |
do ithird=1,kthird | |
if (real(test_array(ithird, ix, iy, it, i5, i6)) .lt. 0) then | |
print *, 'Corruption of array bulk at: ', ithird, ix, iy, it, i5, i6, ' - value: ', test_array(ithird, ix, iy, it, i5, i6) | |
end if | |
end do | |
end do | |
end do | |
end do | |
end do | |
end do | |
end if | |
call MPI_Finalize | |
end program test_halo_6 | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment