CMakeLists.txt

cmake_minimum_required(VERSION 3.17)
project(dm)

#--- MPI -------------------------------------------------------------------
find_package(MPI)
# Need this to comply with CMP004 policy:
string(STRIP "${MPI_CXX_LINK_FLAGS}" MPI_CXX_LINK_FLAGS)
string(STRIP "${MPI_CXX_LIBRARIES}" MPI_CXX_LIBRARIES)

add_library(mpi_target INTERFACE)
target_link_libraries(mpi_target INTERFACE ${MPI_CXX_LIBRARIES})
target_include_directories(mpi_target INTERFACE ${MPI_CXX_INCLUDE_PATH})
target_compile_options(mpi_target INTERFACE
    $<$<CXX_COMPILER_ID:GNU>:-Wno-long-long>
    $<$<CXX_COMPILER_ID:GNU>:-Wno-literal-suffix>
    $<$<CXX_COMPILER_ID:Clang>:-Wno-long-long>
    )

#--- AMGCL -----------------------------------------------------------------
find_package(amgcl)

#---------------------------------------------------------------------------
add_executable(dm dm.cpp)
target_link_libraries(dm amgcl::amgcl mpi_target)

dm.cpp

#include <iostream>
#include <vector>

#include <amgcl/backend/builtin.hpp>
#include <amgcl/adapter/zero_copy.hpp>
#include <amgcl/mpi/distributed_matrix.hpp>

/*
 *       0  1    2  3
 *   -|-------|--------
 * 0 [|  1 -1 |  0  0 ]
 * 1 [| -1  2 | -1  0 ]
 *   [|-------|-------]
 * 2 [|  0 -1 |  2 -1 ]
 * 3 [|  0  0 | -1  1 ]
 */
int main(int argc, char *argv[]) {
    MPI_Init(&argc, &argv);

    amgcl::mpi::communicator comm(MPI_COMM_WORLD);

    ptrdiff_t chunk = 0;
    std::vector<size_t> loc_ptr, loc_col, rem_ptr, rem_col, z_ptr = {0, 0, 0}, z_col;
    std::vector<double> loc_val, rem_val, z_val, x, y;

    if (comm.rank == 0) {
        chunk = 2;

        loc_ptr = {0, 2, 4};
        loc_col = {0, 1, 0, 1};
        loc_val = {1.0, -1.0, -1.0, 2.0};

        rem_ptr = {0, 0, 1};
        rem_col = {2};
        rem_val = {-1.0};

        x = {1.0, 1.0};
        y = {0.0, 0.0};
    } else if (comm.rank == 1) {
        chunk = 2;

        loc_ptr = {0, 2, 4};
        loc_col = {0, 1, 0, 1};
        loc_val = {2.0, -1.0, -1.0, 1.0};

        rem_ptr = {0, 1, 1};
        rem_col = {1};
        rem_val = {-1.0};

        x = {1.0, 1.0};
        y = {0.0, 0.0};
    }

    auto loc_a = amgcl::adapter::zero_copy(chunk, loc_ptr.data(), loc_col.data(), loc_val.data());
    auto rem_a = amgcl::adapter::zero_copy(chunk, rem_ptr.data(), rem_col.data(), rem_val.data());
    rem_a->ncols = 1;

    amgcl::mpi::distributed_matrix<amgcl::backend::builtin<double>> A(comm, loc_a, rem_a);

    auto loc_b = amgcl::adapter::zero_copy(chunk, loc_ptr.data(), loc_col.data(), loc_val.data());
    auto rem_b = amgcl::adapter::zero_copy(chunk, z_ptr.data(), z_col.data(), z_val.data());
    rem_b->ncols = 0;

    amgcl::mpi::distributed_matrix<amgcl::backend::builtin<double>> B(comm, loc_b, rem_b);

    auto Bt = amgcl::mpi::transpose(B);

    auto C = amgcl::mpi::product(A, *Bt);

    C->move_to_backend();
    amgcl::backend::spmv(1.0, *C, x, 0.0, y);

    for(int i = 0; i < comm.size; ++i) {
        if (comm.rank == i) {
            for(auto v : y) std::cout << v << std::endl;
        }
        MPI_Barrier(comm);
    }

    MPI_Finalize();
}