Passing by reference to the embedded python interpreter with pybind11

CMakeLists.txt

cmake_minimum_required(VERSION 3.15...3.22)
project(passByReference LANGUAGES CXX)
find_package(Python COMPONENTS Interpreter Development REQUIRED)
include(FetchContent)
FetchContent_Declare(
    Catch2
    GIT_SHALLOW    TRUE
    GIT_REPOSITORY https://github.com/catchorg/Catch2.git
    GIT_TAG        v3.3.2)
FetchContent_MakeAvailable(Catch2)
FetchContent_Declare(
    pybind11
    GIT_SHALLOW    TRUE
    GIT_REPOSITORY https://github.com/pybind/pybind11
    GIT_TAG        v2.10.4
)
FetchContent_MakeAvailable(pybind11)

include(CTest)
include(Catch)

enable_testing()

add_executable(passByReferenceTest passByReference.cpp)
target_link_libraries(passByReferenceTest PRIVATE
    Catch2::Catch2WithMain
    pybind11::embed)

catch_discover_tests(passByReferenceTest)

# ```bash
# mkdir build ; cd build
# cmake ../ . -DCMAKE_BUILD_TYPE="Release"
# cmake --build . && ctest
# ```

passByReference.cpp

#include <fstream> //used to have everithing in one file

#include <catch2/catch_template_test_macros.hpp>
#include <catch2/catch_test_macros.hpp>
#include <catch2/generators/catch_generators.hpp>
#include <catch2/generators/catch_generators_adapters.hpp>
#include <catch2/generators/catch_generators_random.hpp>

#include <pybind11/embed.h>

namespace mylib {
class SimpleIO {
  int i = 0;

public:
#ifdef _DELETECPYCTOR
  SimpleIO() = default;
  SimpleIO(const SimpleIO &) = delete;
#endif //_DELETECPYCTOR
  int getI() const { return i; }
  void setI(int newI) { i = newI; }
};
} // namespace mylib

namespace py = pybind11;

PYBIND11_EMBEDDED_MODULE(embeddedMylib, m) {
  m.doc() = "mylib"; // optional module docstring

  pybind11::class_<mylib::SimpleIO>(m, "SimpleIO")
      .def(py::init<>(), py::return_value_policy::reference)
      .def("getI",
           &mylib::SimpleIO::getI) //, py::return_value_policy::reference)
      .def("setI",
           &mylib::SimpleIO::setI) //, py::return_value_policy::reference)
      .def_property("I", &mylib::SimpleIO::getI, &mylib::SimpleIO::setI
                    //,py::return_value_policy::reference
      );
}

constexpr auto externalModule = R"(
import embeddedMylib
def parameter(objectToSet:embeddedMylib.SimpleIO, value:int):
    toret = objectToSet.I
    objectToSet.I = value
    toret2 = objectToSet.I
    return toret, toret2

def getterSetter(objectToSet:embeddedMylib.SimpleIO, value:int):
    toret = objectToSet.getI()
    objectToSet.setI(value)
    toret2 = objectToSet.getI()
    return toret, toret2

)";

// this fails with "return_value_policy = copy, but type is non-copyable!"
// when the copy constructor of SimpleIO is deleted
SCENARIO("Directly pass the C++ object") {
  GIVEN("The C++ object") {
    // this is here to make sure that the py::scoped_interpreter goes out of
    // scope
    mylib::SimpleIO mytest;
    // set to not overlap
    auto exampleValue = GENERATE(take(10, random(0, 100)));
    auto setValue = GENERATE(take(10, random(1000, 1100)));
    AND_GIVEN(
        "The embedded interpreter the interface module and a function that "
        "uses the module") {
      {
        std::ofstream f("passbyReference.py");
        f << externalModule;
      }
      py::scoped_interpreter guard{};
      auto embeddedMylib = py::module_::import("embeddedMylib");
      auto py_module = py::module::import("passbyReference");
      auto funcName = GENERATE("parameter", "getterSetter");
      WHEN("We load a function that calls " << funcName) {
        auto fn = py_module.attr(funcName);
        mytest.setI(exampleValue);
        THEN("Setting and getting the values in python should not affect the "
             "C++ object") {
          // the name of the variables help reading the tests errors
          int prevInCpp = mytest.getI();
          auto retInPy = fn(mytest, setValue).cast<py::tuple>();
          int prevInPy = retInPy[0].cast<int>();
          int setInPy = retInPy[1].cast<int>();
          REQUIRE(prevInPy == prevInCpp);
          REQUIRE(setInPy == setValue);
          // this will always fail
          // CHECK_FALSE(mytest.getI() == setValue);
          // better test that nothing is changed
          CHECK(mytest.getI() == exampleValue);
        }
      }
    }

    // WHEN("The interpreter goes out of scope") {
    // THEN("The C+ object should survive") {
    CHECK(mytest.getI() == exampleValue);
    //}
    //}
  }
}

SCENARIO("Cast the C++ objet to a python object") {
  GIVEN("The C++ object") {
    // this is here to make sure that the py::scoped_interpreter goes out of
    // scope
    mylib::SimpleIO mytest;
    // set to not overlap
    auto exampleValue = GENERATE(take(10, random(0, 100)));
    auto setValue = GENERATE(take(10, random(1000, 1100)));
    AND_GIVEN(
        "The embedded interpreter the interface module and a function that "
        "uses the module") {
      {
        std::ofstream f("passbyReference.py");
        f << externalModule;
      }
      py::scoped_interpreter guard{};
      auto embeddedMylib = py::module_::import("embeddedMylib");
      auto py_module = py::module::import("passbyReference");
      auto funcName = GENERATE("parameter", "getterSetter");
      WHEN("We load a function that calls " << funcName) {
        auto fn = py_module.attr(funcName);
        mytest.setI(exampleValue);
        // py::object obj =
        // we cast to python
        auto obj = py::cast(&mytest);
        THEN("Setting and getting the values in python should not affect the "
             "C++ object") {
          // the name of the variables help reading the tests errors
          int prevInCpp = mytest.getI();
          auto retInPy = fn(mytest, setValue).cast<py::tuple>();
          int prevInPy = retInPy[0].cast<int>();
          int setInPy = retInPy[1].cast<int>();
          REQUIRE(prevInPy == prevInCpp);
          REQUIRE(setInPy == setValue);

          CHECK(mytest.getI() == setValue);
        }
      }
    }
    // WHEN("The interpreter goes out of scope") {
    // THEN("The C+ object should survive") {
    // the set happened in python:
    CHECK(mytest.getI() == setValue);
    // }
    // }
  }
}

run.sh

#!/bin/bash
mkdir build ; cd build
cmake ../ -DCMAKE_BUILD_TYPE="Release"
cmake --build . && ctest