Exposing raw C++ buffers as NumPy arrays using Pybind11

__init__.py

# Python side
from . import _testModule

class namespace:
    pass

def getNDArrayFromBuffer(buffer_adrs,shape,typestr='<f4'):
    b=namespace()
    b.__array_interface__={
         'data': (buffer_adrs, False),
         'descr': [('', typestr)],
         'shape': shape,
         'strides': None,
         'typestr': typestr,
         'version': 3}
    return np.asarray(b)


class dataHolder(_testModule.dataHolder):
    def __init__(self,data_size):
        #call the c++ constructor
        _pyWormSim.worm.__init__(self,data_size)
        self.size=data_size
        
        # here we create ndarray views of the internal buffers of the class
        # Changes in the ndarray's ELEMENTS will be visible to the cpp side
        # however changes to the ndarray itself will destroy the link.
        self.PrivateData = getNDArrayFromBuffer(self._PrivateData,(self.size,),'<f8') # double 
        self.PublicData = getNDArrayFromBuffer(self._PublicData,(self.size,),'<f4') # float  
        self.myarray = getNDArrayFromBuffer(self._myarray,(2,3),'<i4') # int

Exposing raw C++ buffers as NumPy arrays using Pybind11.md

Exposing raw C++ buffers as NumPy arrays using Pybind11

Sometimes you have to work with C++ libraries that use raw pointers and arrays that you need to have access to from python. Indeed, working with C styles arrays is sometimes considered bad practice but many people still do...

The solution here is for the C++ side to provide the raw memory address of the data buffer, and then use numpy's array interface mechanism to create an ndarray that uses this buffer.

This allows the user to view and change elements of the buffers with no copies in a way that is visible to both C++ and Python sides.

WARNING: Working just with the memory address removes all the information about the buffer (type, size, etc.) Thus, one have to be extremely careful when accesing the buffer.

testModule.cpp

// C++ side
#include <iostream>
#include <pybind11/pybind11.h>

/*
We define a simple class that holds a few data buffers of a given size with some values.
*/
class dataHolder
{
public:
    dataHolder(int array_size) : size(array_size) {
        private_data = new double[size];
        public_data = new float[size];
                
        for (auto i=0; i<size; ++i){
            private_data[i]=(float)(i);
            public_data[i] = float(i*2.1);
        }
    }
  
    ~dataHolder(){
        delete private_data;
        delete public_data;
    }
    float * getPrivateData() const{
        return private_data;
    }
    
    void print(){
        std::cout<<"private_data: ";
        for (auto i=0; i<size; std::cout<<private_data[i++]<<" ");
        std::cout<<std::endl;
        std::cout<<"public_data: ";
        for (auto i=0; i<size; std::cout<<public_data[i++]<<" ");
        std::cout<<std::endl;
        std::cout<<"myarray: "<<std::endl;
        for (auto i =0; i<2; ++i){
            for (auto j=0; j<3; ++j)
                std::cout<<myarray[i][j]<<" ";
            std::cout<<std::endl;
        }
    }
    
    float * public_data;
    int myarray[2][3] = {{1,2,3},{4,5,6}};
private:
    double * private_data;
    int size;
};

namespace py = pybind11;

template<class T, typename buffer_T> 
std::function<unsigned long (T const &)> expose_private_buffer(buffer_T * (T::*fptr) () const){
    return [fptr](T const & a) {return (unsigned long)((a.*fptr)());};
} 

template<class T, typename buffer_T> 
std::function<unsigned long (T &)> expose_public_buffer(buffer_T T::*ptr) {
    return [ptr](T & a) {return (unsigned long)(a.*ptr);};
} 

template<class T, typename buffer_T> 
std::function<unsigned long (T &)> expose_public_ref_to_buffer(buffer_T T::*ptr) {
    return [ptr](T & a) {return (unsigned long)(&(a.*ptr));};
} 


PYBIND11_MODULE(_testModule,m) {	
        py::class_<dataHolder>(m,"dataHolder")
            .def(py::init<int>())
            .def("print",&dataHolder::print)
            .def_property_readonly("_PrivateData",expose_buffer(&dataHolder::getPrivateData))
            .def_property_readonly("_PublicData",expose_public_buffer(&dataHolder::public_data))
            .def_property_readonly("_myarray",expose_public_ref_to_buffer(&dataHolder::myarray))
         ;
 }