Skip to content

Instantly share code, notes, and snippets.

@hovren
Created December 12, 2017 15:35
Show Gist options
  • Save hovren/5b62175731433c741d07ee6f482e2936 to your computer and use it in GitHub Desktop.
Save hovren/5b62175731433c741d07ee6f482e2936 to your computer and use it in GitHub Desktop.
pybind11 with CMake and setup.py

This shows how to invoke cmake from setup.py and put the generated .so-files where they belong. The initial CMakeBuild and CMakeExtension idea comes from http://www.benjack.io/2017/06/12/python-cpp-tests.html who should be credited accordingly!

Basic operations

  1. CMakeLists.txt defines pybind11 targets using the helper function
  2. On setup.py install|build|develop the CMakeBuild class first instructs cmake to configure which sets the appropriate python paths.
  3. It then instruct cmake to build all targets
  4. The extensions listed in setup.py ext_modules= are then moved from the build directory to its final destination.

Caveats

  1. Tested using both setup.py develop and setup.py install and seems to work, but there is not much testing done.
  2. All targets are built, not just those specified in ext_modules.
  3. Not much documentation, as is obvious :)
# Build a Python extension module using pybind11
# pybindings_add_module(<module>)
# Here <module> should be the fully qualified name for the module,
# e.g. pybindings_add_module(foo.bar._baz)
# <module> becomes the target name in case you wish to do something to it later
# The source for the binding *must* be placed in src/pybindings/{relpath}/py{name}.cc
# E.g. for module=foo.bar._baz -> src/pybindings/bar/py_baz.cc
function(pybindings_add_module module)
set(target_name ${module})
string(REPLACE "." "/" modpath ${module})
string(REPLACE "." ";" modlist ${module})
# The module name is the last entry
list(GET modlist -1 modname)
# Remove everything that is not the root or the module name
#list(REMOVE_AT modlist 0)
list(REMOVE_AT modlist -1)
# Get the relative path
if(modlist)
string(REPLACE ";" "/" relpath "${modlist}")
else()
set(relpath "")
endif()
# Define the binding source file
set(sources src/pybindings/${relpath}/py${modname}.cc)
# Invoke pybind11 and set where the library should go, and what it is called
pybind11_add_module(${target_name} ${sources})
set(outdir ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${relpath})
set_target_properties(${target_name} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${outdir})
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${modname})
endfunction()
import os
import re
import sys
import sysconfig
import platform
import subprocess
from pathlib import Path
from distutils.version import LooseVersion
from setuptools import setup, Extension, find_packages
from setuptools.command.build_ext import build_ext
from setuptools.command.test import test as TestCommand
class CMakeExtension(Extension):
def __init__(self, name):
Extension.__init__(self, name, sources=[])
class CMakeBuild(build_ext):
def run(self):
try:
out = subprocess.check_output(['cmake', '--version'])
except OSError:
raise RuntimeError(
"CMake must be installed to build the following extensions: " +
", ".join(e.name for e in self.extensions))
build_directory = os.path.abspath(self.build_temp)
cmake_args = [
'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + build_directory,
'-DPYTHON_EXECUTABLE=' + sys.executable
]
cfg = 'Debug' if self.debug else 'Release'
build_args = ['--config', cfg]
cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
# Assuming Makefiles
build_args += ['--', '-j2']
self.build_args = build_args
env = os.environ.copy()
env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(
env.get('CXXFLAGS', ''),
self.distribution.get_version())
if not os.path.exists(self.build_temp):
os.makedirs(self.build_temp)
# CMakeLists.txt is in the same directory as this setup.py file
cmake_list_dir = os.path.abspath(os.path.dirname(__file__))
print('-'*10, 'Running CMake prepare', '-'*40)
subprocess.check_call(['cmake', cmake_list_dir] + cmake_args,
cwd=self.build_temp, env=env)
print('-'*10, 'Building extensions', '-'*40)
cmake_cmd = ['cmake', '--build', '.'] + self.build_args
subprocess.check_call(cmake_cmd,
cwd=self.build_temp)
# Move from build temp to final position
for ext in self.extensions:
self.move_output(ext)
def move_output(self, ext):
build_temp = Path(self.build_temp).resolve()
dest_path = Path(self.get_ext_fullpath(ext.name)).resolve()
source_path = build_temp / self.get_ext_filename(ext.name)
dest_directory = dest_path.parents[0]
dest_directory.mkdir(parents=True, exist_ok=True)
self.copy_file(source_path, dest_path)
ext_modules = [
CMakeExtension('foo.bar._baz'),
CMakeExtension('foo.spam._ham')
]
setup(
# ...
packages=find_packages(),
ext_modules=ext_modules,
cmdclass=dict(build_ext=CMakeBuild),
zip_safe=False,
)
@SanPen
Copy link

SanPen commented Sep 29, 2020

How would you add the numpy includes in the setup.py file ?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment