Build a python package with setup.py in CMake

Question

EDIT: The question is a bit too long. Here is my real question: How can I build and install a python package with setuptools (setup.py) inside CMake? The detail of my code is shown below (but with an out-of-source build method, the method with the source is working).

I have a project where I need to distribute my own python package. I made a setup.py script but I would like to build & install it with CMake.

I followed Using CMake with setup.py but it only works with one CMakeLists.txt alongside the setup.py and the python folder and without executing cmake from a build directory.

With this layout :

Project/
--build/
--lib/
----python/
------folder1/
------folder2/
------data/
------...
------__init__.py
----setup.py
----CMakeLists.txt
--CMakeLists.txt

and with CMakeLists.txt:

cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR)
add_subdirectory(lib)
(..)

and with lib/CMakeLists.txt:

find_program(PYTHON "python")

if (PYTHON)
    set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py")
    set(SETUP_PY    "${CMAKE_CURRENT_BINARY_DIR}/setup.py")
    set(DEPS        "${CMAKE_CURRENT_SOURCE_DIR}/python/__init__.py")
    set(OUTPUT      "${CMAKE_CURRENT_BINARY_DIR}/build")

    configure_file(${SETUP_PY_IN} ${SETUP_PY})

    add_custom_command(OUTPUT ${OUTPUT}
                       COMMAND ${PYTHON}
                       ARGS setup.py build
                       DEPENDS ${DEPS})

    add_custom_target(target ALL DEPENDS ${OUTPUT})

    install(CODE "execute_process(COMMAND ${PYTHON} ${SETUP_PY} install)")
endif()

and with setup.py:

from setuptools import setup, find_packages

setup(name="python",
    version="xx",
    author="xx",
    packages = find_packages(),
    package_data = {'': ['*.txt']},
    description="Python lib for xx")

When I run CMake from build directory and then make, the target is built but with nothing. It is as if no packages were found. The installation installs the python package without .py files.

Have you tried setting the WORKING_DIRECTORY in add_custom_command? — cmorse, May 29 '15 at 20:44
You should use all the paths relative to the current source directory. Put a line such as `DIR = os.path.dirname(os.path.realpath(__file__))` and then make all packages relative to that dir so it builds out of source. — Alex Huszagh, Nov 03 '16 at 06:21

score 1 · Answer 1 · answered Jun 14 '16 at 12:10

setuptools doesn't know about the out of source build and therefore doesn't find any python source files (because you do not copy them to the binary dir, only the setup.py file seems to exist there). In order to fix this, you would have to copy the python source tree into the CMAKE_CURRENT_BINARY_DIR.

score 1 · Answer 2 · answered Jan 18 '17 at 20:57

1

https://bloerg.net/2012/11/10/cmake-and-distutils.html suggests setting package_dir to ${CMAKE_CURRENT_SOURCE_DIR} in setup.py.

answered Jan 18 '17 at 20:57

Tudor Bosman

61
1
5

score 1 · Answer 3 · answered Sep 21 '21 at 19:51

As pointed out previously you can copy your python files to the build folder, e.g. something like this

set(TARGET_NAME YourLib)
file(GLOB_RECURSE pyfiles python/*.py)
foreach (filename ${pyfiles})
    get_filename_component(target "${filename}" NAME)
    message(STATUS "Copying ${filename} to ${TARGET_NAME}/${target}")
    configure_file("${filename}" 
    "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}/${target}" COPYONLY)
endforeach (filename)

and then have a build target like this

add_custom_target(PyPackageBuild
        COMMAND "${PYTHON_EXECUTABLE}" -m pip wheel .
        WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
        COMMENT "Building python wheel package"
        )
add_dependencies(PyPackageBuild ${TARGET_NAME})

In case you do not want to use pip you have to adjust the PyPackageBuld target.

If you want to include some shared library, e.g. written in C++, which is build by other parts of your cmake project you have to copy the shared object file as well to the binary folder

set_target_properties(${TARGET_NAME} PROPERTIES
        PREFIX "${PYTHON_MODULE_PREFIX}"
        SUFFIX "${PYTHON_MODULE_EXTENSION}"
        BUILD_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}/libs"
        BUILD_WITH_INSTALL_RPATH TRUE)
set(TARGET_PYMODULE_NAME "${PYTHON_MODULE_PREFIX}${TARGET_NAME}${PYTHON_MODULE_EXTENSION}")

and add it to the package_data in setup.py

....
package_data={
        '': ['libs/YourLib.cpython-39-x86_64-linux-gnu.so']
    }

You can find a working example using pybind11 for `C++´ bindings at https://github.com/maximiliank/cmake_python_r_example

The cython version, archticture args in the shared library name in the `setup.py` can be obtained via `import sysconfig` and `f'libs/YourLib{sysconfig.get_config_var("EXT_SUFFIX")}'`. — Max, Nov 18 '21 at 23:24

Build a python package with setup.py in CMake

3 Answers3