Custom python module bootstrapping .so file intead of .dylib

Question

I'm making a custom Python 3.7 module compiled with CMake (on OSX). It compiles the module to a dylib just fine, however when I run this command:

python3.7 setup.py install

The bootstrapping file that is generated in the egg file looks like this:

def __bootstrap__():
    global __bootstrap__, __loader__, __file__
    import sys, pkg_resources, imp
    __file__ = pkg_resources.resource_filename(__name__, 'spexi.cpython-37m-darwin.so')
    __loader__ = None; del __bootstrap__, __loader__
    imp.load_dynamic(__name__,__file__)
__bootstrap__()

However, the file spexi.cpython-37m-darwin.so does not exist, it should read spexi.dylib instead. If I manually change it, I appear to be able to run the code just fine.

The setup.py script looks like this right now: https://pastebin.com/sMWnJRdn

How do I get the bootstrapping script to generate with a correct reference to the dylib file?

Here is the output of running the setup.py script:

➜  spexi-gis-py python3.7 setup.py install
running install
running bdist_egg
running egg_info
writing spexi.egg-info/PKG-INFO
writing dependency_links to spexi.egg-info/dependency_links.txt
writing top-level names to spexi.egg-info/top_level.txt
reading manifest file 'spexi.egg-info/SOURCES.txt'
writing manifest file 'spexi.egg-info/SOURCES.txt'
installing library code to build/bdist.macosx-10.9-x86_64/egg
running install_lib
running build_ext
cmake /Users/colinbasnett/spexi-gis-py -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=/Users/colinbasnett/spexi-gis-py/build/lib.macosx-10.9-x86_64-3.7 -DCMAKE_BUILD_TYPE=Release
-- The C compiler identification is AppleClang 11.0.0.11000033
-- Check for working C compiler: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc
-- Check for working C compiler: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Found Python: /Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib (found version "3.7") found components: Development
-- Found TIFF: /usr/local/lib/libtiff.dylib (found version "4.0.10")
-- Configuring done
-- Generating done
-- Build files have been written to: /Users/colinbasnett/spexi-gis-py/build/temp.macosx-10.9-x86_64-3.7
cmake --build . --config Release -- -j4
Scanning dependencies of target spexi
[ 50%] Building C object CMakeFiles/spexi.dir/src/vincenty.c.o
[ 50%] Building C object CMakeFiles/spexi.dir/src/dem.c.o
[ 75%] Building C object CMakeFiles/spexi.dir/src/spexi.c.o
[100%] Linking C shared library ../lib.macosx-10.9-x86_64-3.7/spexi.dylib
[100%] Built target spexi
creating build/bdist.macosx-10.9-x86_64
creating build/bdist.macosx-10.9-x86_64/egg
copying build/lib.macosx-10.9-x86_64-3.7/spexi.dylib -> build/bdist.macosx-10.9-x86_64/egg
creating build/bdist.macosx-10.9-x86_64/egg/spexi.cpython-37m-darwin.so
creating stub loader for spexi.cpython-37m-darwin.so
byte-compiling build/bdist.macosx-10.9-x86_64/egg/spexi.py to spexi.cpython-37.pyc
creating build/bdist.macosx-10.9-x86_64/egg/EGG-INFO
copying spexi.egg-info/PKG-INFO -> build/bdist.macosx-10.9-x86_64/egg/EGG-INFO
copying spexi.egg-info/SOURCES.txt -> build/bdist.macosx-10.9-x86_64/egg/EGG-INFO
copying spexi.egg-info/dependency_links.txt -> build/bdist.macosx-10.9-x86_64/egg/EGG-INFO
copying spexi.egg-info/top_level.txt -> build/bdist.macosx-10.9-x86_64/egg/EGG-INFO
writing build/bdist.macosx-10.9-x86_64/egg/EGG-INFO/native_libs.txt
zip_safe flag not set; analyzing archive contents...
__pycache__.spexi.cpython-37: module references __file__
creating 'dist/spexi-1.0.0-py3.7-macosx-10.9-x86_64.egg' and adding 'build/bdist.macosx-10.9-x86_64/egg' to it
removing 'build/bdist.macosx-10.9-x86_64/egg' (and everything under it)
Processing spexi-1.0.0-py3.7-macosx-10.9-x86_64.egg
removing '/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spexi-1.0.0-py3.7-macosx-10.9-x86_64.egg' (and everything under it)
creating /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spexi-1.0.0-py3.7-macosx-10.9-x86_64.egg
Extracting spexi-1.0.0-py3.7-macosx-10.9-x86_64.egg to /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages
spexi 1.0.0 is already the active version in easy-install.pth

Installed /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spexi-1.0.0-py3.7-macosx-10.9-x86_64.egg
Processing dependencies for spexi==1.0.0
Finished processing dependencies for spexi==1.0.0

setup.py

import os
import pathlib
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext as build_ext_orig


class CMakeExtension(Extension):
    def __init__(self, name, cmake_lists_dir='.', **kwa):
        Extension.__init__(self, name, sources=[], **kwa)
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)


class build_ext(build_ext_orig):
    def run(self):
        for ext in self.extensions:
            self.build_cmake(ext)
        super().run()

    def build_cmake(self, ext):
        cwd = pathlib.Path().absolute()

        # these dirs will be created in build_py, so if you don't have
        # any python sources to bundle, the dirs will be missing
        build_temp = pathlib.Path(self.build_temp)
        build_temp.mkdir(parents=True, exist_ok=True)
        extdir = pathlib.Path(self.get_ext_fullpath(ext.name))
        extdir.mkdir(parents=True, exist_ok=True)

        # example of cmake args
        config = 'Debug' if self.debug else 'Release'
        cmake_args = [
            '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(extdir.parent.absolute()),
            '-DCMAKE_BUILD_TYPE=' + config
        ]

        # example of build args
        build_args = [
            '--config', config,
            '--', '-j4'
        ]

        os.chdir(str(build_temp))
        self.spawn(['cmake', str(cwd)] + cmake_args)
        if not self.dry_run:
            self.spawn(['cmake', '--build', '.'] + build_args)
        os.chdir(str(cwd))


def main():
    setup(name="spexi",
          version="1.0.0",
          author="Colin Basnett",
          ext_modules=[CMakeExtension('spexi')],
          package_data={'python_package': ['*.dylib']},
          cmdclass={
              'build_ext': build_ext,
          })


if __name__ == "__main__":
    main()

`CMakeLists.txt`

CMAKE_MINIMUM_REQUIRED(VERSION 3.15)
PROJECT(spexi)

SET(CMAKE_CXX_STANDARD 14)

FIND_PACKAGE(Python 3.5 COMPONENTS Development)
INCLUDE_DIRECTORIES(${Python_INCLUDE_DIRS})
LINK_LIBRARIES(${Python_LIBRARIES})

FIND_PACKAGE(TIFF REQUIRED)
INCLUDE_DIRECTORIES(${TIFF_INCLUDE_DIR})
LINK_LIBRARIES(${TIFF_LIBRARIES})

SET(SRC src/vincenty.h src/data.h src/dem.h src/vincenty.cpp src/dem.cpp src/spexi.cpp src/draw.h src/draw.cpp)

ADD_LIBRARY(spexi SHARED ${SRC})
SET_TARGET_PROPERTIES(
        spexi
        PROPERTIES
        PREFIX ""
        OUTPUT_NAME "spexi"
        LINKER_LANGUAGE "CXX"
)

I think your question is already answered here: https://stackoverflow.com/questions/32757007/distutils-setup-generate-so-and-not-dylib-on-mac-os-x — Grismar, Jan 23 '20 at 01:13
That unfortunately does not solve my problem. I already have it generating a dylib, the issue is the that bootstrap function is pointing to a non-existent `.so` file after install. — Colin Basnett, Jan 23 '20 at 01:26
Can you not just make it a `.so`? See https://stackoverflow.com/questions/2488016/how-to-make-python-load-dylib-on-osx — user228395, Jan 26 '20 at 12:32
@I'L'I CMakeLists.txt posted. As an additional bit of information, I can compile this thing just fine on Linux as-is, it only seems to be an issue on OSX. — Colin Basnett, Jan 27 '20 at 22:22
Could it be a bug with CMake? Which version of CMake are you using? — Anwar Shaikh, Jan 28 '20 at 22:34
Re: “if I manually change it, I appear to be able to run the code just fine.”, manually change what? — l'L'l, Jan 30 '20 at 00:48
Manually change `spexi.cpython-37m-darwin.so` to `spexi.dylib` in the bootstrapping file. — Colin Basnett, Jan 30 '20 at 01:31

score 0 · Answer 1 · answered Jan 29 '20 at 15:41

0

You should be able to fix this by using the following command in CMakeLists.txt:

SET(CMAKE_SHARED_LIBRARY_SUFFIX ".dylib")

answered Jan 29 '20 at 15:41

Dylan

1,335
8
21

The issue is not that a .dylib isn't being created, it's that the bootstrapping file generated by `python setup.py install` does not point to the dylib that was created. – Colin Basnett Jan 29 '20 at 21:09

Custom python module bootstrapping .so file intead of .dylib

setup.py

CMakeLists.txt

1 Answers1

`CMakeLists.txt`