Link against another python C extension

Question

I have two python extensions written in C:

• foo.bar.libbar defined in foo/bar/bar.c
• foo.baz.libbaz defined in foo/baz/baz.c

libbar depends on symbols from libbaz, namely a non-python-exported function called doubleint.

The setup.py for the library looks like this:

from setuptools import setup, find_packages, Extension

setup(
        name="mylib",
        packages=find_packages(),
        ext_modules=[
            Extension("foo.baz.libbaz",
                sources=["foo/baz/baz.c"],
                ),
            Extension("foo.bar.libbar",
                sources=["foo/bar/bar.c"],
                include_dirs=['foo/baz']
                ),
            ],
        )

Then there is the header files:

foo/baz/baz.h:

int doubleint(int a);

and the two C files: foo/baz/baz.c:

#define PY_SSIZE_T_CLEAN
#include <Python.h>

#include "baz.h"

int doubleint(int a){
    return a * 2;
}

static PyObject *myfunction(PyObject *self, PyObject *args){
    printf("inside python baz\n");
    printf("double of %d is %d\n", 2, doubleint(2));
    Py_RETURN_NONE;
}

static PyMethodDef methods[] = {
    {"myfunction", myfunction, METH_VARARGS, "does stuff"},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
        PyModuleDef_HEAD_INIT, "libbaz", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_libbaz(void) {
    return PyModule_Create(&module);
}

foo/bar/bar.c:

#define PY_SSIZE_T_CLEAN
#include <Python.h>

#include "baz.h"

static PyObject *otherfunction(PyObject *self, PyObject *args){
    printf("inside python bar\n");
    printf("double of %d is %d\n", 2, doubleint(2));
    Py_RETURN_NONE;
}

static PyMethodDef methods[] = {
    {"otherfunction", otherfunction, METH_VARARGS, "does same stuff"},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
        PyModuleDef_HEAD_INIT, "libbar", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_libbar(void) {
    return PyModule_Create(&module);
}

If I now call python setup.py build, everthing builds fine, there are no compile errors:

running build
running build_py
creating build
creating build/lib.linux-x86_64-3.5
creating build/lib.linux-x86_64-3.5/foo
copying foo/__init__.py -> build/lib.linux-x86_64-3.5/foo
creating build/lib.linux-x86_64-3.5/foo/baz
copying foo/baz/__init__.py -> build/lib.linux-x86_64-3.5/foo/baz
creating build/lib.linux-x86_64-3.5/foo/bar
copying foo/bar/__init__.py -> build/lib.linux-x86_64-3.5/foo/bar
running build_ext
building 'foo.baz.libbaz' extension
creating build/temp.linux-x86_64-3.5
creating build/temp.linux-x86_64-3.5/foo
creating build/temp.linux-x86_64-3.5/foo/baz
x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -g -fdebug-prefix-map=/build/python3.5-3.5.3=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.5m -I/home/user/mytest/venv/include/python3.5m -c foo/baz/baz.c -o build/temp.linux-x86_64-3.5/foo/baz/baz.o
x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.5-3.5.3=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.5/foo/baz/baz.o -o build/lib.linux-x86_64-3.5/foo/baz/libbaz.cpython-35m-x86_64-linux-gnu.so
building 'foo.bar.libbar' extension
creating build/temp.linux-x86_64-3.5/foo/bar
x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -g -fdebug-prefix-map=/build/python3.5-3.5.3=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -Ifoo/baz -I/usr/include/python3.5m -I/home/user/mytest/venv/include/python3.5m -c foo/bar/bar.c -o build/temp.linux-x86_64-3.5/foo/bar/bar.o
x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.5-3.5.3=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.5/foo/bar/bar.o -o build/lib.linux-x86_64-3.5/foo/bar/libbar.cpython-35m-x86_64-linux-gnu.so

Now I can start up an interpreter and load foo.baz.libbaz and call myfunction. But if I want to import foo.bar.libbar I get an error:

Python 3.5.3 (default, Sep 27 2018, 17:25:39)
[GCC 6.3.0 20170516] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> from foo.baz import libbaz
>>> libbaz.myfunction()
inside python baz
double of 2 is 4
>>> from foo.bar import libbar
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ImportError: /home/user/mytest/build/lib.linux-x86_64-3.5/foo/bar/libbar.cpython-35m-x86_64-linux-gnu.so: undefined symbol: doubleint

I can see, that the library is not linked against libbaz:

$ ldd foo/bar/libbar.cpython-35m-x86_64-linux-gnu.so
        linux-vdso.so.1 (0x00007ffd2d821000)
        libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007fb8850a0000)
        libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fb884d01000)
        /lib64/ld-linux-x86-64.so.2 (0x00007fb8854bf000)

I can work around this problem my adding foo/baz/baz.c to the sources of libbar but I'm not sure if this is the correct way.

I also tried to set runtime_library_dirs=['$ORIGIN/../baz'] which did not helped either. I assume I would have to tell the linker to link against the library, but how do you specify this? I tried many library names like libbaz, baz, foo.baz.libbaz, foo.baz.baz but none of them work with ld, as it can not find the library.

I read this answer https://stackoverflow.com/a/3270302/446140 to a similar question which says you should create a separate import library, but how would this work in the context of Extension then? How could I build the library first and then link it against the extensions?