2

I am trying to import a python extension coded in c++ with boost. While I experienced some problems compiling the extension using cmake, I managed to do so linked to the boost_python27 library. I then used pythons distutils to install the extension into the python framework.

However, when I try to import the module I get the following error:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ImportError: dlopen(./tools.so, 2): Library not loaded: @rpath/libboost_python.dylib
  Referenced from: /Users/DaniBook/CLionProjects/Uebung1/cmake-build-debug/tools.so
  Reason: image not found

I tried everything I found on the internet including reinstalling boost and recompiling using the distutils extensions etc. to get it to work. Can anybody offer some help on this and might also answer what this ominous image thats missing is?

tools.h

#ifndef UEBUNG1_TOOLS_H
#define UEBUNG1_TOOLS_H

#include <string>
#include <vector>
#include <map>
#include <boost/python.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>

using namespace std;

vector<string> *product(string alphabet, int repeats);

vector<string> *product(vector<string> pools);

vector<string>* hammdist(string &pattern, int distance);

#endif //UEBUNG1_TOOLS_H

tools.cpp

#include <boost/python.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
#include "tools.h"
#include <vector>
#include <string>
#include <map>

using namespace std;

vector<string> *product(string alphabet, int repeats) {
    //initializing vector
    auto *results = new vector<string>();
    for(auto character : alphabet) {
        string tmpstr;
        tmpstr = character;
        results->push_back(tmpstr);
    }

    //cartesian product generation
    for(int i = 1; i < repeats; i++) {
        vector<string> tmp = *results;
        results->clear();

        //iterating over temporary list adding elements from pool to each contained string
        for(auto &it : tmp) {
            for(auto &character : alphabet) {
                results->push_back(it + character);
            }
        }
    }
    return results;
}

vector<string> product(vector<string> pools) {
    //initializing vector
    auto *results = new vector<string>();
    for(auto character : pools[0]) {
        string tmpstr;
        tmpstr = character;
        results->push_back(tmpstr);
    }

    //removing the first pool container
    pools.erase(pools.begin());

    //cartesian product generation
    for(const auto &pool : pools) {
        vector<string> tmp = *results;
        results->clear();

        //iterating over temporary list adding elements from pool to each contained string
        for(auto &it : tmp) {
            for(auto character : pool) {
                results->push_back(it + character);
            }
        }
    }
    return results;
}


vector<string>* hammdist(string &pattern, int distance) {
    map<char, string> possibles = {
            {'A', "CGT"},
            {'C', "AGT"},
            {'G', "ACT"},
            {'T', "ACG"}
    };
    auto *results = new vector<string>();
    vector<string> *masks = product("01", pattern.size());
    for(auto &mask : *masks) {
        auto *permute = new vector<string>();
        auto *tmp = new vector<string>();
        if(count(mask.begin(), mask.end(), '1') == distance) {
            for(int i = 0; i < pattern.size(); i++) {
                if(mask[i] != '1') {
                    string tmpstr;
                    tmpstr = pattern[i];
                    tmp->push_back(tmpstr);
                }
                else {
                    tmp->push_back(possibles[pattern[i]]);
                }
            }
            permute = product(*tmp);
            results->insert(results->end(), permute->begin(), permute->end());
        }
        delete permute;
        delete tmp;
    }
    return results;
}

/* 
   initializing function pointers in order to tell boost that we have 
   overloaded functions to expose to python
*/
vector<string> *(*product1)(string, int) = &product;
vector<string> *(*product2)(vector<string> pools) = &product;

//include all functions that are used by the function to expose to the BOOST_PYTHON_MODULE call

using namespace boost::python;

BOOST_PYTHON_MODULE(tools) {
    //telling boost_python that we have overloaded functions which need to be called in the respective situations
    //return_value_policy<manage_new_objects> is required in order for the interface to handle the new invocation and the returned pointer

    def("product", product1, return_value_policy<manage_new_object>());
    def("product", product2, return_value_policy<manage_new_object>());

    def("hammdist", hammdist, return_value_policy<manage_new_object>());

    //vector_indexing_suite handles the wrapping of vector member functions
    //enables handling vector in a pythonic way when using in python

    class_<std::vector<string>>("string_vector")
        .def(vector_indexing_suite<std::vector<string>>());
}

CMakeLists.txt

make_minimum_required(VERSION 3.12)
project(tools)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_INCLUDE_CURRENT_DIR ON)

if(APPLE)
    set(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
endif(APPLE)

find_package(PythonLibs 2.7 REQUIRED)
include_directories(${PYTHON_INCLUDE_DIRS})

set(PROJECT_SOURCE_DIR src/)
include_directories(${PROJECT_SOURCE_DIR})

set(BOOST_ROOT "/Users/DaniBook/miniconda3/pkgs/boost-1.66.0-py27_1")
set(BOOST_LIBRARYDIR "/Users/DaniBook/miniconda3/pkgs/boost-1.66.0-27_1/lib")

find_package(Boost COMPONENTS python REQUIRED)
include_directories(${Boost_INCLUDE_DIRS})

add_library(tools SHARED src/tools.cpp src/tools.h)
target_link_libraries(tools ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})
set_target_properties(tools PROPERTIES PREFIX "")

setup.py

from distutils.core import setup

setup(
    name = 'tools',
    version = '0.1',
    py_modules = ['tools'])
mnmldani
  • 53
  • 6

1 Answers1

2

Okay after some additional research I came to a very interesting conclusion, which, when you look at the error message, is quite obvious.

The issue is not the absence of the shared library itself but the undefined relative path of the reference to the boost library used upon compilation (linker phase). This might be due to the use of an boost installation which resides in miniconda and not in the /usr/local/lib which might be in the pythonpath or something. However, this can be manually resolved using otool and install_name_tool (which can be acquired on mac by installing XCode dev tools).

So you do the following on the command line:

otools -L ${PATH_TO_PYTHONEXTENSION}/someextension.so

which list all libraries and their system path on the terminal like this

./build/lib.macosx-10.13-x86_64-2.7/tools.so:
    @rpath/libboost_python.dylib (compatibility version 0.0.0, current version 0.0.0)
    /usr/lib/libc++.1.dylib (compatibility version 1.0.0, current version 400.9.0)
    /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1252.50.4)

one can easily see that there is a missing path for the boost library which we linked upon compilation, which we resolve using install_name_tool:

install_name_tool -change @rpath/libboost_python.dylib ${ACTUAL_PATH_TO_libbost_python.dylib} ${PATH_TO_PYTHONEXTENSION}/someextension.so

Rerunning setup.py install now resolves the issue and the extension can be imported successfully:

import tools
dir(tools)

gives the following output:

['__doc__', '__file__', '__name__', '__package__', 'hammdist', 'product', 'string_vector']

link to external resource: unsafe use of relative rpath libboost.dylib when making boost.python helloword demo?

I assume it works the same for other Unix-based operating systems like Linux. Hope this helps.

mnmldani
  • 53
  • 6