Boost::python: object destroys itself inside a overriden method

Question

While embedding python in my application I've faced with problem related to python objects lifetime. My application exports some classes with virtual methods to python, so they can be derived and extended by python code. Application is using python interpreter and calls virtual methods of objects. The problem is that when object's reference counter reaches zero inside of python overridden method, which was called from c++ code, interpreter destroys object immediately. So, if we call such method inside another method of object we will get behavior equivalent to delete this statement. Simple test code:

Object:

class Base
{
public:
    virtual ~Base()
    {
        std::cout << "C++ deleted" << std::endl;
        std::cout.flush();
    }

    virtual void virtFunc()
    {
    }

    void rmFunc()
    {
        std::cout << "Precall" << std::endl;
        virtFunc();
        std::cout << "Postcall" << std::endl;
        //Segfault here, this doesn't exists. 
        value = 0;
    }
    
private:
    int value;
};

Boost::Python module library:

#include <boost/python.hpp>
#include <list>
#include "Types.h"
#include <iostream>

// Policies used for reference counting
struct add_reference_policy : boost::python::default_call_policies
{
    static PyObject *postcall(PyObject *args, PyObject *result)
    {
        PyObject *arg = PyTuple_GET_ITEM(args, 0);
        Py_INCREF(arg);
        return result;
    }
};

struct remove_reference_policy : boost::python::default_call_policies
{
    static PyObject *postcall(PyObject *args, PyObject *result)
    {
        PyObject *arg = PyTuple_GET_ITEM(args, 0);
        Py_DecRef(arg);
        return result;
    }
};

struct BaseWrap: Base, boost::python::wrapper<Base>
{
    BaseWrap(): Base()
    {
    }
    
    virtual ~BaseWrap()
    {
        std::cout << "Wrap deleted" << std::endl;
        std::cout.flush();
    }

    void virtFunc()
    {
        if (boost::python::override f = get_override("virtFunc"))
        {
            try 
            { 
                f();
            }
            catch (const boost::python::error_already_set& e)
            {
            }
        }
    }
    
    void virtFunc_()
    {
        Base::virtFunc();
    }
};

std::list<Base*> objects;

void addObject(Base *o)
{
    objects.push_back(o);
}

void removeObject(Base *o)
{
    objects.remove(o);
}

BOOST_PYTHON_MODULE(pytest)
{
    using namespace boost::python;
    class_<BaseWrap, boost::noncopyable>("Base", init<>())
    .def("virtFunc", &Base::virtFunc, &BaseWrap::virtFunc_);
    
    def("addObject", &addObject, add_reference_policy());
    def("removeObject", &removeObject, remove_reference_policy());
}

Application, linked with module:

#include <boost/python.hpp>
#include <list>
#include "Types.h"

extern std::list<Base*> objects;

int main(int argc, char **argv)
{
    Py_Initialize();
    boost::python::object main_module = boost::python::import("__main__");
    boost::python::object main_namespace = main_module.attr("__dict__");

    try
    {
        boost::python::exec_file("fail-test.py", main_namespace);
    }
    catch(boost::python::error_already_set const &)
    {
        PyErr_Print();
    }
    sleep(1);
    objects.front()->rmFunc();
    sleep(1);
}

fail-test.py:

import pytest

class Derived(pytest.Base):
   def __init__(self, parent):
       pytest.Base.__init__(self)
       pytest.addObject(self)
       
   def __del__(self):
       print("Python deleted")
       
   def virtFunc(self):
       pytest.removeObject(self)
           
o1 = Derived(None)
o1 = None

Output:

Precall
Python deleted
Wrap deleted
C++ deleted
Postcall

Is there any good way to avoid such behavior?

score 1 · Accepted Answer · answered May 30 '13 at 20:10

With Boost.Python, this can be accomplished by using boost::shared_ptr to manage the lifetime of the objects. This is normally done by specifying the HeldType when exposing the C++ type via boost::python::class_. However, Boost.Python often provides the desired functionality with boost::shared_ptr. In this case, the boost::python::wrapper type supports conversions.

Here is a complete example:

#include <iostream>
#include <list>
#include <string>

#include <boost/python.hpp>
#include <boost/shared_ptr.hpp>

class Base
{
public:
  virtual ~Base() { std::cout << "C++ deleted" << std::endl; }
  virtual void virtFunc() {}
  void rmFunc()
  {
     std::cout << "Precall" << std::endl;
     virtFunc();
     std::cout << "Postcall" << std::endl;
  }
};

/// @brief Wrap Base to allow for python derived types to override virtFunc.
struct BaseWrap
  : Base,
    boost::python::wrapper<Base>
{
  virtual ~BaseWrap() { std::cout << "Wrap deleted" << std::endl; }
  void virtFunc_() { Base::virtFunc(); }
  void virtFunc()
  {
    namespace python = boost::python;
    if (python::override f = get_override("virtFunc"))
    {
      try { f(); }
      catch (const python::error_already_set&) {}
    }
  }
};


std::list<boost::shared_ptr<Base> > objects;

void addObject(boost::shared_ptr<Base> o)    { objects.push_back(o); }
void removeObject(boost::shared_ptr<Base> o) { objects.remove(o);    }

BOOST_PYTHON_MODULE(pytest)
{
  namespace python = boost::python;
  python::class_<BaseWrap, boost::noncopyable >("Base", python::init<>())
    .def("virtFunc", &Base::virtFunc, &BaseWrap::virtFunc_);

  python::def("addObject", &addObject);
  python::def("removeObject", &removeObject);
}

const char* derived_example_py =
  "import pytest\n"
  "\n"
  "class Derived(pytest.Base):\n"
  "  def __init__(self, parent):\n"
  "    pytest.Base.__init__(self)\n"
  "    pytest.addObject(self)\n"
  "\n"
  "  def __del__(self):\n"
  "    print(\"Python deleted\")\n"
  "\n"
  "  def virtFunc(self):\n"
  "    pytest.removeObject(self)\n"
  "\n"
  "o1 = Derived(None)\n"
  "o1 = None\n"
  ;

int main()
{
  PyImport_AppendInittab("pytest", &initpytest);
  Py_Initialize();

  namespace python = boost::python;
  python::object main_module    = python::import("__main__");
  python::object main_namespace = main_module.attr("__dict__");

  try
  {
    exec(derived_example_py, main_namespace);
  }
  catch (const python::error_already_set&)
  {
    PyErr_Print();
  }

  boost::shared_ptr<Base> o(objects.front());
  o->rmFunc();
  std::cout << "pre reset" << std::endl;
  o.reset();
  std::cout << "post reset" << std::endl;
}

And the output:

Precall
Postcall
pre reset
Python deleted
Wrap deleted
C++ deleted
post reset

One final change to note is that:

objects.front()->rmFunc();

was replaced with:

boost::shared_ptr<Base> o(objects.front());
o->rmFunc();

This was required because std::list::front returns a reference to the element. By creating a copy of the shared_ptr, the lifespan is extended beyond the rmFunc() call.

Boost::python: object destroys itself inside a overriden method

1 Answers1

Linked