Python extension in C - Metaclass

Question

I have the following python code:

class Meta(type):
    def __call__(cls, *args, **kwargs):
        obj = type.__call__(cls, *args, **kwargs)
        # Only do checks for subclasses
        if cls.__name__ == 'Parent':
            return obj
        required_attrs = ['x']
        for ra in required_attrs:
            if ra not in dir(obj):
                fmt = 'Subclasses of Parent must define the %s attribute'
                raise NotImplementedError(fmt % ra)
        return obj

class Parent(metaclass=Meta):
    pass

class Child(Parent):
    def __init__(self):
        self.x = True

Meta is used only to require that Child defines certain attributes. This class structure must remain as is because this is how my project is structured. Parent is actually called DefaultConfig and Child is actually a user-defined class derived from DefaultConfig.

I'm working on translating Meta and Parent into a C extension. This is the module:

#include <Python.h>
#include <structmember.h>

#define ARRLEN(x) sizeof(x)/sizeof(x[0])


typedef struct {
    PyObject_HEAD
} MetaObject;

typedef struct {
    PyObject_HEAD
} ParentObject;


static PyObject *Meta_call(MetaObject *type, PyObject *args, PyObject *kwargs) {
    PyObject *obj = PyType_GenericNew((PyTypeObject *) type, args, kwargs);

    // Only do checks for subclasses of Parent
    if (strcmp(obj->ob_type->tp_name, "Parent") == 0)
        return obj;

    // Get obj's attributes
    PyObject *obj_dir = PyObject_Dir(obj);
    if (obj_dir == NULL)
        return NULL;

    char *required_attrs[] = {"x"};

    // Raise an exception of obj doesn't define all required_attrs
    PyObject *attr_obj;
    int has_attr;
    for (int i=0; i<ARRLEN(required_attrs); i++) {
        attr_obj = PyUnicode_FromString(required_attrs[i]);
        has_attr = PySequence_Contains(obj_dir, attr_obj);
        if (has_attr == 0) {
            printf("Subclasses of Parent must define %s\n", required_attrs[i]);
            // raise NotImplementedError
            return NULL;
        } else if (has_attr == -1) {
            return NULL;
        }
    }

    return obj;
}


static PyTypeObject MetaType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "custom.Meta",
    .tp_basicsize = sizeof(MetaObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
    .tp_call = (ternaryfunc) Meta_call,
};

static PyTypeObject ParentType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "custom.Parent",
    .tp_basicsize = sizeof(ParentObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
};


static PyModuleDef custommodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "custom",
    .m_size = -1,
};


PyMODINIT_FUNC PyInit_custom(void) {
    PyObject *module = PyModule_Create(&custommodule);
    if (module == NULL)
        return NULL;

    // Should Parent inherit from Meta?
    ParentType.tp_base = &MetaType;

    if (PyType_Ready(&MetaType) < 0)
        return NULL;
    Py_INCREF(&MetaType);
    PyModule_AddObject(module, "Meta", (PyObject *) &MetaType);

    if (PyType_Ready(&ParentType) < 0)
        return NULL;
    Py_INCREF(&ParentType);
    PyModule_AddObject(module, "Parent", (PyObject *) &ParentType);

    return module;
}

This is the python code used to test module custom:

import custom

class Child(custom.Parent):
    def __init__(self):
        self.x = True

if __name__ == '__main__':
    c = Child()

Unfortunately, there is no .tp_meta member in the PyTypeObject struct, so how do I specify Meta as the metaclass of Parent?

EDIT:

Modified C code:

#include <Python.h>
#include <structmember.h>

#define ARRLEN(x) sizeof(x)/sizeof(x[0])


typedef struct {
    PyObject_HEAD
    PyTypeObject base;
} MetaObject;

typedef struct {
    PyObject_HEAD
} ParentObject;


static PyObject *Meta_call(MetaObject *type, PyObject *args, PyObject *kwargs) {
    PyObject *obj = PyType_GenericNew((PyTypeObject *) type, args, kwargs);

    // Only do checks for subclasses of Parent
    if (strcmp(obj->ob_type->tp_name, "Parent") == 0)
        return obj;

    // Get obj's attributes
    PyObject *obj_dir = PyObject_Dir(obj);
    if (obj_dir == NULL)
        return NULL;

    char *required_attrs[] = {"x"};

    // Raise an exception of obj doesn't define all required_attrs
    PyObject *attr_obj;
    int has_attr;
    for (int i=0; i<ARRLEN(required_attrs); i++) {
        attr_obj = PyUnicode_FromString(required_attrs[i]);
        has_attr = PySequence_Contains(obj_dir, attr_obj);
        if (has_attr == 0) {
            printf("Subclasses of Parent must define %s\n", required_attrs[i]);
            // raise NotImplementedError
            return NULL;
        } else if (has_attr == -1) {
            return NULL;
        }
    }

    return obj;
}


static PyTypeObject MetaType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "custom.Meta",
    .tp_basicsize = sizeof(MetaObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
    .tp_call = (ternaryfunc) Meta_call,
};

static PyTypeObject ParentType = {
    PyVarObject_HEAD_INIT(&MetaType, 0)
    .tp_name = "custom.Parent",
    .tp_basicsize = sizeof(ParentObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
};


static PyModuleDef custommodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "custom",
    .m_size = -1,
};


PyMODINIT_FUNC PyInit_custom(void) {
    PyObject *module = PyModule_Create(&custommodule);
    if (module == NULL)
        return NULL;

    MetaType.tp_base = &PyType_Type;
    if (PyType_Ready(&MetaType) < 0)
        return NULL;
    Py_INCREF(&MetaType);
    PyModule_AddObject(module, "Meta", (PyObject *) &MetaType);

    if (PyType_Ready(&ParentType) < 0)
        return NULL;
    Py_INCREF(&ParentType);
    PyModule_AddObject(module, "Parent", (PyObject *) &ParentType);

    return module;
}

Antti Haapala -- Слава Україні · Answer 1 · 2022-06-23T16:54:07.610

2

The metaclass is nothing but a type that is used as the type (ob_type!) of the class (type)... (clear, isn't it)... ParentType does not inherit from MetaType but is an instance of `MetaType.

Hence, the place where &MetaType should go (if anywhere), is ParentType.ob_type:

PyModule_AddObject(module, "Meta", (PyObject *) &MetaType);

ParentType.ob_type = &MetaType;

if (PyType_Ready(&ParentType) < 0)

PyType_Ready checks the ob_type field - if it is NULL, it takes the ob_type of the .tp_base; but if ob_type is set already, it is left as is.

Actually you can set it in the ParentType initializer:

PyVarObject_HEAD_INIT(&MetaType, 0)

The first argument goes to the ob_type field.

edited Jun 23 '22 at 16:54

answered Oct 23 '18 at 21:18

Antti Haapala -- Слава Україні

129,958
22
279
321

It seems though the python source code has a special attribute called `PyId_metaclass` that it uses to determine the meta class .... https://github.com/python/cpython/blob/9dfa0fe587eae3626ffc973680c6a17f35de3864/Python/bltinmodule.c#L151 – Josh Weinstein Oct 23 '18 at 21:31
1

@JoshWeinstein on the contrary, this is just the *string* `"metaclass"`, used to get the `metaclass=FooBar` from the kwargs. – Antti Haapala -- Слава Україні Oct 23 '18 at 21:35
2

Setting `ob_type` isn't going to be enough, though, because there are other problems with the code in the question. For example, `MetaObject` doesn't include a `PyTypeObject` base member, and `MetaType`'s `tp_base` needs to be set to `PyType_Type`. – user2357112 Oct 23 '18 at 22:00
Also, things like PyId_metaclass aren't *quite* just strings - they're wrappers used to manage interned, static strings. Python still ends up using it to fetch `metaclass` from the kwargs by string name, though. It's not any sort of special attribute, and it's not how the metaclass of an actual class object is stored. – user2357112 Oct 23 '18 at 22:22
Changing the `ParentType` struct to use `PyVarObject_HEAD_INIT(&MetaType, 0)` (after fixing the [errors mentioned by user2357112](https://stackoverflow.com/questions/52957192/python-extension-in-c-metaclass#comment92824148_52957978)) causes a segfault. See the updated code – Nelson Oct 24 '18 at 00:10
2

@Nelson: From a quick glance, you don't need the `PyObject_HEAD` in `MetaObject`; the `PyTypeObject`'s `PyObject_HEAD` takes care of that. There may be other problems. See the [relevant section](https://docs.python.org/3/extending/newtypes_tutorial.html#subclassing-other-types) of the C API tutorial, and [`Modules/xxsubtype.c`](https://github.com/python/cpython/blob/v3.7.0/Modules/xxsubtype.c) for an example of implementing subclasses of built-in types in C. – user2357112 Oct 24 '18 at 03:09

Josh Weinstein · Answer 2 · 2018-10-23T21:35:22.973

-1

There is no direct way to do this. According to the py docs, there is no members or flags to directly indicate a class is a meta class of another. The attribute responsible for indicating a meta class is inside the class dictionary. You could implement something that modifies the .tp_dict member, but this is actually deemed unsafe if done through the dictionary C-API.

Warning It is not safe to use PyDict_SetItem() on or otherwise modify tp_dict with the dictionary C-API.

EDIT:

From the python source code, it seems meta class is accessed as an id via the C dictionary API, but the methods to do so are prefixed with an _, and don't appear in any documentation.

    meta = _PyDict_GetItemId(mkw, &PyId_metaclass);
    if (meta != NULL) {
        Py_INCREF(meta);
        if (_PyDict_DelItemId(mkw, &PyId_metaclass) < 0) {
            Py_DECREF(meta);
            Py_DECREF(mkw);
            Py_DECREF(bases);
            return NULL;
        }

These methods are apart of the "limited api", and can be used by defining the Py_LIMITED_API macro

PyAPI_FUNC(PyObject *) _PyDict_GetItemId(PyObject *dp, struct _Py_Identifier *key);
#endif /* !Py_LIMITED_API */
PyAPI_FUNC(int) PyDict_SetItemString(PyObject *dp, const char *key, PyObject *item);
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, struct _Py_Identifier *key, PyObject *item);
#endif /* !Py_LIMITED_API */

edited Oct 23 '18 at 21:35

answered Oct 23 '18 at 20:41

Josh Weinstein

2,788
2
21
38

There is a member indicating a class's metaclass; it's part of PyObject_HEAD, and it's the same member used for any other object's class. The `__metaclass__` dict entry isn't used in Python 3, and it was only ever used in Python 2 during class creation, not to determine the metaclass of an already-created class. – user2357112 Oct 23 '18 at 20:44
Setting `__metaclass__` in the dict of an existing class won't affect its metaclass. Also, messing with `tp_dict` after `PyType_Ready` is called is still unsafe. – user2357112 Oct 23 '18 at 20:45
Can you link to where that part of PyObject_HEAD is? not documented here https://docs.python.org/3/c-api/structures.html#c.PyObject – Josh Weinstein Oct 23 '18 at 20:47
1

/usr/include/python3.6/object.h:83 `#define PyObject_HEAD PyObject ob_base;`. I don't understand how a `PyObject` contains the information about a type's metaclass – Nelson Oct 23 '18 at 20:54
Updated my answer to include the place in the python source code that checks an objects meta class. It is *not* in `PyObject_HEAD` – Josh Weinstein Oct 23 '18 at 21:27
@Nelson: That would be the `struct _typeobject *ob_type;` inside the [`PyObject` definition](https://github.com/python/cpython/blob/v3.7.0/Include/object.h#L106), which records the type of an object. A class's metaclass is its type. – user2357112 Oct 23 '18 at 21:56
Also, defining `Py_LIMITED_API` is not required to access any part of the C API. Defining `Py_LIMITED_API` *restricts* you to a subset of the C API. – user2357112 Oct 23 '18 at 22:05

Python extension in C - Metaclass

2 Answers2

Linked