CPython C/C extension: Dealloc never called-CodePudding

Initial situation: I have a Python C extension module which defines a init() method which creates and returns a new Python object. I followed the approach of heaptypes.c in the official python sources.

My source code (my_module.cpp) is almost a 1:1 copy of the example in the Python sources :

typedef struct {
    PyObject_HEAD
    int value;
} HeapCTypeObject;

static int heapctype_init(PyObject *self, PyObject *args, PyObject *kwargs)
{
    std::cout << "Init called" << std::endl;
    ((HeapCTypeObject *)self)->value = 10;
    return 0;
}

static struct PyMemberDef heapctype_members[] = {
    {"value", T_INT, offsetof(HeapCTypeObject, value)},
    {NULL} /* Sentinel */
};

static void heapctype_dealloc(HeapCTypeObject *self)
{
    std::cout << "dealloc called" << std::endl;
    PyTypeObject *tp = Py_TYPE(self);
    PyObject_Free(self);
    Py_DECREF(tp);
}

PyDoc_STRVAR(heapctype__doc__,
"A heap type without GC, but with overridden dealloc.\n\n"
"The 'value' attribute is set to 10 in __init__.");

static PyType_Slot HeapCType_slots[] = {
    {Py_tp_init, (void*)heapctype_init},
    {Py_tp_members, heapctype_members},
    {Py_tp_dealloc, (void*)heapctype_dealloc},
    {Py_tp_doc, (char*)heapctype__doc__},
    {0, 0},
};

static PyType_Spec HeapCType_spec = {
    "MyModule.HeapCType",
    sizeof(HeapCTypeObject),
    0,
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    HeapCType_slots
};

static PyObject* init(PyObject *self, PyObject *args){
    PyObject *HeapCType = PyType_FromSpec(&HeapCType_spec);
    return HeapCType;
}

PyMethodDef method_table[] = {

    {"init", (PyCFunction) init, METH_NOARGS, "Construct create new PyObject"},
    {NULL, NULL, 0, NULL} // Sentinel value ending the table
};

// A struct contains the definition of a module
PyModuleDef mymod_module = {
    PyModuleDef_HEAD_INIT,
    "MyModule", // Module name
    "This is MyModule's docstring",
    -1,   // Optional size of the module state memory
    method_table,
    NULL, // Optional slot definitions
    NULL, // Optional traversal function
    NULL, // Optional clear function
    NULL  // Optional module deallocation function
};

PyMODINIT_FUNC 
PyInit_MyModule(void) {
    return PyModule_Create(&mymod_module);
}

Importing the module und creating a new instance works fine:

import MyModule

x = MyModule.init()

However, when I delete the instance, the heapctype_dealloc(HeapCTypeObject *self) gets never called:

import gc

del x
gc.collect()

Even when I quit the interpreter, the deallocation methods does not get called and I don't know why. As my plan is to combine this approach witch a C class allocated on the heap, I would get a rock-solid memory leak if the deallocation method won't get called.

The behaviour can be reproduces with the following steps:

git clone https://github.com/hANSIc99/PythonCppExtension/tree/stackoverflow
cd PythonCppExtension
cmake -B build
cmake --build buil
source init.sh
python3 -i main.py

I also observed that right after the call to to PyType_FromSpec(...), the newly created class has already an reference count of 4.

What I'm doing wrong here? How can I ensure that the deallocation methods is called when the instance is destroyed?

Thanks for your help!

CodePudding user response：

You are using the API incorrectly.

PyType_FromSpec returns a TYPE, not an Object instance (it is not the same as PyObject_New). So it will never call init (the constructor) of said type.

You are doing: x = MyModule.init(). x is actually a HeapCTypeObject TYPE. You haven't constructed an instance of that type yet.

If you print(x), you'd see <class 'MyModule.HeapCType'>

To do so, you need to do:

import MyModule

HeapCTypeObject = MyModule.init()
x = HeapCTypeObject()

If you then print(x) you'd see <MyModule.HeapCType object at 0x10702bb70>

Now it will call your constructor and destructor correctly. This is useful for creating dynamic types. But if you already know what your type looks like, why not just create it directly and expose it via PyModule_AddObject?

IE:

#include <Python.h>
#include "methodobject.h"
#include "object.h"
#include "pyport.h"
#include <structmember.h>
#include <cstdio>

typedef struct {
    PyObject_HEAD
    int value;
} HeapCTypeObject;

static int heapctype_init(PyObject *self, PyObject *args, PyObject *kwargs)
{
    fprintf(stderr, "INIT CALLED\n");
    ((HeapCTypeObject *)self)->value = 10;
    return 0;
}

static struct PyMemberDef heapctype_members[] = {
    {"value", T_INT, offsetof(HeapCTypeObject, value)},
    {NULL} /* Sentinel */
};

static void
heapctype_dealloc(HeapCTypeObject *self)
{
    fprintf(stderr, "DEALLOC CALLED\n");
    PyTypeObject *tp = Py_TYPE(self);
    PyObject_Free(self);
    Py_DECREF(tp);
}

PyDoc_STRVAR(heapctype__doc__,
"A heap type without GC, but with overridden dealloc.\n\n"
"The 'value' attribute is set to 10 in __init__.");

static PyType_Slot HeapCType_slots[] = {
    {Py_tp_init, (void*)heapctype_init},
    {Py_tp_members, heapctype_members},
    {Py_tp_dealloc, (void*)heapctype_dealloc},
    {Py_tp_doc, (char*)heapctype__doc__},
    {Py_tp_base, (void *)&PyType_Type},
    {0, NULL},
};

static PyTypeObject HeapCType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "HeapCTypeObject",
    .tp_doc = PyDoc_STR("Custom objects"),
    .tp_basicsize = sizeof(HeapCTypeObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE,
    .tp_new = nullptr,
    .tp_init = (initproc) heapctype_init,
    .tp_dealloc = (destructor) heapctype_dealloc,
    .tp_members = heapctype_members,
    .tp_methods = nullptr,
};

PyMethodDef method_table[] = {

    {NULL, NULL, 0, NULL} // Sentinel value ending the table
};

// A struct contains the definition of a module
PyModuleDef mymod_module = {
    PyModuleDef_HEAD_INIT,
    "MyModule", // Module name
    "This is MyModule's docstring",
    -1,   // Optional size of the module state memory
    method_table,
    NULL, // Optional slot definitions
    NULL, // Optional traversal function
    NULL, // Optional clear function
    NULL  // Optional module deallocation function
};

PyMODINIT_FUNC 
PyInit_MyModule(void) {
    if (PyType_Ready(&HeapCType) < 0)
        return NULL;

    PyObject* module = PyModule_Create(&mymod_module);
    Py_INCREF(&HeapCType);
    if (PyModule_AddObject(module, "HeapCTypeObject", (PyObject *) &HeapCType) < 0) {
        Py_DECREF(&HeapCType);
        Py_DECREF(module);
        return NULL;
    }

    return module;
}

Then in Python you'd do:

from MyModule import HeapCTypeObject

x = HeapCTypeObject()

which is much clearer.