Garbage collection problems with a c++ wrapper for a module

Discussion in 'Python' started by James S, Jul 15, 2004.

  1. James S

    James S Guest

    Hi,

    Basically I've been fighting with this code for a few days now and
    can't seem to work around this problem. Included is the output, the
    program I use to get this error and the source code for my wrapper.
    This is acually part of the project, libxmlconf on sourceforge. The
    newest working version isn't there yet, and cvs is lagged by 6 hours
    or so. So if you think you want to have a try at this I can tgz the
    source for you. My libxmlconf.cpp wrapper is based on the xxmodule.c
    included with the source for the 2.3.3 python. That also happens to be
    the version I'm developing and testing python with.

    Any help would be fantastic,

    James

    Output:
    python test.py
    ############ TEST ############
    ['XMLConf', '__doc__', '__file__', '__name__']
    ['__class__', '__delattr__', '__doc__', '__getattribute__',
    '__hash__', '__init__', '__new__', '__reduce__', '__reduce_ex__',
    '__repr__', '__setattr__', '__str__']
    NEW
    P: init
    NEW
    P: init
    <libxmlconf.XMLConf object at 0x402f2050> <libxmlconf.XMLConf object
    at 0x402f2060>

    You can use the following gay ness to eat
    cheese after wor
    And then you can poke your eyes our with a
    pen.
    Don't forget to write!

    ############ DONE TEST ############
    ASFDA
    ASFDA
    ASFDA
    ASFDA
    Exception exceptions.TypeError: 'function takes exactly 0 arguments (1
    given)' in 'garbage collection' ignored
    Fatal Python error: unexpected exception during garbage collection
    make: *** [test] Aborted

    # note this error happes after the program is done, so it would appear
    to be a gc problem. Also, I can get this error to happen before hand
    by importing gc after I have loaded and make an instance of my
    wrapper.


    test.py:
    import libxmlconf


    _d = "############"


    print _d, "TEST",_d

    print dir( libxmlconf )
    print dir( libxmlconf.XMLConf )

    s = libxmlconf.XMLConf( "../../runtest.xml")
    ss = libxmlconf.XMLConf()
    print s,ss
    print s.GetValue("root.help.coin")
    print _d, "DONE TEST",_d

    libxmlconf.cpp:
    /* Use this file as a template to start implementing a module that
    also declares object types. All occurrences of 'Xxo' should be
    changed
    to something reasonable for your objects. After that, all other
    occurrences of 'xx' should be changed to something reasonable for
    your
    module. If your module is named foo your sourcefile should be named
    foomodule.c.

    You will probably want to delete all references to 'x_attr' and add
    your own types of attributes instead. Maybe you want to name your
    local variables other than 'self'. If your object type is needed
    in
    other files, you'll have to create a file "foobarobject.h"; see
    intobject.h for an example. */

    /* Xxo objects */
    #include "libxmlconf.h"
    #include "Python.h"

    extern "C" {

    };

    static PyObject *ErrorObject;

    typedef struct {
    PyObject_HEAD
    struct XMLCONF_STRUCT *root; /* Attributes dictionary */
    int inited;
    } XMLCONFObject;

    extern PyTypeObject XMLCONF_Type;

    //#define XMLCONFObject_Check(v) ((v)->ob_type == &XMLCONF_Type))

    static XMLCONFObject *
    newXMLCONFObject(PyObject *arg)
    {
    printf("NEW\n");
    XMLCONFObject *self;
    self = PyObject_New(XMLCONFObject, &XMLCONF_Type);
    if (self == NULL)
    return NULL;

    self->inited = 0;
    self->root = NULL;
    return self;
    }

    static int
    XMLCONF_Initialize(
    XMLCONFObject *self,
    PyObject *args,
    PyObject *kwargs )
    {
    printf("P: init\n");
    char *filename;
    self->inited = 1;
    self->root = XMLConf_Create( );
    if (PyArg_ParseTuple(args,"" ) )
    return 0;
    if (PyArg_ParseTuple(args,"s", &filename ) ) {
    XMLConf_ParseFile( self->root, filename );
    self->inited = 1;
    return 0;
    }

    return -1;
    }

    static void
    XMLCONF_dealloc(XMLCONFObject *self)
    {
    printf("DEALLOC");
    if ( self->inited == 1 ) {
    XMLConf_Destroy( self->root );
    }
    PyObject_Del(self);
    printf(">DEALLOC\n");
    }

    static PyObject *
    XMLCONF_GetValue(XMLCONFObject *self, PyObject *args)
    {
    PyObject *resultobj;
    char *path;
    char *result;
    if ( PyArg_ParseTuple( args, "s", &path ) == 0 ) {
    Py_INCREF(Py_None);
    return Py_None;
    }

    result = XMLConf_GetValue( self->root, path );
    if ( result == NULL ) {
    Py_INCREF(Py_None);
    return Py_None;
    }
    resultobj = Py_BuildValue("s", result );
    Py_INCREF( resultobj );
    return resultobj;
    }
    static PyObject *
    XMLCONF_SetValue(XMLCONFObject *self, PyObject *args) {
    Py_INCREF(Py_None);
    }

    static PyMethodDef XMLCONF_methods[] = {
    {"GetValue", (PyCFunction)XMLCONF_GetValue, METH_VARARGS,
    PyDoc_STR("Gets the Value")},
    {"SetValue", (PyCFunction)XMLCONF_SetValue, METH_VARARGS,
    PyDoc_STR("Sets the Value")},
    {NULL, NULL} /* sentinel */
    };

    static PyObject *
    XMLCONF_getattr(XMLCONFObject *self, char *name)
    {
    return Py_FindMethod(XMLCONF_methods, (PyObject *)self, name);
    }

    static int
    XMLCONF_setattr(XMLCONFObject *self, char *name, PyObject *v)
    {
    printf("SETATTR\n");
    return 0;
    }

    PyTypeObject XMLCONF_Type = {
    /* The ob_type field must be initialized in the module init function
    * to be portable to Windows without using C++. */
    PyObject_HEAD_INIT(NULL)
    0, /*ob_size*/
    "libxmlconf.XMLConf", /*tp_name*/
    sizeof(XMLCONFObject), /*tp_basicsize*/
    0, /*tp_itemsize*/
    /* methods */
    (destructor)XMLCONF_dealloc, /*tp_dealloc*/
    0, /*tp_print*/
    (getattrfunc)XMLCONF_getattr, /*tp_getattr*/
    (setattrfunc)XMLCONF_setattr, /*tp_setattr*/
    0, /*tp_compare*/
    0, /*tp_repr*/
    0, /*tp_as_number*/
    0, /*tp_as_sequence*/
    0, /*tp_as_mapping*/
    0, /*tp_hash*/
    0, /*tp_call*/
    0, /*tp_str*/
    0, /*tp_getattro*/
    0, /*tp_setattro*/
    0, /*tp_as_buffer*/
    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC,
    /*tp_flags*/
    0, /*tp_doc*/
    0, /*tp_traverse*/
    0, /*tp_clear*/
    0, /*tp_richcompare*/
    0, /*tp_weaklistoffset*/
    0, /*tp_iter*/
    0, /*tp_iternext*/
    0, /*tp_methods*/
    0, /*tp_members*/
    0, /*tp_getset*/
    0, /*tp_base*/
    0, /*tp_dict*/
    0, /*tp_descr_get*/
    0, /*tp_descr_set*/
    0, /*tp_dictoffset*/
    (initproc) XMLCONF_Initialize,/*init*/
    0, /*tp_alloc*/
    (newfunc) newXMLCONFObject, /*tp_new*/
    0, /*tp_free*/
    0, /*tp_is_gc*/
    };
    /* ---------------------------------------------------------------------
    */

    /* List of functions defined in the module */

    static PyMethodDef xx_methods[] = {
    {NULL, NULL} /* sentinel */
    };

    PyDoc_STRVAR(module_doc,
    "This is a template module just for instruction.");

    /* Initialization function for the module (*must* be called initxx) */

    PyMODINIT_FUNC
    initlibxmlconf(void)
    {
    PyObject *m;

    /* Finalize the type object including setting type of the new type
    * object; doing it here is required for portability to Windows
    * without requiring C++. */
    if (PyType_Ready(&XMLCONF_Type) < 0)
    return;

    /* Create the module and add the functions */
    m = Py_InitModule3("libxmlconf", xx_methods, module_doc);
    PyModule_AddObject( m, "XMLConf", (PyObject *)&XMLCONF_Type);
    /* Add some symbolic constants to the module */
    }
     
    James S, Jul 15, 2004
    #1
    1. Advertising

  2. James S

    Greg Chapman Guest

    On 15 Jul 2004 11:15:10 -0700, (James S) wrote:

    >
    >libxmlconf.cpp:
    >
    >/* Xxo objects */
    >#include "libxmlconf.h"
    >#include "Python.h"
    >
    >extern "C" {
    >
    >};
    >
    >static PyObject *ErrorObject;
    >
    >typedef struct {
    > PyObject_HEAD
    > struct XMLCONF_STRUCT *root; /* Attributes dictionary */
    > int inited;
    >} XMLCONFObject;
    >
    >extern PyTypeObject XMLCONF_Type;
    >
    >//#define XMLCONFObject_Check(v) ((v)->ob_type == &XMLCONF_Type))
    >
    >static XMLCONFObject *
    >newXMLCONFObject(PyObject *arg)
    >{
    > printf("NEW\n");
    > XMLCONFObject *self;
    > self = PyObject_New(XMLCONFObject, &XMLCONF_Type);
    > if (self == NULL)
    > return NULL;
    >
    > self->inited = 0;
    > self->root = NULL;
    > return self;
    >}


    In your type definition below, the above is used as your tp_new, but it doesn't
    have the right signature. Also, the class is marked as supporting garbage
    collection, but you're not using the GC allocator. But, as it turns out, you
    don't need to support garbage collection in this class since it doesn't hold any
    references to other python objects (and so can't create a reference cycle). So
    it's better to simplify by removing the HAVE_GC flag. But, since you want this
    to be a base class, you need to use type.tp_alloc to actually get the memory for
    the instance (this allows a subclass to turn on garbage collection and have the
    memory allocated by the GC allocator). Also, I would ditch the inited flag, and
    simply call XMLConf_Create in the new method. A class invariant is then that
    self->root is not NULL (once tp_new has completed).

    (Caveat: I haven't tried to compile or test any of the below):

    typedef struct {
    PyObject_HEAD
    struct XMLCONF_STRUCT *root; /* Attributes dictionary */
    } XMLCONFObject;

    PyObject*
    newXMLCONFObject(PyTypeObject *type, PyObject *args, PyObject *kwds)
    {
    XMLCONFObject *self;

    printf("NEW\n");
    self = (XMLCONFObject *)type->tp_alloc(type, 0);
    if (self == NULL)
    return NULL;
    self->root = XMLConf_Create();
    if (initialization of self->root failed) {
    PyErr_SetString(some appropriate exception);
    type->tp_free(self);
    return NULL;
    }
    return (PyObject *)self;
    }

    >
    >static int
    >XMLCONF_Initialize(
    > XMLCONFObject *self,
    > PyObject *args,
    > PyObject *kwargs )
    >{
    > printf("P: init\n");
    > char *filename;
    > self->inited = 1;
    > self->root = XMLConf_Create( );
    > if (PyArg_ParseTuple(args,"" ) )
    > return 0;
    > if (PyArg_ParseTuple(args,"s", &filename ) ) {
    > XMLConf_ParseFile( self->root, filename );
    > self->inited = 1;
    > return 0;
    > }
    >
    > return -1;
    >}


    In the above, it looks like you want filename to be an optional argument; you
    can specify it as such using a format of "|s". In the above, if the first call
    to ParseTuple fails, it will set an exception. If the second call then
    succeeds, the function returns successfully, but the exception is still set,
    which can cause confusion down the line. In particular, I think this is the
    exception which the garbage collector is detecting.

    static int
    XMLCONF_Initialize(
    XMLCONFObject *self,
    PyObject *args,
    PyObject *kwargs )
    {
    static char *kwlist = {"filename", NULL};
    char *filename = NULL;

    printf("P: init\n");
    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|s:XMLConf", kwlist,
    &filename ))
    return -1;

    if (filename == NULL)
    return 0;

    XMLConf_ParseFile(self->root, filename);
    if (ParseFile failed somehow) {
    PyErr_RaiseAppropriateException;
    return -1;
    }
    return 0;
    }

    Since you want this to be a base class, it is important that your dealloc
    function calls the tp_free method defined in the actual type (self->ob_type) of
    the instance; if the actual type is a subtype which supports garbage collection
    this will call the GC deallocator.

    static void
    XMLCONF_dealloc(XMLCONFObject *self)
    {
    printf("DEALLOC");
    XMLConf_Destroy( self->root );
    self->ob_type->tp_free(self);
    printf(">DEALLOC\n");
    }


    >
    >static PyObject *
    >XMLCONF_GetValue(XMLCONFObject *self, PyObject *args)
    >{
    > PyObject *resultobj;
    > char *path;
    > char *result;
    > if ( PyArg_ParseTuple( args, "s", &path ) == 0 ) {
    > Py_INCREF(Py_None);
    > return Py_None;
    > }
    >
    > result = XMLConf_GetValue( self->root, path );
    > if ( result == NULL ) {
    > Py_INCREF(Py_None);
    > return Py_None;
    > }
    > resultobj = Py_BuildValue("s", result );
    > Py_INCREF( resultobj );
    > return resultobj;
    >}


    First, if ParseTuple fails, an exception is set. In the above, if you really
    want to return None in that case, you should clear the exception
    (PyErr_Clear()). However, it seems much better to report the exception when no
    argument is supplied. Also, Py_BuildValue returns a new reference suitable for
    use as a method result. When you INCREF it, you add a reference which will
    never be DECREF'd. And, anyway, it's easier to use PyString_FromString:

    static PyObject *
    XMLCONF_GetValue(XMLCONFObject *self, PyObject *args)
    {
    char *path;
    char *result;
    if ( !PyArg_ParseTuple( args, "s:GetValue", &path ))
    return NULL;

    result = XMLConf_GetValue( self->root, path );
    if ( result == NULL ) {
    /* perhaps an exception would be better here? */
    Py_INCREF(Py_None);
    return Py_None;
    }
    return PyString_FromString(result);
    }


    >static PyObject *
    >XMLCONF_SetValue(XMLCONFObject *self, PyObject *args) {
    > Py_INCREF(Py_None);
    >}


    I guess you want the above as a stub; you must remember to actually return
    Py_None after INCREFing it. It's probably also worthwhile to at least check the
    parameters, to make sure you got the right number:

    static PyObject *
    XMLCONF_SetValue(XMLCONFObject *self, PyObject *args) {
    char *path, *value;

    if (!PyArg_ParseTuple(args, "ss:SetValue", &path, &value))
    return NULL;

    Py_INCREF(Py_None);
    return Py_None;
    }


    >
    >static PyMethodDef XMLCONF_methods[] = {
    > {"GetValue", (PyCFunction)XMLCONF_GetValue, METH_VARARGS,
    > PyDoc_STR("Gets the Value")},
    > {"SetValue", (PyCFunction)XMLCONF_SetValue, METH_VARARGS,
    > PyDoc_STR("Sets the Value")},
    > {NULL, NULL} /* sentinel */
    >};


    It would be nice to supply more helpful docstrings; at the very least they
    should indicate the method's signature.

    >
    >static PyObject *
    >XMLCONF_getattr(XMLCONFObject *self, char *name)
    >{
    > return Py_FindMethod(XMLCONF_methods, (PyObject *)self, name);
    >}
    >
    >static int
    >XMLCONF_setattr(XMLCONFObject *self, char *name, PyObject *v)
    >{
    > printf("SETATTR\n");
    > return 0;
    >}


    You don't need the above anymore, provided you initialize the type's tp_methods
    slot with XMLCONF_methods (and provided you call PyType_Ready as you are doing).
    So the type object becomes:

    PyTypeObject XMLCONF_Type = {
    /* The ob_type field must be initialized in the module init function
    * to be portable to Windows without using C++. */
    PyObject_HEAD_INIT(NULL)
    0, /*ob_size*/
    "libxmlconf.XMLConf", /*tp_name*/
    sizeof(XMLCONFObject), /*tp_basicsize*/
    0, /*tp_itemsize*/
    /* methods */
    (destructor)XMLCONF_dealloc, /*tp_dealloc*/
    0, /*tp_print*/
    0, /*tp_getattr*/
    0, /*tp_setattr*/
    0, /*tp_compare*/
    0, /*tp_repr*/
    0, /*tp_as_number*/
    0, /*tp_as_sequence*/
    0, /*tp_as_mapping*/
    0, /*tp_hash*/
    0, /*tp_call*/
    0, /*tp_str*/
    0, /*tp_getattro*/
    0, /*tp_setattro*/
    0, /*tp_as_buffer*/
    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/
    0, /*tp_doc*/
    0, /*tp_traverse*/
    0, /*tp_clear*/
    0, /*tp_richcompare*/
    0, /*tp_weaklistoffset*/
    0, /*tp_iter*/
    0, /*tp_iternext*/
    XMLCONF_methods, /*tp_methods*/
    0, /*tp_members*/
    0, /*tp_getset*/
    0, /*tp_base*/
    0, /*tp_dict*/
    0, /*tp_descr_get*/
    0, /*tp_descr_set*/
    0, /*tp_dictoffset*/
    (initproc) XMLCONF_Initialize,/*init*/
    0, /*tp_alloc*/
    newXMLCONFObject, /*tp_new*/
    0, /*tp_free*/
    0, /*tp_is_gc*/
    };


    >
    >/* List of functions defined in the module */
    >
    >static PyMethodDef xx_methods[] = {
    > {NULL, NULL} /* sentinel */
    >};
    >
    >PyDoc_STRVAR(module_doc,
    >"This is a template module just for instruction.");
    >
    >/* Initialization function for the module (*must* be called initxx) */
    >
    >PyMODINIT_FUNC
    >initlibxmlconf(void)
    >{
    > PyObject *m;
    >
    > /* Finalize the type object including setting type of the new type
    > * object; doing it here is required for portability to Windows
    > * without requiring C++. */
    > if (PyType_Ready(&XMLCONF_Type) < 0)
    > return;
    >
    > /* Create the module and add the functions */
    > m = Py_InitModule3("libxmlconf", xx_methods, module_doc);
    > PyModule_AddObject( m, "XMLConf", (PyObject *)&XMLCONF_Type);
    > /* Add some symbolic constants to the module */
    >}
     
    Greg Chapman, Jul 16, 2004
    #2
    1. Advertising

  3. James S

    James S Guest

    Wow. Thank you! Not only did you show be what was wrong, but you told
    me what I was doing wrong. It all makes sense now. Thanks again. It
    works perfect.

    James
     
    James S, Jul 20, 2004
    #3
    1. Advertising

Want to reply to this thread or ask your own question?

It takes just 2 minutes to sign up (and it's free!). Just click the sign up button to choose a username and then you can ask your own questions on the forum.
Similar Threads
  1. Cheung, Jeffrey Jing-Yen
    Replies:
    3
    Views:
    1,563
    Cheung, Jeffrey Jing-Yen
    Jul 10, 2003
  2. Øyvind Isaksen
    Replies:
    1
    Views:
    999
    Øyvind Isaksen
    May 18, 2007
  3. jacob navia

    Garbage collection problems

    jacob navia, Nov 18, 2007, in forum: C Programming
    Replies:
    89
    Views:
    1,696
    David Thompson
    Dec 2, 2007
  4. Carbon Man

    Memory problems (garbage collection)

    Carbon Man, Apr 23, 2009, in forum: Python
    Replies:
    6
    Views:
    377
    Paul Hemans
    Apr 29, 2009
  5. timothytoe
    Replies:
    4
    Views:
    166
Loading...

Share This Page