10.8. Instrumenting the Python Process for Your Structures¶
Some debugging problems can be solved by instrumenting your C extensions for the duration of the Python process and reporting what happened when the process terminates. The data could be: the number of times classes were instantiated, functions called, memory allocations/deallocations or anything else that you wish.
To take a simple case, suppose we have a class that implements a up/down counter and we want to count how often each inc()
and dec()
function is called during the entirety of the Python process. We will create a C extension that has a class that has a single member (an interger) and two functions that increment or decrement that number. If it was in Python it would look like this:
class Counter:
def __init__(self, count=0):
self.count = count
def inc(self):
self.count += 1
def dec(self):
self.count -= 1
What we would like to do is to count how many times inc()
and dec()
are called on all instances of these objects and summarise them when the Python process exits 1.
There is an interpreter hook Py_AtExit()
that allows you to register C functions that will be executed as the Python interpreter exits. This allows you to dump information that you have gathered about your code execution.
10.8.1. An Implementation of a Counter¶
First here is the module pyatexit
with the class pyatexit.Counter
with no intrumentation (it is equivelent to the Python code above). We will add the instrumentation later:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | #include <Python.h>
#include "structmember.h"
#include <stdio.h>
typedef struct {
PyObject_HEAD int number;
} Py_Counter;
static void Py_Counter_dealloc(Py_Counter* self) {
Py_TYPE(self)->tp_free((PyObject*)self);
}
static PyObject* Py_Counter_new(PyTypeObject* type, PyObject* args,
PyObject* kwds) {
Py_Counter* self;
self = (Py_Counter*)type->tp_alloc(type, 0);
if (self != NULL) {
self->number = 0;
}
return (PyObject*)self;
}
static int Py_Counter_init(Py_Counter* self, PyObject* args, PyObject* kwds) {
static char* kwlist[] = { "number", NULL };
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &self->number)) {
return -1;
}
return 0;
}
static PyMemberDef Py_Counter_members[] = {
{ "count", T_INT, offsetof(Py_Counter, number), 0, "count value" },
{ NULL, 0, 0, 0, NULL } /* Sentinel */
};
static PyObject* Py_Counter_inc(Py_Counter* self) {
self->number++;
Py_RETURN_NONE;
}
static PyObject* Py_Counter_dec(Py_Counter* self) {
self->number--;
Py_RETURN_NONE;
}
static PyMethodDef Py_Counter_methods[] = {
{ "inc", (PyCFunction)Py_Counter_inc, METH_NOARGS, "Increments the counter" },
{ "dec", (PyCFunction)Py_Counter_dec, METH_NOARGS, "Decrements the counter" },
{ NULL, NULL, 0, NULL } /* Sentinel */
};
static PyTypeObject Py_CounterType = {
PyVarObject_HEAD_INIT(NULL, 0) "pyatexit.Counter", /* tp_name */
sizeof(Py_Counter), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)Py_Counter_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
"Py_Counter objects", /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
Py_Counter_methods, /* tp_methods */
Py_Counter_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)Py_Counter_init, /* tp_init */
0, /* tp_alloc */
Py_Counter_new, /* tp_new */
0, /* tp_free */
};
static PyModuleDef pyexitmodule = {
PyModuleDef_HEAD_INIT, "pyatexit",
"Extension that demonstrates the use of Py_AtExit().",
-1, NULL, NULL, NULL, NULL,
NULL
};
PyMODINIT_FUNC PyInit_pyatexit(void) {
PyObject* m;
if (PyType_Ready(&Py_CounterType) < 0) {
return NULL;
}
m = PyModule_Create(&pyexitmodule);
if (m == NULL) {
return NULL;
}
Py_INCREF(&Py_CounterType);
PyModule_AddObject(m, "Counter", (PyObject*)&Py_CounterType);
return m;
}
|
If this was a file Py_AtExitDemo.c
then a Python setup.py
file might look like this:
from distutils.core import setup, Extension
setup(
ext_modules=[
Extension("pyatexit", sources=['Py_AtExitDemo.c']),
]
)
Building this with python3 setup.py build_ext --inplace
we can check everything works as expected:
1 2 3 4 5 6 7 8 9 10 11 12 13 | >>> import pyatexit
>>> c = pyatexit.Counter(8)
>>> c.inc()
>>> c.inc()
>>> c.dec()
>>> c.count
9
>>> d = pyatexit.Counter()
>>> d.dec()
>>> d.dec()
>>> d.count
-2
>>> ^D
|
10.8.2. Instrumenting the Counter¶
To add the instrumentation we will declare a macro COUNT_ALL_DEC_INC
to control whether the compilation includes instrumentation.
#define COUNT_ALL_DEC_INC
In the global area of the file declare some global counters and a function to write them out on exit. This must be a void
function taking no arguments:
1 2 3 4 5 6 7 8 9 10 11 12 | #ifdef COUNT_ALL_DEC_INC
/* Counters for operations and a function to dump them at Python process end. */
static size_t count_inc = 0;
static size_t count_dec = 0;
static void dump_inc_dec_count(void) {
fprintf(stdout, "==== dump_inc_dec_count() ====\n");
fprintf(stdout, "Increments: %" PY_FORMAT_SIZE_T "d\n", count_inc);
fprintf(stdout, "Decrements: %" PY_FORMAT_SIZE_T "d\n", count_dec);
fprintf(stdout, "== dump_inc_dec_count() END ==\n");
}
#endif
|
In the Py_Counter_new
function we add some code to register this function. This must be only done once so we use the static has_registered_exit_function
to guard this:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | static PyObject* Py_Counter_new(PyTypeObject* type, PyObject* args,
PyObject* kwds) {
Py_Counter* self;
#ifdef COUNT_ALL_DEC_INC
static int has_registered_exit_function = 0;
if (! has_registered_exit_function) {
if (Py_AtExit(dump_inc_dec_count)) {
return NULL;
}
has_registered_exit_function = 1;
}
#endif
self = (Py_Counter*)type->tp_alloc(type, 0);
if (self != NULL) {
self->number = 0;
}
return (PyObject*)self;
}
|
Note
Py_AtExit
can take, at most, 32 functions. If the function can not be registered then Py_AtExit
will return -1.
Warning
Since Python’s internal finalization will have completed before the cleanup function, no Python APIs should be called by any registered function.
Now we modify the inc()
and dec()
functions thus:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | static PyObject* Py_Counter_inc(Py_Counter* self) {
self->number++;
#ifdef COUNT_ALL_DEC_INC
count_inc++;
#endif
Py_RETURN_NONE;
}
static PyObject* Py_Counter_dec(Py_Counter* self) {
self->number--;
#ifdef COUNT_ALL_DEC_INC
count_dec++;
#endif
Py_RETURN_NONE;
}
|
Now when we build this extension and run it we see the following:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | >>> import pyatexit
>>> c = pyatexit.Counter(8)
>>> c.inc()
>>> c.inc()
>>> c.dec()
>>> c.count
9
>>> d = pyatexit.Counter()
>>> d.dec()
>>> d.dec()
>>> d.count
-2
>>> ^D
==== dump_inc_dec_count() ====
Increments: 2
Decrements: 3
== dump_inc_dec_count() END ==
|
Footnotes
- 1
The
atexit
module in Python can be used to similar effect however registered functions are called at a different stage of interpreted teardown thanPy_AtExit
.