C_Python不释放缓冲存储器

Question

我正在为python（Python C API）编写C代码，并且我注意到python没有释放文件的内存，我想知道问题是否出在我的代码中。

我想尽可能简化一些，但我希望不会丢失任何细节。

该文件是带有缓冲区的二进制文件，前4个字节是缓冲区大小，然后是缓冲区。

二进制文件（big_file.comp）：

du ~/Desktop/TEST_FILES/big_file.comp
4175416 ~/Desktop/TEST_FILES/big_file.comp

python代码（test.py）：

#!/usr/bin/env python3

from struct import unpack_from
from psutil import Process
from os import getpid
import decomplib


def file_handler(file_name):
    with open(file_name, 'rb') as reader:
        while True:
            next_4_bytes = reader.read(4)
            if next_4_bytes == b'':
                break
            next_size, *_ = unpack_from("I", next_4_bytes)
            buffer = reader.read(next_size)
            yield buffer, next_size


def main():
    args = _parse_args()
    decompress = decomplib.Decompress()
    for buf, buf_size in file_handler(args.file):
        for msg in decompress.decompress_buffer(buf, buf_size):
            print(msg)


if __name__ == "__main__":
    pid = getpid()
    ps = Process(pid)
    main()
    print(ps.memory_info())

简化了某些C代码：

#include <Python.h>
#include "structmember.h"

typedef struct {
    PyObject_HEAD
    uint32_t arr_size;
} DecompressObject;


static int Decompress_init(DecompressObject *self, PyObject *args, PyObject *kwds){
    return 0;
}

static PyObject* Decompress_handle_buffer(DecompressObject* self, PyObject* args){
    uint32_t buf_size = 0;
    uint8_t *buf = NULL;

    // get buffer and buffer length from python function
    if(!PyArg_ParseTuple(args, "y*i", &buf, &buf_size)){
        PyErr_SetString(PyExc_Exception, "Failed to parse function arguments");
        return NULL;
    }

    self->arr_size = 10;
    Py_XINCREF(self);
    return (PyObject *) self;
}

static PyObject* Decompress_next(DecompressObject *self, PyObject *Py_UNUSED(ignored)){
    static uint32_t seq_index = 0;
    if (seq_index < self->arr_size) {
        seq_index++;
        Py_RETURN_NONE;
    }
    seq_index = 0;
    return NULL;
}

static void Decompress_dealloc(DecompressObject *self){
    Py_TYPE(self)->tp_free((PyObject *) self);
}


static PyMethodDef Decompress_methods[] = {
    {"decompress_buffer", (PyCFunction) Decompress_handle_buffer, METH_VARARGS, "Decompress a buffer to asc data."},
    {NULL}  /* Sentinel */
};

static PyTypeObject DecompressType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "decomplib.Decompress",
    .tp_doc = "Decompress object",
    .tp_basicsize = sizeof(DecompressObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_alloc = PyType_GenericAlloc,
    .tp_new = PyType_GenericNew,
    .tp_iter = PyObject_SelfIter,
    .tp_init = (initproc) Decompress_init,
    .tp_dealloc = (destructor) Decompress_dealloc,
    .tp_iternext = (iternextfunc) Decompress_next,
    .tp_methods = Decompress_methods,
};

static PyModuleDef Decompressmodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "decomplib",
    .m_doc = "Decompress an compressed file.",
    .m_size = -1,
};


PyMODINIT_FUNC PyInit_decomplib(void){
    PyObject *d;
    if (PyType_Ready(&DecompressType) < 0)
        return NULL;

    d = PyModule_Create(&Decompressmodule);
    if (d == NULL)
        return NULL;

    Py_INCREF(&DecompressType);
    if (PyModule_AddObject(d, "Decompress", (PyObject *) &DecompressType) < 0) {
        Py_DECREF(&DecompressType);
        Py_DECREF(d);
        return NULL;
    }

    return d;
}

结果，我得到以下输出：

./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=4349915136, vms=4412583936, shared=6270976, text=2867200, lib=0, data=4344135680, dirty=0)

[玩耍时，我注意到如果我在C函数Decompress_handle_buffer中进行了更改，则对函数PyArg_ParseTuple的调用是从"y*i"到"Si"的第二个参数，Python会清理内存...

./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=22577152, vms=84869120, shared=6361088, text=2867200, lib=0, data=16420864, dirty=0)

但是，正确读取缓冲区NOT。有任何想法吗？！

额外信息：

我正在使用虚拟机（VMware Workstation 15）

操作系统Ubuntu 18.4

Python 3.6.9

Answer 1

y*与您正在使用的uint8_t不对应。如documentation中所述，它将填充您应提供的Py_buffer结构。

您实际上需要提供一个Py_buffer，完成后，您需要使用Py_buffer release缓冲区。