C_Python 不释放缓冲内存

Question

我正在为 python (Python C API) 编写 C 代码，我注意到 python 没有释放文件的内存，我想知道问题是否出在我的代码中。

我想尽可能地简化，但我希望不会遗漏任何细节。

该文件是带有缓冲区的二进制文件，前 4 个字节是缓冲区大小，然后是缓冲区。

二进制文件（big_file.comp）：

du ~/Desktop/TEST_FILES/big_file.comp
4175416 ~/Desktop/TEST_FILES/big_file.comp

python 代码（test.py）：

#!/usr/bin/env python3

from struct import unpack_from
from psutil import Process
from os import getpid
import decomplib


def file_handler(file_name):
    with open(file_name, 'rb') as reader:
        while True:
            next_4_bytes = reader.read(4)
            if next_4_bytes == b'':
                break
            next_size, *_ = unpack_from("I", next_4_bytes)
            buffer = reader.read(next_size)
            yield buffer, next_size


def main():
    args = _parse_args()
    decompress = decomplib.Decompress()
    for buf, buf_size in file_handler(args.file):
        for msg in decompress.decompress_buffer(buf, buf_size):
            print(msg)


if __name__ == "__main__":
    pid = getpid()
    ps = Process(pid)
    main()
    print(ps.memory_info())

部分 C 代码简化：

#include <Python.h>
#include "structmember.h"

typedef struct {
    PyObject_HEAD
    uint32_t arr_size;
} DecompressObject;


static int Decompress_init(DecompressObject *self, PyObject *args, PyObject *kwds){
    return 0;
}

static PyObject* Decompress_handle_buffer(DecompressObject* self, PyObject* args){
    uint32_t buf_size = 0;
    uint8_t *buf = NULL;

    // get buffer and buffer length from python function
    if(!PyArg_ParseTuple(args, "y*i", &buf, &buf_size)){
        PyErr_SetString(PyExc_Exception, "Failed to parse function arguments");
        return NULL;
    }

    self->arr_size = 10;
    Py_XINCREF(self);
    return (PyObject *) self;
}

static PyObject* Decompress_next(DecompressObject *self, PyObject *Py_UNUSED(ignored)){
    static uint32_t seq_index = 0;
    if (seq_index < self->arr_size) {
        seq_index++;
        Py_RETURN_NONE;
    }
    seq_index = 0;
    return NULL;
}

static void Decompress_dealloc(DecompressObject *self){
    Py_TYPE(self)->tp_free((PyObject *) self);
}


static PyMethodDef Decompress_methods[] = {
    {"decompress_buffer", (PyCFunction) Decompress_handle_buffer, METH_VARARGS, "Decompress a buffer to asc data."},
    {NULL}  /* Sentinel */
};

static PyTypeObject DecompressType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "decomplib.Decompress",
    .tp_doc = "Decompress object",
    .tp_basicsize = sizeof(DecompressObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_alloc = PyType_GenericAlloc,
    .tp_new = PyType_GenericNew,
    .tp_iter = PyObject_SelfIter,
    .tp_init = (initproc) Decompress_init,
    .tp_dealloc = (destructor) Decompress_dealloc,
    .tp_iternext = (iternextfunc) Decompress_next,
    .tp_methods = Decompress_methods,
};

static PyModuleDef Decompressmodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "decomplib",
    .m_doc = "Decompress an compressed file.",
    .m_size = -1,
};


PyMODINIT_FUNC PyInit_decomplib(void){
    PyObject *d;
    if (PyType_Ready(&DecompressType) < 0)
        return NULL;

    d = PyModule_Create(&Decompressmodule);
    if (d == NULL)
        return NULL;

    Py_INCREF(&DecompressType);
    if (PyModule_AddObject(d, "Decompress", (PyObject *) &DecompressType) < 0) {
        Py_DECREF(&DecompressType);
        Py_DECREF(d);
        return NULL;
    }

    return d;
}

结果，我得到了以下输出：

./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=4349915136, vms=4412583936, shared=6270976, text=2867200, lib=0, data=4344135680, dirty=0)

在玩耍时，我注意到如果我更改 C 函数 Decompress_handle_buffer 对函数 PyArg_ParseTuple 的第二个参数从 "y*i" 到 "Si" 的调用，Python 会清理内存...

./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=22577152, vms=84869120, shared=6361088, text=2867200, lib=0, data=16420864, dirty=0)

但是，缓冲区未正确读取。
有任何想法吗？！

额外信息：

我正在使用虚拟机 (VMware Workstation 15)

操作系统 Ubuntu 18.4

Python 3.6.9

Answer 1

y* 与您正在使用的 uint8_t 不对应。如documentation 中所述，它填充了您应该提供的Py_buffer 结构。

您实际上需要提供一个 Py_buffer，当您完成它时，您需要使用PyBuffer_Release释放该缓冲区。