背景
我创建了一个 python 模块，它使用 SWIG 包装了 c++ 程序。它工作得很好，但它有一个非常严重的内存泄漏问题，我认为这是由于指向大型 map 对象的指针处理不当造成的。我对c++的经验很少，并且我对delete[]是否可以在使用new创建的对象上有疑问不同的功能或方法。
该程序是在 2007 年编写的，因此缺少有用的 c++11 技巧。
swig 扩展基本上只包装了一个 C++ 类 (Matrix) 和一些函数。
Matrix.h
#ifndef __MATRIX__
#define __MATRIX__
#include <string>
#include <vector>
#include <map>
#include <cmath>
#include <fstream>
#include <cstdlib>
#include <stdio.h>
#include <unistd.h>
#include "FileException.h"
#include "ParseException.h"
#define ROUND_TO_INT(n) ((long long)floor(n))
#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
using namespace std;
class Matrix {
private:
/**
* Split a string following delimiters
*/
void tokenize(const string& str, vector<string>& tokens, const string& delimiters) {
// Skip delimiters at beginning.
string::size_type lastPos = str.find_first_not_of(delimiters, 0);
// Find first "non-delimiter".
string::size_type pos = str.find_first_of(delimiters, lastPos);
while (string::npos != pos || string::npos != lastPos)
{
// Found a token, add it to the vector.
tokens.push_back(str.substr(lastPos, pos - lastPos));
// Skip delimiters. Note the "not_of"
lastPos = str.find_first_not_of(delimiters, pos);
// Find next "non-delimiter"
pos = str.find_first_of(delimiters, lastPos);
}
}
public:
// used for efficiency tests
long long totalMapSize;
long long totalOp;
double ** mat; // the matrix as it is stored in the matrix file
int length;
double granularity; // the real granularity used, greater than 1
long long ** matInt; // the discrete matrix with offset
double errorMax;
long long *offsets; // offset of each column
long long offset; // sum of offsets
long long *minScoreColumn; // min discrete score at each column
long long *maxScoreColumn; // max discrete score at each column
long long *sum;
long long minScore; // min total discrete score (normally 0)
long long maxScore; // max total discrete score
long long scoreRange; // score range = max - min + 1
long long *bestScore;
long long *worstScore;
double background[4];
Matrix() {
granularity = 1.0;
offset = 0;
background[0] = background[1] = background[2] = background[3] = 0.25;
}
Matrix(double pA, double pC, double pG, double pT) {
granularity = 1.0;
offset = 0;
background[0] = pA;
background[1] = pC;
background[2] = pG;
background[3] = pT;
}
~Matrix() {
for (int k = 0; k < 4; k++ ) {
delete[] matInt[k];
}
delete[] matInt;
delete[] mat;
delete[] offsets;
delete[] minScoreColumn;
delete[] maxScoreColumn;
delete[] sum;
delete[] bestScore;
delete[] worstScore;
}
void toLogOddRatio () {
for (int p = 0; p < length; p++) {
double sum = mat[0][p] + mat[1][p] + mat[2][p] + mat[3][p];
for (int k = 0; k < 4; k++) {
mat[k][p] = log((mat[k][p] + 0.25) /(sum + 1)) - log (background[k]);
}
}
}
void toLog2OddRatio () {
for (int p = 0; p < length; p++) {
double sum = mat[0][p] + mat[1][p] + mat[2][p] + mat[3][p];
for (int k = 0; k < 4; k++) {
mat[k][p] = log2((mat[k][p] + 0.25) /(sum + 1)) - log2 (background[k]);
}
}
}
/**
* Transforms the initial matrix into an integer and offseted matrix.
*/
void computesIntegerMatrix (double granularity, bool sortColumns = true);
// computes the complete score distribution between score min and max
void showDistrib (long long min, long long max) {
map<long long, double> *nbocc = calcDistribWithMapMinMax(min,max);
map<long long, double>::iterator iter;
// computes p values and stores them in nbocc[length]
double sum = 0;
map<long long, double>::reverse_iterator riter = nbocc[length-1].rbegin();
while (riter != nbocc[length-1].rend()) {
sum += riter->second;
nbocc[length][riter->first] = sum;
riter++;
}
iter = nbocc[length].begin();
while (iter != nbocc[length].end() && iter->first <= max) {
//cout << (((iter->first)-offset)/granularity) << " " << (iter->second) << " " << nbocc[length-1][iter->first] << endl;
iter ++;
}
}
/**
* Computes the pvalue associated with the threshold score requestedScore.
*/
void lookForPvalue (long long requestedScore, long long min, long long max, double *pmin, double *pmax);
/**
* Computes the score associated with the pvalue requestedPvalue.
*/
long long lookForScore (long long min, long long max, double requestedPvalue, double *rpv, double *rppv);
/**
* Computes the distribution of scores between score min and max as the DP algrithm proceeds
* but instead of using a table we use a map to avoid computations for scores that cannot be reached
*/
map<long long, double> *calcDistribWithMapMinMax (long long min, long long max);
void readMatrix (string matrix) {
vector<string> str;
tokenize(matrix, str, " \t|");
this->length = 0;
this->length = str.size() / 4;
mat = new double*[4];
int idx = 0;
for (int j = 0; j < 4; j++) {
this->mat[j] = new double[this->length];
for (int i = 0; i < this->length; i++) {
mat[j][i] = atof(str.at(idx).data());
idx++;
}
}
str.clear();
}
}; /* Matrix */
#endif
Matrix.cpp
#include "Matrix.h"
#define MEMORYCOUNT
void Matrix::computesIntegerMatrix (double granularity, bool sortColumns) {
double minS = 0, maxS = 0;
double scoreRange;
// computes precision
for (int i = 0; i < length; i++) {
double min = mat[0][i];
double max = min;
for (int k = 1; k < 4; k++ ) {
min = ((min < mat[k][i])?min:(mat[k][i]));
max = ((max > mat[k][i])?max:(mat[k][i]));
}
minS += min;
maxS += max;
}
// score range
scoreRange = maxS - minS + 1;
if (granularity > 1.0) {
this->granularity = granularity / scoreRange;
} else if (granularity < 1.0) {
this->granularity = 1.0 / granularity;
} else {
this->granularity = 1.0;
}
matInt = new long long *[length];
for (int k = 0; k < 4; k++ ) {
matInt[k] = new long long[length];
for (int p = 0 ; p < length; p++) {
matInt[k][p] = ROUND_TO_INT((double)(mat[k][p]*this->granularity));
}
}
this->errorMax = 0.0;
for (int i = 1; i < length; i++) {
double maxE = mat[0][i] * this->granularity - (matInt[0][i]);
for (int k = 1; k < 4; k++) {
maxE = ((maxE < mat[k][i] * this->granularity - matInt[k][i])?(mat[k][i] * this->granularity - (matInt[k][i])):(maxE));
}
this->errorMax += maxE;
}
if (sortColumns) {
// sort the columns : the first column is the one with the greatest value
long long min = 0;
for (int i = 0; i < length; i++) {
for (int k = 0; k < 4; k++) {
min = MIN(min,matInt[k][i]);
}
}
min --;
long long *maxs = new long long [length];
for (int i = 0; i < length; i++) {
maxs[i] = matInt[0][i];
for (int k = 1; k < 4; k++) {
if (maxs[i] < matInt[k][i]) {
maxs[i] = matInt[k][i];
}
}
}
long long **mattemp = new long long *[4];
for (int k = 0; k < 4; k++) {
mattemp[k] = new long long [length];
}
for (int i = 0; i < length; i++) {
long long max = maxs[0];
int p = 0;
for (int j = 1; j < length; j++) {
if (max < maxs[j]) {
max = maxs[j];
p = j;
}
}
maxs[p] = min;
for (int k = 0; k < 4; k++) {
mattemp[k][i] = matInt[k][p];
}
}
for (int k = 0; k < 4; k++) {
for (int i = 0; i < length; i++) {
matInt[k][i] = mattemp[k][i];
}
}
for (int k = 0; k < 4; k++) {
delete[] mattemp[k];
}
delete[] mattemp;
delete[] maxs;
}
// computes offsets
this->offset = 0;
offsets = new long long [length];
for (int i = 0; i < length; i++) {
long long min = matInt[0][i];
for (int k = 1; k < 4; k++ ) {
min = ((min < matInt[k][i])?min:(matInt[k][i]));
}
offsets[i] = -min;
for (int k = 0; k < 4; k++ ) {
matInt[k][i] += offsets[i];
}
this->offset += offsets[i];
}
// look for the minimum score of the matrix for each column
minScoreColumn = new long long [length];
maxScoreColumn = new long long [length];
sum = new long long [length];
minScore = 0;
maxScore = 0;
for (int i = 0; i < length; i++) {
minScoreColumn[i] = matInt[0][i];
maxScoreColumn[i] = matInt[0][i];
sum[i] = 0;
for (int k = 1; k < 4; k++ ) {
sum[i] = sum[i] + matInt[k][i];
if (minScoreColumn[i] > matInt[k][i]) {
minScoreColumn[i] = matInt[k][i];
}
if (maxScoreColumn[i] < matInt[k][i]) {
maxScoreColumn[i] = matInt[k][i];
}
}
minScore = minScore + minScoreColumn[i];
maxScore = maxScore + maxScoreColumn[i];
//cout << "minScoreColumn[" << i << "] = " << minScoreColumn[i] << endl;
//cout << "maxScoreColumn[" << i << "] = " << maxScoreColumn[i] << endl;
}
this->scoreRange = maxScore - minScore + 1;
bestScore = new long long[length];
worstScore = new long long[length];
bestScore[length-1] = maxScore;
worstScore[length-1] = minScore;
for (int i = length - 2; i >= 0; i--) {
bestScore[i] = bestScore[i+1] - maxScoreColumn[i+1];
worstScore[i] = worstScore[i+1] - minScoreColumn[i+1];
}
}
/**
* Computes the pvalue associated with the threshold score requestedScore.
*/
void Matrix::lookForPvalue (long long requestedScore, long long min, long long max, double *pmin, double *pmax) {
map<long long, double> *nbocc = calcDistribWithMapMinMax(min,max);
map<long long, double>::iterator iter;
// computes p values and stores them in nbocc[length]
double sum = nbocc[length][max+1];
long long s = max + 1;
map<long long, double>::reverse_iterator riter = nbocc[length-1].rbegin();
while (riter != nbocc[length-1].rend()) {
sum += riter->second;
if (riter->first >= requestedScore) s = riter->first;
nbocc[length][riter->first] = sum;
riter++;
}
//cout << " s found : " << s << endl;
iter = nbocc[length].find(s);
while (iter != nbocc[length].begin() && iter->first >= s - errorMax) {
iter--;
}
//cout << " s - E found : " << iter->first << endl;
#ifdef MEMORYCOUNT
// for tests, store the number of memory bloc necessary
for (int pos = 0; pos <= length; pos++) {
totalMapSize += nbocc[pos].size();
}
#endif
*pmax = nbocc[length][s];
*pmin = iter->second;
}
/**
* Computes the score associated with the pvalue requestedPvalue.
*/
long long Matrix::lookForScore (long long min, long long max, double requestedPvalue, double *rpv, double *rppv) {
map<long long, double> *nbocc = calcDistribWithMapMinMax(min,max);
map<long long, double>::iterator iter;
// computes p values and stores them in nbocc[length]
double sum = 0.0;
map<long long, double>::reverse_iterator riter = nbocc[length-1].rbegin();
long long alpha = riter->first+1;
long long alpha_E = alpha;
nbocc[length][alpha] = 0.0;
while (riter != nbocc[length-1].rend()) {
sum += riter->second;
nbocc[length][riter->first] = sum;
if (sum >= requestedPvalue) {
break;
}
riter++;
}
if (sum > requestedPvalue) {
alpha_E = riter->first;
riter--;
alpha = riter->first;
} else {
if (riter == nbocc[length-1].rend()) { // path following the remark of the mail
riter--;
alpha = alpha_E = riter->first;
} else {
alpha = riter->first;
riter++;
sum += riter->second;
alpha_E = riter->first;
}
nbocc[length][alpha_E] = sum;
//cout << "Pv(S) " << riter->first << " " << sum << endl;
}
#ifdef MEMORYCOUNT
// for tests, store the number of memory bloc necessary
for (int pos = 0; pos <= length; pos++) {
totalMapSize += nbocc[pos].size();
}
#endif
if (alpha - alpha_E > errorMax) alpha_E = alpha;
*rpv = nbocc[length][alpha];
*rppv = nbocc[length][alpha_E];
delete[] nbocc;
return alpha;
}
// computes the distribution of scores between score min and max as the DP algrithm proceeds
// but instead of using a table we use a map to avoid computations for scores that cannot be reached
map<long long, double> *Matrix::calcDistribWithMapMinMax (long long min, long long max) {
// maps for each step of the computation
// nbocc[length] stores the pvalue
// nbocc[pos] for pos < length stores the qvalue
map<long long, double> *nbocc = new map<long long, double> [length+1];
map<long long, double>::iterator iter;
long long *maxs = new long long[length+1]; // @ pos i maximum score reachable with the suffix matrix from i to length-1
maxs[length] = 0;
for (int i = length-1; i >= 0; i--) {
maxs[i] = maxs[i+1] + maxScoreColumn[i];
}
// initializes the map at position 0
for (int k = 0; k < 4; k++) {
if (matInt[k][0]+maxs[1] >= min) {
nbocc[0][matInt[k][0]] += background[k];
}
}
// computes q values for scores greater or equal than min
nbocc[length-1][max+1] = 0.0;
for (int pos = 1; pos < length; pos++) {
iter = nbocc[pos-1].begin();
while (iter != nbocc[pos-1].end()) {
for (int k = 0; k < 4; k++) {
long long sc = iter->first + matInt[k][pos];
if (sc+maxs[pos+1] >= min) {
// the score min can be reached
if (sc > max) {
// the score will be greater than max for all suffixes
nbocc[length-1][max+1] += nbocc[pos-1][iter->first] * background[k]; //pow(4,length-pos-1) ;
totalOp++;
} else {
nbocc[pos][sc] += nbocc[pos-1][iter->first] * background[k];
totalOp++;
}
}
}
iter++;
}
//cerr << " map size for " << pos << " " << nbocc[pos].size() << endl;
}
delete[] maxs;
return nbocc;
}
pytfmpval.i
%module pytfmpval
%{
#include "../src/Matrix.h"
#define SWIG_FILE_WITH_INIT
%}
%include "cpointer.i"
%include "std_string.i"
%include "std_vector.i"
%include "typemaps.i"
%include "../src/Matrix.h"
%pointer_class(double, doublep)
%pointer_class(int, intp)
%nodefaultdtor Matrix;
c++ 函数在 python 模块中调用。
我担心 Matrix.cpp 中的 nbocc 没有被正确取消引用或删除。 这种用途有效吗？
我已尝试使用gc.collect()，并且我正在使用此question中推荐的multiprocessing模块。从我的 python 程序中调用这些函数。我还尝试从 python 中删除 Matrix 对象，但无济于事。
我的角色不够，但我会尽力在评论中提供任何其他所需的信息。
更新:我删除了所有 python 代码，因为这不是问题所在，并且导致了一篇冗长的帖子。正如我在下面的评论中所述，通过采纳许多用户的建议并创建一个用纯 C++ 展示该问题的最小示例，最终解决了这个问题。然后，我使用 valgrind 来识别使用 new 创建的有问题的指针，并确保它们被正确取消引用。这修复了几乎所有内存泄漏。仍然存在一个，但它在数千次迭代中只泄漏了几百个字节，并且需要重构整个 Matrix 类，这根本不值得花时间。不好的做法，我知道。对于任何其他 C++ 新手，请认真尝试避免动态内存分配或使用 std::unique_ptr 或 std::shared_ptr。
再次感谢所有提供意见和建议的人。
最佳答案
很难跟踪正在发生的事情，但我很确定你的矩阵没有被正确清理。
在 readMatrix 中，您在 j 上有一个循环，其中包含行 this->mat[j] = new double[this->length]; 。这会分配 mat[j] 指向的内存。该内存需要在某个时刻通过调用delete[] mat[j](或其他一些循环变量)来释放。但是，在析构函数中，您只需调用 delete[] mat，这会泄漏其中的所有数组。
有关清理此问题的一些一般建议:
- 如果您知道数组的边界，例如
matInt的长度始终为 4，则应使用该固定长度声明它 (long long* matInt[4]将创建一个由四个指向long long的指针组成的数组，其中每个指针都可以是指向数组的指针)；这意味着您不需要新建或删除它。 - 如果您有一个像
double ** mat这样的双指针，并且您使用new[]分配第一层和第二层指针，则需要取消分配内层带有delete[](并且您需要在delete[]外层之前执行此操作)。 - 如果您仍然遇到问题，如果删除与问题无关的方法，您的代码将会更加清晰。例如，toLogOddRatio 根本不分配或释放内存；它几乎肯定不会导致问题，您可以从您在此处发布的代码中删除它(一旦您删除了您认为不会造成影响的部分，请再次测试以确保问题仍然存在；如果不是，那么你知道它是导致泄漏的那些部件之一)。
关于python - Swig Python 模块中的 C++ 内存泄漏，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/44935830/