使用 MPI_Alltoallv 时，内存范围重叠

Question

我们正在尝试执行的任务是编写一个程序，使用 MPI 解决 p 个进程上的泊松问题。当试图找到一个矩阵的转置时，问题就出现了，我们必须使用 MPI_Alltoallv，因为对于每个进程，完整的矩阵被分成一个更小的矩阵。我们的代码不完整。当我们到达转置部分时我们停止了，因为 MPI_Alltoallv 给了我们错误：

文件 src/mpid/ch3/src/ch3u_buffer.c 第 77 行的断言失败：FALSE memcpy 参数内存范围重叠，dst_=0x7fff5d5efa80 src_=0x7fff5d5efa88 len_=256，

在 3 个进程上运行且 n = 8 时。(mpirun -np 3 ./program 8)。

链接到失败文件的来源：ch3u_buffer.c

#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
#include <memory.h>
#include <math.h>
#include <mpi.h>

typedef double Real;

/* function prototypes */
Real *createRealArray (int n);
Real **createReal2DArray (int m, int n);
void transpose (Real **bt, Real **b, int m);


int main(int argc, char **argv ) {
Real *diag, **b, **bt, *z; 
double *sndbuf, *rcvbuf;
Real pi, h, umax;
int i, j, n, m, k, nn, size, rank, err, rest, c;

/* the total number of grid points in each spatial direction is (n+1) */
/* the total number of degrees-of-freedom in each spatial direction is (n-1) */
/* this version requires n to be a power of 2 */

if( argc < 2 ) {
    printf("need a problem size\n");
    return 1;
}

//Problem size
n  = atoi(argv[1]);
m  = n-1;
nn = 4*n;

err = MPI_Init(&argc, &argv); /* Initialize MPI */
err = MPI_Comm_size(MPI_COMM_WORLD, &size); /* Get nr of tasks */
err = MPI_Comm_rank(MPI_COMM_WORLD, &rank);    /* Get id of this process */
if (err != MPI_SUCCESS) {
    printf("MPI failed!\n");
    exit(1);
}

int* colArray;
int* sCount;
int* sDispl;
rest = m % size;

colArray = (int*) calloc(size, sizeof(int));
sCount = (int*) calloc(size, sizeof(int));
sDispl = (int*) calloc(size, sizeof(int));

for (i=0; i<size; i++){
    if (rest < size-i) {
        colArray[i] = floor(m/size);
    }
    else {
        colArray[i] = floor(m/size) + 1;
    }  
}

sDispl[0] = 0;
for (i=0; i<size; i++) {
    sCount[i] = colArray[rank]*colArray[i]*sizeof(double);
    if (i>0) sDispl[i] = sDispl[i-1] + sCount[i-1];
    // printf("Rank: %d, sCount[%d]: %d \n", rank, i, sCount[i]);
    // printf("Rank: %d, sDispl[%d]: %d \n", rank, i, sDispl[i]);
}

k = colArray[rank];

diag    = createRealArray (m);
b       = createReal2DArray (m,k);
//sndbuf  = createRealArray(m*k);
//rcvbuf  = createRealArray(m*k);
bt      = createReal2DArray (m,k);
z       = createRealArray (nn);

sndbuf   = (double  *)malloc(m*k*sizeof(double));
rcvbuf   = (double  *)malloc(m*k*sizeof(double));

h    = 1./(Real)n;
pi   = 4.*atan(1.);

for (i=0; i < m; i++) {
    diag[i] = 2.*(1.-cos((i+1)*pi/(Real)n));
}

for (j=0; j < m; j++) {
    for (i=0; i < k; i++) {
        b[j][i] = h*h;
    }
}

if (rank == 2) {
    printf("Matrise for rank: %d \n", rank);
    for (j=0; j < m; j++) {
        for (i=0; i < k; i++) {
            printf("%f \t", b[j][i]);
        }
        printf("\n");
    }
}

c = 0;
for (i=0; i<m; i++) {
    for (j=0; j<m; j++) {
        sndbuf[c] = (double)b[i][j];
        c++;
    }
}

if (rank == 2) {
    printf("sndbuf fra rank: %d \n", rank);
    for (i=0;i<m*k;i++) {
        printf("%f\n", sndbuf[i]);
    }
}   

if (rank == 2) {
    printf("sCount fra rank: %d \n", rank);
    for (i=0; i<size; i++) {
        printf("%d \t", sCount[i]);
    }
    printf("\n");
    printf("sDispl fra rank: %d \n", rank);
    for (i=0; i<size; i++) {
        printf("%d \t", sDispl[i]);
    }
}


err = MPI_Alltoallv(&sndbuf, sCount, sDispl, MPI_DOUBLE, 
                    &rcvbuf, sCount, sDispl, MPI_DOUBLE, 
                    MPI_COMM_WORLD);

if (err != MPI_SUCCESS) {
    printf("Error in MPI_Alltoallv!\n");
    exit(1);
}


// transpose (bt,b,m);


err = MPI_Finalize();          /* Terminate MPI */
if (err != MPI_SUCCESS) {
    printf("Error in MPI_Finalize!\n");
    exit(1);
}

return 0;
}

void transpose (Real **bt, Real **b, int m)
{
int i, j;
for (j=0; j < m; j++) {
for (i=0; i < m; i++) {
bt[j][i] = b[i][j];
}
}
}

Real *createRealArray (int n)
{
Real *a;
int i;
a = (Real *)malloc(n*sizeof(Real));
for (i=0; i < n; i++) {
a[i] = 0.0;
}
return (a);
}

Real **createReal2DArray (int n1, int n2)
{
int i, n;
Real **a;
a    = (Real **)malloc(n1   *sizeof(Real *));
a[0] = (Real  *)malloc(n1*n2*sizeof(Real));
for (i=1; i < n1; i++) {
a[i] = a[i-1] + n2;
}
n = n1*n2;
memset(a[0],0,n*sizeof(Real));
return (a);
}

我猜 MPI_Alltoallv 的输入一定有问题。有人可以帮我找出它是什么吗？如果代码有点乱，请见谅。

编辑：我尝试从 MPI_Alltoallv(&sndbuf .. &rcvbuf, ...); 更改到 MPI_Alltoallv(sndbuf .. rcvbuf, ...);正如有人所说，但它仍然给出了同样的错误。

Answer 1

您错误地计算了给MPI_Alltoallv 的计数和位移。当您以字节为单位计算它们时，两者都应该以相应 MPI 数据类型的数据元素为单位：

sDispl[0] = 0;
for (i=0; i<size; i++) {
    sCount[i] = colArray[rank]*colArray[i]*sizeof(double);
                                          ^^^^^^^^^^^^^^^---- problem
    if (i>0) sDispl[i] = sDispl[i-1] + sCount[i-1];
    // printf("Rank: %d, sCount[%d]: %d \n", rank, i, sCount[i]);
    // printf("Rank: %d, sDispl[%d]: %d \n", rank, i, sDispl[i]);
}

带下划线的乘法很可能是罪魁祸首。 MPI 尝试访问超过其长度 7 倍的已分配内存。在同一线程内连续调用malloc() 可能会返回位置很近的内存块，因此会出现重叠错误。不要乘以sizeof(double)。