使用 MPI_Gather 接收数据结构的正确方法

Question

我尝试使用 MPI_Gather 发送以下数据结构：

struct set {
    int nbits;
    char  bits[];
};

问题是我无法收集上述结构的所有项目，只有第一项。剩下的项目根本没有意义。

这是一个测试用例：

#include <stdio.h>
#include <stdlib.h>

#include "mpi.h"

#define SIZE 10

struct set {
    int nbits;
    char bits[];
};

int main(int argc, char *argv[]) {
    int np, rank, i;
    struct set *subsets, *single;
    void *buf;

    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &np);

    single = malloc(sizeof(struct set) + SIZE);

    if(rank == 0) {
            subsets = malloc((sizeof(struct set) + SIZE) * np);
    }

    buf = &subsets[0];

    MPI_Datatype set_type, oldtypes[2];
    int blockcounts[2];
    MPI_Aint offsets[2];
    MPI_Aint addr[3];

    MPI_Get_address(single, &addr[0]);
    MPI_Get_address(&single->nbits, &addr[1]);
    MPI_Get_address(&single->bits, &addr[2]);

    offsets[0] = addr[1] - addr[0];
    oldtypes[0] = MPI_INT;
    blockcounts[0] = 1;

    offsets[1] = addr[2] - addr[0];
    oldtypes[1] = MPI_CHAR;
    blockcounts[1] = SIZE;

    MPI_Type_create_struct(2, blockcounts, offsets, oldtypes, &set_type);
    MPI_Type_commit(&set_type);

    single->nbits = 2;

    for(i=0; i<single->nbits; i++)
            single->bits[i] = 'A' + rank;

    MPI_Gather(single, 1, set_type, buf, 1, set_type, 0, MPI_COMM_WORLD);

    if(rank == 0) {
            void *ptr;
            struct set *fs;
            int size;

            MPI_Type_size(set_type, &size);

            ptr = buf;

            for(i=0; i<np; i++) {
                    size_t j;

                    fs = ptr;
                    printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                    for(j=0; j<2; j++)
                            printf("from rank %d: buf[%d] = %#x\n",
                                    i, j, fs->bits[j]);
                    ptr += size;
            }
    }

    MPI_Type_free(&set_type);

    MPI_Finalize();
}

任何帮助将不胜感激。

Answer 1

问题不在于MPI，而在于使用结构和 MPI 类型的指针算术。

你有

        void *ptr;
        struct set *fs;
        int size;

        MPI_Type_size(set_type, &size);

        ptr = buf;

        for(i=0; i<np; i++) {
                size_t j;

                fs = ptr;
                printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                for(j=0; j<2; j++)
                        printf("from rank %d: buf[%d] = %#x\n",
                                i, j, fs->bits[j]);
                ptr += size;
        }
}

但是MPI_Type_size实际上给出了类型中的数据量；如果有填充（可能会在这里使字符数组位于单词边界上），这与sizeof 不同。如果你想在这里使用 MPI 函数，如果你将一个函数调用切换到 MPI_Type_extent，它实际上会告诉你该类型跨越的整个范围，你的代码会为我运行......但仍然存在问题。

如果您看一下sizeof(struct set)+SIZE 和MPI_Type_extent() 之间的区别，您会发现它们并不相同；这个：

#define SIZE 10
struct set {
  int nbits
  char nbits[]
}

...

malloc(sizeof(struct set)+SIZE);

不一样

struct set {
  int nbits
  char nbits[SIZE]
}

malloc(sizeof(struct set));

因为padding等原因，这说明subsets的大小不对，调用MPI_Gather时出现内存错误。

您可以通过几种不同的方式解决这个问题，但最简单的（就行数而言也是最短的）是使用已经调整大小的数组定义结构，然后使用数组索引而不是指针算法：

#include <stdio.h>
#include <stdlib.h>

#include "mpi.h"

#define SIZE 10

struct set {
    int nbits;
    char bits[SIZE];
};

int main(int argc, char *argv[]) {
    int np, rank, i;
    struct set *subsets, *single;

    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &np);

    single = malloc(sizeof(struct set));

    if(rank == 0) {
            subsets = malloc(sizeof(struct set) * np);
    }

    MPI_Datatype set_type, oldtypes[2];
    int blockcounts[2];
    MPI_Aint offsets[2];
    MPI_Aint addr[3];

    MPI_Get_address(single, &addr[0]);
    MPI_Get_address(&single->nbits, &addr[1]);
    MPI_Get_address(&single->bits, &addr[2]);

    offsets[0] = addr[1] - addr[0];
    oldtypes[0] = MPI_INT;
    blockcounts[0] = 1;

    offsets[1] = addr[2] - addr[0];
    oldtypes[1] = MPI_CHAR;
    blockcounts[1] = SIZE;

    MPI_Type_create_struct(2, blockcounts, offsets, oldtypes, &set_type);
    MPI_Type_commit(&set_type);

    single->nbits = 2;

    for(i=0; i<single->nbits; i++)
            single->bits[i] = 'A' + rank;

    MPI_Gather(single, 1, set_type, &(subsets[0]), 1, set_type, 0, MPI_COMM_WORLD);

    if(rank == 0) {
            for(i=0; i<np; i++) {
                    struct set *fs = &(subsets[i]);
                    printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                    for(int j=0; j<2; j++)
                            printf("from rank %d: buf[%d] = %#x\n",
                                    i, j, fs->bits[j]);
            }
    }

    MPI_Type_free(&set_type);

    MPI_Finalize();
}

更新到添加如果您不能这样做，只需更改缓冲区分配的大小以将数据收集到：

#include <stdio.h>
#include <stdlib.h>
#include "mpi.h"

#define SIZE 10

struct set {
    int nbits;
    char bits[];
};

int main(int argc, char *argv[]) {
    int np, rank, i;
    struct set *single;
    void *buf;
    ptrdiff_t extent;

    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &np);

    single = malloc(sizeof(struct set) + SIZE);

    MPI_Datatype set_type, oldtypes[2];
    int blockcounts[2];
    MPI_Aint offsets[2];
    MPI_Aint addr[3];

    MPI_Get_address(single, &addr[0]);
    MPI_Get_address(&single->nbits, &addr[1]);
    MPI_Get_address(&single->bits, &addr[2]);

    offsets[0] = addr[1] - addr[0];
    oldtypes[0] = MPI_INT;
    blockcounts[0] = 1;

    offsets[1] = addr[2] - addr[0];
    oldtypes[1] = MPI_CHAR;
    blockcounts[1] = SIZE;

    MPI_Type_create_struct(2, blockcounts, offsets, oldtypes, &set_type);
    MPI_Type_commit(&set_type);

    MPI_Type_extent(set_type, &extent);
    buf = malloc((int)extent * np);

    single->nbits = 2;

    for(i=0; i<single->nbits; i++)
            single->bits[i] = 'A' + rank;

    MPI_Gather(single, 1, set_type, buf, 1, set_type, 0, MPI_COMM_WORLD);

    if(rank == 0) {
            struct set *fs = buf;
            for(i=0; i<np; i++) {
                    printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                    for(int j=0; j<2; j++)
                            printf("from rank %d: buf[%d] = %#x\n",
                                    i, j, fs->bits[j]);

                    fs = (struct set *)((char *)fs + extent);
            }
    }

    MPI_Type_free(&set_type);

    MPI_Finalize();
}