MPI_Scatterv submatrix with MPI_Type_struct

Question

I'm currently working on a MPI-program and I'm trying to send blocks of a matrix with scatterv to all processes.

Process description

The matrix is given as an array. First I produce a datatype with MPI_Type_vector to create the necessary block out of the original array. Second I create a MPI_Type_struct that should hold rows of blocks.

#include <math.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>

#define n 16

int main(int argc, char *argv[])
{
  MPI_Init(&argc, &argv);
  MPI_Comm comm = MPI_COMM_WORLD;
  int p,r;
  MPI_Comm_size(comm, &p);
  MPI_Comm_rank(comm, &r);
  int *arr;
  arr = NULL;
  if (r == 0){
    arr = (int *) malloc(n * n * sizeof(int));
    for (int i = 0; i < n * n; i++) arr[i] = i;
    for (int i = 0; i < n; i++){
      printf("\n");
      for (int j = 0; j < n; j++)
        printf("%4d", arr[i * n + j]);
    }
  }
  printf("\n");
  int ps = sqrt(p);
  int ns = n / ps;

  if (r == 0) {
    printf("ps: %d ns: %d\n", ps, ns);
  }
  /* create datatype */
  MPI_Datatype block;
  MPI_Type_vector(ns, ns, n, MPI_INT, &block);
  int blocks[ps];
  MPI_Aint displs[ps];
  for (int i = 0; i < ps; i++) { 
    blocks[i] = 1;
    displs[i] = i  * sizeof(int);
  }
  MPI_Datatype types[ps];
  //for (int i = 0; i < ps - 1; i++) types[i] = block;
  //types[ps - 1] = MPI_UB;
  types[0] = block;
  for (int i = 1; i < ps; i++) types[i] = MPI_UB; 
  //types[0] = block;
  //types[1] = MPI_UB;
  if (r == 0) {
    printf("displs:\n");
    for(int i = 0; i < ps; i++) printf("%3ld", displs[i]);
    printf("\n");
  }

  MPI_Datatype row;
  MPI_Type_struct(ps, blocks, displs, types, &row);
  MPI_Type_commit(&row);

  /* prepare scatter */
  int sdispl[p]; int sendcounts[p];
  for (int i = 0; i < p; i++) {
    sdispl[i] = (i % ps) + (i / ps) * (ns * ps);
    sendcounts[i] = 1;
  }
  if (r == 0) {
    printf("sdispl: \n");
    for (int i = 0; i < 4; i++) printf("%3d", sdispl[i]);
    printf("\n");
  }

  int rcv[ns * ns];
  MPI_Scatterv(arr, sendcounts, sdispl, row, rcv, ns * ns, MPI_INT, 0, comm);

  int result = 1;
  if (r == result) {
    printf("result for %d:\n", result);
    for (int i = 0; i < ns * ns; i++) {
      printf("%4d", rcv[i]);
      if ((i+1) % ns == 0) printf("\n");
    }
  }

  if (arr != NULL) free(arr);
  MPI_Finalize();
  return 0;
}

So far the structure of the blocks is correct.

The problem

The block, that was sent to process r = 1 starts with 3 instead of 4. The block for process r = 2 also starts with 6 and the one for process r = 3 starts with 9. For r == 4 it jumps to 48.

What it should do

r start
0 0
1 4
2 8
3 12
4 64
5 68
6 ...
15 204

The help I would need

I think, that I'm making some mistake with displ and sdispl.

Compiling and Running the example

The code is compiled with the folowing command:

mpicc -o main main.c -lm

I run the code with:

mpirun -np 16 ./main

Thanks for any help in advance!

You should check out this [excellent answer](http://stackoverflow.com/a/9271753/620382) to a related question. Not sure what you are really trying to achieve with your struct type, I think you are overcomplicating things. Try to avoid leaving all sorts of //commented out statements in your code. — Zulan, Mar 25 '17 at 10:36

score 0 · Answer 1 · edited May 23 '17 at 12:25

With the hint of Zulan I was able to solve my problem.

The following code is based on the excellent answer to subarrays.

#include <math.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>

#define n 8

void print_arr(int *arr, int x) {
  printf("\n");
  for (int i = 0; i < x*x; i++){
    if (i % x == 0) printf("\n");
    printf("%4d", arr[i]);
  }
  printf("\n");
}

int main(int argc, char *argv[])
{
  MPI_Init(&argc, &argv);
  MPI_Comm comm = MPI_COMM_WORLD;
  int p, r;
  MPI_Comm_size(comm, &p);
  MPI_Comm_rank(comm, &r);

  /* number of proceses in dim x and dim y */
  int ps = sqrt(p);

  /* number of elements in dim x and dim y in sarr */
  int ns = n/ps;

  /* array of data - distributed by process 0 */
  int *arr = NULL;
  if (r==0) {
    arr = (int *) malloc(n * n * sizeof(int));
    for (int i = 0; i < n*n; i++) arr[i] = i;
    print_arr(arr, n);
  }

  MPI_Datatype type, resizedtype;
  int sizes[2] = {n,n};
  int subsizes[2] = {ns,ns};
  int starts[2] = {0,0};

  MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_C, MPI_INT, &type);
  MPI_Type_create_resized(type, 0, ns*sizeof(int), &resizedtype);
  MPI_Type_commit(&resizedtype);

  int counts[p];
  for (int i = 0; i < p; i++) counts[i] = 1;
  int displs[p];
  for (int i = 0; i < p; i++) displs[i] = i%ps + i/ps * ns * ps;

  /* subarray to store distributed data */
  int sarr[ns * ns];

  /* send submatrices to all processes */
  MPI_Scatterv(arr, counts, displs, resizedtype, sarr, ns*ns, MPI_INT, 0, comm);

  /* print received data for process pr */
  int pr = 3;
  if (r == pr)
    print_arr(sarr, ns);

  /* free arr */
  if (arr != NULL) free(arr);

  MPI_Finalize();
  return 0;
}

You can compile the example with

mpicc -o main main.c

and run it with

mpirun -np 4 ./main