How to synchronize and sort prints (tasks) with MPI in C++

Question

Before edit:

I am making a simple code in C++ using MPI to do parallel processing. I do simple tasks like sending and receiving messages from one to another process, exchange messages with MPI_Sendrecv, say Hello, and print the execution time for each process. It works, but outputs are not sorted in the order I want (Process 0: tasks... Process 1: tasks ...). I know it is due to no synchronization between processes, and according to my search, I know that MPI_Send,MPI_Recv ... are implicit blocking functions, but it seems that I did not understand how to use this feature. I also tried the MPI_Barrier() function as it is often recommended, but without success. I run 8 processes. Maybe someone can help me? Thanks in advance. Here is my code:


#include <mpi.h>
#include <iostream>
using namespace std;


int main(int argc, char* argv[])  

{

    int rank,nbproc, length;
    char name[80];
    float time;
    double SendData, ReceiveData;
    int tag = 1; 
    int NumTo6 = 500;
    int NumTo7 = 300;
    int ReceiveFrom6, ReceiveFrom7;
 
    
    char message[] = "pokay";
    char receive[] = "none";
    int longueur = strlen(message); 

    SendData = 1254.3356;
    ReceiveData = 0;
    MPI_Init(&argc, &argv); 

    time = MPI_Wtime();
    cout << " " << endl;
    
    MPI_Comm_size(MPI_COMM_WORLD, &nbproc); 
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);    
   // MPI_Get_processor_name(name, &length);
    
    cout << "Hello from process\t" << rank << endl;

    if (rank == 1) 
    {
        cout << "2*5 = " << 2*5 << endl;
    }    

    
    if (rank == 2) 
    {
        MPI_Send(&SendData,1,MPI_DOUBLE,3,tag,MPI_COMM_WORLD); 
    }

    if (rank == 3) 
    {
        cout << "Data before the reception:\t" << ReceiveData << endl; 
        MPI_Recv(&ReceiveData,1,MPI_DOUBLE,2,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE); 
        cout << "Data received is :\t" << ReceiveData << endl;
        tag+=1;
    }


    if (rank == 4)
    {
        MPI_Send(&message[1],4,MPI_CHAR,5,tag,MPI_COMM_WORLD); 
    }

    if (rank == 5)
    {   

        cout << "Message before the reception:\t" << receive << endl;
        MPI_Recv(&receive,longueur+1,MPI_CHAR,4,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        cout << "Message after reception:\t" << receive << endl;
        tag+=1;
    }
// Exchange between 2 processes:
    if(rank == 6)
    {
        MPI_Sendrecv(&NumTo7,1,MPI_INT,7,tag,&ReceiveFrom7,1,MPI_INT,7,tag+1,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        cout << "Num receive from 7:\t" << ReceiveFrom7 << endl;
    }

    if(rank == 7)
    {
        MPI_Sendrecv(&NumTo6,1,MPI_INT,6,tag+1,&ReceiveFrom6,1,MPI_INT,6,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        cout << "Num receive from 6:\t" << ReceiveFrom6 << endl;
        tag+=2;  
    }


    time = MPI_Wtime() - time;
    cout << "Time spend on process " << rank << " is: " << time << " sec" << endl;
    
    MPI_Finalize(); 
  
    return 0;
}

And here are my outputs:

Hello from process      6
 
Hello from process      7
Num receive from 6:     300
Time spend on process 7 is: 6.0746e-05 sec
 
Hello from process      2
Time spend on process 2 is: 5.0439e-05 sec
 
Hello from process      3
Data before the reception:      0
Data received is:      1254.34
Time spend on process 3 is: 0.000439355 sec
 
Hello from process      4
Time spend on process 4 is: 6.2342e-05 sec
 
Hello from process      5
Message before the reception:   none
Message after reception:        okay
Time spend on process 5 is: 0.000168845 sec
 
Hello from process      1
2*5 = 10
Time spend on process 1 is: 0.000132448 sec
 
Hello from process      0
Time spend on the process 0 is: 3.9762e-05 sec
Num receive from 7:     500
Time spend on process 6 is: 0.00206599 sec

EDIT with @VictorEijkhout 's comment:

I can print out almost all I want and how I want it, except with the MPI_Gather() of char... (see after my code).

My new code:

#include <mpi.h>
#include <iostream>
#include <math.h>
#include <string>
#include <cstring>
#include <stdlib.h>
using namespace std;


int main(int argc, char* argv[]) 

{

    int rang,nbproc, taille;
    char name[80];
    float time;
    double SendData, ReceiveData;
    double NumTo6 = 500;
    double NumTo7 = 300;
    double ReceiveFrom6, ReceiveFrom7;

    
    char message[] = "precu";
    int longueur = strlen(message); 


    int len_buffer = 200;
    char Buffer_Time[len_buffer];
    char Buffer_Hello[len_buffer];
    char Buffer_message[len_buffer];

    char receive[] = "none";    

    int mylen = strlen(receive);
    char* Gathered_Char_message = new char[len_buffer];
    double DataMessage;
    double* GatheredDataMessage = new double[20]; 
    double* GateredDataTime = new double[20]; 
    double DataTime;
    int elements[] = {};
    int source = 0;
    int GatheredSources[] = {};
    double NoData = NAN; 
    SendData = 1254.3356;
    ReceiveData = 0;

    cout << " " << endl;

    MPI_Init(&argc, &argv); 
    time = MPI_Wtime();
    
   
    MPI_Comm_size(MPI_COMM_WORLD, &nbproc); 
    MPI_Comm_rank(MPI_COMM_WORLD, &rang);  
    MPI_Get_processor_name(name, &taille);
    
    sprintf(Buffer_Hello,"Hello from process %d among %d of the machine %s",rang,nbproc,name);
    sprintf(Buffer_Time,"Time elapsed in process %d on %d is " ,rang,nbproc);
    sprintf(Buffer_message,"Data received from process ");
    
    
    MPI_Send(Buffer_Time,len_buffer,MPI_CHAR,0,rang+20,MPI_COMM_WORLD);
    MPI_Send(Buffer_Hello,len_buffer,MPI_CHAR,0,rang+10,MPI_COMM_WORLD);
    MPI_Send(Buffer_message,len_buffer,MPI_CHAR,0,rang+30,MPI_COMM_WORLD);

    if (rang == 1) 
    {
        DataMessage = 5*6; 
        source = 1;
    }    
    
    if (rang == 2) 
    {
        MPI_Send(&SendData,1,MPI_DOUBLE,3,1,MPI_COMM_WORLD);
        DataMessage = NoData; 

    }

    if (rang == 3) 
    {
       
        MPI_Recv(&ReceiveData,1,MPI_DOUBLE,2,1,MPI_COMM_WORLD,MPI_STATUS_IGNORE); 
        DataMessage = ReceiveData;
        source = 2;
    }

    if (rang == 4)
    {
  
        MPI_Send(&message[1],longueur+1,MPI_CHAR,5,2,MPI_COMM_WORLD); 
        DataMessage = NoData;
  
        
    }
 
    if (rang == 5)
    {   
        MPI_Recv(&receive,longueur+1,MPI_CHAR,4,2,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        DataMessage = NoData; 
        source = 4;
    }
// Exchange between 2 processes:
    if(rang == 6)
    {
        MPI_Sendrecv(&NumTo7,1,MPI_DOUBLE,7,3,&ReceiveFrom7,1,MPI_DOUBLE,7,4,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        DataMessage = ReceiveFrom7;
        elements[rang] = 1; 
        source = 7;
    }

    if(rang == 7)
    {
        MPI_Sendrecv(&NumTo6,1,MPI_DOUBLE,6,4,&ReceiveFrom6,1,MPI_DOUBLE,6,3,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        DataMessage = ReceiveFrom6;
        elements[rang] = 1; 
        source = 6;
    }

    DataTime = MPI_Wtime() - time;

    MPI_Gather(&DataTime,1,MPI_DOUBLE,GateredDataTime,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
    MPI_Gather(&DataMessage,1,MPI_DOUBLE,GatheredDataMessage,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
    MPI_Gather(&source,1,MPI_INT,GatheredSources,1,MPI_INT,0,MPI_COMM_WORLD);


    // int* recvcounts = new int[nbproc*sizeof(int)];

    
    // MPI_Gather(&mylen,1,MPI_INT,recvcounts,1,MPI_INT,0,MPI_COMM_WORLD); 
    
    // int totlen = 0;
    // int* displs = new int[nbproc*sizeof(int)];
    // //char* totalstring = new char[totlen*sizeof(char)];

    // if(rang == 0)
    // {
    //     displs[0] = 0;
    //     totlen += recvcounts[0] + 1;

    //     for(int i=1; i< nbproc; i++)
    //     {
    //         totlen += recvcounts[i]+1;
    //         displs[i] = displs[i-1] + recvcounts[i-1] + 1;
    //     }
    // }
    // char* totalstring = new char[totlen*sizeof(char)];

    // if(rang == 0)
    // {
    //     for (int i=0; i<totlen-1; i++)
    //         totalstring[i] = ' ';
        
    //     totalstring[totlen-1] = '\0';
    // }

    //    MPI_Gatherv(&receive, mylen, MPI_CHAR,
    //             totalstring, recvcounts, displs, MPI_CHAR,
    //             0, MPI_COMM_WORLD);

   if(rang == 0)
   {
        cout << Buffer_Hello << endl;

        for(int i = 1; i < nbproc; i++)
        {
           MPI_Recv(Buffer_Hello,len_buffer,MPI_CHAR,i,i+10,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
           MPI_Recv(Buffer_message,len_buffer,MPI_CHAR,i,i+30,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
           cout << Buffer_Hello << endl;
        
           if(isnan(GatheredDataMessage[i]))
           {

           }
           else 
           {
               cout << Buffer_message << GatheredSources[i] << ": "<<
               GatheredDataMessage[i] << endl;
           }  


            // cout << totalstring[i] << endl;

            MPI_Recv(Buffer_Time,len_buffer,MPI_CHAR,i,i+20,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
            cout << Buffer_Time << GateredDataTime[i] << " sec" << endl;
            cout << " " << endl;
        }
        
   }

    delete[] GatheredDataMessage; 
    delete[] GateredDataTime;  
    MPI_Finalize(); 
    return 0;
}

And outputs:

Hello from process 0 among 8 of the machine jeremy-SATELLITE-P50-C
Hello from process 1 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 1: 30
Time elapsed in process 1 on 8 is 0.000248922 sec
 
Hello from process 2 among 8 of the machine jeremy-SATELLITE-P50-C
Time elapsed in process 2 on 8 is 0.00013139 sec
 
Hello from process 3 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 2: 1254.34
Time elapsed in process 3 on 8 is 0.000183373 sec
 
Hello from process 4 among 8 of the machine jeremy-SATELLITE-P50-C
Time elapsed in process 4 on 8 is 0.000121771 sec
 
Hello from process 5 among 8 of the machine jeremy-SATELLITE-P50-C
Time elapsed in process 5 on 8 is 0.00027475 sec
 
Hello from process 6 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 7: 500
Time elapsed in process 6 on 8 is 0.00330783 sec
 
Hello from process 7 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 6: 300
Time elapsed in process 7 on 8 is 0.000215519 sec

So I am close to what I want, it misses the gather and print of char that process 4 and 5 exchange, and print out the initial char "nothing" for all others processes. I tried several things like this for instance: https://stackoverflow.com/a/31932283/14901229, you can see that in my code with comments, but then nothing is printed ...

Maybe you could help me with this last thing? Also, if you see ways of optimization for my code (and I think there are...), please don't hesitate to tell me them!

Thanks in advance!

Output is handled by the operating system. You can not synchronize it from within MPI. The only MPI-based solution is to send all the text to process zero and have that one do the printing. In your case you could also `MPI_Gather` the numbers and have process zero print them with accompanying text. — Victor Eijkhout, Aug 14 '21 at 12:07
If process 4 & 5 have an independent print, that again can not be synchronized. Live with it, or send the buffer to process zero. But really, what's the point of printing? Large scale applications never print in parallel preicsely because it can not be synchronized, and if they do , they provide an explicit mechanism for gathering the data. You could also use a shell mechanism (does your mpistarter define an environment variable with the process number?) to send the output of each process to a different file. Or you could write an `MPI_printf` function that prefixes the process number. — Victor Eijkhout, Aug 21 '21 at 16:03

score 0 · Answer 1 · answered Aug 23 '21 at 09:07

I finally found a way to do all these things, for those who are interested I link the code below. It works as I want.

However, I think it is not the better way, so if someone could explain to me how I can optimize it, I will be grateful!

Here is my full code with some explanation comments:


#include <mpi.h>
#include <iostream>
#include <math.h>
using namespace std;


int main(int argc, char* argv[]) 
{

    int rank,nbproc, taille;
    char name[80];
    float time;
    double SendData, ReceiveData;
    double NumTo6 = 500;
    double NumTo7 = 300;
    double ReceiveFrom6, ReceiveFrom7;

    char message[] = "preceived";
    int longueur = strlen(message); 
    int len_buffer = 200;
    char Buffer_Time[len_buffer];
    char Buffer_Hello[len_buffer];
    char Buffer_message[len_buffer];
    char receive[] = "nothing"; 

    char* Gathered_Char_message = new char[len_buffer];
    double DataMessage;
    double* GatheredDataMessage = new double[20]; 
    double* GateredDataTime = new double[20]; 
    double DataTime;
    int source = 0;
    int* GatheredSources = new int[20];
    double NoData = NAN; 
    SendData = 1254.3356;
    ReceiveData = 0;

    cout << " " << endl;

    MPI_Init(&argc, &argv); 
    time = MPI_Wtime();

    
    MPI_Comm_size(MPI_COMM_WORLD, &nbproc); 
    MPI_Comm_rank(MPI_COMM_WORLD, &rank); 
    MPI_Get_processor_name(name, &taille);
    
    sprintf(Buffer_Hello,"Hello from process %d among %d of the machine %s",rank,nbproc,name);
    sprintf(Buffer_Time,"Time elapsed in process %d on %d is " ,rank,nbproc);
    sprintf(Buffer_message,"Data received from process ");

    
    MPI_Send(Buffer_Time,len_buffer,MPI_CHAR,0,rank+20,MPI_COMM_WORLD);
    MPI_Send(Buffer_Hello,len_buffer,MPI_CHAR,0,rank+10,MPI_COMM_WORLD);
    MPI_Send(Buffer_message,len_buffer,MPI_CHAR,0,rank+30,MPI_COMM_WORLD);
    
    if (rank == 1) 
    {
        DataMessage = 5*6; 
        source = 1;
    }    

    
    if (rank == 2) 
    {
        MPI_Send(&SendData,1,MPI_DOUBLE,3,1,MPI_COMM_WORLD); 
        DataMessage = NoData; 
    

    }
  
    if (rank == 3) 
    {
        MPI_Recv(&ReceiveData,1,MPI_DOUBLE,2,1,MPI_COMM_WORLD,MPI_STATUS_IGNORE); 
        DataMessage = ReceiveData;
        source = 2;
    }

    if (rank == 4)
    {
        MPI_Send(&message[1],longueur+1,MPI_CHAR,5,2,MPI_COMM_WORLD); // takes only a part of the message 
        DataMessage = NoData; 
    }

    if (rank == 5)
    {   
        MPI_Recv(&receive,longueur+1,MPI_CHAR,4,2,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        DataMessage = NoData; 
        source = 4;
    }

    if(rank == 6)
    {
        MPI_Sendrecv(&NumTo7,1,MPI_DOUBLE,7,3,&ReceiveFrom7,1,MPI_DOUBLE,7,4,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
        DataMessage = ReceiveFrom7;
        source = 7;
    }

    if(rank == 7)
    {
        MPI_Sendrecv(&NumTo6,1,MPI_DOUBLE,6,4,&ReceiveFrom6,1,MPI_DOUBLE,6,3,MPI_COMM_WORLD,MPI_STATUS_IGNORE); 
        DataMessage = ReceiveFrom6;
        source = 6;
    }

   
   
    MPI_Gather(&DataMessage,1,MPI_DOUBLE,GatheredDataMessage,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
    MPI_Gather(&source,1,MPI_INT,GatheredSources,1,MPI_INT,0,MPI_COMM_WORLD);

   
    int mylen = strlen(receive); // takes back the length of receive 
    int* recvcounts = new int[nbproc*sizeof(int)]; // stores all the lengths after gathering

    MPI_Gather(&mylen,1,MPI_INT,recvcounts,1,MPI_INT,0,MPI_COMM_WORLD); // Gathering all lengths
    
    int totlen = 0; // Total length of the string gathered with spaces
    int* displs = new int[nbproc*sizeof(int)]; // stores all spaces


    if(rank == 0)
    {
        displs[0] = 0; // no space first
        totlen += recvcounts[0] + 1; // first word + space

        for(int i=1; i < nbproc; i++)
        {
            totlen += recvcounts[i]+1; // (+ space and + \0 for the last iteration)
            displs[i] = displs[i-1] + recvcounts[i-1] + 1; // previous space + length of previous word + 1 for the \0
        }
    }


    char* totalstring = new char[totlen*sizeof(char)]; // Big string with all the 'receive' words and spaces

    if(rank == 0)
    {
        for (int i=0; i<totlen; i++) // totlen = 65 -> 7*7 + 8 + 8(spaces + \0)
            totalstring[i] = ' '; // pre allocation
        
        totalstring[totlen] = '\0'; // end of string
    }
  
    // Gather all the 'receive' words from each process to create a big string with spaces between words
       MPI_Gatherv(&receive, mylen, MPI_CHAR,
                totalstring, recvcounts, displs, MPI_CHAR,
                0, MPI_COMM_WORLD);
   

    char* piece = strtok(totalstring," "); // string function for splitting strings/ char array 
    //with a delimiter (here a space)
 

    DataTime = MPI_Wtime() - time;
    MPI_Gather(&DataTime,1,MPI_DOUBLE,GateredDataTime,1,MPI_DOUBLE,0,MPI_COMM_WORLD);

   if(rank == 0)
   {
        cout << Buffer_Hello << endl;
        cout << "Message received: " << piece << endl;
        cout << " " << endl;

        for(int i = 1; i < nbproc; i++)
        {
           MPI_Recv(Buffer_Hello,len_buffer,MPI_CHAR,i,i+10,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
           MPI_Recv(Buffer_message,len_buffer,MPI_CHAR,i,i+30,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
           cout << Buffer_Hello << endl;
          
           if(isnan(GatheredDataMessage[i]))
           {

           }
           else 
           {
               cout << Buffer_message << GatheredSources[i] << ": "<<
               GatheredDataMessage[i] << endl;
           }  

             if(piece != NULL)  // Stop condition when it reads the \0 
            {
                piece = strtok(NULL," "); // Alternativelly, a null pointer may be specified, 
                //in which case the function continues scanning where a previous successful call to the function ended.
                cout << "Message received: " << piece << endl;
            }
        
            

            MPI_Recv(Buffer_Time,len_buffer,MPI_CHAR,i,i+20,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
            cout << Buffer_Time << GateredDataTime[i] << " sec" << endl;
            cout << " " << endl;
        }
        cout << "Time elapsed in process 0 on 8 is " << MPI_Wtime() - time << "sec " << endl;
   }

    delete[] GatheredDataMessage; 
    delete[] GateredDataTime;  
    MPI_Finalize(); 
   
    return 0;
}

And here are my outputs:

Hello from process 0 among 8 of the machine jeremy-SATELLITE-P50-C
Message received: nothing
 
Hello from process 1 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 1: 30
Message received: nothing
Time elapsed in process 1 on 8 is 0.000141763 sec
 
Hello from process 2 among 8 of the machine jeremy-SATELLITE-P50-C
Message received: nothing
Time elapsed in process 2 on 8 is 0.000560879 sec
 
Hello from process 3 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 2: 1254.34
Message received: nothing
Time elapsed in process 3 on 8 is 0.0037692 sec
 
Hello from process 4 among 8 of the machine jeremy-SATELLITE-P50-C
Message received: nothing
Time elapsed in process 4 on 8 is 0.000128876 sec
 
Hello from process 5 among 8 of the machine jeremy-SATELLITE-P50-C
Message received: received
Time elapsed in process 5 on 8 is 0.00140292 sec
 
Hello from process 6 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 7: 500
Message received: nothing
Time elapsed in process 6 on 8 is 0.00226591 sec
 
Hello from process 7 among 8 of the machine jeremy-SATELLITE-P50-C
Data received from process 6: 300
Message received: nothing
Time elapsed in process 7 on 8 is 0.000495762 sec
 
Time elapsed in process 0 on 8 is 0.00330377sec

How to synchronize and sort prints (tasks) with MPI in C++

1 Answers1