mpi

[daniel_unb]

Hi all,
I am mulling about a question on mpi, which is not really a particular
issue of parallel programing. I am trying to spawn a grid of 200x200
independent initial conditions to a smp machine using mpi. After a
while I managed to do a simple code using mpi which sends blocks of
initial conditions to each processor.

It turns out that this is not so efficient because some processors end
the job before others. It would be best if each process begins as the
previous ends, in such a way that all processors keep working until
the last step of the calculation.

Can anyone help me out with this very simple question? I know some
tuturials, but was unable to find an answer.

Thanks,
Daniel

[daniel_unb]

here is the code I did. Also, it could not make it work for a grid of
250x250 for example... for smaller grids it works fine. I don't know
why.

#include <stdio.h>
#include <math.h>
#include "mpi.h"
//int
main (int argc,char *argv[])
{
FILE *fp;
int rank,kntrl,ct;
int dest;
int size,k,i,j,l,idade,n;
int source;
int tag=0;
int N=200;
double mem[2];
MPI_Status status;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&size);
int mens[2][2*N*2*N],s[2][size][2*N*2*N/size+2*N];
double m[2][2*N*2*N],r[2][size][2*N*2*N/size+2*N];
fp=fopen("t.dat","w");
for (i=-N;i<N;i++){
for(l=-N;l<N;l=l+size){
for(k=0;k<size;k++){
if(l+k<N){
if(k==rank){
m[1][(i+N)*(2*N)+l+k+N]=(200.0)*(l+k)/N;
m[0][(i+N)*(2*N)+l+k+N]=(200.0)*i/N;
mens[0][(i+N)*(2*N)+l+k+N]=k+l;
mens[1][(i+N)*(2*N)+l+k+N]=k;
}
}
else
;
}
}
}
for(k=0;k<size;k++){
if(k==rank){
j=0;
for (i=0;i<2*N;i++){
for(l=0;l<2*N;l=l+size){
if(l+k<2*N){
for(n=0;n<2;n++){
s[n][k][j]=mens[n][i*(2*N)+l+k];
r[n][k][j]=m[n][i*(2*N)+l+k];
}
j++;
}
}
}
}
}
for(k=0;k<size;k++){
if(k==rank){
for(n=0;n<2;n++){
MPI_Send(s[n][k],2*N*2*N/size+2*N,MPI_INT,0,tag,MPI_COMM_WORLD);
MPI_Send(r[n][k],2*N*2*N/size+2*N,MPI_DOUBLE,0,tag,MPI_COMM_WORLD);
}
}
}
if(0==rank){
for(k=0;k<size;k++){
for(n=0;n<2;n++){
MPI_Recv(s[n][k],2*N*2*N/size
+2*N,MPI_INT,k,tag,MPI_COMM_WORLD,&status);
MPI_Recv(r[n][k],2*N*2*N/size
+2*N,MPI_DOUBLE,k,tag,MPI_COMM_WORLD,&status);
}
j=0;
for (i=0;i<2*N;i++){
for(l=0;l<2*N;l=l+size){
if(l+k<2*N){
for(n=0;n<2;n++){
mens[n][i*(2*N)+l+k]=s[n][k][j];
m[n][i*(2*N)+l+k]=r[n][k][j];
}
j++;
}
}
}
}
for (i=0;i<2*N;i=i++){
for(l=0;l<2*N;l++){
fprintf(fp,"%.5e %.5e %d %d \n",m[0][i*(2*N)+l],m[1][i*(2*N)
+l],mens[0][i*(2*N)+l],mens[1][i*(2*N)+l]);
}
fprintf(fp,"\n ");
}
}
MPI_Finalize();
fclose(fp);
}

[daniel_unb]

just a correction to both messages in the first 400x400 instead of
200x200. In the second message 500x500 instead of 250x250.

[daniel_unb]

Post by daniel_unb
just a correction to both messages in the first 400x400 instead of
200x200. In the second message 500x500 instead of 250x250.

It seems that this code which is almost the same as the above works
fine. Still it doesn't run if the number of points is too large.
500x500 for example.

----------------------------------------------------------------------------------------------------------------------------------------------
#include <stdio.h>
#include <math.h>
#include "mpi.h"
//int
main (int argc,char *argv[])
{
FILE *fp;
int rank,kntrl,ct;
int dest;
int size,k,i,j,l,n;
int source;
int tag=0;
int N=10;
double mem[2];
j=-N;
MPI_Status status;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&size);
int mens[2][2*N*2*N],s[2][size][2*N*2*N/size+2*N];
double m[2][2*N*2*N],r[2][size][2*N*2*N/size+2*N];
fp=fopen("t.dat","w");
for (i=-N;i<N;i++){
for(l=-N;l<N;l=l+size){
for(k=0;k<size;k++){
//if(k+l<N){
if(k==rank){
m[1][(i+N)*(2*N)+l+k+N]=(200.0)*j/N;
m[0][(i+N)*(2*N)+l+k+N]=(200.0)*i/N;
mens[0][(i+N)*(2*N)+l+k+N]=k+l;
mens[1][(i+N)*(2*N)+l+k+N]=k;
}
//}
//else
//;
j++;
if(j==N)
j=-N;
}
}
}
for(k=0;k<size;k++){
if(k==rank){
j=0;
for (i=0;i<2*N;i++){
for(l=0;l<2*N;l=l+size){
if(l+k<2*N){
for(n=0;n<2;n++){
s[n][k][j]=mens[n][i*(2*N)+l+k];
r[n][k][j]=m[n][i*(2*N)+l+k];
}
j++;
}
}
}
}
}

for(k=0;k<size;k++){
if(k==rank){
for(n=0;n<2;n++){
MPI_Send(s[n][k],2*N*2*N/size+2*N,MPI_INT,0,tag,MPI_COMM_WORLD);
MPI_Send(r[n][k],2*N*2*N/size+2*N,MPI_DOUBLE,0,tag,MPI_COMM_WORLD);
}
}
}
if(0==rank){
for(k=0;k<size;k++){
for(n=0;n<2;n++){
MPI_Recv(s[n][k],2*N*2*N/size
+2*N,MPI_INT,k,tag,MPI_COMM_WORLD,&status);
MPI_Recv(r[n][k],2*N*2*N/size
+2*N,MPI_DOUBLE,k,tag,MPI_COMM_WORLD,&status);
}
j=0;
for (i=0;i<2*N;i++){
for(l=0;l<2*N;l=l+size){
if(l+k<2*N){
for(n=0;n<2;n++){
mens[n][i*(2*N)+l+k]=s[n][k][j];
m[n][i*(2*N)+l+k]=r[n][k][j];
}
j++;
}
}
}
}
for (i=0;i<2*N;i=i++){
for(l=0;l<2*N;l++){
fprintf(fp,"%.5e %.5e %d %d \n",m[0][i*(2*N)+l],m[1][i*(2*N)
+l],mens[0][i*(2*N)+l],mens[1][i*(2*N)+l]);
}
fprintf(fp,"\n ");
}
}
MPI_Finalize();
fclose(fp);
}

[Georg Bisseling]

for(k=0;k<size;k++){
if(k==rank){
for(n=0;n<2;n++){
MPI_Send(s[n][k],2*N*2*N/size+2*N,MPI_INT,0,tag,MPI_COMM_WORLD);
MPI_Send(r[n][k],2*N*2*N/size+2*N,MPI_DOUBLE,0,tag,MPI_COMM_WORLD);
// Here everybody sends to rank 0. The MPI_Send may block until
// rank 0 is ready to receive the message if the message is considered
// too big to be sent "unexpectedly".
// The problem is that even rank 0 sends to itself and may block
// waiting for itself to call MPI_Recv => DEADLOCK.
// Solutions:
// - replace the send/receive pair with a memcopy for rank 0.
// - use MPI_Bsend and provide enough memory via MPI_Buffer_attach
// - structure the code so that each in each phase each rank will
// call MPI_Irecv*; MPI_Send*; MPI_Waitall;
// There is no generic best solution.
}
}
}

hth
Georg

[daniel_unb]

Hi Georg,
thank you for your email. It seems that the send/receive from process
0 is unnecessary. I have removed and it made no difference. Still if
the number of points increases more than 400x400, it generates error
messages

$mpirun -np 4 exec.e
mpirun noticed that job rank 0 with PID 744 on node calculadora exited
on signal 11 (Segmentation fault).
3 additional processes aborted (not shown)

Can you be more specific about the MPI_Bsend and the
MPI_Buffer_attach where can I find examples?

Thanks a lot again,
Daniel

Post by daniel_unb
for(k=0;k<size;k++){
if(k==rank){
for(n=0;n<2;n++){
MPI_Send(s[n][k],2*N*2*N/size+2*N,MPI_INT,0,tag,MPI_COMM_WORLD);
MPI_Send(r[n][k],2*N*2*N/size+2*N,MPI_DOUBLE,0,tag,MPI_COMM_WORLD);
// Here everybody sends to rank 0. The MPI_Send may block until
// rank 0 is ready to receive the message if the message is considered
// too big to be sent "unexpectedly".
// The problem is that even rank 0 sends to itself and may block
// waiting for itself to call MPI_Recv => DEADLOCK.
// - replace the send/receive pair with a memcopy for rank 0.
// - use MPI_Bsend and provide enough memory via MPI_Buffer_attach
// - structure the code so that each in each phase each rank will
// call MPI_Irecv*; MPI_Send*; MPI_Waitall;
// There is no generic best solution.
}
}
}
hth
Georg

[Georg Bisseling]

Post by daniel_unb
Hi Georg,
thank you for your email. It seems that the send/receive from process
0 is unnecessary. I have removed and it made no difference. Still if
the number of points increases more than 400x400, it generates error
messages
$mpirun -np 4 exec.e
mpirun noticed that job rank 0 with PID 744 on node calculadora exited
on signal 11 (Segmentation fault).
3 additional processes aborted (not shown)

Maybe the problem is that you create huge arrays on the stack (as
local variables of main). It may help to declare them static or to
make them global variables. Sooner or later you will have to switch
to dynamically allocated memory anyway. Three dimensional, dynamically
alloced arrays in C are a pain...

Post by daniel_unb
Can you be more specific about the MPI_Bsend and the
MPI_Buffer_attach where can I find examples?

You can find MPI documentation here
http://www.mpi-forum.org/docs/mpi21-report.pdf, section 3.6 Buffer
Allocation and Usage.

If the examples in the standard are not enough then you may refer
to the test suite that comes with the source distribution of MPICH2.
These programs are quite instructive since they do exactly check the
corner cases that are different to deduce from the documentation.

Post by daniel_unb
Thanks a lot again,
Daniel

Post by daniel_unb
for(k=0;k<size;k++){
if(k==rank){
for(n=0;n<2;n++){
MPI_Send(s[n][k],2*N*2*N/size+2*N,MPI_INT,0,tag,MPI_COMM_WORLD);
MPI_Send(r[n][k],2*N*2*N/size+2*N,MPI_DOUBLE,0,tag,MPI_COMM_WORLD);
// Here everybody sends to rank 0. The MPI_Send may block until
// rank 0 is ready to receive the message if the message is considered
// too big to be sent "unexpectedly".
// The problem is that even rank 0 sends to itself and may block
// waiting for itself to call MPI_Recv => DEADLOCK.
// - replace the send/receive pair with a memcopy for rank 0.
// - use MPI_Bsend and provide enough memory via MPI_Buffer_attach
// - structure the code so that each in each phase each rank will
// call MPI_Irecv*; MPI_Send*; MPI_Waitall;
// There is no generic best solution.
}
}
}
hth
Georg