subarray.c

The example for Boor USING MPI2

#include "mpi.h"
int main( int argc, char *argv[] )
{
    int          gsizes[2], distribs[2], dargs[2], psizes[2], rank, size, m, n;
    MPI_Datatype filetype;
    int          local_array_size, num_local_rows, num_local_cols;
    int          row_procs, col_procs, row_rank, col_rank;
    int          dims[2], periods[2], lsizes[2], coords[2], start_indices[2];
    MPI_Comm     comm;
    MPI_File     fh; 
    float        *local_array; 
    MPI_Status   status;

    MPI_Init( &argc, &argv );

    /* ... */

    /* This code is particular to a 2 x 3 process decomposition */
    MPI_Comm_size( MPI_COMM_WORLD, &size );
    if (size != 6) {
	printf( "Communicator size must be 6\n" );
	MPI_Abort( MPI_COMM_WORLD, 1 );
    }

    /* See comments on block distribution */
    row_procs = 2;
    col_procs = 3;
    num_local_rows = (m + row_procs - 1) / row_procs;
    /* adjust for last row */
    if (row_rank == row_procs-1) 
	num_local_rows = m - (row_procs-1) * num_local_rows;
    num_local_cols = (n + col_procs - 1) / col_procs;
    /* adjust for last column */
    if (col_rank == col_procs-1) 
	num_local_cols = n - (col_procs-1) * num_local_cols;

    local_array = (float *)malloc( num_local_rows * num_local_cols * sizeof(float) );

    /* ... set elements of local_array ... */

    gsizes[0] = m;  /* no. of rows in global array */
    gsizes[1] = n;  /* no. of columns in global array*/
    
    psizes[0] = 2;  /* no. of processes in vertical dimension 
		       of process grid */
    psizes[1] = 3;  /* no. of processes in horizontal dimension 
		       of process grid */

    lsizes[0] = m/psizes[0];   /* no. of rows in local array */
    lsizes[1] = n/psizes[1];   /* no. of columns in local array */

    dims[0] = 2;
    dims[1] = 3;
    periods[0] = periods[1] = 1;
    MPI_Cart_create(MPI_COMM_WORLD, 2, dims, periods, 0, &comm);
    MPI_Comm_rank(comm, &rank);
    MPI_Cart_coords(comm, rank, 2, coords);

    /* global indices of the first element of the local array */
    start_indices[0] = coords[0] * lsizes[0];
    start_indices[1] = coords[1] * lsizes[1];

    MPI_Type_create_subarray(2, gsizes, lsizes, start_indices, 
                         MPI_ORDER_C, MPI_FLOAT, &filetype);
    MPI_Type_commit(&filetype);
    
    MPI_File_open(MPI_COMM_WORLD, "/pfs/datafile", 
		  MPI_MODE_CREATE | MPI_MODE_WRONLY, 
		  MPI_INFO_NULL, &fh);
    MPI_File_set_view(fh, 0, MPI_FLOAT, filetype, "native", 
		      MPI_INFO_NULL);
    
    local_array_size = lsizes[0] * lsizes[1];
    MPI_File_write_all(fh, local_array, local_array_size, 
		       MPI_FLOAT, &status);
    
    MPI_File_close(&fh);

    /* ... */
    MPI_Finalize();
    return 0;
}