mpi.h

CCSM Research Tools: Community Atmosphere Model (CAM)

/* $Id: mpi.h,v 1.1.6.1 2002/04/24 03:26:00 erik Exp $ */

/*
   This is a special set of bindings for uni-processor use of MPI by the PETSc library.
 
   NOT ALL THE MPI CALLS ARE IMPLEMENTED CORRECTLY! Only those needed in PETSc.

   For example,
   * Does not implement send to self.
   * Does not implement attributes correctly.
*/

/*
  The following info is a response to one of the petsc-maint questions 
  regarding MPIUNI.

  MPIUNI was developed with the aim of getting PETSc compiled, and
  usable in the absence of MPI. This is the reason each function is
  not documented.  The development strategy was - to make enough
  changes to it so that PETSc source compiles without errors, and runs
  in the uni-processor mode.

  Most PETSc objects have both sequential and parallel
  implementations, which are separate. For eg: We have two types of
  sparse matrix storage formats - SeqAIJ, and MPIAIJ. Some MPI
  routines are used in the Seq part, but most of them are used in the
  MPI part. The send/receive calls can be found mostly in the MPI
  part.

  When MPIUNI is used, only the Seq version of the PETSc objects are
  used, even though the MPI variant of the objects are compiled. Since
  there are no send/receive calls in the Seq variant, PETSc works fine
  with MPIUNI in seq mode.

  The reason some send/receive functions are defined to abort(), is to
  detect sections of code that use send/receive functions, and gets
  executed in the sequential mode. (which shouldn't happen in case of
  PETSc).

  One of the goals with MPIUNI, is to avoid the function call overhead
  of a regular MPI implementation. If this was not the case, we could
  as well have used a regular implementation of MPI as they are
  available on almost all machines. Hence most of the functions are
  implemented as macros. One of the additional benefits we got from
  MPIUNI is, we were able to use PETSc on machines where using a
  proper implementation of MPI was painful (for eg NT).

  Proper implementation of send/receive would involve writing a
  function for each of them. Inside each of these functions, we have
  to check if the send is to self or receive is from self, and then
  doing the buffering accordingly (until the receive is called) - or
  what if a nonblocking receive is called, do a copy etc.. Handling
  the buffering aspects might be complicated enough, that in this
  case, a proper implementation of MPI might as well be used. This is
  the reason the send to self is not implemented in MPIUNI, and never
  will be.
*/

#if !defined(__MPI_H)
#define __MPI_H

#if defined(__BSdependh)
#error You cannot use MPI-uni with BlockSolve95
#endif
#if defined(__basic_H)
#error You cannot use MPI-uni with SPAI
#endif

#define USING_MPIUNI

/*

    MPIUNI_TMP is used in the macros below only to stop various C/C++ compilers
from generating warning messages about unused variables while compiling PETSc.
*/
extern void *MPIUNI_TMP;

#define MPI_COMM_WORLD       1
#define MPI_COMM_SELF        MPI_COMM_WORLD
#define MPI_COMM_NULL        0
#define MPI_SUCCESS          0
#define MPI_IDENT            0
#define MPI_CONGRUENT        0
#define MPI_SIMILAR          0
#define MPI_UNEQUAL          3
#define MPI_ANY_SOURCE     (-2)
#define MPI_KEYVAL_INVALID   0
#define MPI_ERR_UNKNOWN     18
#define MPI_ERR_INTERN      21
#define MPI_ERR_OTHER        1
#define MPI_TAG_UB           0
#define MPI_ERRORS_RETURN    0

/* External types */
typedef int    MPI_Comm;  
typedef void   *MPI_Request;
typedef void   *MPI_Group;
typedef struct {int MPI_TAG,MPI_SOURCE,MPI_ERROR;} MPI_Status;
typedef char*   MPI_Errhandler;

extern int MPIUNI_Memcpy(void*,void*,int);

/* In order to handle datatypes, we make them into "sizeof(raw-type)";
    this allows us to do the MPIUNI_Memcpy's easily */
#define MPI_Datatype        int
#define MPI_FLOAT           sizeof(float)
#define MPI_DOUBLE          sizeof(double)
#define MPI_CHAR            sizeof(char)
#define MPI_BYTE            sizeof(char)
#define MPI_INT             sizeof(int)
#define MPI_LONG            sizeof(long)
#define MPI_SHORT           sizeof(short)
#define MPI_UNSIGNED_CHAR   sizeof(unsigned char)
#define MPI_UNSIGNED_LONG   sizeof(unsigned long)
#define MPIU_PETSCLOGDOUBLE sizeof(PetscLogDouble)
#define MPI_REQUEST_NULL    ((MPI_Request)0)

typedef int MPI_Op;

#define MPI_SUM           0
#define MPI_ANY_TAG     (-1)

/*
  Prototypes of some functions which are implemented in mpi.c
*/
typedef int   (MPI_Copy_function)(MPI_Comm,int,void *,void *,void *,int *);
typedef int   (MPI_Delete_function)(MPI_Comm,int,void *,void *);
typedef void  (MPI_User_function)(void *, void *, int *, MPI_Datatype *); 

/*
  In order that the PETSc MPIUNI can be used with another package that has its
  own MPIUni we map the following function names to a unique PETSc name. Those functions
  are defined in mpi.c
*/
extern int    Petsc_MPI_Abort(MPI_Comm,int);
extern int    Petsc_MPI_Attr_get(MPI_Comm comm,int keyval,void *attribute_val,int *flag);
extern int    Petsc_MPI_Keyval_free(int*);
extern int    Petsc_MPI_Attr_put(MPI_Comm,int,void *);
extern int    Petsc_MPI_Attr_delete(MPI_Comm,int);
extern int    Petsc_MPI_Keyval_create(MPI_Copy_function *,MPI_Delete_function *,int *,void *);
extern int    Petsc_MPI_Comm_free(MPI_Comm*);
extern int    Petsc_MPI_Initialized(int *);
extern int    Petsc_MPI_Comm_dup(MPI_Comm,MPI_Comm *);
extern int    Petsc_MPI_Finalize(void);

#define MPI_Abort         Petsc_MPI_Abort
#define MPI_Attr_get      Petsc_MPI_Attr_get
#define MPI_Keyval_free   Petsc_MPI_Keyval_free
#define MPI_Attr_put      Petsc_MPI_Attr_put
#define MPI_Attr_delete   Petsc_MPI_Attr_delete
#define MPI_Keyval_create Petsc_MPI_Keyval_create
#define MPI_Comm_free     Petsc_MPI_Comm_free
#define MPI_Initialized   Petsc_MPI_Initialized
#define MPI_Comm_dup      Petsc_MPI_Comm_dup
#define MPI_Finalize      Petsc_MPI_Finalize

/* 
    Routines we have replace with macros that do nothing 
    Some return error codes others return success
*/

#define MPI_Init(argc,argv) \
     (MPIUNI_TMP = (void*)(long) (argc),\
      MPIUNI_TMP = (void*)(long) (argv),\
      MPI_SUCCESS)
#define MPI_Send(buf,count,datatype,dest,tag,comm)  \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (dest),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPI_SUCCESS)
#define MPI_Recv(buf,count,datatype,source,tag,comm,status) \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (source),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPIUNI_TMP = (void*)(long) (status),\
      MPI_Abort(MPI_COMM_WORLD,0))
#define MPI_Get_count(status, datatype,count) \
     (MPIUNI_TMP = (void*)(long) (status),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPI_Abort(MPI_COMM_WORLD,0))
#define MPI_Bsend(buf,count,datatype,dest,tag,comm)  \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (dest),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPI_SUCCESS)
#define MPI_Ssend(buf,count, datatype,dest,tag,comm) \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (dest),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPI_SUCCESS)
#define MPI_Rsend(buf,count, datatype,dest,tag,comm) \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (dest),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPI_SUCCESS)
#define MPI_Buffer_attach(buffer,size) \
     (MPIUNI_TMP = (void*)(long) (buffer),\
      MPIUNI_TMP = (void*)(long) (size),\
      MPI_SUCCESS)
#define MPI_Buffer_detach(buffer,size)\
     (MPIUNI_TMP = (void*)(long) (buffer),\
      MPIUNI_TMP = (void*)(long) (size),\
      MPI_SUCCESS)
#define MPI_Ibsend(buf,count, datatype,dest,tag,comm,request) \
     (MPIUNI_TMP = (void*)(long) (buf),\
       MPIUNI_TMP = (void*)(long) (count),\
       MPIUNI_TMP = (void*)(long) (datatype),\
       MPIUNI_TMP = (void*)(long) (dest),\
       MPIUNI_TMP = (void*)(long) (tag),\
       MPIUNI_TMP = (void*)(long) (comm),\
       MPIUNI_TMP = (void*)(long) (request),\
       MPI_SUCCESS)
#define MPI_Issend(buf,count, datatype,dest,tag,comm,request) \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (dest),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPIUNI_TMP = (void*)(long) (request),\
      MPI_SUCCESS)
#define MPI_Irsend(buf,count, datatype,dest,tag,comm,request) \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (dest),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPIUNI_TMP = (void*)(long) (request),\
      MPI_SUCCESS)
#define MPI_Irecv(buf,count, datatype,source,tag,comm,request) \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (source),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPIUNI_TMP = (void*)(long) (request),\
      MPI_Abort(MPI_COMM_WORLD,0))
#define MPI_Isend(buf,count, datatype,dest,tag,comm,request) \
     (MPIUNI_TMP = (void*)(long) (buf),\
      MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (datatype),\
      MPIUNI_TMP = (void*)(long) (dest),\
      MPIUNI_TMP = (void*)(long) (tag),\
      MPIUNI_TMP = (void*)(long) (comm),\
      MPIUNI_TMP = (void*)(long) (request),\
      MPI_Abort(MPI_COMM_WORLD,0))
#define MPI_Wait(request,status) \
     (MPIUNI_TMP = (void*)(long) (request),\
      MPIUNI_TMP = (void*)(long) (status),\
      MPI_SUCCESS)
#define MPI_Test(request,flag,status) \
     (MPIUNI_TMP = (void*)(long) (request),\
      MPIUNI_TMP = (void*)(long) (status),\
      *(flag) = 0, \
      MPI_SUCCESS)
#define MPI_Request_free(request) \
     (MPIUNI_TMP = (void*)(long) (request),\
      MPI_SUCCESS)
#define MPI_Waitany(a,b,c,d) \
     (MPIUNI_TMP = (void*)(long) (a),\
      MPIUNI_TMP = (void*)(long) (b),\
      MPIUNI_TMP = (void*)(long) (c),\
      MPIUNI_TMP = (void*)(long) (d),\
      MPI_SUCCESS)
#define MPI_Testany(a,b,c,d,e) \
     (MPIUNI_TMP = (void*)(long) (a),\
      MPIUNI_TMP = (void*)(long) (b),\
      MPIUNI_TMP = (void*)(long) (c),\
      MPIUNI_TMP = (void*)(long) (d),\
      MPIUNI_TMP = (void*)(long) (e),\
      MPI_SUCCESS)
#define MPI_Waitall(count,array_of_requests,array_of_statuses) \
     (MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (array_of_requests),\
      MPIUNI_TMP = (void*)(long) (array_of_statuses),\
      MPI_SUCCESS)
#define MPI_Testall(count,array_of_requests,flag,array_of_statuses) \
     (MPIUNI_TMP = (void*)(long) (count),\
      MPIUNI_TMP = (void*)(long) (array_of_requests),\
      MPIUNI_TMP = (void*)(long) (flag),\
      MPIUNI_TMP = (void*)(long) (array_of_statuses),\
      MPI_SUCCESS)
#define MPI_Waitsome(incount,array_of_requests,outcount,\
                     array_of_indices,array_of_statuses) \
     (MPIUNI_TMP = (void*)(long) (incount),\