mpid_coll.c

fortran并行计算包

/*  (C)Copyright IBM Corp.  2007, 2008  */
/**
 * \file src/comm/collselect/mpid_coll.c
 * \brief Collective setup
 */
#include "mpido_coll.h"


#warning reasonable hack for now
#define MAXGEOMETRIES 65536


static DCMF_Geometry_t *mpid_geometrytable[MAXGEOMETRIES];
MPIDI_CollectiveProtocol_t MPIDI_CollectiveProtocols;

/*
 * geometries have a 'comm' ID which needs to be equivalently unique as
 * MPIs context_ids. So, we set geometry comm to context_id. Unfortunately
 * there is no trivial way to convert a context_id back to which MPI comm
 * it belongs to so this gross table is here for now. It will be replaced
 * probably with a lazy allocated list. Whatever goes here will have to be
 * cleaned up in comm_destroy as well
 */
static DCMF_Geometry_t *
getGeometryRequest(int comm)
{
   assert(mpid_geometrytable[comm%MAXGEOMETRIES] != NULL);
   return mpid_geometrytable[comm%MAXGEOMETRIES];
}



static int barriers_num=0;
static DCMF_CollectiveProtocol_t *barriers[DCMF_NUM_BARRIER_PROTOCOLS];

static inline int BARRIER_REGISTER(DCMF_Barrier_Protocol proto,
                             DCMF_CollectiveProtocol_t *proto_ptr,
                             DCMF_Barrier_Configuration_t *config)
{
  int rc;
  config->protocol = proto;
  rc = DCMF_Barrier_register(proto_ptr, config);
  if (rc == DCMF_SUCCESS)
    barriers[barriers_num++] = proto_ptr;
  MPID_assert_debug(barriers_num <= DCMF_NUM_BARRIER_PROTOCOLS);
  return rc;
}

static int local_barriers_num=0;
/* Local barriers PLUS room for one standard/global barrier (DCMF_TORUS_BINOMIAL_BARRIER_PROTOCOL)*/
static DCMF_CollectiveProtocol_t *local_barriers[DCMF_NUM_LOCAL_BARRIER_PROTOCOLS+1];

static inline int LOCAL_BARRIER_REGISTER(DCMF_Barrier_Protocol proto,
                                         DCMF_CollectiveProtocol_t *proto_ptr,
                                         DCMF_Barrier_Configuration_t *config)
{
  int rc;
  config->protocol = proto;
  rc = DCMF_Barrier_register(proto_ptr, config);
  if (rc == DCMF_SUCCESS)
    local_barriers[local_barriers_num++] = proto_ptr;
  MPID_assert_debug(local_barriers_num <= DCMF_NUM_LOCAL_BARRIER_PROTOCOLS+1);
  return rc;
}

static inline int BROADCAST_REGISTER(DCMF_Broadcast_Protocol proto,
                              DCMF_CollectiveProtocol_t *proto_ptr,
                              DCMF_Broadcast_Configuration_t *config)
{
   config->protocol = proto;
   return DCMF_Broadcast_register(proto_ptr, config);
}

static inline int ALLREDUCE_REGISTER(DCMF_Allreduce_Protocol proto,
                              DCMF_CollectiveProtocol_t *proto_ptr,
                              DCMF_Allreduce_Configuration_t *config)
{
   config->protocol = proto;
   return DCMF_Allreduce_register(proto_ptr, config);
}

static inline int ALLTOALLV_REGISTER(DCMF_Alltoallv_Protocol proto,
                              DCMF_CollectiveProtocol_t *proto_ptr,
                              DCMF_Alltoallv_Configuration_t *config)
{
   config->protocol = proto;
   return DCMF_Alltoallv_register(proto_ptr, config);
}

static inline int REDUCE_REGISTER(DCMF_Reduce_Protocol proto,
                              DCMF_CollectiveProtocol_t *proto_ptr,
                              DCMF_Reduce_Configuration_t *config)
{
   config->protocol = proto;
   return DCMF_Reduce_register(proto_ptr, config);
}



/** \brief Helper used to register all the collective protocols at initialization */
void MPIDI_Coll_register(void)
{
   DCMF_Barrier_Configuration_t   barrier_config;
   DCMF_Broadcast_Configuration_t broadcast_config;
   DCMF_Allreduce_Configuration_t allreduce_config;
   DCMF_Alltoallv_Configuration_t alltoallv_config;
   DCMF_Reduce_Configuration_t    reduce_config;
   DCMF_GlobalBarrier_Configuration_t gbarrier_config;
   DCMF_GlobalBcast_Configuration_t gbcast_config;
   DCMF_GlobalAllreduce_Configuration_t gallreduce_config;

   DCMF_Result rc;

   /* Register the global functions first */
   
   /* ---------------------------------- */
   /* Register global barrier          */
   /* ---------------------------------- */
   gbarrier_config.protocol = DCMF_GI_GLOBALBARRIER_PROTOCOL;
   rc = DCMF_GlobalBarrier_register(&MPIDI_Protocols.globalbarrier,
                                    &gbarrier_config);
   /* registering the global barrier failed, so don't use it */
   if(rc != DCMF_SUCCESS)
   {
      MPIDI_CollectiveProtocols.barrier.usegi = 0;
   }
   
   
   /* ---------------------------------- */
   /* Register global broadcast          */
   /* ---------------------------------- */
   gbcast_config.protocol = DCMF_TREE_GLOBALBCAST_PROTOCOL;
   rc = DCMF_GlobalBcast_register(&MPIDI_Protocols.globalbcast, &gbcast_config);
   
   /* most likely, we lack shared memory and therefore can't use this */
   if(rc != DCMF_SUCCESS)
   {
      MPIDI_CollectiveProtocols.broadcast.usetree = 0;
   }
   
   /* ---------------------------------- */
   /* Register global allreduce          */
   /* ---------------------------------- */
   gallreduce_config.protocol = DCMF_TREE_GLOBALALLREDUCE_PROTOCOL;
   rc = DCMF_GlobalAllreduce_register(&MPIDI_Protocols.globalallreduce,
                                      &gallreduce_config);
   
   /* most likely, we lack shared memory and therefore can't use this */
   /* reduce uses the allreduce protocol */
   if(rc != DCMF_SUCCESS)
   {
      /* Try the ccmi tree if we were trying global tree */
      MPIDI_CollectiveProtocols.allreduce.useccmitree = MPIDI_CollectiveProtocols.allreduce.usetree;
      MPIDI_CollectiveProtocols.reduce.useccmitree    = MPIDI_CollectiveProtocols.reduce.usetree;
      MPIDI_CollectiveProtocols.allreduce.usetree     = 0;
      MPIDI_CollectiveProtocols.reduce.usetree        = 0;
   }
   

   /* register first barrier protocols now */
   barrier_config.cb_geometry = getGeometryRequest;

   /* set the function that will find the [all]reduce geometry on unexpected callbacks*/
   allreduce_config.cb_geometry = getGeometryRequest;
   reduce_config.cb_geometry = getGeometryRequest;

   /* set configuration flags in the config*/
   allreduce_config.reuse_storage = MPIDI_CollectiveProtocols.allreduce.reusestorage;
   reduce_config.reuse_storage = MPIDI_CollectiveProtocols.reduce.reusestorage;

   /* Other env vars can be checked at communicator creation time
    * but barriers are associated with a geometry and this knowledge
    * isn't available to mpido_barrier
    */
   if(MPIDI_CollectiveProtocols.barrier.usegi)
   {
      if(BARRIER_REGISTER(DCMF_GI_BARRIER_PROTOCOL,
                 &MPIDI_CollectiveProtocols.barrier.gi,
                 &barrier_config) != DCMF_SUCCESS)
         MPIDI_CollectiveProtocols.barrier.usegi = 0;
   }

   /* 
    * Always register a binomial barrier for collectives in subcomms, just
    * choose not to use it at mpido_barrier
    */
   if(BARRIER_REGISTER(DCMF_TORUS_BINOMIAL_BARRIER_PROTOCOL,
              &MPIDI_CollectiveProtocols.barrier.binomial,
              &barrier_config) != DCMF_SUCCESS)
      MPIDI_CollectiveProtocols.barrier.usebinom = 0;

   /* if we don't even get a binomial barrier, we are in trouble */
   MPID_assert_debug(barriers_num >  0);

   /* 
    * Register local barriers for the geometry.  
    * Both a true local lockbox barrier and a global binomial
    * barrier (which can be used non-optimally).  The geometry
    * will decide internally if/which to use.
    * They are not used directly by MPICH but must be initialized.
    */
   if(MPIDI_CollectiveProtocols.localbarrier.uselockbox)
   {
      if(LOCAL_BARRIER_REGISTER(DCMF_LOCKBOX_BARRIER_PROTOCOL,
                 &MPIDI_CollectiveProtocols.localbarrier.lockbox,
                 &barrier_config) != DCMF_SUCCESS)
        MPIDI_CollectiveProtocols.localbarrier.uselockbox = 0;
   }

   /* 
    * Always register a binomial barrier for collectives in subcomms
    */
  if(LOCAL_BARRIER_REGISTER(DCMF_TORUS_BINOMIAL_BARRIER_PROTOCOL,
                &MPIDI_CollectiveProtocols.localbarrier.binomial,
                &barrier_config) != DCMF_SUCCESS)
    MPIDI_CollectiveProtocols.localbarrier.usebinom = 0;

   /* MPID doesn't care if this actually works.  Let someone else 
    * handle problems as needed.  
    * MPID_assert_debug(local_barriers_num >  0); 
    */


   /* Register broadcast protocols */
   if(MPIDI_CollectiveProtocols.broadcast.usetree)
   {
      if(BROADCAST_REGISTER(DCMF_TREE_BROADCAST_PROTOCOL,
                         &MPIDI_CollectiveProtocols.broadcast.tree,
                         &broadcast_config) != DCMF_SUCCESS)
         MPIDI_CollectiveProtocols.broadcast.usetree = 0;
   }

   if(BROADCAST_REGISTER(DCMF_TORUS_RECTANGLE_BROADCAST_PROTOCOL,
                      &MPIDI_CollectiveProtocols.broadcast.rectangle,
                      &broadcast_config) != DCMF_SUCCESS)
      MPIDI_CollectiveProtocols.broadcast.userect = 0;

//   BROADCAST_REGISTER(DCMF_TORUS_RECT_BCAST_3COLOR_PROTOCOL,
//                     &MPIDI_CollectiveProtocols.broadcast.rectangle.threecolor,
//                     &broadcast_config);

   if(BROADCAST_REGISTER(DCMF_TORUS_BINOMIAL_BROADCAST_PROTOCOL,
                      &MPIDI_CollectiveProtocols.broadcast.binomial,
                      &broadcast_config) != DCMF_SUCCESS)
      MPIDI_CollectiveProtocols.broadcast.usebinom = 0;


   /* Register allreduce protocols */
   if(MPIDI_CollectiveProtocols.allreduce.usetree ||
      MPIDI_CollectiveProtocols.allreduce.useccmitree)
   {
      if(ALLREDUCE_REGISTER(DCMF_TREE_ALLREDUCE_PROTOCOL,
                         &MPIDI_CollectiveProtocols.allreduce.tree,
                         &allreduce_config) != DCMF_SUCCESS)
      {
           MPIDI_CollectiveProtocols.allreduce.usetree = 0;
           MPIDI_CollectiveProtocols.allreduce.useccmitree = 0;
      }
   }

   if(ALLREDUCE_REGISTER(DCMF_TREE_PIPELINED_ALLREDUCE_PROTOCOL,
                      &MPIDI_CollectiveProtocols.allreduce.pipelinedtree,
                      &allreduce_config) != DCMF_SUCCESS)
      MPIDI_CollectiveProtocols.allreduce.usepipelinedtree = 0;

   if(ALLREDUCE_REGISTER(DCMF_TORUS_RECTANGLE_ALLREDUCE_PROTOCOL,
                      &MPIDI_CollectiveProtocols.allreduce.rectangle,
                      &allreduce_config) != DCMF_SUCCESS)
      MPIDI_CollectiveProtocols.allreduce.userect = 0;

   if(ALLREDUCE_REGISTER(DCMF_TORUS_RECTANGLE_RING_ALLREDUCE_PROTOCOL,