commutil.c

mpi并行计算的c++代码 可用vc或gcc编译通过 可以用来搭建并行计算试验环境

/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*  $Id: commutil.c,v 1.55 2005/10/14 21:40:23 gropp Exp $
 *
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpiimpl.h"
#include "mpicomm.h"

/* This is the utility file for comm that contains the basic comm items
   and storage management */
#ifndef MPID_COMM_PREALLOC 
#define MPID_COMM_PREALLOC 8
#endif

/* Preallocated comm objects */
MPID_Comm MPID_Comm_builtin[MPID_COMM_N_BUILTIN] = { {0} };
MPID_Comm MPID_Comm_direct[MPID_COMM_PREALLOC] = { {0} };
MPIU_Object_alloc_t MPID_Comm_mem = { 0, 0, 0, 0, MPID_COMM, 
				      sizeof(MPID_Comm), MPID_Comm_direct,
                                      MPID_COMM_PREALLOC};

/* FIXME :
   Reusing context ids can lead to a race condition if (as is desirable)
   MPI_Comm_free does not include a barrier.  Consider the following:
   Process A frees the communicator.
   Process A creates a new communicator, reusing the just released id
   Process B sends a message to A on the old communicator.
   Process A receives the message, and believes that it belongs to the
   new communicator.
   Process B then cancels the message, and frees the communicator.

   The likelyhood of this happening can be reduced by introducing a gap
   between when a context id is released and when it is reused.  An alternative
   is to use an explicit message (in the implementation of MPI_Comm_free)
   to indicate that a communicator is being freed; this will often require
   less communication than a barrier in MPI_Comm_free, and will ensure that 
   no messages are later sent to the same communicator (we may also want to
   have a similar check when building fault-tolerant versions of MPI).
 */

/* Create a new communicator with a context.  
   Do *not* initialize the other fields except for the reference count.
   See MPIR_Comm_copy for a function to produce a copy of part of a
   communicator 
*/
/*  FIXME : comm_create can't use this because the context id must be
   created separately from the communicator (creating the context
   is collective over oldcomm_ptr, but this routine may be called only
   by a subset of processes in the new communicator)

   Only Comm_split currently uses this
 */
int MPIR_Comm_create( MPID_Comm *oldcomm_ptr, MPID_Comm **newcomm_ptr )
{   
    int mpi_errno, new_context_id;
    MPID_Comm *newptr;

    newptr = (MPID_Comm *)MPIU_Handle_obj_alloc( &MPID_Comm_mem );
    /* --BEGIN ERROR HANDLING-- */
    if (!newptr) {
	mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, 
		   "MPIR_Comm_create", __LINE__, MPI_ERR_OTHER, "**nomem", 0 );
	return mpi_errno;
    }
    /* --END ERROR HANDLING-- */
    *newcomm_ptr = newptr;
    MPIU_Object_set_ref( newptr, 1 );

    /* If there is a context id cache in oldcomm, use it here.  Otherwise,
       use the appropriate algorithm to get a new context id */
    newptr->context_id = new_context_id = 
	MPIR_Get_contextid( oldcomm_ptr );
    newptr->attributes = 0;
    /* --BEGIN ERROR HANDLING-- */
    if (new_context_id == 0) {
	mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, 
                                 "MPIR_Comm_create", __LINE__, MPI_ERR_OTHER,
					  "**toomanycomm", 0 );
	return mpi_errno;
    }
    /* --END ERROR HANDLING-- */
    return 0;
}

/* Create a local intra communicator from the local group of the 
   specified intercomm. */
/* FIXME : 
   For the context id, use the intercomm's context id + 2.  (?)
 */
int MPIR_Setup_intercomm_localcomm( MPID_Comm *intercomm_ptr )
{
    MPID_Comm *localcomm_ptr;
    int mpi_errno;

    localcomm_ptr = (MPID_Comm *)MPIU_Handle_obj_alloc( &MPID_Comm_mem );
    /* --BEGIN ERROR HANDLING-- */
    if (!localcomm_ptr) {
	mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, 
                                 "MPIR_Setup_intercomm_localcomm", __LINE__,
					  MPI_ERR_OTHER, "**nomem", 0 );
	return mpi_errno;
    }
    /* --END ERROR HANDLING-- */
    MPIU_Object_set_ref( localcomm_ptr, 1 );
    /* Note that we must not free this context id since we are sharing it
       with the intercomm's context */
    /* FIXME: This was + 2 (in agreement with the docs) but that
       caused some errors with an apparent use of the same context id
       by operations in different communicators.  Switching this to +1
       seems to have fixed that problem, but this isn't the right answer. */
    localcomm_ptr->context_id = intercomm_ptr->context_id + 1;

    /* Duplicate the VCRT references */
    MPID_VCRT_Add_ref( intercomm_ptr->local_vcrt );
    localcomm_ptr->vcrt = intercomm_ptr->local_vcrt;
    localcomm_ptr->vcr  = intercomm_ptr->local_vcr;

    /* Save the kind of the communicator */
    localcomm_ptr->comm_kind   = MPID_INTRACOMM;
    
    /* Set the sizes and ranks */
    localcomm_ptr->remote_size = intercomm_ptr->local_size;
    localcomm_ptr->local_size  = intercomm_ptr->local_size;
    localcomm_ptr->rank        = intercomm_ptr->rank;

    /* More advanced version: if the group is available, dup it by 
       increasing the reference count */
    localcomm_ptr->local_group  = 0;
    localcomm_ptr->remote_group = 0;

    /* This is an internal communicator, so ignore */
    localcomm_ptr->errhandler = 0;
    
    /* FIXME  : No local functions for the collectives */
    localcomm_ptr->coll_fns = 0;

    /* FIXME  : No local functions for the topology routines */
    localcomm_ptr->topo_fns = 0;

    /* We do *not* inherit any name */
    localcomm_ptr->name[0] = 0;

    localcomm_ptr->attributes = 0;

    intercomm_ptr->local_comm = localcomm_ptr;
    return 0;
}
/*
 * Here are the routines to find a new context id.  The algorithm is discussed 
 * in detail in the mpich2 coding document.  There are versions for
 * single threaded and multithreaded MPI.
 * 
 * These assume that int is 32 bits; they should use uint_32 instead, 
 * and an MPI_UINT32 type (should be able to use MPI_INTEGER4)
 */

/* Both the threaded and non-threaded routines use the same mask of available
   context id values. */
#define MAX_CONTEXT_MASK 32
static unsigned int context_mask[MAX_CONTEXT_MASK];
static int initialize_context_mask = 1;

#ifdef MPICH_DEBUG_INTERNAL
static void MPIR_PrintContextMask( FILE *fp )
{
    int i;
    int maxset=0;
    for (i=MAX_CONTEXT_MASK-1; i>=0; i--) {
	if (context_mask[i] != 0) break;
    }
    maxset = i;
    DBG_FPRINTF( fp, "Context mask: " );
    for (i=0; i<maxset; i++) {
	DBG_FPRINTF( fp, "%.8x ", context_mask[i] );
    }
    DBG_FPRINTF( fp, "\n" );
}
#endif

static void MPIR_Init_contextid (void) {
    int i;
    for (i=1; i<MAX_CONTEXT_MASK; i++) {
	context_mask[i] = 0xFFFFFFFF;
    }
    /* the first two values are already used */
    context_mask[0] = 0xFFFFFFFC; 
    initialize_context_mask = 0;
}
/* Return the context id corresponding to the first set bit in the mask.
   Return 0 if no bit found */
static int MPIR_Find_context_bit( unsigned int local_mask[] ) {
    int i, j, context_id = 0;
    for (i=0; i<MAX_CONTEXT_MASK; i++) {
	if (local_mask[i]) {
	    /* There is a bit set in this word. */
	    register unsigned int val, nval;
	    /* The following code finds the highest set bit by recursively
	       checking the top half of a subword for a bit, and incrementing
	       the bit location by the number of bit of the lower sub word if 
	       the high subword contains a set bit.  The assumption is that
	       full-word bitwise operations and compares against zero are 
	       fast */
	    val = local_mask[i];
	    j   = 0;
	    nval = val & 0xFFFF0000;
	    if (nval) {
		j += 16;
		val = nval;
	    }
	    nval = val & 0xFF00FF00;
	    if (nval) {
		j += 8;
		val = nval;
	    }
	    nval = val & 0xF0F0F0F0;
	    if (nval) {
		j += 4;
		val = nval;
	    }
	    nval = val & 0xCCCCCCCC;
	    if (nval) {
		j += 2;
		val = nval;
	    }
	    if (val & 0xAAAAAAAA) { 
		j += 1;
	    }
	    context_mask[i] &= ~(1<<j);
	    context_id = 4 * (32 * i + j);
#ifdef MPICH_DEBUG_INTERNAL
	    if (MPIR_IDebug("context")) {
		DBG_FPRINTF( stderr, "allocating contextid = %d\n", context_id ); 
		DBG_FPRINTF( stderr, "(mask[%d], bit %d\n", i, j );
	    }
#endif
	    return context_id;
	}
    }
    return 0;
}
#if MPID_MAX_THREAD_LEVEL <= MPI_THREAD_SERIALIZED
/* Unthreaded (only one MPI call active at any time) */

int MPIR_Get_contextid( MPID_Comm *comm_ptr )
{
    int          context_id = 0;
    unsigned int local_mask[MAX_CONTEXT_MASK];

    if (initialize_context_mask) {
	MPIR_Init_contextid();
    }
    memcpy( local_mask, context_mask, MAX_CONTEXT_MASK * sizeof(int) );
    MPIR_Nest_incr();
    /* Comm must be an intracommunicator */
    NMPI_Allreduce( MPI_IN_PLACE, local_mask, MAX_CONTEXT_MASK, MPI_INT, 
		    MPI_BAND, comm_ptr->handle );
    MPIR_Nest_decr();

    context_id = MPIR_Find_context_bit( local_mask );

    /* return 0 if no context id found.  The calling routine should 
       check for this and generate the appropriate error code */
#ifdef MPICH_DEBUG_INTERNAL
    if (MPIR_IDebug("context"))
	MPIR_PrintContextMask( stderr );
#endif
    return context_id;
}

#else

/* Additional values needed to maintain thread safety */
static volatile int mask_in_use = 0;
/* lowestContextId is used to break ties when multiple threads
   are contending for the mask */
#define MPIR_MAXID (1 << 30)
static volatile int lowestContextId = MPIR_MAXID;

int MPIR_Get_contextid( MPID_Comm *comm_ptr )
{
    int          context_id = 0;
    unsigned int local_mask[MAX_CONTEXT_MASK];
    int          own_mask = 0;

    /* We lock only around access to the mask.  If another thread is
       using the mask, we take a mask of zero */
    MPIU_DBG_PRINTF_CLASS( MPIU_DBG_COMM | MPIU_DBG_CONTEXTID, 1,
			   ( "Entering; shared state is %d:%d\n", mask_in_use, 
			     lowestContextId ) );
    while (context_id == 0) {
	MPID_Common_thread_lock();
	if (initialize_context_mask) {
	    MPIR_Init_contextid();
	}
	if (mask_in_use || comm_ptr->context_id > lowestContextId) {
	    memset( local_mask, 0, MAX_CONTEXT_MASK * sizeof(int) );
	    if (comm_ptr->context_id < lowestContextId) {
		lowestContextId = comm_ptr->context_id;
	    }
	    MPIU_DBG_PRINTF_CLASS( MPIU_DBG_COMM | MPIU_DBG_CONTEXTID, 2,
				   ( "In in-use, sed lowestContextId to %d\n", 
				     lowestContextId ) );
	}
	else {
	    memcpy( local_mask, context_mask, MAX_CONTEXT_MASK * sizeof(int) );
	    mask_in_use     = 1;
	    own_mask        = 1;