bsendutil.c

刚才是说明 现在是安装程序在 LINUX环境下进行编程的MPICH安装文件

/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*  $Id: bsendutil.c,v 1.16 2002/11/14 22:35:47 gropp Exp $
 *
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpiimpl.h"
#include "bsendutil.h"

/*
 * Description of the Bsend data structures.
 *
 * Bsend is buffered send; a buffer, provided by the user, is used to store
 * both the user's message and information that my be needed to send that
 * message.  In addition, space within that buffer must be allocated, so
 * additional information is required to manage that space allocation.  
 * In the following, the term "segment" denotes a fragment of the user buffer
 * that has been allocated either to free (unused) space or to a particular
 * user message.
 *
 * The following datastructures are used:
 *
 *  BsendMsg_t  - Describes a user message, including the values of tag
 *                and datatype (*could* be used incase the data is already 
 *                contiguous; see below)
 *  BsendData_t - Describes a segment of the user buffer.  This data structure
 *                contains a BsendMsg_t for segments that contain a user 
 *                message.  Each BsendData_t segment belongs to one of 
 *                three lists: avail (unused and free), active (currently
 *                sending) and pending (contains a user message that has
 *                not begun sending because of some resource limit, such
 *                as no more MPID requests available).
 *  BsendBuffer - This global structure contains pointers to the user buffer
 *                and the three lists, along with the size of the user buffer.
 *                Multithreaded versions contain a thread lock as well.
 *
 * Miscellaneous comments
 * By storing total_size along with "size available for messages", we avoid
 * any complexities associated with alignment, since we must ensure that each
 * BsendData_t structure is properly aligned (i.e., we can't simply
 * do (sizeof(BsendData_t) + size) to get total_size).
 */

/* #define DEBUG(a) a;fflush(stdout)  */
#define DEBUG(a) 
#define DEBUG1(a) 
/* #define PRINT_AVAIL */
/* #define DEBUG1(a) a;fflush(stdout)  */
/* #define PRINT_ARENA */

/* Private structures for the bsend buffers */

/* BsendMsg is used to hold all of the message particulars in case
   a request is not currently available */
typedef struct {
    void         *msgbuf;
    int          count;
    MPI_Datatype dtype;
    int          tag;
    MPID_Comm    *comm_ptr;
    int          dest;
} BsendMsg_t;

/* BsendData describes a bsend request */
typedef struct BsendData {
    int              size;             /* size that is available for data */
    int              total_size;       /* total size of this segment, 
					  including all headers */
    struct BsendData *next, *prev;
    BsendKind_t      kind;
    MPID_Request     *request;
    BsendMsg_t       msg;
    double           alignpad;         /* make sure that the struct shares
					  double alignment */
} BsendData_t;
#define BSENDDATA_HEADER_TRUE_SIZE (sizeof(BsendData_t) - sizeof(double))

/* BsendBuffer is the structure that describes the overall Bsend buffer */
static struct {
    void               *buffer;        /* Pointer to the begining of the user-
					  provided buffer */
    int                buffer_size;    /* Size of the user-provided buffer */
    BsendData_t        *avail;         /* Pointer to the first available block
					  of space */
    BsendData_t        *pending;       /* Pointer to the first message that
					  could not be sent because of a 
					  resource limit (e.g., no requests
					  available) */
    BsendData_t        *active;        /* Pointer to the first active (sending)
					  message */
#if MPID_MAX_THREAD_LEVEL >= MPI_THREAD_MULTIPLE
    MPID_Thread_lock_t bsend_lock;     /* Thread lock for bsend access */
#endif

} BsendBuffer = { 0, 0, 0, 0, 0 };

static int initialized = 0;   /* keep track of the first call to any
				 bsend routine */

/* Forward references */
static void MPIR_Bsend_retry_pending( void );
static void MPIR_Bsend_check_active ( void );
static BsendData_t *MPIR_Bsend_find_buffer( int );
static void MPIR_Bsend_take_buffer( BsendData_t *, int );
static int MPIR_Bsend_finalize( void * );

/*
 * Attach a buffer.  This checks for the error conditions and then
 * initialized the avail buffer.
 */    
int MPIR_Bsend_attach( void *buffer, int buffer_size )
{
    BsendData_t *p;

#   ifdef HAVE_ERROR_CHECKING
    {
        MPID_BEGIN_ERROR_CHECKS;
        {
	    if (BsendBuffer.buffer) {
		return MPIR_Err_create_code( MPI_ERR_BUFFER, 
					     "**bufexists", 0 );
	    }
	    if (buffer_size < MPI_BSEND_OVERHEAD) {
		return MPIR_Err_create_code( MPI_ERR_OTHER, 
			     "**bsendbufsmall", "**bsendbufsmall %d %d",
			     buffer_size, MPI_BSEND_OVERHEAD );
	    }
	}
	MPID_END_ERROR_CHECKS;
    }
#   endif /* HAVE_ERROR_CHECKING */

    if (!initialized) {
	initialized = 1;
	MPIR_Add_finalize( MPIR_Bsend_finalize, (void *)0, 10 );
    }

    BsendBuffer.buffer	    = buffer;
    BsendBuffer.buffer_size = buffer_size;
    BsendBuffer.avail	    = buffer;
    BsendBuffer.pending	    = 0;
    BsendBuffer.active	    = 0;
#if MPID_MAX_THREAD_LEVEL >= MPI_THREAD_MULTIPLE
    MPID_Thread_initlock( BsendBuffer.bsend_lock );
#endif

    /* Set the first block */
    p		  = (BsendData_t *)buffer;
    p->size	  = buffer_size - BSENDDATA_HEADER_TRUE_SIZE;
    p->total_size = buffer_size;
    p->next	  = p->prev = 0;
    p->msg.msgbuf = (char *)p + BSENDDATA_HEADER_TRUE_SIZE;

    return MPI_SUCCESS;
}

/* 
 * Detach a buffer.  This routine must wait until any pending bsends 
 * are complete.
 */
int MPIR_Bsend_detach( void *bufferp, int *size )
{
    if (BsendBuffer.pending) {
	return MPIR_Err_create_code( MPI_ERR_OTHER, "**notimpl", 0 );
    }
    if (BsendBuffer.buffer == 0) {
	/* Error - detaching an already detached buffer */
	return MPIR_Err_create_code( MPI_ERR_OTHER, "**bsendnobuf", 0 );
    }
    if (BsendBuffer.active) {
	/* Loop through each active element and wait on it */
	BsendData_t *p = BsendBuffer.active;
	
	MPIR_Nest_incr();
	while (p) {
	    MPI_Request r = p->request->handle;
	    NMPI_Wait( &r, MPI_STATUS_IGNORE );
	    p = p->next;
	}
	MPIR_Nest_decr();
    }

    *(void **) bufferp  = BsendBuffer.buffer;
    *size = BsendBuffer.buffer_size;
    BsendBuffer.buffer  = 0;
    BsendBuffer.avail   = 0;
    BsendBuffer.active  = 0;
    BsendBuffer.pending = 0;

    return MPI_SUCCESS;
}

/*
 * Initiate an ibsend.  We'll used this for Bsend as well.
 */
int MPIR_Bsend_isend( void *buf, int count, MPI_Datatype dtype, 
		      int dest, int tag, MPID_Comm *comm_ptr, 
		      BsendKind_t kind, MPID_Request **request )
{
    BsendData_t *p;
    int packsize, mpi_errno, pass;

    /* Find a free segment and copy the data into it.  If we could 
       have, we would already have used tBsend to send the message with
       no copying.

       We may want to decide here whether we need to pack at all 
       or if we can just use (a memcpy) of the buffer.
    */

    MPIR_Nest_incr();

    /* We check the active buffer first.  This helps avoid storage 
       fragmentation */
    MPIR_Bsend_check_active();

    (void)NMPI_Pack_size( count, dtype, comm_ptr->handle, &packsize );

    DEBUG1(printf("looking for buffer of size %d\n", packsize));
    /*
     * Use two passes.  Each pass is the same; between the two passes,
     * attempt to complete any active requests, and start any pending
     * ones.  If the message can be initiated in the first pass,
     * do not perform the second pass.
     */
    MPID_Thread_lock( &BsendBuffer.bsend_lock );
    for (pass = 0; pass < 2; pass++) {
	
	p = MPIR_Bsend_find_buffer( packsize );
	if (p) {
	    DEBUG(printf("found buffer of size %d with address %x\n",packsize,p));
	    /* Found a segment */
	    
	    /* Pack the data into the buffer */
	    /* We may want to optimize for the special case of
	       either primative or contiguous types, and just
	       use memcpy and the provided datatype */
	    p->msg.count = 0;
	    (void)NMPI_Pack( buf, count, dtype, p->msg.msgbuf, packsize, 
			     &p->msg.count, comm_ptr->handle );
	    /* Try to send the message.  We must use MPID_Isend
	       because this call must not block */
	    mpi_errno = MPID_Isend(p->msg.msgbuf, p->msg.count, MPI_PACKED, 
				   dest, tag, comm_ptr,
				   MPID_CONTEXT_INTRA_PT2PT, &p->request );
	    if(mpi_errno) {
		printf ("Surprise! err = %d\n", mpi_errno );
	    }
	    /* If the error is "request not available", put this on the
	       pending list */
	    /* FIXME: NOT YET DONE */
	    if (mpi_errno == -1) {  /* -1 is a temporary place holder for
				       request-not-available */
		p->msg.dtype     = MPI_PACKED;
		p->msg.tag       = tag;
		p->msg.comm_ptr  = comm_ptr;
		p->msg.dest      = dest;
		
		/* FIXME: Still need to add to pending list */
		
		/* ibsend has a problem.  
		   Use a generalized request here? */
		p->kind = kind;
	    }
	    else if (p->request) {
		DEBUG(printf("saving request %x in %x\n",p->request,p));
		/* Only remove this block from the avail list if the 
		   message has not been sent (MPID_Send may have already 
		   sent the data, in which case it returned a null
		   request) */
#if 0
		/* If the request is already complete, bypass the step
		   of saving the require and taking the buffer */
		if (p->request->cc_ptr == 0) {
		    mpi_errno = MPIR_Request_complete(p->request->handle, 
						      p->requestr, 
					      MPI_STATUS_IGNORE, &active_flag);
		}
#endif
		MPIR_Bsend_take_buffer( p, p->msg.count );
		p->kind  = kind;
		*request = p->request;
	    }
	    else {
		*request = 0;
	    }
	    break;
	}
	if (p && pass == 2) break;
	DEBUG(printf("Could not find storage, checking active\n" ));
	/* Try to complete some pending bsends */
	MPIR_Bsend_check_active( );
	/* Give priority to any pending operations */
	MPIR_Bsend_retry_pending( );
    }
    MPID_Thread_unlock( &BsendBuffer.bsend_lock );
    MPIR_Nest_decr();
    
    if (!p) {
	/* Return error for no buffer space found */
	return MPIR_Err_create_code( MPI_ERR_BUFFER, "**bufbsend",