clog_commset.c

mpi并行计算的c++代码 可用vc或gcc编译通过 可以用来搭建并行计算试验环境

/*
   (C) 2001 by Argonne National Laboratory.
       See COPYRIGHT in top-level directory.
*/
#include "mpe_logging_conf.h"

#if defined( STDC_HEADERS ) || defined( HAVE_STDIO_H )
#include <stdio.h>
#endif
#if defined( STDC_HEADERS ) || defined( HAVE_STDLIB_H )
#include <stdlib.h>
#endif
#if defined( STDC_HEADERS ) || defined( HAVE_STRING_H )
#include <string.h>
#endif

#include "clog_const.h"
#include "clog_uuid.h"
#include "clog_commset.h"
#include "clog_util.h"

#if !defined( CLOG_NOMPI )

#if !defined( HAVE_PMPI_COMM_CREATE_KEYVAL )
#define PMPI_Comm_create_keyval PMPI_Keyval_create
#endif

#if !defined( HAVE_PMPI_COMM_FREE_KEYVAL )
#define PMPI_Comm_free_keyval PMPI_Keyval_free
#endif

#if !defined( HAVE_PMPI_COMM_SET_ATTR )
#define PMPI_Comm_set_attr PMPI_Attr_put
#endif

#if !defined( HAVE_PMPI_COMM_GET_ATTR )
#define PMPI_Comm_get_attr PMPI_Attr_get
#endif

CLOG_CommSet_t* CLOG_CommSet_create( void )
{
    CLOG_CommSet_t  *commset;
    int              table_size;

    commset = (CLOG_CommSet_t *) MALLOC( sizeof(CLOG_CommSet_t) );
    if ( commset == NULL ) {
        fprintf( stderr, __FILE__":CLOG_CommSet_create() - MALLOC() fails.\n" );
        fflush( stderr );
        return NULL;
    }

    /* LID_key Initialized to 'unallocated' */
    commset->LID_key   = MPI_KEYVAL_INVALID;
    commset->max       = CLOG_COMM_TABLE_INCRE;
    commset->count     = 0;
    table_size         = commset->max * sizeof(CLOG_CommIDs_t);
    commset->table     = (CLOG_CommIDs_t *) MALLOC( table_size );

    return commset;
}

void CLOG_CommSet_free( CLOG_CommSet_t **comm_handle )
{
    CLOG_CommSet_t  *commset;

    commset = *comm_handle;
    if ( commset->table != NULL )
        FREE( commset->table );
    PMPI_Comm_free_keyval( &(commset->LID_key) );
    if ( commset != NULL )
        FREE( commset );
    *comm_handle = NULL;
}

/*
   CLOG_CommSet_init() should only be called if MPI is involved.
*/
void CLOG_CommSet_init( CLOG_CommSet_t *commset )
{
    int          *extra_state = NULL; /* unused variable */

    CLOG_Uuid_init();

    /* create LID_Key */
    PMPI_Comm_create_keyval( MPI_COMM_NULL_COPY_FN,
                             MPI_COMM_NULL_DELETE_FN, 
                             &(commset->LID_key), extra_state );

    PMPI_Comm_size( MPI_COMM_WORLD, &(commset->world_size) );
    PMPI_Comm_rank( MPI_COMM_WORLD, &(commset->world_rank) );

    CLOG_CommSet_add_intracomm( commset, MPI_COMM_WORLD );
    CLOG_CommSet_add_intracomm( commset, MPI_COMM_SELF );
}

static CLOG_CommIDs_t* CLOG_CommSet_get_new_IDs( CLOG_CommSet_t *commset )
{
    CLOG_CommIDs_t  *new_table;
    CLOG_CommIDs_t  *new_entry;
    int              new_size;

    /* Enlarge the CLOG_CommSet_t.table if necessary */
    if ( commset->count >= commset->max ) {
        commset->max += CLOG_COMM_TABLE_INCRE;
        new_size      = commset->max * sizeof(CLOG_CommIDs_t);
        new_table     = (CLOG_CommIDs_t *) REALLOC( commset->table, new_size );
        if ( new_table == NULL ) {
            fprintf( stderr, __FILE__":CLOG_CommSet_get_next_IDs() - \n"
                             "\t""REALLOC(%p,%d) fails!\n",
                             commset->table, new_size );
            fflush( stderr );
            CLOG_Util_abort( 1 );
        }
        commset->table = new_table;
    }

    /* return the next available table entry in CLOG_CommSet_t */
    new_entry  = &(commset->table[ commset->count ]);

    /* Set the local Comm ID temporarily equal to the table entry's index */
    new_entry->local_ID   = commset->count;

    /* set the new entry with the process rank in MPI_COMM_WORLD */
    new_entry->world_rank = commset->world_rank;

    /* Set the related CLOG_CommIDs_t* as NULL(Most commIDs refer intracomms. */
    new_entry->next      = NULL;

    /*
       Increment the count to next available slot in the table.
       Also, the count indicates the current used slot in the table.
    */
    commset->count++;

    return new_entry;
}

/*
    This function should be used on every newly created communicator
    The input argument MPI_Comm is assumed to be not MPI_COMM_NULL
*/
const CLOG_CommIDs_t *CLOG_CommSet_add_intracomm( CLOG_CommSet_t *commset,
                                                  MPI_Comm        intracomm )
{
    CLOG_CommIDs_t  *intracommIDs;

    /* Update the next available table entry in CLOG_CommSet_t */
    intracommIDs         = CLOG_CommSet_get_new_IDs( commset );
    intracommIDs->kind   = CLOG_COMM_KIND_INTRA;

    /* Set the input MPI_Comm's LID_key attribute with new local CommID's LID */
    PMPI_Comm_set_attr( intracomm, commset->LID_key,
                        (void *) (MPI_Aint) intracommIDs->local_ID );

    /* Set the Comm field */
    intracommIDs->comm   = intracomm;
    PMPI_Comm_rank( intracommIDs->comm, &(intracommIDs->comm_rank) );

    /* Set the global Comm ID */
    if ( intracommIDs->comm_rank == 0 )
        CLOG_Uuid_generate( intracommIDs->global_ID );
    PMPI_Bcast( intracommIDs->global_ID, CLOG_UUID_SIZE, MPI_CHAR,
                0, intracomm );
#if defined( CLOG_COMMSET_PRINT )
    char uuid_str[CLOG_UUID_STR_SIZE] = {0};
    CLOG_Uuid_sprint( intracommIDs->global_ID, uuid_str );
    fprintf( stdout, "comm_rank=%d, comm_global_ID=%s\n",
                     intracommIDs->comm_rank, uuid_str );
#endif

    return intracommIDs;
}

/*
    Assume that "intracomm" be the LOCAL communicator of the "intercomm".
*/
const CLOG_CommIDs_t*
CLOG_CommSet_add_intercomm(       CLOG_CommSet_t *commset,
                                  MPI_Comm        intercomm,
                            const CLOG_CommIDs_t *orig_intracommIDs )
{
    CLOG_CommIDs_t  *intercommIDs;
    CLOG_CommIDs_t  *local_intracommIDs, *remote_intracommIDs;
    MPI_Status       status;
    MPI_Request      request;
    int              is_intercomm;

    /* Confirm if input intercomm is really an intercommunicator */
    PMPI_Comm_test_inter( intercomm, &is_intercomm );
    if ( !is_intercomm )
        return CLOG_CommSet_add_intracomm( commset, intercomm );

    /* Set the next available table entry in CLOG_CommSet_t with intercomm */
    intercommIDs         = CLOG_CommSet_get_new_IDs( commset );
    intercommIDs->kind   = CLOG_COMM_KIND_INTER;

    /* Set the input MPI_Comm's LID_key attribute with new local CommID */
    PMPI_Comm_set_attr( intercomm, commset->LID_key,
                        (void *) (MPI_Aint) intercommIDs->local_ID );

    /* Set the Comm field with the intercomm's info */
    intercommIDs->comm   = intercomm;
    PMPI_Comm_rank( intercommIDs->comm, &(intercommIDs->comm_rank) );

    /* Set the global Comm ID based on intercomm's local group rank */
    if ( intercommIDs->comm_rank == 0 )
        CLOG_Uuid_generate( intercommIDs->global_ID );

    /*
       Broadcast the (local side of) intercomm ID within local intracomm,
       i.e. orig_intracommIDs->comm
    */
    PMPI_Bcast( intercommIDs->global_ID, CLOG_UUID_SIZE, MPI_CHAR,
                0, orig_intracommIDs->comm );

#if defined( CLOG_COMMSET_PRINT )
    char uuid_str[CLOG_UUID_STR_SIZE] = {0};
    CLOG_Uuid_sprint( intercommIDs->global_ID, uuid_str );
    fprintf( stdout, "comm_rank=%d, comm_global_ID=%s\n",
                     intercommIDs->comm_rank, uuid_str );
#endif

    /* Set the next available table entry with the LOCAL intracomm's info */
    local_intracommIDs            = CLOG_CommSet_get_new_IDs( commset );
    local_intracommIDs->kind      = CLOG_COMM_KIND_LOCAL;
    local_intracommIDs->local_ID  = orig_intracommIDs->local_ID;
    CLOG_Uuid_copy( orig_intracommIDs->global_ID,
                    local_intracommIDs->global_ID );
    local_intracommIDs->comm      = orig_intracommIDs->comm;
    local_intracommIDs->comm_rank = orig_intracommIDs->comm_rank;
    /* NOTE: LOCAL intracommIDs->comm_rank == intercommIDs->comm_rank */

    /* Set the next available table entry with the REMOTE intracomm's info */
    remote_intracommIDs             = CLOG_CommSet_get_new_IDs( commset );
    remote_intracommIDs->kind       = CLOG_COMM_KIND_REMOTE;