log_mpi_core.c

fortran并行计算包

/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
   (C) 2001 by Argonne National Laboratory.
       See COPYRIGHT in top-level directory.
*/
#ifdef MPI_BUILD_PROFILING
#undef MPI_BUILD_PROFILING
#endif
#include "mpe_wrappers_conf.h"
#include "mpi.h"
#include "mpe_log.h"


/* AIX requires this to be the first thing in the file.  */
#ifndef __GNUC__
# if HAVE_ALLOCA_H
#  include <alloca.h>
# else
#  ifdef _AIX
 #pragma alloca
#  else
#   ifndef alloca /* predefined by HP cc +Olibcalls */
char *alloca ();
#   endif
#  endif
# endif
#else
# if defined( HAVE_ALLOCA_H )
#  include <alloca.h>
# endif
#endif

#if defined( STDC_HEADERS ) || defined( HAVE_STDLIB_H )
#include <stdlib.h>
#endif
#if defined( STDC_HEADERS ) || defined( HAVE_STDIO_H )
#include <stdio.h>
#endif
#if defined( STDC_HEADERS ) || defined( HAVE_STRING_H )
#include <string.h>
#endif

#ifdef HAVE_STDARG_H
/* Needed for va_start/end in MPI_Pcontrol */
#include <stdarg.h>
#endif

/* Enable memory tracing.  This requires MPICH's mpid/util/tr2.c codes */
#if defined(MPIR_MEMDEBUG)
#define malloc(a)    MPID_trmalloc((unsigned)(a),__LINE__,__FILE__)
#define free(a)      MPID_trfree(a,__LINE__,__FILE__)
#endif

/* 
   This is a large file and may exceed some compilers ability to handle.
   In that case, compile without optimization.
 */

/*
 * To give better control over the log file generated by these routines,
 * the user can switch on an off individual states as well as entire classes
 * of routines.  By default, only the communication routines, pack, and
 * unpack are logged.  This is a change from the previous implementation.
 *
 * MPI_Init checks for logging control options and environment variables,
 * and MPI_Pcontrol allows control over logging (allowing the user to
 * turn logging on and off).  Note that some routines are ALWAYS logged;
 * principly, these are the communicator constuction routines (needed to
 * avoid using the "context_id" which may not exist in some MPI
 * implementations).
 *
 * kind_mask is used ONLY when setting (or clearing) is_active by class.
 *
 * An additional feature is the RECV_IDLE state; this is the state between
 * when a recv is posted and when it is able to complete (basically the
 * time for an MPI_PROBE to succeed).
 *
 * Additional issues:
 * Must log all communicator creation events, even if logging turned off
 * (needed to match up communicator/contexts if a context_id not available).
 * We might choose to allow an MPIR_Comm_id(MPI_Comm) routine in
 * context/comm_util.c to simplify the implementation.
 *
 * We need begin-state and end-state routines, because we (a) might want
 * to use alternate logging routines (just link with a different logging
 * library to get runtime animation) and (b) some logging libraries might
 * want it that way (state-based instead of event-based).
 *
 * In request completion routines (e.g., wait, test), we need to shadow
 * the requests because a completed non-persistent request is freed and
 * the input value set to zero (a previous version of this code did not
 * do this).
 *
 * Note that some sends/recvs can be to/from MPI_PROC_NULL.  Must decide
 * whether no log event should be generated or not (can upshot/nupshot
 * handle MPI_PROC_NULL partners?  Should they?)
 * Previous code was uneven about that.
 *
 * Previous code was also uneven about placement of log_send/recv and
 * particularly on the size logged (some logged count, some size in bytes).
 * This still needs work.
 *
 * An alternate version of this could use macros (and require recompiling).
 */
typedef struct {
    int  stateID;      /* CLOG state ID */
    int  start_evtID;  /* CLOG Event ID for the beginning event */
    int  final_evtID;  /* CLOG event ID for the ending event */
    int  n_calls;      /* Number of times this state used * 2 */
    int  is_active;    /* Allows each state to be selectively switched off */
    int  kind_mask;    /* Indicates kind of state (message, environment) */
    char *name;        /* Pointer to name */
    char *color;       /* Color */
    char *format;      /* Format printf stype string */
} MPE_State;

typedef struct {
    int  eventID;      /* CLOG event ID */
    int  n_calls;      /* Number of times this event used */
    int  is_active;    /* Allows each event to be selectively switched off */
    int  kind_mask;    /* Indicates kind of state (message, environment) */
    char *name;        /* Pointer to name */
    char *color;       /* Color */
} MPE_Event;

/* Kind_mask values */
#define MPE_KIND_MSG 0x1
#define MPE_KIND_TOPO 0x2
#define MPE_KIND_COLL 0x4
#define MPE_KIND_DATATYPE 0x8
#define MPE_KIND_ENV 0x10
#define MPE_KIND_COMM_INFO 0x20
#define MPE_KIND_COMM 0x40
#define MPE_KIND_ATTR 0x80
#define MPE_KIND_GROUP 0x100
#define MPE_KIND_MSG_INIT 0x200
#define MPE_KIND_FILE 0x400
#define MPE_KIND_RMA 0x800
#define MPE_KIND_SPAWN 0x1000
#define MPE_KIND_INTERNAL 0x10000000

/*
   Because of existence of MPE internal states whose state ID is higher than
   any of MPI states' ID.  MPE_MAX_KNOWN_STATES needs to be defined in full.
*/
#define MPE_MAX_KNOWN_STATES 300

#define MPE_MAX_KNOWN_EVENTS 2

void MPE_Init_states_events( void );
void MPE_Init_mpi_core( void );
void MPE_Init_internal_logging( void );

#ifdef HAVE_MPI_IO
void MPE_Init_mpi_io( void );
#endif

#ifdef HAVE_MPI_RMA
void MPE_Init_mpi_rma( void );
#endif

#ifdef HAVE_MPI_SPAWN
void MPE_Init_mpi_spawn( void );
#endif

/* define known events' ID, i.e. index to the corresponding event in events[] */
#define MPE_COMM_INIT_ID 0
#define MPE_COMM_FINALIZE_ID 1

/* define known states' ID, i.e. index to the corresponding state in states[] */
#define MPE_ALLGATHER_ID 0
#define MPE_ALLGATHERV_ID 1
#define MPE_ALLREDUCE_ID 2
#define MPE_ALLTOALL_ID 3
#define MPE_ALLTOALLV_ID 4
#define MPE_BARRIER_ID 5
#define MPE_BCAST_ID 6
#define MPE_GATHER_ID 7
#define MPE_GATHERV_ID 8
#define MPE_OP_CREATE_ID 9
#define MPE_OP_FREE_ID 10
#define MPE_REDUCE_SCATTER_ID 11
#define MPE_REDUCE_ID 12
#define MPE_SCAN_ID 13
#define MPE_SCATTER_ID 14
#define MPE_SCATTERV_ID 15
#define MPE_ATTR_DELETE_ID 16
#define MPE_ATTR_GET_ID 17
#define MPE_ATTR_PUT_ID 18
#define MPE_COMM_COMPARE_ID 19
#define MPE_COMM_CREATE_ID 20
#define MPE_COMM_DUP_ID 21
#define MPE_COMM_FREE_ID 22
#define MPE_COMM_GROUP_ID 23
#define MPE_COMM_RANK_ID 24
#define MPE_COMM_REMOTE_GROUP_ID 25
#define MPE_COMM_REMOTE_SIZE_ID 26
#define MPE_COMM_SIZE_ID 27
#define MPE_COMM_SPLIT_ID 28
#define MPE_COMM_TEST_INTER_ID 29
#define MPE_GROUP_COMPARE_ID 30
#define MPE_GROUP_DIFFERENCE_ID 31
#define MPE_GROUP_EXCL_ID 32
#define MPE_GROUP_FREE_ID 33
#define MPE_GROUP_INCL_ID 34
#define MPE_GROUP_INTERSECTION_ID 35
#define MPE_GROUP_RANK_ID 36
#define MPE_GROUP_RANGE_EXCL_ID 37
#define MPE_GROUP_RANGE_INCL_ID 38
#define MPE_GROUP_SIZE_ID 39
#define MPE_GROUP_TRANSLATE_RANKS_ID 40
#define MPE_GROUP_UNION_ID 41
#define MPE_INTERCOMM_CREATE_ID 42
#define MPE_INTERCOMM_MERGE_ID 43
#define MPE_KEYVAL_CREATE_ID 44
#define MPE_KEYVAL_FREE_ID 45
#define MPE_ABORT_ID 46
#define MPE_ERROR_CLASS_ID 47
#define MPE_ERRHANDLER_CREATE_ID 48
#define MPE_ERRHANDLER_FREE_ID 49
#define MPE_ERRHANDLER_GET_ID 50
#define MPE_ERROR_STRING_ID 51
#define MPE_ERRHANDLER_SET_ID 52
#define MPE_GET_PROCESSOR_NAME_ID 53
#define MPE_INITIALIZED_ID 54
#define MPE_WTICK_ID 55
#define MPE_WTIME_ID 56
#define MPE_ADDRESS_ID 57
#define MPE_BSEND_ID 58
#define MPE_BSEND_INIT_ID 59
#define MPE_BUFFER_ATTACH_ID 60
#define MPE_BUFFER_DETACH_ID 61
#define MPE_CANCEL_ID 62
#define MPE_REQUEST_FREE_ID 63
#define MPE_RECV_INIT_ID 64
#define MPE_SEND_INIT_ID 65
#define MPE_GET_ELEMENTS_ID 66
#define MPE_GET_COUNT_ID 67
#define MPE_IBSEND_ID 68
#define MPE_IPROBE_ID 69
#define MPE_IRECV_ID 70
#define MPE_IRSEND_ID 71
#define MPE_ISEND_ID 72
#define MPE_ISSEND_ID 73
#define MPE_PACK_ID 74
#define MPE_PACK_SIZE_ID 75
#define MPE_PROBE_ID 76
#define MPE_RECV_ID 77
#define MPE_RSEND_ID 78
#define MPE_RSEND_INIT_ID 79
#define MPE_SEND_ID 80
#define MPE_SENDRECV_ID 81
#define MPE_SENDRECV_REPLACE_ID 82
#define MPE_SSEND_ID 83
#define MPE_SSEND_INIT_ID 84
#define MPE_START_ID 85
#define MPE_STARTALL_ID 86
#define MPE_TEST_ID 87
#define MPE_TESTALL_ID 88
#define MPE_TESTANY_ID 89
#define MPE_TEST_CANCELLED_ID 90
#define MPE_TESTSOME_ID 91
#define MPE_TYPE_COMMIT_ID 92
#define MPE_TYPE_CONTIGUOUS_ID 93
#define MPE_TYPE_EXTENT_ID 94
#define MPE_TYPE_FREE_ID 95
#define MPE_TYPE_HINDEXED_ID 96
#define MPE_TYPE_HVECTOR_ID 97
#define MPE_TYPE_INDEXED_ID 98
#define MPE_TYPE_LB_ID 99
#define MPE_TYPE_SIZE_ID 100
#define MPE_TYPE_STRUCT_ID 101
#define MPE_TYPE_UB_ID 102
#define MPE_TYPE_VECTOR_ID 103
#define MPE_UNPACK_ID 104
#define MPE_WAIT_ID 105
#define MPE_WAITALL_ID 106
#define MPE_WAITANY_ID 107
#define MPE_WAITSOME_ID 108
#define MPE_CART_COORDS_ID 109
#define MPE_CART_CREATE_ID 110
#define MPE_CART_GET_ID 111
#define MPE_CART_MAP_ID 112
#define MPE_CART_SHIFT_ID 113
#define MPE_CARTDIM_GET_ID 114
#define MPE_DIMS_CREATE_ID 115
#define MPE_GRAPH_CREATE_ID 116
#define MPE_GRAPH_GET_ID 117
#define MPE_GRAPH_MAP_ID 118
#define MPE_GRAPH_NEIGHBORS_ID 119
#define MPE_GRAPH_NEIGHBORS_COUNT_ID 120
#define MPE_GRAPHDIMS_GET_ID 121
#define MPE_TOPO_TEST_ID 122
#define MPE_RECV_IDLE_ID 123
#define MPE_CART_RANK_ID 124
#define MPE_CART_SUB_ID 125

/*
   Be sure NO MPE internal states are overlapped with ANY of MPI states
   Also, CLOG's internal states which are defined in clog_record.h
   should be overlapping with MPE internal states as well.

       250 <= MPE's internal stateID < 280
   and 280 <= CLOG's internal stateID < MPE_MAX_KNOWN_STATES

   This is done so the MPE internal stateID/evetIDs are included in
   the clog2TOslog2's predefined MPI uninitialized states.
*/

#define MPE_ISEND_WAITED_ID 250
#define MPE_IRECV_WAITED_ID 251

#include "mpe_requests.h"

#include "mpe_log_thread.h"

/* define global known states and events */
static MPE_State states[MPE_MAX_KNOWN_STATES];
static MPE_Event events[MPE_MAX_KNOWN_EVENTS];

/*
   Global trace control
   is_mpilog_on : a boolean flag indicates if MPI user level profiling is on.
   IS_MPELOG_ON : a boolean flag indicates if internal MPE profiling is on.
                  This allows MPE to turn off logging for safe PMPI calls.
   IS_MPELOG_ON = is_mpelog_on in a single thread program.
   IS_MPELOG_ON = thdstm->is_log_on in multiple threads situation.
   So IS_MPELOG_ON is defined by mpe_log_thread.h.
*/
static int is_mpilog_on = 0;
#if !defined( HAVE_PTHREAD_IN_MPI )
static int is_mpelog_on = 0;
#endif


/*  LOGFILENAME_LEN == CLOG_PATH_STRLEN  */
#define LOGFILENAME_STRLEN  256
static request_list *requests_head_0, *requests_tail_0, *requests_avail_0=0;
static int procid_0;
static char logFileName_0[LOGFILENAME_STRLEN];

/* This is used for the multiple-completion test/wait functions */
#define MPE_MAX_REQUESTS 1024
static MPI_Request req[MPE_MAX_REQUESTS];

/* Function prototypes for MPI_Request processing */
void MPE_Req_add_send( MPI_Request, MPI_Datatype, int,
                       int, int, const CLOG_CommIDs_t*, int );
void MPE_Req_add_recv( MPI_Request, MPI_Datatype, int,
                       int, int, const CLOG_CommIDs_t*, int );
void MPE_Req_cancel( MPI_Request );
void MPE_Req_remove( MPI_Request );
void MPE_Req_start( MPI_Request, MPE_State *, int, int );
void MPE_Req_wait_test( MPI_Request, MPI_Status *, char *, MPE_State *,
                        int, int );

/*
   Temporary MPE log definitions (eventually will replace with more
   permanent changes)
   Note that these include a communicator as well as the state (pointer
   to predefined state structure).  Use MPE_COMM_NULL for no communicator
*/
#define MPE_COMM_NULL  MPI_COMM_WORLD

/*
   To use these, declare
     register MPE_State *state;
   and call around routine
*/
/*
   For GNU CC with warnings turned on, we should use a macro that 
   declares this as register MPE_State *state = 0; when error checking 
   is on, just to suppress unnecessary warnings
*/
/*
   is_thisfn_logged is a logging switch that is local within 
   each profiled MPI function that MPE_LOG_SWITCH_DECL is used,
   i.e. on top of the program stack.  is_thisfn_logged is true
   when all other log switches are true, so it summarizes other
   switches (for performance reason).  When a profiled function
   is executed, is_thisfn_logged is default to false.
*/
#define MPE_LOG_SWITCH_DECL \
    register       int              is_thisfn_logged = 0;

#ifdef GCC_WALL

#define MPE_LOG_STATE_DECL \
    register       MPE_State       *state   = 0; \
    register const CLOG_CommIDs_t  *commIDs = 0; \
    MPE_LOG_SWITCH_DECL
#define MPE_LOG_COMM_DECL \
    register       MPE_Event       *solo_event  = 0; \
    register const CLOG_CommIDs_t  *new_commIDs = 0;
#define MPE_LOG_SOLO_EVENT_DECL \
    register       MPE_Event       *solo_event  = 0;
#define MPE_LOG_BYTEBUF_DECL \
                   MPE_LOG_BYTES    bytebuf = {0};  \
                   int              bytebuf_pos = 0;

#else

#define MPE_LOG_STATE_DECL \
    register       MPE_State       *state; \
    register const CLOG_CommIDs_t  *commIDs; \
    MPE_LOG_SWITCH_DECL
#define MPE_LOG_COMM_DECL \
    register       MPE_Event       *solo_event; \
    register const CLOG_CommIDs_t  *new_commIDs;
#define MPE_LOG_SOLO_EVENT_DECL \
    register       MPE_Event       *solo_event;
#define MPE_LOG_BYTEBUF_DECL \
                   MPE_LOG_BYTES    bytebuf;  \
                   int              bytebuf_pos;

#endif

extern MPEU_DLL_SPEC CLOG_CommSet_t  *CLOG_CommSet;

/*
   All following macros have "comm" as argument, but none of them except
   MPE_LOG_STATE_BEGIN needs comm argument.  Instead all of them need
   commIDs as an argument.  "comm" are used in all macros to indicate
   the macro body needs reference of comm, i.e commIDs.  The goal is that
   the functions that invoke these macros will look clearer and more consistent.
*/
#define MPE_LOG_STATE_BEGIN(comm,name) \
    if (is_mpilog_on && IS_MPELOG_ON) { \
        state = &states[name]; \
        if (state->is_active) { \
            commIDs = CLOG_CommSet_get_IDs( CLOG_CommSet, comm ); \
            MPE_Log_commIDs_event( commIDs, THREADID, \
                                   state->start_evtID, NULL ); \
            is_thisfn_logged = 1; \
        } \
    }
/*    if (is_mpilog_on && IS_MPELOG_ON && state->is_active) { \ */
#define MPE_LOG_STATE_END(comm,bytebuffer) \
    if (is_thisfn_logged) { \
        MPE_Log_commIDs_event( commIDs, THREADID, \
                               state->final_evtID, bytebuffer ); \
        state->n_calls += 2; \
    }

/*    if (is_mpilog_on && IS_MPELOG_ON) { \ */
#define MPE_LOG_SOLO_EVENT(commIDs,thdID,name) \
    if (is_thisfn_logged) { \
        solo_event = &events[name]; \
        if (solo_event->is_active) { \
            MPE_Log_commIDs_event( commIDs, thdID, \
                                   solo_event->eventID, NULL ); \
            solo_event->n_calls += 1; \
        } \
    }

/*    if (is_mpilog_on && IS_MPELOG_ON && state->is_active) { \ */
#define MPE_LOG_COMM_SEND(comm,receiver,tag,size) \
    if (is_thisfn_logged) { \
        MPE_Log_commIDs_send( commIDs, THREADID, receiver, tag, size ); \
    }
/*    if (is_mpilog_on && IS_MPELOG_ON && state->is_active) { \ */
#define MPE_LOG_COMM_RECV(comm,sender,tag,size) \
    if (is_thisfn_logged) { \
        MPE_Log_commIDs_receive( commIDs, THREADID, sender, tag, size ); \
    }

#define MPE_REQ_ADD_SEND(request,datatype,count,dest,tag,comm,is_persistent) \
    if (dest != MPI_PROC_NULL) { \
        MPE_Req_add_send( request, datatype, count, \
                          dest, tag, commIDs, is_persistent ); \
    }
#define MPE_REQ_ADD_RECV(request,datatype,count,source,tag,comm,is_persistent) \
    if (source != MPI_PROC_NULL) { \
       MPE_Req_add_recv( request, datatype, count, \
                         source, tag, commIDs, is_persistent ); \
    }