mpid_accumulate.c

fortran并行计算包

/*  (C)Copyright IBM Corp.  2007, 2008  */
/**
 * \file src/onesided/mpid_accumulate.c
 * \brief MPI-DCMF MPI_Accumulate functionality
 */

#include "mpid_onesided.h"

#define DATATYPE_ADDITIONAL(dt)	\
	(dt == MPI_FLOAT_INT ||	\
	dt == MPI_DOUBLE_INT ||	\
	dt == MPI_LONG_INT ||	\
	dt == MPI_SHORT_INT ||	\
	dt == MPI_LONG_DOUBLE_INT)

#define DATATYPE_PREDEFINED(dt)	\
        (HANDLE_GET_KIND(dt) == HANDLE_KIND_BUILTIN ||	\
				DATATYPE_ADDITIONAL(dt))

/**
 * \brief Utility routine to provide accumulate function locally.
 *
 * Utility routine to provide accumulate function locally.
 *
 * Called from \e target_accumulate(), or \e MPID_Accumulate() in the
 * case of local operation (to self).
 *
 * Non-contiguous datatypes are handled by \e MPID_Accumulate(),
 * which splits the data into contiguous regions. Note that MPICH2
 * states that datatypes used with MPI_Accumulate must be "monotonic",
 * they must be of only one underlying type.
 *
 * We have to cast the 'src' param to '(void *)' in our calls to the
 * operand function because MPI_User_function is declared without the 'const'
 * even though MPID_User_function union elements have the 'const'.
 * Also, cxx_call_op_fn is missing the 'const'.  Even with that, we still
 * have to cast when assigning to uop from op_ptr, because of the way
 * f77_function and c_function are declared.
 *
 * \param[in] win	Pointer to MPID_Win object
 * \param[in] dst	Pointer to destination buffer
 * \param[in] src	Pointer to source buffer
 * \param[in] lpid	lpid of origin
 * \param[in] dt	Local datatype
 * \param[in] op	Operand
 * \param[in] num	number of dt elements
 * \return nothing
 *
 */
static void local_accumulate(MPID_Win *win, char *dst, const char *src,
                int lpid, MPI_Datatype dt, MPI_Op op, int num) {
        MPI_User_function *uop;
        int is_cxx = 0;

        if (HANDLE_GET_KIND(op) == HANDLE_KIND_BUILTIN) {
                uop = MPIR_Op_table[op%16 - 1];
        } else {
                MPID_Op *op_ptr;

                /*
                 * This case can only happen if accumulate originated locally.
                 */
                MPID_Op_get_ptr(op, op_ptr);
                if (op_ptr->language == MPID_LANG_C) {
                        uop = (MPI_User_function *)op_ptr->function.c_function;
#ifdef HAVE_CXX_BINDING
                } else if (op_ptr->language == MPID_LANG_CXX) {
                        uop = (MPI_User_function *)op_ptr->function.c_function;
                        ++is_cxx;
#endif /* HAVE_CXX_BINDING */
                } else {
                        uop = (MPI_User_function *)op_ptr->function.f77_function;
                }
        }
        /* MPI_REPLACE was filtered-out in MPID_Accumulate() */
        /* Also, MPID_Accumulate() only passes builtin types to us */
        /* builtin implies contiguous */
#ifdef HAVE_CXX_BINDING
        if (is_cxx)
                (*MPIR_Process.cxx_call_op_fn)((void *)src, dst, num, dt, uop);
        else
#endif /* HAVE_CXX_BINDING */
                (*uop)((void *)src, dst, &num, &dt);
}

/**
 * \brief Utility routine to provide accumulate function on target.
 *
 * Utility routine to provide accumulate function on target.
 * Updates RMA count.
 *
 * w0 = MPID_MSGTYPE_ACC (not used)
 * w1 = Window handle
 * w2 = Rank of origin
 * w3 = Destination buffer address
 * w4 = "eltype" datatype handle (must be builtin)
 * w5 = Operand
 * w6 = number of datatype elements
 * w7 = (not used)
 *
 * Called from "long message" ACCUMULATE completion callback
 * or "short message" ACCUMULATE receive callback.
 *
 * \param[in] mi	MPIDU_Onesided_info_t for accumulate, as described above
 * \param[in] src	Pointer to source buffer
 * \param[in] lpid	lpid of origin
 * \return nothing
 */
void target_accumulate(MPIDU_Onesided_info_t *mi,
                                        const char *src, int lpid) {
        MPID_Win *win;

        MPID_Win_get_ptr((MPI_Win)mi->mpid_info_w1, win);
        MPID_assert_debug(win != NULL);
        local_accumulate(win, (char *)mi->mpid_info_w3, src,
                mi->mpid_info_w2, mi->mpid_info_w4, mi->mpid_info_w5, mi->mpid_info_w6);
        rma_recvs_cb(win, mi->mpid_info_w2, lpid);
}

/**
 * \brief Callback for Accumulate recv completion
 *
 * "Message receive completion" callback used for MPID_MSGTYPE_ACC
 * to implement the accumulate function. Decodes data from request
 * cache object, frees request, does accumulate, and updates RMA count.
 *
 * Used for "long message" ACCUMULATE.
 *
 * To use this callback, the "xtra" info (DCQuad) must
 * be filled as follows:
 *
 * - \e w0 - ignored
 * - \e w1 - ignored
 * - \e w2 - (int *)multi-struct buffer (int *, DCQuad[], data)
 * - \e w3 - origin lpid
 *
 * \param[in] v	Pointer to DCMF request object
 * \return nothing
 *
 * \ref msginfo_usage
 */
void accum_cb(void *v) {
        MPIDU_Onesided_info_t *info;
        DCQuad xtra;
        char *buf;

        MPIDU_free_req((DCMF_Request_t *)v, &xtra);
        info = (MPIDU_Onesided_info_t *)xtra.w2;
        buf = (char *)(info + 1);

        target_accumulate(info, buf, xtra.w3);
        MPIDU_FREE(info, e, "accum_cb");
}

/**
 * \page accum_design MPID_Accumulate Design
 *
 * A MPID_Accumulate sequence is as follows:
 *
 * <B>Origin node calls MPI_Accumulate</B>
 *
 * - A sanity-check is done to
 * ensure that the window is in a valid state to initiate
 * an accumulate RMA operation.
 * These checks include testing that
 * the epoch currently in affect is not \e NONE or \e POST.
 * Additionally, the target node is checked to ensure it is
 * currently a legitimate target of an RMA operation.
 * - If the target node is the origin node, and the epoch type
 * is \e LOCK, then require that the local lock be acquired.
 * - If the operand is MPI_REPLACE, then perform a Put instead.
 * - Require that the oprand be a built-in operand.
 * - If the origin datatype is non-contiguous, allocate a buffer
 * and pack the origin data into a contiguous buffer.
 * - If the target datatype is not built-in, send the target
 * datatype to the target node for caching there (sending of
 * a particular datatype to a particular target is done only
 * once per job).
 *  - Send a message of type MPID_MSGTYPE_DT_MAP which contains
 * the datatype map structure as created by make_dt_map_vec().
 *  - Send a message of type MPID_MSGTYPE_DT_IOV which contains
 * the datatype iov structure, also created by make_dt_map_vec().
 * - Create the msg info for the accumulate operation.
 * - Call DCMF_Send to send the data and associated parameters
 * to the target node.
 * - Increment the RMA operations count for target node.
 * - Wait for all sends to complete, calling advance in the loop.
 *
 * <B>Target node invokes the receive callback</B>
 *
 * If the datatype was sent, cases \e MPID_MSGTYPE_DT_MAP and
 * \e MPID_MSGTYPE_DT_IOV will be invoked, in sequence.
 *
 * Case: MPID_MSGTYPE_DT_MAP
 *
 * - Sanity-check the message by testing for msginfo count of 1.
 * - Allocate space for the map and store its pointer in the datatype
 * cache element.
 * - Receive the remote map contents.
 *
 * Case: MPID_MSGTYPE_DT_IOV
 *
 * - Sanity-check the message by testing for msginfo count of 1.
 * - Allocate a iov buffer and store its pointer in the datatype
 * cache element's dataloop field.
 * - Receive the remote datatype iov contents.
 *
 * Case: MPID_MSGTYPE_ACC
 *
 * - Sanity-check message...
 * - Setup operand function pointer.
 * - Invoke operand from \e MPIR_Op_table on received data and
 * specified target window buffer.
 * - Increment counter of received RMA operations.
 */
/// \cond NOT_REAL_CODE
#undef FUNCNAME
#define FUNCNAME MPID_Accumulate
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
/// \endcond
/**
 * \brief MPI-DCMF glue for MPI_ACCUMULATE function
 *
 * Perform DEST = DEST (op) SOURCE for \e origin_count number of