mpid_onesided.h

fortran并行计算包

 *
 * \param[in] win	Pointer to MPID_Win object
 * \param[in] orig	Rank of originator of RMA operation
 * \param[in] lpid	lpid of originator of RMA operation
 * \return nothing
 */
void rma_recvs_cb(MPID_Win *win, int orig, int lpid);

/**
 * \brief Generic request cache callback for RMA op completion
 *
 * Callback for incrementing window RMA recvs count.
 * Used only by Put and Accumulate (not used by Get).
 *
 * Only used for a "long message" - i.e. multi-packet - PUT.
 *
 * To use this callback, the "xtra" info (DCQuad) must
 * be filled as follows:
 *
 * - \e w0 - ignored
 * - \e w1 - window handle
 * - \e w2 - origin rank
 * - \e w3 - origin lpid
 *
 * \param[in] v	Pointer to DCMF request object
 * \return nothing
 *
 * \ref rqcache_design
 */
void rma_rqc_cb(void *v);

/**
 * \brief Generic callback for request cache
 *
 * Callback for simply (only) freeing the request cache object.
 *
 * To use this callback, the "xtra" info (DCQuad) must
 * be filled as follows:
 *
 * - \e w0 - ignored
 * - \e w1 - ignored
 * - \e w2 - ignored
 * - \e w3 - ignored
 *
 * \param[in] v	Pointer to DCMF request object
 * \return nothing
 *
 * \ref rqcache_design
 */
void none_rqc_cb(void *v);

#ifdef NOT_USED
/**
 * \brief Generic send done callback
 *
 * Local send callback.
 *
 * Simple "done" callback, currently used only by lock/unlock.
 * Assumes param is an int * and decrements it.
 *
 * \param[in] v	Pointer to integer counter to decrement
 * \return nothing
 */
void done_cb(void *v);
#endif /* NOT_USED */

/**
 *  \brief receive callback for datatype cache messages (map and iov)
 *
 * \param[in] v	Pointer to request object used for transfer
 * \return	nothing
 */
void dtc1_rqc_cb(void *v);

#ifdef NOT_USED
/**
 *  \brief receive callback for datatype cache messages (map and iov)
 *
 * \param[in] v	Pointer to request object used for transfer
 * \return	nothing
 */
void dtc2_rqc_cb(void *v);
#endif /* NOT_USED */

/*
 * * * * * * * * * * * * * * * * * * * * * *
 */

/**
 * \brief Return communicator's VCR table to use for translations
 *
 * \param[in] c	Pointer to communicator object
 * \return VCR table to use for rank translations
 *
 * \todo Confirm this: we only ever talk to remote nodes,
 * so always use (c)->vcr (?).
 */
static inline MPID_VCR *MPIDU_world_vcr_c(MPID_Comm *c) {
        return (c)->vcr;
}

/**
 * \brief Convert comm rank to world rank
 *
 * \param[in] c	Pointer to communicator object
 * \param[in] r	Rank of node in window
 * \return World rank (lpid) of node
 *
 * \todo Confirm this: we only ever talk to remote nodes,
 * so always use (c)->vcr (?).
 */
static inline int MPIDU_world_rank_c(MPID_Comm *c, int r) {
        MPID_VCR *vc;
        vc = MPIDU_world_vcr_c(c);
        return vc[r]->lpid;
}

/**
 * \brief Return the active size of a communicator
 *
 * For inter-comms, we will only talk to the remote side so
 * return that size.  Otherwise, use the local size.
 *
 * \param[in] c	Pointer to communicator object
 * \return Size of communicator that we will talk to
 */
static inline int MPIDU_comm_size_c(MPID_Comm *c) {
        return ((c)->comm_kind == MPID_INTERCOMM ?
                (c)->remote_size :
                (c)->local_size);
}

/**
 * \brief Convert window rank to world rank
 *
 * Used in MPID_Win_{lock|put|get|accumulate} to get the
 * COMM_WORLD rank of a window-comm rank in order to get
 * the destination parameter of a DCMF_ operation.
 *
 * Assumes MPID_Win_* might be called with an intercomm.
 *
 * Note, MPIDU_proto_send() does not call this since it
 * ends up getting the COMM_WORLD rank as a result of the
 * group-to-window-comm rank translation.
 *
 * \param[in] w	Pointer to MPID_Win object
 * \param[in] r	Rank of node in window
 * \return World rank (lpid) of node
 */
#define MPIDU_world_rank(w, r)	MPIDU_world_rank_c((w)->_dev.comm_ptr, r)

/**
 * \brief Return the VCR of a window's communicator
 *
 * \param[in] w	Pointer to MPID_Win object
 * \return VCR table to use for rank translations
 */
#define MPIDU_world_vcr(w)	MPIDU_world_vcr_c((w)->_dev.comm_ptr)

/**
 * \brief Return the active size of a window's communicator
 *
 * \param[in] w	Pointer to MPID_Win object
 * \return Size of communicator
 */
#define MPIDU_comm_size(w)	MPIDU_comm_size_c((w)->_dev.comm_ptr)

/**
 * \brief Send (spray) a protocol message to a group of nodes.
 *
 * Send a protocol message to all members of a group (or the
 * window-comm if no group).
 *
 * Currently, this routine will only be called once per group
 * (i.e. once during an exposure or access epoch). If it ends
 * up being called more than once, it might make sense to build
 * a translation table between the group rank and the window
 * communicator rank.  Or if we can determine that the same
 * group is being used in multiple, successive, epochs. In practice,
 * it takes more work to build a translation table than to lookup
 * ranks ad-hoc.
 *
 * \param[in] win	Pointer to MPID_Win object
 * \param[in] grp	Optional pointer to MPID_Group object
 * \param[in] type	Type of message (MPID_MSGTYPE_*)
 * \return MPI_SUCCESS or error returned from DCMF_Send.
 *
 * \ref msginfo_usage
 */
int MPIDU_proto_send(MPID_Win *win, MPID_Group *grp, int type);

/**
 * \brief Utility routine to provide accumulate function on target.
 *
 * Utility routine to provide accumulate function on target.
 *
 * Called from "long message" ACCUMULATE completion callback
 * or "short message" ACCUMULATE receive callback.
 *
 * \param[in] win	Pointer to MPID_Win object
 * \param[in] dst	Pointer to destination buffer
 * \param[in] src	Pointer to source buffer
 * \param[in] rank	Rank of origin
 * \param[in] fdt	Foreign datatype
 * \param[in] op	Operand
 * \param[in] num	number of Foreign datatype elements
 * \return nothing
 */
void target_accumulate(MPIDU_Onesided_info_t *mi,
                                const char *src, int lpid);

/*
 * * * * * * * * * * * * * * * * * * * * * *
 */

/**
 * \brief validate whether a lpid is in a given group
 *
 * Searches the group lpid list for a match.
 *
 * \param[in] lpid	World rank of the node in question
 * \param[in] grp	Group to validate against
 * \return TRUE is lpid is in group
 */
int MPIDU_valid_group_rank(int lpid, MPID_Group *grp);

/**
 * \brief Test whether a window is in an RMA access epoch
 *
 * Assert that the local window is in the proper mode to receive
 * RMA operations.
 *
 * \note One cause of this to fail is improper use of MPI_MODE_NOPUT.
 *
 * \param[in] w	Pointer to window object
 * \return	TRUE if RMA ops are allowed
 */
#define MPIDU_assert_RMAOK(w)	MPID_assert((w)->_dev.epoch_rma_ok)
/**
 * \brief Test whether a window is in an RMA access epoch
 *
 * Assert that the local window is in the proper mode to receive
 * RMA operations.
 *
 * \note One cause of this to fail is improper use of MPI_MODE_NOPUT.
 * Another is erroneous use of MPI_MODE_NOCHECK.
 *
 * \param[in] w	Pointer to window object
 * \return	TRUE if PUT/ACCUMULATE ops are allowed
 */
#define MPIDU_assert_PUTOK(w)	MPID_assert((w)->_dev.epoch_rma_ok && \
                                !((w)->_dev.epoch_assert & MPI_MODE_NOPUT))

/**
 * \brief validate that an RMA target is legitimate for the epoch type
 *
 * For MPID_EPOTYPE_LOCK requires target to be the same as that
 * used in the MPID_Win_lock call.
 *
 * For MPID_EPOTYPE_FENCE allows any target,
 * assuming that the target was validated against comm_ptr
 * by the MPI layer.
 *
 * For MPID_EPOTYPE_*START valids the rank against group_ptr.
 *
 * \todo Is this check too expensive to be done on every RMA?
 *
 * \param[in] w	Window
 * \param[in] r	Rank
 * \return TRUE if rank is valid for current epoch
 */
#define MPIDU_VALID_RMA_TARGET(w, r)	\
        (((w)->_dev.epoch_type == MPID_EPOTYPE_LOCK && \
                (w)->_dev.epoch_size == (r)) || \
        (w)->_dev.epoch_type == MPID_EPOTYPE_FENCE || \
        (((w)->_dev.epoch_type == MPID_EPOTYPE_START || \
          (w)->_dev.epoch_type == MPID_EPOTYPE_POSTSTART) && \
                MPIDU_valid_group_rank(MPIDU_world_rank(w, r), \
                                        (w)->start_group_ptr)))

/*
 * Remote (receiver) Callbacks.
 */

/**
 * \brief Receive callback for RMA protocol and operations messages
 *
 * "Small" message callback - the entire message is already here.
 * Process it now and return.
 *
 * \param[in] _mi	Pointer to msginfo
 * \param[in] ct	Number of DCQuad's in msginfo
 * \param[in] or	Rank of origin
 * \param[in] sb	Pointer to send buffer (data received)
 * \param[in] sl	Length (bytes) of data
 * \return nothing
 *
 * \ref msginfo_usage
 */
void recv_sm_cb(void *cd, const DCQuad *_mi, unsigned ct, unsigned or,
                        const char *sb, const unsigned sl);

/**
 * \brief Callback for DCMF_Control() messages
 *
 * Simple pass-through to recv_sm_cb() with zero-length data.
 *
 * \param[in] ctl	Control message (one quad)
 * \param[in] or	Origin node lpid
 * \return	nothing
 */
void recv_ctl_cb(void *cd, const DCMF_Control_t *ctl, unsigned or);

/**
 * \brief Callback for Accumulate recv completion
 *
 * "Message receive completion" callback used for MPID_MSGTYPE_ACC
 * to implement the accumulate function. Decodes data from request
 * cache object, frees request, does accumulate, and updates RMA count.
 *
 * Used for "long message" ACCUMULATE.
 *
 * To use this callback, the "xtra" info (DCQuad) must
 * be filled as follows:
 *
 * - \e w0 - ignored
 * - \e w1 - ignored
 * - \e w2 - (int *)multi-struct buffer (int *, DCQuad[], data)
 * - \e w3 - origin lpid
 *
 * \param[in] v Pointer to DCMF request object
 * \return nothing
 *
 * \ref msginfo_usage
 */
void accum_cb(void *v);

/**
 * \brief Receive callback for RMA operations messages
 *
 * "Message receive initiated" callback.
 * This one should never get called for protocol messages.
 * Setup buffers, get a request object, and return so receive can begin.
 * In some cases (e.g. MPID_MSGTYPE_ACC) the processing is done in the
 * receive completion callback, otherwise that callback just frees
 * the request and cleans up (updates counters).
 *
 * \param[in] _mi	Pointer to msginfo
 * \param[in] ct	Number of DCQuad's in msginfo
 * \param[in] or	Rank of origin
 * \param[in] sl	Length (bytes) of sent data
 * \param[out] rl	Length (bytes) of data to receive
 * \param[out] rb	receive buffer
 * \param[out] cb	callback to invoke after receive
 * \return Pointer to DCMF request object to use for receive,
 *	or NULL to discard received data
 *
 * \ref msginfo_usage
 */
DCMF_Request_t *recv_cb(void *cd, const DCQuad *_mi, unsigned ct,
                        unsigned or, const unsigned sl, unsigned *rl,
                        char **rb, DCMF_Callback_t *cb);

void mpidu_init_lock(MPID_Win *win);
void mpidu_free_lock(MPID_Win *win);

/**
 * \brief Lock receive callback.
 *
 * Attempts to acquire the lock.
 * On success, sends ACK to origin.
 * On failure to acquire lock,
 * adds caller to lock wait queue.
 *
 * Does not attempt to acquire lock (counted as failure)
 * if window is currently in some other epoch.
 *
 * \param[in] info	Pointer to msginfo from origin (locker)
 * \param[in] lpid	lpid of origin node (locker)
 * \return nothing
 *
 * \ref msginfo_usage\n
 * \ref lock_design
 */
void lock_cb(const MPIDU_Onesided_ctl_t *info, int lpid);

/**
 * \brief Epoch End callback.
 *
 * Called whenever epoch_type is set to MPID_EPOTYPE_NONE, i.e. an
 * access/exposure epoch ends. Also called when the window lock is
 * released (by the origin node).
 *
 * This is used to prevent locks from being acquired while some other
 * access/exposure epoch is active on a window, and queues the lock
 * attempt until such time as the epoch has ended.
 *
 * \param[in] win	Pointer to MPID_Win whose epoch has ended
 */
void epoch_end_cb(MPID_Win *win);

/**
 * \brief Unlock receive callback.
 *
 * Attempts to release the lock.
 * If the lock cannot be released (due to outstanding RMA ops not
 * yet received) then the unlocker is placed on a queue where its
 * request will be re-evaluated when RMA ops are received.
 * If lock can be released, any lock waiters are woken up in
 * \e epoch_end_cb() and an MPID_MSGTYPE_UNLOCKACK is sent to the unlocker.
 *
 * \param[in] info	Pointer to msginfo from origin (unlocker)
 * \param[in] lpid	lpid of origin node (unlocker)
 * \return nothing
 *
 * \ref msginfo_usage\n
 * \ref lock_design
 */
void unlk_cb(const MPIDU_Onesided_ctl_t *info, int lpid);

/*
 * End of remote callbacks.
 */

#ifdef NOT_USED
/**
 * \brief Send local datatype to target node
 *
 * Routine to send target datatype to target node.
 * These sends are handled by recv callbacks above...
 *
 * \param[in] dt	datatype handle to send
 * \param[in] o_lpid	Origin lpid
 * \param[in] t_lpid	Target lpid
 * \param[out] pending	Pointer to send done counter
 * \param[in,out] consistency	Pointer for consistency used for sends (out)
 * \return MPI_SUCCESS, or error returned by DCMF_Send.
 *
 * \ref msginfo_usage\n
 * \ref dtcache_design
 */
int mpid_queue_datatype(MPI_Datatype dt,
                        int o_lpid, int t_lpid, volatile unsigned *pending,
                        DCMF_Consistency *consistency);
#endif /* NOT_USED */

/*
 * End of utility routines
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 */