communicator.h

MPI stands for the Message Passing Interface. Written by the MPI Forum (a large committee comprising

/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2006 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */

#ifndef OMPI_COMMUNICATOR_H
#define OMPI_COMMUNICATOR_H

#include "opal/class/opal_object.h"
#include "ompi/errhandler/errhandler.h"
#include "opal/threads/mutex.h"

#include "mpi.h"
#include "ompi/group/group.h"
#include "ompi/mca/coll/coll.h"
#include "orte/mca/rml/rml_types.h"
#include "ompi/proc/proc.h"

#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif

OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);

#define OMPI_COMM_INTER      0x00000001
#define OMPI_COMM_CART       0x00000002
#define OMPI_COMM_GRAPH      0x00000004
#define OMPI_COMM_NAMEISSET  0x00000008
#define OMPI_COMM_ISFREED    0x00000010
#define OMPI_COMM_INTRINSIC  0x00000020
#define OMPI_COMM_DYNAMIC    0x00000040
#define OMPI_COMM_INVALID    0x00000080
#define OMPI_COMM_PML_ADDED  0x00000100

/* some utility #defines */
#define OMPI_COMM_IS_INTER(comm) ((comm)->c_flags & OMPI_COMM_INTER)
#define OMPI_COMM_IS_INTRA(comm) (!((comm)->c_flags & OMPI_COMM_INTER))
#define OMPI_COMM_IS_CART(comm) ((comm)->c_flags & OMPI_COMM_CART)
#define OMPI_COMM_IS_GRAPH(comm) ((comm)->c_flags & OMPI_COMM_GRAPH)
#define OMPI_COMM_IS_INTRINSIC(comm) ((comm)->c_flags & OMPI_COMM_INTRINSIC)
#define OMPI_COMM_IS_FREED(comm) ((comm)->c_flags & OMPI_COMM_ISFREED)
#define OMPI_COMM_IS_DYNAMIC(comm) ((comm)->c_flags & OMPI_COMM_DYNAMIC)
#define OMPI_COMM_IS_INVALID(comm) ((comm)->c_flags & OMPI_COMM_INVALID)
#define OMPI_COMM_IS_PML_ADDED(comm) ((comm)->c_flags & OMPI_COMM_PML_ADDED)

#define OMPI_COMM_SET_DYNAMIC(comm) ((comm)->c_flags |= OMPI_COMM_DYNAMIC)
#define OMPI_COMM_SET_INVALID(comm) ((comm)->c_flags |= OMPI_COMM_INVALID)

#define OMPI_COMM_SET_PML_ADDED(comm) ((comm)->c_flags |= OMPI_COMM_PML_ADDED)

/* a set of special tags: */

/*  to recognize an MPI_Comm_join in the comm_connect_accept routine. */
#define OMPI_COMM_JOIN_TAG      -32000

#define OMPI_COMM_ALLGATHER_TAG -31078
#define OMPI_COMM_BARRIER_TAG   -31079
#define OMPI_COMM_ALLREDUCE_TAG -31080

/**
 * Modes required for acquiring the new comm-id.
 * The first (INTER/INTRA) indicates whether the
 * input comm was an inter/intra-comm, the second
 * whether the new communicator will be an inter/intra
 * comm
 */
#define OMPI_COMM_CID_INTRA        0x00000020
#define OMPI_COMM_CID_INTER        0x00000040
#define OMPI_COMM_CID_INTRA_BRIDGE 0x00000080
#define OMPI_COMM_CID_INTRA_OOB    0x00000100

OMPI_DECLSPEC extern ompi_pointer_array_t ompi_mpi_communicators;

struct ompi_communicator_t {
    opal_object_t              c_base;
    opal_mutex_t               c_lock; /* mutex for name and potentially
                                          attributes */
    char  c_name[MPI_MAX_OBJECT_NAME];
    uint32_t              c_contextid;
    int                     c_my_rank;
    uint32_t                  c_flags; /* flags, e.g. intercomm,
                                          topology, etc. */

    ompi_group_t        *c_local_group;
    ompi_group_t       *c_remote_group;

    /* Attributes */
    struct opal_hash_table_t       *c_keyhash;

    /**< inscribing cube dimension */
    int c_cube_dim;

    /* Hooks for topo module to hang things */
    mca_base_component_t *c_topo_component;

    const struct mca_topo_base_module_1_0_0_t* c_topo; 
    /**< structure of function pointers */

    struct mca_topo_base_comm_1_0_0_t* c_topo_comm; 
    /**< structure containing basic information about the topology */

    struct mca_topo_base_module_comm_t *c_topo_module;
    /**< module specific data */

    /* index in Fortran <-> C translation array */

    int c_f_to_c_index;

#ifdef OMPI_WANT_PERUSE
    /*
     * Place holder for the PERUSE events.
     */
    struct ompi_peruse_handle_t** c_peruse_handles;
#endif

    /* Error handling.  This field does not have the "c_" prefix so
       that the OMPI_ERRHDL_* macros can find it, regardless of whether
       it's a comm, window, or file. */

    ompi_errhandler_t                  *error_handler;
    ompi_errhandler_type_t             errhandler_type;

    /* Hooks for PML to hang things */
    struct mca_pml_comm_t  *c_pml_comm;

    mca_coll_base_module_1_0_0_t c_coll;
    /**< Selected collective module, saved by value for speed (instead
         of by reference) */

    const mca_coll_base_component_1_0_0_t *c_coll_selected_component;
    /**< Selected coll component */
    const mca_coll_base_module_1_0_0_t *c_coll_selected_module;
    /**< The selected module, but only when the selected module
         is not* the basic module.  Used during comm_unselect(). */
    struct mca_coll_base_comm_t *c_coll_selected_data;
    /**< Allow the selected module to cache data on the communicator */

    const mca_coll_base_module_1_0_0_t *c_coll_basic_module;
    /**< Save the basic module; only necessary when the selected
         module is *not* the basic module, but was supplemented
         with methods from the basic module. */
    struct mca_coll_base_comm_t *c_coll_basic_data;
    /**< Allow the basic module to cache data on the communicator */
};
    typedef struct ompi_communicator_t ompi_communicator_t;
    OMPI_DECLSPEC extern ompi_communicator_t *ompi_mpi_comm_parent;
    OMPI_DECLSPEC extern ompi_communicator_t ompi_mpi_comm_null;


    /**
     * Is this a valid communicator?  This is a complicated question.
     * :-)
     *
     * According to MPI-1:5.2.4 (p137):
     *
     * "The predefined constant MPI_COMM_NULL is the value used for
     * invalid communicator handles."
     *
     * Hence, MPI_COMM_NULL is not valid.  However, MPI-2:4.12.4 (p50)
     * clearly states that the MPI_*_C2F and MPI_*_F2C functions
     * should treat MPI_COMM_NULL as a valid communicator -- it
     * distinctly differentiates between "invalid" handles and
     * "MPI_*_NULL" handles.  Some feel that the MPI-1 definition
     * still holds for all other MPI functions; others feel that the
     * MPi-2 definitions trump the MPI-1 definition.  Regardless of
     * who is right, there is ambiguity here.  So we have left
     * ompi_comm_invalid() as originally coded -- per the MPI-1
     * definition, where MPI_COMM_NULL is an invalid communicator.
     * The MPI_Comm_c2f() function, therefore, calls
     * ompi_comm_invalid() but also explictily checks to see if the
     * handle is MPI_COMM_NULL.
     */
    static inline int ompi_comm_invalid(ompi_communicator_t* comm)
    {
        if ((NULL == comm) || (MPI_COMM_NULL == comm) ||
            (OMPI_COMM_IS_FREED(comm)) || (OMPI_COMM_IS_INVALID(comm)) )
            return true;
        else
            return false;
    }

    /**
     * rank w/in the communicator
     */
    static inline int ompi_comm_rank(ompi_communicator_t* comm)
    {
        return comm->c_my_rank;
    }

    /**
     * size of the communicator
     */
    static inline int ompi_comm_size(ompi_communicator_t* comm)
    {
        return comm->c_local_group->grp_proc_count;
    }

    /**
     * size of the remote group for inter-communicators.
     * returns zero for an intra-communicator
     */
    static inline int ompi_comm_remote_size(ompi_communicator_t* comm)
    {
        return (comm->c_flags & OMPI_COMM_INTER ? comm->c_remote_group->grp_proc_count : 0);
    }

    /* return pointer to communicator associated with context id cid,
     * No error checking is done*/
    static inline ompi_communicator_t *ompi_comm_lookup(uint32_t cid)
    {
        /* array of pointers to communicators, indexed by context ID */
        return (ompi_communicator_t*)ompi_pointer_array_get_item(&ompi_mpi_communicators, cid);
    }

    static inline struct ompi_proc_t* ompi_comm_peer_lookup(ompi_communicator_t* comm, int peer_id)
    {
#if OMPI_ENABLE_DEBUG
        if(peer_id >= comm->c_remote_group->grp_proc_count) {
            opal_output(0, "ompi_comm_lookup_peer: invalid peer index (%d)", peer_id);
            return (struct ompi_proc_t *) NULL;
        }
#endif
        return comm->c_remote_group->grp_proc_pointers[peer_id];
    }

    static inline bool ompi_comm_peer_invalid(ompi_communicator_t* comm, int peer_id)
    {
        if(peer_id < 0 || peer_id >= comm->c_remote_group->grp_proc_count) {
            return true;
        }
        return false;
    }


    /**
     * Initialise MPI_COMM_WORLD and MPI_COMM_SELF
     */
    int ompi_comm_init(void);
    OMPI_DECLSPEC int ompi_comm_link_function(void);