comm.c

MPI stands for the Message Passing Interface. Written by the MPI Forum (a large committee comprising

        return (1);
    }
    
    /* ok, if the keys are the same then we check the original ranks */
    if (a[1] == b[1]) {
        if (a[0] < b[0]) {
            return (-1);
        }
        if (a[0] == b[0]) {
            return (0);
        }
        if (a[0] > b[0]) {
            return (1);
        }
    }
    return ( 0 );
}


/*************************************************************************************
 * Counterpart of MPI_Cart/Graph_create. This will be called from the
 * top level MPI. The condition for INTER communicator is already
 * checked by the time this has been invoked. This function should do
 * somewhat the same things which ompi_comm_create does. It will
 * however select a component for topology and then call the
 * cart_create on that component so that it can re-arrange the proc
 * structure as required (if the reorder flag is true). It will then
 * use this proc structure to create the communicator using
 * ompi_comm_set.
 */
int ompi_topo_create (ompi_communicator_t *old_comm, 
                      int ndims_or_nnodes,
                      int *dims_or_index,
                      int *periods_or_edges,
                      bool reorder,
                      ompi_communicator_t **comm_topo,
                      int cart_or_graph){

    ompi_communicator_t *new_comm;
    int new_rank;
    ompi_proc_t **topo_procs;
    int num_procs;
    int ret;

    /* allocate a new communicator */

    new_comm = ompi_comm_allocate(ompi_comm_size(old_comm), 0);
    if (NULL == new_comm) {
        return MPI_ERR_INTERN;
    }

    /* allocate the data for the common good */

    new_comm->c_topo_comm = (mca_topo_base_comm_t*)malloc(sizeof(mca_topo_base_comm_t));
    if (NULL == new_comm->c_topo_comm) {
        OBJ_RELEASE(new_comm);
        return OMPI_ERR_OUT_OF_RESOURCE;
    }

    /* select the topology component on the communicator */

    if (OMPI_SUCCESS != (ret = mca_topo_base_comm_select (new_comm, NULL))) {
        /* OBJ_RELEASE also frees new_comm->c_topo_comm */
        OBJ_RELEASE(new_comm);
        return ret;
    }

    /* since the topo component has initialised, let us now initialise
     * the topo comm structure */
    new_comm->c_flags |= cart_or_graph;

    new_comm->c_topo_comm->mtc_ndims_or_nnodes = ndims_or_nnodes;
    
    new_comm->c_topo_comm->mtc_dims_or_index = NULL;
    new_comm->c_topo_comm->mtc_periods_or_edges = NULL;
    new_comm->c_topo_comm->mtc_reorder = reorder;

    new_comm->c_topo_comm->mtc_coords = NULL;

    new_comm->c_topo_comm->mtc_dims_or_index = (int *)malloc (sizeof(int) * ndims_or_nnodes);
    if (NULL == new_comm->c_topo_comm->mtc_dims_or_index) {
        ompi_comm_free (&new_comm);
        *comm_topo = new_comm;
        return OMPI_ERROR;
    }
    memcpy (new_comm->c_topo_comm->mtc_dims_or_index,
            dims_or_index, ndims_or_nnodes * sizeof(int));
    
    /* Now the topology component has been selected, let the component
     * re-arrange the proc ranks if need be. This is a down-call into
     * the topo component and does not have anything to do with this
     * level */

    /* first, copy the proc structure from the previous communicator
     * over to the new one. the topology component can then work on
     * this and rearrange it as it deems fit.
     */
    num_procs = old_comm->c_local_group->grp_proc_count;
    topo_procs = (ompi_proc_t **)malloc (num_procs * sizeof(ompi_proc_t *));
    memcpy (topo_procs, 
            old_comm->c_local_group->grp_proc_pointers,
            num_procs * sizeof(ompi_proc_t *));
    new_rank = old_comm->c_local_group->grp_my_rank;

    if (OMPI_COMM_CART == cart_or_graph) {

        /* A cartesian system has been requested. Call the right function */

        /* Note that we fill in the basic information, i.e, copy the
         * information which was provided to us over into the
         * structure. The base component functions are free to change
         * it as they deem fit */

        new_comm->c_topo_comm->mtc_periods_or_edges = (int *)
	    malloc (sizeof(int) * dims_or_index[ndims_or_nnodes - 1]);
        if (NULL == new_comm->c_topo_comm->mtc_periods_or_edges) {
            ompi_comm_free (&new_comm);
            *comm_topo = new_comm;
            return OMPI_ERROR;
        }
        memcpy (new_comm->c_topo_comm->mtc_periods_or_edges,
                periods_or_edges, dims_or_index[ndims_or_nnodes - 1]
		* sizeof(int));

        new_comm->c_topo_comm->mtc_coords = (int *)malloc (sizeof(int) * ndims_or_nnodes);
        if (NULL == new_comm->c_topo_comm->mtc_coords) {
            ompi_comm_free (&new_comm);
            *comm_topo = new_comm;
            return OMPI_ERROR;
        }

        if (OMPI_SUCCESS != 
            (ret = new_comm->c_topo->topo_cart_create (new_comm->c_topo_comm,
                                                       &num_procs,
                                                       topo_procs,
                                                       &new_rank,
                                                       ndims_or_nnodes,
                                                       dims_or_index,
                                                       periods_or_edges,
                                                       reorder))) {
            return ret;
        }

    } else if (OMPI_COMM_GRAPH == cart_or_graph) {

        /* A graph system has been requested. Call the right function */

        /* Note that we fill in the basic information, i.e, copy the
         * information which was provided to us over into the
         * structure. The base component functions are free to change
         * it as they deem fit */

        new_comm->c_topo_comm->mtc_periods_or_edges = (int *)
                malloc (sizeof(int) * dims_or_index[ndims_or_nnodes-1]);
        if (NULL == new_comm->c_topo_comm->mtc_periods_or_edges) {
            ompi_comm_free (&new_comm);
            *comm_topo = new_comm;
            return OMPI_ERROR;
        }
        memcpy (new_comm->c_topo_comm->mtc_periods_or_edges,
                periods_or_edges, dims_or_index[ndims_or_nnodes-1] * sizeof(int));

        if (OMPI_SUCCESS != 
            (ret = new_comm->c_topo->topo_graph_create (new_comm->c_topo_comm,
                                                        &num_procs,
                                                        topo_procs,
                                                        &new_rank,
                                                        ndims_or_nnodes,
                                                        dims_or_index,
                                                        periods_or_edges,
                                                        reorder))) {
            return ret;
        }

    }

    /* Determine context id. It is identical to f_2_c_handle */

    ret = ompi_comm_nextcid ( new_comm,  /* new communicator */
                              old_comm,     /* old comm */
                              NULL,     /* bridge comm */
                              NULL,     /* local leader */
                              NULL,     /* remote_leader */
                              OMPI_COMM_CID_INTRA,   /* mode */
                             -1 );     /* send first, doesn't matter */
    if (OMPI_SUCCESS != ret) {
        /* something wrong happened during setting the communicator */
        ompi_comm_free (&new_comm);
        *comm_topo = new_comm;
        return ret;
    }


    /* Now, the topology component has been selected and the group
     * which has the topology information has been created. All we
     * need to do now is to fill the rest of the information into the
     * communicator. The following steps are not just similar to
     * ompi_comm_set, but are actually the same */

    ret = ompi_comm_fill_rest(new_comm,                /* the communicator */
                              num_procs,               /* local size */
                              topo_procs,              /* process structure */
                              new_rank,                /* rank of the process */
                              old_comm->error_handler); /* error handler */

    if (OMPI_SUCCESS != ret) {
        /* something wrong happened during setting the communicator */
        ompi_comm_free (&new_comm);
        *comm_topo = new_comm;
        return ret;
    }

    ret = ompi_comm_activate ( new_comm,  /* new communicator */
                               old_comm,     /* old comm */
                               NULL,     /* bridge comm */
                               NULL,     /* local leader */
                               NULL,     /* remote_leader */
                               OMPI_COMM_CID_INTRA,   /* mode */
                               -1,       /* send first, doesn't matter */
                               NULL );   /* coll component */

    if (OMPI_SUCCESS != ret) {
        /* something wrong happened during setting the communicator */
        ompi_comm_free (&new_comm);
        *comm_topo = new_comm;
        return ret;
    }

    
    /* finally, set the communicator to comm_cart */

    /* if the returned rank is -1, then this process is not in the 
     * new topology, so free everything we have allocated and return */
    if (MPI_UNDEFINED == new_rank) {
        ompi_comm_free (&new_comm);
        *comm_topo = new_comm;
    } else {
        *comm_topo = new_comm;
    }

    return OMPI_SUCCESS;
}

static int ompi_comm_fill_rest (ompi_communicator_t *comm,
                                int num_procs,
                                ompi_proc_t **proc_pointers,
                                int my_rank,
                                ompi_errhandler_t *errh )
{
    int ret;
    
    /* properly decrement the ref counts on the groups.
       We are doing this because this function is sort of a redo 
       of what is done in comm.c.. No need to decrement the ref 
       count on the proc pointers 
       This is just a quick fix, and will be looking for a 
       better solution */
    OBJ_RELEASE ( comm->c_local_group );
    OBJ_RELEASE ( comm->c_local_group );

    /* allocate a group structure for the new communicator */
    comm->c_local_group = ompi_group_allocate(num_procs);
    
    /* free the malloced  proc pointers */
    free(comm->c_local_group->grp_proc_pointers);
    
    /* set the group information */
    comm->c_local_group->grp_proc_pointers = proc_pointers;

    /* set the remote group to be the same as local group */
    comm->c_remote_group = comm->c_local_group;
    OBJ_RETAIN ( comm->c_remote_group );

    /* retain these proc pointers */
    ompi_group_increment_proc_count(comm->c_local_group);
        
    /* set the rank information */
    comm->c_local_group->grp_my_rank = my_rank;
    comm->c_my_rank = my_rank;

    /* verify whether to set the flag, that this comm
       contains process from more than one jobid. */
    ompi_comm_mark_dyncomm (comm);

    /* set the error handler */
    comm->error_handler = errh;
    OBJ_RETAIN (comm->error_handler);

    /* set name for debugging purposes */
    /* there is no cid at this stage ... make this right and make edgars
     * code call this function and remove dupli cde
     */  
    snprintf (comm->c_name, MPI_MAX_OBJECT_NAME, "MPI_COMMUNICATOR %d",
              comm->c_contextid);

    /* determine the cube dimensions */
    comm->c_cube_dim = opal_cube_dim(comm->c_local_group->grp_proc_count);

    /* initialize PML stuff on the communicator */
    if (OMPI_SUCCESS != (ret = MCA_PML_CALL(add_comm(comm)))) {
        /* some error has happened */
        return ret;
    }
    OMPI_COMM_SET_PML_ADDED(comm);

    return OMPI_SUCCESS;
}

static int ompi_comm_copy_topo (ompi_communicator_t *oldcomm, 
                                 ompi_communicator_t *newcomm) {


    int index = 
    
    (oldcomm->c_topo_comm->mtc_dims_or_index[oldcomm->c_topo_comm->mtc_ndims_or_nnodes-1] > 0)? 
    oldcomm->c_topo_comm->mtc_dims_or_index[oldcomm->c_topo_comm->mtc_ndims_or_nnodes-1] : 
    -oldcomm->c_topo_comm->mtc_dims_or_index[oldcomm->c_topo_comm->mtc_ndims_or_nnodes-1]; 
                
    /* pointers for the rest of the information have been set up .... simply
       allocate enough space and copy all the information from the previous one */

    newcomm->c_topo_comm->mtc_ndims_or_nnodes = oldcomm->c_topo_comm->mtc_ndims_or_nnodes;
    newcomm->c_topo_comm->mtc_reorder = oldcomm->c_topo_comm->mtc_reorder;

    newcomm->c_topo_comm->mtc_dims_or_index = (int *)malloc(sizeof(int)*
                                                     newcomm->c_topo_comm->mtc_ndims_or_nnodes);   

    if (NULL == newcomm->c_topo_comm->mtc_dims_or_index) {
        return OMPI_ERROR;
    }

    memcpy (newcomm->c_topo_comm->mtc_dims_or_index, 
            oldcomm->c_topo_comm->mtc_dims_or_index ,
            newcomm->c_topo_comm->mtc_ndims_or_nnodes * sizeof(int));

    newcomm->c_topo_comm->mtc_periods_or_edges = (int *)
                 malloc (sizeof(int)*index);
    if (NULL == newcomm->c_topo_comm->mtc_periods_or_edges) {
        return OMPI_ERROR;
    }

    memcpy (newcomm->c_topo_comm->mtc_periods_or_edges, 
            oldcomm->c_topo_comm->mtc_periods_or_edges, 
            sizeof(int) * index );

    if (OMPI_COMM_IS_CART(oldcomm)) {
        newcomm->c_topo_comm->mtc_coords = (int *)malloc(sizeof(int)*
                                                  newcomm->c_topo_comm->mtc_ndims_or_nnodes);   

        if (NULL == newcomm->c_topo_comm->mtc_coords) {
            return OMPI_ERROR;
        }

        memcpy (newcomm->c_topo_comm->mtc_coords,
                oldcomm->c_topo_comm->mtc_coords,
                sizeof(int) * newcomm->c_topo_comm->mtc_ndims_or_nnodes);
    } else {
        newcomm->c_topo_comm->mtc_coords = NULL;
    }

    return OMPI_SUCCESS;
}