btl_mvapi_component.c

来自「MPI stands for the Message Passing Inter」· C语言 代码 · 共 1,098 行 · 第 1/4 页

/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2005 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */


#include "ompi_config.h"

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_MALLOC_H
#include <malloc.h>
#endif

#include "ompi/constants.h"
#include "opal/event/event.h"
#include "opal/util/if.h"
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "ompi/mca/pml/pml.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/proc/proc.h"
#include "opal/util/show_help.h"
#include "opal/mca/base/mca_base_param.h"
#include "orte/mca/errmgr/errmgr.h"
#include "ompi/mca/mpool/base/base.h" 
#include "btl_mvapi.h"
#include "btl_mvapi_frag.h"
#include "btl_mvapi_endpoint.h" 
#include "btl_mvapi_eager_rdma.h"
#include "btl_mvapi_proc.h"
#include "ompi/mca/btl/base/base.h" 
#include <vapi.h> 
#include <vapi_common.h> 
#include "ompi/datatype/convertor.h" 
#include "ompi/mca/mpool/rdma/mpool_rdma.h"
#include "btl_mvapi_endpoint.h"
#include "ompi/mca/pml/base/pml_base_module_exchange.h"

static int mvapi_reg_mr(void *reg_data, void *base, size_t size,
        mca_mpool_base_registration_t *reg);
static int mvapi_dereg_mr(void *reg_data, mca_mpool_base_registration_t *reg);

mca_btl_mvapi_component_t mca_btl_mvapi_component = {
    {
        /* First, the mca_base_component_t struct containing meta information
           about the component itself */
        
        {
            /* Indicate that we are a pml v1.0.0 component (which also implies a
               specific MCA version) */

            MCA_BTL_BASE_VERSION_1_0_1,

            "mvapi", /* MCA component name */
            OMPI_MAJOR_VERSION,  /* MCA component major version */
            OMPI_MINOR_VERSION,  /* MCA component minor version */
            OMPI_RELEASE_VERSION,  /* MCA component release version */
            mca_btl_mvapi_component_open,  /* component open */
            mca_btl_mvapi_component_close  /* component close */
        },

        /* Next the MCA v1.0.0 component meta data */

        {
            /* Whether the component is checkpointable or not */

            false
        },

        mca_btl_mvapi_component_init,  
        mca_btl_mvapi_component_progress,
    }
};


/*
 * utility routines for parameter registration
 */

static inline void mca_btl_mvapi_param_register_string(
                                                       const char* param_name, 
                                                       const char* param_desc,
                                                       const char* default_value, 
                                                       char** out_value)
{
    mca_base_param_reg_string(&mca_btl_mvapi_component.super.btl_version, 
                              param_name, 
                              param_desc, 
                              false, 
                              false, 
                              default_value, 
                              out_value);
}

static inline void  mca_btl_mvapi_param_register_int(
                                                     const char* param_name,
                                                     const char* param_desc,
                                                     int default_value, 
                                                     int* out_value)
{
    mca_base_param_reg_int(&mca_btl_mvapi_component.super.btl_version, 
                           param_name, 
                           param_desc, 
                           false, 
                           false, 
                           default_value, 
                           out_value); 
}

/*
 *  Called by MCA framework to open the component, registers
 *  component parameters.
 */

int mca_btl_mvapi_component_open(void)
{
    char *msg;
    int tmp_int;

    /* initialize state */
    mca_btl_mvapi_component.ib_num_btls=0;
    mca_btl_mvapi_component.mvapi_btls=NULL;
    
    /* initialize objects */ 
    OBJ_CONSTRUCT(&mca_btl_mvapi_component.ib_procs, opal_list_t);

    /* register IB component parameters */
    mca_btl_mvapi_param_register_int ("max_btls", "maximum number of HCAs/ports to use", 
                                      4, &tmp_int);
    mca_btl_mvapi_component.ib_max_btls = tmp_int;
    mca_btl_mvapi_param_register_int ("free_list_num", "initial size of free lists", 
                                       8, &mca_btl_mvapi_component.ib_free_list_num);
    mca_btl_mvapi_param_register_int ("free_list_max", "maximum size of free lists",
                                       -1, &mca_btl_mvapi_component.ib_free_list_max);
    mca_btl_mvapi_param_register_int ("free_list_inc", "increment size of free lists", 
                                       32, &mca_btl_mvapi_component.ib_free_list_inc);
    mca_btl_mvapi_param_register_string("mpool", "name of the memory pool to be used", 
                                         "rdma", &mca_btl_mvapi_component.ib_mpool_name);
    mca_btl_mvapi_param_register_int("reg_mru_len",  "length of the registration cache most recently used list", 
                                      16, (int*) &mca_btl_mvapi_component.reg_mru_len); 
#ifdef VAPI_FEATURE_SRQ 
    mca_btl_mvapi_param_register_int("use_srq", "if 1 use the IB shared receive queue to post receive descriptors", 
                                     0, (int*) &mca_btl_mvapi_component.use_srq); 
#else
    mca_btl_mvapi_component.use_srq = 0;
#endif
    mca_btl_mvapi_param_register_int("ib_cq_size", "size of the IB completion queue",
                                     10000, (int*) &mca_btl_mvapi_component.ib_cq_size); 
    mca_btl_mvapi_param_register_int("ib_sg_list_size", "size of IB segment list", 
                                     1, (int*) &mca_btl_mvapi_component.ib_sg_list_size); 
    mca_btl_mvapi_param_register_int("ib_pkey_ix", "IB pkey index", 
                                      0, (int*) &mca_btl_mvapi_component.ib_pkey_ix); 
    mca_btl_mvapi_param_register_int("ib_psn", "IB Packet sequence starting number", 
                                      0, (int*) &mca_btl_mvapi_component.ib_psn); 
    mca_btl_mvapi_param_register_int("ib_qp_ous_rd_atom", "IB outstanding atomic reads", 
                                     4, (int*) &mca_btl_mvapi_component.ib_qp_ous_rd_atom); 
    tmp_int = MTU1024;
    asprintf(&msg, "IB MTU, in bytes.  Valid values are: %d=256 bytes, %d=512 bytes, %d=1024 bytes, %d=2048 bytes, %d=4096 bytes.",
             MTU256,
             MTU512,
             MTU1024,
             MTU2048,
             MTU4096);
    mca_btl_mvapi_param_register_int("ib_mtu", msg,
                                     tmp_int, (int*) &mca_btl_mvapi_component.ib_mtu); 
    free(msg);
       
    mca_btl_mvapi_param_register_int("ib_min_rnr_timer", "IB min rnr timer", 
                                      5, (int*) &mca_btl_mvapi_component.ib_min_rnr_timer);
    mca_btl_mvapi_param_register_int("ib_timeout", "IB transmit timeout", 
                                      10, (int*) &mca_btl_mvapi_component.ib_timeout); 
    mca_btl_mvapi_param_register_int("ib_retry_count", "IB transmit retry count",
                                      7, (int*) &mca_btl_mvapi_component.ib_retry_count); 
    mca_btl_mvapi_param_register_int("ib_rnr_retry", "IB rnr retry", 
                                     7, (int*) &mca_btl_mvapi_component.ib_rnr_retry); 
    mca_btl_mvapi_param_register_int("ib_max_rdma_dst_ops", "IB max rdma destination operations", 
                                      4, (int*) &mca_btl_mvapi_component.ib_max_rdma_dst_ops); 
    mca_btl_mvapi_param_register_int("ib_service_level", "IB service level", 
                                      0, (int*) &mca_btl_mvapi_component.ib_service_level); 
    mca_btl_mvapi_param_register_int("ib_static_rate", "IB static rate", 
                                      0, (int*) &mca_btl_mvapi_component.ib_static_rate); 
    mca_btl_mvapi_param_register_int("ib_src_path_bits", "IB source path bits", 
                                      0, (int*) &mca_btl_mvapi_component.ib_src_path_bits); 
        
    mca_btl_mvapi_param_register_int("rd_num", "number of receive descriptors to post to a QP", 
                                      8, (int*) &mca_btl_mvapi_component.rd_num);  
    mca_btl_mvapi_param_register_int("rd_low", "low water mark before reposting occurs", 
                                      6,  (int*) &mca_btl_mvapi_component.rd_low); 
    mca_btl_mvapi_param_register_int("rd_win", "window size at which generate explicity credit message", 
                                      4,  (int*) &mca_btl_mvapi_component.rd_win); 
    mca_btl_mvapi_component.rd_rsv = ((mca_btl_mvapi_component.rd_num<<1)-1) / mca_btl_mvapi_component.rd_win;

    mca_btl_mvapi_param_register_int("srq_rd_max", "Maximum number of receive descriptors posted per SRQ.\n",
                                      1000, (int*) &mca_btl_mvapi_component.srq_rd_max); 
    mca_btl_mvapi_param_register_int("srq_rd_per_peer", "receive descriptors posted per peer, SRQ mode only", 
                                      16, (int*) &mca_btl_mvapi_component.srq_rd_per_peer); 
    mca_btl_mvapi_param_register_int("srq_sd_max", "Maximum number of send descriptors posted per process", 
                                      8,  &mca_btl_mvapi_component.srq_sd_max); 

    mca_btl_mvapi_param_register_int("use_eager_rdma", "user RDMA for eager messages", 
            1, (int*) &mca_btl_mvapi_component.use_eager_rdma);
    if (mca_btl_mvapi_component.use_srq)
        mca_btl_mvapi_component.use_eager_rdma = 0;
    mca_btl_mvapi_param_register_int("eager_rdma_threshold", "Open rdma channel for eager messages after this number of messages received from peer (zero to disable)",
            16, (int*)&mca_btl_mvapi_component.eager_rdma_threshold);
    mca_btl_mvapi_param_register_int("max_eager_rdma", "Maximum number of eager RDMA connections",
            16, (int*)&mca_btl_mvapi_component.max_eager_rdma);
    mca_btl_mvapi_param_register_int("eager_rdma_num", "Number of RDMA buffers for eager messages",
            16, (int*)&mca_btl_mvapi_component.eager_rdma_num);
    mca_btl_mvapi_component.eager_rdma_num+=1;
    mca_btl_mvapi_param_register_int ("exclusivity", "BTL exclusivity", 
                                      MCA_BTL_EXCLUSIVITY_DEFAULT, (int*) &mca_btl_mvapi_module.super.btl_exclusivity);
    mca_btl_mvapi_param_register_int ("eager_limit", "eager send limit", 
                                      (12*1024), &tmp_int); 
    mca_btl_mvapi_module.super.btl_eager_limit = tmp_int;  
    mca_btl_mvapi_param_register_int ("min_send_size", "minimum send size", 
                                      (32*1024), &tmp_int);
    mca_btl_mvapi_module.super.btl_min_send_size = tmp_int;
    mca_btl_mvapi_param_register_int ("max_send_size", "maximum send size", 
                                      (64*1024), &tmp_int);
    mca_btl_mvapi_module.super.btl_max_send_size = tmp_int; 
    mca_btl_mvapi_param_register_int("min_rdma_size", "minimum rdma size", 
                                     1024*1024, &tmp_int);
    mca_btl_mvapi_module.super.btl_min_rdma_size = tmp_int;
    mca_btl_mvapi_param_register_int("max_rdma_size", "maximium rdma size", 
                                     1024*1024, &tmp_int);
    mca_btl_mvapi_module.super.btl_max_rdma_size = tmp_int; 
    mca_btl_mvapi_param_register_int("flags", "BTL flags, SEND=1, PUT=2, GET=4", 
                                     MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_GET, (int*) &mca_btl_mvapi_module.super.btl_flags); 
    mca_btl_mvapi_param_register_int("bandwidth", "Approximate maximum bandwidth of interconnect", 
                                      800, (int*) &mca_btl_mvapi_module.super.btl_bandwidth); 
    
    mca_btl_mvapi_component.max_send_size = mca_btl_mvapi_module.super.btl_max_send_size; 
    mca_btl_mvapi_component.eager_limit = mca_btl_mvapi_module.super.btl_eager_limit; 
    
    return OMPI_SUCCESS;
}

/*
 * component cleanup - sanity checking of queue lengths
 */

int mca_btl_mvapi_component_close(void)
{
    return OMPI_SUCCESS;
}



/*
 *  Register MVAPI port information. The MCA framework
 *  will make this available to all peers.
 */

static int
mca_btl_mvapi_modex_send(void)
{
    int         rc;
    size_t      i;
    size_t      size;