btl_mvapi_component.c

来自「MPI stands for the Message Passing Inter」· C语言 代码 · 共 1,098 行 · 第 1/4 页

    mca_btl_mvapi_port_info_t *ports = NULL;

    size = mca_btl_mvapi_component.ib_num_btls * sizeof (mca_btl_mvapi_port_info_t);
    if (size != 0) {
        ports = (mca_btl_mvapi_port_info_t *)malloc (size);
        if (NULL == ports) {
            return OMPI_ERR_OUT_OF_RESOURCE;
        }

        for (i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++) {
            mca_btl_mvapi_module_t *btl = &mca_btl_mvapi_component.mvapi_btls[i];
            ports[i] = btl->port_info;
        }
    }
    rc = mca_pml_base_modex_send (&mca_btl_mvapi_component.super.btl_version, ports, size);
    if (NULL != ports) {
        free (ports);
    }
    return rc;
}

/*
 * Callback function on control message.
 */

static void mca_btl_mvapi_control(
    struct mca_btl_base_module_t* btl,
    mca_btl_base_tag_t tag,
    mca_btl_base_descriptor_t* descriptor,
    void* cbdata)
{
    /* dont return credits used for control messages */
    mca_btl_mvapi_frag_t* frag = (mca_btl_mvapi_frag_t*)descriptor; 
    mca_btl_mvapi_endpoint_t* endpoint = frag->endpoint;
    mca_btl_mvapi_control_header_t *ctl_hdr = frag->segment.seg_addr.pval;
    mca_btl_mvapi_eager_rdma_header_t *rdma_hdr;

    if(frag->size == mca_btl_mvapi_component.eager_limit) {
        /* if not sent via rdma */
        if (!MCA_BTL_MVAPI_RDMA_FRAG(frag) &&
                ctl_hdr->type == MCA_BTL_MVAPI_CONTROL_NOOP)
            OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, -1);

    } else {
        OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, -1);
    }

    switch (ctl_hdr->type) {
    case MCA_BTL_MVAPI_CONTROL_NOOP:
        break;
    case MCA_BTL_MVAPI_CONTROL_RDMA:
        rdma_hdr = (mca_btl_mvapi_eager_rdma_header_t*)ctl_hdr;
        if (endpoint->eager_rdma_remote.base.pval) {
           BTL_ERROR(("Got RDMA connect twise!\n"));
           return;
        }
        endpoint->eager_rdma_remote.rkey = rdma_hdr->rkey;
        endpoint->eager_rdma_remote.base.pval = rdma_hdr->rdma_start.pval;
        endpoint->eager_rdma_remote.tokens =
            mca_btl_mvapi_component.eager_rdma_num - 1;
        break;
    default:
        BTL_ERROR(("Unknown message type sent by BTL\n"));
        break;
    }
}

static int mvapi_reg_mr(void *reg_data, void *base, size_t size,
        mca_mpool_base_registration_t *reg)
{
    mca_btl_mvapi_module_t *mvapi_btl = (mca_btl_mvapi_module_t*)reg_data;
    mca_btl_mvapi_reg_t *mvapi_reg = (mca_btl_mvapi_reg_t*)reg;
    VAPI_mrw_t mr_in, mr_out;
    VAPI_ret_t ret;

    memset(&mr_in, 0, sizeof(VAPI_mrw_t));
    memset(&mr_out, 0, sizeof(VAPI_mrw_t));
    mr_in.acl =
        VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE | VAPI_EN_REMOTE_READ;
    mr_in.pd_hndl = mvapi_btl->ptag;
    mr_in.size = size;
    mr_in.start = (VAPI_virt_addr_t)(MT_virt_addr_t)base;
    mr_in.type = VAPI_MR;
    mvapi_reg->hndl = VAPI_INVAL_HNDL;

    ret = VAPI_register_mr(mvapi_btl->nic, &mr_in, &mvapi_reg->hndl, &mr_out);

    if(ret != VAPI_OK) {
        return OMPI_ERR_OUT_OF_RESOURCE;
    }

    mvapi_reg->l_key = mr_out.l_key;
    mvapi_reg->r_key = mr_out.r_key;
    return OMPI_SUCCESS;
}

static int mvapi_dereg_mr(void *reg_data, mca_mpool_base_registration_t *reg)
{
    mca_btl_mvapi_module_t *mvapi_btl = (mca_btl_mvapi_module_t*)reg_data;
    mca_btl_mvapi_reg_t *mvapi_reg = (mca_btl_mvapi_reg_t*)reg;
    VAPI_ret_t ret;

    if(mvapi_reg->hndl != VAPI_INVAL_HNDL) {
        ret = VAPI_deregister_mr(mvapi_btl->nic, mvapi_reg->hndl);
        if(ret != VAPI_OK) {
            opal_output(0, "%s: error unpinning mvapi memory errno says %s\n",
                    __func__, strerror(errno));
            return OMPI_ERROR;
        }
    }
    return OMPI_SUCCESS;
}

/*
 *  IB component initialization:
 *  (1) read interface list from kernel and compare against component parameters
 *      then create a BTL instance for selected interfaces
 *  (2) setup IB listen socket for incoming connection attempts
 *  (3) register BTL parameters with the MCA
 */

mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules, 
                                                  bool enable_progress_threads,
                                                  bool enable_mpi_threads)
{
    VAPI_ret_t vapi_ret;
    VAPI_hca_id_t* hca_ids;
    VAPI_hca_hndl_t hca_hndl; 
    VAPI_hca_vendor_t hca_vendor; 
    VAPI_hca_cap_t hca_cap; 
    VAPI_hca_port_t hca_port; 
    uint32_t num_hcas; 
    mca_btl_base_module_t** btls;
    uint32_t i,j, length;
    struct mca_mpool_base_resources_t hca_pd; 
    opal_list_t btl_list; 
    mca_btl_mvapi_module_t * mvapi_btl; 
    mca_btl_base_selected_module_t* ib_selected; 
    opal_list_item_t* item; 

    /* initialization */
    *num_btl_modules = 0;

    /* mvapi BTL does not currently support progress threads, so
       disable the component if they were requested */
    if (enable_progress_threads) {
        mca_btl_base_error_no_nics("MVAPI", "HCA");
        mca_btl_mvapi_component.ib_num_btls = 0;
        mca_btl_mvapi_modex_send();
        return NULL;
    }

    /* Determine the number of hca's available on the host */
    vapi_ret=EVAPI_list_hcas(0, &num_hcas, NULL);
    if( VAPI_EAGAIN != vapi_ret || 0 == num_hcas ) {
        mca_btl_base_error_no_nics("MVAPI", "HCA");
        mca_btl_mvapi_component.ib_num_btls = 0;
        mca_btl_mvapi_modex_send();
        return NULL;
    }

    /* Allocate space for the hca's */ 
    hca_ids = (VAPI_hca_id_t*) malloc(num_hcas * sizeof(VAPI_hca_id_t));
    if(NULL == hca_ids) {
        ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
        return NULL;
    }

    /* obtain a list of the hca's on this host */ 
    vapi_ret=EVAPI_list_hcas(num_hcas, &num_hcas, hca_ids);
    if( VAPI_OK != vapi_ret ) {
        ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
        return NULL;
    }
        
    /** We must loop through all the hca id's, get there handles and 
        for each hca we query the number of ports on the hca and set up 
        a distinct btl module for each hca port */ 

    OBJ_CONSTRUCT(&btl_list, opal_list_t); 
    OBJ_CONSTRUCT(&mca_btl_mvapi_component.ib_lock, opal_mutex_t);


    for(i = 0; 
        i < num_hcas && mca_btl_mvapi_component.ib_num_btls < mca_btl_mvapi_component.ib_max_btls; 
        i++){  
        vapi_ret = EVAPI_get_hca_hndl(hca_ids[i], &hca_hndl); 
        if(VAPI_OK != vapi_ret) { 
            BTL_ERROR(("error getting hca handle: %s", VAPI_strerror(vapi_ret))); 
            return NULL; 
        } 
        

        vapi_ret = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap); 
         if(VAPI_OK != vapi_ret) { 
            BTL_ERROR(("error getting hca properties %s", VAPI_strerror(vapi_ret))); 
            return NULL; 
        } 
         
         
         /* Note ports are 1 based hence j = 1 */
         for(j = 1; j <= hca_cap.phys_port_num; j++){ 
             vapi_ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t) j, &hca_port);  
             if(VAPI_OK != vapi_ret) { 
                 BTL_ERROR(("error getting hca port properties %s", VAPI_strerror(vapi_ret))); 
                 return NULL; 
             } 
             
             if( PORT_ACTIVE == hca_port.state ){ 
                 
                 mvapi_btl = (mca_btl_mvapi_module_t*) malloc(sizeof(mca_btl_mvapi_module_t)); 
                 memcpy(mvapi_btl, &mca_btl_mvapi_module, sizeof(mca_btl_mvapi_module));
                 
                 ib_selected = OBJ_NEW(mca_btl_base_selected_module_t); 
                 ib_selected->btl_module = (mca_btl_base_module_t*) mvapi_btl; 
                 memcpy(mvapi_btl->hca_id,   hca_ids[i], sizeof(VAPI_hca_id_t)); 
                 mvapi_btl->nic = hca_hndl; 
                 mvapi_btl->port_id = (IB_port_t) j; 
                 mvapi_btl->port = hca_port; 
                 mvapi_btl->port_info.subnet = hca_port.sm_lid;
                 mvapi_btl->ib_reg[MCA_BTL_TAG_BTL].cbfunc = mca_btl_mvapi_control;
                 mvapi_btl->ib_reg[MCA_BTL_TAG_BTL].cbdata = NULL;
                 
                 opal_list_append(&btl_list, (opal_list_item_t*) ib_selected);
                 if(++mca_btl_mvapi_component.ib_num_btls >= mca_btl_mvapi_component.ib_max_btls)
                     break;
            } 
        }
    }
    
    if(0 == mca_btl_mvapi_component.ib_num_btls){ 
        mca_btl_base_error_no_nics("MVAPI", "HCA");
        mca_btl_mvapi_modex_send();
        return NULL; 
    }
    /* Allocate space for btl modules */
    mca_btl_mvapi_component.mvapi_btls = (mca_btl_mvapi_module_t*) malloc(sizeof(mca_btl_mvapi_module_t) * 
                                                                          mca_btl_mvapi_component.ib_num_btls);
    
    if(NULL == mca_btl_mvapi_component.mvapi_btls) {
        ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
        return NULL;
    }
    btls = (struct mca_btl_base_module_t**) 
        malloc(mca_btl_mvapi_component.ib_num_btls * sizeof(struct mca_btl_mvapi_module_t*));
    if(NULL == btls) {
        ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
        return NULL;
    }

    
    
    for(i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++){
                 
        
        item = opal_list_remove_first(&btl_list); 
        ib_selected = (mca_btl_base_selected_module_t*)item; 
        mvapi_btl = (mca_btl_mvapi_module_t*) ib_selected->btl_module; 
        memcpy(&(mca_btl_mvapi_component.mvapi_btls[i]), mvapi_btl , sizeof(mca_btl_mvapi_module_t)); 
        free(ib_selected); 
        free(mvapi_btl); 

        mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i];
        mvapi_btl->rd_num = mca_btl_mvapi_component.rd_num + mca_btl_mvapi_component.rd_rsv;
        mvapi_btl->rd_low = mca_btl_mvapi_component.rd_low;
        mvapi_btl->num_peers = 0; 
        mvapi_btl->sd_tokens_hp = mvapi_btl->sd_tokens_lp = mca_btl_mvapi_component.srq_sd_max;

        /* Initialize module state */

        OBJ_CONSTRUCT(&mvapi_btl->pending_frags_hp, opal_list_t);
        OBJ_CONSTRUCT(&mvapi_btl->pending_frags_lp, opal_list_t);
    

        OBJ_CONSTRUCT(&mvapi_btl->ib_lock, opal_mutex_t);