btl_mvapi_component.c

来自「MPI stands for the Message Passing Inter」· C语言 代码 · 共 1,098 行 · 第 1/4 页

        OBJ_CONSTRUCT(&mvapi_btl->send_free_eager, ompi_free_list_t);
        OBJ_CONSTRUCT(&mvapi_btl->send_free_max, ompi_free_list_t);
        OBJ_CONSTRUCT(&mvapi_btl->send_free_frag, ompi_free_list_t);
        
        OBJ_CONSTRUCT(&mvapi_btl->recv_free_eager, ompi_free_list_t);
        OBJ_CONSTRUCT(&mvapi_btl->recv_free_max, ompi_free_list_t);
        
        if(mca_btl_mvapi_module_init(mvapi_btl) != OMPI_SUCCESS) {
            free(hca_ids);
            return NULL;
        }
                         
        hca_pd.reg_data = mvapi_btl;
        hca_pd.sizeof_reg = sizeof(mca_btl_mvapi_reg_t);
        hca_pd.register_mem = mvapi_reg_mr;
        hca_pd.deregister_mem = mvapi_dereg_mr;
        /* initialize the memory pool using the hca */ 
        mvapi_btl->super.btl_mpool = 
            mca_mpool_base_module_create(mca_btl_mvapi_component.ib_mpool_name,
                                         &mvapi_btl->super, 
                                         &hca_pd); 
        
        if(NULL == mvapi_btl->super.btl_mpool) { 
            BTL_ERROR(("error creating vapi memory pool! aborting mvapi btl initialization")); 
            return NULL; 
        }
        /* Initialize pool of send fragments */ 
        
        length = sizeof(mca_btl_mvapi_frag_t) + 
            sizeof(mca_btl_mvapi_header_t) + 
            sizeof(mca_btl_mvapi_footer_t) +
            mvapi_btl->super.btl_eager_limit+ 
            2*MCA_BTL_IB_FRAG_ALIGN; 
       
        mvapi_btl->eager_rdma_frag_size =
            length & ~(2 * MCA_BTL_IB_FRAG_ALIGN - 1);

        ompi_free_list_init(&mvapi_btl->send_free_eager,
                            length, 
                            OBJ_CLASS(mca_btl_mvapi_send_frag_eager_t),
                            2*mvapi_btl->rd_num,
                            mca_btl_mvapi_component.ib_free_list_max,
                            mca_btl_mvapi_component.ib_free_list_inc,
                            mvapi_btl->super.btl_mpool);
        
        ompi_free_list_init(&mvapi_btl->recv_free_eager,
                            length, 
                            OBJ_CLASS(mca_btl_mvapi_recv_frag_eager_t),
                            2*mvapi_btl->rd_num,
                            mca_btl_mvapi_component.ib_free_list_max,
                            mca_btl_mvapi_component.ib_free_list_inc,
                            mvapi_btl->super.btl_mpool);
        
        
        
        length = sizeof(mca_btl_mvapi_frag_t) + 
            sizeof(mca_btl_mvapi_header_t) + 
            mvapi_btl->super.btl_max_send_size+
            2*MCA_BTL_IB_FRAG_ALIGN; 
        
        
        ompi_free_list_init(&mvapi_btl->send_free_max,
                            length, 
                            OBJ_CLASS(mca_btl_mvapi_send_frag_max_t),
                            2*mvapi_btl->rd_num,
                            mca_btl_mvapi_component.ib_free_list_max,
                            mca_btl_mvapi_component.ib_free_list_inc,
                            mvapi_btl->super.btl_mpool);
        
        
        
        /* Initialize pool of receive fragments */
        ompi_free_list_init (&mvapi_btl->recv_free_max, 
                             length, 
                             OBJ_CLASS (mca_btl_mvapi_recv_frag_max_t),
                             2*mvapi_btl->rd_num,
                             mca_btl_mvapi_component.ib_free_list_max,
                             mca_btl_mvapi_component.ib_free_list_inc, 
                             mvapi_btl->super.btl_mpool);

        
        
        length = sizeof(mca_btl_mvapi_frag_t) + 
            sizeof(mca_btl_mvapi_header_t)+ 
            2*MCA_BTL_IB_FRAG_ALIGN; 

        
        
        
        ompi_free_list_init(&mvapi_btl->send_free_frag,
                            length, 
                            OBJ_CLASS(mca_btl_mvapi_send_frag_frag_t),
                            mca_btl_mvapi_component.ib_free_list_num,
                            mca_btl_mvapi_component.ib_free_list_max,
                            mca_btl_mvapi_component.ib_free_list_inc,
                            mvapi_btl->super.btl_mpool);
        
        orte_pointer_array_init(&mvapi_btl->eager_rdma_buffers,
                mca_btl_mvapi_component.max_eager_rdma,
                mca_btl_mvapi_component.max_eager_rdma, 
                0);
        mvapi_btl->eager_rdma_buffers_count = 0;
        OBJ_CONSTRUCT(&mvapi_btl->eager_rdma_lock, opal_mutex_t); 

        /* Initialize the rr_desc_post array for posting of rr*/ 
        mvapi_btl->rr_desc_post = (VAPI_rr_desc_t*) malloc(
            ((mca_btl_mvapi_component.rd_num + mca_btl_mvapi_component.rd_rsv) * sizeof(VAPI_rr_desc_t))); 
        btls[i] = &mvapi_btl->super;
    }

    /* Post OOB receive to support dynamic connection setup */
    mca_btl_mvapi_post_recv();
    
    mca_btl_mvapi_modex_send(); 
    *num_btl_modules = mca_btl_mvapi_component.ib_num_btls;
    free(hca_ids);
    return btls;
}

static int mca_btl_mvapi_handle_incoming_hp(mca_btl_mvapi_module_t *,
        mca_btl_mvapi_endpoint_t *,
        mca_btl_mvapi_frag_t *, 
        size_t);
int mca_btl_mvapi_handle_incoming_hp(
        mca_btl_mvapi_module_t *mvapi_btl,
        mca_btl_mvapi_endpoint_t *endpoint,
        mca_btl_mvapi_frag_t *frag, 
        size_t byte_len)
{
    /* advance the segment address past the header and subtract from the length..*/ 
    frag->segment.seg_len =  byte_len-
        ((unsigned char*) frag->segment.seg_addr.pval -
         (unsigned char*) frag->hdr); 

    /* call registered callback */
    mvapi_btl->ib_reg[frag->hdr->tag].cbfunc(&mvapi_btl->super, frag->hdr->tag,
            &frag->base, mvapi_btl->ib_reg[frag->hdr->tag].cbdata);

    if (!MCA_BTL_MVAPI_RDMA_FRAG(frag)) {
        OMPI_FREE_LIST_RETURN(&(mvapi_btl->recv_free_eager),
                (ompi_free_list_item_t*) frag);
    } else {
        mca_btl_mvapi_frag_t *tf;
        OPAL_THREAD_LOCK(&endpoint->eager_rdma_local.lock);
        MCA_BTL_MVAPI_RDMA_MAKE_REMOTE(frag->ftr);
        while (endpoint->eager_rdma_local.tail !=
                endpoint->eager_rdma_local.head) {
             tf = MCA_BTL_MVAPI_GET_LOCAL_RDMA_FRAG(endpoint,
                     endpoint->eager_rdma_local.tail);
             if (MCA_BTL_MVAPI_RDMA_FRAG_LOCAL (tf))
                 break;
             OPAL_THREAD_ADD32(&endpoint->eager_rdma_local.credits, 1);
             MCA_BTL_MVAPI_RDMA_NEXT_INDEX(endpoint->eager_rdma_local.tail);
        }
        OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
    }

    if (!mca_btl_mvapi_component.use_srq) {
        OPAL_THREAD_ADD32(&endpoint->sd_tokens_hp, frag->hdr->credits);
        OPAL_THREAD_ADD32(&endpoint->eager_rdma_remote.tokens,
                frag->hdr->rdma_credits);
    }
    
    if (mca_btl_mvapi_component.use_eager_rdma &&
            !endpoint->eager_rdma_local.base.pval &&
            mvapi_btl->eager_rdma_buffers_count <
            mca_btl_mvapi_component.max_eager_rdma &&
            OPAL_THREAD_ADD32(&endpoint->eager_recv_count, 1) ==
            mca_btl_mvapi_component.eager_rdma_threshold) 
                mca_btl_mvapi_endpoint_connect_eager_rdma(endpoint); 

    /* repost receive descriptors */
#ifdef VAPI_FEATURE_SRQ
    if(mca_btl_mvapi_component.use_srq) {
        OPAL_THREAD_ADD32((int32_t*) &mvapi_btl->srd_posted_hp, -1);
        MCA_BTL_MVAPI_POST_SRR_HIGH(mvapi_btl, 0);
    } else {
#endif
        if (!MCA_BTL_MVAPI_RDMA_FRAG(frag)) {
            OPAL_THREAD_ADD32((int32_t*) &endpoint->rd_posted_hp, -1);
            MCA_BTL_MVAPI_ENDPOINT_POST_RR_HIGH(endpoint, 0);
        }

        /* check to see if we need to progress any pending desciptors */
        if(endpoint->sd_tokens_hp > 0 ||
                endpoint->eager_rdma_remote.tokens > 0) {

            while(!opal_list_is_empty(&endpoint->pending_frags_hp) &&
                    endpoint->sd_wqe_hp > 0 &&
                    (endpoint->sd_tokens_hp > 0 ||
                     endpoint->eager_rdma_remote.tokens > 0)) {
                opal_list_item_t *frag_item;
                OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
                frag_item = opal_list_remove_first(&(endpoint->pending_frags_hp));
                OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
                if(NULL == (frag = (mca_btl_mvapi_frag_t *) frag_item))
                    break;
                if(OMPI_SUCCESS !=
                        mca_btl_mvapi_endpoint_send(frag->endpoint, frag)) {
                    BTL_ERROR(("error in posting pending send\n"));
                    break;
                }
            }
        }

       /* check to see if we need to return credits */
       if((endpoint->rd_credits_hp >= mca_btl_mvapi_component.rd_win ||
                   endpoint->eager_rdma_local.credits >=
                   mca_btl_mvapi_component.rd_win) &&
               OPAL_THREAD_ADD32(&endpoint->sd_credits_hp, 1) == 1) {
           mca_btl_mvapi_endpoint_send_credits_hp(endpoint);
        }
#ifdef VAPI_FEATURE_SRQ
    }
#endif

    return OMPI_SUCCESS;
}

/*
 *  IB component progress.
 */


int mca_btl_mvapi_component_progress( void )
{
    uint32_t i, j, c;
    int count = 0, hret;
    int32_t credits;
    
    mca_btl_mvapi_frag_t* frag;
    mca_btl_mvapi_endpoint_t* endpoint; 
    /* Poll for RDMA completions - if any succeed, we don't process the slower queues */
    for(i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++) {
        mca_btl_mvapi_module_t* mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i];

        OPAL_THREAD_LOCK(&mvapi_btl->eager_rdma_lock);
        c = mvapi_btl->eager_rdma_buffers_count;
        OPAL_THREAD_UNLOCK(&mvapi_btl->eager_rdma_lock);

        for(j = 0; j < c; j++) {
            endpoint = 
                orte_pointer_array_get_item(mvapi_btl->eager_rdma_buffers, j);

            if(!endpoint) /* shouldn't happen */
                continue;

            OPAL_THREAD_LOCK(&endpoint->eager_rdma_local.lock);
            frag = MCA_BTL_MVAPI_GET_LOCAL_RDMA_FRAG (endpoint,
                    endpoint->eager_rdma_local.head);
            
            if (MCA_BTL_MVAPI_RDMA_FRAG_LOCAL (frag)) {
                uint32_t size = MCA_BTL_MVAPI_RDMA_FRAG_GET_SIZE(frag->ftr);
#if OMPI_ENABLE_DEBUG
                if (frag->ftr->seq != endpoint->eager_rdma_local.seq)
                    BTL_ERROR(("Eager RDMA wrong SEQ: received %d expected %d",
                                frag->ftr->seq, 
                                endpoint->eager_rdma_local.seq));
                endpoint->eager_rdma_local.seq++;
#endif
                MCA_BTL_MVAPI_RDMA_NEXT_INDEX(endpoint->eager_rdma_local.head);

                OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
                frag->hdr = (mca_btl_mvapi_header_t*)(((char*)frag->ftr) - 
                        size + sizeof(mca_btl_mvapi_footer_t));
                frag->segment.seg_addr.pval = ((unsigned char* )frag->hdr) + 
                    sizeof(mca_btl_mvapi_header_t);
                
                hret = mca_btl_mvapi_handle_incoming_hp(mvapi_btl,
                        frag->endpoint, frag, 
                        size - sizeof(mca_btl_mvapi_footer_t));
                if (hret != MPI_SUCCESS)
                    return hret;
                count++;
            } else