bml.h

来自「MPI stands for the Message Passing Inter」· C头文件 代码 · 共 653 行 · 第 1/2 页

/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2005 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2006 The Regents of the University of California.
 *                         All rights reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */
/**
 * @file
 *
 * BML Management Layer (BML)
 *
 */

#ifndef MCA_BML_H
#define MCA_BML_H

#include "opal/mca/mca.h"
#include "ompi/mca/btl/btl.h"

#include "ompi/mca/bml/base/bml_base_btl.h"
#include "ompi/mca/bml/base/bml_base_endpoint.h" 

#include "ompi/types.h"
#include "ompi/class/ompi_free_list.h"

#define OMPI_ENABLE_DEBUG_RELIABILITY 0

/*
 * BML types
 */

struct ompi_proc_t; 
struct mca_bml_base_module_t;
struct mca_bml_base_endpoint_t;
struct mca_mpool_base_resources_t;

/*
 * Cached set of information for each btl 
 */

struct mca_bml_base_btl_t {
    int    btl_index;                             /**< index in endpoint array */
    int    btl_flags;                             /**< support for put/get? */
    double btl_weight;                            /**< BTL weight for scheduling */
    size_t btl_eager_limit;                       /**< BTL eager limit */
    size_t btl_min_send_size;                     /**< BTL min send size */
    size_t btl_max_send_size;                     /**< BTL max send size */
    size_t btl_min_rdma_size;                     /**< BTL min rdma size */
    size_t btl_max_rdma_size;                     /**< BTL max rdma size */
    struct mca_btl_base_module_t *btl;            /**< BTL module */
    struct mca_btl_base_endpoint_t* btl_endpoint; /**< BTL addressing info */
    struct mca_btl_base_descriptor_t* btl_cache;
        
    /* BTL function table */
    mca_btl_base_module_alloc_fn_t   btl_alloc;
    mca_btl_base_module_free_fn_t    btl_free;
    mca_btl_base_module_send_fn_t    btl_send;
    mca_btl_base_module_prepare_fn_t btl_prepare_src;
    mca_btl_base_module_prepare_fn_t btl_prepare_dst;
    mca_btl_base_module_put_fn_t     btl_put;
    mca_btl_base_module_get_fn_t     btl_get;
    mca_btl_base_component_progress_fn_t btl_progress; 
    
    mca_mpool_base_module_t*         btl_mpool; 
    
};
typedef struct mca_bml_base_btl_t mca_bml_base_btl_t;



/**
 * A dynamically growable array of mca_bml_base_btl_t instances.
 * Maintains an index into the array that is used for round-robin
 * scheduling across contents.
 */
struct mca_bml_base_btl_array_t {
    opal_object_t super;
    size_t arr_size;                     /**< number available */
    size_t arr_reserve;                  /**< size of allocated btl_proc array */
    size_t arr_index;                    /**< last used index*/
    mca_bml_base_btl_t* bml_btls;   /**< array of  bml btl's */
};
typedef struct mca_bml_base_btl_array_t mca_bml_base_btl_array_t;

OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_bml_base_btl_array_t);


/**
 * If required, reallocate (grow) the array to the indicate size.
 * 
 * @param array (IN)
 * @param size (IN)
 */
int mca_bml_base_btl_array_reserve(mca_bml_base_btl_array_t*, size_t);

static inline size_t mca_bml_base_btl_array_get_size(mca_bml_base_btl_array_t* array)
{
    return array->arr_size;
}

/**
 * Grow the array if required, and set the size.
 * 
 * @param array (IN)
 * @param size (IN)
 */
static inline void mca_bml_base_btl_array_set_size(mca_bml_base_btl_array_t* array, size_t size)
{
    if(array->arr_size > array->arr_reserve)
        mca_bml_base_btl_array_reserve(array, size);
    array->arr_size = size;
}

/**
 * Grow the array size by one and return the item at that index.
 * 
 * @param array (IN)
 */
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_insert(mca_bml_base_btl_array_t* array)
{
#if OMPI_ENABLE_DEBUG
    if(array->arr_size >= array->arr_reserve) {
        opal_output(0, "mca_bml_base_btl_array_insert: invalid array index %d >= %d", 
            array->arr_size, array->arr_reserve);
        return 0;
    }
#endif
    return &array->bml_btls[array->arr_size++];
}

/** 
 * Remove a btl from a bml_btl 
 *
 * @param array (IN)
 * @param btl (IN)
 */
static inline bool mca_bml_base_btl_array_remove( mca_bml_base_btl_array_t* array, 
                                                  struct mca_btl_base_module_t* btl )
{ 
    size_t i = 0;
    /* find the btl */
    for( i = 0; i < array->arr_size; i++ ) {
        if( array->bml_btls[i].btl == btl ) {
            for( ; i < array->arr_size; i++ ) {
                /* move all btl's back by 1, so the found 
                   btl is "removed" */
                array->bml_btls[i] = array->bml_btls[(i+1)];
            }
            array->arr_size--;
            array->arr_index = 0;
            return true;
        }
    }
    return false;
}


/**
 * Return an array item at the specified index.
 * 
 * @param array (IN)
 * @param index (IN)
 */
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_get_index(mca_bml_base_btl_array_t* array, size_t index)
{
#if OMPI_ENABLE_DEBUG
    if(index >= array->arr_size) {
        opal_output(0, "mca_bml_base_btl_array_get_index: invalid array index %d >= %d",
            index, array->arr_size);
        return 0;
    }
#endif
    return &array->bml_btls[index];
}

/**
 * Return the next LRU index in the array.
 * 
 * @param array (IN)
 * @param index (IN)
 */
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_get_next(mca_bml_base_btl_array_t* array)
{
#if OMPI_ENABLE_DEBUG
    if(array->arr_size == 0) {
        opal_output(0, "mca_bml_base_btl_array_get_next: invalid array size");
        return 0;
    }
#endif
    if( 1 == array->arr_size ) {
        return &array->bml_btls[0];  /* force the return to avoid a jump */
    } else {
        uint32_t current_position = array->arr_index;  /* force to always start from zero */
        if( (current_position + 1) == array->arr_size ) {
            array->arr_index = 0;  /* next time serve from the beginning */
        } else {
            array->arr_index = current_position + 1;  /* continue */
        }
        return &array->bml_btls[current_position];
    }
}

/**
 * Locate an element in the array
 * 
 * @param array (IN)
 * @param index (IN)
 */
static inline mca_bml_base_btl_t* mca_bml_base_btl_array_find(
    mca_bml_base_btl_array_t* array, struct mca_btl_base_module_t* btl)
{
    size_t i=0;   
    for(i=0; i<array->arr_size; i++) {
        if(array->bml_btls[i].btl == btl) {
            return &array->bml_btls[i];
        }
    }
    return NULL;
}

/**
 *  Structure associated w/ ompi_proc_t that contains the set
 *  of BTLs used to reach a destination
 */
struct mca_bml_base_endpoint_t {
    opal_list_item_t         super;             /**< base_endpoint is a list item */
    struct ompi_proc_t*      btl_proc;          /**< backpointer to target ompi_proc_t */
    size_t                   btl_rdma_offset;   /**< max of min rdma size for available rmda btls */
    size_t                   btl_max_send_size; /**< min of max send size for available send btls */
    size_t                   btl_rdma_align;    /**< max of min rdma size for available rmda btls */
    mca_bml_base_btl_array_t btl_eager;         /**< array of btls to use for first fragments */
    mca_bml_base_btl_array_t btl_send;          /**< array of btls to use for remaining fragments */
    size_t                   bml_max_send_length;
    size_t                   bml_max_rdma_length;
    mca_bml_base_btl_array_t btl_rdma;          /**< array of btls that support (prefer) rdma */
    size_t btl_rdma_index;                      /**< index of last used BTL for RDMA */
    uint32_t btl_flags_or;                      /**< the bitwise OR of the btl flags */
    uint32_t btl_flags_and;                     /**< the bitwise AND of the btl flags */
};
typedef struct mca_bml_base_endpoint_t mca_bml_base_endpoint_t;
                              
    
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_bml_base_endpoint_t);

static inline void mca_bml_base_alloc(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t** des, size_t size) { 
    *des = bml_btl->btl_alloc(bml_btl->btl, size);
}

static inline void mca_bml_base_free(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t* des) { 
    bml_btl->btl_free( bml_btl->btl, des );   
    /* The previous function is supposed to release the des object
     * so we should not touch it anymore.
     */
}

#if OMPI_ENABLE_DEBUG_RELIABILITY

int mca_bml_base_send(
    mca_bml_base_btl_t* bml_btl, 
    mca_btl_base_descriptor_t* des, 
    mca_btl_base_tag_t tag); 


#else

static inline int mca_bml_base_send(
    mca_bml_base_btl_t* bml_btl, 
    mca_btl_base_descriptor_t* des, 
    mca_btl_base_tag_t tag) 
{ 
    des->des_context = (void*) bml_btl; 
    return bml_btl->btl_send(
                             bml_btl->btl,
                             bml_btl->btl_endpoint, 
                             des,
                             tag);
}

#endif

static inline int mca_bml_base_put(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t* des) { 
    des->des_context = (void*) bml_btl; 
    return bml_btl->btl_put(
                             bml_btl->btl,
                             bml_btl->btl_endpoint, 
                             des);
}

static inline int mca_bml_base_get(mca_bml_base_btl_t* bml_btl, mca_btl_base_descriptor_t* des) { 
    des->des_context = (void*) bml_btl; 
    return bml_btl->btl_get(
                             bml_btl->btl,
                             bml_btl->btl_endpoint, 
                             des);
}


static inline void mca_bml_base_prepare_src(mca_bml_base_btl_t* bml_btl, 
                                            mca_mpool_base_registration_t* reg, 
                                            struct ompi_convertor_t* conv, 
                                            size_t reserve, 
                                            size_t *size, 
                                            mca_btl_base_descriptor_t** des) { 
    *des = bml_btl->btl_prepare_src(
                                   bml_btl->btl, 
                                   bml_btl->btl_endpoint, 
                                   reg, 
                                   conv,
                                   reserve,
                                   size
                                   );
    if((*des) != NULL) { 
        (*des)->des_context = (void*) bml_btl;
    }
}