ibu_regmem.vapi.c

mpi并行计算的c++代码 可用vc或gcc编译通过 可以用来搭建并行计算试验环境

#else

#ifdef TRACK_MEMORY_REGISTRATION
    add_mem(buf);
#endif
 start_over:
    iter = ibu_pin_cache;
    while (iter != NULL)
    {
	if (buf == iter->buf && len == iter->len)
	{
	    iter->ref_count++;
	    *state = iter->mem;
	    return IBU_SUCCESS;
	}
	if ((buf >= iter->buf) && ((char*)buf + len < ((char*)iter->buf + iter->len)))
	{
	    iter->ref_count++;
	    *state = iter->mem;
	    return IBU_SUCCESS;
	}
	if (buf < iter->buf)
	{
	    if (iter->left == NULL)
	    {
		if (g_total_mem_pinned > IBU_MAX_PINNED && EVENODD(g_onoff))
		{
		    ibu_clean_pin_cache();
		    goto start_over;
		}

		iter->left = (ibu_mem_node_t*)MPIU_Malloc(sizeof(ibu_mem_node_t));
		iter = iter->left;
		iter->left = NULL;
		iter->right = NULL;
		iter->buf = buf;
		iter->len = len;
		iter->ref_count = 1;
		break;
	    }
	    iter = iter->left;
	}
	else
	{
	    if (iter->right == NULL)
	    {
		if (g_total_mem_pinned > IBU_MAX_PINNED && EVENODD(g_onoff))
		{
		    ibu_clean_pin_cache();
		    goto start_over;
		}

		iter->right = (ibu_mem_node_t*)MPIU_Malloc(sizeof(ibu_mem_node_t));
		iter = iter->right;
		iter->left = NULL;
		iter->right = NULL;
		iter->buf = buf;
		iter->len = len;
		iter->ref_count = 1;
		break;
	    }
	    iter = iter->right;
	}
    }
    if (iter == NULL)
    {
	iter = (ibu_mem_node_t*)MPIU_Malloc(sizeof(ibu_mem_node_t));
	iter->buf = buf;
	iter->len = len;
	iter->left = NULL;
	iter->right = NULL;
	iter->ref_count = 1;
	ibu_pin_cache = iter;
    }
    memset(&mem, 0, sizeof(VAPI_mrw_t));
    memset(&mem_out, 0, sizeof(VAPI_mrw_t));
    mem.type = VAPI_MR;
    mem.start = (VAPI_virt_addr_t)(MT_virt_addr_t)buf;
    mem.size = len;
    mem.pd_hndl = IBU_Process.pd_handle;
    mem.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE | VAPI_EN_REMOTE_READ;
    mem.l_key = 0;
    mem.r_key = 0;
    MPIDI_FUNC_ENTER(MPID_STATE_VAPI_REGISTER_MR);
    status = VAPI_register_mr(
	IBU_Process.hca_handle,
	&mem,
	&state->handle,
	&mem_out);
    MPIDI_FUNC_EXIT(MPID_STATE_VAPI_REGISTER_MR);
    if (status != IBU_SUCCESS)
    {
	MPIU_Internal_error_printf("ibu_register_memory: VAPI_register_mr failed, error %s\n", VAPI_strerror(status));
	MPIDI_FUNC_EXIT(MPID_STATE_IBU_REGISTER_MEMORY);
	return IBU_FAIL;
    }
    state->lkey = mem_out.l_key;
    state->rkey = mem_out.r_key;

    iter->mem.handle = state->handle;
    iter->mem.lkey = mem_out.l_key;
    iter->mem.rkey = mem_out.r_key;
    g_total_mem_pinned += len;

    MPIU_DBG_PRINTF(("exiting ibu_register_memory\n"));
    MPIDI_FUNC_EXIT(MPID_STATE_IBU_REGISTER_MEMORY);
    return IBU_SUCCESS;

#endif
}

#undef FUNCNAME
#define FUNCNAME ibu_deregister_memory
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int ibu_deregister_memory(void *buf, int len, ibu_mem_t *state)
{
#ifdef USE_NO_PIN_CACHE
    VAPI_ret_t status;
#else
    ibu_mem_node_t *iter;
#endif
    MPIDI_STATE_DECL(MPID_STATE_IBU_DEREGISTER_MEMORY);
    MPIDI_STATE_DECL(MPID_STATE_VAPI_DEREGISTER_MR);

    MPIDI_FUNC_ENTER(MPID_STATE_IBU_DEREGISTER_MEMORY);

#ifdef USE_NO_PIN_CACHE

    /* caching turned off */
    MPIDI_FUNC_ENTER(MPID_STATE_VAPI_DEREGISTER_MR);
    status = VAPI_deregister_mr(IBU_Process.hca_handle, state->handle);
    MPIDI_FUNC_EXIT(MPID_STATE_VAPI_DEREGISTER_MR);
    if (status != IBU_SUCCESS)
    {
	MPIU_Internal_error_printf("ibu_deregister_memory: VAPI_deregister_mr failed, error %s\n", VAPI_strerror(status));
	MPIDI_FUNC_EXIT(MPID_STATE_IBU_DEREGISTER_MEMORY);
	return IBU_FAIL;
    }
    MPIDI_FUNC_EXIT(MPID_STATE_IBU_DEREGISTER_MEMORY);
    return IBU_SUCCESS;

#else

#ifdef TRACK_MEMORY_REGISTRATION
    remove_mem(buf);
#endif
    iter = ibu_pin_cache;
    while (iter != NULL)
    {
	if (buf == iter->buf && len == iter->len)
	{
	    iter->ref_count--;
	    MPIDI_FUNC_EXIT(MPID_STATE_IBU_DEREGISTER_MEMORY);
	    return IBU_SUCCESS;
	}
	if ((buf >= iter->buf) && ((char*)buf + len < ((char*)iter->buf + iter->len)))
	{
	    iter->ref_count--;
	    MPIDI_FUNC_EXIT(MPID_STATE_IBU_DEREGISTER_MEMORY);
	    return IBU_SUCCESS;
	}
	if (buf < iter->buf)
	{
	    if (iter->left == NULL)
	    {
		/* error, node not found */
		/*MPIU_Internal_error_printf("deregister memory not in cache buf = %p len = %d", buf, len);*/
		break;
	    }
	    iter = iter->left;
	}
	else
	{
	    if (iter->right == NULL)
	    {
		/* error, node not found */
		/*MPIU_Internal_error_printf("deregister memory not in cache buf = %p len = %d", buf, len);*/
		break;
	    }
	    iter = iter->right;
	}	
    }
    /* error, node not found */
    /*MPIU_Internal_error_printf("deregister memory not in cache buf = %p len = %d", buf, len);*/
/*
    MPIDI_FUNC_ENTER(MPID_STATE_VAPI_DEREGISTER_MR);
    status = VAPI_deregister_mr(IBU_Process.hca_handle, state->handle);
    MPIDI_FUNC_EXIT(MPID_STATE_VAPI_DEREGISTER_MR);
    if (status != IBU_SUCCESS)
    {
	MPIU_Internal_error_printf("ibu_deregister_memory: VAPI_deregister_mr failed, error %s\n", VAPI_strerror(status));
	MPIDI_FUNC_EXIT(MPID_STATE_IBU_DEREGISTER_MEMORY);
	return IBU_FAIL;
    }
*/
    MPIDI_FUNC_EXIT(MPID_STATE_IBU_DEREGISTER_MEMORY);
    return IBU_SUCCESS;

#endif
}

#undef FUNCNAME
#define FUNCNAME ibu_invalidate_memory
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int ibu_invalidate_memory(void *buf, int len)
{
#ifndef USE_NO_PIN_CACHE
    ibu_mem_node_t *iter;
#endif
    MPIDI_STATE_DECL(MPID_STATE_IBU_INVALIDATE_MEMORY);

    MPIDI_FUNC_ENTER(MPID_STATE_IBU_INVALIDATE_MEMORY);

#ifdef USE_NO_PIN_CACHE

    MPIDI_FUNC_EXIT(MPID_STATE_IBU_INVALIDATE_MEMORY);
    return IBU_SUCCESS;

#else

    iter = ibu_pin_cache;
    while (iter != NULL)
    {
	if (buf == iter->buf && len == iter->len)
	{
	    /* destroy this node */
	    iter->ref_count = 0;
	    ibu_clean_pin_cache();
	    MPIDI_FUNC_EXIT(MPID_STATE_IBU_INVALIDATE_MEMORY);
	    return IBU_SUCCESS;
	}
	if ((buf >= iter->buf) && ((char*)buf + len < ((char*)iter->buf + iter->len)))
	{
	    /* destroy this node */
	    iter->ref_count = 0;
	    ibu_clean_pin_cache();
	    MPIDI_FUNC_EXIT(MPID_STATE_IBU_INVALIDATE_MEMORY);
	    return IBU_SUCCESS;
	}
	if (buf < iter->buf)
	{
	    if (iter->left == NULL)
	    {
		/* node not found */
		break;
	    }
	    iter = iter->left;
	}
	else
	{
	    if (iter->right == NULL)
	    {
		/* node not found */
		break;
	    }
	    iter = iter->right;
	}	
    }

    /* memory not found */

    MPIDI_FUNC_EXIT(MPID_STATE_IBU_INVALIDATE_MEMORY);
    return IBU_SUCCESS;

#endif
}

#undef FUNCNAME
#define FUNCNAME ibu_nocache_register_memory
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int ibu_nocache_register_memory(void *buf, int len, ibu_mem_t *state)
{
    VAPI_ret_t status;
    VAPI_mrw_t mem, mem_out;
    MPIDI_STATE_DECL(MPID_STATE_IBU_NOCACHE_REGISTER_MEMORY);

    MPIDI_FUNC_ENTER(MPID_STATE_IBU_NOCACHE_REGISTER_MEMORY);

    MPIU_DBG_PRINTF(("entering ibu_nocache_register_memory\n"));

    memset(&mem, 0, sizeof(VAPI_mrw_t));
    memset(&mem_out, 0, sizeof(VAPI_mrw_t));
    mem.type = VAPI_MR;
    mem.start = (VAPI_virt_addr_t)(MT_virt_addr_t)buf;
    mem.size = len;
    mem.pd_hndl = IBU_Process.pd_handle;
    mem.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE | VAPI_EN_REMOTE_READ;
    mem.l_key = 0;
    mem.r_key = 0;
    status = VAPI_register_mr(
	IBU_Process.hca_handle,
	&mem,
	&state->handle,
	&mem_out);
    if (status != IBU_SUCCESS)
    {
	MPIU_Internal_error_printf("ibu_nocache_register_memory: VAPI_register_mr failed, error %s\n", VAPI_strerror(status));
	MPIDI_FUNC_EXIT(MPID_STATE_IBU_NOCACHE_REGISTER_MEMORY);
	return IBU_FAIL;
    }
    state->lkey = mem_out.l_key;
    state->rkey = mem_out.r_key;

    MPIDI_FUNC_EXIT(MPID_STATE_IBU_NOCACHE_REGISTER_MEMORY);
    return IBU_SUCCESS;
}

#undef FUNCNAME
#define FUNCNAME ibu_nocache_deregister_memory
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int ibu_nocache_deregister_memory(void *buf, int len, ibu_mem_t *state)
{
    VAPI_ret_t status;
    MPIDI_STATE_DECL(MPID_STATE_IBU_NOCACHE_DEREGISTER_MEMORY);

    MPIDI_FUNC_ENTER(MPID_STATE_IBU_NOCACHE_DEREGISTER_MEMORY);

    status = VAPI_deregister_mr(IBU_Process.hca_handle, state->handle);
    if (status != IBU_SUCCESS)
    {
	MPIU_Internal_error_printf("ibu_nocache_deregister_memory: VAPI_deregister_mr failed, error %s\n", VAPI_strerror(status));
	MPIDI_FUNC_EXIT(MPID_STATE_IBU_NOCACHE_DEREGISTER_MEMORY);
	return IBU_FAIL;
    }
    MPIDI_FUNC_EXIT(MPID_STATE_IBU_NOCACHE_DEREGISTER_MEMORY);
    return IBU_SUCCESS;
}

#endif