mxlnd.c

来自「lustre 1.6.5 source code」· C语言 代码 · 共 938 行 · 第 1/3 页

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 *
 * Copyright (C) 2004 Cluster File Systems, Inc.
 *   Author: Eric Barton <[e-mail]>
 * Copyright (C) 2006 Myricom, Inc.
 *   Author: Scott Atchley <atchley at myri.com>
 *
 *   This file is part of Lustre, http://www.lustre.org.
 *
 *   Lustre is free software; you can redistribute it and/or
 *   modify it under the terms of version 2 of the GNU General Public
 *   License as published by the Free Software Foundation.
 *
 *   Lustre is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with Lustre; if not, write to the Free Software
 *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include "mxlnd.h"

lnd_t the_kmxlnd = {
        .lnd_type       = MXLND,
        .lnd_startup    = mxlnd_startup,
        .lnd_shutdown   = mxlnd_shutdown,
        .lnd_ctl        = mxlnd_ctl,
        .lnd_send       = mxlnd_send,
        .lnd_recv       = mxlnd_recv,
};

kmx_data_t               kmxlnd_data;

/**
 * mxlnd_ctx_free - free ctx struct
 * @ctx - a kmx_peer pointer
 *
 * The calling function should remove the ctx from the ctx list first
 * then free it.
 */
void
mxlnd_ctx_free(struct kmx_ctx *ctx)
{
        if (ctx == NULL) return;

        if (ctx->mxc_page != NULL) {
                __free_page(ctx->mxc_page);
                spin_lock(&kmxlnd_data.kmx_global_lock);
                kmxlnd_data.kmx_mem_used -= MXLND_EAGER_SIZE;
                spin_unlock(&kmxlnd_data.kmx_global_lock);
        }

        if (ctx->mxc_seg_list != NULL) {
                LASSERT(ctx->mxc_nseg > 0);
                MXLND_FREE(ctx->mxc_seg_list, ctx->mxc_nseg * sizeof(mx_ksegment_t));
        }

        MXLND_FREE (ctx, sizeof (*ctx));
        return;
}

/**
 * mxlnd_ctx_alloc - allocate and initialize a new ctx struct
 * @ctxp - address of a kmx_ctx pointer
 *
 * Returns 0 on success and -EINVAL, -ENOMEM on failure
 */
int
mxlnd_ctx_alloc(struct kmx_ctx **ctxp, enum kmx_req_type type)
{
        int             ret     = 0;
        struct kmx_ctx  *ctx    = NULL;

        if (ctxp == NULL) return -EINVAL;

        MXLND_ALLOC(ctx, sizeof (*ctx));
        if (ctx == NULL) {
                CDEBUG(D_NETERROR, "Cannot allocate ctx\n");
                return -ENOMEM;
        }
        memset(ctx, 0, sizeof(*ctx));
        spin_lock_init(&ctx->mxc_lock);

        ctx->mxc_type = type;
        ctx->mxc_page = alloc_page (GFP_KERNEL);
        if (ctx->mxc_page == NULL) {
                CDEBUG(D_NETERROR, "Can't allocate page\n");
                ret = -ENOMEM;
                goto failed;
        }
        spin_lock(&kmxlnd_data.kmx_global_lock);
        kmxlnd_data.kmx_mem_used += MXLND_EAGER_SIZE;
        spin_unlock(&kmxlnd_data.kmx_global_lock);
        ctx->mxc_msg = (struct kmx_msg *)((char *)page_address(ctx->mxc_page));
        ctx->mxc_seg.segment_ptr = MX_PA_TO_U64(lnet_page2phys(ctx->mxc_page));
        ctx->mxc_state = MXLND_CTX_IDLE;

        *ctxp = ctx;
        return 0;

failed:
        mxlnd_ctx_free(ctx);
        return ret;
}

/**
 * mxlnd_ctx_init - reset ctx struct to the default values
 * @ctx - a kmx_ctx pointer
 */
void
mxlnd_ctx_init(struct kmx_ctx *ctx)
{
        if (ctx == NULL) return;

        /* do not change mxc_type */
        ctx->mxc_incarnation = 0;
        ctx->mxc_deadline = 0;
        ctx->mxc_state = MXLND_CTX_IDLE;
        /* ignore mxc_global_list */
        if (ctx->mxc_list.next != NULL && !list_empty(&ctx->mxc_list)) {
                if (ctx->mxc_peer != NULL) spin_lock(&ctx->mxc_lock);
                list_del_init(&ctx->mxc_list);
                if (ctx->mxc_peer != NULL) spin_unlock(&ctx->mxc_lock);
        }
        /* ignore mxc_rx_list */
        /* ignore mxc_lock */
        ctx->mxc_nid = 0;
        ctx->mxc_peer = NULL;
        ctx->mxc_conn = NULL;
        /* ignore mxc_msg */
        /* ignore mxc_page */
        ctx->mxc_lntmsg[0] = NULL;
        ctx->mxc_lntmsg[1] = NULL;
        ctx->mxc_msg_type = 0;
        ctx->mxc_cookie = 0LL;
        ctx->mxc_match = 0LL;
        /* ctx->mxc_seg.segment_ptr points to mxc_page */
        ctx->mxc_seg.segment_length = 0;
        if (ctx->mxc_seg_list != NULL) {
                LASSERT(ctx->mxc_nseg > 0);
                MXLND_FREE(ctx->mxc_seg_list, ctx->mxc_nseg * sizeof(mx_ksegment_t));
        }
        ctx->mxc_seg_list = NULL;
        ctx->mxc_nseg = 0;
        ctx->mxc_nob = 0;
        ctx->mxc_mxreq = NULL;
        memset(&ctx->mxc_status, 0, sizeof(mx_status_t));
        /* ctx->mxc_get */
        /* ctx->mxc_put */

        ctx->mxc_msg->mxm_type = 0;
        ctx->mxc_msg->mxm_credits = 0;
        ctx->mxc_msg->mxm_nob = 0;
        ctx->mxc_msg->mxm_seq = 0;

        return;
}

/**
 * mxlnd_free_txs - free kmx_txs and associated pages
 *
 * Called from mxlnd_shutdown()
 */
void
mxlnd_free_txs(void)
{
        struct kmx_ctx          *tx     = NULL;
        struct kmx_ctx          *next   = NULL;

        list_for_each_entry_safe(tx, next, &kmxlnd_data.kmx_txs, mxc_global_list) {
                list_del_init(&tx->mxc_global_list);
                mxlnd_ctx_free(tx);
        }
        return;
}

/**
 * mxlnd_init_txs - allocate tx descriptors then stash on txs and idle tx lists
 *
 * Called from mxlnd_startup()
 * returns 0 on success, else -ENOMEM
 */
int
mxlnd_init_txs(void)
{
        int             ret     = 0;
        int             i       = 0;
        struct kmx_ctx  *tx      = NULL;

        for (i = 0; i < *kmxlnd_tunables.kmx_ntx; i++) {
                ret = mxlnd_ctx_alloc(&tx, MXLND_REQ_TX);
                if (ret != 0) {
                        mxlnd_free_txs();
                        return ret;
                }
                mxlnd_ctx_init(tx);
                /* in startup(), no locks required */
                list_add_tail(&tx->mxc_global_list, &kmxlnd_data.kmx_txs);
                list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_tx_idle);
        }
        return 0;
}

/**
 * mxlnd_free_rxs - free initial kmx_rx descriptors and associated pages
 *
 * Called from mxlnd_shutdown()
 */
void
mxlnd_free_rxs(void)
{
        struct kmx_ctx          *rx     = NULL;
        struct kmx_ctx          *next   = NULL;

        list_for_each_entry_safe(rx, next, &kmxlnd_data.kmx_rxs, mxc_global_list) {
                list_del_init(&rx->mxc_global_list);
                mxlnd_ctx_free(rx);
        }
        return;
}

/**
 * mxlnd_init_rxs - allocate initial rx descriptors 
 *
 * Called from startup(). We create MXLND_MAX_PEERS plus MXLND_NTX
 * rx descriptors. We create one for each potential peer to handle 
 * the initial connect request. We create on for each tx in case the 
 * send requires a non-eager receive.
 *
 * Returns 0 on success, else -ENOMEM
 */
int
mxlnd_init_rxs(void)
{
        int             ret     = 0;
        int             i       = 0;
        struct kmx_ctx  *rx      = NULL;

        for (i = 0; i < (*kmxlnd_tunables.kmx_ntx + *kmxlnd_tunables.kmx_max_peers); i++) {
                ret = mxlnd_ctx_alloc(&rx, MXLND_REQ_RX);
                if (ret != 0) {
                        mxlnd_free_rxs();
                        return ret;
                }
                mxlnd_ctx_init(rx);
                /* in startup(), no locks required */
                list_add_tail(&rx->mxc_global_list, &kmxlnd_data.kmx_rxs);
                list_add_tail(&rx->mxc_list, &kmxlnd_data.kmx_rx_idle);
        }
        return 0;
}

/**
 * mxlnd_free_peers - free peers
 *
 * Called from mxlnd_shutdown()
 */
void
mxlnd_free_peers(void)
{
        int                      i      = 0;
        struct kmx_peer         *peer   = NULL;
        struct kmx_peer         *next   = NULL;

        for (i = 0; i < MXLND_HASH_SIZE; i++) {
                list_for_each_entry_safe(peer, next, &kmxlnd_data.kmx_peers[i], mxp_peers) {
                        list_del_init(&peer->mxp_peers);
                        if (peer->mxp_conn) mxlnd_conn_decref(peer->mxp_conn);
                        mxlnd_peer_decref(peer);
                }
        }
}

int
mxlnd_host_alloc(struct kmx_host **hostp)
{
        struct kmx_host *host   = NULL;

        MXLND_ALLOC(host, sizeof (*host));
        if (host == NULL) {
                CDEBUG(D_NETERROR, "Cannot allocate host\n");
                return -1;
        }
        memset(host, 0, sizeof(*host));
        spin_lock_init(&host->mxh_lock);

        *hostp = host;

        return 0;
}

void
mxlnd_host_free(struct kmx_host *host)
{
        if (host == NULL) return;

        if (host->mxh_hostname != NULL)
                MXLND_FREE(host->mxh_hostname, strlen(host->mxh_hostname) + 1);

        MXLND_FREE(host, sizeof(*host));
        return;
}

/**
 * mxlnd_free_hosts - free kmx_hosts
 *
 * Called from mxlnd_shutdown()
 */
void