gmlnd_utils.c

非常经典的一个分布式系统

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 *
   Copyright (c) 2003 Los Alamos National Laboratory (LANL)
 * Copyright (C) 2005 Cluster File Systems, Inc. All rights reserved.
 *
 *   This file is part of Lustre, http://www.lustre.org/
 *
 *   Lustre is free software; you can redistribute it and/or
 *   modify it under the terms of version 2 of the GNU General Public
 *   License as published by the Free Software Foundation.
 *
 *   Lustre is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with Lustre; if not, write to the Free Software
 *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include "gmlnd.h"

void
gmnal_free_netbuf_pages (gmnal_netbuf_t *nb, int npages) 
{
        int     i;
        
        for (i = 0; i < npages; i++)
                __free_page(nb->nb_kiov[i].kiov_page);
}

int
gmnal_alloc_netbuf_pages (gmnal_ni_t *gmni, gmnal_netbuf_t *nb, int npages)
{
        int          i;
        gm_status_t  gmrc;

        LASSERT (npages > 0);

        for (i = 0; i < npages; i++) {
                
                nb->nb_kiov[i].kiov_page = alloc_page(GFP_KERNEL);
                nb->nb_kiov[i].kiov_offset = 0;
                nb->nb_kiov[i].kiov_len = PAGE_SIZE;

                if (nb->nb_kiov[i].kiov_page == NULL) {
                        CERROR("Can't allocate page\n");
                        gmnal_free_netbuf_pages(nb, i);
                        return -ENOMEM;
                }

                CDEBUG(D_NET,"[%3d] page %p, phys "LPX64", @ "LPX64"\n",
                       i, nb->nb_kiov[i].kiov_page, 
                       lnet_page2phys(nb->nb_kiov[i].kiov_page),
                       gmni->gmni_netaddr_base);

                gmrc = gm_register_memory_ex_phys(
                        gmni->gmni_port,
                        lnet_page2phys(nb->nb_kiov[i].kiov_page),
                        PAGE_SIZE,
                        gmni->gmni_netaddr_base);
                CDEBUG(D_NET,"[%3d] page %p: %d\n", 
                       i, nb->nb_kiov[i].kiov_page, gmrc);

                if (gmrc != GM_SUCCESS) {
                        CERROR("Can't map page: %d(%s)\n", gmrc,
                               gmnal_gmstatus2str(gmrc));
                        gmnal_free_netbuf_pages(nb, i+1);
                        return -ENOMEM;
                }
                
                if (i == 0) 
                        nb->nb_netaddr = gmni->gmni_netaddr_base;
                
                gmni->gmni_netaddr_base += PAGE_SIZE;
        }
        
        return 0;
}

void
gmnal_free_ltxbuf (gmnal_ni_t *gmni, gmnal_txbuf_t *txb)
{
        int            npages = gmni->gmni_large_pages;

        LASSERT (gmni->gmni_port == NULL);
        /* No unmapping; the port has been closed */

        gmnal_free_netbuf_pages(&txb->txb_buf, gmni->gmni_large_pages);
        LIBCFS_FREE(txb, offsetof(gmnal_txbuf_t, txb_buf.nb_kiov[npages]));
}

int
gmnal_alloc_ltxbuf (gmnal_ni_t *gmni)
{
        int            npages = gmni->gmni_large_pages;
        int            sz = offsetof(gmnal_txbuf_t, txb_buf.nb_kiov[npages]);
        gmnal_txbuf_t *txb;
        int            rc;
        
        LIBCFS_ALLOC(txb, sz);
        if (txb == NULL) {
                CERROR("Can't allocate large txbuffer\n");
                return -ENOMEM;
        }

        rc = gmnal_alloc_netbuf_pages(gmni, &txb->txb_buf, npages);
        if (rc != 0) {
                LIBCFS_FREE(txb, sz);
                return rc;
        }

        list_add_tail(&txb->txb_list, &gmni->gmni_idle_ltxbs);

        txb->txb_next = gmni->gmni_ltxbs;
        gmni->gmni_ltxbs = txb;

        return 0;
}

void
gmnal_free_tx (gmnal_tx_t *tx)
{
        LASSERT (tx->tx_gmni->gmni_port == NULL);

        gmnal_free_netbuf_pages(&tx->tx_buf, 1);
        LIBCFS_FREE(tx, sizeof(*tx));
}

int
gmnal_alloc_tx (gmnal_ni_t *gmni) 
{
        gmnal_tx_t  *tx;
        int          rc;
        
        LIBCFS_ALLOC(tx, sizeof(*tx));
        if (tx == NULL) {
                CERROR("Failed to allocate tx\n");
                return -ENOMEM;
        }
        
        memset(tx, 0, sizeof(*tx));

        rc = gmnal_alloc_netbuf_pages(gmni, &tx->tx_buf, 1);
        if (rc != 0) {
                LIBCFS_FREE(tx, sizeof(*tx));
                return -ENOMEM;
        }

        tx->tx_gmni = gmni;
        
        list_add_tail(&tx->tx_list, &gmni->gmni_idle_txs);

        tx->tx_next = gmni->gmni_txs;
        gmni->gmni_txs = tx;
                
        return 0;
}

void
gmnal_free_rx(gmnal_ni_t *gmni, gmnal_rx_t *rx)
{
        int   npages = rx->rx_islarge ? gmni->gmni_large_pages : 1;
        
        LASSERT (gmni->gmni_port == NULL);

        gmnal_free_netbuf_pages(&rx->rx_buf, npages);
        LIBCFS_FREE(rx, offsetof(gmnal_rx_t, rx_buf.nb_kiov[npages]));
}

int
gmnal_alloc_rx (gmnal_ni_t *gmni, int islarge)
{
        int         npages = islarge ? gmni->gmni_large_pages : 1;
        int         sz = offsetof(gmnal_rx_t, rx_buf.nb_kiov[npages]);
        int         rc;
        gmnal_rx_t *rx;
        gm_status_t gmrc;
        
        LIBCFS_ALLOC(rx, sz);
        if (rx == NULL) {
                CERROR("Failed to allocate rx\n");
                return -ENOMEM;
        }
        
        memset(rx, 0, sizeof(*rx));

        rc = gmnal_alloc_netbuf_pages(gmni, &rx->rx_buf, npages);
        if (rc != 0) {
                LIBCFS_FREE(rx, sz);
                return rc;
        }
        
        rx->rx_islarge = islarge;
        rx->rx_next = gmni->gmni_rxs;
        gmni->gmni_rxs = rx;

        gmrc = gm_hash_insert(gmni->gmni_rx_hash, 
                              GMNAL_NETBUF_LOCAL_NETADDR(&rx->rx_buf), rx);
        if (gmrc != GM_SUCCESS) {
                CERROR("Couldn't add rx to hash table: %d\n", gmrc);
                return -ENOMEM;
        }
        
        return 0;
}

void
gmnal_free_ltxbufs (gmnal_ni_t *gmni)
{
        gmnal_txbuf_t *txb;
        
        while ((txb = gmni->gmni_ltxbs) != NULL) {
                gmni->gmni_ltxbs = txb->txb_next;
                gmnal_free_ltxbuf(gmni, txb);
        }
}

int
gmnal_alloc_ltxbufs (gmnal_ni_t *gmni)
{
        int     nlarge_tx_bufs = *gmnal_tunables.gm_nlarge_tx_bufs;
        int     i;
        int     rc;

        for (i = 0; i < nlarge_tx_bufs; i++) {
                rc = gmnal_alloc_ltxbuf(gmni);
                
                if (rc != 0)
                        return rc;
        }

        return 0;
}

void
gmnal_free_txs(gmnal_ni_t *gmni)
{
	gmnal_tx_t *tx;

        while ((tx = gmni->gmni_txs) != NULL) {
                gmni->gmni_txs = tx->tx_next;
                gmnal_free_tx (tx);
	}
}

int
gmnal_alloc_txs(gmnal_ni_t *gmni)
{
        int           ntxcred = gm_num_send_tokens(gmni->gmni_port);
        int           ntx = *gmnal_tunables.gm_ntx;
        int           i;
        int           rc;

        CDEBUG(D_NET, "ntxcred: %d\n", ntxcred);
        gmni->gmni_tx_credits = ntxcred;

        for (i = 0; i < ntx; i++) {
                rc = gmnal_alloc_tx(gmni);
                if (rc != 0)
                        return rc;
        }

        return 0;
}

void
gmnal_free_rxs(gmnal_ni_t *gmni)
{
	gmnal_rx_t *rx;

	while ((rx = gmni->gmni_rxs) != NULL) {
                gmni->gmni_rxs = rx->rx_next;

                gmnal_free_rx(gmni, rx);
        }

        LASSERT (gmni->gmni_port == NULL);
#if 0
        /* GM releases all resources allocated to a port when it closes */
        if (gmni->gmni_rx_hash != NULL)
                gm_destroy_hash(gmni->gmni_rx_hash);
#endif
}

int
gmnal_alloc_rxs (gmnal_ni_t *gmni)
{
        int          nrxcred = gm_num_receive_tokens(gmni->gmni_port);