router.c

来自「lustre 1.6.5 source code」· C语言 代码 · 共 1,077 行 · 第 1/3 页

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 *
 * Copyright (C) 2002 Cluster File Systems, Inc.
 *
 *   This file is part of Portals
 *   http://sourceforge.net/projects/sandiaportals/
 *
 *   Portals is free software; you can redistribute it and/or
 *   modify it under the terms of version 2 of the GNU General Public
 *   License as published by the Free Software Foundation.
 *
 *   Portals is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with Portals; if not, write to the Free Software
 *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 */

#define DEBUG_SUBSYSTEM S_LNET
#include <lnet/lib-lnet.h>

#if defined(__KERNEL__) && defined(LNET_ROUTER)

static char *forwarding = "";
CFS_MODULE_PARM(forwarding, "s", charp, 0444,
                "Explicitly enable/disable forwarding between networks");

static int tiny_router_buffers = 1024;
CFS_MODULE_PARM(tiny_router_buffers, "i", int, 0444,
                "# of 0 payload messages to buffer in the router");
static int small_router_buffers = 8192;
CFS_MODULE_PARM(small_router_buffers, "i", int, 0444,
                "# of small (1 page) messages to buffer in the router");
static int large_router_buffers = 512;
CFS_MODULE_PARM(large_router_buffers, "i", int, 0444,
                "# of large messages to buffer in the router");

static int auto_down = 1;
CFS_MODULE_PARM(auto_down, "i", int, 0444,
                "Automatically mark peers down on comms error");

static int check_routers_before_use = 0;
CFS_MODULE_PARM(check_routers_before_use, "i", int, 0444,
                "Assume routers are down and ping them before use");

static int dead_router_check_interval = 0;
CFS_MODULE_PARM(dead_router_check_interval, "i", int, 0444,
                "Seconds between dead router health checks (<= 0 to disable)");

static int live_router_check_interval = 0;
CFS_MODULE_PARM(live_router_check_interval, "i", int, 0444,
                "Seconds between live router health checks (<= 0 to disable)");

static int router_ping_timeout = 50;
CFS_MODULE_PARM(router_ping_timeout, "i", int, 0444,
                "Seconds to wait for the reply to a router health query");

int
lnet_peers_start_down(void)
{
        return check_routers_before_use;
}

void
lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, time_t when)
{
        if (when < lp->lp_timestamp) {          /* out of date information */
                CDEBUG(D_NET, "Out of date\n");
                return;
        }

        lp->lp_timestamp = when;                /* update timestamp */
        lp->lp_ping_deadline = 0;               /* disable ping timeout */

        if (lp->lp_alive_count != 0 &&          /* got old news */
            (!lp->lp_alive) == (!alive)) {      /* new date for old news */
                CDEBUG(D_NET, "Old news\n");
                return;
        }

        /* Flag that notification is outstanding */

        lp->lp_alive_count++;
        lp->lp_alive = !(!alive);               /* 1 bit! */
        lp->lp_notify = 1;
        lp->lp_notifylnd |= notifylnd;

        CDEBUG(D_NET, "set %s %d\n", libcfs_nid2str(lp->lp_nid), alive);
}

void
lnet_do_notify (lnet_peer_t *lp) 
{
        lnet_ni_t *ni = lp->lp_ni;
        int        alive;
        int        notifylnd;
        
        LNET_LOCK();
                
        /* Notify only in 1 thread at any time to ensure ordered notification.
         * NB individual events can be missed; the only guarantee is that you
         * always get the most recent news */

        if (lp->lp_notifying) {
                LNET_UNLOCK();
                return;
        }

        lp->lp_notifying = 1;
        
        while (lp->lp_notify) {
                alive     = lp->lp_alive;
                notifylnd = lp->lp_notifylnd;

                lp->lp_notifylnd = 0;
                lp->lp_notify    = 0;

                if (notifylnd && ni->ni_lnd->lnd_notify != NULL) {
                        LNET_UNLOCK();

                        /* A new notification could happen now; I'll handle it
                         * when control returns to me */

                        (ni->ni_lnd->lnd_notify)(ni, lp->lp_nid, alive);

                        LNET_LOCK();
                }
        }

        lp->lp_notifying = 0;

        LNET_UNLOCK();
}

int
lnet_notify (lnet_ni_t *ni, lnet_nid_t nid, int alive, time_t when)
{
        lnet_peer_t         *lp = NULL;
        time_t               now = cfs_time_current_sec();

        LASSERT (!in_interrupt ());

        CDEBUG (D_NET, "%s notifying %s: %s\n",
                (ni == NULL) ? "userspace" : libcfs_nid2str(ni->ni_nid),
                libcfs_nid2str(nid),
                alive ? "up" : "down");

        if (ni != NULL &&
            LNET_NIDNET(ni->ni_nid) != LNET_NIDNET(nid)) {
                CWARN ("Ignoring notification of %s %s by %s (different net)\n",
                        libcfs_nid2str(nid), alive ? "birth" : "death",
                        libcfs_nid2str(ni->ni_nid));
                return -EINVAL;
        }

        /* can't do predictions... */
        if (when > now) {
                CWARN ("Ignoring prediction from %s of %s %s "
                       "%ld seconds in the future\n",
                       (ni == NULL) ? "userspace" : libcfs_nid2str(ni->ni_nid),
                       libcfs_nid2str(nid), alive ? "up" : "down",
                       when - now);
                return -EINVAL;
        }

        if (ni != NULL && !alive &&             /* LND telling me she's down */
            !auto_down) {                       /* auto-down disabled */
                CDEBUG(D_NET, "Auto-down disabled\n");
                return 0;
        }
        
        LNET_LOCK();

        lp = lnet_find_peer_locked(nid);
        if (lp == NULL) {
                /* nid not found */
                LNET_UNLOCK();
                CDEBUG(D_NET, "%s not found\n", libcfs_nid2str(nid));
                return 0;
        }

        lnet_notify_locked(lp, ni == NULL, alive, when);

        LNET_UNLOCK();
        
        lnet_do_notify(lp);
        
        LNET_LOCK();

        lnet_peer_decref_locked(lp);

        LNET_UNLOCK();
        return 0;
}
EXPORT_SYMBOL(lnet_notify);

#else

int
lnet_notify (lnet_ni_t *ni, lnet_nid_t nid, int alive, time_t when)
{
        return -EOPNOTSUPP;
}

#endif

static void
lnet_rtr_addref_locked(lnet_peer_t *lp)
{
        LASSERT (lp->lp_refcount > 0);
        LASSERT (lp->lp_rtr_refcount >= 0);

        lp->lp_rtr_refcount++;
        if (lp->lp_rtr_refcount == 1) {
                struct list_head *pos;

                /* a simple insertion sort */
                list_for_each_prev(pos, &the_lnet.ln_routers) {
                        lnet_peer_t *rtr = list_entry(pos, lnet_peer_t, 
                                                      lp_rtr_list);

                        if (rtr->lp_nid < lp->lp_nid)
                                break;
                }

                list_add(&lp->lp_rtr_list, pos);
                /* addref for the_lnet.ln_routers */
                lnet_peer_addref_locked(lp);
                the_lnet.ln_routers_version++;
        }
}

static void
lnet_rtr_decref_locked(lnet_peer_t *lp)
{
        LASSERT (lp->lp_refcount > 0);
        LASSERT (lp->lp_rtr_refcount > 0);

        lp->lp_rtr_refcount--;
        if (lp->lp_rtr_refcount == 0) {
                list_del(&lp->lp_rtr_list);
                /* decref for the_lnet.ln_routers */
                lnet_peer_decref_locked(lp);
                the_lnet.ln_routers_version++;
        }
}

lnet_remotenet_t *
lnet_find_net_locked (__u32 net)
{
        lnet_remotenet_t *rnet;
        struct list_head *tmp;

        LASSERT (!the_lnet.ln_shutdown);

        list_for_each (tmp, &the_lnet.ln_remote_nets) {
                rnet = list_entry(tmp, lnet_remotenet_t, lrn_list);

                if (rnet->lrn_net == net)
                        return rnet;
        }
        return NULL;
}

int
lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway)
{
        struct list_head     zombies;
	struct list_head    *e;
	lnet_remotenet_t    *rnet;
	lnet_remotenet_t    *rnet2;
	lnet_route_t        *route;
	lnet_route_t        *route2;
        lnet_ni_t           *ni;
        int                  add_route;
        int                  rc;

        CDEBUG(D_NET, "Add route: net %s hops %u gw %s\n",
               libcfs_net2str(net), hops, libcfs_nid2str(gateway));

        if (gateway == LNET_NID_ANY ||
            LNET_NETTYP(LNET_NIDNET(gateway)) == LOLND ||
            net == LNET_NIDNET(LNET_NID_ANY) ||
            LNET_NETTYP(net) == LOLND ||
            LNET_NIDNET(gateway) == net ||
            hops < 1 || hops > 255)
                return (-EINVAL);

        if (lnet_islocalnet(net))               /* it's a local network */
                return 0;                       /* ignore the route entry */

        /* Assume net, route, all new */
        LIBCFS_ALLOC(route, sizeof(*route));
        LIBCFS_ALLOC(rnet, sizeof(*rnet));
        if (route == NULL || rnet == NULL) {
                CERROR("Out of memory creating route %s %d %s\n",
                       libcfs_net2str(net), hops, libcfs_nid2str(gateway));
                if (route != NULL)
                        LIBCFS_FREE(route, sizeof(*route));
                if (rnet != NULL)
                        LIBCFS_FREE(rnet, sizeof(*rnet));
                return -ENOMEM;
        }

        CFS_INIT_LIST_HEAD(&rnet->lrn_routes);
        rnet->lrn_net = net;
        rnet->lrn_hops = hops;

        LNET_LOCK();

        rc = lnet_nid2peer_locked(&route->lr_gateway, gateway);
        if (rc != 0) {
                LNET_UNLOCK();

                LIBCFS_FREE(route, sizeof(*route));
                LIBCFS_FREE(rnet, sizeof(*rnet));

                if (rc == -EHOSTUNREACH)        /* gateway is not on a local net */
                        return 0;               /* ignore the route entry */

                CERROR("Error %d creating route %s %d %s\n", rc,
                       libcfs_net2str(net), hops, libcfs_nid2str(gateway));
                return rc;
        }

        LASSERT (!the_lnet.ln_shutdown);
        CFS_INIT_LIST_HEAD(&zombies);

        rnet2 = lnet_find_net_locked(net);
        if (rnet2 == NULL) {
                /* new network */
                list_add_tail(&rnet->lrn_list, &the_lnet.ln_remote_nets);
                rnet2 = rnet;
        }

        if (hops > rnet2->lrn_hops) {
                /* New route is longer; ignore it */
                add_route = 0;
        } else if (hops < rnet2->lrn_hops) {
                /* new route supercedes all currently known routes to this
                 * net */
                list_add(&zombies, &rnet2->lrn_routes);
                list_del_init(&rnet2->lrn_routes);
                add_route = 1;
        } else {
                add_route = 1;
                /* New route has the same hopcount as existing routes; search
                 * for a duplicate route (it's a NOOP if it is) */
                list_for_each (e, &rnet2->lrn_routes) {
                        route2 = list_entry(e, lnet_route_t, lr_list);

                        if (route2->lr_gateway == route->lr_gateway) {
                                add_route = 0;
                                break;