router.c

来自「lustre 1.6.5 source code」· C语言 代码 · 共 1,077 行 · 第 1/3 页

                        }

                        /* our loopups must be true */
                        LASSERT (route2->lr_gateway->lp_nid != gateway);
                }
        }
        
        if (add_route) {
                ni = route->lr_gateway->lp_ni;
                lnet_ni_addref_locked(ni);
                
                LASSERT (rc == 0);
                list_add_tail(&route->lr_list, &rnet2->lrn_routes);
                the_lnet.ln_remote_nets_version++;

                lnet_rtr_addref_locked(route->lr_gateway);

                LNET_UNLOCK();

                /* XXX Assume alive */
                if (ni->ni_lnd->lnd_notify != NULL)
                        (ni->ni_lnd->lnd_notify)(ni, gateway, 1);

                lnet_ni_decref(ni);
        } else {
                lnet_peer_decref_locked(route->lr_gateway);
                LNET_UNLOCK();
                LIBCFS_FREE(route, sizeof(*route));
        }

        if (rnet != rnet2)
                LIBCFS_FREE(rnet, sizeof(*rnet));

        while (!list_empty(&zombies)) {
                route = list_entry(zombies.next, lnet_route_t, lr_list);
                list_del(&route->lr_list);
                
                LNET_LOCK();
                lnet_rtr_decref_locked(route->lr_gateway);
                lnet_peer_decref_locked(route->lr_gateway);
                LNET_UNLOCK();
                LIBCFS_FREE(route, sizeof(*route));
        }

        return rc;
}

int
lnet_check_routes (void)
{
        lnet_remotenet_t    *rnet;
        lnet_route_t        *route;
        lnet_route_t        *route2;
        struct list_head    *e1;
        struct list_head    *e2;

        LNET_LOCK();

        list_for_each (e1, &the_lnet.ln_remote_nets) {
                rnet = list_entry(e1, lnet_remotenet_t, lrn_list);

                route2 = NULL;
                list_for_each (e2, &rnet->lrn_routes) {
                        route = list_entry(e2, lnet_route_t, lr_list);

                        if (route2 == NULL)
                                route2 = route;
                        else if (route->lr_gateway->lp_ni !=
                                 route2->lr_gateway->lp_ni) {
                                LNET_UNLOCK();
                                
                                CERROR("Routes to %s via %s and %s not supported\n",
                                       libcfs_net2str(rnet->lrn_net),
                                       libcfs_nid2str(route->lr_gateway->lp_nid),
                                       libcfs_nid2str(route2->lr_gateway->lp_nid));
                                return -EINVAL;
                        }
                }
        }
        
        LNET_UNLOCK();
        return 0;
}

int
lnet_del_route (__u32 net, lnet_nid_t gw_nid)
{
        lnet_remotenet_t    *rnet;
        lnet_route_t        *route;
        struct list_head    *e1;
        struct list_head    *e2;
        int                  rc = -ENOENT;

        CDEBUG(D_NET, "Del route: net %s : gw %s\n",
               libcfs_net2str(net), libcfs_nid2str(gw_nid));

        /* NB Caller may specify either all routes via the given gateway
         * or a specific route entry actual NIDs) */

 again:
        LNET_LOCK();

        list_for_each (e1, &the_lnet.ln_remote_nets) {
                rnet = list_entry(e1, lnet_remotenet_t, lrn_list);

                if (!(net == LNET_NIDNET(LNET_NID_ANY) ||
                      net == rnet->lrn_net))
                        continue;

                list_for_each (e2, &rnet->lrn_routes) {
                        route = list_entry(e2, lnet_route_t, lr_list);

                        if (!(gw_nid == LNET_NID_ANY ||
                              gw_nid == route->lr_gateway->lp_nid))
                                continue;

                        list_del(&route->lr_list);
                        the_lnet.ln_remote_nets_version++;

                        if (list_empty(&rnet->lrn_routes))
                                list_del(&rnet->lrn_list);
                        else
                                rnet = NULL;

                        lnet_rtr_decref_locked(route->lr_gateway);
                        lnet_peer_decref_locked(route->lr_gateway);
                        LNET_UNLOCK();

                        LIBCFS_FREE(route, sizeof (*route));

                        if (rnet != NULL)
                                LIBCFS_FREE(rnet, sizeof(*rnet));

                        rc = 0;
                        goto again;
                }
        }

        LNET_UNLOCK();
        return rc;
}

void
lnet_destroy_routes (void)
{
        lnet_del_route(LNET_NIDNET(LNET_NID_ANY), LNET_NID_ANY);
}

int
lnet_get_route (int idx, __u32 *net, __u32 *hops,
               lnet_nid_t *gateway, __u32 *alive)
{
	struct list_head    *e1;
	struct list_head    *e2;
        lnet_remotenet_t    *rnet;
        lnet_route_t        *route;

        LNET_LOCK();

        list_for_each (e1, &the_lnet.ln_remote_nets) {
                rnet = list_entry(e1, lnet_remotenet_t, lrn_list);

                list_for_each (e2, &rnet->lrn_routes) {
                        route = list_entry(e2, lnet_route_t, lr_list);

                        if (idx-- == 0) {
                                *net     = rnet->lrn_net;
                                *hops    = rnet->lrn_hops;
                                *gateway = route->lr_gateway->lp_nid;
                                *alive   = route->lr_gateway->lp_alive;
                                LNET_UNLOCK();
                                return 0;
                        }
                }
        }

        LNET_UNLOCK();
        return -ENOENT;
}

#if defined(__KERNEL__) && defined(LNET_ROUTER)
static void
lnet_router_checker_event (lnet_event_t *event)
{
        /* CAVEAT EMPTOR: I'm called with LNET_LOCKed and I'm not allowed to
         * drop it (that's how come I see _every_ event, even ones that would
         * overflow my EQ) */
        lnet_peer_t   *lp;
        lnet_nid_t     nid;

        if (event->unlinked) {
                /* The router checker thread has unlinked the rc_md
                 * and exited. */
                LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_UNLINKING);
                the_lnet.ln_rc_state = LNET_RC_STATE_UNLINKED; 
                mutex_up(&the_lnet.ln_rc_signal); 
                return;
        }

        LASSERT (event->type == LNET_EVENT_SEND || 
                 event->type == LNET_EVENT_REPLY);
        
        nid = (event->type == LNET_EVENT_SEND) ?
              event->target.nid : event->initiator.nid;

        lp = lnet_find_peer_locked(nid);
        if (lp == NULL) {
                /* router may have been removed */
                CDEBUG(D_NET, "Router %s not found\n", libcfs_nid2str(nid));
                return;
        }

        if (event->type == LNET_EVENT_SEND)     /* re-enable another ping */
                lp->lp_ping_notsent = 0;

        if (lnet_isrouter(lp) &&                /* ignore if no longer a router */
            (event->status != 0 ||
             event->type == LNET_EVENT_REPLY)) {
                
                /* A successful REPLY means the router is up.  If _any_ comms
                 * to the router fail I assume it's down (this will happen if
                 * we ping alive routers to try to detect router death before
                 * apps get burned). */

                lnet_notify_locked(lp, 1, (event->status == 0),
                                   cfs_time_current_sec());

                /* The router checker will wake up very shortly and do the
                 * actual notification.  
                 * XXX If 'lp' stops being a router before then, it will still
                 * have the notification pending!!! */
        }

        /* This decref will NOT drop LNET_LOCK (it had to have 1 ref when it
         * was in the peer table and I've not dropped the lock, so no-one else
         * can have reduced the refcount) */
        LASSERT(lp->lp_refcount > 1);

        lnet_peer_decref_locked(lp);
}

static int
lnet_router_checker(void *arg)
{
        static lnet_ping_info_t   pinginfo;

        int                  rc;
        lnet_handle_md_t     mdh;
        lnet_peer_t         *rtr;
        struct list_head    *entry;
        time_t               now;
        lnet_process_id_t    rtr_id;
        int                  secs;

	cfs_daemonize("router_checker");
	cfs_block_allsigs();

        rtr_id.pid = LUSTRE_SRV_LNET_PID;

        LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_SHUTDOWN);

        rc = LNetMDBind((lnet_md_t){.start     = &pinginfo,
                                    .length    = sizeof(pinginfo),
                                    .threshold = LNET_MD_THRESH_INF,
                                    .options   = LNET_MD_TRUNCATE,
                                    .eq_handle = the_lnet.ln_rc_eqh},
                        LNET_UNLINK,
                        &mdh);

        if (rc < 0) {
                CERROR("Can't bind MD: %d\n", rc);
                the_lnet.ln_rc_state = rc;
                mutex_up(&the_lnet.ln_rc_signal);
                return rc;
        }

        LASSERT (rc == 0);

        the_lnet.ln_rc_state = LNET_RC_STATE_RUNNING;
        mutex_up(&the_lnet.ln_rc_signal);       /* let my parent go */

	while (the_lnet.ln_rc_state == LNET_RC_STATE_RUNNING) {
                __u64 version;

                LNET_LOCK();
rescan:
                version = the_lnet.ln_routers_version;

                list_for_each (entry, &the_lnet.ln_routers) {
                        rtr = list_entry(entry, lnet_peer_t, lp_rtr_list);

                        lnet_peer_addref_locked(rtr);

                        now = cfs_time_current_sec();

                        if (rtr->lp_ping_deadline != 0 && /* ping timed out? */
                            now > rtr->lp_ping_deadline)
                                lnet_notify_locked(rtr, 1, 0, now);

                        LNET_UNLOCK();

                        /* Run any outstanding notificiations */
                        lnet_do_notify(rtr);

                        if (rtr->lp_alive) {
                                secs = live_router_check_interval;
                        } else {
                                secs = dead_router_check_interval;
                        }
                        if (secs <= 0)
                                secs = 0;
                        
                        if (secs != 0 &&
                            !rtr->lp_ping_notsent &&
                            now > rtr->lp_ping_timestamp + secs) {
                                CDEBUG(D_NET, "Check: %s\n",
                                       libcfs_nid2str(rtr->lp_nid));

                                LNET_LOCK();
                                rtr_id.nid = rtr->lp_nid;
                                rtr->lp_ping_notsent = 1;
                                rtr->lp_ping_timestamp = now;

                                if (rtr->lp_ping_deadline == 0)
                                        rtr->lp_ping_deadline = 
                                                now + router_ping_timeout;

                                LNET_UNLOCK();

                                LNetGet(LNET_NID_ANY, mdh, rtr_id,
                                        LNET_RESERVED_PORTAL,
                                        LNET_PROTO_PING_MATCHBITS, 0);
                        }
                        
                        LNET_LOCK();
                        lnet_peer_decref_locked(rtr);

                        if (version != the_lnet.ln_routers_version) {
                                /* the routers list has changed */
                                goto rescan;
                        }
                }

                LNET_UNLOCK();

                /* Call cfs_pause() here always adds 1 to load average 
                 * because kernel counts # active tasks as nr_running 
                 * + nr_uninterruptible. */
                set_current_state(CFS_TASK_INTERRUPTIBLE);
                cfs_schedule_timeout(CFS_TASK_INTERRUPTIBLE,
                                     cfs_time_seconds(1));
	}

        LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_STOPTHREAD);
        the_lnet.ln_rc_state = LNET_RC_STATE_UNLINKING;
        
        rc = LNetMDUnlink(mdh);
        LASSERT (rc == 0);