api-ni.c

lustre 1.6.5 source code

#endif /* defined(__KERNEL__) || defined(HAVE_LIBPTHREAD) */
        return count;
}

void
lnet_shutdown_lndnis (void)
{
        int                i;
        int                islo;
        lnet_ni_t         *ni;

        /* NB called holding the global mutex */

        /* All quiet on the API front */
        LASSERT (!the_lnet.ln_shutdown);
        LASSERT (the_lnet.ln_refcount == 0);
        LASSERT (list_empty(&the_lnet.ln_zombie_nis));
        LASSERT (the_lnet.ln_nzombie_nis == 0);
        LASSERT (list_empty(&the_lnet.ln_remote_nets));

        LNET_LOCK();
        the_lnet.ln_shutdown = 1;               /* flag shutdown */

        /* Unlink NIs from the global table */
        while (!list_empty(&the_lnet.ln_nis)) {
                ni = list_entry(the_lnet.ln_nis.next,
                                lnet_ni_t, ni_list);
                list_del (&ni->ni_list);

                the_lnet.ln_nzombie_nis++;
                lnet_ni_decref_locked(ni); /* drop apini's ref */
        }

        /* Drop the cached eqwait NI. */
        if (the_lnet.ln_eqwaitni != NULL) {
                lnet_ni_decref_locked(the_lnet.ln_eqwaitni);
                the_lnet.ln_eqwaitni = NULL;
        }

        /* Drop the cached loopback NI. */
        if (the_lnet.ln_loni != NULL) {
                lnet_ni_decref_locked(the_lnet.ln_loni);
                the_lnet.ln_loni = NULL;
        }

        LNET_UNLOCK();

        /* Clear lazy portals and drop delayed messages which hold refs
         * on their lnet_msg_t::msg_rxpeer */
        for (i = 0; i < the_lnet.ln_nportals; i++)
                LNetClearLazyPortal(i);

        /* Clear the peer table and wait for all peers to go (they hold refs on
         * their NIs) */
        lnet_clear_peer_table();

        LNET_LOCK();
        /* Now wait for the NI's I just nuked to show up on apini_zombie_nis
         * and shut them down in guaranteed thread context */
        i = 2;
        while (the_lnet.ln_nzombie_nis != 0) {

                while (list_empty(&the_lnet.ln_zombie_nis)) {
                        LNET_UNLOCK();
                        ++i;
                        if ((i & (-i)) == i)
                                CDEBUG(D_WARNING,"Waiting for %d zombie NIs\n",
                                       the_lnet.ln_nzombie_nis);
                        cfs_pause(cfs_time_seconds(1));
                        LNET_LOCK();
                }

                ni = list_entry(the_lnet.ln_zombie_nis.next,
                                lnet_ni_t, ni_list);
                list_del(&ni->ni_list);
                ni->ni_lnd->lnd_refcount--;

                LNET_UNLOCK();

                islo = ni->ni_lnd->lnd_type == LOLND;

                LASSERT (!in_interrupt ());
                (ni->ni_lnd->lnd_shutdown)(ni);

                /* can't deref lnd anymore now; it might have unregistered
                 * itself...  */

                if (!islo)
                        CDEBUG(D_LNI, "Removed LNI %s\n",
                               libcfs_nid2str(ni->ni_nid));

                LIBCFS_FREE(ni, sizeof(*ni));

                LNET_LOCK();
                the_lnet.ln_nzombie_nis--;
        }

        the_lnet.ln_shutdown = 0;
        LNET_UNLOCK();

        if (the_lnet.ln_network_tokens != NULL) {
                LIBCFS_FREE(the_lnet.ln_network_tokens,
                            the_lnet.ln_network_tokens_nob);
                the_lnet.ln_network_tokens = NULL;
        }
}

int
lnet_startup_lndnis (void)
{
        lnd_t             *lnd;
        lnet_ni_t         *ni;
        struct list_head   nilist;
        int                rc = 0;
        int                lnd_type;
        int                nicount = 0;
        char              *nets = lnet_get_networks();

        CFS_INIT_LIST_HEAD(&nilist);

        if (nets == NULL)
                goto failed;

        rc = lnet_parse_networks(&nilist, nets);
        if (rc != 0)
                goto failed;

        while (!list_empty(&nilist)) {
                ni = list_entry(nilist.next, lnet_ni_t, ni_list);
                lnd_type = LNET_NETTYP(LNET_NIDNET(ni->ni_nid));

                LASSERT (libcfs_isknown_lnd(lnd_type));

                LNET_MUTEX_DOWN(&the_lnet.ln_lnd_mutex);
                lnd = lnet_find_lnd_by_type(lnd_type);

#ifdef __KERNEL__
                if (lnd == NULL) {
                        LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
                        rc = request_module(libcfs_lnd2modname(lnd_type));
                        LNET_MUTEX_DOWN(&the_lnet.ln_lnd_mutex);

                        lnd = lnet_find_lnd_by_type(lnd_type);
                        if (lnd == NULL) {
                                LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
                                CERROR("Can't load LND %s, module %s, rc=%d\n",
                                       libcfs_lnd2str(lnd_type),
                                       libcfs_lnd2modname(lnd_type), rc);
#ifndef CONFIG_KMOD
                                LCONSOLE_ERROR_MSG(0x104, "Your kernel must be "
                                         "compiled with CONFIG_KMOD set for "
                                         "automatic module loading.");
#endif
                                goto failed;
                        }
                }
#else
                if (lnd == NULL) {
                        LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
                        CERROR("LND %s not supported\n",
                               libcfs_lnd2str(lnd_type));
                        goto failed;
                }
#endif

                ni->ni_refcount = 1;

                LNET_LOCK();
                lnd->lnd_refcount++;
                LNET_UNLOCK();

                ni->ni_lnd = lnd;

                rc = (lnd->lnd_startup)(ni);

                LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);

                if (rc != 0) {
                        LCONSOLE_ERROR_MSG(0x105, "Error %d starting up LNI %s"
                                           "\n",
                                           rc, libcfs_lnd2str(lnd->lnd_type));
                        LNET_LOCK();
                        lnd->lnd_refcount--;
                        LNET_UNLOCK();
                        goto failed;
                }

                list_del(&ni->ni_list);

                LNET_LOCK();
                list_add_tail(&ni->ni_list, &the_lnet.ln_nis);
                LNET_UNLOCK();

                if (lnd->lnd_type == LOLND) {
                        lnet_ni_addref(ni);
                        LASSERT (the_lnet.ln_loni == NULL);
                        the_lnet.ln_loni = ni;
                        continue;
                }

#ifndef __KERNEL__
                if (lnd->lnd_wait != NULL) {
                        if (the_lnet.ln_eqwaitni == NULL) {
                                lnet_ni_addref(ni);
                                the_lnet.ln_eqwaitni = ni;
                        }
                } else {
# ifndef HAVE_LIBPTHREAD
                        LCONSOLE_ERROR_MSG(0x106, "LND %s not supported in a "
                                           "single-threaded runtime\n",
                                           libcfs_lnd2str(lnd_type));
                        goto failed;
# endif
                }
#endif
                if (ni->ni_peertxcredits == 0 ||
                    ni->ni_maxtxcredits == 0) {
                        LCONSOLE_ERROR_MSG(0x107, "LNI %s has no %scredits\n",
                                           libcfs_lnd2str(lnd->lnd_type),
                                           ni->ni_peertxcredits == 0 ?
                                           "" : "per-peer ");
                        goto failed;
                }

                ni->ni_txcredits = ni->ni_mintxcredits = ni->ni_maxtxcredits;

                CDEBUG(D_LNI, "Added LNI %s [%d/%d]\n",
                       libcfs_nid2str(ni->ni_nid),
                       ni->ni_peertxcredits, ni->ni_txcredits);

                /* Handle nidstrings for network 0 just like this one */
                if (the_lnet.ln_ptlcompat > 0) {
                        if (nicount > 0) {
                                LCONSOLE_ERROR_MSG(0x108, "Can't run > 1 "
                                       "network when portals_compatibility is "
                                       "set\n");
                                goto failed;
                        }
                        libcfs_setnet0alias(lnd->lnd_type);
                }
                
                nicount++;
        }

        if (the_lnet.ln_eqwaitni != NULL && nicount > 1) {
                lnd_type = the_lnet.ln_eqwaitni->ni_lnd->lnd_type;
                LCONSOLE_ERROR_MSG(0x109, "LND %s can only run single-network"
                                   "\n",
                                   libcfs_lnd2str(lnd_type));
                goto failed;
        }

        return 0;

 failed:
        lnet_shutdown_lndnis();

        while (!list_empty(&nilist)) {
                ni = list_entry(nilist.next, lnet_ni_t, ni_list);
                list_del(&ni->ni_list);
                LIBCFS_FREE(ni, sizeof(*ni));
        }

        return -ENETDOWN;
}

int
LNetInit(void)
{
        int    rc;

        lnet_assert_wire_constants ();
        LASSERT (!the_lnet.ln_init);

        memset(&the_lnet, 0, sizeof(the_lnet));

        rc = lnet_get_portals_compatibility();
        if (rc < 0)
                return rc;

        lnet_init_locks();
        CFS_INIT_LIST_HEAD(&the_lnet.ln_lnds);
        the_lnet.ln_ptlcompat = rc;
        the_lnet.ln_refcount = 0;
        the_lnet.ln_init = 1;

#ifdef __KERNEL__
        /* All LNDs apart from the LOLND are in separate modules.  They
         * register themselves when their module loads, and unregister
         * themselves when their module is unloaded. */
#else
        /* Register LNDs
         * NB the order here determines default 'networks=' order */
# ifdef CRAY_XT3
        LNET_REGISTER_ULND(the_ptllnd);
# endif
# ifdef HAVE_LIBPTHREAD
        LNET_REGISTER_ULND(the_tcplnd);
# endif
#endif
        lnet_register_lnd(&the_lolnd);
        return 0;
}

void
LNetFini(void)
{
        LASSERT (the_lnet.ln_init);
        LASSERT (the_lnet.ln_refcount == 0);

        while (!list_empty(&the_lnet.ln_lnds))
                lnet_unregister_lnd(list_entry(the_lnet.ln_lnds.next,
                                               lnd_t, lnd_list));
        lnet_fini_locks();

        the_lnet.ln_init = 0;
}

int
LNetNIInit(lnet_pid_t requested_pid)
{
        int         im_a_router = 0;
        int         rc;

        LNET_MUTEX_DOWN(&the_lnet.ln_api_mutex);

        LASSERT (the_lnet.ln_init);
        CDEBUG(D_OTHER, "refs %d\n", the_lnet.ln_refcount);

        if (the_lnet.ln_refcount > 0) {
                rc = the_lnet.ln_refcount++;
                goto out;
        }

        if (requested_pid == LNET_PID_ANY) {
                /* Don't instantiate LNET just for me */
                rc = -ENETDOWN;
                goto failed0;
        }

        rc = lnet_prepare(requested_pid);
        if (rc != 0)
                goto failed0;

        rc = lnet_startup_lndnis();
        if (rc != 0)
                goto failed1;

        rc = lnet_parse_routes(lnet_get_routes(), &im_a_router);
        if (rc != 0)
                goto failed2;

        rc = lnet_check_routes();
        if (rc != 0)
                goto failed2;

        rc = lnet_alloc_rtrpools(im_a_router);
        if (rc != 0)
                goto failed2;

        rc = lnet_acceptor_start();
        if (rc != 0)
                goto failed2;

        the_lnet.ln_refcount = 1;
        /* Now I may use my own API functions... */

        rc = lnet_router_checker_start();
        if (rc != 0)
                goto failed3;

        rc = lnet_ping_target_init();
        if (rc != 0)
                goto failed4;

        lnet_proc_init();
        goto out;

 failed4:
        lnet_router_checker_stop();
 failed3:
        the_lnet.ln_refcount = 0;
        lnet_acceptor_stop();
 failed2:
        lnet_destroy_routes();
        lnet_shutdown_lndnis();
 failed1:
        lnet_unprepare();
 failed0:
        LASSERT (rc < 0);
 out:
        LNET_MUTEX_UP(&the_lnet.ln_api_mutex);
        return rc;
}

int
LNetNIFini()
{
        LNET_MUTEX_DOWN(&the_lnet.ln_api_mutex);

        LASSERT (the_lnet.ln_init);
        LASSERT (the_lnet.ln_refcount > 0);

        if (the_lnet.ln_refcount != 1) {
                the_lnet.ln_refcount--;
        } else {
                LASSERT (!the_lnet.ln_niinit_self);

                lnet_proc_fini();
                lnet_ping_target_fini();
                lnet_router_checker_stop();

                /* Teardown fns that use my own API functions BEFORE here */
                the_lnet.ln_refcount = 0;

                lnet_acceptor_stop();
                lnet_destroy_routes();
                lnet_shutdown_lndnis();
                lnet_unprepare();
        }

        LNET_MUTEX_UP(&the_lnet.ln_api_mutex);
        return 0;
}

int
LNetCtl(unsigned int cmd, void *arg)
{
        struct libcfs_ioctl_data *data = arg;
        lnet_process_id_t         id;
        lnet_ni_t                *ni;
        int                       rc;

        LASSERT (the_lnet.ln_init);
        LASSERT (the_lnet.ln_refcount > 0);

        switch (cmd) {
        case IOC_LIBCFS_GET_NI:
                rc = LNetGetId(data->ioc_count, &id);
                data->ioc_nid = id.nid;