openiblnd.c

非常经典的一个分布式系统

#endif
        case IBNAL_INIT_PD:
                rc = ib_pd_destroy(kibnal_data.kib_pd);
                if (rc != 0)
                        CERROR ("Destroy PD error: %d\n", rc);
                /* fall through */

        case IBNAL_INIT_DATA:
                /* Module refcount only gets to zero when all peers
                 * have been closed so all lists must be empty */
                LASSERT (atomic_read(&kibnal_data.kib_npeers) == 0);
                LASSERT (kibnal_data.kib_peers != NULL);
                for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
                        LASSERT (list_empty (&kibnal_data.kib_peers[i]));
                }
                LASSERT (atomic_read (&kibnal_data.kib_nconns) == 0);
                LASSERT (list_empty (&kibnal_data.kib_sched_rxq));
                LASSERT (list_empty (&kibnal_data.kib_sched_txq));
                LASSERT (list_empty (&kibnal_data.kib_reaper_conns));
                LASSERT (list_empty (&kibnal_data.kib_connd_peers));
                LASSERT (list_empty (&kibnal_data.kib_connd_acceptq));

                /* flag threads to terminate; wake and wait for them to die */
                kibnal_data.kib_shutdown = 1;
                wake_up_all (&kibnal_data.kib_sched_waitq);
                wake_up_all (&kibnal_data.kib_reaper_waitq);
                wake_up_all (&kibnal_data.kib_connd_waitq);

                i = 2;
                while (atomic_read (&kibnal_data.kib_nthreads) != 0) {
                        i++;
                        CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
                               "Waiting for %d threads to terminate\n",
                               atomic_read (&kibnal_data.kib_nthreads));
                        cfs_pause(cfs_time_seconds(1));
                }
                /* fall through */
                
        case IBNAL_INIT_NOTHING:
                break;
        }

        if (kibnal_data.kib_tx_descs != NULL)
                LIBCFS_FREE (kibnal_data.kib_tx_descs,
                             IBNAL_TX_MSGS() * sizeof(kib_tx_t));

        if (kibnal_data.kib_peers != NULL)
                LIBCFS_FREE (kibnal_data.kib_peers,
                             sizeof (struct list_head) * 
                             kibnal_data.kib_peer_hash_size);

        CDEBUG(D_MALLOC, "after NAL cleanup: kmem %d\n",
               atomic_read (&libcfs_kmemory));

        kibnal_data.kib_init = IBNAL_INIT_NOTHING;
        PORTAL_MODULE_UNUSE;
}

int
kibnal_get_ipoibidx(void)
{
        /* NB single threaded! */
        static struct ib_port_properties port_props;

        int               ipoibidx = 0;
        int               devidx;
        int               port;
        int               rc;
        struct ib_device *device;

        for (devidx = 0; devidx <= kibnal_data.kib_hca_idx; devidx++) {
                device = ib_device_get_by_index(devidx);
                
                if (device == NULL) {
                        CERROR("Can't get IB device %d\n", devidx);
                        return -1;
                }
                
                for (port = 1; port <= 2; port++) {
                        if (devidx == kibnal_data.kib_hca_idx &&
                            port == kibnal_data.kib_port)
                                return ipoibidx;
                        
                        rc = ib_port_properties_get(device, port,
                                                    &port_props);
                        if (rc == 0)
                                ipoibidx++;
                }
        }

        LBUG();
        return -1;
}

int
kibnal_startup (lnet_ni_t *ni)
{
        char              ipif_name[32];
        __u32             ip;
        __u32             netmask;
        int               up;
        struct timeval    tv;
        int               rc;
        int               hca;
        int               port;
        int               i;
        int               nob;

        LASSERT (ni->ni_lnd == &the_kiblnd);

        /* Only 1 instance supported */
        if (kibnal_data.kib_init != IBNAL_INIT_NOTHING) {
                CERROR ("Only 1 instance supported\n");
                return -EPERM;
        }

        if (*kibnal_tunables.kib_credits > *kibnal_tunables.kib_ntx) {
                CERROR ("Can't set credits(%d) > ntx(%d)\n",
                        *kibnal_tunables.kib_credits,
                        *kibnal_tunables.kib_ntx);
                return -EINVAL;
        }

        memset (&kibnal_data, 0, sizeof (kibnal_data)); /* zero pointers, flags etc */

        ni->ni_maxtxcredits = *kibnal_tunables.kib_credits;
        ni->ni_peertxcredits = *kibnal_tunables.kib_peercredits;

        CLASSERT (LNET_MAX_INTERFACES > 1);


        kibnal_data.kib_hca_idx = 0;            /* default: first HCA */
        kibnal_data.kib_port = 0;               /* any port */

        if (ni->ni_interfaces[0] != NULL) {
                /* hca.port specified in 'networks=openib(h.p)' */
                if (ni->ni_interfaces[1] != NULL) {
                        CERROR("Multiple interfaces not supported\n");
                        return -EPERM;
                }
                
                nob = strlen(ni->ni_interfaces[0]);
                i = sscanf(ni->ni_interfaces[0], "%d.%d%n", &hca, &port, &nob);
                if (i >= 2 && nob == strlen(ni->ni_interfaces[0])) {
                        kibnal_data.kib_hca_idx = hca;
                        kibnal_data.kib_port = port;
                } else {
                        nob = strlen(ni->ni_interfaces[0]);
                        i = sscanf(ni->ni_interfaces[0], "%d%n", &hca, &nob);

                        if (i >= 1 && nob == strlen(ni->ni_interfaces[0])) {
                                kibnal_data.kib_hca_idx = hca;
                        } else {
                                CERROR("Can't parse interface '%s'\n",
                                       ni->ni_interfaces[0]);
                                return -EINVAL;
                        }
                }
        }
        
        kibnal_data.kib_ni = ni;
        ni->ni_data = &kibnal_data;
        
        do_gettimeofday(&tv);
        kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;

        PORTAL_MODULE_USE;

        rwlock_init(&kibnal_data.kib_global_lock);

        kibnal_data.kib_peer_hash_size = IBNAL_PEER_HASH_SIZE;
        LIBCFS_ALLOC (kibnal_data.kib_peers,
                      sizeof (struct list_head) * kibnal_data.kib_peer_hash_size);
        if (kibnal_data.kib_peers == NULL) {
                goto failed;
        }
        for (i = 0; i < kibnal_data.kib_peer_hash_size; i++)
                INIT_LIST_HEAD(&kibnal_data.kib_peers[i]);

        spin_lock_init (&kibnal_data.kib_reaper_lock);
        INIT_LIST_HEAD (&kibnal_data.kib_reaper_conns);
        init_waitqueue_head (&kibnal_data.kib_reaper_waitq);

        spin_lock_init (&kibnal_data.kib_connd_lock);
        INIT_LIST_HEAD (&kibnal_data.kib_connd_acceptq);
        INIT_LIST_HEAD (&kibnal_data.kib_connd_peers);
        init_waitqueue_head (&kibnal_data.kib_connd_waitq);

        spin_lock_init (&kibnal_data.kib_sched_lock);
        INIT_LIST_HEAD (&kibnal_data.kib_sched_txq);
        INIT_LIST_HEAD (&kibnal_data.kib_sched_rxq);
        init_waitqueue_head (&kibnal_data.kib_sched_waitq);

        spin_lock_init (&kibnal_data.kib_tx_lock);
        INIT_LIST_HEAD (&kibnal_data.kib_idle_txs);

        LIBCFS_ALLOC (kibnal_data.kib_tx_descs,
                      IBNAL_TX_MSGS() * sizeof(kib_tx_t));
        if (kibnal_data.kib_tx_descs == NULL) {
                CERROR ("Can't allocate tx descs\n");
                goto failed;
        }

        /* lists/ptrs/locks initialised */
        kibnal_data.kib_init = IBNAL_INIT_DATA;
        /*****************************************************/

        for (i = 0; i < IBNAL_N_SCHED; i++) {
                rc = kibnal_thread_start (kibnal_scheduler,
                                          (void *)((unsigned long)i));
                if (rc != 0) {
                        CERROR("Can't spawn openibnal scheduler[%d]: %d\n",
                               i, rc);
                        goto failed;
                }
        }

        /* must have at least 2 connds to remain responsive to svcqry while
         * connecting */
        if (*kibnal_tunables.kib_n_connd < 2)
                *kibnal_tunables.kib_n_connd = 2;


        for (i = 0; i < *kibnal_tunables.kib_n_connd; i++) {
                rc = kibnal_thread_start (kibnal_connd,
                                          (void *)((unsigned long)i));
                if (rc != 0) {
                        CERROR("Can't spawn openibnal connd[%d]: %d\n",
                               i, rc);
                        goto failed;
                }
        }

        rc = kibnal_thread_start (kibnal_reaper, NULL);
        if (rc != 0) {
                CERROR ("Can't spawn openibnal reaper: %d\n", rc);
                goto failed;
        }

        kibnal_data.kib_device = ib_device_get_by_index(kibnal_data.kib_hca_idx);
        if (kibnal_data.kib_device == NULL) {
                CERROR ("Can't open ib device %d\n",
                        kibnal_data.kib_hca_idx);
                goto failed;
        }
        
        rc = ib_device_properties_get(kibnal_data.kib_device,
                                      &kibnal_data.kib_device_props);
        if (rc != 0) {
                CERROR ("Can't get device props: %d\n", rc);
                goto failed;
        }

        CDEBUG(D_NET, "Max Initiator: %d Max Responder %d\n", 
               kibnal_data.kib_device_props.max_initiator_per_qp,
               kibnal_data.kib_device_props.max_responder_per_qp);

        if (kibnal_data.kib_port != 0) {
                rc = ib_port_properties_get(kibnal_data.kib_device, 
                                            kibnal_data.kib_port,
                                            &kibnal_data.kib_port_props);
                if (rc != 0) {
                        CERROR("Error %d open port %d on HCA %d\n", rc,
                               kibnal_data.kib_port,
                               kibnal_data.kib_hca_idx);
                        goto failed;
                }
        } else {
                for (i = 1; i <= 2; i++) {
                        rc = ib_port_properties_get(kibnal_data.kib_device, i,
                                                    &kibnal_data.kib_port_props);
                        if (rc == 0) {
                                kibnal_data.kib_port = i;
                                break;
                        }
                }
                if (kibnal_data.kib_port == 0) {
                        CERROR ("Can't find a port\n");
                        goto failed;
                }
        }

        i = kibnal_get_ipoibidx();
        if (i < 0)
                goto failed;
        
        snprintf(ipif_name, sizeof(ipif_name), "%s%d",
                 *kibnal_tunables.kib_ipif_basename, i);
        if (strlen(ipif_name) == sizeof(ipif_name) - 1) {
                CERROR("IPoIB interface name %s truncated\n", ipif_name);
                return -EINVAL;
        }
        
        rc = libcfs_ipif_query(ipif_name, &up, &ip, &netmask);
        if (rc != 0) {
                CERROR("Can't query IPoIB interface %s: %d\n", ipif_name, rc);
                goto failed;
        }
        
        if (!up) {
                CERROR("Can't query IPoIB interface %s: it's down\n", ipif_name);
                goto failed;
        }
        
        ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ip);

        rc = ib_pd_create(kibnal_data.kib_device,
                          NULL, &kibnal_data.kib_pd);
        if (rc != 0) {
                CERROR ("Can't create PD: %d\n", rc);
                goto failed;
        }
        
        /* flag PD initialised */
        kibnal_data.kib_init = IBNAL_INIT_PD;
        /*****************************************************/
#if IBNAL_FMR
        {
                const int pool_size = *kibnal_tunables.kib_ntx;
                struct ib_fmr_pool_param params = {
                        .max_pages_per_fmr = LNET_MAX_PAYLOAD/PAGE_SIZE,
                        .access            = (IB_ACCESS_LOCAL_WRITE |
                                              IB_ACCESS_REMOTE_WRITE |
                                              IB_ACCESS_REMOTE_READ),
                        .pool_size         = pool_size,
                        .dirty_watermark   = (pool_size * 3)/4,
                        .flush_function    = NULL,
                        .flush_arg         = NULL,
                        .cache             = 1,
                };
                rc = ib_fmr_pool_create(kibnal_data.kib_pd, &params,
                                        &kibnal_data.kib_fmr_pool);
                if (rc != 0) {
                        CERROR ("Can't create FMR pool size %d: %d\n", 
                                pool_size, rc);
                        goto failed;
                }
        }

        /* flag FMR pool initialised */
        kibnal_data.kib_init = IBNAL_INIT_FMR;
#endif
        /*****************************************************/

        rc = kibnal_setup_tx_descs();
        if (rc != 0) {
                CERROR ("Can't register tx descs: %d\n", rc);
                goto failed;
        }
        
        /* flag TX descs initialised */
        kibnal_data.kib_init = IBNAL_INIT_TXD;
        /*****************************************************/
        
        {
                struct ib_cq_callback callback = {
                        .context        = IBNAL_CALLBACK_CTXT,
                        .policy         = IB_CQ_PROVIDER_REARM,
                        .function       = {
                                .entry  = kibnal_callback,
                        },
                        .arg            = NULL,
                }