socklnd_lib-linux.c

非常经典的一个分布式系统

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 */

#include "socklnd.h"

# if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM
static cfs_sysctl_table_t ksocknal_ctl_table[21];

cfs_sysctl_table_t ksocknal_top_ctl_table[] = {
        {
                .ctl_name = 200,
                .procname = "socknal",
                .data     = NULL,
                .maxlen   = 0,
                .mode     = 0555,
                .child    = ksocknal_ctl_table
        },
        { 0 }
};

int
ksocknal_lib_tunables_init ()
{
        int    i = 0;
        int    j = 1;

        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "timeout",
                .data     = ksocknal_tunables.ksnd_timeout,
                .maxlen   = sizeof (int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "credits",
                .data     = ksocknal_tunables.ksnd_credits,
                .maxlen   = sizeof (int),
                .mode     = 0444,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "peer_credits",
                .data     = ksocknal_tunables.ksnd_peercredits,
                .maxlen   = sizeof (int),
                .mode     = 0444,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "nconnds",
                .data     = ksocknal_tunables.ksnd_nconnds,
                .maxlen   = sizeof (int),
                .mode     = 0444,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "min_reconnectms",
                .data     = ksocknal_tunables.ksnd_min_reconnectms,
                .maxlen   = sizeof (int),
                .mode     = 0444,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "max_reconnectms",
                .data     = ksocknal_tunables.ksnd_max_reconnectms,
                .maxlen   = sizeof (int),
                .mode     = 0444,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "eager_ack",
                .data     = ksocknal_tunables.ksnd_eager_ack,
                .maxlen   = sizeof (int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "zero_copy",
                .data     = ksocknal_tunables.ksnd_zc_min_frag,
                .maxlen   = sizeof (int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "typed",
                .data     = ksocknal_tunables.ksnd_typed_conns,
                .maxlen   = sizeof (int),
                .mode     = 0444,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "min_bulk",
                .data     = ksocknal_tunables.ksnd_min_bulk,
                .maxlen   = sizeof (int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "rx_buffer_size",
                .data     = ksocknal_tunables.ksnd_rx_buffer_size,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "tx_buffer_size",
                .data     = ksocknal_tunables.ksnd_tx_buffer_size,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "nagle",
                .data     = ksocknal_tunables.ksnd_nagle,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
#ifdef CPU_AFFINITY
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "irq_affinity",
                .data     = ksocknal_tunables.ksnd_irq_affinity,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
#endif
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "keepalive_idle",
                .data     = ksocknal_tunables.ksnd_keepalive_idle,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "keepalive_count",
                .data     = ksocknal_tunables.ksnd_keepalive_count,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "keepalive_intvl",
                .data     = ksocknal_tunables.ksnd_keepalive_intvl,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
#ifdef SOCKNAL_BACKOFF
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "backoff_init",
                .data     = ksocknal_tunables.ksnd_backoff_init,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "backoff_max",
                .data     = ksocknal_tunables.ksnd_backoff_max,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
#endif
#if SOCKNAL_VERSION_DEBUG
        ksocknal_ctl_table[i++] = (cfs_sysctl_table_t) {
                .ctl_name = j++,
                .procname = "protocol",
                .data     = ksocknal_tunables.ksnd_protocol,
                .maxlen   = sizeof(int),
                .mode     = 0644,
                .proc_handler = &proc_dointvec
        };
#endif
        ksocknal_ctl_table[i++] =  (cfs_sysctl_table_t) { 0 };

        LASSERT (j == i);
        LASSERT (i <= sizeof(ksocknal_ctl_table)/sizeof(ksocknal_ctl_table[0]));

        ksocknal_tunables.ksnd_sysctl =
                cfs_register_sysctl_table(ksocknal_top_ctl_table, 0);

        if (ksocknal_tunables.ksnd_sysctl == NULL)
                CWARN("Can't setup /proc tunables\n");

        return 0;
}

void
ksocknal_lib_tunables_fini ()
{
        if (ksocknal_tunables.ksnd_sysctl != NULL)
                cfs_unregister_sysctl_table(ksocknal_tunables.ksnd_sysctl);
}
#else
int
ksocknal_lib_tunables_init ()
{
        return 0;
}

void
ksocknal_lib_tunables_fini ()
{
}
#endif /* # if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM */

void
ksocknal_lib_bind_irq (unsigned int irq)
{
#if (defined(CONFIG_SMP) && defined(CPU_AFFINITY))
        int              bind;
        int              cpu;
        char             cmdline[64];
        ksock_irqinfo_t *info;
        char            *argv[] = {"/bin/sh",
                                   "-c",
                                   cmdline,
                                   NULL};
        char            *envp[] = {"HOME=/",
                                   "PATH=/sbin:/bin:/usr/sbin:/usr/bin",
                                   NULL};

        LASSERT (irq < NR_IRQS);
        if (irq == 0)              /* software NIC or affinity disabled */
                return;

        info = &ksocknal_data.ksnd_irqinfo[irq];

        write_lock_bh (&ksocknal_data.ksnd_global_lock);

        LASSERT (info->ksni_valid);
        bind = !info->ksni_bound;
        info->ksni_bound = 1;

        write_unlock_bh (&ksocknal_data.ksnd_global_lock);

        if (!bind)                              /* bound already */
                return;

        cpu = ksocknal_irqsched2cpu(info->ksni_sched);
        snprintf (cmdline, sizeof (cmdline),
                  "echo %d > /proc/irq/%u/smp_affinity", 1 << cpu, irq);

        LCONSOLE_INFO("Binding irq %u to CPU %d with cmd: %s\n",
                      irq, cpu, cmdline);

        /* FIXME: Find a better method of setting IRQ affinity...
         */

        USERMODEHELPER(argv[0], argv, envp);
#endif
}

int
ksocknal_lib_get_conn_addrs (ksock_conn_t *conn)
{
        int rc = libcfs_sock_getaddr(conn->ksnc_sock, 1,
                                     &conn->ksnc_ipaddr,
                                     &conn->ksnc_port);

        /* Didn't need the {get,put}connsock dance to deref ksnc_sock... */
        LASSERT (!conn->ksnc_closing);

        if (rc != 0) {
                CERROR ("Error %d getting sock peer IP\n", rc);
                return rc;
        }

        rc = libcfs_sock_getaddr(conn->ksnc_sock, 0,
                                 &conn->ksnc_myipaddr, NULL);
        if (rc != 0) {
                CERROR ("Error %d getting sock local IP\n", rc);
                return rc;
        }

        return 0;
}

unsigned int
ksocknal_lib_sock_irq (struct socket *sock)
{
        int                irq = 0;
#ifdef CPU_AFFINITY
        struct dst_entry  *dst;

        if (!*ksocknal_tunables.ksnd_irq_affinity)
                return 0;

        dst = sk_dst_get (sock->sk);
        if (dst != NULL) {
                if (dst->dev != NULL) {
                        irq = dst->dev->irq;
                        if (irq >= NR_IRQS) {
                                CERROR ("Unexpected IRQ %x\n", irq);
                                irq = 0;
                        }
                }
                dst_release (dst);
        }

#endif
        return irq;
}

int
ksocknal_lib_zc_capable(struct socket *sock)
{
        int  caps = sock->sk->sk_route_caps;

        /* ZC if the socket supports scatter/gather and doesn't need software
         * checksums */
        return ((caps & NETIF_F_SG) != 0 &&
                (caps & (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM)) != 0);
}