mxlnd.h

非常经典的一个分布式系统

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 *
 * Copyright (C) 2004 Cluster File Systems, Inc.
 *   Author: Eric Barton <[e-mail]>
 * Copyright (C) 2006 Myricom, Inc.
 *   Author: Scott Atchley <atchley at myri.com>
 *
 *   This file is part of Lustre, http://www.lustre.org.
 *
 *   Lustre is free software; you can redistribute it and/or
 *   modify it under the terms of version 2 of the GNU General Public
 *   License as published by the Free Software Foundation.
 *
 *   Lustre is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with Lustre; if not, write to the Free Software
 *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#ifndef EXPORT_SYMTAB
#define EXPORT_SYMTAB
#endif
#ifndef AUTOCONF_INCLUDED
#include <linux/config.h>
#endif
#include <linux/module.h>       /* module */
#include <linux/kernel.h>       /* module */
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/smp_lock.h>
#include <linux/unistd.h>
#include <linux/uio.h>

#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/io.h>

#include <linux/init.h>         /* module */
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/stat.h>
#include <linux/list.h>
#include <linux/kmod.h>
#include <linux/sysctl.h>
#include <linux/random.h>
#include <linux/utsname.h>

#include <net/sock.h>
#include <linux/in.h>

#include <linux/netdevice.h>    /* these are needed for ARP */
#include <linux/if_arp.h>
#include <net/arp.h>
#include <linux/inetdevice.h>

#define DEBUG_SUBSYSTEM S_LND

#include "libcfs/kp30.h"
#include "lnet/lnet.h"
#include "lnet/lib-lnet.h"

#define MX_KERNEL 1
#include "mx_extensions.h"
#include "myriexpress.h"

#if LNET_MAX_IOV > MX_MAX_SEGMENTS
    #error LNET_MAX_IOV is greater then MX_MAX_SEGMENTS
#endif

/* Using MX's 64 match bits
 * We are using the match bits to specify message type and the cookie.  The
 * highest four bits (60-63) are reserved for message type. Below we specify
 * the types. MXLND_MASK_ICON_REQ and MXLND_MASK_ICON_ACK are used for
 * mx_iconnect().  We reserve the remaining combinations for future use.  The
 * next 8 bits (52-59) are reserved for returning a status code for failed
 * GET_DATA (payload) messages. The last 52 bits are used for cookies. That
 * should allow unique cookies for 4 KB messages at 10 Gbps line rate without
 * rollover for about 8 years. That should be enough. */

/* constants */
#define MXLND_MASK_ICON_REQ (0xBLL << 60) /* it is a mx_iconnect() completion */
#define MXLND_MASK_CONN_REQ (0xCLL << 60) /* CONN_REQ msg */
#define MXLND_MASK_ICON_ACK (0x9LL << 60) /* it is a mx_iconnect() completion */
#define MXLND_MASK_CONN_ACK (0xALL << 60) /* CONN_ACK msg*/
#define MXLND_MASK_EAGER    (0xELL << 60) /* EAGER msg */
#define MXLND_MASK_NOOP     (0x1LL << 60) /* NOOP msg */
#define MXLND_MASK_PUT_REQ  (0x2LL << 60) /* PUT_REQ msg */
#define MXLND_MASK_PUT_ACK  (0x3LL << 60) /* PUT_ACK msg */
#define MXLND_MASK_PUT_DATA (0x4LL << 60) /* PUT_DATA msg */
#define MXLND_MASK_GET_REQ  (0x5LL << 60) /* GET_REQ msg */
#define MXLND_MASK_GET_DATA (0x6LL << 60) /* GET_DATA msg */
//#define MXLND_MASK_NAK      (0x7LL << 60) /* NAK msg */

#define MXLND_MAX_COOKIE    ((1LL << 52) - 1)         /* when to roll-over the cookie value */
#define MXLND_NCOMPLETIONS  (MXLND_N_SCHED + 2)   /* max threads for completion array */

/* defaults for configurable parameters */
#define MXLND_N_SCHED           1               /* # schedulers (mx_wait_any() threads) */
#define MXLND_MX_BOARD          0               /* Use the first MX NIC if more than 1 avail */
#define MXLND_MX_EP_ID          3               /* MX endpoint ID */
#define MXLND_COMM_TIMEOUT      (20 * HZ)       /* timeout for send/recv (jiffies) */
#define MXLND_WAIT_TIMEOUT      HZ              /* timeout for wait (jiffies) */
#define MXLND_POLLING           0               /* poll iterations before blocking */
#define MXLND_MAX_PEERS         1024            /* number of nodes talking to me */
#define MXLND_EAGER_NUM         MXLND_MAX_PEERS /* number of pre-posted receives */
#define MXLND_EAGER_SIZE        PAGE_SIZE       /* pre-posted eager message size */
#define MXLND_MSG_QUEUE_DEPTH   8               /* msg queue depth */
#define MXLND_CREDIT_HIGHWATER  (MXLND_MSG_QUEUE_DEPTH - 2)
                                                /* when to send a noop to return credits */
#define MXLND_NTX               256             /* # of kmx_tx - total sends in flight 
                                                   1/2 are reserved for connect messages */

#define MXLND_HASH_BITS         6               /* the number of bits to hash over */
#define MXLND_HASH_SIZE         (1<<MXLND_HASH_BITS)
                                                /* number of peer lists for lookup.
                                                   we hash over the last N bits of
                                                   the IP address converted to an int. */
#define MXLND_HASH_MASK         (MXLND_HASH_SIZE - 1)
                                                /* ensure we use only the last N bits */

/* debugging features */
#define MXLND_CKSUM             0               /* checksum kmx_msg_t */
#define MXLND_DEBUG             0               /* turn on printk()s */

extern inline void mxlnd_noop(char *s, ...);
#if MXLND_DEBUG
        #define MXLND_PRINT printk
#else
        #define MXLND_PRINT mxlnd_noop
#endif

/* provide wrappers around LIBCFS_ALLOC/FREE to keep MXLND specific
 * memory usage stats that include pages */

#define MXLND_ALLOC(x, size) \
        do { \
                spin_lock(&kmxlnd_data.kmx_global_lock); \
                kmxlnd_data.kmx_mem_used += size; \
                spin_unlock(&kmxlnd_data.kmx_global_lock); \
                LIBCFS_ALLOC(x, size); \
                if (x == NULL) { \
                        spin_lock(&kmxlnd_data.kmx_global_lock); \
                        kmxlnd_data.kmx_mem_used -= size; \
                        spin_unlock(&kmxlnd_data.kmx_global_lock); \
                } \
        } while (0)

#define MXLND_FREE(x, size) \
        do { \
                spin_lock(&kmxlnd_data.kmx_global_lock); \
                kmxlnd_data.kmx_mem_used -= size; \
                spin_unlock(&kmxlnd_data.kmx_global_lock); \
                LIBCFS_FREE(x, size); \
        } while (0)


typedef struct kmx_tunables {
        int     *kmx_n_waitd;           /* # completion threads */
        int     *kmx_max_peers;         /* max # of potential peers */
        int     *kmx_cksum;             /* checksum small msgs? */
        int     *kmx_ntx;               /* total # of tx (1/2 for LNET 1/2 for CONN_REQ */
        int     *kmx_credits;           /* concurrent sends to 1 peer */
        int     *kmx_board;             /* MX board (NIC) number */
        int     *kmx_ep_id;             /* MX endpoint number */
        int     *kmx_polling;           /* if 0, block. if > 0, poll this many
                                           iterations before blocking */
        char    **kmx_hosts;            /* Location of hosts file, if used */
} kmx_tunables_t;

/* structure to hold IP-to-hostname resolution data */
struct kmx_host {
        struct kmx_peer    *mxh_peer;           /* pointer to matching peer */
        u32                 mxh_addr;           /* IP address as int */
        char               *mxh_hostname;       /* peer's hostname */
        u32                 mxh_board;          /* peer's board rank */
        u32                 mxh_ep_id;          /* peer's MX endpoint ID */
        struct list_head    mxh_list;           /* position on kmx_hosts */
        spinlock_t          mxh_lock;           /* lock */
};

/* global interface state */
typedef struct kmx_data
{
        int                 kmx_init;           /* initialization state */
        int                 kmx_shutdown;       /* shutting down? */
        atomic_t            kmx_nthreads;       /* number of threads */
        struct completion  *kmx_completions;    /* array of completion structs */
        lnet_ni_t          *kmx_ni;             /* the LND instance */
        u64                 kmx_incarnation;    /* my incarnation value - unused */
        long                kmx_mem_used;       /* memory used */
        struct kmx_host    *kmx_localhost;      /* pointer to my kmx_host info */
        mx_endpoint_t       kmx_endpt;          /* the MX endpoint */

        spinlock_t          kmx_global_lock;    /* global lock */

        struct list_head    kmx_conn_req;       /* list of connection requests */
        spinlock_t          kmx_conn_lock;      /* connection list lock */
        struct semaphore    kmx_conn_sem;       /* semaphore for connection request list */

        struct list_head    kmx_hosts;          /* host lookup info */
        spinlock_t          kmx_hosts_lock;     /* hosts list lock */