iiblnd.h

非常经典的一个分布式系统

typedef struct
{
	__u64             rd_addr;             	/* IO VMA address */
	__u32             rd_nob;              	/* # of bytes */
	__u32             rd_key;		/* remote key */
} WIRE_ATTR kib_rdma_desc_t;
#else
typedef struct
{
        __u32             rf_nob;               /* # of bytes */
        __u64             rf_addr;              /* remote io vaddr */
} WIRE_ATTR kib_rdma_frag_t;

typedef struct
{
        __u32             rd_key;               /* local/remote key */
        __u32             rd_nfrag;             /* # fragments */
        kib_rdma_frag_t   rd_frags[0];          /* buffer frags */
} WIRE_ATTR kib_rdma_desc_t;
#endif

typedef struct
{
        lnet_hdr_t        ibprm_hdr;            /* LNET header */
        __u64             ibprm_cookie;         /* opaque completion cookie */
} WIRE_ATTR kib_putreq_msg_t;

typedef struct
{
        __u64             ibpam_src_cookie;     /* reflected completion cookie */
        __u64             ibpam_dst_cookie;     /* opaque completion cookie */
        kib_rdma_desc_t   ibpam_rd;             /* sender's sink buffer */
} WIRE_ATTR kib_putack_msg_t;

typedef struct
{
        lnet_hdr_t        ibgm_hdr;             /* LNET header */
        __u64             ibgm_cookie;          /* opaque completion cookie */
        kib_rdma_desc_t   ibgm_rd;              /* sender's sink buffer */
} WIRE_ATTR kib_get_msg_t;

typedef struct
{
        __u64             ibcm_cookie;          /* opaque completion cookie */
        __u32             ibcm_status;          /* completion status */
} WIRE_ATTR kib_completion_msg_t;

typedef struct
{
        /* First 2 fields fixed FOR ALL TIME */
        __u32             ibm_magic;            /* I'm an openibnal message */
        __u16             ibm_version;          /* this is my version number */

        __u8              ibm_type;             /* msg type */
        __u8              ibm_credits;          /* returned credits */
        __u32             ibm_nob;              /* # bytes in whole message */
        __u32             ibm_cksum;            /* checksum (0 == no checksum) */
        __u64             ibm_srcnid;           /* sender's NID */
        __u64             ibm_srcstamp;         /* sender's incarnation */
        __u64             ibm_dstnid;           /* destination's NID */
        __u64             ibm_dststamp;         /* destination's incarnation */
        __u64             ibm_seq;              /* sequence number */

        union {
                kib_connparams_t      connparams;
                kib_immediate_msg_t   immediate;
                kib_putreq_msg_t      putreq;
                kib_putack_msg_t      putack;
                kib_get_msg_t         get;
                kib_completion_msg_t  completion;
        } WIRE_ATTR ibm_u;
} WIRE_ATTR kib_msg_t;

#define IBNAL_MSG_MAGIC LNET_PROTO_IIB_MAGIC    /* unique magic */
#define IBNAL_MSG_VERSION              2        /* current protocol version */
#define IBNAL_MSG_VERSION_RDMAREPLYNOTRSRVD 1   /* previous version */

#define IBNAL_MSG_CONNREQ           0xc0        /* connection request */
#define IBNAL_MSG_CONNACK           0xc1        /* connection acknowledge */
#define IBNAL_MSG_NOOP              0xd0        /* nothing (just credits) */
#define IBNAL_MSG_IMMEDIATE         0xd1        /* immediate */
#define IBNAL_MSG_PUT_REQ           0xd2        /* putreq (src->sink) */
#define IBNAL_MSG_PUT_NAK           0xd3        /* completion (sink->src) */
#define IBNAL_MSG_PUT_ACK           0xd4        /* putack (sink->src) */
#define IBNAL_MSG_PUT_DONE          0xd5        /* completion (src->sink) */
#define IBNAL_MSG_GET_REQ           0xd6        /* getreq (sink->src) */
#define IBNAL_MSG_GET_DONE          0xd7        /* completion (src->sink: all OK) */

/* connection rejection reasons */
#define IBNAL_REJECT_CONN_RACE       0          /* You lost connection race */
#define IBNAL_REJECT_NO_RESOURCES    1          /* Out of memory/conns etc */
#define IBNAL_REJECT_FATAL           2          /* Anything else */

/***********************************************************************/

typedef struct kib_rx                           /* receive message */
{
        struct list_head          rx_list;      /* queue for attention */
        struct kib_conn          *rx_conn;      /* owning conn */
        int                       rx_nob;       /* # bytes received (-1 while posted) */
        __u64                     rx_hca_msg;   /* pre-mapped buffer (hca vaddr) */
        kib_msg_t                *rx_msg;       /* pre-mapped buffer (host vaddr) */
        IB_WORK_REQ2              rx_wrq;
        IB_LOCAL_DATASEGMENT      rx_gl;        /* and its memory */
} kib_rx_t;

typedef struct kib_tx                           /* transmit message */
{
        struct list_head          tx_list;      /* queue on idle_txs ibc_tx_queue etc. */
        struct kib_conn          *tx_conn;      /* owning conn */
        int                       tx_mapped;    /* mapped for RDMA? */
        int                       tx_sending;   /* # tx callbacks outstanding */
        int                       tx_queued;    /* queued for sending */
        int                       tx_waiting;   /* waiting for peer */
        int                       tx_status;    /* completion status */
        unsigned long             tx_deadline;  /* completion deadline */
        __u64                     tx_cookie;    /* completion cookie */
        lnet_msg_t               *tx_lntmsg[2]; /* lnet msgs to finalize on completion */
        kib_msg_t                *tx_msg;       /* pre-mapped buffer (host vaddr) */
        __u64                     tx_hca_msg;   /* pre-mapped buffer (HCA vaddr) */
        int                       tx_nwrq;      /* # send work items */
#if IBNAL_USE_FMR
        IB_WORK_REQ2              tx_wrq[2];    /* send work items... */
        IB_LOCAL_DATASEGMENT      tx_gl[2];     /* ...and their memory */
        kib_rdma_desc_t           tx_rd[1];     /* rdma descriptor */
        kib_md_t                  tx_md;        /* mapping */
        __u64                    *tx_pages;     /* page phys addrs */
#else
        IB_WORK_REQ2             *tx_wrq;       /* send work items... */
        IB_LOCAL_DATASEGMENT     *tx_gl;        /* ...and their memory */
        kib_rdma_desc_t          *tx_rd;        /* rdma descriptor (src buffers) */
#endif
} kib_tx_t;

typedef struct
{
        /* scratchpad during connection establishment */
        IB_QP_ATTRIBUTES_QUERY cv_qpattrs;
        QUERY                  cv_query;
        IB_SERVICE_RECORD      cv_svcrec;
        IB_PATH_RECORD         cv_path;
        CM_CONN_INFO           cv_cmci;
} kib_connvars_t;

typedef struct kib_conn
{
        struct kib_peer    *ibc_peer;           /* owning peer */
        struct list_head    ibc_list;           /* stash on peer's conn list */
        __u64               ibc_incarnation;    /* which instance of the peer */
        __u64               ibc_txseq;          /* tx sequence number */
        __u64               ibc_rxseq;          /* rx sequence number */
        __u32               ibc_version;        /* peer protocol version */
        atomic_t            ibc_refcount;       /* # users */
        int                 ibc_state;          /* what's happening */
        int                 ibc_nsends_posted;  /* # uncompleted sends */
        int                 ibc_credits;        /* # credits I have */
        int                 ibc_outstanding_credits; /* # credits to return */
        int                 ibc_reserved_credits; /* # credits for ACK/DONE msgs */
        unsigned long       ibc_last_send;      /* time of last send */
        struct list_head    ibc_early_rxs;      /* rxs completed before ESTABLISHED */
        struct list_head    ibc_tx_queue_nocred; /* sends that don't need a cred */
        struct list_head    ibc_tx_queue_rsrvd; /* sends that need a reserved cred */
        struct list_head    ibc_tx_queue;       /* send queue */
        struct list_head    ibc_active_txs;     /* active tx awaiting completion */
        spinlock_t          ibc_lock;           /* serialise */
        kib_rx_t           *ibc_rxs;            /* the rx descs */
        kib_pages_t        *ibc_rx_pages;       /* premapped rx msg pages */
        IB_HANDLE           ibc_qp;             /* queue pair */
        IB_HANDLE           ibc_cep;            /* CM endpoint */
        kib_connvars_t     *ibc_cvars;          /* connection scratchpad */
} kib_conn_t;

#define IBNAL_CONN_INIT_NOTHING      0          /* initial state */
#define IBNAL_CONN_INIT_QP           1          /* ibc_qp set up */
#define IBNAL_CONN_CONNECTING        2          /* started to connect */
#define IBNAL_CONN_ESTABLISHED       3          /* connection established */
#define IBNAL_CONN_DISCONNECTING     4          /* to send disconnect req */
#define IBNAL_CONN_DISCONNECTED      5          /* no more QP or CM traffic */

/* types of connection */
#define IBNAL_CONN_ACTIVE            0          /* active connect */
#define IBNAL_CONN_PASSIVE           1          /* passive connect */
#define IBNAL_CONN_WAITING           2          /* waiting for connect */

typedef struct kib_peer
{
        struct list_head    ibp_list;           /* stash on global peer list */
        struct list_head    ibp_connd_list;     /* schedule on kib_connd_peers */
        lnet_nid_t          ibp_nid;            /* who's on the other end(s) */
        atomic_t            ibp_refcount;       /* # users */
        int                 ibp_persistence;    /* "known" peer refs */
        int                 ibp_version;        /* protocol version */
        struct list_head    ibp_conns;          /* all active connections */
        struct list_head    ibp_tx_queue;       /* msgs waiting for a conn */
        int                 ibp_connecting;     /* active connects in progress */
        int                 ibp_accepting;      /* passive connects in progress */
        int                 ibp_passivewait;    /* waiting for peer to connect */
        unsigned long       ibp_passivewait_deadline; /* when passive wait must complete */
        unsigned long       ibp_reconnect_time; /* when reconnect may be attempted */
        unsigned long       ibp_reconnect_interval; /* exponential backoff */
        int                 ibp_error;          /* errno on closing this peer */
        cfs_time_t          ibp_last_alive;     /* when (in jiffies) I was last alive */
} kib_peer_t;


extern kib_data_t      kibnal_data;
extern kib_tunables_t  kibnal_tunables;

/******************************************************************************/

/* these are purposely avoiding using local vars so they don't increase
 * stack consumption. */

#define kibnal_conn_addref(conn)                                \
do {                                                            \
        CDEBUG(D_NET, "conn[%p] (%d)++\n",                      \
               (conn), atomic_read(&(conn)->ibc_refcount));     \
        LASSERT(atomic_read(&(conn)->ibc_refcount) > 0);        \
        atomic_inc(&(conn)->ibc_refcount);                      \
} while (0)

#define kibnal_conn_decref(conn)                                              \
do {                                                                          \
        unsigned long   flags;                                                \
                                                                              \
        CDEBUG(D_NET, "conn[%p] (%d)--\n",                                    \
               (conn), atomic_read(&(conn)->ibc_refcount));                   \
        LASSERT(atomic_read(&(conn)->ibc_refcount) > 0);                      \
        if (atomic_dec_and_test(&(conn)->ibc_refcount)) {                     \
                spin_lock_irqsave(&kibnal_data.kib_connd_lock, flags);        \
                list_add_tail(&(conn)->ibc_list,                              \
                              &kibnal_data.kib_connd_zombies);                \
                wake_up(&kibnal_data.kib_connd_waitq);                        \
                spin_unlock_irqrestore(&kibnal_data.kib_connd_lock, flags);   \
        }                                                                     \
} while (0)

#define kibnal_peer_addref(peer)                                \
do {                                                            \
        CDEBUG(D_NET, "peer[%p] -> %s (%d)++\n",                \
               (peer), libcfs_nid2str((peer)->ibp_nid),         \
               atomic_read (&(peer)->ibp_refcount));            \
        LASSERT(atomic_read(&(peer)->ibp_refcount) > 0);        \
        atomic_inc(&(peer)->ibp_refcount);                      \
} while (0)

#define kibnal_peer_decref(peer)                                \