lib-types.h

非常经典的一个分布式系统

        
        int  (*lnd_startup) (struct lnet_ni *ni);
        void (*lnd_shutdown) (struct lnet_ni *ni);
        int  (*lnd_ctl)(struct lnet_ni *ni, unsigned int cmd, void *arg);

        /* In data movement APIs below, payload buffers are described as a set
         * of 'niov' fragments which are...
         * EITHER 
         *    in virtual memory (struct iovec *iov != NULL)
         * OR
         *    in pages (kernel only: plt_kiov_t *kiov != NULL).
         * The LND may NOT overwrite these fragment descriptors.
         * An 'offset' and may specify a byte offset within the set of
         * fragments to start from 
         */

        /* Start sending a preformatted message.  'private' is NULL for PUT and
	 * GET messages; otherwise this is a response to an incoming message
	 * and 'private' is the 'private' passed to lnet_parse().  Return
	 * non-zero for immediate failure, otherwise complete later with
	 * lnet_finalize() */
	int (*lnd_send)(struct lnet_ni *ni, void *private, lnet_msg_t *msg);

        /* Start receiving 'mlen' bytes of payload data, skipping the following
         * 'rlen' - 'mlen' bytes. 'private' is the 'private' passed to
         * lnet_parse().  Return non-zero for immedaite failure, otherwise
         * complete later with lnet_finalize().  This also gives back a receive
         * credit if the LND does flow control. */
	int (*lnd_recv)(struct lnet_ni *ni, void *private, lnet_msg_t *msg,
                        int delayed, unsigned int niov, 
                        struct iovec *iov, lnet_kiov_t *kiov,
                        unsigned int offset, unsigned int mlen, unsigned int rlen);

        /* lnet_parse() has had to delay processing of this message
         * (e.g. waiting for a forwarding buffer or send credits).  Give the
         * LND a chance to free urgently needed resources.  If called, return 0
         * for success and do NOT give back a receive credit; that has to wait
         * until lnd_recv() gets called.  On failure return < 0 and
         * release resources; lnd_recv() will not be called. */
	int (*lnd_eager_recv)(struct lnet_ni *ni, void *private, lnet_msg_t *msg,
                              void **new_privatep);

        /* notification of peer health */
        void (*lnd_notify)(struct lnet_ni *ni, lnet_nid_t peer, int alive);

#ifdef __KERNEL__
        /* accept a new connection */
        int (*lnd_accept)(struct lnet_ni *ni, cfs_socket_t *sock);
#else
        /* wait for something to happen */
        void (*lnd_wait)(struct lnet_ni *ni, int milliseconds);

        /* ensure non-RDMA messages can be received outside liblustre */
        int (*lnd_setasync)(struct lnet_ni *ni, lnet_process_id_t id, int nasync);

#ifdef HAVE_LIBPTHREAD
        int (*lnd_accept)(struct lnet_ni *ni, int sock);
#endif
#endif
} lnd_t;

#define LNET_MAX_INTERFACES   16

typedef struct lnet_ni {
        struct list_head  ni_list;              /* chain on ln_nis */
        struct list_head  ni_txq;               /* messages waiting for tx credits */
        int               ni_maxtxcredits;      /* # tx credits  */
        int               ni_txcredits;         /* # tx credits free */
        int               ni_mintxcredits;      /* lowest it's been */
        int               ni_peertxcredits;     /* # per-peer send credits */
        lnet_nid_t        ni_nid;               /* interface's NID */
        void             *ni_data;              /* instance-specific data */
        lnd_t            *ni_lnd;               /* procedural interface */
        int               ni_refcount;          /* reference count */
        char             *ni_interfaces[LNET_MAX_INTERFACES]; /* equivalent interfaces to use */
} lnet_ni_t;

typedef struct lnet_peer {
        struct list_head  lp_hashlist;          /* chain on peer hash */
        struct list_head  lp_txq;               /* messages blocking for tx credits */
        struct list_head  lp_rtrq;              /* messages blocking for router credits */
        struct list_head  lp_rtr_list;          /* chain on router list */
        int               lp_txcredits;         /* # tx credits available */
        int               lp_mintxcredits;      /* low water mark */
        int               lp_rtrcredits;        /* # router credits */
        int               lp_minrtrcredits;     /* low water mark */
        unsigned int      lp_alive:1;           /* alive/dead? */
        unsigned int      lp_notify:1;          /* notification outstanding? */
        unsigned int      lp_notifylnd:1;       /* outstanding notification for LND? */
        unsigned int      lp_notifying:1;       /* some thread is handling notification */
        unsigned int      lp_ping_notsent;      /* SEND event outstanding from ping */
        int               lp_alive_count;       /* # times router went dead<->alive */
        long              lp_txqnob;            /* bytes queued for sending */
        time_t            lp_timestamp;         /* time of last aliveness news */
        time_t            lp_ping_timestamp;    /* time of last ping attempt */
        time_t            lp_ping_deadline;     /* != 0 if ping reply expected */
        lnet_ni_t        *lp_ni;                /* interface peer is on */
        lnet_nid_t        lp_nid;               /* peer's NID */
        int               lp_refcount;          /* # refs */
        int               lp_rtr_refcount;      /* # refs from lnet_route_t::lr_gateway */
} lnet_peer_t;

typedef struct {
	struct list_head  lr_list;              /* chain on net */
        lnet_peer_t      *lr_gateway;           /* router node */
} lnet_route_t;

typedef struct {
        struct list_head        lrn_list;       /* chain on ln_remote_nets */
        struct list_head        lrn_routes;     /* routes to me */
        __u32                   lrn_net;        /* my net number */
        unsigned int            lrn_hops;       /* how far I am */
} lnet_remotenet_t;

typedef struct {
        struct list_head  rbp_bufs;             /* my free buffer pool */
        struct list_head  rbp_msgs;             /* messages blocking for a buffer */
        int               rbp_npages;           /* # pages in each buffer */
        int               rbp_nbuffers;         /* # buffers */
        int               rbp_credits;          /* # free buffers / blocked messages */
        int               rbp_mincredits;       /* low water mark */
} lnet_rtrbufpool_t;

typedef struct {
        struct list_head   rb_list;             /* chain on rbp_bufs */
        lnet_rtrbufpool_t *rb_pool;             /* owning pool */
        lnet_kiov_t        rb_kiov[0];          /* the buffer space */
} lnet_rtrbuf_t;

typedef struct {
        __u32        msgs_alloc;
        __u32        msgs_max;
        __u32        errors;
        __u32        send_count;
        __u32        recv_count;
        __u32        route_count;
        __u32        drop_count;
        __u64        send_length;
        __u64        recv_length;
        __u64        route_length;
        __u64        drop_length;
} lnet_counters_t;

#define LNET_PEER_HASHSIZE   503                /* prime! */

#define LNET_NRBPOOLS         3                 /* # different router buffer pools */

#define LNET_PROTO_PING_MATCHBITS     0x8000000000000000LL
#define LNET_PROTO_PING_VERSION       1
typedef struct {
        __u32          pi_magic;
        __u32          pi_version;
        lnet_pid_t     pi_pid;
        __u32          pi_nnids;
        lnet_nid_t     pi_nid[0];
} WIRE_ATTR lnet_ping_info_t;

/* Options for lnet_portal_t::ptl_options */
#define LNET_PTL_LAZY               (1 << 0)
typedef struct {
        struct list_head ptl_ml;  /* match list */
        struct list_head ptl_msgq; /* messages blocking for MD */
        __u64            ptl_msgq_version;  /* validity stamp */
        unsigned int     ptl_options;
} lnet_portal_t;

/* Router Checker */
/*                               < 0 == startup error */
#define LNET_RC_STATE_SHUTDOWN     0            /* not started */
#define LNET_RC_STATE_RUNNING      1            /* started up OK */
#define LNET_RC_STATE_STOPTHREAD   2            /* telling thread to stop */
#define LNET_RC_STATE_UNLINKING    3            /* unlinking RC MD */
#define LNET_RC_STATE_UNLINKED     4            /* RC's MD has been unlinked */

typedef struct
{
        /* Stuff initialised at LNetInit() */
        int                ln_init;             /* LNetInit() called? */
        int                ln_refcount;         /* LNetNIInit/LNetNIFini counter */
        int                ln_niinit_self;      /* Have I called LNetNIInit myself? */

        int                ln_ptlcompat;        /* do I support talking to portals? */
        
        struct list_head   ln_lnds;             /* registered LNDs */

#ifdef __KERNEL__
        spinlock_t         ln_lock;
        cfs_waitq_t        ln_waitq;
        struct semaphore   ln_api_mutex;
        struct semaphore   ln_lnd_mutex;
#else
# ifndef HAVE_LIBPTHREAD
        int                ln_lock;
        int                ln_api_mutex;
        int                ln_lnd_mutex;
# else
        pthread_cond_t     ln_cond;
        pthread_mutex_t    ln_lock;
        pthread_mutex_t    ln_api_mutex;
        pthread_mutex_t    ln_lnd_mutex;
# endif
#endif

        /* Stuff initialised at LNetNIInit() */

        int                ln_shutdown;         /* shutdown in progress */
        int                ln_nportals;         /* # portals */
        lnet_portal_t     *ln_portals;          /* the vector of portals */

        lnet_pid_t         ln_pid;              /* requested pid */

        struct list_head   ln_nis;              /* LND instances */
        lnet_ni_t         *ln_loni;             /* the loopback NI */
        lnet_ni_t         *ln_eqwaitni;         /* NI to wait for events in */
        struct list_head   ln_zombie_nis;       /* dying LND instances */
        int                ln_nzombie_nis;      /* # of NIs to wait for */

        struct list_head   ln_remote_nets;      /* remote networks with routes to them */
        __u64              ln_remote_nets_version; /* validity stamp */

        struct list_head   ln_routers;          /* list of all known routers */
        __u64              ln_routers_version;  /* validity stamp */

        struct list_head  *ln_peer_hash;        /* NID->peer hash */
        int                ln_npeers;           /* # peers extant */
        int                ln_peertable_version; /* /proc validity stamp */
        
        int                ln_routing;          /* am I a router? */
        lnet_rtrbufpool_t  ln_rtrpools[LNET_NRBPOOLS]; /* router buffer pools */
        
        int                ln_lh_hash_size;     /* size of lib handle hash table */
        struct list_head  *ln_lh_hash_table;    /* all extant lib handles, this interface */
        __u64              ln_next_object_cookie; /* cookie generator */
        __u64              ln_interface_cookie; /* uniquely identifies this ni in this epoch */

        char              *ln_network_tokens;   /* space for network names */
        int                ln_network_tokens_nob;

        int                ln_testprotocompat;  /* test protocol compatibility flags */

        struct list_head   ln_finalizeq;        /* msgs waiting to complete finalizing */
#ifdef __KERNEL__
        void             **ln_finalizers;       /* threads doing finalization */
        int                ln_nfinalizers;      /* max # threads finalizing */
#else
        int                ln_finalizing;
#endif
        struct list_head   ln_test_peers;       /* failure simulation */

        lnet_handle_md_t   ln_ping_target_md;
        lnet_handle_eq_t   ln_ping_target_eq;
        lnet_ping_info_t  *ln_ping_info;

#ifdef __KERNEL__
	int                ln_rc_state;         /* router checker startup/shutdown state */
	struct semaphore   ln_rc_signal;        /* serialise startup/shutdown */
        lnet_handle_eq_t   ln_rc_eqh;           /* router checker's event queue */
#endif
        
#ifdef LNET_USE_LIB_FREELIST
        lnet_freelist_t    ln_free_mes;
        lnet_freelist_t    ln_free_msgs;
        lnet_freelist_t    ln_free_mds;
        lnet_freelist_t    ln_free_eqs;
#endif
        struct list_head   ln_active_msgs;
        struct list_head   ln_active_mds;
        struct list_head   ln_active_eqs;

        lnet_counters_t    ln_counters;

#ifndef __KERNEL__
        /* Temporary workaround to allow uOSS and test programs force
         * server mode in userspace. The only place where we use it is
         * lnet_prepare(). The only way to turn this flag on is to
         * call lnet_server_mode() */

        int                ln_server_mode_flag;
#endif        
} lnet_t;

#endif