ralnd.h

非常经典的一个分布式系统

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 *
 * Copyright (C) 2004 Cluster File Systems, Inc.
 *   Author: Eric Barton <[e-mail]>
 *
 *   This file is part of Lustre, http://www.lustre.org.
 *
 *   Lustre is free software; you can redistribute it and/or
 *   modify it under the terms of version 2 of the GNU General Public
 *   License as published by the Free Software Foundation.
 *
 *   Lustre is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with Lustre; if not, write to the Free Software
 *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 */

#ifndef EXPORT_SYMTAB
# define EXPORT_SYMTAB
#endif
#ifndef AUTOCONF_INCLUDED
#include <linux/config.h>
#endif
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/smp_lock.h>
#include <linux/unistd.h>
#include <linux/uio.h>

#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/io.h>

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/stat.h>
#include <linux/list.h>
#include <linux/kmod.h>
#include <linux/sysctl.h>

#include <net/sock.h>
#include <linux/in.h>

#define DEBUG_SUBSYSTEM S_LND

#include <libcfs/kp30.h>
#include <lnet/lnet.h>
#include <lnet/lib-lnet.h>

#include <rapl.h>

/* tunables determined at compile time */
#define RANAL_RESCHED             100           /* # scheduler loops before reschedule */

#define RANAL_PEER_HASH_SIZE      101           /* # peer lists */
#define RANAL_CONN_HASH_SIZE      101           /* # conn lists */

#define RANAL_MIN_TIMEOUT         5             /* minimum timeout interval (seconds) */
#define RANAL_TIMEOUT2KEEPALIVE(t) (((t)+1)/2)  /* timeout -> keepalive interval */

/* fixed constants */
#define RANAL_MAXDEVS             2             /* max # devices RapidArray supports */
#define RANAL_FMA_MAX_PREFIX      232           /* max bytes in FMA "Prefix" we can use */
#define RANAL_FMA_MAX_DATA        ((7<<10)-256) /* Max FMA MSG is 7K including prefix */


typedef struct
{
        int              *kra_n_connd;          /* # connection daemons */
        int              *kra_min_reconnect_interval; /* first failed connection retry... */
        int              *kra_max_reconnect_interval; /* ...exponentially increasing to this */
        int              *kra_ntx;              /* # tx descs */
        int              *kra_credits;          /* # concurrent sends */
        int              *kra_peercredits;      /* # concurrent sends to 1 peer */
        int              *kra_fma_cq_size;      /* # entries in receive CQ */
        int              *kra_timeout;          /* comms timeout (seconds) */
        int              *kra_max_immediate;    /* immediate payload breakpoint */

#if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
        cfs_sysctl_table_header_t *kra_sysctl;  /* sysctl interface */
#endif
} kra_tunables_t;

typedef struct
{
        RAP_PVOID               rad_handle;     /* device handle */
        RAP_PVOID               rad_fma_cqh;    /* FMA completion queue handle */
        RAP_PVOID               rad_rdma_cqh;   /* rdma completion queue handle */
        int                     rad_id;         /* device id */
        int                     rad_idx;        /* index in kra_devices */
        int                     rad_ready;      /* set by device callback */
        struct list_head        rad_ready_conns;/* connections ready to tx/rx */
        struct list_head        rad_new_conns;  /* new connections to complete */
        wait_queue_head_t       rad_waitq;      /* scheduler waits here */
        spinlock_t              rad_lock;       /* serialise */
        void                   *rad_scheduler;  /* scheduling thread */
        unsigned int            rad_nphysmap;   /* # phys mappings */
        unsigned int            rad_nppphysmap; /* # phys pages mapped */
        unsigned int            rad_nvirtmap;   /* # virt mappings */
        unsigned long           rad_nobvirtmap; /* # virt bytes mapped */
} kra_device_t;

typedef struct
{
        int               kra_init;             /* initialisation state */
        int               kra_shutdown;         /* shut down? */
        atomic_t          kra_nthreads;         /* # live threads */
        lnet_ni_t        *kra_ni;               /* _the_ nal instance */
        
        kra_device_t      kra_devices[RANAL_MAXDEVS]; /* device/ptag/cq etc */
        int               kra_ndevs;            /* # devices */

        rwlock_t          kra_global_lock;      /* stabilize peer/conn ops */

        struct list_head *kra_peers;            /* hash table of all my known peers */
        int               kra_peer_hash_size;   /* size of kra_peers */
        atomic_t          kra_npeers;           /* # peers extant */
        int               kra_nonewpeers;       /* prevent new peers */

        struct list_head *kra_conns;            /* conns hashed by cqid */
        int               kra_conn_hash_size;   /* size of kra_conns */
        __u64             kra_peerstamp;        /* when I started up */
        __u64             kra_connstamp;        /* conn stamp generator */
        int               kra_next_cqid;        /* cqid generator */
        atomic_t          kra_nconns;           /* # connections extant */

        long              kra_new_min_timeout;  /* minimum timeout on any new conn */
        wait_queue_head_t kra_reaper_waitq;     /* reaper sleeps here */
        spinlock_t        kra_reaper_lock;      /* serialise */

        struct list_head  kra_connd_peers;      /* peers waiting for a connection */
        struct list_head  kra_connd_acceptq;    /* accepted sockets to handshake */
        wait_queue_head_t kra_connd_waitq;      /* connection daemons sleep here */
        spinlock_t        kra_connd_lock;       /* serialise */

        struct list_head  kra_idle_txs;         /* idle tx descriptors */
        __u64             kra_next_tx_cookie;   /* RDMA completion cookie */
        spinlock_t        kra_tx_lock;          /* serialise */
} kra_data_t;

#define RANAL_INIT_NOTHING         0
#define RANAL_INIT_DATA            1
#define RANAL_INIT_ALL             2

typedef struct kra_acceptsock                   /* accepted socket queued for connd */
{
        struct list_head     ras_list;          /* queue for attention */
        struct socket       *ras_sock;          /* the accepted socket */
} kra_acceptsock_t;

/************************************************************************
 * Wire message structs.  These are sent in sender's byte order
 * (i.e. receiver checks magic and flips if required).
 */

typedef struct kra_connreq                      /* connection request/response */
{                                               /* (sent via socket) */
        __u32             racr_magic;           /* I'm an ranal connreq */
        __u16             racr_version;         /* this is my version number */
        __u16             racr_devid;           /* sender's device ID */
        __u64             racr_srcnid;          /* sender's NID */
        __u64             racr_dstnid;          /* who sender expects to listen */
        __u64             racr_peerstamp;       /* sender's instance stamp */
        __u64             racr_connstamp;       /* sender's connection stamp */
        __u32             racr_timeout;         /* sender's timeout */
        RAP_RI_PARAMETERS racr_riparams;        /* sender's endpoint info */
} kra_connreq_t;

typedef struct
{
        RAP_MEM_KEY       rard_key;
        RAP_PVOID64       rard_addr;
        RAP_UINT32        rard_nob;
} kra_rdma_desc_t;

typedef struct
{
        lnet_hdr_t        raim_hdr;             /* portals header */
        /* Portals payload is in FMA "Message Data" */
} kra_immediate_msg_t;

typedef struct
{
        lnet_hdr_t        raprm_hdr;            /* portals header */
        __u64             raprm_cookie;         /* opaque completion cookie */
} kra_putreq_msg_t;

typedef struct
{
        __u64             rapam_src_cookie;     /* reflected completion cookie */
        __u64             rapam_dst_cookie;     /* opaque completion cookie */
        kra_rdma_desc_t   rapam_desc;           /* sender's sink buffer */
} kra_putack_msg_t;

typedef struct
{
        lnet_hdr_t        ragm_hdr;             /* portals header */
        __u64             ragm_cookie;          /* opaque completion cookie */
        kra_rdma_desc_t   ragm_desc;            /* sender's sink buffer */
} kra_get_msg_t;

typedef struct
{
        __u64             racm_cookie;          /* reflected completion cookie */
} kra_completion_msg_t;

typedef struct                                  /* NB must fit in FMA "Prefix" */
{
        __u32             ram_magic;            /* I'm an ranal message */
        __u16             ram_version;          /* this is my version number */
        __u16             ram_type;             /* msg type */
        __u64             ram_srcnid;           /* sender's NID */
        __u64             ram_connstamp;        /* sender's connection stamp */
        union {
                kra_immediate_msg_t   immediate;
                kra_putreq_msg_t      putreq;
                kra_putack_msg_t      putack;
                kra_get_msg_t         get;