lustre_idl.h

非常经典的一个分布式系统

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 *
 *   This file is part of Lustre, http://www.lustre.org
 *
 * Lustre wire protocol definitions.
 *
 * ALL structs passing over the wire should be declared here.  Structs
 * that are used in interfaces with userspace should go in lustre_user.h.
 *
 * All structs being declared here should be built from simple fixed-size
 * types (__u8, __u16, __u32, __u64) or be built from other types or
 * structs also declared in this file.  Similarly, all flags and magic
 * values in those structs should also be declared here.  This ensures
 * that the Lustre wire protocol is not influenced by external dependencies.
 *
 * The only other acceptable items in this file are VERY SIMPLE accessor
 * functions to avoid callers grubbing inside the structures, and the
 * prototypes of the swabber functions for each struct.  Nothing that
 * depends on external functions or definitions should be in here.
 *
 * Structs must be properly aligned to put 64-bit values on an 8-byte
 * boundary.  Any structs being added here must also be added to
 * utils/wirecheck.c and "make newwiretest" run to regenerate the
 * utils/wiretest.c sources.  This allows us to verify that wire structs
 * have the proper alignment/size on all architectures.
 *
 * DO NOT CHANGE any of the structs, flags, values declared here and used
 * in released Lustre versions.  Some structs may have padding fields that
 * can be used.  Some structs might allow addition at the end (verify this
 * in the code to ensure that new/old clients that see this larger struct
 * do not fail, otherwise you need to implement protocol compatibility).
 *
 * We assume all nodes are either little-endian or big-endian, and we
 * always send messages in the sender's native format.  The receiver
 * detects the message format by checking the 'magic' field of the message
 * (see lustre_msg_swabbed() below).
 *
 * Each wire type has corresponding 'lustre_swab_xxxtypexxx()' routines,
 * implemented either here, inline (trivial implementations) or in
 * ptlrpc/pack_generic.c.  These 'swabbers' convert the type from "other"
 * endian, in-place in the message buffer.
 *
 * A swabber takes a single pointer argument.  The caller must already have
 * verified that the length of the message buffer >= sizeof (type).
 *
 * For variable length types, a second 'lustre_swab_v_xxxtypexxx()' routine
 * may be defined that swabs just the variable part, after the caller has
 * verified that the message buffer is large enough.
 */

#ifndef _LUSTRE_IDL_H_
#define _LUSTRE_IDL_H_

#if defined(__linux__)
#include <linux/lustre_types.h>
#elif defined(__APPLE__)
#include <darwin/lustre_types.h>
#elif defined(__WINNT__)
#include <winnt/lustre_types.h>
#else
#error Unsupported operating system.
#endif

/* Defn's shared with user-space. */
#include <lustre/lustre_user.h>

/*
 * this file contains all data structures used in Lustre interfaces:
 * - obdo and obd_request records
 * - mds_request records
 * - ldlm data
 * - ioctl's
 */

/*
 *  GENERAL STUFF
 */
/* FOO_REQUEST_PORTAL is for incoming requests on the FOO
 * FOO_REPLY_PORTAL   is for incoming replies on the FOO
 * FOO_BULK_PORTAL    is for incoming bulk on the FOO
 */

#define CONNMGR_REQUEST_PORTAL          1
#define CONNMGR_REPLY_PORTAL            2
//#define OSC_REQUEST_PORTAL            3
#define OSC_REPLY_PORTAL                4
//#define OSC_BULK_PORTAL               5
#define OST_IO_PORTAL                   6
#define OST_CREATE_PORTAL               7
#define OST_BULK_PORTAL                 8
//#define MDC_REQUEST_PORTAL            9
#define MDC_REPLY_PORTAL               10
//#define MDC_BULK_PORTAL              11
#define MDS_REQUEST_PORTAL             12
//#define MDS_REPLY_PORTAL             13
#define MDS_BULK_PORTAL                14
#define LDLM_CB_REQUEST_PORTAL         15
#define LDLM_CB_REPLY_PORTAL           16
#define LDLM_CANCEL_REQUEST_PORTAL     17
#define LDLM_CANCEL_REPLY_PORTAL       18
//#define PTLBD_REQUEST_PORTAL           19
//#define PTLBD_REPLY_PORTAL             20
//#define PTLBD_BULK_PORTAL              21
#define MDS_SETATTR_PORTAL             22
#define MDS_READPAGE_PORTAL            23

#define MGC_REPLY_PORTAL               25
#define MGS_REQUEST_PORTAL             26
#define MGS_REPLY_PORTAL               27
#define OST_REQUEST_PORTAL             28

#define SVC_KILLED               1
#define SVC_EVENT                2
#define SVC_SIGNAL               4
#define SVC_RUNNING              8
#define SVC_STOPPING            16
#define SVC_STOPPED             32

/* packet types */
#define PTL_RPC_MSG_REQUEST 4711
#define PTL_RPC_MSG_ERR     4712
#define PTL_RPC_MSG_REPLY   4713

/* DON'T use swabbed values of MAGIC as magic! */
#define LUSTRE_MSG_MAGIC_V1 0x0BD00BD0
#define LUSTRE_MSG_MAGIC_V2 0x0BD00BD3

#define LUSTRE_MSG_MAGIC_V1_SWABBED 0xD00BD00B
#define LUSTRE_MSG_MAGIC_V2_SWABBED 0xD30BD00B

#define LUSTRE_MSG_MAGIC LUSTRE_MSG_MAGIC_V2

#define PTLRPC_MSG_VERSION  0x00000003
#define LUSTRE_VERSION_MASK 0xffff0000
#define LUSTRE_OBD_VERSION  0x00010000
#define LUSTRE_MDS_VERSION  0x00020000
#define LUSTRE_OST_VERSION  0x00030000
#define LUSTRE_DLM_VERSION  0x00040000
#define LUSTRE_LOG_VERSION  0x00050000
#define LUSTRE_MGS_VERSION  0x00060000

struct lustre_handle {
        __u64 cookie;
};
#define DEAD_HANDLE_MAGIC 0xdeadbeefcafebabeULL

static inline int lustre_handle_is_used(struct lustre_handle *lh)
{
        return lh->cookie != 0ull;
}

static inline int lustre_handle_equal(struct lustre_handle *lh1,
                                      struct lustre_handle *lh2)
{
        return lh1->cookie == lh2->cookie;
}

static inline void lustre_handle_copy(struct lustre_handle *tgt,
                                      struct lustre_handle *src)
{
        tgt->cookie = src->cookie;
}

/* we depend on this structure to be 8-byte aligned */
/* this type is only endian-adjusted in lustre_unpack_msg() */
struct lustre_msg_v1 {
        struct lustre_handle lm_handle;
        __u32 lm_magic;
        __u32 lm_type;
        __u32 lm_version;
        __u32 lm_opc;
        __u64 lm_last_xid;
        __u64 lm_last_committed;
        __u64 lm_transno;
        __u32 lm_status;
        __u32 lm_flags;
        __u32 lm_conn_cnt;
        __u32 lm_bufcount;
        __u32 lm_buflens[0];
};

#define lustre_msg lustre_msg_v2
/* we depend on this structure to be 8-byte aligned */
/* this type is only endian-adjusted in lustre_unpack_msg() */
struct lustre_msg_v2 {
        __u32 lm_bufcount;
        __u32 lm_secflvr;
        __u32 lm_magic;
        __u32 lm_repsize;
        __u32 lm_cksum;
        __u32 lm_padding_1;
        __u32 lm_padding_2;
        __u32 lm_padding_3;
        __u32 lm_buflens[0];
};

/* without security, ptlrpc_body is put in the first buffer. */
struct ptlrpc_body {
        struct lustre_handle pb_handle;
        __u32 pb_type;
        __u32 pb_version;
        __u32 pb_opc;
        __u32 pb_status;
        __u64 pb_last_xid;
        __u64 pb_last_seen;
        __u64 pb_last_committed;
        __u64 pb_transno;
        __u32 pb_flags;
        __u32 pb_op_flags;
        __u32 pb_conn_cnt;
        __u32 pb_timeout;  /* for req, the deadline, for rep, the service est */
        __u32 pb_service_time; /* for rep, actual service time */
        __u32 pb_limit;
        __u64 pb_slv;
};

extern void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);

/* message body offset for lustre_msg_v2 */
/* ptlrpc body offset in all request/reply messages */
#define MSG_PTLRPC_BODY_OFF             0

/* normal request/reply message record offset */
#define REQ_REC_OFF                     1
#define REPLY_REC_OFF                   1

/* ldlm request message body offset */
#define DLM_LOCKREQ_OFF                 1 /* lockreq offset */
#define DLM_REQ_REC_OFF                 2 /* normal dlm request record offset */

/* ldlm intent lock message body offset */
#define DLM_INTENT_IT_OFF               2 /* intent lock it offset */
#define DLM_INTENT_REC_OFF              3 /* intent lock record offset */

/* ldlm reply message body offset */
#define DLM_LOCKREPLY_OFF               1 /* lockrep offset */
#define DLM_REPLY_REC_OFF               2 /* reply record offset */

/* Flags that are operation-specific go in the top 16 bits. */
#define MSG_OP_FLAG_MASK   0xffff0000
#define MSG_OP_FLAG_SHIFT  16

/* Flags that apply to all requests are in the bottom 16 bits */
#define MSG_GEN_FLAG_MASK      0x0000ffff
#define MSG_LAST_REPLAY        1
#define MSG_RESENT             2
#define MSG_REPLAY             4
#define MSG_AT_SUPPORT         8

/*
 * Flags for all connect opcodes (MDS_CONNECT, OST_CONNECT)
 */

#define MSG_CONNECT_RECOVERING  0x1
#define MSG_CONNECT_RECONNECT   0x2
#define MSG_CONNECT_REPLAYABLE  0x4
//#define MSG_CONNECT_PEER        0x8
#define MSG_CONNECT_LIBCLIENT   0x10
#define MSG_CONNECT_INITIAL     0x20
#define MSG_CONNECT_ASYNC       0x40
#define MSG_CONNECT_NEXT_VER    0x80 /* use next version of lustre_msg */

/* Connect flags */
#define OBD_CONNECT_RDONLY            0x1ULL /*client allowed read-only access*/
#define OBD_CONNECT_INDEX             0x2ULL /*connect to specific LOV idx */
#define OBD_CONNECT_GRANT             0x8ULL /*OSC acquires grant at connect */
#define OBD_CONNECT_SRVLOCK          0x10ULL /*server takes locks for client */
#define OBD_CONNECT_VERSION          0x20ULL /*Lustre versions in ocd */
#define OBD_CONNECT_REQPORTAL        0x40ULL /*Separate non-IO request portal */
#define OBD_CONNECT_ACL              0x80ULL /*access control lists */
#define OBD_CONNECT_XATTR           0x100ULL /*client use extended attributes */
#define OBD_CONNECT_CROW            0x200ULL /*MDS+OST create objects on write*/
#define OBD_CONNECT_TRUNCLOCK       0x400ULL /*locks on server for punch */
#define OBD_CONNECT_TRANSNO         0x800ULL /*replay sends initial transno */
#define OBD_CONNECT_IBITS          0x1000ULL /*support for inodebits locks */
#define OBD_CONNECT_JOIN           0x2000ULL /*files can be concatenated */
#define OBD_CONNECT_ATTRFID        0x4000ULL /*Server supports GetAttr By Fid */
#define OBD_CONNECT_NODEVOH        0x8000ULL /*No open handle on special nodes*/
#define OBD_CONNECT_LCL_CLIENT    0x10000ULL /*local 1.8 client */
#define OBD_CONNECT_RMT_CLIENT    0x20000ULL /*Remote 1.8 client */
#define OBD_CONNECT_BRW_SIZE      0x40000ULL /*Max bytes per rpc */
#define OBD_CONNECT_QUOTA64       0x80000ULL /*64bit qunit_data.qd_count */
#define OBD_CONNECT_MDS_CAPA     0x100000ULL /*MDS capability */
#define OBD_CONNECT_OSS_CAPA     0x200000ULL /*OSS capability */
#define OBD_CONNECT_CANCELSET    0x400000ULL /*Early batched cancels. */
#define OBD_CONNECT_SOM        0x00800000ULL /*Size on MDS */
#define OBD_CONNECT_AT         0x01000000ULL /*client uses adaptive timeouts */
#define OBD_CONNECT_LRU_RESIZE 0x02000000ULL /*Lru resize feature. */
#define OBD_CONNECT_MDS_MDS    0x04000000ULL /*MDS-MDS connection */
#define OBD_CONNECT_REAL       0x08000000ULL /*real connection */
/* also update obd_connect_names[] for lprocfs_rd_connect_flags()
 * and lustre/utils/wirecheck.c */

#ifdef HAVE_LRU_RESIZE_SUPPORT
#define LRU_RESIZE_CONNECT_FLAG OBD_CONNECT_LRU_RESIZE
#else
#define LRU_RESIZE_CONNECT_FLAG 0
#endif

#define MDS_CONNECT_SUPPORTED  (OBD_CONNECT_RDONLY | OBD_CONNECT_VERSION | \
                                OBD_CONNECT_ACL | OBD_CONNECT_XATTR | \
                                OBD_CONNECT_IBITS | OBD_CONNECT_JOIN | \
                                OBD_CONNECT_NODEVOH | OBD_CONNECT_ATTRFID | \
                                OBD_CONNECT_CANCELSET | LRU_RESIZE_CONNECT_FLAG)

#define OST_CONNECT_SUPPORTED  (OBD_CONNECT_SRVLOCK | OBD_CONNECT_GRANT | \
                                OBD_CONNECT_REQPORTAL | OBD_CONNECT_VERSION | \
                                OBD_CONNECT_TRUNCLOCK | OBD_CONNECT_INDEX | \
                                OBD_CONNECT_BRW_SIZE | OBD_CONNECT_QUOTA64 | \
                                OBD_CONNECT_CANCELSET | LRU_RESIZE_CONNECT_FLAG)
#define ECHO_CONNECT_SUPPORTED (0)
#define MGS_CONNECT_SUPPORTED  (OBD_CONNECT_VERSION)

#define MAX_QUOTA_COUNT32 (0xffffffffULL)

#define OBD_OCD_VERSION(major,minor,patch,fix) (((major)<<24) + ((minor)<<16) +\
                                                ((patch)<<8) + (fix))
#define OBD_OCD_VERSION_MAJOR(version) ((int)((version)>>24)&255)
#define OBD_OCD_VERSION_MINOR(version) ((int)((version)>>16)&255)
#define OBD_OCD_VERSION_PATCH(version) ((int)((version)>>8)&255)
#define OBD_OCD_VERSION_FIX(version)   ((int)(version)&255)

/* This structure is used for both request and reply.
 *
 * If we eventually have separate connect data for different types, which we
 * almost certainly will, then perhaps we stick a union in here. */
struct obd_connect_data {
        __u64 ocd_connect_flags;        /* OBD_CONNECT_* per above */
        __u32 ocd_version;              /* lustre release version number */
        __u32 ocd_grant;                /* initial cache grant amount (bytes) */
        __u32 ocd_index;                /* LOV index to connect to */
        __u32 ocd_brw_size;             /* Maximum BRW size in bytes */
        __u64 ocd_ibits_known;          /* inode bits this client understands */
        __u32 ocd_nllu;                 /* non-local-lustre-user */
        __u32 ocd_nllg;                 /* non-local-lustre-group */
        __u64 padding1;                 /* also fix lustre_swab_connect */
        __u64 padding2;                 /* also fix lustre_swab_connect */
        __u64 padding3;                 /* also fix lustre_swab_connect */
        __u64 padding4;                 /* also fix lustre_swab_connect */
};

extern void lustre_swab_connect(struct obd_connect_data *ocd);

/*
 *   OST requests: OBDO & OBD request records
 */

/* opcodes */
typedef enum {
        OST_REPLY      =  0,       /* reply ? */
        OST_GETATTR    =  1,
        OST_SETATTR    =  2,
        OST_READ       =  3,
        OST_WRITE      =  4,
        OST_CREATE     =  5,
        OST_DESTROY    =  6,
        OST_GET_INFO   =  7,
        OST_CONNECT    =  8,
        OST_DISCONNECT =  9,
        OST_PUNCH      = 10,
        OST_OPEN       = 11,
        OST_CLOSE      = 12,
        OST_STATFS     = 13,
/*      OST_SAN_READ   = 14,    deprecated */
/*      OST_SAN_WRITE  = 15,    deprecated */
        OST_SYNC       = 16,
        OST_SET_INFO   = 17,
        OST_QUOTACHECK = 18,
        OST_QUOTACTL   = 19,
        OST_LAST_OPC
} ost_cmd_t;
#define OST_FIRST_OPC  OST_REPLY