ptllnd.c

非常经典的一个分布式系统

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
 * vim:expandtab:shiftwidth=8:tabstop=8:
 *
 * Copyright (C) 2005 Cluster File Systems, Inc. All rights reserved.
 *   Author: PJ Kirner <[e-mail]>
 *
 *   This file is part of the Lustre file system, http://www.lustre.org
 *   Lustre is a trademark of Cluster File Systems, Inc.
 *
 *   This file is confidential source code owned by Cluster File Systems.
 *   No viewing, modification, compilation, redistribution, or any other
 *   form of use is permitted except through a signed license agreement.
 *
 *   If you have not signed such an agreement, then you have no rights to
 *   this file.  Please destroy it immediately and contact CFS.
 *
 */

#include "ptllnd.h"

lnd_t kptllnd_lnd = {
        .lnd_type       = PTLLND,
        .lnd_startup    = kptllnd_startup,
        .lnd_shutdown   = kptllnd_shutdown,
        .lnd_ctl        = kptllnd_ctl,
        .lnd_send       = kptllnd_send,
        .lnd_recv       = kptllnd_recv,
        .lnd_eager_recv = kptllnd_eager_recv,
};

kptl_data_t kptllnd_data;

char *
kptllnd_ptlid2str(ptl_process_id_t id)
{
        static char    strs[64][32];
        static int     idx = 0;

        unsigned long  flags;
        char          *str;
        
        spin_lock_irqsave(&kptllnd_data.kptl_ptlid2str_lock, flags);
        str = strs[idx++];
        if (idx >= sizeof(strs)/sizeof(strs[0]))
                idx = 0;
        spin_unlock_irqrestore(&kptllnd_data.kptl_ptlid2str_lock, flags);

        snprintf(str, sizeof(strs[0]), FMT_PTLID, id.pid, id.nid);
        return str;
}

void 
kptllnd_assert_wire_constants (void)
{
        /* Wire protocol assertions generated by 'wirecheck'
         * running on Linux fedora 2.6.11-co-0.6.4 #1 Mon Jun 19 05:36:13 UTC 2006 i686 i686 i386 GNU
         * with gcc version 4.1.1 20060525 (Red Hat 4.1.1-1) */


        /* Constants... */
        CLASSERT (PTL_RESERVED_MATCHBITS == 0x100);
        CLASSERT (LNET_MSG_MATCHBITS == 0);
        CLASSERT (PTLLND_MSG_MAGIC == 0x50746C4E);
        CLASSERT (PTLLND_MSG_VERSION == 0x04);
        CLASSERT (PTLLND_RDMA_OK == 0x00);
        CLASSERT (PTLLND_RDMA_FAIL == 0x01);
        CLASSERT (PTLLND_MSG_TYPE_INVALID == 0x00);
        CLASSERT (PTLLND_MSG_TYPE_PUT == 0x01);
        CLASSERT (PTLLND_MSG_TYPE_GET == 0x02);
        CLASSERT (PTLLND_MSG_TYPE_IMMEDIATE == 0x03);
        CLASSERT (PTLLND_MSG_TYPE_NOOP == 0x04);
        CLASSERT (PTLLND_MSG_TYPE_HELLO == 0x05);
        CLASSERT (PTLLND_MSG_TYPE_NAK == 0x06);

        /* Checks for struct kptl_msg_t */
        CLASSERT ((int)sizeof(kptl_msg_t) == 136);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_magic) == 0);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_magic) == 4);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_version) == 4);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_version) == 2);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_type) == 6);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_type) == 1);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_credits) == 7);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_credits) == 1);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_nob) == 8);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_nob) == 4);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_cksum) == 12);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_cksum) == 4);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_srcnid) == 16);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_srcnid) == 8);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_srcstamp) == 24);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_srcstamp) == 8);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_dstnid) == 32);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_dstnid) == 8);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_dststamp) == 40);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_dststamp) == 8);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_srcpid) == 48);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_srcpid) == 4);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_dstpid) == 52);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_dstpid) == 4);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_u.immediate) == 56);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_u.immediate) == 72);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_u.rdma) == 56);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_u.rdma) == 80);
        CLASSERT ((int)offsetof(kptl_msg_t, ptlm_u.hello) == 56);
        CLASSERT ((int)sizeof(((kptl_msg_t *)0)->ptlm_u.hello) == 12);

        /* Checks for struct kptl_immediate_msg_t */
        CLASSERT ((int)sizeof(kptl_immediate_msg_t) == 72);
        CLASSERT ((int)offsetof(kptl_immediate_msg_t, kptlim_hdr) == 0);
        CLASSERT ((int)sizeof(((kptl_immediate_msg_t *)0)->kptlim_hdr) == 72);
        CLASSERT ((int)offsetof(kptl_immediate_msg_t, kptlim_payload[13]) == 85);
        CLASSERT ((int)sizeof(((kptl_immediate_msg_t *)0)->kptlim_payload[13]) == 1);

        /* Checks for struct kptl_rdma_msg_t */
        CLASSERT ((int)sizeof(kptl_rdma_msg_t) == 80);
        CLASSERT ((int)offsetof(kptl_rdma_msg_t, kptlrm_hdr) == 0);
        CLASSERT ((int)sizeof(((kptl_rdma_msg_t *)0)->kptlrm_hdr) == 72);
        CLASSERT ((int)offsetof(kptl_rdma_msg_t, kptlrm_matchbits) == 72);
        CLASSERT ((int)sizeof(((kptl_rdma_msg_t *)0)->kptlrm_matchbits) == 8);

        /* Checks for struct kptl_hello_msg_t */
        CLASSERT ((int)sizeof(kptl_hello_msg_t) == 12);
        CLASSERT ((int)offsetof(kptl_hello_msg_t, kptlhm_matchbits) == 0);
        CLASSERT ((int)sizeof(((kptl_hello_msg_t *)0)->kptlhm_matchbits) == 8);
        CLASSERT ((int)offsetof(kptl_hello_msg_t, kptlhm_max_msg_size) == 8);
        CLASSERT ((int)sizeof(((kptl_hello_msg_t *)0)->kptlhm_max_msg_size) == 4);
}

const char *kptllnd_evtype2str(int type)
{
#define DO_TYPE(x) case x: return #x;
        switch(type)
        {
                DO_TYPE(PTL_EVENT_GET_START);
                DO_TYPE(PTL_EVENT_GET_END);
                DO_TYPE(PTL_EVENT_PUT_START);
                DO_TYPE(PTL_EVENT_PUT_END);
                DO_TYPE(PTL_EVENT_REPLY_START);
                DO_TYPE(PTL_EVENT_REPLY_END);
                DO_TYPE(PTL_EVENT_ACK);
                DO_TYPE(PTL_EVENT_SEND_START);
                DO_TYPE(PTL_EVENT_SEND_END);
                DO_TYPE(PTL_EVENT_UNLINK);
        default:
                return "<unknown event type>";
        }
#undef DO_TYPE
}

const char *kptllnd_msgtype2str(int type)
{
#define DO_TYPE(x) case x: return #x;
        switch(type)
        {
                DO_TYPE(PTLLND_MSG_TYPE_INVALID);
                DO_TYPE(PTLLND_MSG_TYPE_PUT);
                DO_TYPE(PTLLND_MSG_TYPE_GET);
                DO_TYPE(PTLLND_MSG_TYPE_IMMEDIATE);
                DO_TYPE(PTLLND_MSG_TYPE_HELLO);
                DO_TYPE(PTLLND_MSG_TYPE_NOOP);
                DO_TYPE(PTLLND_MSG_TYPE_NAK);
        default:
                return "<unknown msg type>";
        }
#undef DO_TYPE
}

const char *kptllnd_errtype2str(int type)
{
#define DO_TYPE(x) case x: return #x;
        switch(type)
        {
                DO_TYPE(PTL_OK);
                DO_TYPE(PTL_SEGV);
                DO_TYPE(PTL_NO_SPACE);
                DO_TYPE(PTL_ME_IN_USE);
                DO_TYPE(PTL_NAL_FAILED);
                DO_TYPE(PTL_NO_INIT);
                DO_TYPE(PTL_IFACE_DUP);
                DO_TYPE(PTL_IFACE_INVALID);
                DO_TYPE(PTL_HANDLE_INVALID);
                DO_TYPE(PTL_MD_INVALID);
                DO_TYPE(PTL_ME_INVALID);
                DO_TYPE(PTL_PROCESS_INVALID);
                DO_TYPE(PTL_PT_INDEX_INVALID);
                DO_TYPE(PTL_SR_INDEX_INVALID);
                DO_TYPE(PTL_EQ_INVALID);
                DO_TYPE(PTL_EQ_DROPPED);
                DO_TYPE(PTL_EQ_EMPTY);
                DO_TYPE(PTL_MD_NO_UPDATE);
                DO_TYPE(PTL_FAIL);
                DO_TYPE(PTL_AC_INDEX_INVALID);
                DO_TYPE(PTL_MD_ILLEGAL);
                DO_TYPE(PTL_ME_LIST_TOO_LONG);
                DO_TYPE(PTL_MD_IN_USE);
                DO_TYPE(PTL_NI_INVALID);
                DO_TYPE(PTL_PID_INVALID);
                DO_TYPE(PTL_PT_FULL);
                DO_TYPE(PTL_VAL_FAILED);
                DO_TYPE(PTL_NOT_IMPLEMENTED);
                DO_TYPE(PTL_NO_ACK);
                DO_TYPE(PTL_EQ_IN_USE);
                DO_TYPE(PTL_PID_IN_USE);
                DO_TYPE(PTL_INV_EQ_SIZE);
                DO_TYPE(PTL_AGAIN);
        default:
                return "<unknown event type>";
        }
#undef DO_TYPE
}

__u32
kptllnd_cksum (void *ptr, int nob)
{
        char  *c  = ptr;
        __u32  sum = 0;

        while (nob-- > 0)
                sum = ((sum << 1) | (sum >> 31)) + *c++;

        /* ensure I don't return 0 (== no checksum) */
        return (sum == 0) ? 1 : sum;
}

void
kptllnd_init_msg(kptl_msg_t *msg, int type, int body_nob)
{
        msg->ptlm_type = type;
        msg->ptlm_nob  = (offsetof(kptl_msg_t, ptlm_u) + body_nob + 7) & ~7;
        
        LASSERT(msg->ptlm_nob <= *kptllnd_tunables.kptl_max_msg_size);
}

void
kptllnd_msg_pack(kptl_msg_t *msg, kptl_peer_t *peer)
{
        msg->ptlm_magic    = PTLLND_MSG_MAGIC;
        msg->ptlm_version  = PTLLND_MSG_VERSION;
        /* msg->ptlm_type  Filled in kptllnd_init_msg()  */
        msg->ptlm_credits  = peer->peer_outstanding_credits;
        /* msg->ptlm_nob   Filled in kptllnd_init_msg()  */
        msg->ptlm_cksum    = 0;
        msg->ptlm_srcnid   = kptllnd_data.kptl_ni->ni_nid;
        msg->ptlm_srcstamp = peer->peer_myincarnation;
        msg->ptlm_dstnid   = peer->peer_id.nid;
        msg->ptlm_dststamp = peer->peer_incarnation;
        msg->ptlm_srcpid   = the_lnet.ln_pid;
        msg->ptlm_dstpid   = peer->peer_id.pid;

        if (*kptllnd_tunables.kptl_checksum) {
                /* NB ptlm_cksum zero while computing cksum */
                msg->ptlm_cksum = kptllnd_cksum(msg, 
                                                offsetof(kptl_msg_t, ptlm_u));
        }
}

int
kptllnd_msg_unpack(kptl_msg_t *msg, int nob)
{
        const int hdr_size = offsetof(kptl_msg_t, ptlm_u);
        __u32     msg_cksum;
        __u16     msg_version;
        int       flip;

        /* 6 bytes are enough to have received magic + version */
        if (nob < 6) {
                CERROR("Very Short message: %d\n", nob);
                return -EPROTO;
        }

        /*
         * Determine if we need to flip
         */
        if (msg->ptlm_magic == PTLLND_MSG_MAGIC) {
                flip = 0;
        } else if (msg->ptlm_magic == __swab32(PTLLND_MSG_MAGIC)) {
                flip = 1;
        } else {
                CERROR("Bad magic: %08x\n", msg->ptlm_magic);
                return -EPROTO;
        }

        msg_version = flip ? __swab16(msg->ptlm_version) : msg->ptlm_version;

        if (msg_version != PTLLND_MSG_VERSION) {
                CERROR("Bad version: got %04x expected %04x\n",
                        (__u32)msg_version, PTLLND_MSG_VERSION);
                return -EPROTO;
        }

        if (nob < hdr_size) {
                CERROR("Short message: got %d, wanted at least %d\n",
                       nob, hdr_size);
                return -EPROTO;
        }