sips.c

一个用在mips体系结构中的操作系统

/*
 * Copyright (C) 1996-1998 by the Board of Trustees
 *    of Leland Stanford Junior University.
 * 
 * This file is part of the SimOS distribution. 
 * See LICENSE file for terms of the license. 
 *
 */

/*****************************************************************
 * simsips.c
 *
 * Short Inter-Processor Sends.
 *
 * Dan Teodosiu, 07/96
 * John Chapin,  08/96 added queueing at receivers, NAK if queue overflows
 * Dan Teodosiu, 06/97 removed NAK, simulate ACK/blocking send.
 *
 ****************************************************************/

#include "sim.h"

#include <stdio.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <sys/file.h>
#include <sys/signal.h>
#ifndef __alpha
#ifndef i386
#include <sys/unistd.h>
#include <sys/ioccom.h>
#include <sys/filio.h>
#endif
#endif
#include <sys/time.h>
#include <sys/uio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>

#include "syslimits.h"
#include "simtypes.h"
#include "machine_params.h"
#include "cpu_interface.h"
#include "cpu_state.h"
#include "sim_error.h"
#include "addr_layout.h"
#include "simutil.h"

#include "sips.h"

static void SipsRestoreArrivals(int cpuNum, EventCallbackHdr *hdr, void *arg);

/* Prints out debugging information to the log if DEBUG == 1.
 * Prints out much more info (about GETs) if DEBUG == 2.
 */

#define DEBUG 0

/* There are two SIPS channels: request and reply. 
 * Additionally we may send a NAK message.
 */

#define SIPS_NUM_CHANNELS 2
typedef enum {SIPS_REQUEST_CHAN, SIPS_REPLY_CHAN} SipChannel;

/* Each channel has a finite buffer capacity.
 */
#define BUFS_PER_SIPS_CHANNEL  32
#define BUF_EMPTY              ((uint64) -1LL)

typedef struct SIPSBuf {
   SimTime             arrival;         /* when arrives at receive node */
   char                data[SIPS_SIZE];
} SIPSBuf;

typedef struct SIPSChannel {
   unsigned int        num_active;
   unsigned int        num_delivered;
   unsigned int        head;
   SIPSBuf             buf[BUFS_PER_SIPS_CHANNEL];
} SIPSChannel;

typedef struct SIPSData {
   SIPSChannel         chan[SIPS_NUM_CHANNELS];
   void                (*int_f)(int n, int lo, int on);
} SIPSData;


static SIPSData sps[SIM_MAXCPUS]; /* SIPS data for all the nodes */
static int      nsps;             /* number of initialized structures */


#ifndef DATA_HANDLING
static int      sips_disabled;
#endif

static EventCallbackHdr sipsCallbackHdr;
static int callbackInstalled = 0;


/*** SimOS interface ***/

void
sim_sips_init(int n, void (*int_f)(int n, int lo, int on))
{
  int i, j, k;
  ASSERT(SIPS_LATENCY > 0);           /* this implementation requires this */
  ASSERT(0 < n && n <= SIM_MAXCPUS);
  for (i = 0; i < n; i++) {
     sps[i].int_f                              = int_f;
     for (j = 0; j < SIPS_NUM_CHANNELS; j++) {
        sps[i].chan[j].num_active              = 0;
        sps[i].chan[j].num_delivered           = 0;
        sps[i].chan[j].head                    = 0;
        for (k = 0; k < BUFS_PER_SIPS_CHANNEL; k++) {
           sps[i].chan[j].buf[k].arrival = BUF_EMPTY;

        }
     }
  }
  nsps = n;

#ifndef DATA_HANDLING
  /* data handling needs to be turned on in mipsy/mxs or sips
   * won't work (multiple cpus fetching ospc's at the same time
   * will smash each other).
   */

  sips_disabled = 0;
  if (   !strcmp(machines.CpuModel, "MIPSY") 
      || !strcmp(machines.CpuModel, "MXS")) {
     sips_disabled = 1;
  }
#endif

}


void
sim_sips_deliver(int n, int chan_sender)
{
   int          chan;
   SIPSChannel* spchan;
   SimTime      dtime;

#if (DEBUG == 0)
   chan = chan_sender;
#else
   int sender = chan_sender >> 4;
   chan = chan_sender & ((1<<4) - 1);
#endif

   ASSERT(0 <= n && n < nsps);

   spchan = &sps[n].chan[chan];
   if (spchan->num_active == 0) {
     /* race condition:  embra delays callbacks some time after the
      * scheduled arrival.  During that time, the cpu may have taken
      * an interrupt for some other reason, polled the sips queue with
      * sips_get, found the waiting sips, and acked it.
      * Do nothing here.
      */
#if (DEBUG > 0) 
     LogEntry("sim_sips_deliver", n, 
	      "sender %d chan %d race condition 1\n",
	      sender, chan);
#endif
     return;
   }
   
   dtime = spchan->buf[spchan->head].arrival;
   ASSERT(dtime != BUF_EMPTY);
   
   if (dtime > CPUVec.CycleCount(n) )  {
     /* another race condition.  Same as before, but additionally,
      * some other cpu has sent another sips which is pending
      * but hasn't arrived yet
      */
#if (DEBUG > 0) 
     LogEntry("sim_sips_deliver", n, 
	      "sender %d chan %d race condition 2\n",
	      sender, chan);
#endif
     return;
   }
   
#if (DEBUG > 0)
   LogEntry("sim_sips_deliver", n, "sender %d chan %d head %d active %d sr 0x%x\n",
	    sender, chan, spchan->head, spchan->num_active);
#endif
   
   /* note that the buffer at the head might not be the one
    * just delivered.  That's ok.  We'll interrupt now (which
    * is probably redundant) and the ack routine will interrupt
    * again if the queue has anything in it that has already
    * arrived.
    */
   spchan->num_delivered++;
   
   if (sps[n].int_f) {
     sps[n].int_f(n, (chan == SIPS_REQUEST_CHAN), 1); /* 0->1 */
   }
}


/*** OS interface ***/

SipsErr
sim_sips_send(int n, int lo, void* data, int delayDelivery)
{
  int    chan  = lo ? SIPS_REQUEST_CHAN : SIPS_REPLY_CHAN;
  uint64 first = *(uint64*)data;
  int    r     = (int)first; /* destination: drop bits over 32 */
  int    dbuf;
  SIPSChannel* spchan;

#ifndef DATA_HANDLING
  ASSERT(!sips_disabled);
#endif

  if (r < 0 || r >= nsps) return SIPS_BADDEST; /* bad dest */

  /* XXX
   * XXX should check nodemap here
   * XXX
   */
  if (0) return SIPS_REMOTEDEAD;

  spchan = &sps[r].chan[chan];
  if (spchan->num_active >= BUFS_PER_SIPS_CHANNEL) {
#if (DEBUG > 0)
     LogEntry("sim_sips_send   ", n, "to %d chan %d FULL\n", r, !lo);
#endif
     /* return "remote busy, retry" status */
     return SIPS_RETRYREMOTE;
  }

  dbuf = (spchan->head + spchan->num_active) % BUFS_PER_SIPS_CHANNEL;
  ASSERT(spchan->buf[dbuf].arrival == BUF_EMPTY);
  spchan->num_active += 1;

  /* place SIPS in destination buffer, but don't deliver it yet.
   * This is a simple way to model SIPS propagation delays.
   *
   * Note that we have to take the cyclecount from the receiver
   * to ensure that cycle counts in the queue increase monotonically
   * (this is an issue on cpus like embra with cycle count skews).
   *
   * Also note that at this point, the sender CPU # is written to the
   * first word in the destination buffer.
   */
  bcopy(data, spchan->buf[dbuf].data, SIPS_SIZE);
  spchan->buf[dbuf].arrival = CPUVec.CycleCount(r) + SIPS_LATENCY;
  *((int*)&spchan->buf[dbuf].data[0]) = n; /* XXX assumes big endian */


  /* set-up a callback to be interrupted later. The callback will give
   * the receiving cpu the interrupt.
   */
  if (delayDelivery) {
     /* use when switching CPUs.  Need to delay RestoreArrivals until
        CPUVec is set for new cpu.  Also because cycle count drops to
        zero at CPU switch, adjust arrival time */
     spchan->buf[dbuf].arrival -= CPUVec.CycleCount(r);
     if (!callbackInstalled) {
        EventDoCallback(0, SipsRestoreArrivals, &sipsCallbackHdr, NULL, 1);
        callbackInstalled = 1;
     }
  } else {
#if (DEBUG == 0)
     CPUVec.Deliver_SIPS(r, chan, SIPS_LATENCY);
#else
     CPUVec.Deliver_SIPS(r, (n << 4) | chan, SIPS_LATENCY);
     LogEntry("sim_sips_send   ", n, 
              "to %d chan %d head %d active %d arrival %lld\n", 
              r, !lo, spchan->head, 
              spchan->num_active, spchan->buf[dbuf].arrival);
#endif
  }
  return SIPS_OK;
}


int
sim_sips_get(int n, int lo, void *data)
{
  int chan = lo ? SIPS_REQUEST_CHAN : SIPS_REPLY_CHAN;
  SIPSChannel* spchan;
  SIPSBuf* spbuf;
  ASSERT(0 <= n && n < nsps);
  spchan = &sps[n].chan[chan];

  /* is there anything in the queue? */
  if (spchan->num_delivered > 0) {
     spbuf = &spchan->buf[spchan->head];
     ASSERT(spbuf->arrival != BUF_EMPTY);
     /* has the thing at the head of the queue arrived at the receiver? */
     if (spbuf->arrival <= CPUVec.CycleCount(n)) {

#if (DEBUG > 1)
        LogEntry("sim_sips_get    ", n, "chan %d buf %d\n", !lo, spchan->head);
#endif
        bcopy(spbuf->data, data, SIPS_SIZE);
        return 0;
     }
  }
#if (DEBUG > 1)
  LogEntry("sim_sips_get    ", n, "chan %d EMPTY\n", !lo);
#endif
  return 1;  /* fail */
}


void
sim_sips_ack(int n, int lo)
{
  int chan = lo ? SIPS_REQUEST_CHAN : SIPS_REPLY_CHAN;
  SIPSChannel* spchan;
  SIPSBuf* spbuf;

  ASSERT(0 <= n && n < nsps);

  spchan = &sps[n].chan[chan];

  if (spchan->num_delivered > 0) {
     spbuf = &spchan->buf[spchan->head];
     ASSERT(spbuf->arrival != BUF_EMPTY);
     if (spbuf->arrival <= CPUVec.CycleCount(n)) {
#if (DEBUG > 0)
        int oldhead = spchan->head;
#endif
        spbuf->arrival = BUF_EMPTY;
        spchan->num_active -= 1;
        spchan->num_delivered -= 1;
        spchan->head = (spchan->head + 1) % BUFS_PER_SIPS_CHANNEL;
        if ((spchan->num_delivered == 0) && sps[n].int_f)
           sps[n].int_f(n, (chan == SIPS_REQUEST_CHAN), 0); /* 1->0 */
#if (DEBUG > 0)
        LogEntry("sim_sips_ack    ", n, 
                 "chan %d buf %d newhead %d active %d\n", 
                 !lo, oldhead, spchan->head, spchan->num_active);
#endif
     } else {
#if (DEBUG > 0)
        LogEntry("sim_sips_ack    ", n, "chan %d PENDING\n", !lo);
#endif
     }
  } else {
#if (DEBUG > 0)
     LogEntry("sim_sips_ack    ", n, "chan %d EMPTY\n", !lo);
#endif
  }
}


/*** Checkpointing support ***/


/* After a checkpoint restore, need to ensure that all interrupts
 * will occur at the right times. Unfortunately can't do this
 * from the checkpoint restore code itself because CPUVec.Deliver_SIPS
 * isn't set up.  We set SipsRestoreArrivals on cycle 1 to
 * do the work.
 *
 * If the interrupt was already pending (ie. stored as set in the
 * cpu state in the checkpoint) that's fine, the cpu will do sipsGet
 * perhaps before this annotation fires but the state is set correctly
 * in sps.
 *
 * oops: this code doesn't work because EventDoCallback tries to
 * check cycle number from the cpu vector.  Disable for now.
 */

static void
SipsRestoreArrivals(int cpuNum, EventCallbackHdr *hdr, void *arg)
{
   int          i, j, k, ind;
   SimTime      ctime, arrival;
   SIPSChannel* spchan;
   
   ASSERT(callbackInstalled);
   callbackInstalled = 0;

   for (i=0; i<nsps; i++) {
      for (j=0; j<SIPS_NUM_CHANNELS; j++) {
         spchan = &sps[i].chan[j];
         ctime  = CPUVec.CycleCount(i);

         for (k = 0; k < spchan->num_active; k++) {
            ind     = (spchan->head + k) % BUFS_PER_SIPS_CHANNEL;
            arrival = spchan->buf[ind].arrival;

            ASSERT(arrival != BUF_EMPTY);
            if (arrival > ctime) {
               CPUVec.Deliver_SIPS(i, j, arrival - ctime);
            } else {
               CPUVec.Deliver_SIPS(i, j, 1);
            }
         }
      }
   }
}

void
sim_sips_cpt_from_magic(CptDescriptor* cptd)
{
  int i, j, k;
  int pending_arrivals = 0;
  SimTime arrival;
  uint version = 1;

  Simcpt_OptionalUint(cptd, "sips_version", 0, 0, 0);
  Simcpt_CptUint(cptd, "sips_version", 0, 0, &version);

  if (version == 0) {
     /* restoring old checkpoint; only simport versions of those,
      * and they don't use sips.  Toss data.
      */
     char toss_buf[SIPS_SIZE];
     uint toss_state;

     for(i = 0; i < nsps; i++) {
        for (j=0; j<SIPS_NUM_CHANNELS; j++) {
           Simcpt_CptUint(cptd, "sips_state",i,j, &toss_state);
           Simcpt_CptBlock(cptd, "sips_data",i,j, &toss_buf, SIPS_SIZE, -1);
        }
     }
     return;
  }
     
  for (i=0; i< nsps; i++) {
     for (j = 0; j < SIPS_NUM_CHANNELS; j++) {
        Simcpt_CptUint(cptd, "sips_num_active", i, j,
                       &sps[i].chan[j].num_active);
        Simcpt_CptUint(cptd, "sips_queue_head", i, j,
                       &sps[i].chan[j].head);
        for (k = 0; k < BUFS_PER_SIPS_CHANNEL; k++) {
           if (cptd->mode == CPT_SAVE) {
              /* if saving a checkpoint, convert arrival times to
               * time-in-future so will arrive at the right delta
               * after checkpoint restore 
               */
              arrival = sps[i].chan[j].buf[k].arrival;
              if (arrival != BUF_EMPTY) {
                 if (arrival > CPUVec.CycleCount(i))
                    arrival -= CPUVec.CycleCount(i);
                 else
                    arrival = 0;
              }
              Simcpt_CptULL(cptd, "sips_buf_arrival", i, (j * 1000) + k,
                             &arrival);
           } else {
              /* if restoring a checkpoint, need to set the correct
               * callback or the interrupt will get lost
               */
              Simcpt_CptULL(cptd, "sips_buf_arrival", i, (j * 1000) + k,
                             & arrival);
              if (arrival != BUF_EMPTY) {
                 /*
                   ASSERT(CPUVec.Deliver_SIPS);
                   CPUVec.Deliver_SIPS(i, j, arrival);

                   Can't do this yet given initialization order of
                   simos.  Delay to cycle 1;
                 */
                 pending_arrivals = 1;
                 sps[i].chan[j].buf[k].arrival = arrival;
              }
           }
              
           Simcpt_CptBlock(cptd, "sips_buf_data", i, (j * 1000) + k,
                           sps[i].chan[j].buf[k].data, SIPS_SIZE, -1);
        }
     }
  }

  if (pending_arrivals) {
     Sim_Warning("SIPS: Checkpoint had pending sips, but can't set callbacks for them\n");
     Sim_Warning("SIPS: SIPS will be delayed if restoring into embra/busuma\n");
     Sim_Warning("SIPS: SIPS will be *dropped* if restoring into flashlite\n");
#ifdef notdef
     EventDoCallback(0, SipsRestoreArrivals, &sipsCallbackHdr, 0, 1);
#endif
  }
}