numa.c

一个用在mips体系结构中的操作系统

   List_Insert(INVAL_TO_LIST(lreq), LIST_ATREAR(&freeInvalReqList));
}



/* 
 * Send the required invalidates or downgrades 
 * Local invalidates have only a local delay
 * Remote invalidates have a net + dc + local delay
 */
static void
DoInvalidates(MemRequest *mreq) 
{
   DirState *dirState = mreq->dirState;
   int i;
   List_Links *itemPtr;
   InvalRequest *lreq;

   dirState->acks = 0;
   mreq->state = INVAL_WAIT;
   
   for (i = 0; i < TOTAL_CPUS; i++) {
      if (i == mreq->cpunum && mreq->transId>=0) {
         /* Requesting CPU */
         dirState->acks++;
      } else if (!DirIsASharer(dirState, i)) {
         /* 
          * Not a sharer
          * MemStat must still know of invalidations
          */
         CacheFakeInvalidate(i, BACKMAP_PADDR(mreq->addr, i), SCACHE_LINE_SIZE);
         dirState->acks++;
      } else {
         itemPtr = List_First(&freeInvalReqList);
         ASSERT(itemPtr != &freeInvalReqList);
         List_Remove(itemPtr);
         lreq = LIST_TO_INVAL(itemPtr);
         lreq->cpu = i;
         lreq->mreq = mreq;

         if (NumaLocalMem(i) != mreq->memnum) {
            /* CPU to be invalidated is remote */
            lreq->state = INVAL_NET_WAIT;
            EventDoCallback(mreq->cpunum, InvalRequestNet, (EventCallbackHdr *) lreq, 0,
                            NUMA_NET_TIME);
         } else {
            /* CPU to be invalidated is local */
            lreq->state = INVAL_LOCAL_WAIT;
            EventDoCallback(mreq->cpunum, InvalCollectAcks, (EventCallbackHdr *) lreq, 0,
                            NUMA_BUS_TIME);
         }
      }
   }
   StatsInc(COUNT_INVALORDNGRADESENT, mreq, (TOTAL_CPUS - dirState->acks));

   /* Make sure that there is at least 1 outstanding inval */
   ASSERT(dirState->acks != TOTAL_CPUS);
}
   
 
/* 
 * net delay done for invalidations that go remote
 * next step is the DC delay
 */
static void
InvalRequestNet(int num, EventCallbackHdr *hdr, void *v) 
{
   InvalRequest *lreq = (InvalRequest *)hdr;
   int memnum = NumaLocalMem(lreq->cpu);
   MemState  *mState = memState[memnum];

   lreq->hdr.rout = InvalRemoteDC;
   lreq->state = INVAL_REMOTE_DC_WAIT;
   lreq->dc_delay_time = NUMA_NILOCAL_DC_TIME;
   if(List_IsEmpty(&mState->localQueue)) {
      EventDoCallback(memnum, lreq->hdr.rout, 
                      (EventCallbackHdr *) lreq, 
                      0, lreq->dc_delay_time);
   }
   List_Insert(INVAL_TO_LIST(lreq), LIST_ATREAR(&mState->localQueue));
}
   

/* 
 * DC delay done for remote invals
 * Next do a local delay
 */
static void
InvalRemoteDC(int num, EventCallbackHdr *hdr, void *v) 
{
   InvalRequest *lreq = (InvalRequest *)hdr;
   int memnum = NumaLocalMem(lreq->cpu);
   MemState  *mState = memState[memnum];

   /* 
    * Remove request from list and schedule next callback
    */
   List_Remove(INVAL_TO_LIST(lreq));
   if (!List_IsEmpty(&mState->localQueue)) {
      InvalRequest *newReq;
      /*
       * Start the next memory request. 
       */
      newReq = LIST_TO_INVAL(List_First(&(mState->localQueue)));
      EventDoCallback(num, newReq->hdr.rout, (EventCallbackHdr *) newReq, 0,
                      lreq->dc_delay_time);
   }


   lreq->state = INVAL_REMOTE_LOCAL_WAIT;
   EventDoCallback(num, InvalAckNet, (EventCallbackHdr *) lreq, 0,
                      NUMA_BUS_TIME);
  
}

/*
 * Cache has been invalidated. Now send the ack to home
 */
static void
InvalAckNet(int num, EventCallbackHdr *hdr, void *v) 
{
   InvalRequest *lreq = (InvalRequest *)hdr;

   lreq->state = INVAL_ACK_NET_WAIT;
   EventDoCallback(num, InvalHomeDC, (EventCallbackHdr *) lreq, 0,
                      NUMA_NET_TIME);
}


/*
 * Do a DC delay for the inval acks returning to the home DC
 */
static void
InvalHomeDC(int num, EventCallbackHdr *hdr, void *v) 
{
   InvalRequest *lreq = (InvalRequest *)hdr;
   int memnum = lreq->mreq->memnum;
   MemState  *mState = memState[memnum];

   lreq->hdr.rout = InvalRemoteAckDone;
   lreq->state = INVAL_HOME_DC_WAIT;
   lreq->dc_delay_time = NUMA_NIREMOTE_DC_TIME;
   if(List_IsEmpty(&mState->localQueue)) {
      EventDoCallback(memnum, lreq->hdr.rout, 
                      (EventCallbackHdr *) lreq, 
                      0, lreq->dc_delay_time);
   }
   List_Insert(INVAL_TO_LIST(lreq), LIST_ATREAR(&mState->localQueue));
}



/* 
 * Remote ack received. Schedule next callback in DC queue
 * and pass this one on to InvalCollectAcks
 */
static void
InvalRemoteAckDone(int num, EventCallbackHdr *hdr, void *v) 
{

   InvalRequest *lreq = (InvalRequest *)hdr;
   int memnum = lreq->mreq->memnum;
   MemState  *mState = memState[memnum];

   /* 
    * Remove request from list and schedule next callback
    */
   List_Remove(INVAL_TO_LIST(lreq));
   if (!List_IsEmpty(&mState->localQueue)) {
      InvalRequest *newReq;
      /*
       * Start the next memory request. 
       */
      newReq = LIST_TO_INVAL(List_First(&(mState->localQueue)));
      EventDoCallback(num, newReq->hdr.rout, (EventCallbackHdr *) newReq, 0,
                      newReq->dc_delay_time);
   }

   InvalCollectAcks(memnum, (EventCallbackHdr *)lreq, 0);

}

/*
 * Invalidation acks for a cache line are counted here
 * When they are all received, we move on.
 * The invals could have crossed writebacks along the way.
 * If this happens, the request is NAKed.
 */
static void
InvalCollectAcks(int num, EventCallbackHdr *hdr, void *v) 
{
   InvalRequest *lreq = (InvalRequest *)hdr;
   int invalCpu = lreq->cpu;
   MemRequest *mreq = lreq->mreq;
   DirState *dirState = mreq->dirState;
   byte *data;
   unsigned wbNeeded = dirState->dirty;
   int wasDirty;

   
   data = (byte*) DATA_ADDR(mreq->machnum, mreq->addr);
   /* 
    * The following check should actually be done at the remote DC, 
    * But since we have the pending flag on the directory controller,
    * it should not matter. This does increase the window for the Inval
    * to cross the writeback.
    */

   if (mreq->mode == MEMSYS_EXCLUSIVE) { 
      if (CacheExtract(invalCpu, 
                       BACKMAP_PADDR(mreq->addr, invalCpu), 
                       SCACHE_LINE_SIZE, &wasDirty, data)) {
         /* Clear only if successful */
         DirClearASharer(dirState,invalCpu); 
      } else {
         wbNeeded = 0;
         /* Writeback or replacement hint in flight, NAK this request */
         mreq->status = MEMSYS_STATUS_NAK;
         StatsInc(COUNT_NAKS, mreq, 1);
      }
   } else {
      if (CacheWriteback(invalCpu, 
                         BACKMAP_PADDR(mreq->addr, invalCpu), 
                         SCACHE_LINE_SIZE, data)) {
         /* 
          * This line could have been replaced while we were waiting for the ack
          * Set it to be a sharer only if the return values says so
          */
         DirSetASharer(dirState,invalCpu,0); 
      } else {
          wbNeeded = 0;
          /* Writeback in flight, NAK this request */
          mreq->status = MEMSYS_STATUS_NAK;
          StatsInc(COUNT_NAKS, mreq, 1);
      }
   }

   if (wbNeeded) {
      /* need a sharing writeback to memory. Use the current Inval structure */
      lreq->hdr.rout = SharingWBDone;
      SharingWBDelay(mreq->memnum, lreq);
      if (ENABLE_COUNT)
         MigRepDoWriteCounter(mreq->reqAddr, mreq->memnum);
   } else {
      /* Put this inval structure back */
      lreq->state = ALL_DONE;
      List_Insert(INVAL_TO_LIST(lreq), LIST_ATREAR(&freeInvalReqList));
   }

   if (++(dirState->acks) == TOTAL_CPUS) {
      /* all acks are in, go to the next step */
      mreq->state = IN_MEM_DC;
      if (mreq->status == MEMSYS_STATUS_NAK) {
         /* this one got NAKed during inval, turn off the pending bit */
         mreq->dirState->pending = 0;
      }
      DirContDone(mreq->memnum, mreq);
   }
}


/*
 * All directory processing is over at this point
 * Send reply, local or remote
 */
static void
DirContDone(int num, MemRequest *mreq) 
{
   DirState *dirState = mreq->dirState;

   /*
    * DMA Data handling
    */
   if( mreq->cmd&MEMSYS_DMAFLAVOR ) { 
      byte *memAddr = (byte*)DATA_ADDR(mreq->machnum, mreq->reqAddr);
      if( mreq->cmd&MEMSYS_GETX ) {
         bcopy(mreq->data,memAddr,mreq->len);
      } else { 
         bcopy(memAddr,mreq->data,mreq->len);
      }
   }
   
   /* Update directory state */
   if (mreq->status != MEMSYS_STATUS_NAK) { 
      switch (mreq->cmd & MEMSYS_CMDMASK) {
      case MEMSYS_GET:
         dirState->pending = 0;
         if (!(mreq->cmd&MEMSYS_DMAFLAVOR)) {
            DirSetASharer(dirState, mreq->cpunum, 0);
         }
         if (ENABLE_COUNT)
            MigRepDoCmissCounter(mreq);
         break;

      case MEMSYS_GETX:
      case MEMSYS_UPGRADE:
         dirState->pending = 0;
         if (!(mreq->cmd&MEMSYS_DMAFLAVOR)) {
            DirSetASharer(dirState, mreq->cpunum, 1);
         }
         if (ENABLE_COUNT)
            MigRepDoCmissCounter(mreq);
         break;

      case MEMSYS_UNCWRITE:
      case MEMSYS_UNCWRITE_ACCELERATED:
      case MEMSYS_UNCREAD:
         break;

      case MEMSYS_WRITEBACK:
         /*
          * End of the road
          * update memory if data handling
          */
         if (mreq->data) { 
            byte *memAddr = (byte*)DATA_ADDR(mreq->machnum, mreq->replacedPaddr);
            bcopy(mreq->data, memAddr, SCACHE_LINE_SIZE);
            free(mreq->data);
         }
         DirClearASharer(dirState, mreq->cpunum);
         List_Insert(MREQ_TO_LIST(mreq), LIST_ATREAR(&freeMemReqList));
         mreq->drainNeeded = 0;
#ifdef MIG_REP_WBACK
         if (ENABLE_COUNT)
            MigRepDoWriteCounter(mreq->replacedPaddr, mreq->memnum);
#endif         
         return;

      case MEMSYS_REPLACEMENT_HINT:
         /*
          * End of the road
          */
         DirClearASharer(dirState, mreq->cpunum);
         List_Insert(MREQ_TO_LIST(mreq), LIST_ATREAR(&freeMemReqList));
         mreq->drainNeeded = 0;
         return;

      default:
         CPUError("Unknown memsys command (0x%x) in UmaCmd\n", mreq->cmd);
      }
   }

   /* Send this one on its way */
   if (mreq->memnum == mreq->localMemNum) {
      mreq->state = REPLY_LOCAL_WAIT;
      EventDoCallback(mreq->cpunum, SendReplyToCPU, 
                      (EventCallbackHdr *) mreq, 
                      0, NUMA_BUS_TIME);
   } else {
      mreq->state = REPLY_NET_WAIT;
      EventDoCallback(mreq->cpunum, ReplyNet, 
                      (EventCallbackHdr *) mreq, 
                      0, NUMA_NET_TIME);
   }
}

/*
 * Network delay done for the reply
 * Send it to the requesting DC
 */
static void
ReplyNet(int num, EventCallbackHdr *hdr, void *v)
{
   MemRequest *mreq = (MemRequest *)hdr;

   mreq->hdr.rout = ReplyDC;
   mreq->state = REPLY_LOCAL_DC_WAIT;
   mreq->dc_delay_time = NUMA_NIREMOTE_DC_TIME;

   DoDCDelay(mreq->localMemNum, mreq);
}


/*
 * Requesting DC is done
 * Do a local delay
 */ 
static void
ReplyDC(int num, EventCallbackHdr *hdr, void *v)
{
   MemRequest *mreq = (MemRequest *)hdr;
   MemState  *mState = memState[mreq->localMemNum];

   /* 
    * Remove request from list and schedule next callback
    */
   List_Remove(MREQ_TO_LIST(mreq));
   if (!List_IsEmpty(&mState->localQueue)) {
      MemRequest *newReq;
      /*
       * Start the next memory request. 
       */
      newReq = LIST_TO_MREQ(List_First(&(mState->localQueue)));
      EventDoCallback(num, newReq->hdr.rout, (EventCallbackHdr *) newReq, 0,
                      newReq->dc_delay_time);
   } 

   mreq->state = REPLY_LOCAL_WAIT;
   EventDoCallback(mreq->cpunum, SendReplyToCPU, (EventCallbackHdr *) mreq, 0,
                      NUMA_BUS_TIME);
 
}


/* 
 * All done. update the caches
 */
static void
SendReplyToCPU(int cpuNum, EventCallbackHdr *hdr, void *v)
{
   MemRequest *mreq = (MemRequest *)hdr;
   MemState  *mState = memState[mreq->memnum];
   StatsHist *hist;
   int isLocal = NumaIsLocal(mreq->cpunum, mreq->memnum) ? 1 : 0;
   byte* data;
   uint difftime = CPUVec.CycleCount(mreq->cpunum) - mreq->starttime;

   /* 
    * DMA requests are not necessarily aligned!!!
    */