numa.c

一个用在mips体系结构中的操作系统

NumaDumpStats(void)
{

   int i, j, isLocal;

   for (i = 0; i < NUM_MEMORIES(0); i++) {
      CPUPrint("MEM %d", i);
      for (j=0; j< COUNT_TOTAL; j++) {
         CPUPrint(" %lld", memState[i]->stats.counts[j]);
      }
      CPUPrint(" %lld %lld\n", memState[i]->stats.reqtime.count, memState[i]->stats.reqtime.sum);
   }

   isLocal = 1;
   CPUPrint("MEM LOCAL");
   for (j=0; j< COUNT_TOTAL; j++) {
      CPUPrint(" %lld", globalStats[isLocal].counts[j]);
   }
   CPUPrint(" %lld %lld\n", globalStats[isLocal].reqtime.count, globalStats[isLocal].reqtime.sum);
   
   isLocal = 0;
   CPUPrint("MEM REMOTE");
   for (j=0; j< COUNT_TOTAL; j++) {
      CPUPrint(" %lld", globalStats[isLocal].counts[j]);
   }
   CPUPrint(" %lld %lld\n", globalStats[isLocal].reqtime.count, globalStats[isLocal].reqtime.sum);
}


void
NumaStatus(void) 
{
   NumaDumpStats();
}


/*
 * Currently only writebacks and replacement hints need to
 * be drained. The others will cause processors to be stalled
 * and the clock will be run forward till all processors are unstalled.
 * For writebacks with data handling, the data needs to be written back.
 */
void
NumaDrain(void)
{
   int i;

   for (i = 0; i < MAX_OUTSTANDING; i++) { 
      MemRequest *mreq = MemRequestStorage + i;
      
      if (mreq->drainNeeded) {
         EventCallbackRemove((EventCallbackHdr*)mreq);
      
         if ((mreq->cmd & MEMSYS_CMDMASK) == MEMSYS_WRITEBACK) {
            if (mreq->data) { 
               byte *memAddr = (byte*)DATA_ADDR(mreq->machnum, mreq->replacedPaddr);
               bcopy(mreq->data, memAddr, SCACHE_LINE_SIZE);
               free(mreq->data);
            }
         }
      }
   }
}

/* Cache miss counting code for migration/replication */

/* 
 * Resets a cache line worth of counters every time
 * Mirrors Flashlite implementation and cache-line size (128)
 */
#define CMISS_COUNTERS_PER_CACHELINE (128/NUM_CPUS(0))
#define WRITE_COUNTERS_PER_CACHELINE (128*8)

#define MIGREP_LOCAL_PAGENUM(_a) (((_a)/PAGE_SIZE)%numaStripeSizePages)

/* Init all the structures for counting cache misses */
static void
MigRepNumaInit(void)
{
   int i, j;
   unsigned long size;

   ASSERT(NUMA_STRIPE_SIZE < 0);
   
   for(i=0; i<NUM_MEMORIES(0);i++) {
      migRepInfo[i].sampleCounter = SAMPLE_COUNT;
      for (j=0; j<MIGREP_PEND; j++) {
         migRepInfo[i].pendingPages[j] = MIGREP_NULLPAGE;
      }
      migRepInfo[i].pendingPagesCount = 0;

      if (!numaDoneInit) {
         size = numaStripeSizePages*NUM_MEMORIES(0);
         migRepInfo[i].cmissCounter = (unsigned char *) MALLOC(size, "migRep");
         size = numaStripeSizePages/8;
         migRepInfo[i].writeCounter = (unsigned char *) ZMALLOC(size, "migRep");
      }

      size = numaStripeSizePages/8;
      bzero((char *) migRepInfo[i].writeCounter, size);
      size = numaStripeSizePages*NUM_MEMORIES(0);
      for (j=0; j<size; j++) {
         migRepInfo[i].cmissCounter[j] = TRIGGER_THRESHOLD;
      }
   }

   /* calculate the period for calling the counter reset routine */
   counterResetInterval = (unsigned long)
      NanoSecsToCycles(RESET_INTERVAL*1000*1000)
      / (numaStripeSizePages/CMISS_COUNTERS_PER_CACHELINE);

   /* Init the write counter countdown variable, i.e. 1 in
      resetWriteCounters invocations */

   resetWriteCounters = WRITE_COUNTERS_PER_CACHELINE
      / CMISS_COUNTERS_PER_CACHELINE;
   /* Set up the first callback */
   bzero((char *) (&resetCounterCB), sizeof(EventCallbackHdr));
   EventDoCallback(0, MigRepPeriodic, &resetCounterCB, 0, counterResetInterval);
}


/* mark the page as written */
static void
MigRepDoWriteCounter(unsigned long addr, int memnum)
{
   unsigned long localPageNum = MIGREP_LOCAL_PAGENUM(addr);
   unsigned long cindex = localPageNum/8;
   unsigned long cbit = localPageNum % 8;

   migRepInfo[memnum].writeCounter[cindex] |= (1<<cbit);
}



/* Increment cache miss count for the page on a sampled basis */
static void
MigRepDoCmissCounter(MemRequest *mreq)
{
   int memnum = mreq->memnum;
   int cpunum = mreq->cpunum;
   int localmemnum = mreq->localMemNum;
   
   if (!(--migRepInfo[memnum].sampleCounter)) {
      unsigned long localPageNum = MIGREP_LOCAL_PAGENUM(mreq->reqAddr);
      unsigned long cindex = localPageNum*NUM_CPUS(0) + localmemnum;

      /* If GETX or UPGRADE mark it as a write */
      if ((mreq->cmd == MEMSYS_GETX) || (mreq->cmd == MEMSYS_UPGRADE)) {
         MigRepDoWriteCounter(mreq->reqAddr, memnum);
         if (ZERO_ON_WRITE) {
            int i;
            unsigned long ci;
            ASSERT(NUM_MEMORIES(0) <= 16);
            for(i=0; i<NUM_MEMORIES(0); i++) {
               if (i != localmemnum) {
                  ci = localPageNum*NUM_CPUS(0) + i;
                  migRepInfo[memnum].cmissCounter[ci] = TRIGGER_THRESHOLD;
               }
            }
         }
      }

      /* Only decrement if it has not hit the trigger */
      if(migRepInfo[memnum].cmissCounter[cindex]) {
         if(!(--migRepInfo[memnum].cmissCounter[cindex])) {
            /* This page is officially hot */
            if ((PAGE_NUM(mreq->reqAddr) > MAX_KERN) && (localmemnum != memnum)) {
               /* 
                * Interrupt only if it is not a static kernel page
                * and it is not a local page
                */
               migRepHotPage(localPageNum, cpunum, memnum, localmemnum);
            }
         }
      }

      migRepInfo[memnum].sampleCounter = SAMPLE_COUNT;
   }
}



/* Add hot page to list of pending pages and potentially interrupt the CPU */
static void
migRepHotPage(unsigned long localPageNum, int cpunum, int memnum, int localmemnum)
{
   int i, cputointr;
   unsigned long page;
   int qnode;
   
   if (INTR_HOT) {
      cputointr = cpunum;
      qnode = localmemnum;
      page = localPageNum + numaStripeSizePages*memnum;
   } else {
      cputointr = NumaMemFirstCPU(memnum);
      qnode = memnum;
      page = localPageNum;
   }
    
   for (i=0; i<MIGREP_PEND; i++) {
      if (migRepInfo[qnode].pendingPages[i] == MIGREP_NULLPAGE) {
         migRepInfo[qnode].pendingPages[i] = page;
         if (++migRepInfo[qnode].pendingPagesCount >= MIGREP_PEND_INTR) {
            /* Interrupt the CPU, enough pending pages */
            CPUVec.MigRepStart(cputointr);
         }
         break;
      } else if (migRepInfo[qnode].pendingPages[i] == page) {
         /* page already there as hot. Ignore! */
         break;
      }
   }
   if (i == MIGREP_PEND)
      CPUWarning("NUMA WARNING: pending page count exceeded on %d\n", qnode);
/*   ASSERT(i != MIGREP_PEND); */
}


/*
 * Called from the kernel through simmagic.
 * Returns a pending page or 0 if no pages
 * Resets the interrupt when all pages are gone
 */
uint64
MigRepGetHotPage(int cpunum)
{
   int i, localmemnum;
   unsigned long page;
   
   localmemnum = NumaLocalMem(cpunum);

   page = MIGREP_NULLPAGE;
   for (i=0; i<MIGREP_PEND; i++) {
      if (migRepInfo[localmemnum].pendingPages[i] != MIGREP_NULLPAGE) {
         page = migRepInfo[localmemnum].pendingPages[i];
         migRepInfo[localmemnum].pendingPages[i] = MIGREP_NULLPAGE;
         migRepInfo[localmemnum].pendingPagesCount--;
         break;
      }
   }

   if (!migRepInfo[localmemnum].pendingPagesCount) {
      /* All pages done, reset the interrupt */
      CPUVec.MigRepEnd(cpunum);
   }

   return ((uint64)page);
}


/*
 * Called from the kernel through simmagic.
 * Returns the requested counter value as int64
 * This is because flashlite does it this way as a int64
 * Bit 19 of the address (20 active bits) is used to demux cache and write counters
 * Bottom 3 bits cannot be used.
 */
uint64
MigRepGetInfo(unsigned long addr, int memnum, unsigned long countType, 
              unsigned long countAddr)
{
   int i;
   uint64 counterVal = 0;
   unsigned long bitVal;
   uint64 dummy;
   unsigned long cindex, cbit;
   unsigned char *counter;

   if (!(ENABLE_COUNT)) {
      return ((uint64) 0);
   }

   if (countType) {
      /* Write counter */
      cindex = countAddr/8;
      cbit = countAddr%8;
      bitVal = (migRepInfo[memnum].writeCounter[cindex])&(1<<cbit);
      if (bitVal) {
         cbit = countAddr%64;
         counterVal = ((uint64) 1) << cbit;
      }
   } else {
      /* cache miss counter */
      cindex = countAddr*NUM_MEMORIES(0);
      counter = &(migRepInfo[memnum].cmissCounter[cindex]);
      if (FOUR_BIT_COUNTERS) {
         ASSERT(NUM_MEMORIES(0) <= 16);
         ASSERT(NUM_MACHINES == 1);
         for(i=0; i<NUM_MEMORIES(0); i++) {
            dummy =  counter[i];
            ASSERT(dummy < 16);
            counterVal |= ((dummy & 0xf) << (i*4));
         }
      } else {
         ASSERT(NUM_MEMORIES(0) <= 8);
         ASSERT(NUM_MACHINES == 1);
         for(i=0; i<NUM_MEMORIES(0); i++) {
            dummy =  counter[i];
            ASSERT(dummy < 256);
            counterVal |= ((dummy & 0xff) << (i*8));
         }
      }
   }
   return (counterVal);
}



/*
 * Called from the kernel through simmagic.
 * Sets the requested counter value as int64
 * This is because flashlite does it this way as a int64
 * Bit 19 of the address (20 active bits) is used to demux cache and write counters
 * Bottom 3 bits cannot be used.
 */
void
MigRepSetInfo(unsigned long addr, int memnum, uint64 val, 
              unsigned long countType, unsigned long countAddr)
{
   int i;
   unsigned long cindex, cbit, bitVal;
   unsigned char *counter;

   if (!(ENABLE_COUNT)) {
      return ;
   }

   if (countType) {
      /* Write counter */
      cindex = countAddr/8;
      cbit = countAddr%8;
      bitVal = 1 << cbit;
      if (val) {
         migRepInfo[memnum].writeCounter[cindex] |= bitVal;
      } else {
         migRepInfo[memnum].writeCounter[cindex] &= ~(bitVal);
      }
   } else {
      /* cache miss counter */
      cindex = countAddr*NUM_MEMORIES(0);
      counter = &(migRepInfo[memnum].cmissCounter[cindex]);
      if (FOUR_BIT_COUNTERS) {
         ASSERT(NUM_MEMORIES(0) <= 16);
         ASSERT(NUM_MACHINES == 1);
         for(i=0; i<NUM_MEMORIES(0); i++) {
            counter[i] = (char) ((val >> (i*4)) & 0xf);
            /* counter[i] = (char) ((val >> (i*8)) & 0x0ff); */
         }
      } else {
         ASSERT(NUM_MEMORIES(0) <= 8);
         ASSERT(NUM_MACHINES == 1);
         for(i=0; i<NUM_MEMORIES(0); i++) {
            counter[i] = (char) ((val >> (i*8)) & 0x0ff);
         }
      }
   }

}


/*
 * Called periodically to reset some miss counters
 * Each time through a MAGIC cache line (128 bytes) of cache-miss counters are reset
 * When resetWriteCounters goes to zero 128 bytes of write counters are reset.
 * Uses the callback mechanism
 */
static void
MigRepPeriodic(int cpuNum, EventCallbackHdr *hdr, void *v)
{
   int i, j, k;
   unsigned long nextResetIndex = cmissResetIndex + CMISS_COUNTERS_PER_CACHELINE;
   unsigned long countersToReset = CMISS_COUNTERS_PER_CACHELINE;
   unsigned char *counter;

   if (nextResetIndex >= numaStripeSizePages) {
      countersToReset = CMISS_COUNTERS_PER_CACHELINE - (nextResetIndex - numaStripeSizePages);
      nextResetIndex = 0;
   }

   for (k=0; k<NUM_MEMORIES(0); k++) {
      counter = &(migRepInfo[k].cmissCounter[cmissResetIndex]);
      for (i=0; i<countersToReset; i++) {
         for (j=0; j<TOTAL_CPUS; j++, counter++) {
            *counter = TRIGGER_THRESHOLD;
         }
      }
   }
   cmissResetIndex = nextResetIndex;

   if (!(--resetWriteCounters)) {
      resetWriteCounters = WRITE_COUNTERS_PER_CACHELINE/CMISS_COUNTERS_PER_CACHELINE;

      nextResetIndex = writeResetIndex + WRITE_COUNTERS_PER_CACHELINE;
      countersToReset = WRITE_COUNTERS_PER_CACHELINE;

      if (nextResetIndex >= numaStripeSizePages) {
         countersToReset = WRITE_COUNTERS_PER_CACHELINE - (nextResetIndex - numaStripeSizePages);
         nextResetIndex = 0;
      }

      for (k=0; k<NUM_MEMORIES(0); k++) {
         counter = &(migRepInfo[k].writeCounter[writeResetIndex/8]);
         for (i=0; i<(countersToReset/8); i++, counter++) {
               *counter = 0;
         }
      }
      writeResetIndex = nextResetIndex;
   }

   EventDoCallback(0, MigRepPeriodic, &resetCounterCB, 0, counterResetInterval);

}