numa.c

一个用在mips体系结构中的操作系统

/*
 * Copyright (C) 1996-1998 by the Board of Trustees
 *    of Leland Stanford Junior University.
 * 
 * This file is part of the SimOS distribution. 
 * See LICENSE file for terms of the license. 
 *
 */

/*****************************************************************
 *  File:  numa.c 
 *
 * NUMA memory system. 
 *
 * Parameters are:
 * NUMA_BUS_TIME: fixed delay for bus/interconnect between CPU and DC
 * NUMA_PILOCAL_DC_TIME
 * NUMA_PIREMOTE_DC_TIME
 * NUMA_NILOCAL_DC_TIME
 * NUMA_NIREMOTE_DC_TIME
 * NUMA_NET_TIME: fixed delay for network transit
 * NUMA_MEM_TIME: contention based delay for access to DRAM
 * Minimum local miss time is 2*(NUMA_BUS_TIME) + (NUMA_PILOCAL_DC_TIME) + (NUMA_MEM_TIME)
 * Minimum remote miss time is 2*(NUMA_BUS_TIME)
 *              + NUMA_PIREMOTE_DC_TIME
 *              + NUMA_NILOCAL_DC_TIME
 *              + NUMA_NIREMOTE_DC_TIME
 *              + (NUMA_MEM_TIME) + 2*(NUMA_NET_TIME)
 *
 * Author: $Author: bosch $
 * Date:   $Date: 1998/02/10 00:37:20 $
 *****************************************************************/

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include "syslimits.h"
#include "scache.h"
#include "memsys.h"
#include "sim_error.h"
#include "simutil.h"
#include "eventcallback.h"
#include "list.h"
#include "cpu_interface.h"
#include "registry.h"

/* 
 * Flag for memory system init to prevent repeated mallocs when switching
 */
static uint numaDoneInit = 0;  

#define NUMA_MAX_MEMORIES    64

#define PAGE_NUM(x) (x)/PAGE_SIZE
static uint numaStripeSizePages;
static uint numaStripeSizeBytes;
static uint numaStripeSizeLines;
static uint numaStripeChunk;

/*
 * Numa memsystem version of the counting code in flashlite
 * Counts, at the home, cache misses/CPU (countdown from Trigger) and write misses.
 * Things are done to mirror the flashlite counting implementation closely
 * Count misses in DirContDone for Get and GetX
 * Mark writes in DirContDone for Writeback and InvalCollectAcks for sharing writebacks
 * 
 */
/* # of hot pages pending */
#define MIGREP_PEND 10
/* # of hot pages before interrupt */
#define MIGREP_PEND_INTR migRepPendIntr
int migRepPendIntr = 2;
/* value returned when no pages are left */
#define MIGREP_NULLPAGE 0xffffffff

typedef struct {
   unsigned long sampleCounter;         /* sample count frequency */
   unsigned long pendingPages[MIGREP_PEND];     /* hot pages pending */
   unsigned long pendingPagesCount;     /* number of hot pending pages */
   unsigned char *cmissCounter;         /* cache-miss counter, a byte perc CPU per page */
   unsigned char *writeCounter;         /* write counter, a bit per page */
} MigRepNodeInfo;
MigRepNodeInfo migRepInfo[NUMA_MAX_MEMORIES];   /* One per memory */

/* All variables for resetting counters */
unsigned long counterResetInterval;     /* Cycles between invocations of the reset routine */
unsigned long cmissResetIndex = 0;      /* Next cache-miss counter to reset */
unsigned long writeResetIndex = 0;      /* Next write counter to reset */
unsigned long resetWriteCounters;       /* countdown for resetting write counters */
EventCallbackHdr resetCounterCB;        /* periodic call back for resetting counters */

/* Policy parameters. Will be eventually set through tcl */
/* Sample 1:N */
#define SAMPLE_COUNT migRepSampleCount
int migRepSampleCount = 10;
/* Assuming sampling 1:10 */
int migRepTriggerThreshold = 9;
#define TRIGGER_THRESHOLD migRepTriggerThreshold
/* Reset interval in ms */
int migRepResetInterval = 500;
#define RESET_INTERVAL migRepResetInterval
/* enable counting */
int migRepEnableCounting = 0;
#define ENABLE_COUNT migRepEnableCounting
/* static kern size in pages */
int migRepMaxKern = 0;
#define MAX_KERN migRepMaxKern
/* which node to interrupt */
int migRepIntrHot = 0;
#define INTR_HOT migRepIntrHot
int migRepZeroOnWrite = 0;
#define ZERO_ON_WRITE migRepZeroOnWrite
int migRep4BitCounters = 0;
#define FOUR_BIT_COUNTERS migRep4BitCounters

#ifdef SOLO
#define NumaAddrToMemnum(_addr)  SOLO_PA_NODE(SoloDecompressAddr(0,_addr))
#define NumaAddrToMemline(_addr) (SOLO_PA_OFFSET(SoloDecompressAddr(0,_addr))/SCACHE_LINE_SIZE)
#else
#define NumaAddrToMemnum(_addr) ((_addr/numaStripeSizeBytes)%NUM_MEMORIES(0))
#define NumaAddrToMemline(_addr) (((_addr/numaStripeChunk)*numaStripeSizeLines) \
                                  + ((_addr/SCACHE_LINE_SIZE)%numaStripeSizeLines))

#endif

/*
 * Memory number that is local to this CPU
 * This is the same as the CPU, except if clustering (a la DASH)is used
 */
#define NumaLocalMem(c) ((c)*NUM_MEMORIES(0)/NUM_CPUS(0))
/* Test if cpu c is local to memory m */
#define NumaIsLocal(c, m) ((m) == NumaLocalMem(c))

#define NumaMemFirstCPU(m) ((m)*NUM_CPUS(0)/NUM_MEMORIES(0));
   
/* States for memory and invalidation requests. Helps debugging */
enum MemReqState { REQ_LOCAL_WAIT,      /* Waiting for the local bus */
                   REQ_LOCAL_DC_WAIT,   /* Waiting for the local DC */
                   REQ_NET_WAIT,        /* Sending to the remote DC */
                   REQ_REMOTE_DC_WAIT,  /* Waiting for the remote DC */
                   IN_MEM_DC,           /* In the DC (not really a wait) */
                   INVAL_WAIT,          /* Waiting for invalidates to be done */
                   MEM_WAIT,            /* Waiting to write to DRAM */
                   INVAL_LOCAL_WAIT,    /* Invalidate: waiting for local bus */
                   INVAL_NET_WAIT,      /* Invalidate: Sending to remote DC */
                   INVAL_REMOTE_DC_WAIT,        /* Invalidate: waiting for remote DC */ 
                   INVAL_REMOTE_LOCAL_WAIT,     /* Invalidate: waiting for remote bus */
                   INVAL_ACK_NET_WAIT,  /* Invalidate: Sending Ack to home */
                   INVAL_HOME_DC_WAIT,   /* Invalidate: waiting for home DC */
                   WB_MEM_WAIT,         /* Sharing writeback going to memory */
                   REPLY_NET_WAIT,      /* Sending reply to requesting DC */
                   REPLY_LOCAL_WAIT,    /* Sending reply on bus to requesting processor */
                   REPLY_LOCAL_DC_WAIT, /* Reply waiting for requesting DC */
                   ALL_DONE };

/* A memory request structure */
#define MAX_OUTSTANDING (SIM_MAXCPUS*MEMSYS_MAX_OUTSTANDING)
typedef struct MemRequest  {
   EventCallbackHdr hdr;  /* Must be first!! */
   List_Links  link; /* Must be second (after EventCallbackhdr */
#define MREQ_TO_LIST(_m) (&((_m)->link))
#define LIST_TO_MREQ(_l) ((MemRequest *) (((char *)(_l)) - sizeof(EventCallbackHdr)))
   enum MemReqState state;
   uint dc_delay_time;
   /* Fields till here are common with the InvalRequest and should be kept that way */
   int cmd;
   int transId;
   uint reqAddr;
   uint addr;    /* Address scache aligned */
   unsigned int mode;
   PA replacedPaddr;
   int replaceWasDirty;
   int status;
   int result;   
   int memnum;
   int machnum;
   int cpunum;
   int localMemNum;
   struct DirState *dirState;
   uint starttime;
   byte *data;  /* data handling - dma */
   int len;
   int drainNeeded;
}  MemRequest;

static MemRequest MemRequestStorage[MAX_OUTSTANDING];
static List_Links freeMemReqList;  /* List of available mreq structures */

/*
 * The inval structure must resemble the request structure 
 * for the first few fields, as they are mixed in DC and memory queues
 */
#define MAX_INVALS (MAX_OUTSTANDING*5)
typedef struct InvalRequest  {
   EventCallbackHdr hdr;  /* Must be first!! */
   List_Links  link; /* Must be second (after EventCallbackhdr */
#define INVAL_TO_LIST(_m) (&((_m)->link))
#define LIST_TO_INVAL(_l) ((InvalRequest *) (((char *)(_l)) - sizeof(EventCallbackHdr)))
   enum MemReqState state;
   uint dc_delay_time;
   /* Fields till here are common with the MemRequest and should be kept that way */
   int cpu;             /* CPU to be invalidated */
   MemRequest *mreq;    /* The memory request for this invalidate */
} InvalRequest;

static InvalRequest InvalRequestStorage[MAX_INVALS];
static List_Links freeInvalReqList;  /* List of available inval structures */

/*
 * Directory state for a memory line.
 */
typedef struct DirState {
   unsigned char  numsharers;  /* Number of cpus with line cached. */
   unsigned char  pending:1;   /* 1 if invalidates are pending. */
   unsigned char  dirty:1;     /* Someone has it exclusive */
   unsigned char  bitmap[(SIM_MAXCPUS+7)/8];
   unsigned char  acks;        /* inval acks pending for this directory line */
} DirState;

#define STAT_HIST_BUCKETS 100
#define DEFAULT_HIST_BUCKET_SCALE 30

typedef struct StatsHist {
   unsigned int scaledivisor;
   SimCounter sum;
   SimCounter count;
   SimCounter counts[STAT_HIST_BUCKETS];
} StatsHist;

/*
 * Stats collected per memory system.
 */
typedef struct NumaMemStats {
   SimCounter counts[COUNT_TOTAL];
   StatsHist  reqtime;                  /* latency */
} NumaStats;

/* Global Info about misses, local and remote home */
static NumaStats globalStats[2];

/*
 * State associated with a memory in the system.
 */

typedef struct MemState  { 
   List_Links localQueue;   /* Queue for the Directory Controller */
   List_Links memoryQueue;    /* Queue to memory */
   NumaStats stats;   
   DirState dirState[1]; /* really NUMA_MAX_BYTES_PER_MEMORY/SCACHE_LINE_SIZE */
} MemState;

static MemState *memState[NUMA_MAX_MEMORIES];

#ifdef SOLO
#include "solo_page.h"
#define DATA_ADDR(_m, _pa) SoloGetMemoryAddr(SoloDecompressAddr(0,_pa))
#else
#define DATA_ADDR(_m, _pa) PHYS_TO_MEMADDR(_m, _pa)
#endif

static void DoDCDelay(int num, MemRequest *mreq) ;
static void RequestLocal(int cpuNum, EventCallbackHdr *hdr, void *v);
static void RequestLocalDC(int num, EventCallbackHdr *hdr, void *v) ;
static void RequestNet(int cpuNum, EventCallbackHdr *hdr, void *v) ;
static void RequestRemoteDC(int num, EventCallbackHdr *hdr, void *v) ;
static void DirContEnter(int num, MemRequest *mreq) ;
static void MemContDelay(int num, MemRequest *mreq) ;
static void MemContDone(int num, EventCallbackHdr *hdr, void *v) ;
static void SharingWBDelay(int num, InvalRequest *lreq);
static void SharingWBDone(int num, EventCallbackHdr *hdr, void *v); 
static void DoInvalidates(MemRequest *mreq) ;
static void InvalRequestNet(int num, EventCallbackHdr *hdr, void *v) ;
static void InvalRemoteDC(int num, EventCallbackHdr *hdr, void *v) ;
static void InvalAckNet(int num, EventCallbackHdr *hdr, void *v) ;
static void InvalHomeDC(int num, EventCallbackHdr *hdr, void *v) ;
static void InvalCollectAcks(int num, EventCallbackHdr *hdr, void *v) ;
static void InvalRemoteAckDone(int num, EventCallbackHdr *hdr, void *v); 
static void DirContDone(int num, MemRequest *mreq) ;
static void ReplyNet(int num, EventCallbackHdr *hdr, void *v);
static void ReplyDC(int num, EventCallbackHdr *hdr, void *v);
static void SendReplyToCPU(int cpuNum, EventCallbackHdr *hdr, void *v);

static void MigRepNumaInit(void);
static void MigRepDoWriteCounter(unsigned long addr, int memnum);
static void MigRepDoCmissCounter(MemRequest *mreq);
static void migRepHotPage(unsigned long localPageNum, int cpunum, int memnum, int localmemnum);
uint64 MigRepGetHotPage(int memnum);
uint64 MigRepGetInfo(unsigned long addr, int memnum, 
                                 unsigned long countType, unsigned long countAddr );
void MigRepSetInfo(unsigned long addr, int memnum, uint64 val,
                   unsigned long countType, unsigned long countAddr );
static void MigRepPeriodic(int cpuNum, EventCallbackHdr *hdr, void *v);

void NumaInit(void);
Result NumaCmd(int cpuNum, int cmd, PA addr, 
               int transId, PA replacedPaddr, int writeback, byte *data);
void NumaDone(void);
void NumaDumpStats(void);
void NumaStatus(void);
void NumaDrain(void);


/* Increment a histogram structure */
static void 
StatsIncHist(StatsHist *s, uint value)
{
   uint bucket;
   if (s->scaledivisor == 0) {
      bucket = value / DEFAULT_HIST_BUCKET_SCALE;
   } else { 
      bucket = value / s->scaledivisor;
   }
   if (bucket > STAT_HIST_BUCKETS-1) {
      bucket = STAT_HIST_BUCKETS-1;
   }
   s->counts[bucket]++;
   s->sum += value;
   s->count++;
}


/* increment a stat bucket in the memory and local/remote home structures */
static void
StatsInc(int bucket, MemRequest *mreq, int i)
{
   MemState *mState = memState[mreq->memnum];
   int isLocal = NumaIsLocal(mreq->cpunum, mreq->memnum) ? 1 : 0;

   mState->stats.counts[bucket] += i;
   globalStats[isLocal].counts[bucket] += i;
}



#ifndef SOLO

/*****************************************************************
 * remap region support 
 *****************************************************************/

static PA backmapMask;
static PA nodeaddrMask;

static void
NumaUpdateBackmapMask(void)
{
   /* the backmapMask is an optimization that summarizes all the enabled
    * remap masks; it has ones in any bit position which, if set, means
    * this physical address could not be the target of a remap on any
    * CPU.
    *
    * nodeaddrMask masks out the node id bits.
    */

   int i;
   backmapMask = nodeaddrMask;
   for (i=0; i<TOTAL_CPUS; i++) {
      if (remapVec->RemapEnable[i]) {
	 backmapMask &= remapVec->RemapMask[i];
      }
   }
}

static void
NumaInitRemap(void)
{
   int i;

   /* We initialize backmapMask to all ones except for the
    * bits that might be set in the node id field
    */

   nodeaddrMask = ~0;
   for (i=0; i<TOTAL_CPUS; i++) {
      if (!remapVec->NodeAddrInitialized) {
         int m = M_FROM_CPU(i);
#ifdef TORNADO
         remapVec->NodeAddr[i] = (i*NUM_MEMORIES(m)/NUM_CPUS(m)) *
            (MEM_SIZE(m) / NUM_CPUS(m));
         CPUWarning("numa: nodeaddr for %d is %lx\n",
                    i, (unsigned long)(remapVec->NodeAddr[i]));
#elif defined(SIM_ORIGIN)
         remapVec->NodeAddr[i] = 
            MEMADDR_TO_PHYS(m, SIM_MEM_ADDR(m) + 
                            (MCPU_FROM_CPU(i)/2) * 2 * (MEM_SIZE(m) / NUM_CPUS(m)));
#else
         remapVec->NodeAddr[i] = 
            MEMADDR_TO_PHYS(m, SIM_MEM_ADDR(m) + 
                            i * (MEM_SIZE(m) / NUM_CPUS(m)));
#endif

      }
      nodeaddrMask &= ~remapVec->NodeAddr[i];
   }
   remapVec->NodeAddrInitialized = 1;

   /* now zero out the low bits of the backmapmask */
   NumaUpdateBackmapMask();
}

static void
NumaSetRemap(int cpunum, PA mask)
{
   remapVec->RemapMask[cpunum] = mask;
   NumaUpdateBackmapMask();
}

static void
NumaControlRemap(int cpunum, int isEnabled)
{
   remapVec->RemapEnable[cpunum] = isEnabled;
   NumaUpdateBackmapMask();
}

static PA
NumaGetNodeAddress(int cpunum)
{
   return remapVec->NodeAddr[cpunum];
}