pagefault.cxx

C++ 编写的EROS RTOS

 * should not have authority.
 * 
 * A case can be made, however, that a kept red segment should be
 * permitted to contain a NON-kept red segment that specifies a
 * background segment.  The main reason to desire this is to allow
 * (e.g.) VCSK to operate on a background segment that contains a
 * window.
 * 
 * For the moment, we do not support this, and I am inclined to
 * believe that it is unwise to do so until I see a case in which it
 * is useful.
 * 
 * Local windows are not yet supported by the SegWalk code, but none
 * of this is an issue for local windows,
 */

inline uint64_t
BLSS_MASK64(uint32_t blss, uint32_t frameBits)
{
  uint32_t bits_to_shift =
    (blss - EROS_PAGE_BLSS) * EROS_NODE_LGSIZE + frameBits; 

  uint64_t mask = (1ull << bits_to_shift);
  mask -= 1ull;
  
  return mask;
}

/* #define WALK_LOUD */
#define FAST_TRAVERSAL
bool
proc_DoPageFault(Process * p, ula_t la, bool isWrite, bool prompt)
{
  const int walk_root_blss = 4 + EROS_PAGE_BLSS;
  const int walk_top_blss = 2 + EROS_PAGE_BLSS;
  
#ifdef DBG_WILD_PTR
  if (dbg_wild_ptr)
    Check::Consistency("Top of DoPageFault()");
#endif

  DoPageFault_CallCounter++;
  
  DEBUG(pgflt) {
    printf("DoPageFault: ctxt=0x%08x EIP 0x%08x la=0x%08x, isWrite=%c prompt=%c\n",
		   p,
		   p->fixRegs.EIP,
		   la,
		   isWrite ? 't' : 'f',
		   prompt ? 't' : 'f');
  }
  
#ifdef OPTION_SMALL_SPACES
  uva_t va = la - p->bias;
#else
  uva_t va = la;
#endif
  
  /* If LA is simply out of range, then forget the whole thing: */
  if ( la >= KVA ) {
    dprintf(true, "Domain accessed kernel or small space la\n");
    p->SetFault(FC_InvalidAddr, va, false);
    return false;
  }

#ifdef OPTION_SMALL_SPACES
  if (p->smallPTE)
    return proc_DoSmallPageFault(p, la, isWrite, prompt);
#endif
			    
  /* If LA is simply out of range, then forget the whole thing: */
  if ( la >= UMSGTOP ) {
    dprintf(true, "Large domain accessed small space la\n");
    p->SetFault(FC_InvalidAddr, va, false);
    return false;
  }

  /* If we discover on the way to load the process that it's mapping
   * table register was voided, we substituted KERNPAGEDIR.  Notice
   * that here:
   */
  if ( p->fixRegs.MappingTable == KERNPAGEDIR )
    p->fixRegs.MappingTable = PTE_ZAPPED;

  /* Set up a WalkInfo structure and start building the necessary
   * mapping table, PDE, and PTE entries.
   */

  SegWalk wi;
  wi.faultCode = FC_NoFault;
  wi.traverseCount = 0;
  wi.segObj = 0;
  wi.vaddr = va;
  wi.frameBits = EROS_PAGE_ADDR_BITS;
  wi.writeAccess = isWrite;
  wi.invokeKeeperOK = !prompt;
  wi.invokeProcessKeeperOK = !prompt;
  wi.wantLeafNode = false;

  segwalk_init(wi, p->GetSegRoot());

  /* See if there is already a page table. If not, go find/build one. */
  PTE *pTable = (PTE *) (PTOV(p->fixRegs.MappingTable) & ~EROS_PAGE_MASK);

#ifdef WALK_LOUD
  dprintf(false, "pTable is 0x%x\n", pTable);
#endif
  
  if (pTable == 0)
    goto need_pgdir;
  
  {
    ObjectHeader *pTableHdr = ObjectCache::PhysPageToObHdr(PtoKPA(pTable));

    if (isWrite && !pTableHdr->rwProduct) {
      dprintf(true, "DoPageFault(): isWrite && !pTableHdr->rwProduct hdr 0x%x\n", pTableHdr);
      goto access_fault;
    }

    wi.canCall = pTableHdr->caProduct;
    
#ifdef FAST_TRAVERSAL
    /* We have a page directory conveying suitable access rights from
       the top.  See if the PDE has the necessary permissions: */
    {
      uint32_t pdeNdx = (la >> 22) & 0x3ffu;
      PTE& thePDE = pTable[pdeNdx];

      if ( PTE_IS(thePDE, PTE_V|PTE_USER) ) {

	/* We could short-circuit the walk in this case by remembering
	 * the status of /wi.canWrite/ in a spare bit in the PTE, but
	 * at the moment we do not do so because in most cases the
	 * write restriction appears lower down in the segment tree.
	 */

	if ( PTE_IS(thePDE, PTE_W) || !isWrite ) {
	  /* We have a valid PDE with the necessary permissions! */

	  pTable = (PTE *) thePDE.PageFrame();
	  pTableHdr = ObjectCache::PhysPageToObHdr(PtoKPA(pTable));
	  
	  wi.offset = wi.vaddr & ((1u << 22) - 1u);
	  wi.segBlss = pTableHdr->producerBlss;
	  wi.segObj = pTableHdr->producer;
	  wi.redSeg = pTableHdr->mp.redSeg;
	  if (wi.redSeg) {
	    wi.redSpanBlss = pTableHdr->mp.redSpanBlss;
	    wi.redSegOffset =
	      ((uint64_t) wi.vaddr) & BLSS_MASK64(wi.redSpanBlss, wi.frameBits);
	  }
#ifdef KT_Wrapper
	  wi.segObjIsWrapper = pTableHdr->mp.wrapperProducer;
#else
	  wi.segObjIsRed = pTableHdr->mp.redProducer;
#endif
	  wi.canWrite = pTableHdr->rwProduct ? true : false;
	  wi.canCall &= pTableHdr->caProduct;

#if 0
	  if (wi.redSeg && wi.offset != wi.redSegOffset) {
      dprintf(false, "pdr proc_WalkSeg: wi.producer 0x%x, wi.prodBlss %d wi.isRed %d\n"
		      "wi.vaddr 0x%x wi.offset 0x%X flt %d  wa %d segKey 0x%x\n"
		      "canCall %d canWrite %d\n"
		      "redSeg 0x%x redOffset 0x%X\n",
		      wi.segObj, wi.segBlss, wi.segObjIsRed,
		      wi.vaddr, wi.offset, wi.segFault, wi.writeAccess,
		      0x0,
		      wi.canCall, wi.canWrite,
		      wi.redSeg, wi.redSegOffset);

	    dprintf(true, "Found pg dir. Offset 0x%X RedSegOffset 0x%X spanBlss %d\n",
			    wi.offset, wi.redSegOffset, wi.redSpanBlss);
	  }
#endif
	  
	  /* NOTE: This is doing the wrong thing when the red segment
	   * needs to be grown upwards, because the segBlss in that
	   * case is less than the *potential* span of the red
	   * segment.
	   */
#ifdef WALK_LOUD
	  dprintf(false, "have_good_pde\n");
#endif
	  goto have_good_pde;
	}
      }
    }
#endif /* FAST_TRAVERSAL */
    
    wi.offset = wi.vaddr;
    wi.segBlss = pTableHdr->producerBlss;
    wi.segObj = pTableHdr->producer;
    wi.redSeg = pTableHdr->mp.redSeg;
    if (wi.redSeg) {
      wi.redSpanBlss = pTableHdr->mp.redSpanBlss;
      wi.redSegOffset =
	((uint64_t) wi.vaddr) & BLSS_MASK64(wi.redSpanBlss, wi.frameBits);
    }
#ifdef KT_Wrapper
    wi.segObjIsWrapper = pTableHdr->mp.wrapperProducer;
#else
    wi.segObjIsRed = pTableHdr->mp.redProducer;
#endif
    wi.canWrite = pTableHdr->rwProduct ? true : false;

#ifdef WALK_LOUD
    dprintf(false, "have_pgdir\n");
#endif
    goto have_pgdir;
  }
  
 need_pgdir:
  /* No page directory was found, so we need to construct a page
   * directory. */
  
  p->fixRegs.MappingTable = PTE_IN_PROGRESS;
  
  /* Begin the traversal... */
  if ( !proc_WalkSeg(p, wi, walk_root_blss,
		     (PTE *)&p->fixRegs.MappingTable, 0, false) ) { 
    p->SetFault(wi.faultCode, va, false);
    return false;
  }

  /* If the wrong depend entry was reclaimed, we may have just lost
   * the mapping table entry. If we are still good to go, : */
  if (p->fixRegs.MappingTable == PTE_ZAPPED)
    Thread::Current()->Yield();

  /* Since the address space pointer register lacks permission bits,
   * we cannot be here due to lack of permissions at this
   * level. Therefore, if we are processing this path at all the
   * mapping table must have been invalid, in which case it should now
   * be PTE_IN_PROGRESS. */
  assert (p->fixRegs.MappingTable == PTE_IN_PROGRESS);

  /* We can now reset the value to the zap guard. */
  p->fixRegs.MappingTable = PTE_ZAPPED;
  
  assert (pTable == 0);
  if (pTable == 0) {
    /* See if a mapping table has already been built for this address
     * space.  If so, just use it.  Using wi.segBlss is okay here
     * because the mapping table pointer will be zapped if anything
     * above this point gets changes, whereupon the gunk the the page
     * directory will no longer matter.
     */
    ObjectHeader *pTableHdr =
      wi.segObj->FindProduct(wi, EROS_NODE_LGSIZE /* ndx */,
			     wi.canWrite, wi.canCall);
    
    if (pTableHdr == 0)
      pTableHdr = proc_MakeNewPageDirectory(wi);

    pTable = (PTE *) ObjectCache::ObHdrToPage(pTableHdr);

    p->fixRegs.MappingTable = VTOP(pTable);
  }

 have_pgdir:
  
#ifndef NDEBUG
  {
    ObjectHeader *pTableHdr = ObjectCache::PhysPageToObHdr(PtoKPA(pTable));

    assert(wi.segBlss == pTableHdr->producerBlss);
    assert(wi.segObj == pTableHdr->producer);
    assert(wi.redSeg == pTableHdr->mp.redSeg);
    assert(wi.canWrite == (pTableHdr->rwProduct ? true : false));
  }
#endif

  {
    /* Start building the PDE entry: */
    uint32_t pdeNdx = (la >> 22) & 0x3ffu;
    PTE& thePDE = pTable[pdeNdx];

    if (PTE_ISNOT(thePDE, PTE_V))
      thePDE.w_value = PTE_IN_PROGRESS;

    /* Translate the top 8 (10) bits of the address: */
    if ( !proc_WalkSeg(p, wi, walk_top_blss, &thePDE, 0, true) )
      return false;

    if (thePDE.w_value == PTE_ZAPPED)
      Thread::Current()->Yield();

    if (thePDE.w_value == PTE_IN_PROGRESS)
      thePDE.w_value = PTE_ZAPPED;    

    /* If we get this far, we need the page table to proceed further.
     * See if we need to build a new page table:
     */

    if (PTE_IS(thePDE, PTE_V)) {
      pTable = (PTE *) PTOV(thePDE.PageFrame());

      if (wi.canWrite && !PTE_IS(thePDE, PTE_W)) {
	/* We could only have taken this fault because writing was not
	 * enabled at the directory level, which means that we need to
	 * flush the PDE from the hardware TLB.
	 */
	thePDE.Invalidate();
      }
    }
    else {
      uint32_t productNdx = 0;

      /* Level 0 product need never be a read-only product.  We use
       * the write permission bit at the PDE level.
       */
      
      ObjectHeader *pTableHdr =
	wi.segObj->FindProduct(wi, productNdx, true, true);

      if (pTableHdr == 0)
	pTableHdr = proc_MakeNewPageTable(wi, productNdx);

      assert(wi.segBlss == pTableHdr->producerBlss);
      assert(wi.segObj == pTableHdr->producer);
      assert(wi.redSeg == pTableHdr->mp.redSeg);

      /* On x86, the page table is always RW product, and we rely on
	 the write permission bit at the PDE level: */
      assert(pTableHdr->rwProduct == true);

      pTable = (PTE *) ObjectCache::ObHdrToPage(pTableHdr);
    }

    /* The level 0 page table is still contentless - there is no
     * need to build depend table entries covering it's contents.
     * We simply need to fill in the page directory entry:
     */

    PTE_SET(thePDE, (VTOP((kva_t)pTable) & PTE_FRAMEBITS));
    PTE_SET(thePDE, PTE_ACC|PTE_USER|PTE_V);
    
    /* Using /canWrite/ instead of /isWrite/ reduces the number of
     * cases in which we need to rebuild the PDE without altering the
     * actual permissions, and does not require us to dirty a page.
     * This is legal only because on this architecture the node tree
     * and the page tables are congruent at this level.
     */
    if (wi.canWrite)
      PTE_SET(thePDE, PTE_W);
  }

#ifdef FAST_TRAVERSAL
 have_good_pde:
#endif
  
  {
    uint32_t pteNdx = (la >> 12) & 0x3ffu;
    PTE& thePTE = pTable[pteNdx];

    if (PTE_ISNOT(thePTE, PTE_V))
      thePTE.w_value = PTE_IN_PROGRESS;
    
    /* Translate the remaining bits of the address: */
    if ( !proc_WalkSeg(p, wi, EROS_PAGE_BLSS, &thePTE, 0, true) )
      return false;

    /* Depend entry triggered -- retry the fault */
    if (thePTE.w_value == PTE_ZAPPED)
      Thread::Current()->Yield();

    if (thePTE.w_value == PTE_IN_PROGRESS)
      thePTE.w_value = PTE_ZAPPED;    
    
    assert(wi.segObj);
    assert(wi.segObj->obType == ObType::PtDataPage ||
	   wi.segObj->obType == ObType::PtDevicePage);
    
    kpa_t pageAddr = 0;

    if (isWrite)
      wi.segObj->MakeObjectDirty();

    pageAddr = VTOP(ObjectCache::ObHdrToPage(wi.segObj));

    if (pageAddr == 0)
      dprintf(true, "wi.segObj 0x%08x at addr 0x%08x!! (wi=0x%08x)\n",
		      wi.segObj, pageAddr, &wi);

    assert ((pageAddr & EROS_PAGE_MASK) == 0);
    assert (pageAddr < PtoKPA(start) || pageAddr >= PtoKPA(end));
	  
    if (isWrite && PTE_IS(thePTE, PTE_V)) {
      /* We are doing this because the old PTE had insufficient
       * permission, so we must zap the TLB.
       */
      thePTE.Invalidate();
    }
  
    PTE_SET(thePTE, (pageAddr & PTE_FRAMEBITS) );
    PTE_SET(thePTE, DATA_PAGE_FLAGS);
    if (isWrite)
      PTE_SET(thePTE, PTE_W);
#ifdef WRITE_THROUGH
    if (CpuType >= 5)
      PTE_SET(thePTE, PTE_W);
#endif

  }
    
  if (PteZapped)
    Machine::FlushTLB();
    
#ifdef DBG_WILD_PTR
  if (dbg_wild_ptr)
    Check::Consistency("End of DoPageFault()");
#endif

  return true;

 access_fault:
  wi.faultCode = FC_Access;
  goto fault_exit;

 fault_exit:
#ifdef WALK_LOUD
    dprintf(true, "flt proc_WalkSeg: wi.producer 0x%x, wi.prodBlss %d wi.isRed %d\n"
		    "wi.vaddr 0x%x wi.offset 0x%X flt %d  wa %d\n"
		    "canCall %d canWrite %d\n",
		    wi.segObj, wi.segBlss, wi.segObjIsRed,
		    wi.vaddr, wi.offset, wi.segFault, wi.writeAccess,
		    wi.canCall, wi.canWrite);
#endif
  if (wi.invokeKeeperOK)
    return p->InvokeSegmentKeeper(wi);

  return false;
}

static ObjectHeader*
proc_MakeNewPageDirectory(SegWalk& wi)
{
  ObjectHeader *pTable = ObjectCache::GrabPageFrame();
  assert (pTable->kr.IsValid(pTable));
  pTable->obType = ObType::PtMappingPage;
  pTable->producerNdx = EROS_NODE_LGSIZE;
  pTable->producerBlss = wi.segBlss;

  pTable->mp.redSeg = wi.redSeg;
#ifdef KT_Wrapper
  pTable->mp.wrapperProducer = wi.segObjIsWrapper;
#else
  pTable->mp.redProducer = wi.segObjIsRed;
#endif
  pTable->mp.redSpanBlss = wi.redSpanBlss;
  pTable->rwProduct = wi.canWrite ? 1 : 0;
  pTable->caProduct = wi.canCall ? 1 : 0;
  pTable->SetDirtyFlag();

  kva_t tableAddr = ObjectCache::ObHdrToPage(pTable);

  bzero((void *) tableAddr, EROS_PAGE_SIZE);

  {
    uint32_t *kpgdir = (uint32_t *) PTOV(KERNPAGEDIR);
    uint32_t *upgdir = (uint32_t *) tableAddr;
    
    for (uint32_t i = (UMSGTOP >> 22); i < NPTE_PER_PAGE; i++)
      upgdir[i] = kpgdir[i];
  }    

  wi.segObj->AddProduct(pTable);

  return pTable;
}

static ObjectHeader*
proc_MakeNewPageTable(SegWalk& wi, uint32_t ndx)
{
  /* Need to make a new mapping table: */

  ObjectHeader *pTable = ObjectCache::GrabPageFrame();
  assert (pTable->kr.IsValid(pTable));
  pTable->obType = ObType::PtMappingPage;
  pTable->producerNdx = ndx;
  pTable->producerBlss = wi.segBlss;
  
  pTable->mp.redSeg = wi.redSeg;
#ifdef KT_Wrapper
  pTable->mp.wrapperProducer = wi.segObjIsWrapper;
#else
  pTable->mp.redProducer = wi.segObjIsRed;
#endif
  pTable->mp.redSpanBlss = wi.redSpanBlss;
  pTable->rwProduct = 1;
  pTable->caProduct = 1;	/* we use spare bit in PTE */
  pTable->SetDirtyFlag();

  kva_t tableAddr = ObjectCache::ObHdrToPage(pTable);

  bzero((void *)tableAddr, EROS_PAGE_SIZE);

#if 0
  printf("0x%08x->MkPgTbl(blss=%d,ndx=%d,rw=%c,ca=%c,"
		 "producerTy=%d) => 0x%08x\n",
		 wi.segObj,
		 wi.segBlss, ndx, 'y', 'y', wi.segObj->obType,
		 pTable);
#endif

  wi.segObj->AddProduct(pTable);

  return pTable;
}