kern_invoke.cxx

C++ 编写的EROS RTOS

  inv.suppressXfer = false;  /* suppress compiler bitching */
  
  /* If preparation causes a depend entry to get zapped, it may be US
   * that gets zapped.  Set up to check for that...
   */
  PteZapped = false;
  ObjectHeader::BeginTransaction();
  
  SetupEntryBlock(inv);

#ifdef OPTION_DDB
  if ( ddb_inv_flags )
    goto general_path;
#endif

  /* Send is a pain in the butt.  Fuck 'em if they can't take a joke. */
  if (inv.invType == IT_Send)
    goto general_path;

  /* If it's a segment key, it might be a red segment key.  Take the
   * long way:
   */
  if (inv.keyType == KT_Segment)
    goto general_path;

#ifdef KT_Wrapper
  /* If it's a wrapper key, take the long way:
   */
  if (inv.keyType == KT_Wrapper)
    goto general_path;
#endif

  /* Handle the return via returner case efficiently: */
  if (inv.key->IsType(KT_Returner)) {
    inv.key = inv.entry.key[3];
    inv.keyType = inv.key->GetType();
    if (inv.keyType != KT_Resume) {
      /* NOTE: this also captures key being unprepared!!  */
      goto general_path;
    }
    inv.entry.key[3] = &Key::VoidKey;
  }
    
  if (inv.keyType <= KT_Resume) {
    inv.invokee = inv.key->gk.pContext;
    if (inv.invokee->IsWellFormed() == false)
      goto general_path;

    /* If this is a resume key, we know the process is in the
     * RS_Waiting state.  If it is a start key, the process might be
     * in any state.  Do not solve the problem here to avoid loss of
     * I-cache locality in this case; we are going to sleep anyway.
     */
    if (inv.keyType == KT_Start && inv.invokee->runState != RS_Available)
      goto general_path;
  }
  else {
    if (inv.invType != IT_Call)
      goto general_path;
    
    inv.invokee = this;
  }
  
#ifdef OPTION_PURE_ENTRY_STRINGS
  if (inv.entry.len != 0)
    SetupEntryString(inv);
#endif

  if (inv.invokee->IsRunnable() == false)
    goto general_path;
  
  if (inv.keyType == KT_Resume && inv.key->keyData != KsitResume)
    inv.suppressXfer = true;

  /* Following is a truly disgusting piece of numerology, which is
   * done deliberately:
   */
  assert (IT_Call == RS_Waiting);
  assert (IT_Reply == RS_Available);
  runState = inv.invType;

  /* It does not matter if the invokee fails to prepare!
   * Note that we may be calling this with 'this == 0'
   */
  inv.invokee->SetupExitBlock(inv);
#ifdef OPTION_PURE_EXIT_STRINGS
  if (inv.validLen != 0)
    inv.invokee->SetupExitString(inv, inv.validLen);
#endif

  {
#if defined(DBG_WILD_PTR) || defined(TESTING_INVOCATION)
    if (dbg_wild_ptr)
      Check::Consistency("DoKeyInvocation() before invoking handler\n");
#endif

#if defined(OPTION_KERN_TIMING_STATS)
    pre_handler = rdtsc();
#endif
    keyHandler[inv.keyType](inv);
#if defined(OPTION_KERN_TIMING_STATS)
    {
      extern uint32_t inv_delta_reset;
      if (inv_delta_reset == 0) {
	extern uint64_t inv_handler_cy;
	uint64_t post_handler = rdtsc();
	inv_handler_cy += (post_handler - pre_handler);
	Invocation::KeyHandlerCycles[inv.keyType][inv.invType] +=
	  (post_handler - pre_handler);
	Invocation::KeyHandlerCounts[inv.keyType][inv.invType] ++;
      }
    }  
#endif

#if defined(DBG_WILD_PTR) || defined(TESTING_INVOCATION)
    if (dbg_wild_ptr)
      Check::Consistency("DoKeyInvocation() after invoking handler\n");
#endif

    assert (InvocationCommitted);
#ifndef NDEBUG
    InvocationCommitted = false;    
#endif
  }

  /* Now for the tricky part.  It's possible that the proces did an
   * invocation whose effect was to blow the invokee apart.  I know of
   * no way to speed these checks:
   */

  if (inv.invokee->IsNotRunnable()) {
    if (inv.invokee->procRoot == 0)
      goto invokee_died;

    inv.invokee->Prepare();

    if (inv.invokee->IsNotRunnable())
      goto invokee_died;
  }

  inv.invokee->runState = RS_Running;
  if (!inv.suppressXfer) {
    inv.invokee->DeliverResult(inv);
#if defined(DBG_WILD_PTR) || defined(TESTING_INVOCATION)
    if (dbg_wild_ptr)
      Check::Consistency("DoKeyInvocation() after DeliverResult()\n");
#endif
  }

  if (inv.invokee != this) {
    /* Following is only safe because we handle non-call on primary
     * key in the general path.
     */
    if (inv.keyType == KT_Resume)
      inv.invokee->kr.ZapResumeKeys();

    Thread::Current()->MigrateTo(inv.invokee);
#ifdef OPTION_DDB
    if (inv.invokee->cpuReserve == &CpuReserve::InactiveReserve)
      dprintf(true, "Thread now in ctxt 0x%08x w/ bad schedule\n", 
		      inv.invokee);
#endif
  }
  
  if (runState == RS_Available)
    stallQ.WakeAll();
    
  inv.Cleanup();
  inv.invokee = 0;

#ifdef OPTION_KERN_TIMING_STATS
  {
    extern uint32_t inv_delta_reset;

    if (inv_delta_reset == 0) {
      extern uint64_t inv_delta_cy;

      uint64_t bot_time = rdtsc();
#ifdef OPTION_KERN_EVENT_TRACING
      extern uint64_t inv_delta_cnt0;
      extern uint64_t inv_delta_cnt1;

      uint64_t bot_cnt0 = Machine::ReadCounter(0);
      uint64_t bot_cnt1 = Machine::ReadCounter(1);
      inv_delta_cnt0 += (bot_cnt0 - top_cnt0);
      inv_delta_cnt1 += (bot_cnt1 - top_cnt1);
#endif
      inv_delta_cy += (bot_time - top_time);
    }
    else
      inv_delta_reset = 0;
  }
#endif

#if defined(DBG_WILD_PTR) || defined(TESTING_INVOCATION)
  if (dbg_wild_ptr)
    Check::Consistency("bottom DoKeyInvocation()");
#endif
  
  return;

 general_path:
  runState = RS_Running;
  /* This path has its own, entirely separate recovery logic.... */
  DoGeneralKeyInvocation();
  return;
  
 invokee_died:
  inv.invokee = 0;
  inv.Cleanup();
  Thread::Current()->MigrateTo(0);
}

Thread *threadToRelease = 0;
  
void
Process::DoGeneralKeyInvocation()
{
#ifdef OPTION_KERN_TIMING_STATS
  uint64_t top_time = rdtsc();
  uint64_t pre_handler;
#ifdef OPTION_KERN_EVENT_TRACING
  uint64_t top_cnt0 = Machine::ReadCounter(0);
  uint64_t top_cnt1 = Machine::ReadCounter(1);
#endif
#endif

  threadToRelease = 0;
  
  /* Set up the entry block, faulting in any necessary data pages and
   * constructing an appropriate kernel mapping:
   */
  SetupEntryBlock(inv);

#ifdef OPTION_PURE_ENTRY_STRINGS
  if (inv.entry.len != 0)
    SetupEntryString(inv);
#endif

#ifdef GATEDEBUG
  dprintf(true, "Populated entry block\n");
#endif

  assert(inv.key->IsPrepared());
  
  /* There are two cases where the actual invocation may proceed on a
   * key other than the invoked key:
   * 
   *   Invocation of kept red segment key proceeds as invocation on
   *     the keeper, AND observes the slot 2 convention of the format
   *     key!!!   Because this must overwrite slot 2, it must occur
   *     following the entry block preparation.
   * 
   *   Gate key to malformed domain proceeds as invocation on void.
   * 
   * The red seg test is done first because the extracted gate key (if
   * any) needs to pass the well-formed test too.
   *
   * In the latest revised kernel, the Wrapper node is beginning to
   * subsume the role of the red segment node. We do either/or logic
   * here because we don't want them to nest.
   */
  
#ifdef KT_Wrapper
  if ( inv.key->IsType(KT_Wrapper)
       && inv.key->IsReadOnly() == false
       && inv.key->IsNoCall() == false
       && inv.key->IsWeak() == false ) {

    /* The original plan here was to hide all of the sanity checking
     * for the wrapper node in the PrepAsWrapper() logic. That doesn't
     * work out -- it causes the wrapper node to need deprepare as a
     * unit when slots are altered, with the unfortunate consequence
     * that perfectly good mapping tables can get discarded. It is
     * therefore better to check the necessary constraints here.
     */
    Node *wrapperNode = (Node *) inv.key->GetObjectPtr();
    Key& fmtKey = wrapperNode->slot[WrapperFormat];

    if (fmtKey.IsType(KT_Number)
	&& wrapperNode->slot[WrapperKeeper].IsGateKey() ) {
      /* Unlike the older red segment logic, the format key has
       * preassigned slots for the keeper, address space, and so
       * forth.
       */

      if (fmtKey.nk.value[0] & WRAPPER_SEND_NODE) {
	/* Not hazarded because invocation key */
	inv.scratchKey.NH_Set(*inv.key);
	inv.scratchKey.SetType(KT_Node);
	inv.entry.key[2] = &inv.scratchKey;
	inv.flags |= INV_SCRATCHKEY;
      }

      if (fmtKey.nk.value[0] & WRAPPER_SEND_WORD)
	inv.entry.w1 = fmtKey.nk.value[1];

      /* Not hazarded because invocation key */
      inv.key = &(wrapperNode->slot[WrapperKeeper]);
      inv.keyType = inv.key->GetType();

      /* Prepared resume keys can only reside in dirty objects! */
      if (inv.keyType == KT_Resume)
	wrapperNode->MakeObjectDirty();
	    
      inv.key->Prepare();	/* MAY YIELD!!! */
    }
  } 
#else
  if ( inv.key->IsType(KT_Segment)
       && inv.key->IsRedSegmentKey()
       && inv.key->IsReadOnly() == false
       && inv.key->IsNoCall() == false
       && inv.key->IsWeak() == false ) {
    /*    dprintf(false, "It's a red segment key\n"); */

    Node *redSegNode = (Node *) inv.key->GetObjectPtr();
    Key& fmtKey = redSegNode->slot[RedSegFormat];

    /* Only pass through if red segment is well-formed and has gate
     * key as keeper!
     */
    
    if ( fmtKey.IsType(KT_Number) ) {
      uint32_t kprSlot = REDSEG_GET_KPR_SLOT(fmtKey.nk);

      if ( kprSlot != EROS_NODE_SLOT_MASK
	   && redSegNode->slot[kprSlot].IsGateKey() ) {

	/* See if we need to clobber slot 2 with a node key to the red
	 * segment.  Need to do this BEFORE we bash inv.key... :-)
	 */
	if ( REDSEG_GET_SENDNODE(fmtKey.nk) == REDSEG_SEND_NODE ) {
	  /* Not hazarded because invocation key */
	  inv.scratchKey.NH_Set(*inv.key);
	  inv.scratchKey.SetType(KT_Node);
	  inv.entry.key[2] = &inv.scratchKey;
	  inv.flags |= INV_SCRATCHKEY;
	}

	/* Not hazarded because invocation key */
	inv.key = &(redSegNode->slot[kprSlot]);
	inv.keyType = inv.key->GetType();

	/* Prepared resume keys can only reside in dirty objects! */
	if (inv.keyType == KT_Resume)
	  redSegNode->MakeObjectDirty();
	    
	inv.key->Prepare();	/* MAY YIELD!!! */
	inv.entry.w1 = fmtKey.nk.value[1];
      }
      else
	dprintf(true, "Keeper not gate key or no keeper\n");
    }
    else
      dprintf(true, "Format key is bad\n");
  }
#endif
  
  assert(inv.key->IsPrepared());

  inv.invokee = 0;		/* until proven otherwise */
  
  /* Right now a corner case here is buggered because we have not yet
   * updated the caller's runstate according to the call type.  As a
   * result, a return on a start key to yourself won't work in this
   * code.
   */
  
  if ( inv.key->IsGateKey() ) {
    assert (inv.key->IsPrepared());
    /* Make a local copy (subvert alias analysis pessimism) */
    Process *p = inv.key->gk.pContext;
    inv.invokee = p;
    p->Prepare();		/* may yield */

    /* This is now checked in pk_GateKey.cxx */
    if (inv.keyType == KT_Resume && inv.key->keyData != KsitResume)
      inv.suppressXfer = true;

    if (p->IsWellFormed() == false) {
      /* Not hazarded because invocation key */
      /* Pretend he invoked a void key. */
      inv.key = &Key::VoidKey;
      inv.keyType = KT_Void;
      inv.invokee = this;
      inv.entry.key[3] = &Key::VoidKey;
#ifndef NDEBUG
      printf("Jumpee malformed\n");
#endif
    }
#ifndef NDEBUG
    else if (inv.keyType == KT_Resume &&
	     p->runState != RS_Waiting) {
      fatal("Resume key to wrong-state context\n");
    }
#endif
    else if (inv.keyType == KT_Start && p->runState != RS_Available) {
      Thread::Current()->SleepOn(p->stallQ);
      Thread::Current()->Yield();
    }
#if 0
    else if ( inv.invType == IT_Call ) {
      BuildResumeKey(inv.resumeKey);
      inv.entry.key[3] = &inv.resumeKey;
      inv.flags |= INV_RESUMEKEY;
    }
#endif
  }
  else if (inv.invType == IT_Call) {
    /* Call on non-gate key always returns to caller and requires no
     * resume key.
     *
     * ISSUE: This will not look right to DDB, but it's correct. When
     * you next look at puzzling DDB state and wonder why the resume
     * key never got generated, it is because I did not want to deal
     * with the necessary key destruction overhead. The register
     * renaming key fabrication strategy should regularize this rather
     * nicely when we get there.
     */
    inv.invokee = this;
    inv.entry.key[3] = &Key::VoidKey;
  }
  else {
    /* Kernel key invoked with REPLY or SEND.  Key in slot 3 must be a
     * resume key, and if so the process must be waiting, else we will
     * not return to anyone.
     */

    Key& rk = *inv.entry.key[3];
    rk.Prepare();

    /* Kernel keys return as though via the returner, so the key in
     * slot four must be a resume key to a party in the right state.
     */

    assert(rk.IsHazard() == false);
    
    if (rk.IsPreparedResumeKey()) {
      Process *p = rk.gk.pContext;
      p->Prepare();

      assert(p->runState == RS_Waiting);
      inv.invokee = p;

      /* it can, however, be a fault key.  Since we are not going via
       * GateKey(), request xfer suppression here.
       */
      if (rk.keyData == KsitFault)
	inv.suppressXfer = true;
    }
    else
      assert (inv.invokee == 0);
  }

  /* Pointer to the domain root (if any) whose sleepers we should
   * awaken on successful completion:
   */
  Process *wakeRoot = 0;

  /* Revise the invoker runState to what it will be when this is all
   * over.  We'll fix it above if we yield.
   */
  static uint8_t newState[3] = {
    RS_Available,		/* IT_Reply */
    RS_Waiting,			/* IT_Call */
    RS_Running,			/* IT_Send */
  };

  runState = newState[inv.invType];