rf_engine.c

RAIDFrame是个非常好的磁盘阵列RAID仿真工具

/*
 * Copyright (c) 1995 Carnegie-Mellon University.
 * All rights reserved.
 *
 * Author: William V. Courtright II, Mark Holland, Rachad Youssef
 *
 * Permission to use, copy, modify and distribute this software and
 * its documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 *
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 *
 * Carnegie Mellon requests users of this software to return to
 *
 *  Software Distribution Coordinator  or  [e-mail]
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 *
 * any improvements or extensions that they make and grant Carnegie the
 * rights to redistribute these changes.
 */

/****************************************************************************
 *                                                                          *
 * engine.c -- code for DAG execution engine                                *
 *                                                                          *
 * Modified to work as follows (holland):                                   *
 *   A user-thread calls into DispatchDAG, which fires off the nodes that   *
 *   are direct successors to the header node.  DispatchDAG then returns,   *
 *   and the rest of the I/O continues asynchronously.  As each node        *
 *   completes, the node execution function calls FinishNode().  FinishNode *
 *   scans the list of successors to the node and increments the antecedent *
 *   counts.  Each node that becomes enabled is placed on a central node    *
 *   queue.  A dedicated dag-execution thread grabs nodes off of this       *
 *   queue and fires them.                                                  *
 *                                                                          *
 *   NULL nodes are never fired.                                            *
 *                                                                          *
 *   Terminator nodes are never fired, but rather cause the callback        *
 *   associated with the DAG to be invoked.                                 *
 *                                                                          *
 *   If a node fails, the dag either rolls forward to the completion or     *
 *   rolls back, undoing previously-completed nodes and fails atomically.   *
 *   The direction of recovery is determined by the location of the failed  *
 *   node in the graph.  If the failure occured before the commit node in   *
 *   the graph, backward recovery is used.  Otherwise, forward recovery is  *
 *   used.                                                                  *
 *                                                                          *
 ****************************************************************************/

/* 
 * $Locker:  $
 *
 * $Log: rf_engine.c,v $
 * Revision 1.56  1996/07/28  20:31:39  jimz
 * i386netbsd port
 * true/false fixup
 *
 * Revision 1.55  1996/07/22  19:52:16  jimz
 * switched node params to RF_DagParam_t, a union of
 * a 64-bit int and a void *, for better portability
 * attempted hpux port, but failed partway through for
 * lack of a single C compiler capable of compiling all
 * source files
 *
 * Revision 1.54  1996/07/17  21:00:58  jimz
 * clean up timer interface, tracing
 *
 * Revision 1.53  1996/07/15  17:22:18  jimz
 * nit-pick code cleanup
 * resolve stdlib problems on DEC OSF
 *
 * Revision 1.52  1996/06/17  03:17:08  jimz
 * correctly shut down engine thread in kernel
 *
 * Revision 1.51  1996/06/14  15:02:10  jimz
 * make new engine code happy in simulator
 *
 * Revision 1.50  1996/06/14  14:19:48  jimz
 * use diskgroup to control engine thread, make all engine-thread-related
 * stuff per-array
 *
 * Revision 1.49  1996/06/10  11:55:47  jimz
 * Straightened out some per-array/not-per-array distinctions, fixed
 * a couple bugs related to confusion. Added shutdown lists. Removed
 * layout shutdown function (now subsumed by shutdown lists).
 *
 * Revision 1.48  1996/06/09  02:36:46  jimz
 * lots of little crufty cleanup- fixup whitespace
 * issues, comment #ifdefs, improve typing in some
 * places (esp size-related)
 *
 * Revision 1.47  1996/06/06  01:23:23  jimz
 * fix bug in node traversal when firing multiple nodes simultaneously
 *
 * Revision 1.46  1996/06/05  18:06:02  jimz
 * Major code cleanup. The Great Renaming is now done.
 * Better modularity. Better typing. Fixed a bunch of
 * synchronization bugs. Made a lot of global stuff
 * per-desc or per-array. Removed dead code.
 *
 * Revision 1.45  1996/05/30  12:59:18  jimz
 * make etimer happier, more portable
 *
 * Revision 1.44  1996/05/30  11:29:41  jimz
 * Numerous bug fixes. Stripe lock release code disagreed with the taking code
 * about when stripes should be locked (I made it consistent: no parity, no lock)
 * There was a lot of extra serialization of I/Os which I've removed- a lot of
 * it was to calculate values for the cache code, which is no longer with us.
 * More types, function, macro cleanup. Added code to properly quiesce the array
 * on shutdown. Made a lot of stuff array-specific which was (bogusly) general
 * before. Fixed memory allocation, freeing bugs.
 *
 * Revision 1.43  1996/05/27  18:56:37  jimz
 * more code cleanup
 * better typing
 * compiles in all 3 environments
 *
 * Revision 1.42  1996/05/24  22:17:04  jimz
 * continue code + namespace cleanup
 * typed a bunch of flags
 *
 * Revision 1.41  1996/05/24  04:28:55  jimz
 * release cleanup ckpt
 *
 * Revision 1.40  1996/05/23  00:33:23  jimz
 * code cleanup: move all debug decls to rf_options.c, all extern
 * debug decls to rf_options.h, all debug vars preceded by rf_
 *
 * Revision 1.39  1996/05/20  16:15:17  jimz
 * switch to rf_{mutex,cond}_{init,destroy}
 *
 * Revision 1.38  1996/05/18  20:09:54  jimz
 * bit of cleanup to compile cleanly in kernel, once again
 *
 * Revision 1.37  1996/05/18  19:51:34  jimz
 * major code cleanup- fix syntax, make some types consistent,
 * add prototypes, clean out dead code, et cetera
 *
 * Revision 1.36  1996/05/15  20:24:19  wvcii
 * fixed syntax bug in SIMULATE clause above ProcessNode
 *
 * Revision 1.35  1996/05/08  21:01:24  jimz
 * fixed up enum type names that were conflicting with other
 * enums and function names (ie, "panic")
 * future naming trends will be towards RF_ and rf_ for
 * everything raidframe-related
 *
 * Revision 1.34  1996/05/08  15:25:28  wvcii
 * eliminated dead code
 * merged common cases (sim/user/kernel)
 * entire node lists (arrays) now fired atomically
 * reordered source code for readability
 * beefed-up & corrected comments
 *
 * Revision 1.33  1996/05/07  19:39:40  jimz
 * 1. fixed problems in PropogateResults() with nodes being referenced
 * after they were no longer valid
 * 2. fixed problems in PropogateResults() with the node list being
 * incorrectly threaded
 *
 * Revision 1.32  1996/05/07  19:03:56  wvcii
 * in PropagateResults, fixed a bug in the  rollBackward case:
 *   node data is copied before the call to FinishNode which
 *   frees the node and destroys its data.
 *
 * Revision 1.31  1996/05/07  17:45:17  jimz
 * remove old #if 0 code from PropogateResults() (was kept in
 * previous version for archival purposes (rcsdiff))
 *
 * Revision 1.30  1996/05/07  17:44:19  jimz
 * fix threading of nodes to be fired in PropagateResults()
 * fix iteration through skiplist in PropagateResults()
 * fix incorrect accesses to freed memory (dereferencing a
 *   node that was freed by the action of calling FinishNode()
 *   on it, which in turn completed its DAG) in PropagateResults()
 *
 * Revision 1.29  1996/05/02  15:04:15  wvcii
 * fixed bad array index in PropagateResults
 *
 * Revision 1.28  1995/12/12  18:10:06  jimz
 * MIN -> RF_MIN, MAX -> RF_MAX, ASSERT -> RF_ASSERT
 * fix 80-column brain damage in comments
 *
 * Revision 1.27  1995/12/08  15:07:03  arw
 * cache code cleanup
 *
 * Revision 1.26  1995/11/07  16:18:01  wvcii
 * numerous changes associated with roll-away error recovery
 * when a node fails, dag enters rollForward or rollBackward state
 *
 * Revision 1.25  1995/09/06  19:27:17  wvcii
 * added debug vars enableRollAway and debugRecovery
 *
 */

#include "rf_threadstuff.h"

#ifndef KERNEL
#include <stdio.h>
#include <stdlib.h>
#endif /* !KERNEL */

#include <sys/errno.h>

#include "rf_dag.h"
#include "rf_engine.h"
#include "rf_threadid.h"
#include "rf_etimer.h"
#include "rf_general.h"
#include "rf_dagutils.h"
#include "rf_shutdown.h"
#include "rf_raid.h"

#ifndef SIMULATE
static void DAGExecutionThread();
#endif /* !SIMULATE */

#define DO_INIT(_l_,_r_) { \
  int _rc; \
  _rc = rf_create_managed_mutex(_l_,&(_r_)->node_queue_mutex); \
  if (_rc) { \
    return(_rc); \
  } \
  _rc = rf_create_managed_cond(_l_,&(_r_)->node_queue_cond); \
  if (_rc) { \
    return(_rc); \
  } \
}

/* synchronization primitives for this file.  DO_WAIT should be enclosed in a while loop. */
#ifndef KERNEL

#define DO_LOCK(_r_)      RF_LOCK_MUTEX((_r_)->node_queue_mutex)
#define DO_UNLOCK(_r_)    RF_UNLOCK_MUTEX((_r_)->node_queue_mutex)
#define DO_WAIT(_r_)      RF_WAIT_COND((_r_)->node_queue_cond, (_r_)->node_queue_mutex)
#define DO_SIGNAL(_r_)    RF_SIGNAL_COND((_r_)->node_queue_cond)

#else /* !KERNEL */

/*
 * XXX Is this spl-ing really necessary?
 */
#define DO_LOCK(_r_)      { ks = splbio(); RF_LOCK_MUTEX((_r_)->node_queue_mutex); }
#define DO_UNLOCK(_r_)    { RF_UNLOCK_MUTEX((_r_)->node_queue_mutex); splx(ks); }
#define DO_WAIT(_r_)      mpsleep(&(_r_)->node_queue, PZERO, "raidframe nq", 0, (void *) simple_lock_addr((_r_)->node_queue_mutex), MS_LOCK_SIMPLE)
#define DO_SIGNAL(_r_)    wakeup(&(_r_)->node_queue)

#endif /* !KERNEL */

static void rf_ShutdownEngine(arg)
  void  *arg;
{
  RF_Raid_t *raidPtr;

  raidPtr = (RF_Raid_t *)arg;
#ifndef SIMULATE
  raidPtr->shutdown_engine = 1;
  DO_SIGNAL(raidPtr);
  RF_THREADGROUP_WAIT_STOP(&raidPtr->engine_tg);
#endif /* !SIMULATE */
}

int rf_ConfigureEngine(
  RF_ShutdownList_t  **listp,
  RF_Raid_t           *raidPtr,
  RF_Config_t         *cfgPtr)
{
  int rc, tid;

  if (rf_engineDebug) {
    rf_get_threadid(tid);
  }
  DO_INIT(listp,raidPtr);

  raidPtr->node_queue = NULL;
  raidPtr->dags_in_flight = 0;

#ifndef SIMULATE
  rc = rf_init_managed_threadgroup(listp, &raidPtr->engine_tg);
  if (rc)
    return(rc);
  
  /* we create the execution thread only once per system boot.
   * no need to check return code b/c the kernel panics if it can't create the thread.
   */
  if (rf_engineDebug) {
    printf("[%d] Creating engine thread\n", tid);
  }
  if (RF_CREATE_THREAD(raidPtr->engine_thread, DAGExecutionThread, raidPtr)) {
    RF_ERRORMSG("RAIDFRAME: Unable to create engine thread\n");
    return(ENOMEM);
  }
  if (rf_engineDebug) {
    printf("[%d] Created engine thread\n", tid);
  }
  RF_THREADGROUP_STARTED(&raidPtr->engine_tg);
  RF_THREADGROUP_WAIT_START(&raidPtr->engine_tg);
  /* engine thread is now running and waiting for work */
  if (rf_engineDebug) {
    printf("[%d] Engine thread running and waiting for events\n", tid);
  }
#endif /* !SIMULATE */

  rc = rf_ShutdownCreate(listp, rf_ShutdownEngine, raidPtr);
  if (rc) {
    RF_ERRORMSG3("Unable to add to shutdown list file %s line %d rc=%d\n", __FILE__,
      __LINE__, rc);
    rf_ShutdownEngine(NULL);
  }

  return(rc);
}

static int BranchDone(RF_DagNode_t *node)
{
  int i;

  /* return true if forward execution is completed for a node and it's succedents */
  switch (node->status) {
  case rf_wait :
    /* should never be called in this state */
    RF_PANIC();
    break;
  case rf_fired :
    /* node is currently executing, so we're not done */
    return(RF_FALSE);
  case rf_good :
    for (i = 0; i < node->numSuccedents; i++) /* for each succedent */
      if (!BranchDone(node->succedents[i]))   /* recursively check branch */
	return RF_FALSE;
    return RF_TRUE;  /* node and all succedent branches aren't in fired state */
    break;
  case rf_bad :
    /* succedents can't fire */
    return(RF_TRUE);
  case rf_recover :
    /* should never be called in this state */
    RF_PANIC();
    break;
  case rf_undone :
  case rf_panic :