rf_driver.c

RAIDFrame是个非常好的磁盘阵列RAID仿真工具

  int rc;

  if (!raidPtr->valid) {
    RF_ERRORMSG("Attempt to shut down unconfigured RAIDframe driver.  Aborting shutdown\n");
    return(EINVAL);
  }

  /*
   * wait for outstanding IOs to land
   * As described in rf_raid.h, we use the rad_freelist lock
   * to protect the per-array info about outstanding descs
   * since we need to do freelist locking anyway, and this
   * cuts down on the amount of serialization we've got going
   * on.
   */
  RF_FREELIST_DO_LOCK(rf_rad_freelist);
  if (raidPtr->waitShutdown) {
    RF_FREELIST_DO_UNLOCK(rf_rad_freelist);
    return(EBUSY);
  }
  raidPtr->waitShutdown = 1;
  while (raidPtr->nAccOutstanding) {
    RF_WAIT_COND(raidPtr->outstandingCond, RF_FREELIST_MUTEX_OF(rf_rad_freelist));
  }
  RF_FREELIST_DO_UNLOCK(rf_rad_freelist);

#if !defined(KERNEL) && !defined(SIMULATE)
  rf_PrintThroughputStats(raidPtr);
#endif /* !KERNEL && !SIMULATE */

  raidPtr->valid = 0;

#if !defined(KERNEL) && !defined(SIMULATE)
  rf_TerminateDiskQueues(raidPtr);           /* tell all disk queues to release any waiting threads */
  rf_ShutdownDiskThreads(raidPtr);           /* wait for all threads to exit */
#endif /* !KERNEL && !SIMULATE */

  rf_ShutdownList(&raidPtr->shutdownList);

  rf_UnconfigureArray();

  return(0);
}

#define DO_INIT_CONFIGURE(f) { \
	rc = f (&globalShutdown); \
	if (rc) { \
		RF_ERRORMSG2("RAIDFRAME: failed %s with %d\n", RF_STRING(f), rc); \
		rf_ShutdownList(&globalShutdown); \
		configureCount--; \
		RF_UNLOCK_MUTEX(configureMutex); \
		return(rc); \
	} \
}

#define DO_RAID_FAIL() { \
	rf_ShutdownList(&raidPtr->shutdownList); \
	rf_UnconfigureArray(); \
}

#define DO_RAID_INIT_CONFIGURE(f) { \
	rc = f (&raidPtr->shutdownList, raidPtr, cfgPtr); \
	if (rc) { \
		RF_ERRORMSG2("RAIDFRAME: failed %s with %d\n", RF_STRING(f), rc); \
		DO_RAID_FAIL(); \
		return(rc); \
	} \
}

#define DO_RAID_MUTEX(_m_) { \
	rc = rf_create_managed_mutex(&raidPtr->shutdownList, (_m_)); \
	if (rc) { \
		RF_ERRORMSG3("Unable to init mutex file %s line %d rc=%d\n", \
			__FILE__, __LINE__, rc); \
		DO_RAID_FAIL(); \
		return(rc); \
	} \
}

#define DO_RAID_COND(_c_) { \
	rc = rf_create_managed_cond(&raidPtr->shutdownList, (_c_)); \
	if (rc) { \
		RF_ERRORMSG3("Unable to init cond file %s line %d rc=%d\n", \
			__FILE__, __LINE__, rc); \
		DO_RAID_FAIL(); \
		return(rc); \
	} \
}

int rf_Configure(raidPtr, cfgPtr)
  RF_Raid_t    *raidPtr;
  RF_Config_t  *cfgPtr;
{
  RF_RowCol_t row, col;
  int i, status, rc;

  if (raidPtr->valid) {
    RF_ERRORMSG("RAIDframe configuration not shut down.  Aborting configure.\n");
    return(EINVAL);
  }

  RF_LOCK_MUTEX(configureMutex);
  configureCount++;
  if (isconfigged == 0) {
    rc = rf_create_managed_mutex(&globalShutdown, &rf_printf_mutex);
    if (rc) {
      RF_ERRORMSG3("Unable to init mutex file %s line %d rc=%d\n", __FILE__,
        __LINE__, rc);
      rf_ShutdownList(&globalShutdown);
      return(rc);
    }

    /* initialize globals */
    printf("RAIDFRAME: protectedSectors is %ld\n",rf_protectedSectors);

    rf_clear_debug_print_buffer();

    DO_INIT_CONFIGURE(rf_ConfigureAllocList);
    DO_INIT_CONFIGURE(rf_ConfigureEtimer);
    /*
     * Yes, this does make debugging general to the whole system instead
     * of being array specific. Bummer, drag.
     */
    rf_ConfigureDebug(cfgPtr);
    DO_INIT_CONFIGURE(rf_ConfigureDebugMem);
#ifdef SIMULATE
    rf_default_disk_names();
    DO_INIT_CONFIGURE(rf_DDEventInit);
#endif /* SIMULATE */

    DO_INIT_CONFIGURE(rf_ConfigureAccessTrace);
    DO_INIT_CONFIGURE(rf_ConfigureMapModule);
    DO_INIT_CONFIGURE(rf_ConfigureReconEvent);
    DO_INIT_CONFIGURE(rf_ConfigureCallback);
    DO_INIT_CONFIGURE(rf_ConfigureMemChunk);
    DO_INIT_CONFIGURE(rf_ConfigureRDFreeList);
    DO_INIT_CONFIGURE(rf_ConfigureNWayXor);
    DO_INIT_CONFIGURE(rf_ConfigureStripeLockFreeList);
    DO_INIT_CONFIGURE(rf_ConfigureMCPair);
#ifndef SIMULATE
    DO_INIT_CONFIGURE(rf_ConfigureCamLayer);
#endif /* !SIMULATE */
    DO_INIT_CONFIGURE(rf_ConfigureDAGs);
    DO_INIT_CONFIGURE(rf_ConfigureDAGFuncs);
    DO_INIT_CONFIGURE(rf_ConfigureDebugPrint);
    DO_INIT_CONFIGURE(rf_ConfigureReconstruction);
    DO_INIT_CONFIGURE(rf_ConfigureCopyback);
    DO_INIT_CONFIGURE(rf_ConfigureDiskQueueSystem);
    DO_INIT_CONFIGURE(rf_ConfigureCpuMonitor);
    isconfigged = 1;
  }
  RF_UNLOCK_MUTEX(configureMutex);

  /*
   * Null out the entire raid descriptor to avoid problems when we reconfig.
   * This also clears the valid bit.
   */
  bzero((char *)raidPtr, sizeof(RF_Raid_t));
  DO_RAID_MUTEX(&raidPtr->mutex);
  /* set up the cleanup list.  Do this after ConfigureDebug so that value of memDebug will be set */
  rf_MakeAllocList(raidPtr->cleanupList);
  if (raidPtr->cleanupList == NULL) {
    DO_RAID_FAIL();
    return(ENOMEM);
  }
  rc = rf_ShutdownCreate(&raidPtr->shutdownList, (void (*)())rf_FreeAllocList, raidPtr->cleanupList);
  if (rc) {
    RF_ERRORMSG3("Unable to add to shutdown list file %s line %d rc=%d\n",
      __FILE__, __LINE__, rc);
    DO_RAID_FAIL();
    return(rc);
  }

  raidPtr->numRow = cfgPtr->numRow;
  raidPtr->numCol = cfgPtr->numCol;
  raidPtr->numSpare = cfgPtr->numSpare;
  RF_CallocAndAdd(raidPtr->status, raidPtr->numRow, sizeof(RF_RowStatus_t),
    (RF_RowStatus_t *), raidPtr->cleanupList);
  if (raidPtr->status == NULL) {
    DO_RAID_FAIL();
    return(ENOMEM);
  }
  RF_CallocAndAdd(raidPtr->reconControl, raidPtr->numRow,
    sizeof(RF_ReconCtrl_t *), (RF_ReconCtrl_t **), raidPtr->cleanupList);
  if (raidPtr->reconControl == NULL) {
    DO_RAID_FAIL();
    return(ENOMEM);
  }
  for (i=0; i<raidPtr->numRow; i++) {
    raidPtr->status[i] = rf_rs_optimal;
    raidPtr->reconControl[i] = NULL;
  }
  DO_RAID_INIT_CONFIGURE(rf_ConfigureEngine);
#if !defined(KERNEL) && !defined(SIMULATE)
  DO_RAID_INIT_CONFIGURE(rf_InitThroughputStats);
#endif /* !KERNEL && !SIMULATE */
  DO_RAID_INIT_CONFIGURE(rf_ConfigureStripeLocks);
  DO_RAID_COND(&raidPtr->outstandingCond);
  raidPtr->nAccOutstanding = 0;
  raidPtr->waitShutdown = 0;

  DO_RAID_MUTEX(&raidPtr->access_suspend_mutex);
  DO_RAID_COND(&raidPtr->quiescent_cond);

  DO_RAID_COND(&raidPtr->waitForReconCond);

  DO_RAID_MUTEX(&raidPtr->recon_done_proc_mutex);

  DO_RAID_INIT_CONFIGURE(rf_ConfigureDisks);

  DO_RAID_INIT_CONFIGURE(rf_ConfigureSpareDisks);

  /* do this after ConfigureDisks & ConfigureSpareDisks to be sure dev no. is set */
  DO_RAID_INIT_CONFIGURE(rf_ConfigureDiskQueues);

#ifndef KERNEL
  DO_RAID_INIT_CONFIGURE(rf_ConfigureDiskThreads);
#endif /* !KERNEL */

  DO_RAID_INIT_CONFIGURE(rf_ConfigureLayout);

  DO_RAID_INIT_CONFIGURE(rf_ConfigurePSStatus);

  for(row=0;row<raidPtr->numRow;row++) {
    for(col=0;col<raidPtr->numCol;col++) {
      /*
       * XXX better distribution
       */
      raidPtr->hist_diskreq[row][col] = 0;
    }
  }

  if (rf_keepAccTotals) {
    raidPtr->keep_acc_totals = 1;
  }

  rf_StartUserStats(raidPtr);

  raidPtr->valid = 1;
  return(0);
}

static int init_rad(desc)
  RF_RaidAccessDesc_t  *desc;
{
	int rc;

	rc = rf_mutex_init(&desc->mutex);
	if (rc) {
		RF_ERRORMSG3("Unable to init mutex file %s line %d rc=%d\n", __FILE__,
			__LINE__, rc);
		return(rc);
	}
	rc = rf_cond_init(&desc->cond);
	if (rc) {
		RF_ERRORMSG3("Unable to init cond file %s line %d rc=%d\n", __FILE__,
			__LINE__, rc);
		rf_mutex_destroy(&desc->mutex);
		return(rc);
	}
	return(0);
}

static void clean_rad(desc)
  RF_RaidAccessDesc_t  *desc;
{
	rf_mutex_destroy(&desc->mutex);
	rf_cond_destroy(&desc->cond);
}

static void rf_ShutdownRDFreeList(ignored)
  void  *ignored;
{
	RF_FREELIST_DESTROY_CLEAN(rf_rad_freelist,next,(RF_RaidAccessDesc_t *),clean_rad);
}

static int rf_ConfigureRDFreeList(listp)
  RF_ShutdownList_t **listp;
{
	int rc;

	RF_FREELIST_CREATE(rf_rad_freelist, RF_MAX_FREE_RAD,
		RF_RAD_INC, sizeof(RF_RaidAccessDesc_t));
	if (rf_rad_freelist == NULL) {
		return(ENOMEM);
	}
	rc = rf_ShutdownCreate(listp, rf_ShutdownRDFreeList, NULL);
	if (rc) {
		RF_ERRORMSG3("Unable to add to shutdown list file %s line %d rc=%d\n", __FILE__,
			__LINE__, rc);
		rf_ShutdownRDFreeList(NULL);
		return(rc);
	}
	RF_FREELIST_PRIME_INIT(rf_rad_freelist, RF_RAD_INITIAL,next,
		(RF_RaidAccessDesc_t *),init_rad);
	return(0);
}

RF_RaidAccessDesc_t *rf_AllocRaidAccDesc(
  RF_Raid_t                    *raidPtr,
  RF_IoType_t                   type,
  RF_RaidAddr_t                 raidAddress,
  RF_SectorCount_t              numBlocks,
  caddr_t                       bufPtr,
  void                         *bp,
  RF_DagHeader_t              **paramDAG,
  RF_AccessStripeMapHeader_t  **paramASM,
  RF_RaidAccessFlags_t          flags,
  void                        (*cbF)(),
  void                         *cbA,
  RF_AccessState_t             *states)
{
  RF_RaidAccessDesc_t *desc;

  RF_FREELIST_GET_INIT_NOUNLOCK(rf_rad_freelist,desc,next,(RF_RaidAccessDesc_t *),init_rad);
  if (raidPtr->waitShutdown) {
    /*
     * Actually, we're shutting the array down. Free the desc
     * and return NULL.
     */
    RF_FREELIST_DO_UNLOCK(rf_rad_freelist);
    RF_FREELIST_FREE_CLEAN(rf_rad_freelist,desc,next,clean_rad);
    return(NULL);
  }
  raidPtr->nAccOutstanding++;
  RF_FREELIST_DO_UNLOCK(rf_rad_freelist);
  
  desc->raidPtr     = (void*)raidPtr;
  desc->type        = type;
  desc->raidAddress = raidAddress;
  desc->numBlocks   = numBlocks;
  desc->bufPtr      = bufPtr;
  desc->bp          = bp;
  desc->paramDAG    = paramDAG;
  desc->paramASM    = paramASM;
  desc->flags       = flags;
  desc -> states    = states;
  desc -> state     = 0;

  desc->status      = 0;
  bzero((char *)&desc->tracerec, sizeof(RF_AccTraceEntry_t));
  desc->callbackFunc= cbF;
  desc->callbackArg = cbA;
  desc->next        = NULL;
  desc->head	    = desc;
  desc->numPending  = 0;
  desc->cleanupList = NULL;
  rf_MakeAllocList(desc->cleanupList);
  rf_get_threadid(desc->tid);
#ifdef SIMULATE
  desc->owner = rf_GetCurrentOwner();
#endif /* SIMULATE */
  return(desc);
}

void rf_FreeRaidAccDesc(RF_RaidAccessDesc_t *desc)
{
  RF_Raid_t *raidPtr = desc->raidPtr;

  RF_ASSERT(desc);

#if !defined(KERNEL) && !defined(SIMULATE)
  rf_StopThroughputStats(raidPtr);
#endif /* !KERNEL && !SIMULATE */

  rf_FreeAllocList(desc->cleanupList);
  RF_FREELIST_FREE_CLEAN_NOUNLOCK(rf_rad_freelist,desc,next,clean_rad);
    raidPtr->nAccOutstanding--;
    if (raidPtr->waitShutdown) {
      RF_SIGNAL_COND(raidPtr->outstandingCond);
    }
  RF_FREELIST_DO_UNLOCK(rf_rad_freelist);
}

#ifdef JIMZ
#define THREAD_NUMDESC 1024
#define THREAD_NUM 600
static RF_RaidAccessDesc_t *dest_hist[THREAD_NUM*THREAD_NUMDESC];
int jimz_access_num[THREAD_NUM];
#endif /* JIMZ */

/*********************************************************************
 * Main routine for performing an access.
 * Accesses are retried until a DAG can not be selected.  This occurs
 * when either the DAG library is incomplete or there are too many
 * failuers in a parity group.
 ********************************************************************/
int rf_DoAccess(
  RF_Raid_t                    *raidPtr,
  RF_IoType_t                   type,
  int                           async_flag,
  RF_RaidAddr_t                 raidAddress, 
  RF_SectorCount_t              numBlocks,
  caddr_t                       bufPtr,
  void                         *bp_in,
  RF_DagHeader_t              **paramDAG,
  RF_AccessStripeMapHeader_t  **paramASM,
  RF_RaidAccessFlags_t          flags,
  RF_RaidAccessDesc_t         **paramDesc,
  void                        (*cbF)(),
  void                         *cbA)
/*
type should be read or write
async_flag should be RF_TRUE or RF_FALSE
bp_in is a buf pointer.  void * to facilitate ignoring it outside the kernel
*/
{
  int tid;
  RF_RaidAccessDesc_t *desc;
  caddr_t lbufPtr = bufPtr;
#ifdef KERNEL
  struct buf *bp = (struct buf *) bp_in;