rf_driver.c

RAIDFrame是个非常好的磁盘阵列RAID仿真工具

#if DFSTRACE > 0
  struct { RF_uint64 raidAddr; int numBlocks; char type;} dfsrecord;  
#endif /* DFSTRACE > 0 */
#else /* KERNEL */
  void *bp = bp_in;
#endif /* KERNEL */

  raidAddress += rf_raidSectorOffset;

  if (!raidPtr->valid) {
    RF_ERRORMSG("RAIDframe driver not successfully configured.  Rejecting access.\n");
    IO_BUF_ERR(bp, EINVAL, raidPtr->raidid);
    return(EINVAL);
  } 

#if defined(KERNEL) && DFSTRACE > 0
  if (rf_DFSTraceAccesses) {
    dfsrecord.raidAddr  = raidAddress;
    dfsrecord.numBlocks = numBlocks;
    dfsrecord.type      = type;
    dfs_log(DFS_NOTE, (char *) &dfsrecord, sizeof(dfsrecord), 0);
  }
#endif /* KERNEL && DFSTRACE > 0 */

  rf_get_threadid(tid);
  if (rf_accessDebug) {
    printf("[%d] %s raidAddr %ld (stripeid %lu-%lu) numBlocks %lu (%d bytes) buf 0x%lx\n",tid,
	   (type==RF_IO_TYPE_READ) ? "READ":"WRITE", raidAddress, 
	   (u_long) rf_RaidAddressToStripeID(&raidPtr->Layout, raidAddress),
	   (u_long) rf_RaidAddressToStripeID(&raidPtr->Layout, raidAddress+numBlocks-1),
	   (u_long) numBlocks, rf_RaidAddressToByte(raidPtr,numBlocks), bufPtr);
  }

  if (raidAddress + numBlocks > raidPtr->totalSectors) {

    printf("DoAccess: raid addr %lu too large to access %lu sectors.  Max legal addr is %lu\n",
           (u_long)raidAddress,(u_long)numBlocks,(u_long)raidPtr->totalSectors);

#ifdef KERNEL
    if (type == RF_IO_TYPE_READ) {
      IO_BUF_ERR(bp, ENOSPC, raidPtr->raidid);
      return(ENOSPC);
    } else {
      IO_BUF_ERR(bp, ENOSPC, raidPtr->raidid);
      return(ENOSPC);
    }
#elif defined(SIMULATE)
    RF_PANIC();
#else /* SIMULATE */
    return(EIO);
#endif /* SIMULATE */
  }

#if !defined(KERNEL) && !defined(SIMULATE)
  rf_StartThroughputStats(raidPtr);
#endif /* !KERNEL && !SIMULATE */

  desc = rf_AllocRaidAccDesc(raidPtr, type, raidAddress,
			  numBlocks, lbufPtr, bp, paramDAG, paramASM,
			  flags, cbF, cbA, raidPtr->Layout.map->states);
  if (desc == NULL) {
    return(ENOMEM);
  }
#ifdef JIMZ
  dest_hist[(tid*THREAD_NUMDESC)+jimz_access_num[tid]]; jimz_access_num[tid]++;
#endif /* JIMZ */

  RF_ETIMER_START(desc->tracerec.tot_timer);

#ifdef SIMULATE
  /* simulator uses paramDesc to continue dag from test function */
  desc->async_flag=async_flag;

  *paramDesc=desc;
  
  return(0);
#endif /* SIMULATE */

  rf_ContinueRaidAccess(desc);

#ifndef KERNEL
  if (!(flags & RF_DAG_NONBLOCKING_IO)) {
    RF_LOCK_MUTEX(desc->mutex);
    while (!(desc->flags & RF_DAG_ACCESS_COMPLETE)) {
      RF_WAIT_COND(desc->cond, desc->mutex);
    }
    RF_UNLOCK_MUTEX(desc->mutex);
    rf_FreeRaidAccDesc(desc);
  }
#endif /* !KERNEL */

  return(0);
}

/* force the array into reconfigured mode without doing reconstruction */
int rf_SetReconfiguredMode(raidPtr, row, col)
  RF_Raid_t  *raidPtr;
  int         row;
  int         col;
{
  if (!(raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) {
    printf("Can't set reconfigured mode in dedicated-spare array\n");
    RF_PANIC();
  }
  RF_LOCK_MUTEX(raidPtr->mutex);
  raidPtr->numFailures++;
  raidPtr->Disks[row][col].status = rf_ds_dist_spared;
  raidPtr->status[row] = rf_rs_reconfigured;
  /* install spare table only if declustering + distributed sparing architecture. */
  if ( raidPtr->Layout.map->flags & RF_BD_DECLUSTERED )
     rf_InstallSpareTable(raidPtr, row, col);
  RF_UNLOCK_MUTEX(raidPtr->mutex);
  return(0);
}

extern int fail_row, fail_col, fail_time;
extern delayed_recon;

int rf_FailDisk(
  RF_Raid_t  *raidPtr,
  int         frow,
  int         fcol,
  int         initRecon)
{
  int tid;

  rf_get_threadid(tid);
  printf("[%d] Failing disk r%d c%d\n",tid,frow,fcol);
  RF_LOCK_MUTEX(raidPtr->mutex);
  raidPtr->numFailures++;
  raidPtr->Disks[frow][fcol].status = rf_ds_failed;
  raidPtr->status[frow] = rf_rs_degraded;
  RF_UNLOCK_MUTEX(raidPtr->mutex);
#ifdef SIMULATE
#if RF_DEMO > 0
  if (rf_demoMode) {
    rf_demo_update_mode (RF_DEMO_DEGRADED);
    fail_col = fcol; fail_row = frow; 
    fail_time = rf_CurTime(); /* XXX */
    if (initRecon)
      delayed_recon = RF_TRUE;
  }
  else {
    if (initRecon)
      rf_ReconstructFailedDisk(raidPtr, frow, fcol);
  }
#else /* RF_DEMO > 0 */
  if (initRecon)
    rf_ReconstructFailedDisk(raidPtr, frow, fcol);
#endif /* RF_DEMO > 0 */
#else /* SIMULATE */
  if (initRecon)
    rf_ReconstructFailedDisk(raidPtr, frow, fcol);
#endif /* SIMULATE */
  return(0);
}

#ifdef SIMULATE
extern RF_Owner_t recon_owner;

void rf_ScheduleContinueReconstructFailedDisk(reconDesc)
  RF_RaidReconDesc_t  *reconDesc;
{
  rf_DDEventRequest(rf_CurTime(), rf_ContinueReconstructFailedDisk,
    (void *) reconDesc, recon_owner, -4, -4, reconDesc->raidPtr, NULL);
}
#endif /* SIMULATE */

/* releases a thread that is waiting for the array to become quiesced.
 * access_suspend_mutex should be locked upon calling this
 */
void rf_SignalQuiescenceLock(raidPtr, reconDesc)
  RF_Raid_t           *raidPtr;
  RF_RaidReconDesc_t  *reconDesc;
{
  int tid;

  if (rf_quiesceDebug) {
    rf_get_threadid(tid);
    printf("[%d] Signalling quiescence lock\n", tid);
  }
  raidPtr->access_suspend_release = 1;

  if (raidPtr->waiting_for_quiescence) {
#ifndef SIMULATE
    SIGNAL_QUIESCENT_COND(raidPtr);
#else /* !SIMULATE */
    if (reconDesc) {
      rf_ScheduleContinueReconstructFailedDisk(reconDesc);
    }
#endif /* !SIMULATE */
  }
}

/* suspends all new requests to the array.  No effect on accesses that are in flight.  */
int rf_SuspendNewRequestsAndWait(raidPtr)
  RF_Raid_t  *raidPtr;
{
  if (rf_quiesceDebug)
    printf("Suspending new reqs\n");

  RF_LOCK_MUTEX(raidPtr->access_suspend_mutex);
  raidPtr->accesses_suspended++;
  raidPtr->waiting_for_quiescence = (raidPtr->accs_in_flight == 0) ? 0 : 1;
  
#ifndef SIMULATE
  if (raidPtr->waiting_for_quiescence) {
    raidPtr->access_suspend_release=0;
    while (!raidPtr->access_suspend_release) {
      WAIT_FOR_QUIESCENCE(raidPtr);
      raidPtr->waiting_for_quiescence = 0;
    }
  }
#endif /* !SIMULATE */

  RF_UNLOCK_MUTEX(raidPtr->access_suspend_mutex);
  return (raidPtr->waiting_for_quiescence);
}

/* wake up everyone waiting for quiescence to be released */
void rf_ResumeNewRequests(raidPtr)
  RF_Raid_t  *raidPtr;
{
  RF_CallbackDesc_t *t, *cb;

  if (rf_quiesceDebug)
    printf("Resuming new reqs\n");
  
  RF_LOCK_MUTEX(raidPtr->access_suspend_mutex);
  raidPtr->accesses_suspended--;
  if (raidPtr->accesses_suspended == 0)
    cb = raidPtr->quiesce_wait_list;
  else
    cb = NULL;
  raidPtr->quiesce_wait_list = NULL;
  RF_UNLOCK_MUTEX(raidPtr->access_suspend_mutex);
  
  while (cb) {
    t = cb;
    cb = cb->next;
    (t->callbackFunc)(t->callbackArg);
    rf_FreeCallbackDesc(t);
  }
}

/*****************************************************************************************
 *
 * debug routines
 *
 ****************************************************************************************/

static void set_debug_option(name, val)
  char  *name;
  long   val;
{
  RF_DebugName_t *p;

  for (p = rf_debugNames; p->name; p++) {
    if (!strcmp(p->name, name)) {
      *(p->ptr) = val;
      printf("[Set debug variable %s to %ld]\n",name,val);
      return;
    }
  }
  RF_ERRORMSG1("Unknown debug string \"%s\"\n",name);
}


/* would like to use sscanf here, but apparently not available in kernel */
/*ARGSUSED*/
static void rf_ConfigureDebug(cfgPtr)
  RF_Config_t  *cfgPtr;
{
  char *val_p, *name_p, *white_p;
  long val;
  int i;

  rf_ResetDebugOptions();
  for (i=0; cfgPtr->debugVars[i][0] && i < RF_MAXDBGV; i++) {
    name_p  = rf_find_non_white(&cfgPtr->debugVars[i][0]);
    white_p = rf_find_white(name_p);                                   /* skip to start of 2nd word */
    val_p   = rf_find_non_white(white_p);
    if (*val_p == '0' && *(val_p+1) == 'x') val = rf_htoi(val_p+2);
    else val = rf_atoi(val_p);
    *white_p = '\0';
    set_debug_option(name_p, val);
  }
}

/* performance monitoring stuff */

#define TIMEVAL_TO_US(t) (((long) t.tv_sec) * 1000000L + (long) t.tv_usec)

#if !defined(KERNEL) && !defined(SIMULATE)

/*
 * Throughput stats currently only used in user-level RAIDframe
 */

static int rf_InitThroughputStats(
  RF_ShutdownList_t  **listp,
  RF_Raid_t           *raidPtr,
  RF_Config_t         *cfgPtr)
{
  int rc;

  /* these used by user-level raidframe only */
  rc = rf_create_managed_mutex(listp, &raidPtr->throughputstats.mutex);
  if (rc) {
    RF_ERRORMSG3("Unable to init mutex file %s line %d rc=%d\n", __FILE__,
      __LINE__, rc);
    return(rc);
  }
  raidPtr->throughputstats.sum_io_us = 0;
  raidPtr->throughputstats.num_ios = 0;
  raidPtr->throughputstats.num_out_ios = 0;
  return(0);
}

void rf_StartThroughputStats(RF_Raid_t *raidPtr)
{
  RF_LOCK_MUTEX(raidPtr->throughputstats.mutex);
  raidPtr->throughputstats.num_ios++;
  raidPtr->throughputstats.num_out_ios++;
  if (raidPtr->throughputstats.num_out_ios == 1)
    RF_GETTIME(raidPtr->throughputstats.start);
  RF_UNLOCK_MUTEX(raidPtr->throughputstats.mutex);
}

static void rf_StopThroughputStats(RF_Raid_t *raidPtr)
{
  struct timeval diff;

  RF_LOCK_MUTEX(raidPtr->throughputstats.mutex);
  raidPtr->throughputstats.num_out_ios--;
  if (raidPtr->throughputstats.num_out_ios == 0) {
    RF_GETTIME(raidPtr->throughputstats.stop);
    RF_TIMEVAL_DIFF(&raidPtr->throughputstats.start, &raidPtr->throughputstats.stop, &diff);
    raidPtr->throughputstats.sum_io_us += TIMEVAL_TO_US(diff);
  }
  RF_UNLOCK_MUTEX(raidPtr->throughputstats.mutex);  
}

static void rf_PrintThroughputStats(RF_Raid_t *raidPtr)
{
  RF_ASSERT(raidPtr->throughputstats.num_out_ios == 0);
  if ( raidPtr->throughputstats.sum_io_us != 0 ) {
     printf("[Througphut: %8.2f IOs/second]\n", raidPtr->throughputstats.num_ios
       / (raidPtr->throughputstats.sum_io_us / 1000000.0));
  }
}

#endif /* !KERNEL && !SIMULATE */

void rf_StartUserStats(RF_Raid_t *raidPtr)
{
  RF_GETTIME(raidPtr->userstats.start);
  raidPtr->userstats.sum_io_us = 0;
  raidPtr->userstats.num_ios = 0;
  raidPtr->userstats.num_sect_moved = 0;
}

void rf_StopUserStats(RF_Raid_t *raidPtr)
{
  RF_GETTIME(raidPtr->userstats.stop);
}

void rf_UpdateUserStats(raidPtr, rt, numsect)
  RF_Raid_t  *raidPtr;
  int         rt;       /* resp time in us */
  int         numsect;  /* number of sectors for this access */
{
  raidPtr->userstats.sum_io_us += rt;
  raidPtr->userstats.num_ios++;
  raidPtr->userstats.num_sect_moved += numsect;
}

void rf_PrintUserStats(RF_Raid_t *raidPtr)
{
  long elapsed_us, mbs, mbs_frac;
  struct timeval diff;

  RF_TIMEVAL_DIFF(&raidPtr->userstats.start, &raidPtr->userstats.stop, &diff);
  elapsed_us = TIMEVAL_TO_US(diff);

  /* 2000 sectors per megabyte, 10000000 microseconds per second */
  if (elapsed_us)
    mbs = (raidPtr->userstats.num_sect_moved / 2000) / (elapsed_us / 1000000);
  else
    mbs = 0;

  /* this computes only the first digit of the fractional mb/s moved */
  if (elapsed_us) {
    mbs_frac = ((raidPtr->userstats.num_sect_moved / 200) / (elapsed_us / 1000000))
      - (mbs * 10);
  }
  else {
    mbs_frac = 0;
  }

  printf("Number of I/Os:             %ld\n",raidPtr->userstats.num_ios);
  printf("Elapsed time (us):          %ld\n",elapsed_us);
  printf("User I/Os per second:       %ld\n",RF_DB0_CHECK(raidPtr->userstats.num_ios, (elapsed_us/1000000)));
  printf("Average user response time: %ld us\n",RF_DB0_CHECK(raidPtr->userstats.sum_io_us, raidPtr->userstats.num_ios));
  printf("Total sectors moved:        %ld\n",raidPtr->userstats.num_sect_moved);
  printf("Average access size (sect): %ld\n",RF_DB0_CHECK(raidPtr->userstats.num_sect_moved, raidPtr->userstats.num_ios));
  printf("Achieved data rate:         %ld.%ld MB/sec\n",mbs,mbs_frac);
}