rf_testcode.c

RAIDFrame是个非常好的磁盘阵列RAID仿真工具

    }
    if ( (nactual = write(fd, buf, RF_MIN(BUFSIZE, filesize - offset))) < 0) {
      perror("File write error"); exit(1);
    }
    offset += nactual;
  }
  close(fd);
  RF_Free(buf, BUFSIZE);
  
  printf("File %s (%d bytes) successfully transferred to file system\n",fname,offset);
}

static char rw_type;
static int terminate_random_read_test = 0, do_sequential = 0;

void RandomReadTest(raidPtr, listp)
  RF_Raid_t           *raidPtr;
  RF_ShutdownList_t  **listp;
{
  int nthreads, i, status, rc;
  RF_GRpair_t *grp, *fgrp;
  RF_Thread_t *tthreads;
  RF_Thread_t fthread;
  char cbuf[50];

  grp = rf_GetGrpair(listp);
  if (grp == NULL) {
    RF_PANIC();
  }
  grp->raidPtr = raidPtr;
  fgrp = rf_GetGrpair(listp);
  if (fgrp == NULL) {
    RF_PANIC();
  }
  fgrp->raidPtr = raidPtr;

  fprintf(stderr,"How many parallel threads? [0 is ok] ");
  gets(cbuf);
  nthreads = atoi(cbuf);
  fprintf(stderr,"Reads or Writes [r/w]? ");
  gets(cbuf);
  rw_type = *cbuf;
  fprintf(stderr,"Degraded mode: none, constant, constant double degraded,async, async+init recon [n/c/2/a/r]? ");
  gets(cbuf);
  rf_testcode_degr_mode_type = *cbuf;
  fprintf(stderr,"Random or sequential I/Os? [r/s] ");
  gets(cbuf);
  if (*cbuf == 's') do_sequential = 1;

  /* ios_per_thread is don't-care when rf_testcode_degr_mode_type == 'r' */
  if (rf_testcode_degr_mode_type != 'r') {
    fprintf(stderr,"How many I/Os per thread? ");
    gets(cbuf);
    ios_per_thread = atoi(cbuf);
  }
 

  RF_CallocAndAdd(tthreads, nthreads, sizeof(RF_Thread_t), (RF_Thread_t *), raidPtr->cleanupList);

#if RF_DEMO > 0
  if (rf_demoMode)
    rf_startup_iops_demo(rf_demoMeterVpos, raidPtr->numCol, raidPtr->Layout.numDataCol+raidPtr->Layout.numParityCol);
#endif /* RF_DEMO > 0 */
  if (rf_testcode_degr_mode_type == 'c')
    rf_FailDisk(raidPtr, 0, 0, 0);
  if (rf_testcode_degr_mode_type == '2') 
    {
      rf_FailDisk(raidPtr, 0, 0, 0);
      rf_FailDisk(raidPtr, 0, 2, 0);
    }

  for (i=0; i<nthreads; i++) {
    if (RF_CREATE_THREAD(tthreads[i], RRTest, grp)) {
      RF_ERRORMSG1("Unable to create test thread number %d\n",i);
      punt(1);
    }
    RF_THREADGROUP_STARTED(&grp->group);
  }

  RF_THREADGROUP_WAIT_START(&grp->group);

  if (rf_testcode_degr_mode_type == 'a' || rf_testcode_degr_mode_type == 'r') {
    if (RF_CREATE_THREAD(fthread, DiskFailingThread, fgrp)) {
      RF_ERRORMSG("Unable to create disk failing thread\n");
      punt(1);
    }
    RF_THREADGROUP_STARTED(&fgrp->group);
    RF_THREADGROUP_WAIT_START(&fgrp->group);
    RF_THREADGROUP_WAIT_STOP(&fgrp->group);
    if (rf_testcode_degr_mode_type == 'r')
      terminate_random_read_test = 1;
  }

  RF_THREADGROUP_WAIT_STOP(&grp->group);
#if RF_DEMO > 0
  if (rf_demoMode)
    rf_finish_iops_demo();
#endif /* RF_DEMO > 0 */
}


static void RRTest(arg)
  RF_ThreadArg_t  arg;
{
  RF_GRpair_t *grp = (RF_GRpair_t *)arg;
  RF_Raid_t *raidPtr = grp->raidPtr;
  int numsect, ionum, tid;
  int numbytes;
  RF_RaidAddr_t offset;
  long tot_resp_time_us = 0;
  long *rbuf;
  RF_Etimer_t etime;
  struct timeval starttime, endtime, elpsd;
  float elpsd_time;
  unsigned long rv;
  RF_DECLARE_RANDOM;

  rf_assign_threadid();
  rf_get_threadid(tid);

  RF_THREADGROUP_RUNNING(&grp->group);

  printf("[%d] starting\n",tid);
  gettimeofday(&starttime, NULL);
  RF_INIT_RANDOM(starttime.tv_sec + tid);

  rv = RF_RANDOM();
  numsect = (rf_accSizeKB) ? rf_accSizeKB*2 : (rv % (rf_maxRandomSizeKB*2)) +1;       /* must access at least one sector */
  numbytes = numsect << raidPtr->logBytesPerSector;

  RF_Malloc(rbuf, numbytes, (long *));
  gettimeofday(&starttime, NULL);
  for (ionum=0; (rf_testcode_degr_mode_type == 'r') || (ionum<ios_per_thread); ionum++) {
    /*
    rv = RF_RANDOM();
    delay_ms = rv % 300;
    RF_DELAY_THREAD( 0, delay_ms );
    */
    if (do_sequential) offset = (ionum * numsect) % (raidPtr->totalSectors - numsect);
    else {
      rv = RF_RANDOM();
      offset = rv % (raidPtr->totalSectors-numsect);
      offset /= numsect; offset *= numsect;
    }
    RF_ETIMER_START(etime);
    rf_DoAccess(raidPtr, rw_type, NULL, offset, numsect, (caddr_t)rbuf,
        NULL, NULL, NULL, RF_DAG_FLAGS_NONE,NULL,NULL,NULL);
    RF_ETIMER_STOP(etime);
    RF_ETIMER_EVAL(etime);
    tot_resp_time_us += RF_ETIMER_VAL_US(etime);
#if RF_DEMO > 0
    if (rf_demoMode)
      rf_update_user_stats(RF_ETIMER_VAL_MS(etime));
#endif /* RF_DEMO > 0 */
    if (terminate_random_read_test) break;
  }
  gettimeofday(&endtime, NULL);
  RF_TIMEVAL_DIFF(&starttime, &endtime, &elpsd);
  elpsd_time = (float) elpsd.tv_sec + ((float) elpsd.tv_usec)/1000000;
  RF_Free(rbuf, numbytes);
#if RF_DEMO > 0
  if (!rf_demoMode)
#endif /* RF_DEMO > 0 */
  {
    printf("[%d] %d %d KB reads in %0.2f secs, avg resp time %0.2f ms\n",
      tid, ionum, rf_accSizeKB, elpsd_time, RF_DB0_CHECK( ((float) tot_resp_time_us)/1000.0, ionum));
  }
  RF_THREADGROUP_DONE(&grp->group);
}


static unsigned char *writemap = NULL;       /* used to determine what data should be in the array after recon */
static int terminate_recon_test = 0;         /* used to cause termination */
RF_DECLARE_STATIC_MUTEX(writemap_mutex)      /* used to protect writemap modifications */
static int painful_test = 0;                 /* causes full-array verifies after each write */
static long *painful_test_buf = NULL;


/* run a bunch of I/Os against the array.  If "writemap" is non-null, it means we're
 * doing a test where we want to verify that all the data in the array is correct after
 * the test completes, and so we maintain the writemap during the run.  If demoMode is
 * on, we suppress some of the more severe correctness checks.
 */


static void RealLoopTest(RF_ThreadArg_t arg)
{
  RF_GRpair_t *grp = (RF_GRpair_t *)arg;
  RF_Raid_t *raidPtr = grp->raidPtr;
  RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout);
  long *wbuf, *rbuf, *sbuf;
  RF_SectorNum_t secno;
  RF_RaidAddr_t offset, t;
  int numsect, i, j, numbytes, ionum, tid, resptime, mustWait, rv;
  RF_DagHeader_t *rdag, *wdag;
  RF_AccessStripeMapHeader_t *rasm, *wasm;
  RF_AccessStripeMap_t *p;
  int bytesPerSect = (1<<raidPtr->logBytesPerSector);
  int longsPerSect = bytesPerSect/sizeof(long), status;
  unsigned long rval;
  RF_Etimer_t wtimer, rtimer;
  struct timeval tstart, tend, diff;
  unsigned long iocount, tot_wtime=0, tot_rtime=0, tot_size=0;
  RF_StripeNum_t lastStripeID;
  RF_MCPair_t *mcpair;
  RF_DECLARE_RANDOM;

  rf_assign_threadid();
  rf_get_threadid(tid);

  RF_THREADGROUP_RUNNING(&grp->group);

  mcpair = rf_AllocMCPair();
  RF_INIT_RANDOM( (different_seeds) ? tid+5 : 1 );
  if (rf_accessDebug)
    printf("[%d] starting\n",tid);

  /* when doing recon-mode test, run until we're told to stop */
  gettimeofday(&tstart, NULL);
  iocount = 0;
  for (ionum=0; writemap || iocount<ios_per_thread; ) {

    if (terminate_recon_test)
      break;
    if (rf_accSizeKB) {
      numsect = (rf_accSizeKB * 1024) >> raidPtr->logBytesPerSector;
    }
    else {
      rv = RF_RANDOM();
      rv %= ((rf_maxRandomSizeKB*1024)>>raidPtr->logBytesPerSector);
      numsect = rv + 1;
    }

    rv = RF_RANDOM();
    offset = rv % (raidPtr->totalSectors-numsect);
    if (rf_alignAccesses) {offset /= numsect; offset *= numsect;}
    numbytes = numsect << raidPtr->logBytesPerSector;
    RF_Malloc(rbuf, numbytes, (long *));
    RF_Malloc(sbuf, numbytes, (long *));
#if RF_DEMO > 0
    if (!rf_demoMode)
#endif /* RF_DEMO > 0 */
    {
      RF_Malloc(wbuf, numbytes, (long *));
    }

    /* acquire locks on the stripes we're going to write */
    wasm = rf_MapAccess(raidPtr, offset, numsect, (caddr_t)wbuf, RF_DONT_REMAP);
    lastStripeID = -1;
    for (p=wasm->stripeMap; p; p=p->next) {
      RF_ASSERT(p->stripeID > lastStripeID);                      /* locks must be acquired hierarchically */
      lastStripeID = p->stripeID;
      RF_LOCK_MUTEX(mcpair->mutex);
      mcpair->flag = 0;
      RF_INIT_LOCK_REQ_DESC(p->lockReqDesc, RF_IO_TYPE_WRITE, rf_MCPairWakeupFunc, mcpair, p, layoutPtr->dataSectorsPerStripe);
      mustWait = rf_AcquireStripeLock(lockTable, p->stripeID, &p->lockReqDesc);
      if (mustWait) {
        while (!mcpair->flag) {
          RF_WAIT_COND(mcpair->cond, mcpair->mutex);
        }
      }
      RF_UNLOCK_MUTEX(mcpair->mutex);
    }

    if (!writemap) {
      for (i=0; i<numbytes/sizeof(long); i++) {
        unsigned long lrv;
        lrv = RF_RANDOM();
#if RF_DEMO > 0
        sbuf[i] = (rf_demoMode) ? i*i : lrv;
#else /* RF_DEMO > 0 */
        sbuf[i] = lrv;
#endif /* RF_DEMO > 0 */
      }
    } else {
      for (secno = offset; secno<offset+numsect; secno++) {
	RF_LOCK_MUTEX(writemap_mutex);
        rv = RF_RANDOM();
	writemap[secno] = (unsigned char) (rv & 0xFF);   /* pick a random char */
	RF_UNLOCK_MUTEX(writemap_mutex);
	rval = (unsigned long) writemap[secno];                /* convert it to a long */
	for (j=0; j < longsPerSect; j++) {                     /* fill the data buffer with rval,rval^2,rval^4,... */
	  sbuf[(secno-offset)*longsPerSect+j] = rval;
	  rval *= rval;
	}
      }
    }
#if RF_DEMO > 0
    if (!rf_demoMode)
#endif /* RF_DEMO > 0 */
    {
      bcopy((char *)sbuf, (char *)wbuf, numbytes);
    }
    
    /* verify that parity is correct in all stripes to be accessed */
#if RF_DEMO > 0
    if (rf_demoMode || rf_demoSuppressReconInitVerify)
      rf_disableParityVerify = RF_TRUE;
#endif /* RF_DEMO > 0 */
#if 1
    if (layoutPtr->map->faultsTolerated == 0)
      rf_disableParityVerify = RF_TRUE;
    if ((rf_testcode_degr_mode_type != 'n') && (rf_testcode_degr_mode_type != 'r'))
      rf_disableParityVerify = RF_TRUE;
#endif
    if (!rf_disableParityVerify) {
      status = rf_VerifyParity(raidPtr, wasm->stripeMap, RF_DO_CORRECT_PARITY, RF_DAG_FLAGS_NONE);
      if (status)
        PrintParityVerifyString(status);
    }
    
    RF_ETIMER_START(wtimer);
#if RF_DEMO > 0
    rf_DoAccess(raidPtr, RF_IO_TYPE_WRITE,NULL, offset, numsect, (caddr_t)((rf_demoMode) ? sbuf : wbuf), NULL, &wdag, &wasm, RF_DAG_RETURN_DAG,NULL,NULL,NULL);
    if (rf_looptestShowWrites) {
      char *wwbuf = (char *)((rf_demoMode) ? sbuf : wbuf);
      printf("write off=%lx buf=%lx  %02x %02x %02x %02x %02x    psid=%ld\n",
        offset, wwbuf, wwbuf[0]&0xff, wwbuf[1]&0xff, wwbuf[2]&0xff,
        wwbuf[3]&0xff, wwbuf[4]&0xff, wasm->stripeMap->stripeID);
    }
#else /* RF_DEMO > 0 */
    rf_DoAccess(raidPtr, RF_IO_TYPE_WRITE,NULL, offset, numsect, (caddr_t)wbuf, NULL, &wdag, &wasm, RF_DAG_RETURN_DAG,NULL,NULL,NULL);
    if (rf_looptestShowWrites) {
      printf("write off=%lx buf=%lx  %02x %02x %02x %02x %02x    psid=%ld\n",
        offset, wbuf, wbuf[0]&0xff, wbuf[1]&0xff, wbuf[2]&0xff,
        wbuf[3]&0xff, wbuf[4]&0xff, wasm->stripeMap->stripeID);
    }
#endif /* RF_DEMO > 0 */
    RF_ETIMER_STOP(wtimer);
    RF_ETIMER_EVAL(wtimer);
    resptime = RF_ETIMER_VAL_MS(wtimer);
    tot_wtime += resptime;
#if RF_DEMO > 0
    if (rf_demoMode)
      rf_update_user_stats(resptime);
#endif /* RF_DEMO > 0 */
    
    bzero((char *)rbuf, numbytes);
    RF_ETIMER_START(rtimer);
    rf_DoAccess(raidPtr, RF_IO_TYPE_READ,NULL, offset, numsect, (caddr_t)rbuf, NULL, &rdag, &rasm, RF_DAG_RETURN_ASM|RF_DAG_RETURN_DAG, NULL, NULL, NULL);
    if (rasm == NULL)
      {
	rasm = rf_MapAccess(raidPtr, offset, numsect, (caddr_t)rbuf, RF_DONT_REMAP);
      }

    RF_ETIMER_STOP(rtimer);
    RF_ETIMER_EVAL(rtimer);
    resptime = RF_ETIMER_VAL_MS(rtimer);
    tot_rtime += resptime;
#if RF_DEMO > 0
    if (rf_demoMode)
      rf_update_user_stats(resptime);
#endif /* RF_DEMO > 0 */
    
    tot_size += numsect;
    iocount++;
    
    /* hold off on collecting stats until disk has failed */
    if ((rf_testcode_degr_mode_type == 'r'
          || rf_testcode_degr_mode_type == 'a'
          || rf_testcode_degr_mode_type == 'd')
        && (!disk_has_failed))
    {
      iocount = tot_wtime = tot_rtime = tot_size = 0;
    }

    for (i=0; i<numbytes/sizeof(long); i++) {
#if RF_DEMO > 0
      if (rbuf[i] != sbuf[i] && !rf_demoSMM && !(rf_testcode_degr_mode_type == 'r' && rf_demoSuppressReconInitVerify))
#else /* RF_DEMO > 0 */
      if (rbuf[i] != sbuf[i] && !(rf_testcode_degr_mode_type == 'r'))
#endif /* RF_DEMO > 0 */
      {
	printf("[%d] MISCOMPARE! raidAddr %ld numSect %d: longword %d is 0x%x should be 0x%x\n",tid,
	       offset,numsect,i, (unsigned long) rbuf[i], (unsigned long) sbuf[i]);
	PrintAccessInfo(raidPtr, offset, numsect, wasm, wdag, rasm, rdag);
	exit(1);
      }
    }
    if (painful_test) if (VerifyAllData(raidPtr, painful_test_buf)) {
      PrintAccessInfo(raidPtr, offset, numsect, wasm, wdag, rasm, rdag);
      exit(1);
    }

    /* verify that parity is _still_ correct in all stripes to be accessed */
    if (!rf_disableParityVerify) {
      status = rf_VerifyParity(raidPtr, rasm->stripeMap, RF_DONT_CORRECT_PARITY, RF_DAG_FLAGS_NONE);
      if (status == RF_PARITY_COULD_NOT_VERIFY) {
        PrintParityVerifyString(status);
      }
      else {
        if (status) {
          PrintParityVerifyString(status);
          RF_PANIC();
        }
      }
    }
    else {
#if RF_DEMO > 0
      if (layoutPtr->map->faultsTolerated && !rf_demoMode)
#else /* RF_DEMO > 0 */
      if (layoutPtr->map->faultsTolerated)
#endif /* RF_DEMO > 0 */
      {
        for (i=0,p = wasm->stripeMap; p; p = p->next,i++) {
          if (p->numDataFailed + p->numParityFailed == 1) {
            /* not currently implemented:  a stub right now */
            if (rf_VerifyDegrModeWrite(raidPtr, wasm)) {
              printf("Degr mode write (ASM %d in following list) failed to verify\n", i);