rf_diskthreads.c

RAIDFrame是个非常好的磁盘阵列RAID仿真工具

  rf_assign_threadid();
  /* open raw device file */
  if (!rf_camlayerIOs) {
    fd = open_raw_disk_file(id->disk->devname);
    id->fd = fd;
  } else {
    rf_extract_ids(id->disk->devname, &bus, &target, &lun);
    r_handle = rf_CreateCamIOHandle();
    w_handle = rf_CreateCamIOHandle();
    rf_SCSI_setup_rw_handle(RF_IO_TYPE_READ,r_handle, bus, target, lun);
    rf_SCSI_setup_rw_handle(RF_IO_TYPE_WRITE,w_handle, bus, target, lun);
  }
  if (rf_dtDebug || rf_doDebug) {
    rf_get_threadid(tid);
  }
  if (rf_dtDebug) {
    printf("[%d] Disk thread for r%d c%d: file %s, numblocks %ld blocksize %ld\n",tid,
		      id->row,id->col,id->disk->devname,id->disk->numBlocks,id->disk->blockSize);
  }
  RF_LOCK_MUTEX(raidPtr->diskthread_count_mutex);
  raidPtr->diskthreads_running++;
  RF_UNLOCK_MUTEX(raidPtr->diskthread_count_mutex);
  RF_SIGNAL_COND(raidPtr->diskthread_count_cond);

  while (1) {
    req = rf_DiskIODequeue(id->queue);       /* blocks thread until something available.  Can also terminate thread. */
    if (!initialized) {gettimeofday(&st_time, NULL); initialized = 1;}
    status=0;
    if (!req) break;


    /* reject the request if the disk is in any of the dead states */
    if (RF_DEAD_DISK(id->disk->status)) {
      printf("DISKTHREAD: Rejecting access to dead disk at r %d c %d\n",id->row, id->col);
      status = -1; goto out;
    }

    sectOffset = req->sectorOffset;
    byteoffs = sectOffset * id->disk->blockSize;
    numbytes = req->numSector * id->disk->blockSize;
    /* the mapping code must guarantee this property */
    RF_ASSERT(req->sectorOffset + req->numSector <= id->disk->numBlocks + rf_protectedSectors);
    if (req->type != RF_IO_TYPE_NOP && numbytes == 0) {
      if (id->row >= 0) printf("WARNING: zero-byte transfer to disk row %d col %d\n",id->row,id->col);
      else printf("WARNING: zero-byte transfer to spare disk %d\n",id->col);
    }
    
    if (req->type != RF_IO_TYPE_NOP && req->numSector) {           /* allow zero-sector accesses with no errors */
      if (!rf_camlayerIOs) {
	if (lseek(fd, byteoffs, SEEK_SET) < 0) {
	  RF_ERRORMSG3("Error: Unable to lseek to sector %ld on disk at row %d col %d\n", req->sectorOffset, id->row, id->col);
	  perror("disk_rw_func");
	  status = -1;
	  goto out;
	}
      }

      RF_ETIMER_START(timer);
      switch (req->type)
	{
	case RF_IO_TYPE_NOP:
	  break;
	case RF_IO_TYPE_READ:
	  if (rf_doDebug) {
		fflush(stdout);
		printf("[%d] +Read fd=%d buf=%lx ebuf=%lx len=%d\n",
			tid, fd, req->buf, req->buf+numbytes, numbytes);
		fflush(stdout);
	  }
	  id->disk->nreads++;
	  numactual = (rf_camlayerIOs) ? (rf_SCSI_do_read(id->raidPtr->logBytesPerSector, r_handle, sectOffset, req->numSector, req->buf) ? -1 : numbytes) : read(fd, (char *) req->buf, numbytes);
	  if (rf_doDebug) {
		printf("[%d] -Read fd=%d buf=%lx ebuf=%lx len=%d\n",
			tid, fd, req->buf, req->buf+numbytes, numbytes);
		fflush(stdout);
	  }
	  break;
	case RF_IO_TYPE_WRITE:
	  if (rf_doDebug) {
		fflush(stdout);
		printf("[%d] +Write fd=%d buf=%lx ebuf=%lx len=%d\n",
			tid, fd, req->buf, req->buf+numbytes, numbytes);
		fflush(stdout);
	  }
	  id->disk->nwrites++;
	  numactual = (rf_camlayerIOs) ? (rf_SCSI_do_write(id->raidPtr->logBytesPerSector, w_handle, sectOffset, req->numSector, req->buf) ? -1 : numbytes) : write(fd, (char *) req->buf, numbytes);
	  if (rf_doDebug) {
		printf("[%d] -Write fd=%d buf=%lx ebuf=%lx len=%d\n",
			tid, fd, req->buf, req->buf+numbytes, numbytes);
		fflush(stdout);
	  }
	  break;
	default:
	  RF_ERRORMSG1("Error: unknown access type '%c' in disk_rw_func\n",req->type);
	  status = -1;
	  goto out;
	  break;
	} /* end case */

      RF_ETIMER_STOP(timer); RF_ETIMER_EVAL(timer);
      
      if (req->type != RF_IO_TYPE_NOP && numactual != numbytes) {
	fprintf(stderr,"Error: failed disk acc (r %d c %d): type %c offs %ld numSect %d returned %d instead of %d\n",
		  id->row, id->col, req->type, req->sectorOffset, req->numSector, numactual, numbytes);
	perror("disk_rw_func");
	status = -1;
	/* mark the disk as failed, don't initialize reconstruction */
	rf_FailDisk(id->raidPtr, id->row, id->col, RF_FALSE);
	goto out;
      }

      id->queue->numOutstanding--;
      RF_ASSERT(id->queue->numOutstanding >= 0);

      switch (req->type)
	{
	case RF_IO_TYPE_NOP:
	  break;
	case RF_IO_TYPE_READ:
	  rdRtSum += RF_ETIMER_VAL_US(timer); 
	  nReads++;
	  break;
	case RF_IO_TYPE_WRITE:
	  wrRtSum += RF_ETIMER_VAL_US(timer); 
	  nWrites++;	  
	  break;
	default:
	  RF_PANIC();
	}


      if (req->tracerec) {
	/* assume disk media rate is 3 MB/sec */
	media_time_ms = (numbytes / 1024) / 3;
	if (rf_warnLongIOs && (( (int) RF_ETIMER_VAL_MS(timer) - (int) media_time_ms) > (int) 80)) 
		printf("Warning: %d ms for %dk I/O (media time %d)\n", RF_ETIMER_VAL_MS(timer), numbytes>>10, media_time_ms);
	RF_LOCK_MUTEX(rf_tracing_mutex);
	req->tracerec->diskwait_us += RF_ETIMER_VAL_US(timer);
	req->tracerec->phys_io_us += RF_ETIMER_VAL_US(timer);
	req->tracerec->num_phys_ios++;
	RF_UNLOCK_MUTEX(rf_tracing_mutex);
      }
    } else if (req->type != RF_IO_TYPE_NOP) {
      printf("Warning: zero-sector access to offset %d of disk r %d c %d\n",req->sectorOffset,id->row,id->col);
    }
#if RF_DEMO > 0
    if (rf_demoMode) {
      rf_update_disk_iops(1);
    }
#endif /* RF_DEMO > 0 */

    /* fail the request if the disk was marked as bad during the access */
    if (RF_DEAD_DISK(id->disk->status)) {
      printf("DISKTHREAD: Rejecting access to dead disk at r %d c %d\n",id->row, id->col);
      status = -1; goto out;
    }
    
out:

    /* unlock the disk queue if this is an unlocking req, independent of failure status of I/O */
    if (RF_UNLOCKING_REQ(req)) {
      if (rf_queueDebug) printf("[%d] unlocking q after pri %d req to r %d c %d\n",tid,req->priority,id->queue->row, id->queue->col);
      RF_LOCK_MUTEX(id->queue->mutex);
      RF_ASSERT(RF_QUEUE_LOCKED(id->queue));
      RF_UNLOCK_QUEUE(id->queue);
      id->queue->nextLockingOp = id->queue->unlockingOp = NULL;
      rf_BroadcastOnQueue(id->queue);    /* wake up all disk threads & have them recheck the queue */
      RF_UNLOCK_MUTEX(id->queue->mutex);
    } else if (RF_LOCKING_REQ(req)) {
      if (rf_queueDebug) printf("[%d] completed pri %d locking req on r %d c %d\n",tid,req->priority,id->queue->row, id->queue->col);
    } else if (rf_queueDebug) printf("[%d] completed pri %d regular req on r %d c %d\n",tid,req->priority,id->queue->row, id->queue->col);

    if (rf_dtDebug > 1) {
      RF_LOCK_MUTEX(rf_printf_mutex);
      printf("Disk op (r%d c%d) %c %ld %d buf 0x%lx stat %d ",
			id->row,id->col,req->type,req->sectorOffset, req->numSector, req->buf, status);
      if (req->type != RF_IO_TYPE_NOP) {
	printf("data ");
	for (i=0;i<RF_MIN(10,numbytes); i++) printf("%02x",(unsigned char) (req->buf[i]));
	printf("...\n");
      }
      RF_UNLOCK_MUTEX(rf_printf_mutex);
    }
    (req->CompleteFunc)(req->argument, status);
  }
  if (!rf_camlayerIOs) {
    close(fd);
  }
  else {
    rf_DestroyCamIOHandle(r_handle);
    rf_DestroyCamIOHandle(w_handle);
  }
  gettimeofday(&end_time, NULL);
  RF_TIMEVAL_DIFF(&st_time, &end_time, &elpsd);
  util = RF_DB0_CHECK( (100*(rdRtSum+wrRtSum)), (long) (elpsd.tv_sec * 1000000 + elpsd.tv_usec) );
  if (rf_dtDebug) {
    int tid;
    rf_get_threadid(tid);
    printf("[%d] disk r %2d c %2d : %d %% util, %d reads, %d writes, %d total, %d avg rd RT, %d avg wr RT, %d avg acc RT\n",
      tid, id->row, id->col, util, nReads, nWrites, nReads+nWrites,
      RF_DB0_CHECK(rdRtSum,nReads), RF_DB0_CHECK(wrRtSum,nWrites),
      RF_DB0_CHECK( (rdRtSum+wrRtSum),(nReads+nWrites) ) );
  }
  else {
    printf("disk r %2d c %2d : %d %% util, %d reads, %d writes, %d total, %d avg rd RT, %d avg wr RT, %d avg acc RT\n",
      id->row, id->col, util, nReads, nWrites, nReads+nWrites,
      RF_DB0_CHECK(rdRtSum,nReads), RF_DB0_CHECK(wrRtSum,nWrites),
      RF_DB0_CHECK( (rdRtSum+wrRtSum),(nReads+nWrites) ) );
  }
  RF_LOCK_MUTEX(raidPtr->diskthread_count_mutex);
  raidPtr->diskthreads_shutdown++;
  RF_UNLOCK_MUTEX(raidPtr->diskthread_count_mutex);
  RF_SIGNAL_COND(raidPtr->diskthread_count_cond);
}

#endif /* SIMULATE */


#ifdef SIMULATE

void rf_PrintDiskStat(RF_Raid_t *raidPtr)
{
    RF_RowCol_t row, col;
    RF_DiskState_t *disk;
    long     numEvents;
    double  avgSeek, avgRotate, avgTransfer, avgAccess;
    double  Sleep, Idle, Rw, Spinup;
    long    sumNumEvents;
    double  sumSeek, sumRotate, sumTransfer, sumAccess;
    double  sumSleep, sumIdle, sumRw, sumSpinup;
    long    sumAllNumEvents;
    double  sumAllSeek, sumAllRotate, sumAllTransfer, sumAllAccess;
   
    double  sumAllSleep, sumAllIdle, sumAllRw, sumAllSpinup;
    
    double  mJSleep, mJIdle, mJRw, mJSpinup;
    
    sumAllNumEvents = 0;
    sumAllSeek = sumAllRotate = sumAllTransfer = sumAllAccess = 0.0;
    sumAllSleep =  sumAllIdle =  sumAllRw =  sumAllSpinup= 0.0;

    for (col = 0; col < raidPtr->numCol; col++) {
        sumNumEvents = 0;
        sumSeek = sumRotate = sumTransfer = sumAccess = 0.0;
        sumSleep = sumIdle =  sumRw =  sumSpinup=0.0;
        for (row = 0; row < raidPtr->numRow; row++) {
		 
           disk= &raidPtr->diskids[row][col].disk->diskState;

	   rf_StopStats (disk, rf_cur_time);

	   rf_Report_stats(disk, &numEvents,
                         &avgSeek, &avgRotate, &avgTransfer, &avgAccess,
                         &Sleep, &Idle, &Rw, &Spinup);
#if RF_KEEP_DISKSTATS > 0
           printf("disk(%2d %2d) Sleep:%5g Idle:%5g RW:%5g Spinup:%5g Reads:%lu Writes:%lu\n",
                  col, row, Sleep, Idle, Rw, Spinup,
                  raidPtr->diskids[row][col].disk->nreads,
                  raidPtr->diskids[row][col].disk->nwrites);
#else /* RF_KEEP_DISKSTATS > 0 */
           printf("disk(%2d %2d) Sleep:%5g Idle:%5g RW:%5g Spinup:%5g\n",
                  col, row, Sleep, Idle, Rw, Spinup);
#endif /* RF_KEEP_DISKSTATS > 0 */
        
           sumNumEvents += numEvents;
           sumSeek      += numEvents * avgSeek;
           sumRotate    += numEvents * avgRotate;
           sumTransfer  += numEvents * avgTransfer;
           sumAccess    += numEvents * avgAccess;

           sumSleep     += Sleep;
           sumIdle      += Idle;
           sumRw        += Rw;
           sumSpinup    += Spinup;
          
        }
        /* sumNumEvents can be zero if the disk was failed & not replaced during simulation */

        sumAllNumEvents += sumNumEvents;
        sumAllSeek      += sumSeek;
        sumAllRotate    += sumRotate;
        sumAllTransfer  += sumTransfer;
        sumAllAccess    += sumAccess;
        sumAllSleep     += sumSleep;
        sumAllIdle      +=  sumIdle;
        sumAllRw        +=  sumRw;
        sumAllSpinup    +=  sumSpinup;
    }

    printf("disk(%2s %2s) Sleep:%5g Idle:%5g RW:%5g Spinup:%5g\n",
	   "*", "*",  sumAllSleep, sumAllIdle, sumAllRw, sumAllSpinup);

    mJSleep=sumAllSleep*rf_mWsleep;
    mJIdle=sumAllIdle*rf_mWidle;
    mJRw= sumAllRw*   rf_mWactive;
    mJSpinup=sumAllSpinup*rf_mWspinup;

    printf("Energy in J  Sleep:%5g Idle:%5g RW:%5g Spinup:%5g\n",
	   mJSleep/1000.0, mJIdle/1000.0, mJRw/1000.0, mJSpinup/1000.0);
	   
    printf("Energy in J  Total:%5g\n",(mJSleep + mJIdle + mJRw + mJSpinup)/1000.0);

    printf ("Total number of events = %d\n", sumAllNumEvents);
    printf ("[Total elapsed time = %8.2f ]\n", rf_cur_time);
    printf ("number of disks %d\n",raidPtr->numRow*raidPtr->numCol);
    
    printf ("Total time %g\n",sumAllSleep+sumAllIdle+sumAllRw+sumAllSpinup);
    printf ("Number of spinups %d \n",(int)(sumAllSpinup/rf_globalSpinup));
    printf ("Total time divided by mumber of disks %g\n",
	    (sumAllSleep+sumAllIdle+sumAllRw+sumAllSpinup)/(raidPtr->numRow*raidPtr->numCol));
}

#endif /* SIMULATE */