rf_reconstruct.c

RAIDFrame是个非常好的磁盘阵列RAID仿真工具

}


/* computes a new minimum head sep, and wakes up anyone who needs to be woken as a result */
static void CheckForNewMinHeadSep(raidPtr, row, hsCtr)
  RF_Raid_t          *raidPtr;
  RF_RowCol_t         row;
  RF_HeadSepLimit_t   hsCtr;
{
  RF_ReconCtrl_t *reconCtrlPtr = raidPtr->reconControl[row];
  RF_HeadSepLimit_t new_min;
  RF_RowCol_t i;
  RF_CallbackDesc_t *p;
  RF_ASSERT(hsCtr >= reconCtrlPtr->minHeadSepCounter);           /* from the definition of a minimum */
  

  RF_LOCK_MUTEX(reconCtrlPtr->rb_mutex);
  
  new_min = ~ (1L<< (8*sizeof(long)-1));         /* 0x7FFF....FFF */
  for (i=0; i<raidPtr->numCol; i++) if (i != reconCtrlPtr->fcol) {
    if (reconCtrlPtr->perDiskInfo[i].headSepCounter < new_min) new_min = reconCtrlPtr->perDiskInfo[i].headSepCounter;
  }

  /* set the new minimum and wake up anyone who can now run again */
  if (new_min != reconCtrlPtr->minHeadSepCounter) {
    reconCtrlPtr->minHeadSepCounter = new_min;
    Dprintf1("RECON:  new min head pos counter val is %ld\n",new_min);
    while (reconCtrlPtr->headSepCBList) {
      if (reconCtrlPtr->headSepCBList->callbackArg.v > new_min) break;
      p = reconCtrlPtr->headSepCBList;
      reconCtrlPtr->headSepCBList = p->next;
      p->next = NULL;
      rf_CauseReconEvent(raidPtr, p->row, p->col, NULL, RF_REVENT_HEADSEPCLEAR);
      rf_FreeCallbackDesc(p);
    }
    
  }

  RF_UNLOCK_MUTEX(reconCtrlPtr->rb_mutex);
}

/* checks to see that the maximum head separation will not be violated
 * if we initiate a reconstruction I/O on the indicated disk.  Limiting the
 * maximum head separation between two disks eliminates the nasty buffer-stall
 * conditions that occur when one disk races ahead of the others and consumes
 * all of the floating recon buffers.  This code is complex and unpleasant
 * but it's necessary to avoid some very nasty, albeit fairly rare,
 * reconstruction behavior.
 *
 * returns non-zero if and only if we have to stop working on the indicated disk
 * due to a head-separation delay.
 */
static int CheckHeadSeparation(
  RF_Raid_t              *raidPtr,
  RF_PerDiskReconCtrl_t  *ctrl,
  RF_RowCol_t             row,
  RF_RowCol_t             col,
  RF_HeadSepLimit_t       hsCtr,
  RF_ReconUnitNum_t       which_ru)
{
  RF_ReconCtrl_t *reconCtrlPtr = raidPtr->reconControl[row];
  RF_CallbackDesc_t *cb, *p, *pt;
  int retval = 0, tid;
  
  /* if we're too far ahead of the slowest disk, stop working on this disk
   * until the slower ones catch up.  We do this by scheduling a wakeup callback
   * for the time when the slowest disk has caught up.  We define "caught up"
   * with 20% hysteresis, i.e. the head separation must have fallen to at most
   * 80% of the max allowable head separation before we'll wake up.
   * 
   */
  rf_get_threadid(tid);
  RF_LOCK_MUTEX(reconCtrlPtr->rb_mutex);
  if ((raidPtr->headSepLimit >= 0) &&
      ((ctrl->headSepCounter - reconCtrlPtr->minHeadSepCounter) > raidPtr->headSepLimit))
  {
    Dprintf6("[%d] RECON: head sep stall: row %d col %d hsCtr %ld minHSCtr %ld limit %ld\n",
			   tid,row,col,ctrl->headSepCounter, reconCtrlPtr->minHeadSepCounter, raidPtr->headSepLimit);
    cb = rf_AllocCallbackDesc();
    /* the minHeadSepCounter value we have to get to before we'll wake up.  build in 20% hysteresis. */
    cb->callbackArg.v  = (ctrl->headSepCounter - raidPtr->headSepLimit + raidPtr->headSepLimit/5);
    cb->row = row; cb->col = col;
    cb->next = NULL;

    /* insert this callback descriptor into the sorted list of pending head-sep callbacks */
    p = reconCtrlPtr->headSepCBList;
    if (!p) reconCtrlPtr->headSepCBList = cb;
    else if (cb->callbackArg.v < p->callbackArg.v) {
      cb->next = reconCtrlPtr->headSepCBList;
      reconCtrlPtr->headSepCBList = cb;
    }
    else {
      for (pt=p, p=p->next; p && (p->callbackArg.v < cb->callbackArg.v); pt=p,p=p->next);
      cb->next = p;
      pt->next = cb;
    }
    retval = 1;
#if RF_RECON_STATS > 0
    ctrl->reconCtrl->reconDesc->hsStallCount++;
#endif /* RF_RECON_STATS > 0 */
  }
  RF_UNLOCK_MUTEX(reconCtrlPtr->rb_mutex);

  return(retval);
}

/* checks to see if reconstruction has been either forced or blocked by a user operation.
 * if forced, we skip this RU entirely.
 * else if blocked, put ourselves on the wait list.
 * else return 0.
 *
 * ASSUMES THE PSS MUTEX IS LOCKED UPON ENTRY
 */
static int CheckForcedOrBlockedReconstruction(
  RF_Raid_t                     *raidPtr,
  RF_ReconParityStripeStatus_t  *pssPtr,
  RF_PerDiskReconCtrl_t         *ctrl,
  RF_RowCol_t                    row,
  RF_RowCol_t                    col,
  RF_StripeNum_t                 psid,
  RF_ReconUnitNum_t              which_ru)
{
  RF_CallbackDesc_t *cb;
  int retcode = 0;

  if ((pssPtr->flags & RF_PSS_FORCED_ON_READ) || (pssPtr->flags & RF_PSS_FORCED_ON_WRITE)) retcode = RF_PSS_FORCED_ON_WRITE;
  else if (pssPtr->flags & RF_PSS_RECON_BLOCKED) {
    Dprintf4("RECON: row %d col %d blocked at psid %ld ru %d\n",row, col, psid, which_ru);
    cb = rf_AllocCallbackDesc();   /* append ourselves to the blockage-wait list */
    cb->row = row; cb->col = col;
    cb->next = pssPtr->blockWaitList;
    pssPtr->blockWaitList = cb;
    retcode = RF_PSS_RECON_BLOCKED;
  }

  if (!retcode) pssPtr->flags |= RF_PSS_UNDER_RECON;    /* mark this RU as under reconstruction */
  
  return(retcode);
}

/* if reconstruction is currently ongoing for the indicated stripeID, reconstruction
 * is forced to completion and we return non-zero to indicate that the caller must
 * wait.  If not, then reconstruction is blocked on the indicated stripe and the
 * routine returns zero.  If and only if we return non-zero, we'll cause the cbFunc
 * to get invoked with the cbArg when the reconstruction has completed.
 */
int rf_ForceOrBlockRecon(raidPtr, asmap, cbFunc, cbArg)
  RF_Raid_t              *raidPtr;
  RF_AccessStripeMap_t   *asmap;
  void                  (*cbFunc)();
  void                   *cbArg;
{
  RF_RowCol_t row = asmap->physInfo->row;               /* which row of the array we're working on */
  RF_StripeNum_t stripeID = asmap->stripeID;                    /* the stripe ID we're forcing recon on */
  RF_SectorCount_t sectorsPerRU = raidPtr->Layout.sectorsPerStripeUnit * raidPtr->Layout.SUsPerRU;   /* num sects in one RU */
  RF_ReconParityStripeStatus_t *pssPtr;                /* a pointer to the parity stripe status structure */
  RF_StripeNum_t psid; /* parity stripe id */
  RF_SectorNum_t offset, fd_offset;                        /* disk offset, failed-disk offset */
  RF_RowCol_t *diskids;
  RF_RowCol_t stripe;
  int tid;
  RF_ReconUnitNum_t which_ru; /* RU within parity stripe */
  RF_RowCol_t fcol, diskno, i;
  RF_ReconBuffer_t *new_rbuf;                               /* ptr to newly allocated rbufs */
  RF_DiskQueueData_t *req;                                  /* disk I/O req to be enqueued */
  RF_CallbackDesc_t *cb;
  int created = 0, nPromoted;

  rf_get_threadid(tid);
  psid = rf_MapStripeIDToParityStripeID(&raidPtr->Layout, stripeID, &which_ru);

  RF_LOCK_PSS_MUTEX(raidPtr, row, psid);

  pssPtr = rf_LookupRUStatus(raidPtr, raidPtr->reconControl[row]->pssTable, psid, which_ru, RF_PSS_CREATE|RF_PSS_RECON_BLOCKED, &created);

  /* if recon is not ongoing on this PS, just return */
  if (!(pssPtr->flags & RF_PSS_UNDER_RECON)) {
    RF_UNLOCK_PSS_MUTEX(raidPtr, row, psid);
    return(0);
  }

  /* otherwise, we have to wait for reconstruction to complete on this RU. */
  /* In order to avoid waiting for a potentially large number of low-priority accesses to
   * complete, we force a normal-priority (i.e. not low-priority) reconstruction
   * on this RU.
   */
  if (!(pssPtr->flags & RF_PSS_FORCED_ON_WRITE) && !(pssPtr->flags & RF_PSS_FORCED_ON_READ)) {
    DDprintf1("Forcing recon on psid %ld\n",psid);
    pssPtr->flags |= RF_PSS_FORCED_ON_WRITE;     /* mark this RU as under forced recon */
    pssPtr->flags &= ~RF_PSS_RECON_BLOCKED;      /* clear the blockage that we just set */
    fcol = raidPtr->reconControl[row]->fcol;
    
    /* get a listing of the disks comprising the indicated stripe */
    (raidPtr->Layout.map->IdentifyStripe)(raidPtr, asmap->raidAddress, &diskids, &stripe);
    RF_ASSERT(row == stripe);
    
    /* For previously issued reads, elevate them to normal priority.  If the I/O has already completed,
     * it won't be found in the queue, and hence this will be a no-op.
     * For unissued reads, allocate buffers and issue new reads.  The fact that we've set the
     * FORCED bit means that the regular recon procs will not re-issue these reqs
     */
    for (i=0; i<raidPtr->Layout.numDataCol+raidPtr->Layout.numParityCol; i++) if ( (diskno = diskids[i]) != fcol) {
      if (pssPtr->issued[diskno]) {
	nPromoted = rf_DiskIOPromote(&raidPtr->Queues[row][diskno], psid, which_ru);
	if (rf_reconDebug && nPromoted) printf("[%d] promoted read from row %d col %d\n",tid,row,diskno);
      } else {
	new_rbuf = rf_MakeReconBuffer(raidPtr, row, diskno, RF_RBUF_TYPE_FORCED);              /* create new buf */
	ComputePSDiskOffsets(raidPtr, psid, row, diskno, &offset, &fd_offset,
			     &new_rbuf->spRow, &new_rbuf->spCol, &new_rbuf->spOffset);   /* find offsets & spare location */
	new_rbuf->parityStripeID = psid;                                                 /* fill in the buffer */
	new_rbuf->which_ru = which_ru;
	new_rbuf->failedDiskSectorOffset = fd_offset;
	new_rbuf->priority = RF_IO_NORMAL_PRIORITY;

	/* use NULL b_proc b/c all addrs should be in kernel space */
	req = rf_CreateDiskQueueData(RF_IO_TYPE_READ, offset + which_ru * sectorsPerRU, sectorsPerRU, new_rbuf->buffer,
				  psid, which_ru, (int(*)())ForceReconReadDoneProc, (void *) new_rbuf, NULL,
				  NULL,(void *)raidPtr, 0, NULL);

	RF_ASSERT(req);          /* XXX -- fix this -- XXX */

	new_rbuf->arg = req;
	rf_DiskIOEnqueue(&raidPtr->Queues[row][diskno], req, RF_IO_NORMAL_PRIORITY);     /* enqueue the I/O */
	Dprintf3("[%d] Issued new read req on row %d col %d\n",tid,row,diskno);
      }
    }

    /* if the write is sitting in the disk queue, elevate its priority */
    if (rf_DiskIOPromote(&raidPtr->Queues[row][fcol], psid, which_ru)) printf("[%d] promoted write to row %d col %d\n",tid,row,fcol);
  }
  
  /* install a callback descriptor to be invoked when recon completes on this parity stripe. */
  cb = rf_AllocCallbackDesc();
  cb->callbackFunc = cbFunc;
  cb->callbackArg.p = (void *) cbArg;
  cb->next = pssPtr->procWaitList;
  pssPtr->procWaitList = cb;
  DDprintf2("[%d] Waiting for forced recon on psid %ld\n",tid,psid);
  
  RF_UNLOCK_PSS_MUTEX(raidPtr, row, psid);
  return(1);
}

/* called upon the completion of a forced reconstruction read.
 * all we do is schedule the FORCEDREADONE event.
 * called at interrupt context in the kernel, so don't do anything illegal here.
 */
static void ForceReconReadDoneProc(arg, status)
  void  *arg;
  int    status;
{
  RF_ReconBuffer_t *rbuf = arg;

  if (status) {printf("Forced recon read failed!\n"); /*fprintf(stderr,"Forced recon read failed!\n");*/ RF_PANIC();}
  rf_CauseReconEvent((RF_Raid_t *) rbuf->raidPtr, rbuf->row, rbuf->col, (void *) rbuf, RF_REVENT_FORCEDREADDONE);
}

/* releases a block on the reconstruction of the indicated stripe */
int rf_UnblockRecon(raidPtr, asmap)
  RF_Raid_t             *raidPtr;
  RF_AccessStripeMap_t  *asmap;
{
  RF_RowCol_t row = asmap->origRow;
  RF_StripeNum_t stripeID = asmap->stripeID;
  RF_ReconParityStripeStatus_t *pssPtr;
  RF_ReconUnitNum_t which_ru;
  RF_StripeNum_t psid;
  int tid, created = 0;
  RF_CallbackDesc_t *cb;

  rf_get_threadid(tid);
  psid = rf_MapStripeIDToParityStripeID(&raidPtr->Layout, stripeID, &which_ru);
  RF_LOCK_PSS_MUTEX( raidPtr, row, psid);
  pssPtr = rf_LookupRUStatus(raidPtr, raidPtr->reconControl[row]->pssTable, psid, which_ru, RF_PSS_NONE, &created);

  /* When recon is forced, the pss desc can get deleted before we get back to unblock recon.
   * But, this can _only_ happen when recon is forced.
   * It would be good to put some kind of sanity check here, but how to decide if recon
   * was just forced or not?
   */
  if (!pssPtr) {
    /*printf("Warning: no pss descriptor upon unblock on psid %ld RU %d\n",psid,which_ru);*/
    if (rf_reconDebug || rf_pssDebug) printf("Warning: no pss descriptor upon unblock on psid %ld RU %d\n",psid,which_ru);
    goto out;
  }

  pssPtr->blockCount--;
  Dprintf3("[%d] unblocking recon on psid %ld: blockcount is %d\n",tid,psid,pssPtr->blockCount);
  if (pssPtr->blockCount == 0) {     /* if recon blockage has been released */

    /* unblock recon before calling CauseReconEvent in case CauseReconEvent causes us to
     * try to issue a new read before returning here.
     */
    pssPtr->flags &= ~RF_PSS_RECON_BLOCKED;

    
    while (pssPtr->blockWaitList) {  /* spin through the block-wait list and release all the waiters */
      cb = pssPtr->blockWaitList;
      pssPtr->blockWaitList = cb->next;
      cb->next = NULL;
      rf_CauseReconEvent(raidPtr, cb->row, cb->col, NULL, RF_REVENT_BLOCKCLEAR);
      rf_FreeCallbackDesc(cb);
    }
    if (!(pssPtr->flags & RF_PSS_UNDER_RECON)) {     /* if no recon was requested while recon was blocked */
      rf_PSStatusDelete(raidPtr, raidPtr->reconControl[row]->pssTable, pssPtr);
    }
  }
  
out:
  RF_UNLOCK_PSS_MUTEX( raidPtr, row, psid );
  return(0);
}