xferpkt.c

This is the library for all storage drivers. It simplifies writing a storage driver by implementing

            /*
             *  We are in stress and have run out of lookaside packets.
             *  In order to service the current transfer,
             *  allocate an extra packet.
             *  We will free it lazily when we are out of stress.
             */
            pkt = NewTransferPacket(Fdo);
            if (pkt){
                InterlockedIncrement(&fdoData->NumTotalTransferPackets);
                fdoData->DbgPeakNumTransferPackets = max(fdoData->DbgPeakNumTransferPackets, fdoData->NumTotalTransferPackets);
            }
            else {
                DBGWARN(("DequeueFreeTransferPacket: packet allocation failed"));
            }
        }
        else {
            pkt = NULL;
        }
    }

    return pkt;
}



/*
 *  SetupReadWriteTransferPacket
 *
 *        This function is called once to set up the first attempt to send a packet.
 *        It is not called before a retry, as SRB fields may be modified for the retry.
 *
 *      Set up the Srb of the TRANSFER_PACKET for the transfer.
 *        The Irp is set up in SubmitTransferPacket because it must be reset
 *        for each packet submission.
 */
VOID SetupReadWriteTransferPacket(  PTRANSFER_PACKET Pkt,
                                            PVOID Buf,
                                            ULONG Len,
                                            LARGE_INTEGER DiskLocation,
                                            PIRP OriginalIrp)
{
    PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
    PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
    PIO_STACK_LOCATION origCurSp = IoGetCurrentIrpStackLocation(OriginalIrp);
    UCHAR majorFunc = origCurSp->MajorFunction;
    LARGE_INTEGER logicalBlockAddr;
    ULONG numTransferBlocks;
    PCDB pCdb;

    logicalBlockAddr.QuadPart = Int64ShrlMod32(DiskLocation.QuadPart, fdoExt->SectorShift);
    numTransferBlocks = Len >> fdoExt->SectorShift;

    /*
     *  Slap the constant SRB fields in from our pre-initialized template.
     *  We'll then only have to fill in the unique fields for this transfer.
     *  Tell lower drivers to sort the SRBs by the logical block address
     *  so that disk seeks are minimized.
     */
    Pkt->Srb = fdoData->SrbTemplate;    // copies _contents_ of SRB blocks
    Pkt->Srb.DataBuffer = Buf;
    Pkt->Srb.DataTransferLength = Len;
    Pkt->Srb.QueueSortKey = logicalBlockAddr.LowPart;
    if (logicalBlockAddr.QuadPart > 0xFFFFFFFF) {
        //
        // If the requested LBA is more than max ULONG set the
        // QueueSortKey to the maximum value, so that these
        // requests can be added towards the end of the queue.
        //

        Pkt->Srb.QueueSortKey = 0xFFFFFFFF;
    }
    Pkt->Srb.OriginalRequest = Pkt->Irp;
    Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
    Pkt->Srb.TimeOutValue = (Len/0x10000) + ((Len%0x10000) ? 1 : 0);
    Pkt->Srb.TimeOutValue *= fdoExt->TimeOutValue;

    /*
     *  Arrange values in CDB in big-endian format.
     */
    pCdb = (PCDB)Pkt->Srb.Cdb;
    if (TEST_FLAG(fdoExt->DeviceFlags, DEV_USE_16BYTE_CDB)) {
        REVERSE_BYTES_QUAD(&pCdb->CDB16.LogicalBlock, &logicalBlockAddr);
        REVERSE_BYTES(&pCdb->CDB16.TransferLength, &numTransferBlocks);
        pCdb->CDB16.OperationCode = (majorFunc==IRP_MJ_READ) ? SCSIOP_READ16 : SCSIOP_WRITE16;
        Pkt->Srb.CdbLength = 16;
    } else {
        pCdb->CDB10.LogicalBlockByte0 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte3;
        pCdb->CDB10.LogicalBlockByte1 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte2;
        pCdb->CDB10.LogicalBlockByte2 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte1;
        pCdb->CDB10.LogicalBlockByte3 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte0;
        pCdb->CDB10.TransferBlocksMsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte1;
        pCdb->CDB10.TransferBlocksLsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte0;
        pCdb->CDB10.OperationCode = (majorFunc==IRP_MJ_READ) ? SCSIOP_READ : SCSIOP_WRITE;
    }

    /*
     *  Set SRB and IRP flags
     */
    Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
    if (TEST_FLAG(OriginalIrp->Flags, IRP_PAGING_IO) ||
        TEST_FLAG(OriginalIrp->Flags, IRP_SYNCHRONOUS_PAGING_IO)){
        SET_FLAG(Pkt->Srb.SrbFlags, SRB_CLASS_FLAGS_PAGING);
    }
    SET_FLAG(Pkt->Srb.SrbFlags, (majorFunc==IRP_MJ_READ) ? SRB_FLAGS_DATA_IN : SRB_FLAGS_DATA_OUT);

    /*
     *  Allow caching only if this is not a write-through request.
     *  If write-through and caching is enabled on the device, force
     *  media access.
     *  Ignore SL_WRITE_THROUGH for reads; it's only set because the file handle was opened with WRITE_THROUGH.
     */
    if (TEST_FLAG(origCurSp->Flags, SL_WRITE_THROUGH) && (majorFunc != IRP_MJ_READ))
    {
        if (TEST_FLAG(fdoExt->DeviceFlags, DEV_WRITE_CACHE) &&
            !TEST_FLAG(fdoExt->DeviceFlags, DEV_POWER_PROTECTED) &&
            !TEST_FLAG(fdoExt->ScanForSpecialFlags, CLASS_SPECIAL_FUA_NOT_SUPPORTED) )
        {
            pCdb->CDB10.ForceUnitAccess = TRUE;
        }
    }
    else {
        SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_ADAPTER_CACHE_ENABLE);
    }

    /*
     *  Remember the buf and len in the SRB because miniports
     *  can overwrite SRB.DataTransferLength and we may need it again
     *  for the retry.
     */
    Pkt->BufPtrCopy = Buf;
    Pkt->BufLenCopy = Len;
    Pkt->TargetLocationCopy = DiskLocation;

    Pkt->OriginalIrp = OriginalIrp;
    Pkt->NumRetries = NUM_IO_RETRIES;
    Pkt->SyncEventPtr = NULL;
    Pkt->CompleteOriginalIrpWhenLastPacketCompletes = TRUE;

    DBGLOGFLUSHINFO(fdoData, TRUE, (BOOLEAN)(pCdb->CDB10.ForceUnitAccess), FALSE);
}


/*
 *  SubmitTransferPacket
 *
 *        Set up the IRP for the TRANSFER_PACKET submission and send it down.
 */
NTSTATUS SubmitTransferPacket(PTRANSFER_PACKET Pkt)
{
    PCOMMON_DEVICE_EXTENSION commonExtension = Pkt->Fdo->DeviceExtension;
    PDEVICE_OBJECT nextDevObj = commonExtension->LowerDeviceObject;
    PIO_STACK_LOCATION nextSp;

    ASSERT(Pkt->Irp->CurrentLocation == Pkt->Irp->StackCount+1);

    /*
     *  Attach the SRB to the IRP.
     *  The reused IRP's stack location has to be rewritten for each retry
     *  call because IoCompleteRequest clears the stack locations.
     */
    IoReuseIrp(Pkt->Irp, STATUS_NOT_SUPPORTED);
    nextSp = IoGetNextIrpStackLocation(Pkt->Irp);
    nextSp->MajorFunction = IRP_MJ_SCSI;
    nextSp->Parameters.Scsi.Srb = &Pkt->Srb;
    Pkt->Srb.ScsiStatus = Pkt->Srb.SrbStatus = 0;
    Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
    if (Pkt->CompleteOriginalIrpWhenLastPacketCompletes){
        /*
         *  Only dereference the "original IRP"'s stack location
         *  if its a real client irp (as opposed to a static irp
         *  we're using just for result status for one of the non-IO scsi commands).
         *
         *  For read/write, propagate the storage-specific IRP stack location flags
         *  (e.g. SL_OVERRIDE_VERIFY_VOLUME, SL_WRITE_THROUGH).
         */
        PIO_STACK_LOCATION origCurSp = IoGetCurrentIrpStackLocation(Pkt->OriginalIrp);
        nextSp->Flags = origCurSp->Flags;
    }

    //
    // If the request is not split, we can use the original IRP MDL.  If the
    // request needs to be split, we need to use a partial MDL.  The partial MDL
    // is needed because more than one driver might be mapping the same MDL
    // and this causes problems.
    //
    if (Pkt->UsePartialMdl == FALSE) {
        Pkt->Irp->MdlAddress = Pkt->OriginalIrp->MdlAddress;
    } else {
        IoBuildPartialMdl(Pkt->OriginalIrp->MdlAddress, Pkt->PartialMdl, Pkt->Srb.DataBuffer, Pkt->Srb.DataTransferLength);
        Pkt->Irp->MdlAddress = Pkt->PartialMdl;
    }

    DBGLOGSENDPACKET(Pkt);

    IoSetCompletionRoutine(Pkt->Irp, TransferPktComplete, Pkt, TRUE, TRUE, TRUE);
    return IoCallDriver(nextDevObj, Pkt->Irp);
}


NTSTATUS TransferPktComplete(IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
{
    PTRANSFER_PACKET pkt = (PTRANSFER_PACKET)Context;
    PFUNCTIONAL_DEVICE_EXTENSION fdoExt = pkt->Fdo->DeviceExtension;
    PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
    PIO_STACK_LOCATION origCurrentSp = IoGetCurrentIrpStackLocation(pkt->OriginalIrp);
    BOOLEAN packetDone = FALSE;

    /*
     *  Put all the assertions and spew in here so we don't have to look at them.
     */
    DBGLOGRETURNPACKET(pkt);
    DBGCHECKRETURNEDPKT(pkt);

    //
    // If partial MDL was used, unmap the pages.  When the packet is retried, the
    // MDL will be recreated.  If the packet is done, the MDL will be ready to be reused.
    //
    if (pkt->UsePartialMdl) {
        MmPrepareMdlForReuse(pkt->PartialMdl);
    }

    if (SRB_STATUS(pkt->Srb.SrbStatus) == SRB_STATUS_SUCCESS){

        fdoData->LoggedTURFailureSinceLastIO = FALSE;

        /*
         *  The port driver should not have allocated a sense buffer
         *  if the SRB succeeded.
         */
        ASSERT(!PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb));

        /*
         *  Add this packet's transferred length to the original IRP's.
         */
        InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
                              (LONG)pkt->Srb.DataTransferLength);

        if (pkt->InLowMemRetry){
            packetDone = StepLowMemRetry(pkt);
        }
        else {
            packetDone = TRUE;
        }

    }
    else {
        /*
         *  The packet failed.  We may retry it if possible.
         */
        BOOLEAN shouldRetry;

        /*
         *  Make sure IRP status matches SRB error status (since we propagate it).
         */
        if (NT_SUCCESS(Irp->IoStatus.Status)){
            Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
        }

        /*
         *  The packet failed.
         *  So when sending the packet down we either saw either an error or STATUS_PENDING,
         *  and so we returned STATUS_PENDING for the original IRP.
         *  So now we must mark the original irp pending to match that, _regardless_ of
         *  whether we actually switch threads here by retrying.
         *  (We also have to mark the irp pending if the lower driver marked the irp pending;
         *   that is dealt with farther down).
         */
        if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
            IoMarkIrpPending(pkt->OriginalIrp);
        }

        /*
         *  Interpret the SRB error (to a meaningful IRP status)
         *  and determine if we should retry this packet.
         *  This call looks at the returned SENSE info to figure out what to do.
         */
        shouldRetry = InterpretTransferPacketError(pkt);

        /*
         *  Sometimes the port driver can allocates a new 'sense' buffer
         *  to report transfer errors, e.g. when the default sense buffer
         *  is too small.  If so, it is up to us to free it.
         *  Now that we're done interpreting the sense info, free it if appropriate.
         *  Then clear the sense buffer so it doesn't pollute future errors returned in this packet.
         */
        if (PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb)) {
            DBGTRACE(ClassDebugSenseInfo, ("Freeing port-allocated sense buffer for pkt %ph.", pkt));
            FREE_PORT_ALLOCATED_SENSE_BUFFER(fdoExt, &pkt->Srb);
            pkt->Srb.SenseInfoBuffer = &pkt->SrbErrorSenseData;
            pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
        }
        else {
            ASSERT(pkt->Srb.SenseInfoBuffer == &pkt->SrbErrorSenseData);
            ASSERT(pkt->Srb.SenseInfoBufferLength <= sizeof(SENSE_DATA));
        }
        RtlZeroMemory(&pkt->SrbErrorSenseData, sizeof(SENSE_DATA));

        /*
         *  If the SRB queue is locked-up, release it.
         *  Do this after calling the error handler.
         */
        if (pkt->Srb.SrbStatus & SRB_STATUS_QUEUE_FROZEN){
            ClassReleaseQueue(pkt->Fdo);
        }

        if (NT_SUCCESS(Irp->IoStatus.Status)){
            /*
             *  The error was recovered above in the InterpretTransferPacketError() call.
             */

            ASSERT(!shouldRetry);

            /*
             *  In the case of a recovered error,
             *  add the transfer length to the original Irp as we would in the success case.
             */
            InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
                                  (LONG)pkt->Srb.DataTransferLength);

            if (pkt->InLowMemRetry){
                packetDone = StepLowMemRetry(pkt);
            }
            else {
                packetDone = TRUE;
            }
        }
        else {
            if (shouldRetry && (pkt->NumRetries > 0)){
                packetDone = RetryTransferPacket(pkt);
            }
            else {
                packetDone = TRUE;
            }
        }
    }

    /*
     *  If the packet is completed, put it back in the free list.
     *  If it is the last packet servicing the original request, complete the original irp.
     */
    if (packetDone){
        LONG numPacketsRemaining;
        PIRP deferredIrp;
        PDEVICE_OBJECT Fdo = pkt->Fdo;
        UCHAR uniqueAddr;

        /*
         *  In case a remove is pending, bump the lock count so we don't get freed
         *  right after we complete the original irp.
         */
        ClassAcquireRemoveLock(Fdo, (PIRP)&uniqueAddr);

        /*
         *  The original IRP should get an error code
         *  if any one of the packets failed.
         */
        if (!NT_SUCCESS(Irp->IoStatus.Status)){