//{ **************************************************************************** }
//{ FileName............: DEVQUEUE.CPP                                           }
//{ Project.............: Generic                                                }
//{ Author(s)...........: M.Majoor                                               }
//{ Original contents...: Walter Oney ('Programming the Windows Driver Model')   }
//{ Version.............: 1.01                                                   }
//{ ---------------------------------------------------------------------------- }
//{ Custom IRP queuing support.                                                  }
//{                                                                              }
//{ Version  Date      Comment                                                   }
//{ 1.00     20050410  - Initial release based on SAA7146A driver V2.02          }
//{ 1.01     20050428  - Minor changes to compile with Windows Server 2003 DDK   }
//{ **************************************************************************** }


#include "stddcls.h"
#include "DevQueue.h"

//{ **************************************************************************** }
//{ Descript : Structures.                                                       }
//{ **************************************************************************** }
typedef struct _NOTIFY_CONTEXT
{
  PQNOTIFYFUNC notify;                                     // Real notification function
  PVOID        context;                                    // Context arg for notification function
  LONG         count;                                      // Number of busy queues
} NOTIFY_CONTEXT, *PNOTIFY_CONTEXT;


//{ **************************************************************************** }
//{ Descript : Forwards.                                                         }
//{ **************************************************************************** }
static VOID NotificationCallback(PNOTIFY_CONTEXT ctx);


//{ **************************************************************************** }
//{ Params   : <q>       Device queue                                            }
//{            <nq>      Number of queues                                        }
//{            <status>  Status to pass through                                  }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Abort all requests.                                               }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI AbortAllRequests(PDEVQUEUE* q, ULONG nq, NTSTATUS status)
{
  for (ULONG i = 0; i < nq; ++i)
    AbortRequests(q[i], status);
}


//{ **************************************************************************** }
//{ Params   : <pdq>     Device queue                                            }
//{            <status>  Status to pass through                                  }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Aborts current and future requests.                               }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI AbortRequests(PDEVQUEUE pdq, NTSTATUS status)
{
  pdq->abortstatus = status;
  CleanupRequests(pdq, NULL, status);
}


//{ **************************************************************************** }
//{ Params   : <q>   Device queue                                                }
//{            <nq>  Number of queues                                            }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Allow all requests of all queues.                                 }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI AllowAllRequests(PDEVQUEUE* q, ULONG nq)
{
  for (ULONG i = 0; i < nq; ++i)
    AllowRequests(q[i]);
}


//{ **************************************************************************** }
//{ Params   : <pdq>  Device queue                                               }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Undoes effect of previous <AbortRequests>.                        }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI AllowRequests(PDEVQUEUE pdq)
{
  pdq->abortstatus = STATUS_SUCCESS;
}


//{ **************************************************************************** }
//{ Params   : <pdq>     Device queue                                            }
//{ Return   : <Result>  ABort status                                            }
//{                                                                              }
//{ Descript : Are we currently aborting new requests?                           }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
NTSTATUS AreRequestsBeingAborted(PDEVQUEUE pdq)
{
  return pdq->abortstatus;
}


//{ **************************************************************************** }
//{ Params   : <pdq>  Device queue                                               }
//{            <Irp>  I/O request packet                                         }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Generic cancel routine.                                           }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI CancelRequest(PDEVQUEUE pdq, PIRP Irp)
{
  KIRQL oldirql = Irp->CancelIrql;

  // Release the global cancel spin lock as soon as possible
  IoReleaseCancelSpinLock(DISPATCH_LEVEL);

  // Acquire our queue-specific queue lock. Note that we stayed at DISPATCH_LEVEL
  // when we released the cancel spin lock
  KeAcquireSpinLockAtDpcLevel(&pdq->lock);

  // (After Hanrahan & Peretz) The IRP is guaranteed to be on *some* queue (maybe a degenerate one),
  // so we unconditionally remove it and complete it.
  RemoveEntryList(&Irp->Tail.Overlay.ListEntry);
  KeReleaseSpinLock(&pdq->lock, oldirql);

  Irp->IoStatus.Status = STATUS_CANCELLED;
  IoCompleteRequest(Irp, IO_NO_INCREMENT);
}


//{ **************************************************************************** }
//{ Params   : <q>       Device queue                                            }
//{            <nq>      Number of queues                                        }
//{            <fdo>     Functional device object                                }
//{ Return   : <Result>  TRUE if a queue is busy                                 }
//{                                                                              }
//{ Descript : Check if any queue is busy and stall if so.                       }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
BOOLEAN NTAPI CheckAnyBusyAndStall(PDEVQUEUE* q, ULONG nq, PDEVICE_OBJECT fdo)
{
  ULONG i;

  // Call CheckBusyAndStall for each queue. If one of them is busy,
  // back out by unstalling the queues we stalled.
  for (i = 0; i < nq; ++i)
    if (CheckBusyAndStall(q[i]))
    {                                                      // If a queue is busy
      for (--i; (int) i >= 0; --i)
        RestartRequests(q[i], fdo);
      return TRUE;                                         // Indicate at least one queue is busy
    }

  // Return FALSE because no queue was busy and all are now stalled
  return FALSE;
}


//{ **************************************************************************** }
//{ Params   : <pdq>     Device queue                                            }
//{ Return   : <Result>  TRUE if a queue is busy                                 }
//{                                                                              }
//{ Descript : Checks for idle device and stalls requests in one atomic          }
//{            operation.                                                        }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
BOOLEAN NTAPI CheckBusyAndStall(PDEVQUEUE pdq)
{
  KIRQL oldirql;
  KeAcquireSpinLock(&pdq->lock, &oldirql);
  BOOLEAN busy = pdq->CurrentIrp != NULL;
  if (!busy)
    InterlockedIncrement(&pdq->stallcount);
  KeReleaseSpinLock(&pdq->lock, oldirql);
  return busy;
}


//{ **************************************************************************** }
//{ Params   : <q>       Device queue                                            }
//{            <nq>      Number of queues                                        }
//{            <fop>     File object                                             }
//{            <status>  Status to pass through                                  }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Cleanup all request of all queues.                                }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI CleanupAllRequests(PDEVQUEUE* q, ULONG nq, PFILE_OBJECT fop, NTSTATUS status)
{
  for (ULONG i = 0; i < nq; ++i)
    CleanupRequests(q[i], fop, status);
}


//{ **************************************************************************** }
//{ Params   : <pdq>     Device queue                                            }
//{            <fop>     File object                                             }
//{            <status>  Status to pass through                                  }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Cancels all requests for a given file object in order to service  }
//{            IRP_MJ_CLEANUP.                                                   }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI CleanupRequests(PDEVQUEUE pdq, PFILE_OBJECT fop, NTSTATUS status)
{
  LIST_ENTRY cancellist;
  InitializeListHead(&cancellist);

  // Create a list of IRPs that belong to the same file object
  KIRQL oldirql;
  KeAcquireSpinLock(&pdq->lock, &oldirql);

  PLIST_ENTRY first = &pdq->head;
  PLIST_ENTRY next;
  for (next = first->Flink; next != first; )
  {                                                        // For each queued IRP
    PIRP Irp = CONTAINING_RECORD(next, IRP, Tail.Overlay.ListEntry);
    PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(Irp);

    // Follow the chain to the next IRP now (so that the next iteration of
    // the loop is properly setup whether we dequeue this IRP or not)
    PLIST_ENTRY current = next;
    next = next->Flink;

    // Skip this IRP if it's not for the same file object as the
    // current IRP_MJ_CLEANUP.
    if (fop && stack->FileObject != fop)
      continue;                                            // Not for same file object

    // (After Hanrahan) Set the CancelRoutine pointer to NULL. If it was
    // already NULL, someone is trying to cancel this IRP right now, so just
    // leave it on the queue and let them do it as soon as we release the spin lock.
    if (!IoSetCancelRoutine(Irp, NULL))
      continue;
    RemoveEntryList(current);
    InsertTailList(&cancellist, current);
  }

  // Release the spin lock. We're about to undertake a potentially time-consuming
  // operation that might conceivably result in a deadlock if we keep the lock.
  KeReleaseSpinLock(&pdq->lock, oldirql);

  // Complete the selected requests.
  while (!IsListEmpty(&cancellist))
  {                                                        // Cancel selected requests
    next = RemoveHeadList(&cancellist);
    PIRP Irp = CONTAINING_RECORD(next, IRP, Tail.Overlay.ListEntry);
    Irp->IoStatus.Status = status;
    IoCompleteRequest(Irp, IO_NO_INCREMENT);
  }
}


//{ **************************************************************************** }
//{ Params   : <pdq>     Device queue                                            }
//{ Return   : <Result>  Current I/O request packet                              }
//{                                                                              }
//{ Descript : Determines which IRP is currently being processed by associated   }
//{            StartIo routine.                                                  }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
PIRP NTAPI GetCurrentIrp(PDEVQUEUE pdq)
{
  return pdq->CurrentIrp;
}


//{ **************************************************************************** }
//{ Params   : <pdq>      Device queue                                           }
//{            <StartIo>  Start I/O routine for queue                            }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Initializes DEVQUEUE object.                                      }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI InitializeQueue(PDEVQUEUE pdq, PDRIVER_STARTIO StartIo)
{
  InitializeListHead(&pdq->head);
  KeInitializeSpinLock(&pdq->lock);
  pdq->StartIo = StartIo;
  pdq->stallcount = 1;
  pdq->CurrentIrp = NULL;
  KeInitializeEvent(&pdq->evStop, NotificationEvent, FALSE);
  pdq->abortstatus = (NTSTATUS) 0;
  pdq->notify = NULL;
  pdq->notifycontext = 0;
}


//{ **************************************************************************** }
//{ Params   : <ctx>  Notification context                                       }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Notification callback procedure.                                  }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
static VOID NotificationCallback(PNOTIFY_CONTEXT ctx)
{
  if (InterlockedDecrement(&ctx->count) > 0)
    return;                                                // If still in use

  (*ctx->notify)(ctx->context);                            // If no longer in use
  ExFreePool(ctx);                                         // Free it
}


//{ **************************************************************************** }
//{ Params   : <q>    Device queue                                               }
//{            <nq>   Number of queues                                           }
//{            <fdo>  Function device object                                     }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Restart all request for all queues.                               }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI RestartAllRequests(PDEVQUEUE* q, ULONG nq, PDEVICE_OBJECT fdo)
{
  for (ULONG i = 0; i < nq; ++i)
    RestartRequests(q[i], fdo);
}


//{ **************************************************************************** }
//{ Params   : <pdq>  Device queue                                               }
//{            <fdo>  Function device object                                     }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Restarts a stalled queue.                                         }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI RestartRequests(PDEVQUEUE pdq, PDEVICE_OBJECT fdo)
{
  KIRQL oldirql;
  KeAcquireSpinLock(&pdq->lock, &oldirql);

  if (InterlockedDecrement(&pdq->stallcount) > 0)
  {                                                        // If still stalled
    KeReleaseSpinLock(&pdq->lock, oldirql);
    return;
  }

  ASSERT(pdq->stallcount == 0);                            // Guard against excessive restart calls
  ASSERT(!pdq->CurrentIrp);                                // Should not be busy right now!

  // Dequeue and start the IRP at the head of the list. See the comments in
  // StartNextPacket for an explanation of the cancel logic.
  while (!pdq->stallcount && !pdq->abortstatus && !IsListEmpty(&pdq->head))
  {                                                        // Start first queued IRP
    PLIST_ENTRY next = RemoveHeadList(&pdq->head);
    PIRP        Irp  = CONTAINING_RECORD(next, IRP, Tail.Overlay.ListEntry);

    if (!IoSetCancelRoutine(Irp, NULL))
    {                                                      // IRP being cancelled right now
      ASSERT(Irp->Cancel);                                 // Else CancelRoutine shouldn't be NULL!
      InitializeListHead(&Irp->Tail.Overlay.ListEntry);
      continue;                                            // With "start first queued IRP"
    }

    pdq->CurrentIrp = Irp;
    KeReleaseSpinLockFromDpcLevel(&pdq->lock);
    (*pdq->StartIo)(fdo, Irp);
    KeLowerIrql(oldirql);
    return;
  }

  // No IRPs need to be started (or else all queued IRPs were being cancelled)
  KeReleaseSpinLock(&pdq->lock, oldirql);
}


//{ **************************************************************************** }
//{ Params   : <q>   Device queue                                                }
//{            <nq>  Number of queues                                            }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Stall all requests for all queues.                                }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI StallAllRequests(PDEVQUEUE* q, ULONG nq)
{
  for (ULONG i = 0; i < nq; ++i)
    StallRequests(q[i]);
}


//{ **************************************************************************** }
//{ Params   : <pdq>  Device queue                                               }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Stalls the queue.                                                 }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI StallRequests(PDEVQUEUE pdq)
{
  InterlockedIncrement(&pdq->stallcount);                  // Just increase the stall counter
}


//{ **************************************************************************** }
//{ Params   : <q>        Device queue                                           }
//{            <nq>       Number of queues                                       }
//{            <notify>   Notify function                                        }
//{            <context>  Context of notification                                }
//{ Return   : <Result>   NTSTATUS                                               }
//{                                                                              }
//{ Descript : Stall all requests and notify for all queues.                     }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
NTSTATUS NTAPI StallAllRequestsAndNotify(PDEVQUEUE* q, ULONG nq, PQNOTIFYFUNC notify, PVOID context)
{
  NTSTATUS status;
  KIRQL    oldirql;
  ULONG    i;

  // Acquire all of the queue locks. We're not worried about a deadlock because
  // this is the only function that ever simultaneously locks more than one queue
  KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
  for (i = 0; i < nq; ++i)
    KeAcquireSpinLockAtDpcLevel(&q[i]->lock);

  // Examine each of the queues in a manner similar to the single-queue version
  // of this function
  ULONG nbusy = 0;                                         // Number of busy devices
  ULONG ibusy;                                             // Index of last busy device

  for (i = 0; i < nq; ++i)
  {                                                        // Examine each queue
    PDEVQUEUE pdq = q[i];
    if (pdq->notify)
      break;
    else
    {                                                      // If notify not pending
      InterlockedIncrement(&pdq->stallcount);              // Stall this queue
      if (pdq->CurrentIrp)
        ++nbusy, ibusy = i;                                // Device busy with this queue
    }
  }

  // If we didn't finish the loop, we found a queue for which a notification is
  // already pending, which is an error. Unstall any queues that we just stalled
  // in order to backout from this function.
  if (i < nq)
  {                                                        // If backout from error
    for (--i; (int) i >= 0; --i)
      InterlockedDecrement(&q[i]->stallcount);
    status = STATUS_INVALID_DEVICE_REQUEST;                // Indicate we have an error
  }
  else if (nbusy == 0)                                     // If none of the queues is busy, we can just return STATUS_SUCCESS
         status = STATUS_SUCCESS;                          // If device not busy
                                                           // If just one of the queues is busy, arrange for it call the notification
                                                           // procedure once the current IRP finishes (whereupon somebody will call
                                                           // StartNextPacket on this queue)
       else if (nbusy == 1)
            {                                              // If one queue busy
              q[ibusy]->notify = notify;
              q[ibusy]->notifycontext = context;
              status = STATUS_PENDING;
            }
                                                           // More than one queue is currently busy. We need to arrange for each queue to
                                                           // finish before calling the callback function.
            else
            {                                              // If multiple queues busy
              PNOTIFY_CONTEXT ctx = (PNOTIFY_CONTEXT) ExAllocatePool(NonPagedPool, sizeof(NOTIFY_CONTEXT));
              if (!ctx)
              {                                            // If we can't allocate context block
                for (i = 0; i < nq; ++i)
                  InterlockedDecrement(&q[i]->stallcount); // Unstall the queues we stalled
                status = STATUS_INSUFFICIENT_RESOURCES;
              }
              else
              {                                            // If context block allocated, arrange for notifications
                ctx->context = context;
                ctx->notify = notify;
                ctx->count = nbusy;
                for (i = 0; i < nq; ++i)
                {                                          // For each queue
                  PDEVQUEUE pdq = q[i];
                  if (!pdq->CurrentIrp)
                    continue;                              // This queue not busy
                  pdq->notify = (PQNOTIFYFUNC) NotificationCallback;
                  pdq->notifycontext = (PVOID) ctx;
                }
                status = STATUS_PENDING;
              }
            }

  // Release all the queue locks
  for (i = nq - 1; (int) i >= 0; --i)
    KeReleaseSpinLockFromDpcLevel(&q[i]->lock);
  KeLowerIrql(oldirql);

  return status;
}


//{ **************************************************************************** }
//{ Params   : <pdq>      Device queue                                           }
//{            <notify>   Notify function                                        }
//{            <context>  Context of notification                                }
//{ Return   : <Result>   NTSTATUS                                               }
//{                                                                              }
//{ Descript : Stall all requests and notify for one queue.                      }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
NTSTATUS NTAPI StallRequestsAndNotify(PDEVQUEUE pdq, PQNOTIFYFUNC notify, PVOID context)
{
  NTSTATUS status;
  KIRQL    oldirql;
  KeAcquireSpinLock(&pdq->lock, &oldirql);

  if (pdq->notify)
    status = STATUS_INVALID_DEVICE_REQUEST;
  else
  {                                                        // If valid request
    InterlockedIncrement(&pdq->stallcount);
    if (pdq->CurrentIrp)
    {                                                      // If device is busy
      pdq->notify = notify;
      pdq->notifycontext = context;
      status = STATUS_PENDING;
    }
    else
      status = STATUS_SUCCESS;                             // If device not busy, device is idle
  }

  KeReleaseSpinLock(&pdq->lock, oldirql);
  return status;
}


//{ **************************************************************************** }
//{ Params   : <pdq>     Device queue                                            }
//{            <fdo>     Functional device object                                }
//{ Return   : <Result>  Next I/O request packet                                 }
//{                                                                              }
//{ Descript : Dequeues and starts the next request.                             }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
PIRP NTAPI StartNextPacket(PDEVQUEUE pdq, PDEVICE_OBJECT fdo)
{
  KIRQL oldirql;
  KeAcquireSpinLock(&pdq->lock, &oldirql);

  // Nullify the current IRP pointer after remembering the current one.
  // We'll return the current IRP pointer as our return value so that
  // a DPC routine has a way to know whether an active request got
  // aborted.
//  PIRP CurrentIrp = (PIRP) InterlockedExchangePointer(&pdq->CurrentIrp, NULL); // V1.01
  PIRP CurrentIrp = (PIRP) InterlockedExchangePointer((volatile PVOID *)&pdq->CurrentIrp, NULL);

  // If we just finished processing a request, set the event on which
  // WaitForCurrentIrp may be waiting in some other thread.
  if (CurrentIrp)
    KeSetEvent(&pdq->evStop, 0, FALSE);

  // If someone is waiting for notification that this IRP has finished,
  // we'll provide the notification after we release the spin lock. We shouldn't
  // find the queue unstalled if there is a notification routine in place, by
  // the way.
  PQNOTIFYFUNC notify        = pdq->notify;
  PVOID        notifycontext = pdq->notifycontext;
  pdq->notify = NULL;

  // Start the next IRP
  while (!pdq->stallcount && !pdq->abortstatus && !IsListEmpty(&pdq->head))
  {                                                        // Start next packet
    PLIST_ENTRY next = RemoveHeadList(&pdq->head);
    PIRP        Irp  = CONTAINING_RECORD(next, IRP, Tail.Overlay.ListEntry);

    // (After Hanrahan & Peretz in part) Nullify the cancel pointer in this IRP. If it was
    // already NULL, someone is trying to cancel this IRP right now. Reinitialize
    // the link pointers so the cancel routine's call to RemoveEntryList won't
    // do anything harmful and look for another IRP. The cancel routine will
    // take over as soon as we release the spin lock
    if (!IoSetCancelRoutine(Irp, NULL))
    {                                                      // If IRP being cancelled right now
      ASSERT(Irp->Cancel);                                 // Else CancelRoutine shouldn't be NULL!
      InitializeListHead(&Irp->Tail.Overlay.ListEntry);
      continue;                                            // With "start next packet"
    }

    pdq->CurrentIrp = Irp;
    KeReleaseSpinLockFromDpcLevel(&pdq->lock);
    (*pdq->StartIo)(fdo, Irp);
    KeLowerIrql(oldirql);
    return CurrentIrp;
  }

  KeReleaseSpinLock(&pdq->lock, oldirql);
  if (notify)
    (*notify)(notifycontext);

  return CurrentIrp;
}


//{ **************************************************************************** }
//{ Params   : <pdq>     Device queue                                            }
//{            <fdo>     Functional device object                                }
//{            <Irp>     I/O request packet                                      }
//{            <Cancel>  Cancelation routine                                     }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Starts or queues a new request.                                   }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI StartPacket(PDEVQUEUE pdq, PDEVICE_OBJECT fdo, PIRP Irp, PDRIVER_CANCEL cancel)
{
  KIRQL oldirql;
  KeAcquireSpinLock(&pdq->lock, &oldirql);

  ASSERT(Irp->CancelRoutine == NULL);                      // Mmaybe left over from a higher level?

  // If the device has been removed by surprise, complete IRP immediately. Do not
  // pass GO. Do not collect $200.
  NTSTATUS abortstatus = pdq->abortstatus;
  if (abortstatus)
  {                                                        // Aborting all requests now
    KeReleaseSpinLock(&pdq->lock, oldirql);
    Irp->IoStatus.Status = abortstatus;
    IoCompleteRequest(Irp, IO_NO_INCREMENT);
  }
                                                           // If the device is busy with another request, or if the queue has
                                                           // been stalled due to some PnP or power event, just put the new IRP
                                                           // onto the queue and set a cancel routine pointer.
  else if (pdq->CurrentIrp || pdq->stallcount)
       {                                                   // If queue this irp
         // (After Peretz) See if this IRP was cancelled before it got to us. If so,
         // make sure either we or the cancel routine completes it
         IoSetCancelRoutine(Irp, cancel);
         if (Irp->Cancel && IoSetCancelRoutine(Irp, NULL))
         {                                                 // If IRP has already been cancelled
           KeReleaseSpinLock(&pdq->lock, oldirql);
           Irp->IoStatus.Status = STATUS_CANCELLED;
           IoCompleteRequest(Irp, IO_NO_INCREMENT);
         }
         else
         {                                                 // If queue IRP
           InsertTailList(&pdq->head, &Irp->Tail.Overlay.ListEntry);
           KeReleaseSpinLock(&pdq->lock, oldirql);
         }
       }
                                                           // If the device is idle and not stalled, pass the IRP to the StartIo
                                                           // routine associated with this queue
       else
       {                                                   // If to start this irp
         pdq->CurrentIrp = Irp;
         KeReleaseSpinLock(&pdq->lock, DISPATCH_LEVEL);
         (*pdq->StartIo)(fdo, Irp);
         KeLowerIrql(oldirql);
       }
}


//{ **************************************************************************** }
//{ Params   : <pdq>  Device queue                                               }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Waits for current IRP to finish.                                  }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI WaitForCurrentIrp(PDEVQUEUE pdq)
{
  // First reset the event that StartNextPacket sets each time.
  KeClearEvent(&pdq->evStop);

  // Under protection of our spin lock, check to see if there's a current IRP.
  // Since whoever called us should also have stalled requests, no-one can sneak
  // in after we release the spin lock and start a new request behind our back.
  ASSERT(pdq->stallcount != 0);                            // Should be stalled now!

  KIRQL oldirql;
  KeAcquireSpinLock(&pdq->lock, &oldirql);
  BOOLEAN mustwait = pdq->CurrentIrp != NULL;
  KeReleaseSpinLock(&pdq->lock, oldirql);

  if (mustwait)
    KeWaitForSingleObject(&pdq->evStop, Executive, KernelMode, FALSE, NULL);
}


//{ **************************************************************************** }
//{ Params   : <q>   Device queue                                                }
//{            <nq>  Number of queues                                            }
//{ Return   : -                                                                 }
//{                                                                              }
//{ Descript : Wait for current I/O request packets to finish for all queues.    }
//{ Notes    :                                                                   }
//{ **************************************************************************** }
VOID NTAPI WaitForCurrentIrps(PDEVQUEUE* q, ULONG nq)
{
  for (ULONG i = 0; i < nq; ++i)
    WaitForCurrentIrp(q[i]);
}
