add support for batched queue submissions (#19769)

Adding a new feature: batched queue submissions.

Batched queues enable submission of operations to the driver in batches,
therefore reducing the overhead of submitting every single operation
individually. Similarly to command buffers in L0v2, they use regular
command lists (later referenced as 'batches'). Operations enqueued on
regular command lists are not executed immediately, but only after
enqueueing the regular command list on an immediate command list.
However, in contrast to command buffers, batched queues also handle
submission of batches (regular command lists) instead of only collecting
enqueued operations, by using an internal immediate command list.

Batched queues introduce:
- batch_manager stores the current batch, the command list manager with
an immediate command list for batch submissions, the vector of submitted
batches, the generation number of the current batch.
- The current batch is a command list manager with a regular command
list; operations requested by users are enqueued on the current batch.
The current batch may be submitted for execution on the immediate
command list, replaced by a new regular command list and stored for
execution completion in the vector of submitted batches.
- The number of regular command lists stored for execution is limited.
- The generation number of the current batch is assigned to events
associated with operations enqueued on the given batch. It is
incremented during every replacement of the current batch. When an event
created by a batched queue appears in an eventWaitList, the batch
assigned to the given event might not have been executed yet and the
event might never be signalled. Comparing generation numbers enables
determining whether the current batch should be submitted for execution.
If the generation number of the current batch is higher than the number
assigned to the given event, the batch associated with the event has
already been submitted for execution and additional submission of the
current batch is not needed.
- Regular command lists use the regular pool cache type, whereas
immediate command lists use the immediate pool cache type. Since
user-requested operations are enqueued on regular command lists and
immediate command lists are only used internally by the batched queue
implementation, events are not created for immediate command lists (in
most cases; see below).
- When a user requests the command list manager to enqueue a command
buffer, the regular command list from the command buffer is appended to
the command list of the given command list manager. Since regular
command lists cannot be enqueued on other regular command lists, but
only on immediate command lists, enqueueing command buffers must be
performed on an immediate command list. Therefore, an additional event
pool with the immediate cache type is introduced in order to provide
events for operations requested by users and enqueued directly on an
immediate command list.
- wait_list_view is modified. Previously, it only stored the waitlist
(as a ze_event_handle buffer created from events) and the corresponding
event count in a single container, which could be passed as an argument
to the driver API. Currently, the constructor also ensures that all
associated operations will eventually be executed. Since regular command
lists are not executed immediately, but only after enqueueing on
immediate lists, it is necessary to enqueue the regular command list
associated with the given event. Otherwise, the event would never be
signalled.

Additionally, support for UR_QUEUE_INFO_FLAGS in urQueueGetInfo has been
added for native CPU, which is required by the enqueueTimestampRecording
tests. Currently, enqueueTimestampRecording is not supported by batched
queues.

Batched queues can be enabled by setting
UR_QUEUE_FLAG_SUBMISSION_BATCHED in ur_queue_flags_t or globally,
through the environment variable UR_L0_V2_FORCE_BATCHED=1.

Batched queues are intended to improve performance on platforms, where
eager submission is not efficient due to driver limitations. Such
hardware includes Xe (and older GPUs) on Windows. There are also
workloads which benefit from batched submissions (e.g., dl-cifar). SYCL
graphs should be preferred for new software, since they allow for better
control of grouped commands submissions.

Benchmark results for default in-order queues (sycl branch, commit hash:
b76f12e554760c3fcfc55f1f815a76b0d8b208ad) and batched queues:
api_overhead_benchmark_ur SubmitKernel in order: 20.839 μs
api_overhead_benchmark_ur SubmitKernel batched:  12.183 μs

Author: EuphoricThinking

scripts/templates/queue_api.hpp.mako

@@ -25,8 +25,9 @@ from templates import helper as th
 #pragma once
 
 #include <ur_api.h>
+#include "queue_extensions.hpp"
 
-struct ur_queue_t_ {
+struct ur_queue_t_ : ur_queue_extensions {
     virtual ~ur_queue_t_();
 
     %for obj in th.get_queue_related_functions(specs, n, tags):

source/adapters/level_zero/CMakeLists.txt

@@ -171,6 +171,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/lockable.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.hpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.hpp
@@ -187,6 +188,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_create.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.cpp

source/adapters/level_zero/v2/command_buffer.cpp

@@ -12,6 +12,7 @@
 #include "../command_buffer_command.hpp"
 #include "../helpers/kernel_helpers.hpp"
 #include "../ur_interface_loader.hpp"
+#include "command_list_manager.hpp"
 #include "logger/ur_logger.hpp"
 #include "queue_handle.hpp"
 
@@ -328,9 +329,12 @@ ur_result_t urCommandBufferAppendKernelLaunchExp(
   auto eventsWaitList = commandBuffer->getWaitListFromSyncPoints(
       syncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendKernelLaunch(
       hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize,
-      nullptr, numSyncPointsInWaitList, eventsWaitList,
+      nullptr, waitListView,
       commandBuffer->createEventIfRequested(retSyncPoint)));
 
   return UR_RESULT_SUCCESS;
@@ -353,8 +357,11 @@ ur_result_t urCommandBufferAppendUSMMemcpyExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendUSMMemcpy(
-      false, pDst, pSrc, size, numSyncPointsInWaitList, eventsWaitList,
+      false, pDst, pSrc, size, waitListView,
       hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
@@ -380,9 +387,12 @@ ur_result_t urCommandBufferAppendMemBufferCopyExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendMemBufferCopy(
-      hSrcMem, hDstMem, srcOffset, dstOffset, size, numSyncPointsInWaitList,
-      eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
+      hSrcMem, hDstMem, srcOffset, dstOffset, size, waitListView,
+      hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
 } catch (...) {
@@ -407,9 +417,12 @@ ur_result_t urCommandBufferAppendMemBufferWriteExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendMemBufferWrite(
-      hBuffer, false, offset, size, pSrc, numSyncPointsInWaitList,
-      eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
+      hBuffer, false, offset, size, pSrc, waitListView,
+      hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
 } catch (...) {
@@ -432,9 +445,12 @@ ur_result_t urCommandBufferAppendMemBufferReadExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendMemBufferRead(
-      hBuffer, false, offset, size, pDst, numSyncPointsInWaitList,
-      eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
+      hBuffer, false, offset, size, pDst, waitListView,
+      hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
 } catch (...) {
@@ -461,10 +477,13 @@ ur_result_t urCommandBufferAppendMemBufferCopyRectExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendMemBufferCopyRect(
       hSrcMem, hDstMem, srcOrigin, dstOrigin, region, srcRowPitch,
-      srcSlicePitch, dstRowPitch, dstSlicePitch, numSyncPointsInWaitList,
-      eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
+      srcSlicePitch, dstRowPitch, dstSlicePitch, waitListView,
+      hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
 } catch (...) {
@@ -491,10 +510,12 @@ ur_result_t urCommandBufferAppendMemBufferWriteRectExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendMemBufferWriteRect(
       hBuffer, false, bufferOffset, hostOffset, region, bufferRowPitch,
-      bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc,
-      numSyncPointsInWaitList, eventsWaitList,
+      bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, waitListView,
       hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
@@ -522,10 +543,12 @@ ur_result_t urCommandBufferAppendMemBufferReadRectExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendMemBufferReadRect(
       hBuffer, false, bufferOffset, hostOffset, region, bufferRowPitch,
-      bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst,
-      numSyncPointsInWaitList, eventsWaitList,
+      bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, waitListView,
       hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
@@ -548,9 +571,12 @@ ur_result_t urCommandBufferAppendUSMFillExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendUSMFill(
-      pMemory, patternSize, pPattern, size, numSyncPointsInWaitList,
-      eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
+      pMemory, patternSize, pPattern, size, waitListView,
+      hCommandBuffer->createEventIfRequested(pSyncPoint)));
   return UR_RESULT_SUCCESS;
 } catch (...) {
   return exceptionToResult(std::current_exception());
@@ -572,9 +598,12 @@ ur_result_t urCommandBufferAppendMemBufferFillExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendMemBufferFill(
-      hBuffer, pPattern, patternSize, offset, size, numSyncPointsInWaitList,
-      eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
+      hBuffer, pPattern, patternSize, offset, size, waitListView,
+      hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
 } catch (...) {
@@ -598,8 +627,11 @@ ur_result_t urCommandBufferAppendUSMPrefetchExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendUSMPrefetch(
-      pMemory, size, flags, numSyncPointsInWaitList, eventsWaitList,
+      pMemory, size, flags, waitListView,
       hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
@@ -622,8 +654,11 @@ ur_result_t urCommandBufferAppendUSMAdviseExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
   UR_CALL(commandListLocked->appendUSMAdvise(
-      pMemory, size, advice, numSyncPointsInWaitList, eventsWaitList,
+      pMemory, size, advice, waitListView,
       hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
@@ -672,15 +707,19 @@ ur_result_t urCommandBufferAppendNativeCommandExp(
   auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
       pSyncPointWaitList, numSyncPointsInWaitList);
 
-  UR_CALL(commandListLocked->appendEventsWaitWithBarrier(
-      numSyncPointsInWaitList, eventsWaitList, nullptr));
+  wait_list_view waitListView =
+      wait_list_view(eventsWaitList, numSyncPointsInWaitList);
+
+  UR_CALL(
+      commandListLocked->appendEventsWaitWithBarrier(waitListView, nullptr));
 
   // Call user-defined function immediately
   pfnNativeCommand(pData);
 
+  wait_list_view emptyWaitList = wait_list_view(nullptr, 0);
   // Barrier on all commands after user defined commands.
   UR_CALL(commandListLocked->appendEventsWaitWithBarrier(
-      0, nullptr, hCommandBuffer->createEventIfRequested(pSyncPoint)));
+      emptyWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
 
   return UR_RESULT_SUCCESS;
 }