[v2] add initial Command Buffers support (#2629)

* [v2] add initial Command Buffers support

---------

Co-authored-by: Mikołaj Komar <[email protected]>
Co-authored-by: Łukasz Ślusarczyk <[email protected]>

include/ur_api.h

@@ -11101,7 +11101,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferReleaseCommandExp(
 ///         is not nullptr.
 ///     - ::UR_RESULT_ERROR_INVALID_OPERATION
 ///         + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true
-///         on creation of the command buffer `hCommand` belongs to.
+///         on creation of the command-buffer `hCommand` belongs to.
 ///         + If the command-buffer `hCommand` belongs to has not been
 ///         finalized.
 ///     - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
@@ -11154,7 +11154,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp(
 ///         supported by the device associated with `hCommand`.
 ///     - ::UR_RESULT_ERROR_INVALID_OPERATION
 ///         + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true
-///         on creation of the command buffer `hCommand` belongs to.
+///         on creation of the command-buffer `hCommand` belongs to.
 ///         + If the command-buffer `hCommand` belongs to has not been
 ///         finalized.
 ///         + If no `phEvent` parameter was set on creation of the command
@@ -11185,7 +11185,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp(
 ///         supported by the device associated with `hCommand`.
 ///     - ::UR_RESULT_ERROR_INVALID_OPERATION
 ///         + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true
-///         on creation of the command buffer `hCommand` belongs to.
+///         on creation of the command-buffer `hCommand` belongs to.
 ///         + If the command-buffer `hCommand` belongs to has not been
 ///         finalized.
 ///     - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP

scripts/core/exp-command-buffer.yml

@@ -1211,7 +1211,7 @@ returns:
         - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET is not supported by the device but `pUpdateKernelLaunch->pNewGlobalWorkOffset` is not nullptr."
         - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE is not supported by the device but `pUpdateKernelLaunch->hNewKernel` is not nullptr."
     - $X_RESULT_ERROR_INVALID_OPERATION:
-        - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to."
+        - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command-buffer `hCommand` belongs to."
         - "If the command-buffer `hCommand` belongs to has not been finalized."
     - $X_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP:
         - "If `hCommand` is not a kernel execution command."
@@ -1244,7 +1244,7 @@ returns:
     - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
         - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`."
     - $X_RESULT_ERROR_INVALID_OPERATION:
-        - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to."
+        - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command-buffer `hCommand` belongs to."
         - "If the command-buffer `hCommand` belongs to has not been finalized."
         - "If no `phEvent` parameter was set on creation of the command associated with `hCommand`."
     - $X_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
@@ -1270,7 +1270,7 @@ returns:
     - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
         - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`."
     - $X_RESULT_ERROR_INVALID_OPERATION:
-        - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to."
+        - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command-buffer `hCommand` belongs to."
         - "If the command-buffer `hCommand` belongs to has not been finalized."
     - $X_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
     - $X_RESULT_ERROR_INVALID_EVENT_WAIT_LIST:

scripts/templates/queue_api.cpp.mako

@@ -20,6 +20,8 @@ from templates import helper as th
  *
  */
 
+// Do not edit. This file is auto generated from a template: scripts/templates/queue_api.cpp.mako
+
 #include "queue_api.hpp"
 #include "ur_util.hpp"
 

scripts/templates/queue_api.hpp.mako

@@ -20,9 +20,12 @@ from templates import helper as th
  *
  */
 
+// Do not edit. This file is auto generated from a template: scripts/templates/queue_api.hpp.mako
+
 #pragma once
 
 #include <ur_api.h>
+#include <ze_api.h>
 
 struct ur_queue_handle_t_ {
     virtual ~ur_queue_handle_t_();
@@ -32,4 +35,8 @@ struct ur_queue_handle_t_ {
     %for obj in th.get_queue_related_functions(specs, n, tags):
     virtual ${x}_result_t ${th.transform_queue_related_function_name(n, tags, obj, format=["type"])} = 0;
     %endfor
+
+    virtual ur_result_t
+    enqueueCommandBuffer(ze_command_list_handle_t, ur_event_handle_t *,
+    uint32_t, const ur_event_handle_t *) = 0;
 };

source/adapters/cuda/command_buffer.cpp

@@ -1313,7 +1313,7 @@ updateKernelArguments(kernel_command_handle *Command,
 }
 
 /**
- * Updates the command buffer command with new values from the update
+ * Updates the command-buffer command with new values from the update
  * description.
  * @param[in] Command The command to be updated.
  * @param[in] UpdateCommandDesc The update command description.

source/adapters/cuda/command_buffer.hpp

@@ -362,7 +362,7 @@ struct ur_exp_command_buffer_handle_t_ {
 
   // UR context associated with this command-buffer
   ur_context_handle_t Context;
-  // Device associated with this command buffer
+  // Device associated with this command-buffer
   ur_device_handle_t Device;
   // Whether commands in the command-buffer can be updated
   bool IsUpdatable;

source/adapters/hip/command_buffer.cpp

@@ -1024,7 +1024,7 @@ updateKernelArguments(ur_exp_command_buffer_command_handle_t Command,
 }
 
 /**
- * Updates the command buffer command with new values from the update
+ * Updates the command-buffer command with new values from the update
  * description.
  * @param[in] Command The command to be updated.
  * @param[in] UpdateCommandDesc The update command description.

source/adapters/hip/command_buffer.hpp

@@ -168,7 +168,7 @@ struct ur_exp_command_buffer_handle_t_ {
 
   // UR context associated with this command-buffer
   ur_context_handle_t Context;
-  // Device associated with this command buffer
+  // Device associated with this command-buffer
   ur_device_handle_t Device;
   // Whether commands in the command-buffer can be updated
   bool IsUpdatable;

source/adapters/level_zero/CMakeLists.txt

@@ -143,7 +143,9 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
         # v2-only sources
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_buffer.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_list_cache.hpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_list_manager.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/context.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_pool_cache.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_pool.hpp
@@ -157,7 +159,9 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/api.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_buffer.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_list_cache.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_list_manager.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/context.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_pool_cache.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_pool.cpp

source/adapters/level_zero/command_buffer.cpp

@@ -173,7 +173,7 @@ ur_result_t getEventsFromSyncPoints(
 /**
  * If needed, creates a sync point for a given command and returns the L0
  * events associated with the sync point.
- * This operations is skipped if the command buffer is in order.
+ * This operations is skipped if the command-buffer is in order.
  * @param[in] CommandType The type of the command.
  * @param[in] CommandBuffer The CommandBuffer where the command is appended.
  * @param[in] NumSyncPointsInWaitList Number of sync points that are
@@ -252,7 +252,7 @@ ur_result_t enqueueCommandBufferMemCopyHelper(
 }
 
 // Helper function for common code when enqueuing rectangular memory operations
-// to a command buffer.
+// to a command-buffer.
 ur_result_t enqueueCommandBufferMemCopyRectHelper(
     ur_command_t CommandType, ur_exp_command_buffer_handle_t CommandBuffer,
     void *Dst, const void *Src, ur_rect_offset_t SrcOrigin,
@@ -584,10 +584,10 @@ ur_result_t createMainCommandList(ur_context_handle_t Context,
 }
 
 /**
- * Checks whether the command buffer can be constructed using in order
+ * Checks whether the command-buffer can be constructed using in order
  * command-lists.
- * @param[in] Context The Context associated with the command buffer.
- * @param[in] CommandBufferDesc The description of the command buffer.
+ * @param[in] Context The Context associated with the command-buffer.
+ * @param[in] CommandBufferDesc The description of the command-buffer.
  * @return Returns true if in order command-lists can be enabled.
  */
 bool canBeInOrder(ur_context_handle_t Context,
@@ -810,7 +810,7 @@ finalizeImmediateAppendPath(ur_exp_command_buffer_handle_t CommandBuffer) {
                 CommandBuffer->AllResetEvent->ZeEvent));
 
     // All the events are reset by default. So signal the all reset event for
-    // the first run of the command buffer
+    // the first run of the command-buffer
     ZE2UR_CALL(zeEventHostSignal, (CommandBuffer->AllResetEvent->ZeEvent));
   }
 
@@ -887,7 +887,7 @@ urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t CommandBuffer) {
 
 /**
  * Sets the kernel arguments for a kernel command that will be appended to the
- * command buffer.
+ * command-buffer.
  * @param[in] Device The Device associated with the command-buffer where the
  * kernel command will be appended.
  * @param[in,out] Arguments stored in the ur_kernel_handle_t object to be set
@@ -918,7 +918,7 @@ ur_result_t setKernelPendingArguments(
 }
 
 /**
- * Creates a new command handle to use in future updates to the command buffer.
+ * Creates a new command handle to use in future updates to the command-buffer.
  * @param[in] CommandBuffer The CommandBuffer associated with the new command.
  * @param[in] Kernel  The Kernel associated with the new command.
  * @param[in] WorkDim Dimensions of the kernel associated with the new command.
@@ -1315,7 +1315,7 @@ ur_result_t urCommandBufferAppendUSMPrefetchExp(
   std::ignore = Flags;
 
   if (CommandBuffer->IsInOrderCmdList) {
-    // Add the prefetch command to the command buffer.
+    // Add the prefetch command to the command-buffer.
     // Note that L0 does not handle migration flags.
     ZE2UR_CALL(zeCommandListAppendMemoryPrefetch,
                (CommandBuffer->ZeComputeCommandList, Mem, Size));
@@ -1332,7 +1332,7 @@ ur_result_t urCommandBufferAppendUSMPrefetchExp(
                   ZeEventList.data()));
     }
 
-    // Add the prefetch command to the command buffer.
+    // Add the prefetch command to the command-buffer.
     // Note that L0 does not handle migration flags.
     ZE2UR_CALL(zeCommandListAppendMemoryPrefetch,
                (CommandBuffer->ZeComputeCommandList, Mem, Size));
@@ -1463,7 +1463,7 @@ ur_result_t urCommandBufferAppendUSMFillExp(
 
 /**
  * Gets an L0 command queue that supports the chosen engine.
- * @param[in] Queue The UR queue used to submit the command buffer.
+ * @param[in] Queue The UR queue used to submit the command-buffer.
  * @param[in] UseCopyEngine Which engine to use. true for the copy engine and
  * false for the compute engine.
  * @param[out] ZeCommandQueue The L0 command queue.
@@ -1478,9 +1478,9 @@ ur_result_t getZeCommandQueue(ur_queue_handle_t Queue, bool UseCopyEngine,
 }
 
 /**
- * Waits for the all the dependencies of the command buffer
- * @param[in] CommandBuffer The command buffer.
- * @param[in] Queue The UR queue used to submit the command buffer.
+ * Waits for the all the dependencies of the command-buffer
+ * @param[in] CommandBuffer The command-buffer.
+ * @param[in] Queue The UR queue used to submit the command-buffer.
  * @param[in] NumEventsInWaitList The number of events to wait for.
  * @param[in] EventWaitList List of events to wait for.
  * @return UR_RESULT_SUCCESS or an error code on failure
@@ -1546,10 +1546,10 @@ ur_result_t appendProfilingQueries(ur_exp_command_buffer_handle_t CommandBuffer,
                                    ur_event_handle_t SignalEvent,
                                    ur_event_handle_t WaitEvent,
                                    ur_event_handle_t ProfilingEvent) {
-  // Multiple submissions of a command buffer implies that we need to save
-  // the event timestamps before resubmiting the command buffer. We
+  // Multiple submissions of a command-buffer implies that we need to save
+  // the event timestamps before resubmiting the command-buffer. We
   // therefore copy these timestamps in a dedicated USM memory section
-  // before completing the command buffer execution, and then attach this
+  // before completing the command-buffer execution, and then attach this
   // memory to the event returned to users to allow the profiling
   // engine to recover these timestamps.
   command_buffer_profiling_t *Profiling = new command_buffer_profiling_t();
@@ -2129,9 +2129,9 @@ ur_result_t updateKernelCommand(
  */
 ur_result_t
 waitForOngoingExecution(ur_exp_command_buffer_handle_t CommandBuffer) {
-  // Calling function has taken a lock for the command buffer so we can safely
+  // Calling function has taken a lock for the command-buffer so we can safely
   // check and modify this value here.
-  // If command buffer was recently synchronized we can return early.
+  // If command-buffer was recently synchronized we can return early.
   if (!CommandBuffer->NeedsUpdateSynchronization) {
     return UR_RESULT_SUCCESS;
   }
@@ -2147,7 +2147,7 @@ waitForOngoingExecution(ur_exp_command_buffer_handle_t CommandBuffer) {
   } else if (ze_fence_handle_t &ZeFence = CommandBuffer->ZeActiveFence) {
     ZE2UR_CALL(zeFenceHostSynchronize, (ZeFence, UINT64_MAX));
   }
-  // Mark that command buffer was recently synchronized
+  // Mark that command-buffer was recently synchronized
   CommandBuffer->NeedsUpdateSynchronization = false;
   return UR_RESULT_SUCCESS;
 }
@@ -2162,7 +2162,7 @@ ur_result_t urCommandBufferUpdateKernelLaunchExp(
 
   UR_ASSERT(KernelCommandHandle->Kernel, UR_RESULT_ERROR_INVALID_NULL_HANDLE);
 
-  // Lock command, kernel and command buffer for update.
+  // Lock command, kernel and command-buffer for update.
   std::scoped_lock<ur_shared_mutex, ur_shared_mutex, ur_shared_mutex> Guard(
       Command->Mutex, Command->CommandBuffer->Mutex,
       KernelCommandHandle->Kernel->Mutex);

source/adapters/level_zero/command_buffer.hpp

@@ -73,7 +73,7 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
 
   // UR context associated with this command-buffer
   ur_context_handle_t Context;
-  // Device associated with this command buffer
+  // Device associated with this command-buffer
   ur_device_handle_t Device;
   // Level Zero command list handle that has the compute engine commands for
   // this command-buffer.
@@ -129,7 +129,7 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
 
   // Indicates if command-buffer commands can be updated after it is closed.
   bool IsUpdatable = false;
-  // Indicates if command buffer was finalized.
+  // Indicates if command-buffer was finalized.
   bool IsFinalized = false;
   // Command-buffer profiling is enabled.
   bool IsProfilingEnabled = false;
@@ -141,8 +141,8 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
   // This list is needed to release all kernels retained by the
   // command_buffer.
   std::vector<ur_kernel_handle_t> KernelsList;
-  // Track whether synchronization is required when updating the command buffer
-  // Set this value to true when a command buffer is enqueued, and false after
+  // Track whether synchronization is required when updating the command-buffer
+  // Set this value to true when a command-buffer is enqueued, and false after
   // any fence or event synchronization to avoid repeated calls to synchronize.
   bool NeedsUpdateSynchronization = false;
 };

source/adapters/level_zero/helpers/kernel_helpers.hpp

@@ -33,7 +33,7 @@ ur_result_t calculateKernelWorkDimensions(
 
 /**
  * Sets the global offset for a kernel command that will be appended to the
- * command buffer.
+ * command-buffer.
  * @param[in] Context Context associated with the queue.
  * @param[in] Kernel The handle to the kernel that will be appended.
  * @param[in] WorkDim The number of work dimensions.