Skip to content

Commit

Permalink
[EXP][Command-buffer] OpenCL kernel command update
Browse files Browse the repository at this point in the history 
Implement the API for updating the kernel commands in a command-buffer
defined by #1089 for
the OpenCL adapter.

However, the following changes to the UR kernel update API have been
made based on implementation experience:

1. Forbid updating the work-dim of the kernel, see KhronosGroup/OpenCL-Docs#1057
2. Remove struct fields to update exec info, after [DPC++ implementation
   prototype](intel/llvm#12840) shows this isn't
   needed.
3. Forbid changing the local work size from user to impl defined and
   vice-versa. See discussion in [L0 implementation
PR](#1353 (comment)).

This adapter implementation depends on support for the
[cl_khr_command_buffer_mutable_dispatch](https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_Ext.html#cl_khr_command_buffer_mutable_dispatch)
extension.

Tested on Intel GPU/CPUs OpenCL implementations with the
[command-buffer emulation
layer](https://github.com/bashbaug/SimpleOpenCLSamples/tree/main/layers/10_cmdbufemu).

```bash
$ OPENCL_LAYERS=<path/to/SimpleOpenCLSamples/build/layers/10_cmdbufemu/libCmdBufEmu.so> ./bin/test-exp_command_buffer --platform="Intel(R) OpenCL Graphics"
```

DPC++ PR intel/llvm#12724
  • Loading branch information
@EwanC
EwanC committed Mar 25, 2024
1 parent a504ead commit df68355
Show file tree
Hide file tree
Showing 30 changed files with 712 additions and 417 deletions.
32 changes: 11 additions & 21 deletions include/ur_api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,6 @@ typedef enum ur_structure_type_t {
UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC = 0x1002, ///< ::ur_exp_command_buffer_update_memobj_arg_desc_t
UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC = 0x1003, ///< ::ur_exp_command_buffer_update_pointer_arg_desc_t
UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC = 0x1004, ///< ::ur_exp_command_buffer_update_value_arg_desc_t
UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC = 0x1005, ///< ::ur_exp_command_buffer_update_exec_info_desc_t
UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES = 0x2000, ///< ::ur_exp_sampler_mip_properties_t
UR_STRUCTURE_TYPE_EXP_INTEROP_MEM_DESC = 0x2001, ///< ::ur_exp_interop_mem_desc_t
UR_STRUCTURE_TYPE_EXP_INTEROP_SEMAPHORE_DESC = 0x2002, ///< ::ur_exp_interop_semaphore_desc_t
Expand Down Expand Up @@ -7934,19 +7933,6 @@ typedef struct ur_exp_command_buffer_update_value_arg_desc_t {

} ur_exp_command_buffer_update_value_arg_desc_t;

///////////////////////////////////////////////////////////////////////////////
/// @brief Descriptor type for updating kernel command execution info.
typedef struct ur_exp_command_buffer_update_exec_info_desc_t {
ur_structure_type_t stype; ///< [in] type of this structure, must be
///< ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC
const void *pNext; ///< [in][optional] pointer to extension-specific structure
ur_kernel_exec_info_t propName; ///< [in] Name of execution attribute.
size_t propSize; ///< [in] Size of execution attribute.
const ur_kernel_exec_info_properties_t *pProperties; ///< [in][optional] Pointer to execution info properties.
const void *pNewExecInfo; ///< [in] Pointer to memory location holding the execution info value.

} ur_exp_command_buffer_update_exec_info_desc_t;

///////////////////////////////////////////////////////////////////////////////
/// @brief Descriptor type for updating a kernel launch command.
typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t {
Expand All @@ -7956,24 +7942,23 @@ typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t {
uint32_t numNewMemObjArgs; ///< [in] Length of pNewMemObjArgList.
uint32_t numNewPointerArgs; ///< [in] Length of pNewPointerArgList.
uint32_t numNewValueArgs; ///< [in] Length of pNewValueArgList.
uint32_t numNewExecInfos; ///< [in] Length of pNewExecInfoList.
uint32_t newWorkDim; ///< [in] Number of work dimensions in the kernel ND-range, from 1-3.
const ur_exp_command_buffer_update_memobj_arg_desc_t *pNewMemObjArgList; ///< [in][optional][range(0, numNewMemObjArgs)] An array describing the new
///< kernel mem obj arguments for the command.
const ur_exp_command_buffer_update_pointer_arg_desc_t *pNewPointerArgList; ///< [in][optional][range(0, numNewPointerArgs)] An array describing the
///< new kernel pointer arguments for the command.
const ur_exp_command_buffer_update_value_arg_desc_t *pNewValueArgList; ///< [in][optional][range(0, numNewValueArgs)] An array describing the new
///< kernel value arguments for the command.
const ur_exp_command_buffer_update_exec_info_desc_t *pNewExecInfoList; ///< [in][optional][range(0, numNewExecInfos)] An array describing the
///< execution info objects for the command.
size_t *pNewGlobalWorkOffset; ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
///< values that describe the offset used to calculate the global ID.
size_t *pNewGlobalWorkSize; ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
///< values that describe the number of global work-items.
size_t *pNewLocalWorkSize; ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
///< values that describe the number of work-items that make up a
///< work-group. If nullptr, the runtime implementation will choose the
///< work-group size.
///< work-group. If newWorkDim is non-zero and pNewLocalWorkSize is
///< nullptr, then runtime implementation will choose the work-group size.
///< If newWorkDim is zero and pNewLocalWorkSize is nullptr, then the local
///< work size is unchanged.

} ur_exp_command_buffer_update_kernel_launch_desc_t;

Expand Down Expand Up @@ -8008,6 +7993,8 @@ typedef struct ur_exp_command_buffer_command_handle_t_ *ur_exp_command_buffer_co
/// + `NULL == phCommandBuffer`
/// - ::UR_RESULT_ERROR_INVALID_CONTEXT
/// - ::UR_RESULT_ERROR_INVALID_DEVICE
/// - ::UR_RESULT_ERROR_INVALID_OPERATION
/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` does not support UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP.
/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
UR_APIEXPORT ur_result_t UR_APICALL
Expand Down Expand Up @@ -8088,7 +8075,6 @@ urCommandBufferFinalizeExp(
/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
/// + `NULL == pGlobalWorkOffset`
/// + `NULL == pGlobalWorkSize`
/// + `NULL == pLocalWorkSize`
/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP
/// - ::UR_RESULT_ERROR_INVALID_KERNEL
/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION
Expand All @@ -8107,7 +8093,7 @@ urCommandBufferAppendKernelLaunchExp(
uint32_t workDim, ///< [in] Dimension of the kernel execution.
const size_t *pGlobalWorkOffset, ///< [in] Offset to use when executing kernel.
const size_t *pGlobalWorkSize, ///< [in] Global work size to use when executing kernel.
const size_t *pLocalWorkSize, ///< [in] Local work size to use when executing kernel.
const size_t *pLocalWorkSize, ///< [in][optional] Local work size to use when executing kernel.
uint32_t numSyncPointsInWaitList, ///< [in] The number of sync points in the provided dependency list.
const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, ///< [in][optional] A list of sync points that this command depends on.
ur_exp_command_buffer_sync_point_t *pSyncPoint, ///< [out][optional] Sync point associated with this command.
Expand Down Expand Up @@ -8609,6 +8595,10 @@ urCommandBufferReleaseCommandExp(
/// - ::UR_RESULT_ERROR_INVALID_OPERATION
/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to.
/// + If the command-buffer `hCommand` belongs to has not been finalized.
/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and different from the work-dim used on creation of `hCommand`.
/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and `pUpdateKernelLaunch->pNewLocalWorkSize` is set to a non-NULL value and `pUpdateKernelLaunch->pNewGlobalWorkSize` is NULL.
/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and `pUpdateKernelLaunch->pNewLocalWorkSize` is set to a non-NULL value when `hCommand` was created with a NULL local work size.
/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and `pUpdateKernelLaunch->pNewLocalWorkSize` is set to a NULL value when `hCommand` was created with a non-NULL local work size.
/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT
/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX
Expand Down
8 changes: 0 additions & 8 deletions include/ur_print.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -978,14 +978,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdatePointerArgDesc(
/// - `buff_size < out_size`
UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdateValueArgDesc(const struct ur_exp_command_buffer_update_value_arg_desc_t params, char *buffer, const size_t buff_size, size_t *out_size);

///////////////////////////////////////////////////////////////////////////////
/// @brief Print ur_exp_command_buffer_update_exec_info_desc_t struct
/// @returns
/// - ::UR_RESULT_SUCCESS
/// - ::UR_RESULT_ERROR_INVALID_SIZE
/// - `buff_size < out_size`
UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdateExecInfoDesc(const struct ur_exp_command_buffer_update_exec_info_desc_t params, char *buffer, const size_t buff_size, size_t *out_size);

///////////////////////////////////////////////////////////////////////////////
/// @brief Print ur_exp_command_buffer_update_kernel_launch_desc_t struct
/// @returns
Expand Down
65 changes: 0 additions & 65 deletions include/ur_print.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,6 @@ inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct
inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_memobj_arg_desc_t params);
inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_pointer_arg_desc_t params);
inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_value_arg_desc_t params);
inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_exec_info_desc_t params);
inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_kernel_launch_desc_t params);
inline std::ostream &operator<<(std::ostream &os, enum ur_exp_peer_info_t value);

Expand Down Expand Up @@ -1047,9 +1046,6 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_structure_type_t value
case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC:
os << "UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC";
break;
case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC:
os << "UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC";
break;
case UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES:
os << "UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES";
break;
Expand Down Expand Up @@ -1285,11 +1281,6 @@ inline ur_result_t printStruct(std::ostream &os, const void *ptr) {
printPtr(os, pstruct);
} break;

case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC: {
const ur_exp_command_buffer_update_exec_info_desc_t *pstruct = (const ur_exp_command_buffer_update_exec_info_desc_t *)ptr;
printPtr(os, pstruct);
} break;

case UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES: {
const ur_exp_sampler_mip_properties_t *pstruct = (const ur_exp_sampler_mip_properties_t *)ptr;
printPtr(os, pstruct);
Expand Down Expand Up @@ -9516,46 +9507,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
return os;
}
///////////////////////////////////////////////////////////////////////////////
/// @brief Print operator for the ur_exp_command_buffer_update_exec_info_desc_t type
/// @returns
/// std::ostream &
inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_buffer_update_exec_info_desc_t params) {
os << "(struct ur_exp_command_buffer_update_exec_info_desc_t){";

os << ".stype = ";

os << (params.stype);

os << ", ";
os << ".pNext = ";

ur::details::printStruct(os,
(params.pNext));

os << ", ";
os << ".propName = ";

os << (params.propName);

os << ", ";
os << ".propSize = ";

os << (params.propSize);

os << ", ";
os << ".pProperties = ";

os << (params.pProperties);

os << ", ";
os << ".pNewExecInfo = ";

os << (params.pNewExecInfo);

os << "}";
return os;
}
///////////////////////////////////////////////////////////////////////////////
/// @brief Print operator for the ur_exp_command_buffer_update_kernel_launch_desc_t type
/// @returns
/// std::ostream &
Expand Down Expand Up @@ -9587,11 +9538,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu

os << (params.numNewValueArgs);

os << ", ";
os << ".numNewExecInfos = ";

os << (params.numNewExecInfos);

os << ", ";
os << ".newWorkDim = ";

Expand Down Expand Up @@ -9630,17 +9576,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
}
os << "}";

os << ", ";
os << ".pNewExecInfoList = {";
for (size_t i = 0; (params.pNewExecInfoList) != NULL && i < params.numNewExecInfos; ++i) {
if (i != 0) {
os << ", ";
}

os << ((params.pNewExecInfoList))[i];
}
os << "}";

os << ", ";
os << ".pNewGlobalWorkOffset = {";
for (size_t i = 0; (params.pNewGlobalWorkOffset) != NULL && i < params.newWorkDim; ++i) {
Expand Down
2 changes: 0 additions & 2 deletions scripts/core/EXP-COMMAND-BUFFER.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,6 @@ Enums
* ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC
* ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC
* ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC
* ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC
* ${x}_command_t
* ${X}_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP
* ${x}_function_t
Expand Down Expand Up @@ -290,7 +289,6 @@ Types
* ${x}_exp_command_buffer_update_memobj_arg_desc_t
* ${x}_exp_command_buffer_update_pointer_arg_desc_t
* ${x}_exp_command_buffer_update_value_arg_desc_t
* ${x}_exp_command_buffer_update_exec_info_desc_t
* ${x}_exp_command_buffer_sync_point_t
* ${x}_exp_command_buffer_handle_t
* ${x}_exp_command_buffer_command_handle_t
Expand Down
Loading

0 comments on commit df68355

Please sign in to comment.