From e59a8a2744d3b1ad420a69f5bc652b876ff48cc2 Mon Sep 17 00:00:00 2001 From: Yuan Hu Date: Mon, 1 Jul 2024 18:46:47 +0800 Subject: [PATCH 01/15] [CPU] Add test case for precision fp16 (#19721) ### Details: - *https://github.com/openvinotoolkit/openvino/pull/16500#issuecomment-1602581987* - *add test case for conv dconv fullconnect matmul mvn pad pooling subgraph softmax* ### Tickets: - *CVS-110112* --------- Signed-off-by: HU Yuan2 --- .../dev_api/openvino/runtime/system_conf.hpp | 7 + src/inference/src/system_conf.cpp | 7 + src/plugins/intel_cpu/src/graph.cpp | 4 +- .../classes/convolution.cpp | 7 +- .../classes/convolution_backprop_data.cpp | 8 +- .../single_layer_tests/classes/matmul.cpp | 9 +- .../single_layer_tests/classes/pooling.cpp | 30 +- .../single_layer_tests/classes/pooling.hpp | 7 +- .../single_layer_tests/classes/softmax.cpp | 17 +- .../single_layer_tests/classes/softmax.hpp | 3 +- .../instances/common/pooling.cpp | 39 ++- .../instances/common/softmax.cpp | 12 +- .../instances/x64/convolution.cpp | 108 ++++++- .../x64/convolution_backprop_data.cpp | 57 ++++ .../instances/x64/matmul.cpp | 273 +++++++++++++++++- .../instances/x64/pooling.cpp | 127 +++++++- .../instances/x64/softmax.cpp | 94 ++++++ .../custom/single_layer_tests/pad.cpp | 124 +++++--- .../src/concat_const_inplace.cpp | 14 +- .../subgraph_tests/src/conv_sum_broadcast.cpp | 35 ++- .../fullyconnected_strided_inputs_outputs.cpp | 16 + .../subgraph_tests/src/inplace_edge.cpp | 2 +- .../src/input_noreorder_eltwise_bf16.cpp | 31 +- .../custom/subgraph_tests/src/interaction.cpp | 16 + .../src/matmul_strided_inputs_outputs.cpp | 15 + .../subgraph_tests/src/remove_convert.cpp | 51 +++- .../skip_tests_config.cpp | 7 + .../functional/utils/arm/filter_cpu_info.cpp | 4 + .../tests/functional/utils/cpu_test_utils.cpp | 71 +++++ .../tests/functional/utils/cpu_test_utils.hpp | 6 +- .../functional/utils/filter_cpu_info.hpp | 1 + .../functional/utils/x64/filter_cpu_info.cpp | 18 ++ 32 files changed, 1110 insertions(+), 110 deletions(-) create mode 100644 src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/softmax.cpp diff --git a/src/inference/dev_api/openvino/runtime/system_conf.hpp b/src/inference/dev_api/openvino/runtime/system_conf.hpp index c3d3364c550f1a..b6f22ad4f8daa7 100644 --- a/src/inference/dev_api/openvino/runtime/system_conf.hpp +++ b/src/inference/dev_api/openvino/runtime/system_conf.hpp @@ -146,6 +146,13 @@ OPENVINO_RUNTIME_API bool with_cpu_x86_avx512_core_amx_int8(); */ OPENVINO_RUNTIME_API bool with_cpu_x86_avx512_core_amx_bf16(); +/** + * @brief Checks whether CPU supports AMX fp16 capability + * @ingroup ov_dev_api_system_conf + * @return `True` is tAMX_FP16 instructions are available, `false` otherwise + */ +OPENVINO_RUNTIME_API bool with_cpu_x86_avx512_core_amx_fp16(); + /** * @brief Checks whether CPU supports AMX capability * @ingroup ov_dev_api_system_conf diff --git a/src/inference/src/system_conf.cpp b/src/inference/src/system_conf.cpp index 4987e72355a5a6..fd8f97d93addd5 100644 --- a/src/inference/src/system_conf.cpp +++ b/src/inference/src/system_conf.cpp @@ -92,6 +92,10 @@ bool with_cpu_x86_avx512_core_amx_bf16() { return get_cpu_info().has(Xbyak::util::Cpu::tAMX_BF16); } +bool with_cpu_x86_avx512_core_amx_fp16() { + return get_cpu_info().has(Xbyak::util::Cpu::tAMX_FP16); +} + bool with_cpu_x86_avx512_core_amx() { return with_cpu_x86_avx512_core_amx_int8() || with_cpu_x86_avx512_core_amx_bf16(); } @@ -131,6 +135,9 @@ bool with_cpu_x86_avx512_core_amx_int8() { bool with_cpu_x86_avx512_core_amx_bf16() { return false; } +bool with_cpu_x86_avx512_core_amx_fp16() { + return false; +} bool with_cpu_x86_avx512_core_amx() { return false; } diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index e17d4d7ae029ef..2ff2ee2636ec02 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -1832,7 +1832,7 @@ void Graph::EnforceInferencePrecision() { for (size_t i = 0; i < node->getOriginalInputsNumber(); i++) { auto keepOriginalInputPrecisionAtPort = [](const NodePtr& node, const size_t inPort) { - // keep non-float precisions + // keep non-float32 precisions if (node->getOriginalInputPrecisionAtPort(inPort) != ov::element::f32) return true; @@ -1876,7 +1876,7 @@ void Graph::EnforceInferencePrecision() { } for (size_t i = 0; i < node->getOriginalOutputsNumber(); i++) { - // keep non-float precisions + // keep non-float32 precisions if (node->getOriginalOutputPrecisionAtPort(i) != ov::element::f32) continue; diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution.cpp index 7f294b3fc7fc75..207e522c32f823 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution.cpp @@ -160,11 +160,16 @@ void ConvolutionLayerCPUTest::SetUp() { init_input_shapes({inputShape}); auto it = configuration.find(ov::hint::inference_precision.name()); - if (it != configuration.end() && it->second.as() == ov::element::bf16) { + ov::element::Type inference_precision = (it != configuration.end()) ? + it->second.as() : ov::element::undefined; + if (inference_precision == ov::element::bf16) { selectedType += "_BF16"; rel_threshold = 1e-2f; if (selectedType == "jit_gemm_BF16") rel_threshold = 0.05f; + } else if (inference_precision == ov::element::f16) { + selectedType += "_FP16"; + rel_threshold = 0.00125f; } else { selectedType = makeSelectedTypeStr(selectedType, netType); } diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution_backprop_data.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution_backprop_data.cpp index 93f1b11cf117a2..415515ef7f40a2 100755 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution_backprop_data.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/convolution_backprop_data.cpp @@ -196,9 +196,15 @@ void DeconvolutionLayerCPUTest::SetUp() { std::tie(kernel, stride, padBegin, padEnd, dilation, convOutChannels, padType, outPadding) = basicParamsSet; - if (additionalConfig[ov::hint::inference_precision.name()] == ov::element::bf16) { + auto it = configuration.find(ov::hint::inference_precision.name()); + ov::element::Type inference_precision = (it != configuration.end()) ? + it->second.as() : ov::element::undefined; + if (inference_precision == ov::element::bf16) { inType = outType = prec = ElementType::bf16; rel_threshold = 1e-2f; + } else if (inference_precision == ov::element::f16) { + inType = outType = prec = ElementType::f16; + rel_threshold = 0.00125f; } else { inType = outType = prec; } diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/matmul.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/matmul.cpp index 3e55d3368cefb3..dd58da28c8ad05 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/matmul.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/matmul.cpp @@ -118,9 +118,14 @@ void MatMulLayerCPUTest::SetUp() { configuration.insert(additionalConfig.begin(), additionalConfig.end()); auto it = additionalConfig.find(ov::hint::inference_precision.name()); - if (it != additionalConfig.end() && it->second.as() == ov::element::bf16) { + ov::element::Type inference_precision = (it != additionalConfig.end()) ? + it->second.as() : ov::element::undefined; + if (inference_precision == ov::element::bf16) { inType = outType = netType = ElementType::bf16; rel_threshold = abs_threshold = 1e-2f; + } else if (inference_precision == ov::element::f16) { + inType = outType = netType = ElementType::f16; + rel_threshold = abs_threshold = 1e-4f; } else { inType = outType = netType; rel_threshold = 1e-4f; @@ -128,7 +133,7 @@ void MatMulLayerCPUTest::SetUp() { } cpuNodeType = nodeType == MatMulNodeType::MatMul ? "MatMul" : "FullyConnected"; - selectedType = makeSelectedTypeStr(selectedType, outType); + selectedType = makeSelectedTypeStr(selectedType, deduce_expected_precision(outType, configuration)); ov::ParameterVector params{std::make_shared(netType, inShapeA)}; diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.cpp index 22bf0994deb37e..65f48713185f59 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.cpp @@ -19,7 +19,8 @@ std::string PoolingLayerCPUTest::getTestCaseName(const testing::TestParamInfo kernel, stride; @@ -53,6 +54,12 @@ std::string PoolingLayerCPUTest::getTestCaseName(const testing::TestParamInfo kernel, stride, dilation; std::vector padBegin, padEnd; @@ -146,6 +156,12 @@ std::string MaxPoolingV8LayerCPUTest::getTestCaseName( results << "PE" << ov::test::utils::vec2str(padEnd) << "_"; results << "Rounding=" << roundingType << "_"; results << "AutoPad=" << padType << "_"; + if (!additionalConfig.empty()) { + results << "_PluginConf"; + for (auto& item : additionalConfig) { + results << "_" << item.first << "=" << item.second.as(); + } + } results << CPUTestsBase::getTestCaseName(cpuParams); return results.str(); @@ -158,7 +174,9 @@ void MaxPoolingV8LayerCPUTest::SetUp() { InputShape inputShapes; ElementType inPrc; CPUSpecificParams cpuParams; - std::tie(basicParamsSet, inputShapes, inPrc, cpuParams) = this->GetParam(); + ov::AnyMap additionalConfig; + std::tie(basicParamsSet, inputShapes, inPrc, cpuParams, additionalConfig) = this->GetParam(); + configuration.insert(additionalConfig.begin(), additionalConfig.end()); std::vector kernel, stride, dilation; std::vector padBegin, padEnd; @@ -172,7 +190,7 @@ void MaxPoolingV8LayerCPUTest::SetUp() { if (selectedType.empty()) { selectedType = getPrimitiveType(); } - selectedType = makeSelectedTypeStr(selectedType, inPrc); + selectedType = makeSelectedTypeStr(selectedType, deduce_expected_precision(inPrc, configuration)); init_input_shapes({inputShapes}); diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.hpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.hpp index a2add35826accd..a1d04c633e94f1 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.hpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/pooling.hpp @@ -19,12 +19,15 @@ using poolLayerCpuTestParamsSet = std::tuple; + fusingSpecificParams, + ov::AnyMap>; + using maxPoolV8LayerCpuTestParamsSet = std::tuple; + CPUSpecificParams, + ov::AnyMap>; class PoolingLayerCPUTest : public testing::WithParamInterface, virtual public SubgraphBaseTest, public CpuTestWithFusing { diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/softmax.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/softmax.cpp index fe95030df307ae..bf2cc9617d8835 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/softmax.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/softmax.cpp @@ -16,7 +16,8 @@ std::string SoftMaxLayerCPUTest::getTestCaseName(const testing::TestParamInfo specificParams_FP16_AMX { + CPUSpecificParams{{}, {}, {"brgemm_avx512_amx"}, "brgemm_avx512_amx"} +}; + const std::vector IS_x64 = { {static_shapes_to_test_representation({{7, 32, 120}, {3, 7, 120, 50}}), {false, false}}, {static_shapes_to_test_representation({{7, 32, 120}, {3, 7, 120, 50}}), {true, false}}, @@ -47,6 +56,7 @@ std::vector fusingParamsSet2DBF16 { fusingPReluPerTensor, }; +std::vector fusingParamsSet2DFP16 = fusingParamsSet2DBF16; const std::vector matmulFusingParamsNightly { emptyFusingSpec, fusingElu, @@ -71,6 +81,21 @@ const auto testParams_Static_IS_x64 = ::testing::Combine(matMulParams_x64, INSTANTIATE_TEST_SUITE_P(smoke_MM_Static_IS_x64, MatMulLayerCPUTest, testParams_Static_IS_x64, MatMulLayerCPUTest::getTestCaseName); +const auto matMulParams_x64_FP16 = ::testing::Combine(::testing::ValuesIn(IS_x64), + ::testing::Values(ov::element::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testParams_Static_IS_x64_FP16 = ::testing::Combine(matMulParams_x64_FP16, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::ValuesIn(matmulFusingParams()), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + +INSTANTIATE_TEST_SUITE_P(smoke_MM_Static_IS_x64_FP16, MatMulLayerCPUTest, testParams_Static_IS_x64_FP16, MatMulLayerCPUTest::getTestCaseName); + const auto testParams2D_smoke = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_smoke()), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), @@ -93,8 +118,20 @@ const auto testParams2DBF16_smoke = ::testing::Combine(::testing::Combine(::test ::testing::ValuesIn(fusingParamsSet2DBF16), ::testing::ValuesIn(filterCPUInfo(filterSpecificParams()))); +const auto testParams2DFP16_smoke = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_smoke()), + ::testing::Values(ov::element::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)), + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2DFP16), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + INSTANTIATE_TEST_SUITE_P(smoke_FC_2D, MatMulLayerCPUTest, testParams2D_smoke, MatMulLayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_BF16, MatMulLayerCPUTest, testParams2DBF16_smoke, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_FP16, MatMulLayerCPUTest, testParams2DFP16_smoke, MatMulLayerCPUTest::getTestCaseName); const auto testParams2D_nightly = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_nightly()), ::testing::Values(ElementType::f32), @@ -212,12 +249,26 @@ const auto fullyConnectedParams2D_Brgemm_smoke = ::testing::Combine(::testing::V ::testing::Values(ov::test::utils::DEVICE_CPU), ::testing::ValuesIn(filterAdditionalConfig_Brgemm())); +const auto fullyConnectedParams2D_Brgemm_FP16_smoke = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm_smoke), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + const auto testParams2D_Brgemm_smoke = ::testing::Combine(fullyConnectedParams2D_Brgemm_smoke, ::testing::Values(MatMulNodeType::FullyConnected), ::testing::ValuesIn(fusingParamsSet2D_Brgemm_smoke), ::testing::ValuesIn(filterSpecificParams_Brgemm(true))); +const auto testParams2D_Brgemm_FP16_smoke = ::testing::Combine(fullyConnectedParams2D_Brgemm_FP16_smoke, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2D_Brgemm_smoke), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_Brgemm, MatMulLayerCPUTest, testParams2D_Brgemm_smoke, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_Brgemm_FP16, MatMulLayerCPUTest, testParams2D_Brgemm_FP16_smoke, MatMulLayerCPUTest::getTestCaseName); const std::vector IS_brgemm_smoke = { {static_shapes_to_test_representation({{1, 2, 32, 120}, {120, 5}}), {false, false}}, @@ -248,6 +299,21 @@ const auto testBrgemmParams_smoke = ::testing::Combine(matMulBrgemmParams_smoke, INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Static, MatMulLayerCPUTest, testBrgemmParams_smoke, MatMulLayerCPUTest::getTestCaseName); +const auto matMulBrgemmParams_FP16_smoke = ::testing::Combine(::testing::ValuesIn(IS_brgemm_smoke), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testBrgemmParams_FP16_smoke = ::testing::Combine(matMulBrgemmParams_FP16_smoke, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::ValuesIn(matmulFusingParams()), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + +INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Static_FP16, MatMulLayerCPUTest, testBrgemmParams_FP16_smoke, MatMulLayerCPUTest::getTestCaseName); + const std::vector IS_brgemm_nightly = { {static_shapes_to_test_representation({{1, 2, 32, 120}, {120, 5}}), {false, true}}, {static_shapes_to_test_representation({{1, 2, 32, 120}, {120, 5}}), {true, true}}, @@ -277,6 +343,21 @@ const auto testBrgemmParams_nightly = ::testing::Combine(matMulBrgemmParams_nigh INSTANTIATE_TEST_SUITE_P(nightly_MM_Brgemm_Static, MatMulLayerCPUTest, testBrgemmParams_nightly, MatMulLayerCPUTest::getTestCaseName); +const auto matMulBrgemmParams_FP16_nightly = ::testing::Combine(::testing::ValuesIn(IS_brgemm_nightly), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testBrgemmParams_FP16_nightly = ::testing::Combine(matMulBrgemmParams_FP16_nightly, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::ValuesIn(matmulFusingParams()), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + +INSTANTIATE_TEST_SUITE_P(nightly_MM_Brgemm_Static_FP16, MatMulLayerCPUTest, testBrgemmParams_FP16_nightly, MatMulLayerCPUTest::getTestCaseName); + const std::vector IS_Brgemm_Dynamic = { { { @@ -351,6 +432,21 @@ const auto testBrgemmParamsDynamic = ::testing::Combine(matMulBrgemmParamsDynami INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Dynamic, MatMulLayerCPUTest, testBrgemmParamsDynamic, MatMulLayerCPUTest::getTestCaseName); +const auto matMulBrgemmParamsDynamic_FP16 = ::testing::Combine(::testing::ValuesIn(IS_Brgemm_Dynamic), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testBrgemmParamsDynamic_FP16 = ::testing::Combine(matMulBrgemmParamsDynamic_FP16, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::Values(emptyFusingSpec), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + +INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Dynamic_FP16, MatMulLayerCPUTest, testBrgemmParamsDynamic_FP16, MatMulLayerCPUTest::getTestCaseName); + const std::vector IS_Dynamic_Fusing = { { { //dynamic case description each pair per each input has {{dynamic shape}, {{static shape case1}, {static shape case2}, ...} @@ -415,6 +511,21 @@ const auto testParamsDynamicFusing = ::testing::Combine(matMulParamsDynamicFusin INSTANTIATE_TEST_SUITE_P(smoke_MM_Dynamic_Fusing, MatMulLayerCPUTest, testParamsDynamicFusing, MatMulLayerCPUTest::getTestCaseName); +const auto matMulParamsDynamicFusing_FP16 = ::testing::Combine(::testing::ValuesIn(IS_Dynamic_Fusing), + ::testing::Values(ov::element::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testParamsDynamicFusing_FP16 = ::testing::Combine(matMulParamsDynamicFusing_FP16, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::ValuesIn(matmulFusingParams()), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + +INSTANTIATE_TEST_SUITE_P(smoke_MM_Dynamic_Fusing_FP16, MatMulLayerCPUTest, testParamsDynamicFusing_FP16, MatMulLayerCPUTest::getTestCaseName); + const auto matMulParamsBrgemmDynamicFusing = ::testing::Combine(::testing::ValuesIn(IS_Dynamic_Fusing), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), @@ -430,6 +541,21 @@ const auto testParamsBrgemmDynamicFusing = ::testing::Combine(matMulParamsBrgemm INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Dynamic_Fusing, MatMulLayerCPUTest, testParamsBrgemmDynamicFusing, MatMulLayerCPUTest::getTestCaseName); +const auto matMulParamsBrgemmDynamicFusing_FP16 = ::testing::Combine(::testing::ValuesIn(IS_Dynamic_Fusing), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testParamsBrgemmDynamicFusing_FP16 = ::testing::Combine(matMulParamsBrgemmDynamicFusing_FP16, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::ValuesIn(matmulFusingParams()), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + +INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Dynamic_Fusing_FP16, MatMulLayerCPUTest, testParamsBrgemmDynamicFusing_FP16, MatMulLayerCPUTest::getTestCaseName); + std::vector filterAdditionalConfig_BrgemmAmx() { std::vector additionalConfig; if (with_cpu_x86_bfloat16()) { @@ -485,6 +611,21 @@ const auto testBrgemmAmxParams_smoke = ::testing::Combine(matMulBrgemmAmxParams_ INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Amx_Static, MatMulLayerCPUTest, testBrgemmAmxParams_smoke, MatMulLayerCPUTest::getTestCaseName); +const auto matMulBrgemmAmxParams_FP16_smoke = ::testing::Combine(::testing::ValuesIn(IS_brgemm_Amx_smoke), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testBrgemmAmxParams_FP16_smoke = ::testing::Combine(matMulBrgemmAmxParams_FP16_smoke, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::ValuesIn(matmulBrgemmAmxFusingParams), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16_AMX))); + +INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Amx_Static_FP16, MatMulLayerCPUTest, testBrgemmAmxParams_FP16_smoke, MatMulLayerCPUTest::getTestCaseName); + const auto matMulBrgemmAmxParams_nightly = ::testing::Combine(::testing::ValuesIn(IS_brgemm_Amx_smoke), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), @@ -500,6 +641,21 @@ const auto testBrgemmAmxParams_nightly = ::testing::Combine(matMulBrgemmAmxParam INSTANTIATE_TEST_SUITE_P(nightly_MM_Brgemm_Amx_Static, MatMulLayerCPUTest, testBrgemmAmxParams_nightly, MatMulLayerCPUTest::getTestCaseName); +const auto matMulBrgemmAmxParams_FP16_nightly = ::testing::Combine(::testing::ValuesIn(IS_brgemm_Amx_smoke), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testBrgemmAmxParams_FP16_nightly = ::testing::Combine(matMulBrgemmAmxParams_FP16_nightly, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::ValuesIn(matmulBrgemmAmxFusingParams), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16_AMX))); + +INSTANTIATE_TEST_SUITE_P(nightly_MM_Brgemm_Amx_Static_FP16, MatMulLayerCPUTest, testBrgemmAmxParams_FP16_nightly, MatMulLayerCPUTest::getTestCaseName); + const auto matMulBrgemmAmxParamsDynamic = ::testing::Combine(::testing::ValuesIn(IS_Brgemm_Dynamic), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), @@ -515,6 +671,21 @@ const auto testBrgemmAmxParamsDynamic = ::testing::Combine(matMulBrgemmAmxParams INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Amx_Dynamic, MatMulLayerCPUTest, testBrgemmAmxParamsDynamic, MatMulLayerCPUTest::getTestCaseName); +const auto matMulBrgemmAmxParamsDynamic_FP16 = ::testing::Combine(::testing::ValuesIn(IS_Brgemm_Dynamic), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::PARAMETER), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testBrgemmAmxParamsDynamic_FP16 = ::testing::Combine(matMulBrgemmAmxParamsDynamic_FP16, + ::testing::Values(MatMulNodeType::MatMul), + ::testing::Values(emptyFusingSpec), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16_AMX))); + +INSTANTIATE_TEST_SUITE_P(smoke_MM_Brgemm_Amx_Dynamic_FP16, MatMulLayerCPUTest, testBrgemmAmxParamsDynamic_FP16, MatMulLayerCPUTest::getTestCaseName); + const std::vector IS2D_Brgemm_Amx_smoke = { {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {true, false}}, {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {true, true}}, @@ -630,6 +801,21 @@ const auto testParams2D_Brgemm_Amx_smoke = ::testing::Combine(fullyConnectedPara INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_Brgemm_Amx, MatMulLayerCPUTest, testParams2D_Brgemm_Amx_smoke, MatMulLayerCPUTest::getTestCaseName); +const auto fullyConnectedParams2D_FP16_Brgemm_Amx_smoke = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm_Amx_smoke), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testParams2D_FP16_Brgemm_Amx_smoke = ::testing::Combine(fullyConnectedParams2D_FP16_Brgemm_Amx_smoke, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2D_Brgemm_smoke), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16_AMX))); + +INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_Brgemm_Amx_FP16, MatMulLayerCPUTest, testParams2D_FP16_Brgemm_Amx_smoke, MatMulLayerCPUTest::getTestCaseName); + const std::vector IS2D_Brgemm_nightly = { {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {false, false}}, {static_shapes_to_test_representation({{59, 16}, {16, 120}}), {false, true}}, @@ -668,6 +854,21 @@ const auto testParams2D_Brgemm_nightly = ::testing::Combine(fullyConnectedParams INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_Brgemm, MatMulLayerCPUTest, testParams2D_Brgemm_nightly, MatMulLayerCPUTest::getTestCaseName); +const auto fullyConnectedParams2D_FP16_Brgemm_nightly = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm_nightly), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testParams2D_FP16_Brgemm_nightly = ::testing::Combine(fullyConnectedParams2D_FP16_Brgemm_nightly, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2D_nightly), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + +INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_Brgemm_FP16, MatMulLayerCPUTest, testParams2D_FP16_Brgemm_nightly, MatMulLayerCPUTest::getTestCaseName); + const auto fullyConnectedParams2D_Brgemm_Amx_nightly = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm_nightly), ::testing::Values(ElementType::f32), ::testing::Values(ElementType::undefined), @@ -683,6 +884,21 @@ const auto testParams2D_Brgemm_Amx_nightly = ::testing::Combine(fullyConnectedPa INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_Brgemm_Amx, MatMulLayerCPUTest, testParams2D_Brgemm_Amx_nightly, MatMulLayerCPUTest::getTestCaseName); +const auto fullyConnectedParams2D_FP16_Brgemm_Amx_nightly = ::testing::Combine(::testing::ValuesIn(IS2D_Brgemm_nightly), + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + +const auto testParams2D_FP16_Brgemm_Amx_nightly = ::testing::Combine(fullyConnectedParams2D_FP16_Brgemm_Amx_nightly, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2D_nightly), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16_AMX))); + +INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_Brgemm_Amx_FP16, MatMulLayerCPUTest, testParams2D_FP16_Brgemm_Amx_nightly, MatMulLayerCPUTest::getTestCaseName); + const auto testParams2DBF16_nightly = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_nightly()), ::testing::ValuesIn(netPRCs()), ::testing::Values(ElementType::undefined), @@ -694,8 +910,20 @@ const auto testParams2DBF16_nightly = ::testing::Combine(::testing::Combine(::te ::testing::ValuesIn(fusingParamsSet2DBF16), ::testing::ValuesIn(filterCPUInfo(filterSpecificParams()))); +const auto testParams2DFP16_nightly = ::testing::Combine(::testing::Combine(::testing::ValuesIn(IS2D_nightly()), + ::testing::Values(ov::element::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)), + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet2DFP16), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + INSTANTIATE_TEST_SUITE_P(nightly_FC_2D, MatMulLayerCPUTest, testParams2D_nightly, MatMulLayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_BF16, MatMulLayerCPUTest, testParams2DBF16_nightly, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(nightly_FC_2D_FP16, MatMulLayerCPUTest, testParams2DFP16_nightly, MatMulLayerCPUTest::getTestCaseName); std::vector fusingParamsSet3D_smoke { // The following three patterns are convered by MLAS test @@ -714,6 +942,12 @@ std::vector fusingParamsSet3DBF16 { fusingMultiplyPerChannel, }; +std::vector fusingParamsSet3DFP16 { + emptyFusingSpec, + fusingBias, + fusingMultiplyPerChannel, +}; + const auto fullyConnectedParams3DBF16_smoke = ::testing::Combine(::testing::ValuesIn(IS3D_smoke()), ::testing::ValuesIn(netPRCs()), ::testing::Values(ElementType::undefined), @@ -722,12 +956,26 @@ const auto fullyConnectedParams3DBF16_smoke = ::testing::Combine(::testing::Valu ::testing::Values(ov::test::utils::DEVICE_CPU), ::testing::ValuesIn(additionalConfig())); +const auto fullyConnectedParams3DFP16_smoke = ::testing::Combine(::testing::ValuesIn(IS3D_smoke()), + ::testing::Values(ov::element::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); + const auto testParams3DBF16_smoke = ::testing::Combine(fullyConnectedParams3DBF16_smoke, ::testing::Values(MatMulNodeType::FullyConnected), ::testing::ValuesIn(fusingParamsSet3DBF16), ::testing::ValuesIn(filterCPUInfo(filterSpecificParams()))); +const auto testParams3DFP16_smoke = ::testing::Combine(fullyConnectedParams3DFP16_smoke, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet3DFP16), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_BF16, MatMulLayerCPUTest, testParams3DBF16_smoke, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_FP16, MatMulLayerCPUTest, testParams3DFP16_smoke, MatMulLayerCPUTest::getTestCaseName); const auto fullyConnectedParams3D_smoke = ::testing::Combine(::testing::ValuesIn(IS3D_smoke()), ::testing::Values(ElementType::f32), @@ -794,19 +1042,32 @@ const auto fullyConnectedParams3DBF16_nightly = ::testing::Combine(::testing::Va ::testing::Values(ov::test::utils::DEVICE_CPU), ::testing::ValuesIn(additionalConfig())); -const auto testParams3DBF16_nightly = ::testing::Combine(fullyConnectedParams3DBF16_nightly, - ::testing::Values(MatMulNodeType::FullyConnected), - ::testing::ValuesIn(fusingParamsSet3DBF16), - ::testing::ValuesIn(filterCPUInfo(filterSpecificParams()))); - -INSTANTIATE_TEST_SUITE_P(nightly_FC_3D_BF16, MatMulLayerCPUTest, testParams3DBF16_nightly, MatMulLayerCPUTest::getTestCaseName); +const auto fullyConnectedParams3DFP16_nightly = ::testing::Combine(::testing::ValuesIn(IS3D_nightly), + ::testing::Values(ov::element::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::Values(utils::InputLayerType::CONSTANT), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(cpu_f16_plugin_config)); const auto testParams3D_nightly = ::testing::Combine(fullyConnectedParams3D_nightly, ::testing::Values(MatMulNodeType::FullyConnected), ::testing::ValuesIn(fusingParamsSet3D_nightly), ::testing::ValuesIn(filterCPUInfo(filterSpecificParams()))); +const auto testParams3DBF16_nightly = ::testing::Combine(fullyConnectedParams3DBF16_nightly, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet3DBF16), + ::testing::ValuesIn(filterCPUInfo(filterSpecificParams()))); + +const auto testParams3DFP16_nightly = ::testing::Combine(fullyConnectedParams3DFP16_nightly, + ::testing::Values(MatMulNodeType::FullyConnected), + ::testing::ValuesIn(fusingParamsSet3DFP16), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(specificParams_FP16))); + INSTANTIATE_TEST_SUITE_P(nightly_FC_3D, MatMulLayerCPUTest, testParams3D_nightly, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(nightly_FC_3D_BF16, MatMulLayerCPUTest, testParams3DBF16_nightly, MatMulLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(nightly_FC_3D_FP16, MatMulLayerCPUTest, testParams3DFP16_nightly, MatMulLayerCPUTest::getTestCaseName); class MatMulLayerCPUTestUndefShapes : public MatMulLayerCPUTest { }; diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/pooling.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/pooling.cpp index 753b671dd7bace..4127d0c9deddd8 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/pooling.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/pooling.cpp @@ -38,7 +38,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MaxPoolV8_CPU_4D_ref, MaxPoolingV8LayerCPUTest, ::testing::ValuesIn(paramsMaxV84D_ref), ::testing::ValuesIn(inputShapes4D()), ::testing::ValuesIn((inpOutPrecision())), - ::testing::Values(ref)), + ::testing::Values(ref), + ::testing::Values(CPUTestUtils::empty_plugin_config)), MaxPoolingV8LayerCPUTest::getTestCaseName); const auto avx512_nwc = CPUSpecificParams{{nwc}, {nwc}, {"jit_avx512"}, "jit_avx512"}; @@ -100,7 +101,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_AvgPool_CPU_4D_I8, PoolingLayerCPUTest, ::testing::Values(ElementType::f32), ::testing::Values(true), ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_4D)), - ::testing::ValuesIn(fusingParamsSet)), + ::testing::ValuesIn(fusingParamsSet), + ::testing::Values(CPUTestUtils::empty_plugin_config)), PoolingLayerCPUTest::getTestCaseName); const std::vector inputShapes5D_int8 = { @@ -140,9 +142,116 @@ INSTANTIATE_TEST_SUITE_P(smoke_AvgPool_CPU_5D_I8, PoolingLayerCPUTest, ::testing::Values(ElementType::f32), ::testing::Values(true), ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_5D)), - ::testing::ValuesIn(fusingParamsSet)), + ::testing::ValuesIn(fusingParamsSet), + ::testing::Values(CPUTestUtils::empty_plugin_config)), PoolingLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_AvgPool_CPU_4D_I8_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsAvg4D()), + ::testing::ValuesIn(inputShapes4D_int8), + ::testing::Values(ElementType::f32), + ::testing::Values(true), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigsFusing_4D)), + ::testing::ValuesIn(fusingParamsSet), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_AvgPool_CPU_5D_I8_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsAvg5D()), + ::testing::ValuesIn(inputShapes5D_int8), + ::testing::Values(ElementType::f32), + ::testing::Values(true), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigsFusing_5D)), + ::testing::ValuesIn(fusingParamsSet), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_MaxPool_CPU_3D_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsMax3D()), + ::testing::ValuesIn(inputShapes3D()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::Values(false), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_AvgPool_CPU_3D_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsAvg3D()), + ::testing::ValuesIn(inputShapes3D()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::Values(false), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_MaxPool_CPU_4D_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsMax4D()), + ::testing::ValuesIn(inputShapes4D()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::Values(false), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_MaxPoolV8_CPU_4D_FP16, MaxPoolingV8LayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsMaxV84D()), + ::testing::ValuesIn(inputShapes4D()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(cpu_f16_plugin_config)), + MaxPoolingV8LayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_AvgPool_CPU_4D_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsAvg4D()), + ::testing::ValuesIn(inputShapes4D()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::Values(false), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_AvgPool_CPU_Large_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsAvg4D_Large()), + ::testing::ValuesIn(inputShapes4D_Large()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::Values(false), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_MaxPool_CPU_5D_FP16, PoolingLayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsMax5D()), + ::testing::ValuesIn(inputShapes5D()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::Values(false), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(cpu_f16_plugin_config)), + PoolingLayerCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_MaxPoolV8_CPU_5D_FP16, MaxPoolingV8LayerCPUTest, + ::testing::Combine( + ::testing::ValuesIn(paramsMaxV85D()), + ::testing::ValuesIn(inputShapes5D()), + ::testing::ValuesIn(inpOutPrecision()), + ::testing::ValuesIn(filterCPUInfoForDeviceWithFP16(vecCpuConfigs)), + ::testing::Values(cpu_f16_plugin_config)), + MaxPoolingV8LayerCPUTest::getTestCaseName); + INSTANTIATE_TEST_SUITE_P(smoke_MaxPool_CPU_3D, PoolingLayerCPUTest, ::testing::Combine( ::testing::ValuesIn(paramsMax3D()), @@ -150,7 +259,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MaxPool_CPU_3D, PoolingLayerCPUTest, ::testing::ValuesIn((inpOutPrecision())), ::testing::Values(false), ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_3D)), - ::testing::Values(emptyFusingSpec)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(CPUTestUtils::empty_plugin_config)), PoolingLayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(smoke_MaxPoolV8_CPU_3D, MaxPoolingV8LayerCPUTest, @@ -158,7 +268,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MaxPoolV8_CPU_3D, MaxPoolingV8LayerCPUTest, ::testing::ValuesIn(paramsMaxV83D()), ::testing::ValuesIn(inputShapes3D()), ::testing::ValuesIn((inpOutPrecision())), - ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_3D))), + ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_3D)), + ::testing::Values(CPUTestUtils::empty_plugin_config)), MaxPoolingV8LayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(smoke_MaxPool_CPU_5D, PoolingLayerCPUTest, @@ -168,7 +279,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MaxPool_CPU_5D, PoolingLayerCPUTest, ::testing::ValuesIn((inpOutPrecision())), ::testing::Values(false), ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_5D)), - ::testing::Values(emptyFusingSpec)), + ::testing::Values(emptyFusingSpec), + ::testing::Values(CPUTestUtils::empty_plugin_config)), PoolingLayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(smoke_MaxPoolV8_CPU_5D, MaxPoolingV8LayerCPUTest, @@ -176,7 +288,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_MaxPoolV8_CPU_5D, MaxPoolingV8LayerCPUTest, ::testing::ValuesIn(paramsMaxV85D()), ::testing::ValuesIn(inputShapes5D()), ::testing::ValuesIn((inpOutPrecision())), - ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_5D))), + ::testing::ValuesIn(filterCPUInfoForDevice(vecCpuConfigsFusing_5D)), + ::testing::Values(CPUTestUtils::empty_plugin_config)), MaxPoolingV8LayerCPUTest::getTestCaseName); } // namespace } // namespace Pooling diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/softmax.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/softmax.cpp new file mode 100644 index 00000000000000..2a22f629c29661 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/softmax.cpp @@ -0,0 +1,94 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "custom/single_layer_tests/classes/softmax.hpp" +#include +#include "utils/cpu_test_utils.hpp" +#include "utils/filter_cpu_info.hpp" +#include + +using namespace CPUTestUtils; + +namespace ov { +namespace test { +namespace SoftMax { +namespace { +const auto optimizedCPUSpec = []()-> std::vector{ + const auto avx512 = CPUSpecificParams{{}, {}, {"jit"}, "jit_avx512"}; + const auto avx2 = CPUSpecificParams{{}, {}, {"jit"}, "jit_avx2"}; + const auto sse42 = CPUSpecificParams{{}, {}, {"jit"}, "jit_sse42"}; + const std::vector vecCpuConfigs = {avx512, avx2, sse42}; + auto supportConfigure = CPUTestUtils::filterCPUInfoForDevice(vecCpuConfigs); + // only the MAX ISA of vecCpuConfigs will be tested + if (supportConfigure.size() > 0) { + return std::vector{supportConfigure[0]}; + } else { + return std::vector{}; + } +}; + +const std::vector optimizedConfigsFP32 = { + // Static shapes + {ov::test::InputShape{ov::PartialShape{1, 100}, {ov::Shape{1, 100}}}, 1}, + {ov::test::InputShape{ov::PartialShape{10, 10}, {ov::Shape{10, 10}}}, 1}, + {ov::test::InputShape{ov::PartialShape{100, 1}, {ov::Shape{100, 1}}}, 0}, + {ov::test::InputShape{ov::PartialShape{100, 1}, {ov::Shape{100, 1}}}, 1}, + {ov::test::InputShape{ov::PartialShape{5, 5, 1}, {ov::Shape{5, 5, 1}}}, 1}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5}, {ov::Shape{5, 5, 5}}}, 2}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5}}}, 0}, + {ov::test::InputShape{ov::PartialShape{5, 5, 1, 1}, {ov::Shape{5, 5, 1, 1}}}, 1}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5}}}, 1}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 1}, {ov::Shape{5, 5, 5, 1}}}, 2}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5}}}, 2}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5}}}, 3}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5, 5}}}, 0}, + {ov::test::InputShape{ov::PartialShape{5, 5, 1, 1, 1}, {ov::Shape{5, 5, 1, 1, 1}}}, 1}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5, 5}}}, 1}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 1, 1}, {ov::Shape{5, 5, 5, 1, 1}}}, 2}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5, 5}}}, 2}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 1, 1}, {ov::Shape{5, 5, 5, 1, 1}}}, 3}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5, 5}}}, 3}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5, 1}, {ov::Shape{5, 5, 5, 5, 1}}}, 4}, + {ov::test::InputShape{ov::PartialShape{5, 5, 5, 5, 5}, {ov::Shape{5, 5, 5, 5, 5}}}, 4}, + // Dynamic shapes + {ov::test::InputShape{// dynamic shape + ov::PartialShape{-1, -1}, + {// target static shapes + ov::Shape{10, 10}, + ov::Shape{15, 15}, + ov::Shape{10, 10}, + ov::Shape{10, 5}}}, + 1}, + {ov::test::InputShape{// dynamic shape + ov::PartialShape{-1, -1, 1, 1, 1}, + {// target static shapes + ov::Shape{5, 5, 1, 1, 1}, + ov::Shape{10, 7, 1, 1, 1}, + ov::Shape{5, 5, 1, 1, 1}}}, + 1}, + {ov::test::InputShape{// dynamic shape + ov::PartialShape{{1, 10}, 10}, + {// target static shapes + ov::Shape{10, 10}, + ov::Shape{5, 10}}}, + 1}, +}; + +const auto OptimizedParams = testing::Combine(testing::Values(ElementType::f32, ElementType::bf16), + testing::ValuesIn(optimizedConfigsFP32), + testing::Values(ov::test::utils::DEVICE_CPU), + testing::ValuesIn(optimizedCPUSpec()), + testing::Values(CPUTestUtils::empty_plugin_config)); + +INSTANTIATE_TEST_SUITE_P(smoke_SoftMax_Optimized_CPU, + SoftMaxLayerCPUTest, + OptimizedParams, + SoftMaxLayerCPUTest::getTestCaseName); + +//TODO CVS-143812 + +} // namespace +} // namespace SoftMax +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/pad.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/pad.cpp index f3b8a603f86e1b..7d35fe6b822cb2 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/pad.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/pad.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "openvino/runtime/system_conf.hpp" #include "common_test_utils/ov_tensor_utils.hpp" #include "common_test_utils/test_enums.hpp" @@ -23,7 +24,8 @@ using PadLayerCPUTestParamSet = std::tuple< std::vector, // padsEnd float, // argPadValue ov::op::PadMode, // padMode - CPUSpecificParams + CPUSpecificParams, + ov::AnyMap // cpu device config >; class PadLayerCPUTest : public testing::WithParamInterface, @@ -37,7 +39,8 @@ class PadLayerCPUTest : public testing::WithParamInterface inputPrecisions = { ElementType::i8 }; +const std::vector deviceConfig = { + CPUTestUtils::empty_plugin_config, + cpu_f16_plugin_config +}; + const std::vector inputLayerTypes = { ov::test::utils::InputLayerType::CONSTANT, ov::test::utils::InputLayerType::PARAMETER @@ -198,7 +214,7 @@ const std::vector> padsEnd4D_Full = {{0, 0, -2, 0}, {0, 0 const std::vector CPUParams4DBlocked = { cpuParams_nChw16c, - cpuParams_nChw8c, + cpuParams_nChw8c }; INSTANTIATE_TEST_SUITE_P( @@ -212,7 +228,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DConstBlocked_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -227,7 +244,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4D_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::Values(cpuParams_nhwc)), + ::testing::Values(cpuParams_nhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -242,7 +260,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DBlocked_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -257,7 +276,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DBlocked_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::Values(cpuParams_nhwc)), + ::testing::Values(cpuParams_nhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -272,7 +292,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DConstBlocked_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -287,7 +308,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4D_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::Values(cpuParams_nhwc)), + ::testing::Values(cpuParams_nhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -302,7 +324,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DBlocked_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -317,7 +340,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DBlocked_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::Values(cpuParams_nhwc)), + ::testing::Values(cpuParams_nhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -352,7 +376,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4D_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams4DDynamic)), + ::testing::ValuesIn(CPUParams4DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -367,7 +392,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DConstBlocked_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -382,7 +408,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4D_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams4DDynamic)), + ::testing::ValuesIn(CPUParams4DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -397,7 +424,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DBlocked_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -412,7 +440,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4D_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams4DDynamic)), + ::testing::ValuesIn(CPUParams4DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -427,7 +456,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DConstBlocked_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -442,7 +472,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4D_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams4DDynamic)), + ::testing::ValuesIn(CPUParams4DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -457,7 +488,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd4DBlocked_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams4DBlocked)), + ::testing::ValuesIn(CPUParams4DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -499,7 +531,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DConstBlocked_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -514,7 +547,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::Values(cpuParams_ndhwc)), + ::testing::Values(cpuParams_ndhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -529,7 +563,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DBlocked_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -544,7 +579,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::Values(cpuParams_ndhwc)), + ::testing::Values(cpuParams_ndhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -559,7 +595,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DConstBlocked_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -574,7 +611,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::Values(cpuParams_ndhwc)), + ::testing::Values(cpuParams_ndhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -589,7 +627,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DBlocked_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -604,7 +643,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::Values(cpuParams_ndhwc)), + ::testing::Values(cpuParams_ndhwc), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -639,7 +679,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams5DDynamic)), + ::testing::ValuesIn(CPUParams5DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -654,7 +695,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DConstBlocked_Smoke), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -669,7 +711,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams5DDynamic)), + ::testing::ValuesIn(CPUParams5DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -684,7 +727,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DBlocked_Smoke), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -699,7 +743,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams5DDynamic)), + ::testing::ValuesIn(CPUParams5DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -714,7 +759,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DConstBlocked_Full), ::testing::ValuesIn(argPadValue), ::testing::Values(ov::op::PadMode::CONSTANT), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -729,7 +775,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5D_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams5DDynamic)), + ::testing::ValuesIn(CPUParams5DDynamic), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); @@ -744,7 +791,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(padsEnd5DBlocked_Full), ::testing::Values(0), ::testing::ValuesIn(padMode), - ::testing::ValuesIn(CPUParams5DBlocked)), + ::testing::ValuesIn(CPUParams5DBlocked), + ::testing::ValuesIn(deviceConfig)), PadLayerCPUTest::getTestCaseName ); diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/concat_const_inplace.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/concat_const_inplace.cpp index ff3f0bad3f302a..1ae0d661746987 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/concat_const_inplace.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/concat_const_inplace.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "openvino/runtime/system_conf.hpp" #include "shared_test_classes/base/ov_subgraph.hpp" #include "utils/cpu_test_utils.hpp" #include "common_test_utils/ov_tensor_utils.hpp" @@ -37,8 +38,13 @@ class ConcatConstantInPlaceTest : public testing::WithParamInterfaceGetParam())) - configuration.insert({ov::hint::inference_precision.name(), ov::element::bf16}); + if (ov::element::bf16 == (inType = outType = this->GetParam())) { + configuration.insert({ov::hint::inference_precision(ov::element::bf16)}); + } else if (ov::element::f16 == (inType = outType = this->GetParam())) { + configuration.insert({ov::hint::inference_precision(ov::element::f16)}); + } else { + configuration.insert({ov::hint::inference_precision(ov::element::f32)}); + } const ov::Shape inputShape = {1, 3, 3, 11}; ov::ParameterVector inputParams{std::make_shared(ov::element::f32, inputShape)}; @@ -79,11 +85,11 @@ TEST_P(ConcatConstantInPlaceTest, smoke_ConcatConstantInPlaceTest_CPU) { CheckNumberOfNodesWithType(compiledModel, "Reorder", 2); } + INSTANTIATE_TEST_SUITE_P(smoke_ConcatConstantInPlaceTest_CPU, ConcatConstantInPlaceTest, - testing::Values(ov::element::f32, ov::element::bf16), + testing::Values(ov::element::f32, ov::element::bf16, ov::element::f16), ConcatConstantInPlaceTest::getTestCaseName); - } // namespace } // namespace test } // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/conv_sum_broadcast.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/conv_sum_broadcast.cpp index 4ac54dcd19b009..002ca921c2afcf 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/conv_sum_broadcast.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/conv_sum_broadcast.cpp @@ -118,8 +118,12 @@ class ConvSumInPlaceTest : public testing::WithParamInterfacesecond.as() == ov::element::bf16) { + ov::element::Type inference_precision = (it != configuration.end()) ? + it->second.as() : ov::element::undefined; + if (inference_precision == ov::element::bf16) { runtimeType = ov::element::Type_t::bf16; + } else if (inference_precision == ov::element::f16) { + runtimeType = ov::element::Type_t::f16; } if (inputParams.front()->get_element_type() == ov::element::i8 || inputParams.front()->get_element_type() == ov::element::u8) { @@ -138,12 +142,25 @@ class ConvSumInPlaceTest : public testing::WithParamInterface fusingParamsSetBF16{ fusingReluScaleShift }; +const std::vector fusingParamsSetFP16 = fusingParamsSetBF16; + InputShape convInpShape = { //dynamic shapes {-1, 32, -1, -1}, @@ -465,6 +484,16 @@ INSTANTIATE_TEST_SUITE_P(smoke_Conv_Sum_Broadcast_BF16, ::testing::Values(cpu_bf16_plugin_config)), ConvSumInPlaceTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Conv_Sum_Broadcast_FP16, + ConvSumInPlaceTest, + ::testing::Combine( + ::testing::Values(convInpShape), + ::testing::ValuesIn(secondInp), + ::testing::Values(true, false), + ::testing::ValuesIn(fusingParamsSetFP16), + ::testing::Values(cpu_f16_plugin_config)), + ConvSumInPlaceTest::getTestCaseName); + INSTANTIATE_TEST_SUITE_P(smoke_Conv_Sum_Broadcast_INT8, ConvSumInPlaceTestInt8, ::testing::Combine( ::testing::Values(convInpShape), diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/fullyconnected_strided_inputs_outputs.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/fullyconnected_strided_inputs_outputs.cpp index b0a5855fd4de5c..1a81e6019fc6c1 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/fullyconnected_strided_inputs_outputs.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/fullyconnected_strided_inputs_outputs.cpp @@ -102,6 +102,16 @@ TEST_P(FullyConnectedStridedInputsOutputsTest, CompareWithRefs) { run(); } +using FullyConnectedStridedInputsOutputsTest_FP16 = FullyConnectedStridedInputsOutputsTest; +TEST_P(FullyConnectedStridedInputsOutputsTest_FP16, CompareWithRefs) { + if (!(ov::with_cpu_x86_avx512_core_fp16())) { + GTEST_SKIP() << "Skipping test, platform don't support precision f16"; + } + configuration.insert({ov::hint::inference_precision.name(), ov::element::f16}); + + run(); +} + namespace { INSTANTIATE_TEST_SUITE_P(smoke_Check, @@ -110,6 +120,12 @@ INSTANTIATE_TEST_SUITE_P(smoke_Check, ::testing::Values(2, 3)), FullyConnectedStridedInputsOutputsTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Check, + FullyConnectedStridedInputsOutputsTest_FP16, + ::testing::Combine(::testing::Values(ov::element::f32), + ::testing::Values(2, 3)), + FullyConnectedStridedInputsOutputsTest::getTestCaseName); + } // namespace } // namespace test diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/inplace_edge.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/inplace_edge.cpp index 8f7f6780048827..feae1769e0e439 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/inplace_edge.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/inplace_edge.cpp @@ -37,7 +37,7 @@ class NonInputInPlaceTest : public testing::WithParamInterface, vir void SetUp() override { targetDevice = utils::DEVICE_CPU; - configuration.insert({ov::hint::inference_precision.name(), ov::element::f16.to_string()}); + configuration.insert({ov::hint::inference_precision.name(), ov::element::f16}); const ov::Shape inputShape = {1, 11, 3, 3}; targetStaticShapes = {{inputShape, inputShape}}; ElementType prc = this->GetParam(); diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/input_noreorder_eltwise_bf16.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/input_noreorder_eltwise_bf16.cpp index 9cfa14baa8a277..a947c765b009bb 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/input_noreorder_eltwise_bf16.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/input_noreorder_eltwise_bf16.cpp @@ -14,12 +14,15 @@ namespace test { class InputNoReorderEltwiseBF16 : virtual public SubgraphBaseStaticTest, public CPUTestsBase { protected: - void SetUp() override { - auto netPrecision = inType = ov::element::f32; + virtual void set_output_type_and_config() { outType = ov::element::bf16; - targetDevice = ov::test::utils::DEVICE_CPU; ov::AnyMap additional_config{ov::hint::inference_precision(ov::element::bf16)}; configuration.insert(additional_config.begin(), additional_config.end()); + } + void SetUp() override { + auto netPrecision = inType = ov::element::f32; + set_output_type_and_config(); + targetDevice = ov::test::utils::DEVICE_CPU; ov::Shape inputShape{2, 4, 4, 1}; auto eltwiseType = ov::test::utils::EltwiseTypes::ADD; @@ -56,5 +59,27 @@ TEST_F(InputNoReorderEltwiseBF16, smoke_CompareWithRefs) { CheckNumberOfNodesWithType(compiledModel, "Convert", 0); CheckNumberOfNodesWithTypes(compiledModel, {"Eltwise", "Subgraph"}, 1); } + +class InputNoReorderEltwiseFP16 : public InputNoReorderEltwiseBF16 { +protected: + void set_output_type_and_config() override { + outType = ov::element::f16; + ov::AnyMap additional_config{ov::hint::inference_precision(ov::element::f16)}; + configuration.insert(additional_config.begin(), additional_config.end()); + } +}; + +TEST_F(InputNoReorderEltwiseFP16, smoke_CompareWithRefs) { + if (!(ov::with_cpu_x86_avx512_core_fp16() || ov::with_cpu_x86_avx512_core_amx_fp16())) { + GTEST_SKIP() << "Skipping test, platform don't support precision f16"; + } + + run(); + + CheckNumberOfNodesWithType(compiledModel, "Reorder", 0); + CheckNumberOfNodesWithType(compiledModel, "Convert", 0); + CheckNumberOfNodesWithTypes(compiledModel, {"Eltwise", "Subgraph"}, 1); +} + } // namespace test } // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/interaction.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/interaction.cpp index b670b565a3903a..1be7035c08285f 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/interaction.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/interaction.cpp @@ -171,8 +171,20 @@ TEST_P(IntertactionCPUTest, CompareWithRefs) { CheckNumberOfNodesWithType(compiledModel, "Interaction", 1); } +using IntertactionCPUTest_FP16 = IntertactionCPUTest; +TEST_P(IntertactionCPUTest_FP16, CompareWithRefs) { + if (!(ov::with_cpu_x86_avx512_core_fp16())) { + GTEST_SKIP() << "Skipping test, platform don't support precision f16"; + } + configuration.insert({ov::hint::inference_precision.name(), ov::element::f16}); + + run(); + CheckNumberOfNodesWithType(compiledModel, "Interaction", 1); +} + namespace { const std::vector inPrecisions = {ElementType::f32, ElementType::bf16, ElementType::i32, ElementType::i8}; +const std::vector inPrecisions_FP16 = {ElementType::f32, ElementType::i32}; // the model has 27 inputs with same shape const std::vector input_shapes = { // temporarily disable dynamic shape for performance issue @@ -194,4 +206,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_Interaction, IntertactionCPUTest, ::testing::Combine(::testing::ValuesIn(inPrecisions), ::testing::ValuesIn(input_shapes)), IntertactionCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Interaction, + IntertactionCPUTest_FP16, + ::testing::Combine(::testing::ValuesIn(inPrecisions_FP16), ::testing::ValuesIn(input_shapes)), + IntertactionCPUTest::getTestCaseName); } // namespace diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/matmul_strided_inputs_outputs.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/matmul_strided_inputs_outputs.cpp index d30c4378921553..7ce57dbfe28ee6 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/matmul_strided_inputs_outputs.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/matmul_strided_inputs_outputs.cpp @@ -89,6 +89,16 @@ TEST_P(MatmulStridedInputsOutputsTest, CompareWithRefs) { run(); } +using MatmulStridedInputsOutputsTest_FP16 = MatmulStridedInputsOutputsTest; +TEST_P(MatmulStridedInputsOutputsTest_FP16, CompareWithRefs) { + if (!(ov::with_cpu_x86_avx512_core_fp16())) { + GTEST_SKIP() << "Skipping test, platform don't support precision f16"; + } + configuration.insert({ov::hint::inference_precision.name(), ov::element::f16}); + + run(); +} + namespace { INSTANTIATE_TEST_SUITE_P(smoke_Check, @@ -96,6 +106,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_Check, ::testing::Values(ov::element::f32, ov::element::bf16), MatmulStridedInputsOutputsTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Check, + MatmulStridedInputsOutputsTest_FP16, + ::testing::Values(ov::element::f32), + MatmulStridedInputsOutputsTest::getTestCaseName); + } // namespace } // namespace test diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/remove_convert.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/remove_convert.cpp index 1f62e678802328..b2d57bc64b138f 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/remove_convert.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/remove_convert.cpp @@ -12,7 +12,7 @@ using namespace CPUTestUtils; namespace ov { namespace test { -using RemoveConvertCPUTestParams = std::tuple; +using RemoveConvertCPUTestParams = std::tuple; class RemoveUselessBF16ConvertCPUTest : public testing::WithParamInterface, virtual public SubgraphBaseTest, @@ -21,21 +21,27 @@ class RemoveUselessBF16ConvertCPUTest : public testing::WithParamInterface& obj) { ElementType inType; InputShape inputShape; - std::tie(inType, inputShape) = obj.param; + ov::AnyMap additionalConfig; + std::tie(inType, inputShape, additionalConfig) = obj.param; std::ostringstream result; result << "IS=" << inputShape << "_"; result << "Prc=" << inType; + if (!additionalConfig.empty()) { + result << "_PluginConf"; + for (auto& item : additionalConfig) { + result << "_" << item.first << "=" << item.second.as(); + } + } return result.str(); } void SetUp() override { ElementType inType; InputShape inputShape; - std::tie(inType, inputShape) = this->GetParam(); + ov::AnyMap additionalConfig; + std::tie(inType, inputShape, additionalConfig) = this->GetParam(); + configuration.insert(additionalConfig.begin(), additionalConfig.end()); targetDevice = ov::test::utils::DEVICE_CPU; - if (inType == ElementType::bf16) { - configuration.insert({ov::hint::inference_precision(ov::element::bf16)}); - } std::tie(inFmts, outFmts, priority, selectedType) = CPUSpecificParams{{}, {}, {}, makeSelectedTypeStr("ref", inType)}; init_input_shapes({inputShape}); @@ -65,17 +71,25 @@ class RemoveUselessConvertCPUTest : public testing::WithParamInterface& obj) { ElementType inType; InputShape inputShape; - std::tie(inType, inputShape) = obj.param; + ov::AnyMap additionalConfig; + std::tie(inType, inputShape, additionalConfig) = obj.param; std::ostringstream result; result << "IS=" << inputShape << "_"; result << "Prc=" << inType; + if (!additionalConfig.empty()) { + result << "_PluginConf"; + for (auto& item : additionalConfig) { + result << "_" << item.first << "=" << item.second.as(); + } + } return result.str(); } void SetUp() override { ElementType inType; InputShape inputShape; - std::tie(inType, inputShape) = this->GetParam(); + ov::AnyMap additionalConfig; + std::tie(inType, inputShape, additionalConfig) = this->GetParam(); targetDevice = ov::test::utils::DEVICE_CPU; init_input_shapes({inputShape}); @@ -110,6 +124,15 @@ TEST_P(RemoveUselessConvertCPUTest, CompareWithRefs) { CheckNumberOfNodesWithType(compiledModel, "Convert", 0); } +using RemoveUselessFP16ConvertCPUTest = RemoveUselessBF16ConvertCPUTest; +TEST_P(RemoveUselessFP16ConvertCPUTest, CompareWithRefs) { + auto implType = deduce_expected_precision(ov::element::f16, configuration); + selectedType = makeSelectedTypeStr("ref", implType); + run(); + CheckNumberOfNodesWithTypes(compiledModel, {"Convert", "Subgraph"}, 0); + CheckPluginRelatedResults(compiledModel, "StridedSlice"); +} + namespace { const std::vector inputShapes = { // dynamic batch @@ -121,12 +144,20 @@ const std::vector inputShapes = { INSTANTIATE_TEST_SUITE_P(smoke_RemoveConvert, RemoveUselessBF16ConvertCPUTest, - ::testing::Combine(::testing::Values(ElementType::bf16), ::testing::ValuesIn(inputShapes)), + ::testing::Combine(::testing::Values(ElementType::bf16), ::testing::ValuesIn(inputShapes), + ::testing::Values(cpu_bf16_plugin_config)), RemoveUselessBF16ConvertCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_RemoveConvert, + RemoveUselessFP16ConvertCPUTest, + ::testing::Combine(::testing::Values(ElementType::f16), ::testing::ValuesIn(inputShapes), + ::testing::Values(cpu_f16_plugin_config)), + RemoveUselessFP16ConvertCPUTest::getTestCaseName); + INSTANTIATE_TEST_SUITE_P(smoke_RemoveConvert, RemoveUselessConvertCPUTest, - ::testing::Combine(::testing::Values(ElementType::f32), ::testing::Values(inputShapes[0])), + ::testing::Combine(::testing::Values(ElementType::f32), ::testing::Values(inputShapes[0]), + ::testing::Values(empty_plugin_config)), RemoveUselessConvertCPUTest::getTestCaseName); } // namespace } // namespace test diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index 017e97bfac874d..70dd06ecfb44d9 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -533,6 +533,13 @@ std::vector disabledTestPatterns() { // Issue: 142465 retVector.emplace_back(R"(smoke_Reduce_MultiAxis_4D_fusing_CPU/ReduceCPULayerTest.CompareWithRefs.*INFERENCE_PRECISION_HINT=f16.*)"); retVector.emplace_back(R"(smoke_Reduce_MultiAxis_5D_fusing_CPU/ReduceCPULayerTest.CompareWithRefs.*INFERENCE_PRECISION_HINT=f16.*)"); + // Issue: 143852 + retVector.emplace_back(R"((smoke|nightly)_FC_3D_FP16/.*_Fused=Multiply\(PerChannel\).*)"); + retVector.emplace_back(R"((smoke|nightly)_MM_Brgemm_Static_FP16.*TS=\(\(55\.12\)\).*_Fused=Multiply\(PerChannel\).*)"); + retVector.emplace_back(R"(smoke_MM_Dynamic_Fusing_FP16/.*TS=\(\(16\.12\)_\(33\.7\)_\(16\.12\)\).*_Fused=Multiply\(PerChannel\).*)"); + retVector.emplace_back(R"(smoke_MM_Brgemm_Dynamic_Fusing_FP16/.*TS=\(\(16\.12\)_\(33\.7\)_\(16\.12\)\).*_Fused=Multiply\(PerChannel\).*)"); + retVector.emplace_back(R"(smoke_Conv_.*_FP16/.*_Fused=PRelu1D\.Multiply\(PerChannel\)\.Add\(PerChannel\).*)"); + retVector.emplace_back(R"(smoke_Conv_Sum_Broadcast_FP16/ConvSumInPlaceTest.*Relu\.Multiply\(PerChannel\)\.Add\(PerChannel\).*)"); } return retVector; diff --git a/src/plugins/intel_cpu/tests/functional/utils/arm/filter_cpu_info.cpp b/src/plugins/intel_cpu/tests/functional/utils/arm/filter_cpu_info.cpp index 3bb42fe47e0615..2d348b323219dd 100644 --- a/src/plugins/intel_cpu/tests/functional/utils/arm/filter_cpu_info.cpp +++ b/src/plugins/intel_cpu/tests/functional/utils/arm/filter_cpu_info.cpp @@ -38,4 +38,8 @@ std::vector filterCPUInfoForDevice(const std::vector filterCPUInfoForDeviceWithFP16(const std::vector& allParams) { + std::vector resCPUParams; + return resCPUParams; +} } // namespace CPUTestUtils diff --git a/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.cpp b/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.cpp index f3fde176331885..fab8365bc28e49 100644 --- a/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.cpp +++ b/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.cpp @@ -8,6 +8,7 @@ #include "cpu_test_utils.hpp" #include "openvino/core/type/element_type.hpp" +#include "openvino/runtime/system_conf.hpp" #include "transformations/rt_info/primitives_priority_attribute.hpp" #include "utils/general_utils.h" #include "utils/rt_info/memory_formats_attribute.hpp" @@ -471,4 +472,74 @@ void CheckNumberOfNodesWithType(const ov::CompiledModel& compiledModel, size_t expectedCount) { CheckNumberOfNodesWithTypes(compiledModel, {nodeType}, expectedCount); } + + +// deduce the actual precision of the operation given the ngraph level operation precision and the plugin config +ov::element::Type +CPUTestsBase::deduce_expected_precision(const ov::element::Type& opPrecision, + const ov::AnyMap& configuration) { +#if defined(OPENVINO_ARCH_ARM) || defined(OPENVINO_ARCH_ARM64) + return opPrecision; +#endif +#if defined(OPENVINO_ARCH_RISCV64) + return opPrecision; +#endif +#if defined(OPENVINO_ARCH_X86_64) + // if is not float + if (!opPrecision.is_real()) { + return opPrecision; + } + ov::element::Type inferencePrecision = ov::element::f32; + bool inferencePrecisionSetExplicitly = false; + const std::string precisionKey = ov::hint::inference_precision.name(); + const auto& it = configuration.find(precisionKey); + if (it != configuration.end()) { + auto inferencePrecisionConfig = it->second.as(); + inferencePrecisionSetExplicitly = true; + // TODO also need to check (dnnl::impl::cpu::x64::avx2_vnni_2) + if ((inferencePrecisionConfig == ov::element::bf16 && ov::with_cpu_x86_avx512_core()) + || (inferencePrecisionConfig == ov::element::f16 && ov::with_cpu_x86_avx512_core_fp16()) + || (inferencePrecisionConfig == ov::element::f32) + || (inferencePrecisionConfig == ov::element::undefined)) { + inferencePrecision = inferencePrecisionConfig; + } + } + if (!inferencePrecisionSetExplicitly) { + const std::string executionModeKey = ov::hint::execution_mode.name(); + const auto& configIt = configuration.find(executionModeKey); + if (configIt != configuration.end() && configIt->second.as() == ov::hint::ExecutionMode::PERFORMANCE) { + inferencePrecision = ov::element::f32; + if (ov::with_cpu_x86_bfloat16()) { + inferencePrecision = ov::element::bf16; + } + } else { + inferencePrecision = ov::element::undefined; + } + } + + ov::element::Type deducedType = opPrecision; + // enforceInferPrecision stage + if (inferencePrecision == ov::element::bf16) { + deducedType = ov::with_cpu_x86_avx512_core() ? ov::element::bf16 : ov::element::f32; + } + + // ngraph transform pipeline stage + if (inferencePrecision == ov::element::f16) { + if (deducedType == ov::element::f32) { + deducedType = ov::element::f16; + } + } + if (deducedType == ov::element::bf16) { + deducedType = ov::with_cpu_x86_avx512_core() ? ov::element::bf16 : ov::element::f32; + } else if (deducedType == ov::element::f16) { + if (inferencePrecision != ov::element::f16 && inferencePrecision != ov::element::undefined) { + deducedType = ov::element::f32; + } + } else { + deducedType = ov::element::f32; + } + + return deducedType; +#endif +} } // namespace CPUTestUtils diff --git a/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.hpp b/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.hpp index 2a15f89c72ad85..792e84a050f095 100644 --- a/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.hpp +++ b/src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.hpp @@ -4,6 +4,7 @@ #pragma once +#include "openvino/core/any.hpp" #include "openvino/runtime/compiled_model.hpp" #include "openvino/runtime/exec_model_info.hpp" #include "openvino/runtime/system_conf.hpp" @@ -128,6 +129,8 @@ class CPUTestsBase { const std::vector& priority); // TODO: change to setter method static std::string makeSelectedTypeStr(std::string implString, ov::element::Type_t elType); + static ov::element::Type deduce_expected_precision(const ov::element::Type& opPrecision, + const ov::AnyMap& configuration); void updateSelectedType(const std::string& primitiveType, const ov::element::Type netType, const ov::AnyMap& config); @@ -171,8 +174,9 @@ class CPUTestsBase { // common parameters const auto emptyCPUSpec = CPUSpecificParams{{}, {}, {}, {}}; const ov::AnyMap empty_plugin_config{}; -const ov::AnyMap cpu_bf16_plugin_config = {{ov::hint::inference_precision(ov::element::bf16)}}; const ov::AnyMap cpu_f16_plugin_config = {{ov::hint::inference_precision(ov::element::f16)}}; +const ov::AnyMap cpu_bf16_plugin_config = {{ov::hint::inference_precision(ov::element::bf16)}}; +const ov::AnyMap cpu_f32_plugin_config = {{ov::hint::inference_precision(ov::element::f32)}}; // utility functions std::vector filterCPUSpecificParams(const std::vector& paramsVector); diff --git a/src/plugins/intel_cpu/tests/functional/utils/filter_cpu_info.hpp b/src/plugins/intel_cpu/tests/functional/utils/filter_cpu_info.hpp index 8a92dea80575ab..d8f9f6ea1b3470 100644 --- a/src/plugins/intel_cpu/tests/functional/utils/filter_cpu_info.hpp +++ b/src/plugins/intel_cpu/tests/functional/utils/filter_cpu_info.hpp @@ -11,4 +11,5 @@ namespace CPUTestUtils { std::vector filterCPUInfo(const std::vector& CPUParams); std::vector filterCPUInfoForArch(const std::vector& CPUParams); std::vector filterCPUInfoForDevice(const std::vector& CPUParams); +std::vector filterCPUInfoForDeviceWithFP16(const std::vector& allParams); } // namespace CPUTestUtils diff --git a/src/plugins/intel_cpu/tests/functional/utils/x64/filter_cpu_info.cpp b/src/plugins/intel_cpu/tests/functional/utils/x64/filter_cpu_info.cpp index cd30f756db66db..f0c663375cee5b 100644 --- a/src/plugins/intel_cpu/tests/functional/utils/x64/filter_cpu_info.cpp +++ b/src/plugins/intel_cpu/tests/functional/utils/x64/filter_cpu_info.cpp @@ -63,4 +63,22 @@ std::vector filterCPUInfoForDevice(const std::vector filterCPUInfoForDeviceWithFP16(const std::vector& allParams) { + std::vector specificParams; + if (!ov::with_cpu_x86_avx512_core_fp16()) { + return specificParams; + } + std::copy_if(allParams.begin(), allParams.end(), std::back_inserter(specificParams), [](const CPUSpecificParams& item) { + const auto &selected = std::get<3>(item); + bool isValid = false; + if (selected.find("avx512") != std::string::npos) { + isValid = true; + } + if ((!ov::with_cpu_x86_avx512_core_amx_fp16()) && selected.find("amx") != std::string::npos) { + isValid = false; + } + return isValid; + }); + return specificParams; +} } // namespace CPUTestUtils From cc29db8ab590830132d78ed3c4cc576cf57b2326 Mon Sep 17 00:00:00 2001 From: Andrzej Kopytko Date: Mon, 1 Jul 2024 13:08:07 +0200 Subject: [PATCH 02/15] [DOCS] fixed footer file (#25306) ### Details: - *item1* - *...* ### Tickets: - *ticket-id* --- docs/sphinx_setup/_static/html/footer.html | 30 +++++----------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/docs/sphinx_setup/_static/html/footer.html b/docs/sphinx_setup/_static/html/footer.html index 61278050b69173..2194cf3cec66b0 100644 --- a/docs/sphinx_setup/_static/html/footer.html +++ b/docs/sphinx_setup/_static/html/footer.html @@ -12,17 +12,13 @@ .footer-link-list a { float: left; + margin-right: 30px; text-decoration: none; background-color: transparent; } .footer-link-list a:nth-child(1) { - margin-right: 50px; - } - - .footer-link-list a:nth-child(0) { - clear: both; - margin-right: 150px; + margin-right: 20px; } .footer-contents { @@ -31,6 +27,8 @@ box-sizing: border-box; font-family: sans-serif; padding-top: 10px; + margin-top: 0; + margin-bottom: 0; line-height: 1.65; margin-left: 0px; margin-right: 0px; @@ -67,23 +65,9 @@ } ul { - list-style-type: none; - margin: 0; - padding: 0; - overflow: hidden; + margin-bottom: 0px; } - li { - float: left; - } - - li a { - display: block; - padding-left: 12px; - padding-right: 12px; - padding-top: 1px; - text-decoration: none !important; - } .no-bullets { list-style-type: none; } @@ -155,13 +139,13 @@ alt="glossary">Glossary
  • Case Studies
    • - -

    Intel technologies may require enabled hardware, software or service activation. // No product or component can be absolutely secure. // Your costs and results may vary. // Performance varies by use, configuration and other factors. // See our complete legal Notices and Disclaimers. // Intel is committed to respecting human rights and avoiding causing or contributing to adverse impacts on human rights. See Intel’s Global Human Rights Principles. Intel’s products and software are intended only to be used in applications that do not cause or contribute to adverse impacts on human rights.

    +

    Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or + its subsidiaries. Other names and brands may be claimed as the property of others.

    \ No newline at end of file From 5480d4148abf6d37f142a00992a4a4d0083bba94 Mon Sep 17 00:00:00 2001 From: Alicja Miloszewska Date: Mon, 1 Jul 2024 13:39:12 +0200 Subject: [PATCH 03/15] [OV JS] Fix hardcoded error msg (#25272) ### Details: - Each error msg concerned the 'compiledModelSync' method. - A new parameter with method name was added to `get_parameters_error_msg` --------- Co-authored-by: Vishniakov Nikolai --- src/bindings/js/node/src/core_wrap.cpp | 8 ++------ src/bindings/js/node/src/type_validation.cpp | 4 ++-- src/bindings/js/node/tests/basic.test.js | 4 ++-- src/bindings/js/node/tests/read_model.test.js | 7 +++++++ 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/bindings/js/node/src/core_wrap.cpp b/src/bindings/js/node/src/core_wrap.cpp index 8264bff0c1a103..33350056fc443c 100644 --- a/src/bindings/js/node/src/core_wrap.cpp +++ b/src/bindings/js/node/src/core_wrap.cpp @@ -86,9 +86,7 @@ Napi::Value CoreWrap::read_model_sync(const Napi::CallbackInfo& info) { } else if (ov::js::validate(info, allowed_signatures)) { model = _core.read_model(info[0].ToString()); } else { - const auto error_message = ov::js::get_parameters_error_msg(info, allowed_signatures); - - OPENVINO_THROW(error_message); + OPENVINO_THROW("'readModelSync'", ov::js::get_parameters_error_msg(info, allowed_signatures)); } return ModelWrap::wrap(info.Env(), model); @@ -168,10 +166,8 @@ Napi::Value CoreWrap::compile_model_sync_dispatch(const Napi::CallbackInfo& info return compile_model_sync(info, info[0].ToObject(), info[1].ToString(), config); } + OPENVINO_THROW("'compileModelSync'", ov::js::get_parameters_error_msg(info, allowed_signatures)); - const auto error_message = ov::js::get_parameters_error_msg(info, allowed_signatures); - - OPENVINO_THROW(error_message); } catch (std::exception& e) { reportError(info.Env(), e.what()); diff --git a/src/bindings/js/node/src/type_validation.cpp b/src/bindings/js/node/src/type_validation.cpp index 737aba21c14f27..4852e0e4051c2a 100644 --- a/src/bindings/js/node/src/type_validation.cpp +++ b/src/bindings/js/node/src/type_validation.cpp @@ -147,8 +147,8 @@ bool validate_value(const Napi::Env& env, const Napi::Value& value) } std::string get_parameters_error_msg(const Napi::CallbackInfo& info, std::vector& allowed_signatures) { - return "Method 'compileModelSync' called with incorrect parameters.\nProvided signature: " + - js::get_current_signature(info) + " \nAllowed signatures:\n- " + ov::util::join(allowed_signatures, "\n- "); + return " method called with incorrect parameters.\nProvided signature: " + js::get_current_signature(info) + + " \nAllowed signatures:\n- " + ov::util::join(allowed_signatures, "\n- "); } } // namespace js } // namespace ov diff --git a/src/bindings/js/node/tests/basic.test.js b/src/bindings/js/node/tests/basic.test.js index e95f76a6470609..ba3e585e40ce7d 100644 --- a/src/bindings/js/node/tests/basic.test.js +++ b/src/bindings/js/node/tests/basic.test.js @@ -75,7 +75,7 @@ describe('Core.compileModelSync()', () => { }); it('compileModelSync(model, device, config) throws when config is a string', () => { - const expectedMsg = ("Method 'compileModelSync' called with incorrect parameters.\n" + + const expectedMsg = ("'compileModelSync' method called with incorrect parameters.\n" + 'Provided signature: (object, string, string) \n' + 'Allowed signatures:\n' + '- (string, string)\n' + @@ -97,7 +97,7 @@ describe('Core.compileModelSync()', () => { }); it('compileModelSync(model) throws if the number of arguments is invalid', () => { - const expectedMsg = ("Method 'compileModelSync' called with incorrect parameters.\n" + + const expectedMsg = ("'compileModelSync' method called with incorrect parameters.\n" + 'Provided signature: (object) \n' + 'Allowed signatures:\n' + '- (string, string)\n' + diff --git a/src/bindings/js/node/tests/read_model.test.js b/src/bindings/js/node/tests/read_model.test.js index 407627c5ae9cb0..731001fbb861b7 100644 --- a/src/bindings/js/node/tests/read_model.test.js +++ b/src/bindings/js/node/tests/read_model.test.js @@ -27,6 +27,13 @@ describe('Core.readModeSync', () => { assert.equal(model.inputs.length, 1); }); + it('readModeSync throws', () => { + assert.throws( + () => core.readModelSync(core), + /'readModelSync' method called with incorrect parameters./, + ) + }); + it('readModeSync(modelUint8ArrayBuffer, weightsUint8ArrayBuffer) ', () => { const model = core.readModelSync( new Uint8Array(modelFile.buffer), From c31ecf4a5cc988266e0b10a583aebae8949943e9 Mon Sep 17 00:00:00 2001 From: Karol Blaszczak Date: Mon, 1 Jul 2024 14:32:51 +0200 Subject: [PATCH 04/15] [DOCS] tiny fix for benchmarks master (#25302) --- .../generative-ai-performance.rst | 13 ++++++++++--- .../openvino_sphinx_theme/templates/navbar-nav.html | 7 +++++++ .../_static/{ => download}/llm_models.csv | 3 +-- 3 files changed, 18 insertions(+), 5 deletions(-) rename docs/sphinx_setup/_static/{ => download}/llm_models.csv (92%) diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst index 03588eeb441f38..73c9f255ff3dbb 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst @@ -1,8 +1,15 @@ Most Efficient Large Language Models for AI PC ============================================== -The table below lists key performance indicators for a selection of Large Language Models -running on an Intel® Core™ Ultra 7-165H based system. +This page is regularly updated to help you identify the best-performing LLMs on the +Intel® Core™ Ultra processor family and AI PCs. + +The table below lists the key performance indicators for a selection of Large Language Models, +running on an Intel® Core™ Ultra 7-165H based system, on built-in GPUs. + +For complete information on the system config, see: +`Hardware Platforms [PDF] `__ + .. raw:: html @@ -54,7 +61,7 @@ running on an Intel® Core™ Ultra 7-165H based system. :class: modeldata stripe :name: supportedModelsTable :header-rows: 1 - :file: ../../_static/llm_models.csv + :file: ../../_static/download/llm_models.csv This page is regularly updated to help you identify the best-performing LLMs on the diff --git a/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/navbar-nav.html b/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/navbar-nav.html index faba6fc1538a62..8d428571f18b12 100644 --- a/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/navbar-nav.html +++ b/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/navbar-nav.html @@ -31,6 +31,13 @@ +
  • + + Performance Benchmarks + +
    • +
  • GitHub diff --git a/docs/sphinx_setup/_static/llm_models.csv b/docs/sphinx_setup/_static/download/llm_models.csv similarity index 92% rename from docs/sphinx_setup/_static/llm_models.csv rename to docs/sphinx_setup/_static/download/llm_models.csv index 28b730c6c7dd1c..28ac9f2c79ec83 100644 --- a/docs/sphinx_setup/_static/llm_models.csv +++ b/docs/sphinx_setup/_static/download/llm_models.csv @@ -19,5 +19,4 @@ Mistral-7b-v0.1,10.5,4462,7242,1007,128,INT4,1,1,PT, Zephyr-7b-beta,10.5,4500,7039,1024,128,INT4,1,1,PT, Qwen1.5-7b-chat,9.9,4318,7034,1024,128,INT4,1,1,PT, Baichuan2-7b-chat,9.8,4668,6724,1024,128,INT4,1,1,PT, -Qwen-7b-chat,9.0,5141,6996,1024,128,INT4,1,1,PT, -Vicuna-7b-v1.5,0.0,3982,7022,1024,128,INT4,1,1,PT, \ No newline at end of file +Qwen-7b-chat,9.0,5141,6996,1024,128,INT4,1,1,PT, \ No newline at end of file From 639697a5060e20800d59b3d921a82628e529a80d Mon Sep 17 00:00:00 2001 From: Yury Gaydaychuk Date: Mon, 1 Jul 2024 14:45:02 +0200 Subject: [PATCH 05/15] [Commit Slider] AC / e2e compatibility (#24410) ### Details: - *tool - specific substitutions moved to config* --- .../commit_slider/tests/commit_slider_test.py | 70 +++++++++++++++- .../tools/commit_slider/utils/cfg_manager.py | 51 ++++++++++++ .../utils/cfg_samples/AC_for_CI.json | 83 +++++++++++++++++++ .../utils/cfg_samples/bm_output.json | 7 +- .../utils/cfg_samples/e2e_for_CI.json | 67 +++++++++++++++ .../tools/commit_slider/utils/helpers.py | 21 ++--- 6 files changed, 281 insertions(+), 18 deletions(-) create mode 100644 src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/AC_for_CI.json create mode 100644 src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py index ce4abfcdc277e5..2a5125870ef4dd 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py @@ -142,7 +142,7 @@ def testBmSteppedBreak2(self): ) @skip_commit_slider_devtest - def testForsubstitutionRule(self): + def testForMapSubstitutionRule(self): from utils.helpers import applySubstitutionRules cfg = { "serviceConfig": { @@ -228,6 +228,74 @@ def applyByRef(cfg: map, rules: list, substitution: str): "{commitHash1} is unchanged" ) + @skip_commit_slider_devtest + def testForStaticSubstitutionRule(self): + from utils.helpers import applySubstitutionRules + cfg = { + "serviceConfig": { + "previousKey": "previousValue" + }, + "wrongDst": "{pathOne} is unchanged", + "dst": { + "complex": { + "path": [ + "{pathOne} is natural number", + "{pathTwo} is natural number", + "{pathOne} is not {pathTwo}" + ] + } + }, + "src": { + "complex": { + "path": { + "one": "1", + "two": "2" + } + } + } + } + rules = [ + { + "name": "testRule1", + "enabled": True, + "type": "static", + "placeholder": "pathOne", + "from": "$.src.complex.path.one", + "to": "$.dst.complex.path" + }, + { + "name": "testRule2", + "enabled": True, + "type": "static", + "placeholder": "pathTwo", + "from": "$.src.complex.path.two", + "to": "$.dst.complex.path" + } + ] + def applyByRef(cfg: map, rules: list, substitution: str): + applySubstitutionRules(cfg, rules, substitution) + + applyByRef(cfg, rules, "mustBeIgnored") + + # assert substitutions + self.assertEqual( + cfg["dst"]["complex"]["path"][0], + "1 is natural number" + ) + + self.assertEqual( + cfg["dst"]["complex"]["path"][1], + "2 is natural number" + ) + self.assertEqual( + cfg["dst"]["complex"]["path"][2], + "1 is not 2" + ) + self.assertEqual( + cfg["wrongDst"], + "{pathOne} is unchanged" + ) + @skip_commit_slider_devtest def testForDeepUpdate(self): from utils.helpers import deepMapUpdate diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py index 5982cda733ea88..85d1b7754946bc 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py @@ -9,14 +9,26 @@ class CfgManager(): def __init__(self, cfg) -> None: self.cfg = cfg + @staticmethod + def multistepStrFormat(input: str, placeholder: str, substitution: str): + return input.replace( + '{}{}{}'.format('{', placeholder, '}'), + substitution + ) + def applyTemplate(self): if not "template" in self.cfg: return self.cfg logPath = self.cfg["logPath"] tmplName = self.cfg["template"]["name"] fullCfg = {} + # todo: generalize tmplcfg generator if tmplName == "bm_simple": fullCfg = self.generatebmSimpleTemplate() + elif tmplName == "e2e": + fullCfg = self.generateE2ETemplate() + elif tmplName == "bm_functional": + fullCfg = self.generatebmFunctionalTemplate() else: raise Exception( "Unknown template '{}'".format(tmplName) @@ -32,6 +44,45 @@ def readJsonTmpl(self, tmplFileName: str): tmplJSON = json.load(cfgFile) return tmplJSON + def generateE2ETemplate(self): + tmpl = self.cfg["template"] + tmpJSON = self.readJsonTmpl("e2e_for_CI.json") + + if "errorPattern" in tmpl and\ + "precommitPath" in tmpl and\ + "testCmd" in tmpl: + tmpJSON["runConfig"]["stopPattern"] = tmpl["errorPattern"] + tmpJSON["dlbConfig"]["commonPath"] = tmpl["precommitPath"] + tmpJSON["cachedPathConfig"]["commonPath"] = tmpl["precommitPath"] + tmpJSON["appCmd"] = CfgManager.multistepStrFormat( + tmpJSON["appCmd"], + tmpl["appCmd"], + "testCmd" + ) + else: + raise("Template is incomplete.") + subPath = "private_linux_manylinux2014_release/" + if "subPath" in tmpl: + subPath = tmpl["subPath"] + tmpJSON["dlbConfig"]["subPath"] = subPath + tmpJSON["cachedPathConfig"]["subPath"] = subPath + + return tmpJSON + + def generatebmFunctionalTemplate(self): + tmpl = self.cfg["template"] + tmpJSON = self.readJsonTmpl("bm_output.json") + stopPattern = "stopPattern" + if "appCmd" in tmpl: + tmpJSON["appCmd"] = tmpl["appCmd"] + else: + raise("No 'appcmd' in template") + if stopPattern in tmpl: + tmpJSON["runConfig"][stopPattern] = tmpl[stopPattern] + else: + raise("No 'stopPattern' in template") + return tmpJSON + def generatebmSimpleTemplate(self): tmpl = self.cfg["template"] tmpJSON = self.readJsonTmpl("bm_perf_for_CI.json") diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/AC_for_CI.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/AC_for_CI.json new file mode 100644 index 00000000000000..a0562532c7736d --- /dev/null +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/AC_for_CI.json @@ -0,0 +1,83 @@ +{ + "appCmd":"source {venvName}/bin/activate && cd {appPath} && {py} -m pip install --upgrade pip && {py} -m pip install openvino-dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=={wheelVersion} --find-links={precommitPath}wheels/ && {py} -m pip install -r requirements_dev.txt && {testCmd}", + "venvCfg":{ + "venvEnabled":true, + "venvDir":"{workPath}/venv/", + "venvName":"tempVenv" + }, + "commandList":[ + + ], + "runConfig":{ + "mode":"checkOutput", + "traversal":"firstFailedVersion", + "stopPattern":"{ErrorPattern}" + }, + "dlbConfig":{ + "launchedAsJob":true, + "toolName":"ac", + "wheelVersionsMap":{ + + }, + "commonPath":"{precommitPath}", + "subPath":"{subPath}", + "appPath":"", + "appCmd":"" + }, + "cachedPathConfig":{ + "enabled":true, + "scheme":"mandatory", + "passCmdList":false, + "changeAppPath":false, + "commonPath":"{precommitPath}", + "subPath":"{subPath}", + "cashMap":{ + + } + }, + "substitutionRules":[ + { + "name":"precommitPath", + "enabled":true, + "type":"map", + "placeholder":"precommitPath", + "from":"$.cachedPathConfig.cashMap", + "to":"$.appCmd" + }, + { + "name":"wheelVersion", + "enabled":true, + "type":"map", + "placeholder":"wheelVersion", + "from":"$.dlbConfig.wheelVersionsMap", + "to":"$.appCmd" + }, + { + "name":"command line for client application", + "enabled":true, + "type":"static", + "placeholder":"appCmd", + "from":"$.dlbConfig.appCmd", + "to":"$.appCmd" + }, + { + "name":"path to AC requirements", + "enabled":true, + "type":"static", + "placeholder":"toolPath", + "from":"$.dlbConfig.toolPath", + "to":"$.appCmd" + } + ], + "subscriptions":[ + { + "name":"wheelPathsMap", + "enabled":true + }, + { + "name":"wheelVersionsMap", + "enabled":true + } + ], + "verboseOutput":false + } \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json index 3ec3d96d5963d8..99fce43d90d2ad 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json @@ -1,5 +1,5 @@ { - "appCmd" : "./benchmark_app -m -d CPU -t 10", + "appCmd" : "{appCmd}", "makeCmd" : "cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=OFF -DTHREADING=TBB -DENABLE_INTEL_GPU=OFF -DENABLE_SAMPLES=ON -DENABLE_TESTS=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_CPU_DEBUG_CAPS=OFF -DENABLE_DEBUG_CAPS=OFF -DENABLE_OPENVINO_DEBUG=OFF -DCMAKE_CXX_FLAGS=-Wno-deprecated -DCMAKE_C_FLAGS=-Wno-deprecated -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_FLAGS=-Wno-deprecated-declarations ..", "runConfig" : { "commitList" : { @@ -7,6 +7,7 @@ }, "mode" : "checkOutput", "traversal" : "firstFailedVersion", - "stopPattern" : "(.)*(.)*" - } + "stopPattern" : "(.)*{bm_error_message}(.)*" + }, + "extendBuildCommand":true } \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json new file mode 100644 index 00000000000000..484e80cabe81dc --- /dev/null +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json @@ -0,0 +1,67 @@ +{ + "appCmd":"source {venvName}/bin/activate && cd {appPath} && {py} -m pip install --upgrade pip && {py} -m pip install openvino-dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=={wheelVersion} --find-links={precommitPath}wheels/ && {py} -m pip install -r requirements_dev.txt && {testCmd}", + "venvCfg":{ + "venvEnabled":true, + "venvDir":"{workPath}/venv/", + "venvName":"tempVenv" + }, + "commandList":[ + + ], + "runConfig":{ + "mode":"checkOutput", + "traversal":"firstFailedVersion", + "stopPattern":"{ErrorPattern}" + }, + "dlbConfig":{ + "launchedAsJob":true, + "toolName":"e2e", + "wheelVersionsMap":{ + + }, + "commonPath":"{precommitPath}", + "subPath":"{subPath}", + "appPath":"", + "appCmd":"" + }, + "cachedPathConfig":{ + "enabled":true, + "scheme":"mandatory", + "passCmdList":false, + "changeAppPath":false, + "commonPath":"{precommitPath}", + "subPath":"{subPath}", + "cashMap":{ + + } + }, + "substitutionRules":[ + { + "name":"precommitPath", + "enabled":true, + "type":"map", + "placeholder":"precommitPath", + "from":"$.cachedPathConfig.cashMap", + "to":"$.appCmd" + }, + { + "name":"wheelVersion", + "enabled":true, + "type":"map", + "placeholder":"wheelVersion", + "from":"$.dlbConfig.wheelVersionsMap", + "to":"$.appCmd" + } + ], + "subscriptions":[ + { + "name":"wheelPathsMap", + "enabled":true + }, + { + "name":"wheelVersionsMap", + "enabled":true + } + ], + "verboseOutput":false + } \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py b/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py index c705e8b362c56c..8185946c1ae54a 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py @@ -290,12 +290,9 @@ def fetchAppOutput(cfg, commit): for item in [ {"src": cfg["venvCfg"]["venvName"], "dst": "venvName"}, {"src": cfg["appPath"], "dst": "appPath"}, - {"src": sys.executable, "dst": "py"}, - # for AC case - {"src": cfg["dlbConfig"]["appCmd"], "dst": "appCmd"}, - {"src": cfg["dlbConfig"]["toolPath"], "dst": "toolPath"} + {"src": sys.executable, "dst": "py"} ]: - appCmd = multistepStrFormat( + appCmd = CfgManager.multistepStrFormat( appCmd, item["dst"], item["src"] @@ -646,7 +643,8 @@ def formatJSON(content, formatLambda): return content def applySubstitutionRules(cfg: map, rules: list, commit: str=None): - # if commit is None, the rule is considered as static, + # if commit is None or rule['type'] == 'static', + # the rule is considered as static, # substitution proceeds as simple string replacing serviceCfg = cfg["serviceConfig"] @@ -682,14 +680,15 @@ def applySubstitutionRules(cfg: map, rules: list, commit: str=None): srcPos = srcPos[item] for item in pathToDst: dstPos = dstPos[item] + ruleIsStatic = True if rule["type"] == "static" else False dstPos = formatJSON( dstPos, lambda content: - multistepStrFormat( + CfgManager.multistepStrFormat( content, rule["placeholder"], getMapValueByShortHash(srcPos, commit)\ - if commit is not None\ + if commit is not None and not ruleIsStatic\ else srcPos ) ) @@ -704,12 +703,6 @@ def getMapValueByShortHash(map: dict, commit: str): commit, map.keys() )) -def multistepStrFormat(input: str, placeholder: str, substitution: str): - return input.replace( - '{}{}{}'.format('{', placeholder, '}'), - substitution - ) - def deepMapUpdate(content: map, path: list, substitution): if not path: return substitution From 1d7daae78488745324599e494b3b9324cda39700 Mon Sep 17 00:00:00 2001 From: Gorokhov Dmitriy Date: Mon, 1 Jul 2024 18:04:10 +0400 Subject: [PATCH 06/15] [CPU] FullyConnected acceleration for BF16 compressed weights (#24633) ### Details: - Improved LLMs latency for models with BF16 weights (by extending compressed IP kernel on BF16 data type) - Improved compilation time for models with BF16 weights on avx2 systems (enabled jit reorder for BF16 data type) OneDNN fork PR: https://github.com/openvinotoolkit/oneDNN/pull/250 ### Tickets: - *[CVS-142057](https://jira.devtools.intel.com/browse/CVS-142057)* --------- Co-authored-by: Maksim Kutakov --- .../src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp | 2 +- src/plugins/intel_cpu/thirdparty/onednn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp index 432a78419e9497..ef1e127f44f636 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp @@ -122,7 +122,7 @@ bool DnnlFCPrimitive::useWeightsDecompressionImpl(const ov::element::Type inputT // f16c kernel saves memory footprint with additional decompression computational overhead // which is only meaningful on LLM with small batch-size. // TODO: fall-back to use f32 weights on large batch-size - if (inputType == f32 && weightsType == f16) + if (inputType == f32 && one_of(weightsType, f16, bf16)) return true; } } diff --git a/src/plugins/intel_cpu/thirdparty/onednn b/src/plugins/intel_cpu/thirdparty/onednn index a320d02d6e733c..d19b5d85880ba9 160000 --- a/src/plugins/intel_cpu/thirdparty/onednn +++ b/src/plugins/intel_cpu/thirdparty/onednn @@ -1 +1 @@ -Subproject commit a320d02d6e733c775724901675cbc8944391459d +Subproject commit d19b5d85880ba917552e57183670f7a3e2fcd086 From a3d2b6ab8562b0f33272a21b0f44d942e145ae36 Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Mon, 1 Jul 2024 16:17:10 +0200 Subject: [PATCH 07/15] [PT FE] Support setting types/shapes for nested structures (#25064) ### Details: - *Some PyTorch models may have `dicts`, `lists`, `tuples` as inputs. We unpack them on `normalize` step in PT FE, but this means that those unpacked inputs do not exist before conversion and `convert_model` can't set shapes/types for them. This PR adds ability for `convert_model` to set shapes/types to any arbitrary input obtained by index or name.* ### Tickets: - *CVS-142939* --- .../openvino/frontend/pytorch/fx_decoder.py | 1 + .../openvino/frontend/pytorch/ts_decoder.py | 3 + .../src/pyopenvino/frontend/input_model.cpp | 12 ++ .../include/openvino/frontend/input_model.hpp | 5 + src/frontends/common/src/input_model.cpp | 7 + .../onnx/frontend/src/input_model.cpp | 4 + .../onnx/frontend/src/input_model.hpp | 1 + src/frontends/paddle/src/input_model.cpp | 4 + src/frontends/paddle/src/input_model.hpp | 1 + src/frontends/pytorch/src/frontend.cpp | 73 ++++++- src/frontends/pytorch/src/input_model.cpp | 56 ++++- src/frontends/pytorch/src/input_model.hpp | 6 + src/frontends/pytorch/src/place.cpp | 26 +++ src/frontends/pytorch/src/place.hpp | 13 +- .../pytorch/src/translate_session.cpp | 13 +- src/frontends/tensorflow/src/input_model.cpp | 4 + src/frontends/tensorflow/src/input_model.hpp | 1 + .../tensorflow_lite/src/input_model.cpp | 4 + .../tensorflow_lite/src/input_model.hpp | 1 + .../ovc_python_api_tests/test_pytorch.py | 202 +++++++++++++++--- tools/ovc/openvino/tools/ovc/convert_impl.py | 3 +- .../tools/ovc/moc_frontend/pipeline.py | 84 ++++++-- .../moc_frontend/pytorch_frontend_utils.py | 130 +++++------ 23 files changed, 537 insertions(+), 117 deletions(-) diff --git a/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py b/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py index 596c4415868910..e8ca651ccae918 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py @@ -30,6 +30,7 @@ def __init__(self, pt_module, fx_gm=None, nodes=None, mark_node_callback=None, i self.input_shapes = input_shapes self._input_signature = [] + self._example_input = None if issubclass(type(pt_module), torch.fx.graph_module.GraphModule): diff --git a/src/bindings/python/src/openvino/frontend/pytorch/ts_decoder.py b/src/bindings/python/src/openvino/frontend/pytorch/ts_decoder.py index 29b09c6e55e04f..4fa45850d78f8b 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/ts_decoder.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/ts_decoder.py @@ -96,6 +96,7 @@ def _get_scripted_model(self, pt_module, example_inputs=None, skip_freeze=False) if isinstance(pt_module, torch.nn.Module): pt_module.eval() input_signature = None + input_parameters = None if isinstance(pt_module, torch.nn.Module) and not isinstance(pt_module, (torch.jit._trace.TopLevelTracedModule, torch.jit._script.RecursiveScriptModule)): # input params is dictionary contains input names and their signature values (type hints and default values if any) input_params = inspect.signature(pt_module.forward if hasattr( @@ -150,8 +151,10 @@ def _get_scripted_model(self, pt_module, example_inputs=None, skip_freeze=False) scripted, preserved_attrs=preserved_attrs) else: f_model = scripted + self._example_input = input_parameters["example_inputs"] if input_parameters else None else: f_model = pt_module + self._example_input = example_inputs self._input_signature = input_signature return f_model diff --git a/src/bindings/python/src/pyopenvino/frontend/input_model.cpp b/src/bindings/python/src/pyopenvino/frontend/input_model.cpp index 20315a74338316..66a41c2ed4ee44 100644 --- a/src/bindings/python/src/pyopenvino/frontend/input_model.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/input_model.cpp @@ -33,6 +33,18 @@ void regclass_frontend_InputModel(py::module m) { :rtype: openvino.frontend.Place )"); + im.def("get_place_by_input_index", + &ov::frontend::InputModel::get_place_by_input_index, + py::arg("input_idx"), + R"( + Returns a tensor place by an input index. + + :param input_idx: Index of model input. + :type input_idx: int + :return: Tensor place corresponding to specified input index or nullptr. + :rtype: openvino.frontend.Place + )"); + im.def("get_place_by_operation_name", &ov::frontend::InputModel::get_place_by_operation_name, py::arg("operation_name"), diff --git a/src/frontends/common/include/openvino/frontend/input_model.hpp b/src/frontends/common/include/openvino/frontend/input_model.hpp index fab6d3d3e4f40b..268bfd8625747b 100644 --- a/src/frontends/common/include/openvino/frontend/input_model.hpp +++ b/src/frontends/common/include/openvino/frontend/input_model.hpp @@ -82,6 +82,11 @@ class FRONTEND_API InputModel { /// \return Tensor place corresponding to specified tensor name or nullptr if not exists virtual Place::Ptr get_place_by_tensor_name(const std::string& tensor_name) const; + /// \brief Returns a tensor place by an input index. + /// \param input_idx Index of model input + /// \return Tensor place corresponding to specified input index or nullptr + virtual Place::Ptr get_place_by_input_index(size_t input_idx) const; + /// \brief Returns an operation place by an operation name following framework /// conventions, or nullptr if an operation with this name doesn't exist. /// \param operation_name Name of operation diff --git a/src/frontends/common/src/input_model.cpp b/src/frontends/common/src/input_model.cpp index f725eca0f31da0..e373593d26a3d5 100644 --- a/src/frontends/common/src/input_model.cpp +++ b/src/frontends/common/src/input_model.cpp @@ -33,6 +33,13 @@ Place::Ptr InputModel::get_place_by_tensor_name(const std::string& tensor_name) FRONTEND_RETURN_STATEMENT("get_place_by_tensor_name", m_actual->get_place_by_tensor_name(tensor_name)) } +Place::Ptr InputModel::get_place_by_input_index(size_t input_idx) const { + if (!m_actual) { + return {}; + } + FRONTEND_RETURN_STATEMENT("get_place_by_input_index", m_actual->get_place_by_input_index(input_idx)) +} + Place::Ptr InputModel::get_place_by_operation_name(const std::string& operation_name) const { if (!m_actual) { return {}; diff --git a/src/frontends/onnx/frontend/src/input_model.cpp b/src/frontends/onnx/frontend/src/input_model.cpp index 9a7c6e62f5d9d2..e7cd7b6efc1d49 100644 --- a/src/frontends/onnx/frontend/src/input_model.cpp +++ b/src/frontends/onnx/frontend/src/input_model.cpp @@ -64,6 +64,10 @@ ov::frontend::Place::Ptr InputModel::get_place_by_tensor_name(const std::string& return nullptr; } +ov::frontend::Place::Ptr InputModel::get_place_by_input_index(size_t input_idx) const { + FRONT_END_NOT_IMPLEMENTED(get_place_by_input_index); +} + ov::frontend::Place::Ptr InputModel::get_place_by_operation_name(const std::string& operation_name) const { if (m_editor->is_correct_and_unambiguous_node(operation_name)) { const auto node_index = m_editor->get_node_index(EditorNode{operation_name}); diff --git a/src/frontends/onnx/frontend/src/input_model.hpp b/src/frontends/onnx/frontend/src/input_model.hpp index 54d3694f0e1106..9bf44a5672fb28 100644 --- a/src/frontends/onnx/frontend/src/input_model.hpp +++ b/src/frontends/onnx/frontend/src/input_model.hpp @@ -37,6 +37,7 @@ class InputModel : public ov::frontend::InputModel { std::vector get_inputs() const override; std::vector get_outputs() const override; ov::frontend::Place::Ptr get_place_by_tensor_name(const std::string& tensor_name) const override; + ov::frontend::Place::Ptr get_place_by_input_index(size_t input_idx) const override; ov::frontend::Place::Ptr get_place_by_operation_name(const std::string& operation_name) const override; ov::frontend::Place::Ptr get_place_by_operation_name_and_input_port(const std::string& operation_name, int input_port_index) override; diff --git a/src/frontends/paddle/src/input_model.cpp b/src/frontends/paddle/src/input_model.cpp index 51deb0637ba257..b448a0d4641561 100644 --- a/src/frontends/paddle/src/input_model.cpp +++ b/src/frontends/paddle/src/input_model.cpp @@ -612,6 +612,10 @@ Place::Ptr InputModel::get_place_by_tensor_name(const std::string& tensorName) c return _impl->get_place_by_tensor_name(tensorName); } +Place::Ptr InputModel::get_place_by_input_index(size_t input_idx) const { + FRONT_END_NOT_IMPLEMENTED(get_place_by_input_index); +} + void InputModel::override_all_outputs(const std::vector& outputs) { _impl->override_all_outputs(outputs); } diff --git a/src/frontends/paddle/src/input_model.hpp b/src/frontends/paddle/src/input_model.hpp index 6eedb6759a70dd..e401e84416455f 100644 --- a/src/frontends/paddle/src/input_model.hpp +++ b/src/frontends/paddle/src/input_model.hpp @@ -25,6 +25,7 @@ class InputModel : public ov::frontend::InputModel { std::vector get_inputs() const override; std::vector get_outputs() const override; Place::Ptr get_place_by_tensor_name(const std::string& tensorName) const override; + Place::Ptr get_place_by_input_index(size_t input_idx) const override; void override_all_outputs(const std::vector& outputs) override; void override_all_inputs(const std::vector& inputs) override; void extract_subgraph(const std::vector& inputs, const std::vector& outputs) override; diff --git a/src/frontends/pytorch/src/frontend.cpp b/src/frontends/pytorch/src/frontend.cpp index 4763ee15c91942..c54d37e28fb037 100644 --- a/src/frontends/pytorch/src/frontend.cpp +++ b/src/frontends/pytorch/src/frontend.cpp @@ -11,6 +11,7 @@ #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/util/log.hpp" +#include "place.hpp" #include "pt_framework_node.hpp" #include "transformations/common_optimizations/push_constant_to_subgraph.hpp" #include "transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.hpp" @@ -113,15 +114,37 @@ std::string pack_detailed_failure_report(const std::map& param, + const Place::Ptr& place, + const std::shared_ptr ov_model) { + const auto& pt_place = std::dynamic_pointer_cast(place); + FRONT_END_GENERAL_CHECK(pt_place, "Wrong place type."); + if (pt_place->get_element_type().is_static()) + param->set_element_type(pt_place->get_element_type()); + auto pshape = param->get_partial_shape(); + FRONT_END_GENERAL_CHECK(PartialShape::merge_into(pshape, pt_place->get_partial_shape()), + "Incompatible shape was requested for input #", + ov_model->get_parameter_index(param), + ", ", + param, + "\nOriginal shape: ", + pshape, + " update shape: ", + pt_place->get_partial_shape()); + param->set_partial_shape(pshape); +} } // namespace FrontEnd::FrontEnd() {} std::shared_ptr FrontEnd::convert(const ov::frontend::InputModel::Ptr& model) const { - FRONT_END_GENERAL_CHECK(std::dynamic_pointer_cast(model), "Invalid input model"); + auto pt_model = std::dynamic_pointer_cast(model); + FRONT_END_GENERAL_CHECK(pt_model, "Invalid input model"); std::map supported_ops = get_supported_ops(model); std::shared_ptr converted_model; { + pt_model->flush_places(); TranslateSession translate_session(model, supported_ops, m_telemetry); converted_model = translate_session.get_converted_model(); } @@ -141,6 +164,50 @@ std::shared_ptr FrontEnd::convert(const ov::frontend::InputModel::Ptr& mo } bool is_conversion_successful = unconverted_ops.size() == 0 && norm_err.empty(); FRONT_END_OP_CONVERSION_CHECK(is_conversion_successful, pack_detailed_failure_report(unconverted_ops, norm_err)); + + if (pt_model->m_requested_places.size() != 0) { + // Fake tensors mean that types were set to non-existent before conversion inputs. + // Here we resolve this. If input doesn't exist after conversion exception is raised. + auto parameters = converted_model->get_parameters(); + std::set input_names; + for (const auto& param : parameters) + for (const auto& name : param->get_output_tensor(0).get_names()) + input_names.insert(name); + const auto& inputs = pt_model->get_inputs(); + for (size_t i = 0; i < inputs.size(); i++) { + auto place = inputs[i]; + if (place->get_names().size() != 0 && input_names.find(place->get_names().at(0)) != input_names.end()) { + auto input = converted_model->input(place->get_names().at(0)); + auto param = std::dynamic_pointer_cast(input.get_node_shared_ptr()); + FRONT_END_GENERAL_CHECK(param, "Input is not a Parameter."); + update_parameter_info(param, place, converted_model); + } else { + FRONT_END_OP_CONVERSION_CHECK(i < parameters.size(), + "Type/shape was set to non-existent input. Converted model:\n", + converted_model); + update_parameter_info(parameters[i], inputs[i], converted_model); + } + } + for (const auto& fplace : pt_model->m_requested_places) { + const auto& pt_place = std::dynamic_pointer_cast(fplace); + FRONT_END_GENERAL_CHECK(pt_place, "Wrong place type."); + if (fplace->get_names().size() == 0) { + // Fake place was set by input index + auto idx = pt_place->get_input_index(); + FRONT_END_GENERAL_CHECK(idx < parameters.size(), + "Type/shape was set to non-existent input. Converted model:\n", + converted_model); + update_parameter_info(parameters[idx], fplace, converted_model); + } else { + auto input = converted_model->input(fplace->get_names().at(0)); + auto param = std::dynamic_pointer_cast(input.get_node_shared_ptr()); + FRONT_END_GENERAL_CHECK(param, "Input is not a Parameter."); + update_parameter_info(param, fplace, converted_model); + } + } + converted_model->validate_nodes_and_infer_types(); + } + return converted_model; } @@ -149,10 +216,12 @@ void FrontEnd::convert(const std::shared_ptr& partiallyConverted) const { } std::shared_ptr FrontEnd::convert_partially(const ov::frontend::InputModel::Ptr& model) const { - FRONT_END_GENERAL_CHECK(std::dynamic_pointer_cast(model), "Invalid input model"); + auto pt_model = std::dynamic_pointer_cast(model); + FRONT_END_GENERAL_CHECK(pt_model, "Invalid input model"); std::map supported_ops = get_supported_ops(model); std::shared_ptr partial_model; { + pt_model->flush_places(); TranslateSession translate_session(model, supported_ops, m_telemetry); partial_model = translate_session.get_converted_model(); } diff --git a/src/frontends/pytorch/src/input_model.cpp b/src/frontends/pytorch/src/input_model.cpp index 4adfa1669b974c..72140881c89698 100644 --- a/src/frontends/pytorch/src/input_model.cpp +++ b/src/frontends/pytorch/src/input_model.cpp @@ -49,11 +49,22 @@ std::vector InputModel::get_outputs() const { } Place::Ptr InputModel::get_place_by_tensor_name(const std::string& tensor_name) const { + if (tensor_name.empty()) + return {}; auto place_it = m_name_to_place.find(tensor_name); if (place_it != m_name_to_place.end()) { return place_it->second; + } else { + // Return fake place that can be used to change shape or type of inputs that will exist after conversion + auto place = std::make_shared(*this, tensor_name, 0); + return place; } - return nullptr; +} + +Place::Ptr InputModel::get_place_by_input_index(size_t input_idx) const { + // Return place that can be used to change shape or type of inputs that will exist after conversion + auto place = std::make_shared(*this, "", input_idx); + return place; } void InputModel::set_partial_shape(const Place::Ptr& place, const ov::PartialShape& shape) { @@ -61,6 +72,18 @@ void InputModel::set_partial_shape(const Place::Ptr& place, const ov::PartialSha "Provided place is invalid, only inputs are supported for setting shape."); auto pytorch_place = std::dynamic_pointer_cast(place); FRONT_END_GENERAL_CHECK(pytorch_place, "Only place produced by PyTorch Frontend is supported"); + if (pytorch_place->m_is_fake) { + bool is_new = true; + for (auto& p : m_requested_places) { + if (p->is_equal(pytorch_place)) { + is_new = false; + pytorch_place = std::dynamic_pointer_cast(p); + FRONT_END_GENERAL_CHECK(pytorch_place, "Only place produced by PyTorch Frontend is supported"); + } + } + if (is_new) + m_requested_places.push_back(place); + } pytorch_place->m_pshape = shape; } @@ -77,6 +100,18 @@ void InputModel::set_element_type(const Place::Ptr& place, const ov::element::Ty "Provided place is invalid, only inputs are supported for setting element type."); auto pytorch_place = std::dynamic_pointer_cast(place); FRONT_END_GENERAL_CHECK(pytorch_place, "Only place produced by PyTorch Frontend is supported"); + if (pytorch_place->m_is_fake) { + bool is_new = true; + for (auto& p : m_requested_places) { + if (p->is_equal(pytorch_place)) { + is_new = false; + pytorch_place = std::dynamic_pointer_cast(p); + FRONT_END_GENERAL_CHECK(pytorch_place, "Only place produced by PyTorch Frontend is supported"); + } + } + if (is_new) + m_requested_places.push_back(place); + } pytorch_place->m_type = type; } @@ -171,6 +206,25 @@ std::shared_ptr InputModel::get_decoder() const { return m_model_decoder; } +void InputModel::flush_places() { + auto input_places = get_inputs(); + if (m_requested_places.size() > input_places.size()) + return; + for (auto place : m_requested_places) { + auto pt_place = std::dynamic_pointer_cast(place); + if (!pt_place || pt_place->get_input_index() >= input_places.size() || pt_place->m_names.size() != 0) + return; + auto to_update_place = std::dynamic_pointer_cast(input_places[pt_place->get_input_index()]); + if (!to_update_place || !to_update_place->m_type.is_dynamic() || !to_update_place->m_pshape.rank().is_dynamic()) + return; + if (pt_place->m_type.is_static()) + to_update_place->m_type = pt_place->m_type; + if (pt_place->m_pshape.rank().is_static()) + to_update_place->m_pshape = pt_place->m_pshape; + } + m_requested_places = {}; +} + } // namespace pytorch } // namespace frontend } // namespace ov diff --git a/src/frontends/pytorch/src/input_model.hpp b/src/frontends/pytorch/src/input_model.hpp index 58d91eae5764ca..b647da29465660 100644 --- a/src/frontends/pytorch/src/input_model.hpp +++ b/src/frontends/pytorch/src/input_model.hpp @@ -12,6 +12,7 @@ namespace ov { namespace frontend { namespace pytorch { +class FrontEnd; class TranslateSession; class Place; class TorchDecoder; @@ -23,6 +24,7 @@ struct PlaceDesc { class InputModel : public ov::frontend::InputModel { friend class ::ov::frontend::pytorch::TranslateSession; + friend class ::ov::frontend::pytorch::FrontEnd; public: explicit InputModel(const std::shared_ptr& model_decoder); @@ -30,6 +32,7 @@ class InputModel : public ov::frontend::InputModel { std::vector get_inputs() const override; std::vector get_outputs() const override; frontend::Place::Ptr get_place_by_tensor_name(const std::string& tensor_name) const override; + frontend::Place::Ptr get_place_by_input_index(size_t input_idx) const override; void set_partial_shape(const frontend::Place::Ptr& place, const ov::PartialShape& shape) override; ov::PartialShape get_partial_shape(const frontend::Place::Ptr& place) const override; void set_element_type(const frontend::Place::Ptr& place, const ov::element::Type& type) override; @@ -39,12 +42,15 @@ class InputModel : public ov::frontend::InputModel { void override_all_inputs(const std::vector& inputs) override; const std::string& decoder_type_name() const; std::shared_ptr get_decoder() const; + // update input places and erase requested places if possible + void flush_places(); private: std::shared_ptr m_model_decoder; std::unordered_map> m_name_to_place; std::vector> m_inputs; std::vector> m_outputs; + std::vector> m_requested_places; std::unordered_map m_descriptors; }; diff --git a/src/frontends/pytorch/src/place.cpp b/src/frontends/pytorch/src/place.cpp index 968cad19988dd6..6a6886b1b97210 100644 --- a/src/frontends/pytorch/src/place.cpp +++ b/src/frontends/pytorch/src/place.cpp @@ -25,6 +25,7 @@ Place::Place(const ov::frontend::InputModel& input_model, size_t tensor_index) auto in_it = std::find(inputs.begin(), inputs.end(), tensor_index); if (in_it != inputs.end()) { m_is_input = true; + m_input_index = std::distance(inputs.begin(), in_it); auto idx = std::distance(inputs.begin(), in_it); const auto& signature_name = decoder->get_input_signature_name(idx); m_names.push_back(signature_name); @@ -55,6 +56,31 @@ Place::Place(const ov::frontend::InputModel& input_model, size_t tensor_index) } } +Place::Place(const ov::frontend::InputModel& input_model, const std::string& name, size_t input_index) + : m_input_model(input_model), + m_tensor_index(0), + m_is_fake(true), + m_input_index(input_index), + m_pshape(PartialShape::dynamic()), + m_type(element::dynamic), + m_is_input(true) { + if (!name.empty()) + m_names = {name}; +} + +bool Place::is_equal(const Ptr& another) const { + const auto& pt_place = std::dynamic_pointer_cast(another); + if (!pt_place) + return this == another.get(); + if (m_is_fake || pt_place->m_is_fake) { + if ((m_is_fake && m_names.size() != 0) || (pt_place->m_is_fake && pt_place->m_names.size() != 0)) + // named fake place can only be equal to itself + return this == another.get(); + return m_input_index == pt_place->get_input_index(); + } + return this == another.get(); +} + } // namespace pytorch } // namespace frontend } // namespace ov diff --git a/src/frontends/pytorch/src/place.hpp b/src/frontends/pytorch/src/place.hpp index c823a126516aa6..e286861fc2d4fd 100644 --- a/src/frontends/pytorch/src/place.hpp +++ b/src/frontends/pytorch/src/place.hpp @@ -19,6 +19,8 @@ class Place : public ov::frontend::Place { public: Place(const ov::frontend::InputModel& input_model, size_t tensor_index); + // Fake place of the input. If name is non-empty it is used as id of the input, otherwise input_index is used + Place(const ov::frontend::InputModel& input_model, const std::string& name, size_t input_index); ~Place() override = default; @@ -28,9 +30,7 @@ class Place : public ov::frontend::Place { bool is_output() const override { return m_is_output; } - bool is_equal(const Ptr& another) const override { - return this == another.get(); - } + bool is_equal(const Ptr& another) const override; std::vector get_names() const override { return m_names; } @@ -50,13 +50,18 @@ class Place : public ov::frontend::Place { } return m_tensor_index == another_pt->get_tensor_index(); } + size_t get_input_index() const { + return m_input_index; + } private: const ov::frontend::InputModel& m_input_model; const size_t m_tensor_index; std::vector m_names; - element::Type m_type = element::dynamic; + bool m_is_fake = false; + size_t m_input_index = 0; // Only used for fake place PartialShape m_pshape; + element::Type m_type = element::dynamic; bool m_is_input = false; bool m_is_output = false; }; diff --git a/src/frontends/pytorch/src/translate_session.cpp b/src/frontends/pytorch/src/translate_session.cpp index 60410f97c33df4..e2c41e70a0254a 100644 --- a/src/frontends/pytorch/src/translate_session.cpp +++ b/src/frontends/pytorch/src/translate_session.cpp @@ -82,9 +82,10 @@ std::shared_ptr TranslateSession::convert_pytorch_model( auto mutated_tensors = std::make_shared>(); std::vector inserted_params; - if (input_model) { + if (input_model && input_model->m_requested_places.size() == 0) { // When we have input model we should use its inputs order to create Parameters // We use m_inputs instead of get_inputs() because latter doesn't have "self" input + // If there are fake places we don't need to use model inputs in that case for (auto& input_p : input_model->m_inputs) { auto pytorch_place = std::dynamic_pointer_cast(input_p); FRONT_END_GENERAL_CHECK(pytorch_place, "Only place produced by PyTorch Frontend is supported."); @@ -98,10 +99,6 @@ std::shared_ptr TranslateSession::convert_pytorch_model( parameters->push_back(parameter); (*tensor_map)[tensor_id] = parameter; } - // Add all tensors that were frozen - for (auto& desc : input_model->m_descriptors) { - (*tensor_map)[desc.first] = desc.second.m_value; - } } else { // Go over all pytorch_model inputs and register them in the tensor map: auto inputs = pytorch_model->inputs(); @@ -120,6 +117,12 @@ std::shared_ptr TranslateSession::convert_pytorch_model( (*tensor_map)[inputs.at(i)] = parameter; } } + if (input_model) { + // Add all tensors that were frozen + for (auto& desc : input_model->m_descriptors) { + (*tensor_map)[desc.first] = desc.second.m_value; + } + } auto node_visitor = [&](std::shared_ptr node) { // Explore all inputs of node. Node may refer to input value that hasn't been created in the current scope. diff --git a/src/frontends/tensorflow/src/input_model.cpp b/src/frontends/tensorflow/src/input_model.cpp index 14474eec64125a..5f0a8acf9e64e4 100644 --- a/src/frontends/tensorflow/src/input_model.cpp +++ b/src/frontends/tensorflow/src/input_model.cpp @@ -824,6 +824,10 @@ ov::frontend::Place::Ptr InputModel::get_place_by_tensor_name(const std::string& return _impl->get_place_by_tensor_name(tensorName); } +ov::frontend::Place::Ptr InputModel::get_place_by_input_index(size_t input_idx) const { + FRONT_END_NOT_IMPLEMENTED(get_place_by_input_index); +} + void InputModel::override_all_outputs(const std::vector& outputs) { _impl->override_all_outputs(outputs); } diff --git a/src/frontends/tensorflow/src/input_model.hpp b/src/frontends/tensorflow/src/input_model.hpp index 11f4033dcf4384..39847c12d5fd88 100644 --- a/src/frontends/tensorflow/src/input_model.hpp +++ b/src/frontends/tensorflow/src/input_model.hpp @@ -44,6 +44,7 @@ class InputModel : public ov::frontend::InputModel { std::vector get_inputs() const override; std::vector get_outputs() const override; ov::frontend::Place::Ptr get_place_by_tensor_name(const std::string& tensorName) const override; + ov::frontend::Place::Ptr get_place_by_input_index(size_t input_idx) const override; void override_all_outputs(const std::vector& outputs) override; void override_all_inputs(const std::vector& inputs) override; void extract_subgraph(const std::vector& inputs, diff --git a/src/frontends/tensorflow_lite/src/input_model.cpp b/src/frontends/tensorflow_lite/src/input_model.cpp index 7f2ad296e9b38d..d120d3898945b2 100644 --- a/src/frontends/tensorflow_lite/src/input_model.cpp +++ b/src/frontends/tensorflow_lite/src/input_model.cpp @@ -406,6 +406,10 @@ ov::frontend::Place::Ptr InputModel::get_place_by_tensor_name(const std::string& return _impl->get_place_by_tensor_name(tensorName); } +ov::frontend::Place::Ptr InputModel::get_place_by_input_index(size_t input_idx) const { + FRONT_END_NOT_IMPLEMENTED(get_place_by_input_index); +} + void InputModel::set_partial_shape(const Place::Ptr& place, const PartialShape& shape) { _impl->set_partial_shape(place, shape); } diff --git a/src/frontends/tensorflow_lite/src/input_model.hpp b/src/frontends/tensorflow_lite/src/input_model.hpp index 4ac655628f80d2..b1112fb14d6c9e 100644 --- a/src/frontends/tensorflow_lite/src/input_model.hpp +++ b/src/frontends/tensorflow_lite/src/input_model.hpp @@ -35,6 +35,7 @@ class InputModel : public ov::frontend::InputModel { std::vector get_inputs() const override; std::vector get_outputs() const override; ov::frontend::Place::Ptr get_place_by_tensor_name(const std::string& tensorName) const override; + ov::frontend::Place::Ptr get_place_by_input_index(size_t input_idx) const override; ///// Naming and annotation ///// void set_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name) override; diff --git a/tests/layer_tests/ovc_python_api_tests/test_pytorch.py b/tests/layer_tests/ovc_python_api_tests/test_pytorch.py index bbc4ddda62de9f..7096c56f815056 100644 --- a/tests/layer_tests/ovc_python_api_tests/test_pytorch.py +++ b/tests/layer_tests/ovc_python_api_tests/test_pytorch.py @@ -81,6 +81,42 @@ def forward(self, x, y=None, z=None): return NeuralNetwork() +def make_pt_model_with_tuple_input(): + from torch import nn + + class NeuralNetwork(nn.Module): + def __init__(self): + super(NeuralNetwork, self).__init__() + self.linear_relu_stack = nn.Sequential( + nn.ReLU(), + nn.Sigmoid(), + ) + + def forward(self, x: tuple, y): + logits = self.linear_relu_stack(x[0] + x[1] + x[2] + y) + return logits + + return NeuralNetwork() + + +def make_pt_model_with_dict_input(): + from torch import nn + + class NeuralNetwork(nn.Module): + def __init__(self): + super(NeuralNetwork, self).__init__() + self.linear_relu_stack = nn.Sequential( + nn.ReLU(), + nn.Sigmoid(), + ) + + def forward(self, x: dict): + logits = self.linear_relu_stack(x["x"] + x["y"] + x["z"]) + return logits + + return NeuralNetwork() + + def make_ref_pt_model_one_input(shape, dtype=np.float32): shape = PartialShape(shape) param1 = ov.opset8.parameter(shape, name="input_0", dtype=dtype) @@ -94,7 +130,7 @@ def make_ref_pt_model_one_input(shape, dtype=np.float32): return model -def make_ref_pt_model_two_inputs(shape, dtype=np.float32): +def make_ref_pt_model_two_inputs(shape, dtype=np.float32, reorder=False): if len(shape) == 2: param1 = ov.opset8.parameter(PartialShape( shape[0]), name="input_0", dtype=dtype) @@ -104,6 +140,8 @@ def make_ref_pt_model_two_inputs(shape, dtype=np.float32): shape = PartialShape(shape) param1 = ov.opset8.parameter(shape, name="input_0", dtype=dtype) param2 = ov.opset8.parameter(shape, name="input_1", dtype=dtype) + if reorder: + param1, param2 = param2, param1 if dtype == Type.dynamic: cl = ov.opset8.convert_like(param2, param1) mul = ov.opset8.multiply(param1, cl) @@ -142,6 +180,49 @@ def make_ref_pt_model_with_optional_inputs(shape, dtype=np.float32, z_exist=Fals return model +def make_ref_pt_model_with_tuple_input(shape, dtype=np.float32): + param1 = ov.opset8.parameter(PartialShape( + shape[0]), name="input_0", dtype=dtype) + param2 = ov.opset8.parameter(PartialShape( + shape[1]), name="input_1", dtype=dtype) + param3 = ov.opset8.parameter(PartialShape( + shape[2]), name="input_2", dtype=dtype) + param4 = ov.opset8.parameter(PartialShape( + shape[3]), name="y", dtype=dtype) + + op = ov.opset8.add(param1, param2) + op2 = ov.opset8.add(op, param3) + op3 = ov.opset8.add(op2, param4) + relu = ov.opset8.relu(op3) + if dtype != np.float32: + relu = ov.opset8.convert(relu, np.float32) + sigm = ov.opset8.sigmoid(relu) + + parameter_list = [param1, param2, param3, param4] + model = Model([sigm], parameter_list, "test") + return model + + +def make_ref_pt_model_with_dict_input(shape, dtype=np.float32): + param1 = ov.opset8.parameter(PartialShape( + shape[0]), name="x", dtype=dtype) + param2 = ov.opset8.parameter(PartialShape( + shape[1]), name="y", dtype=dtype) + param3 = ov.opset8.parameter(PartialShape( + shape[2]), name="z", dtype=dtype) + + op = ov.opset8.add(param1, param2) + op2 = ov.opset8.add(op, param3) + relu = ov.opset8.relu(op2) + if dtype != np.float32: + relu = ov.opset8.convert(relu, np.float32) + sigm = ov.opset8.sigmoid(relu) + + parameter_list = [param1, param2, param3] + model = Model([sigm], parameter_list, "test") + return model + + def create_pytorch_nn_module_case1(tmp_dir): pt_model = make_pt_model_two_inputs() ref_model = make_ref_pt_model_two_inputs([-1, -1, -1, -1]) @@ -499,6 +580,14 @@ def create_pytorch_nn_module_shapes_list_static_via_input(tmp_dir): return pt_model, ref_model, {'input': [([1, 3, 20, 20], np.float32), ([1, 3, 20, 20], np.float32)]} +def create_pytorch_nn_module_reorder_inputs(tmp_dir): + pt_model = make_pt_model_two_inputs() + ref_model = make_ref_pt_model_two_inputs([[-1, -1, -1, -1], [-1]]) + + return pt_model, ref_model, {"input": ["x", "y"], + "example_input": [torch.zeros((1, 3, 10, 10)), torch.ones((10))],} + + def create_pytorch_nn_module_shapes_list_dynamic(tmp_dir): pt_model = make_pt_model_two_inputs() inp_shapes = [[Dimension(-1), 3, 20, Dimension(20, -1)], @@ -684,7 +773,8 @@ def forward(self, x): net = ConvModel() shape = PartialShape([-1, 3, -1, -1]) param1 = ov.opset10.parameter(shape, dtype=np.float32) - weights = ov.opset10.constant(net.weights.numpy(force=True), dtype=np.float16) + weights = ov.opset10.constant( + net.weights.numpy(force=True), dtype=np.float16) decompress_weights = ov.opset10.convert(weights, np.float32) conv = ov.opset10.convolution(param1, decompress_weights, strides=[1, 1], pads_begin=[0, 0], pads_end=[0, 0], @@ -720,6 +810,40 @@ def create_pytorch_module_with_optional_inputs_case3(tmp_dir): return net, ref_model, {"example_input": example_input, "input": [[3, 3, 3, 3], [3, 3, 3, 3]]} +def create_pytorch_module_with_tuple_case1(tmp_dir): + net = make_pt_model_with_tuple_input() + example_input = ((torch.zeros((1, 3, 10, 10)), torch.ones((1, 3, 10, 10)), torch.ones((1, 3, 10, 10))), torch.ones((1, 3, 10, 10))) + ref_model = make_ref_pt_model_with_tuple_input(([-1, -1, -1, -1], [-1, -1, -1, -1], [-1, -1, -1, -1], [-1, -1, -1, -1])) + return net, ref_model, {"example_input": example_input} + +def create_pytorch_module_with_tuple_case2(tmp_dir): + net = make_pt_model_with_tuple_input() + example_input = {"x": (torch.zeros((1, 3, 10, 10)), torch.ones((1, 3, 10, 10)), torch.ones((1, 3, 10, 10))), "y": torch.ones((1, 3, 10, 10))} + ref_model = make_ref_pt_model_with_tuple_input(([-1, -1, -1, -1], [-1, -1, -1, -1], [-1, -1, -1, -1], [-1, -1, -1, -1])) + return net, ref_model, {"example_input": example_input} + + +def create_pytorch_module_with_tuple_case3(tmp_dir): + net = make_pt_model_with_tuple_input() + example_input = ((torch.zeros((1, 3, 10, 10)), torch.ones((1, 3, 10, 10)), torch.ones((1, 3, 10, 10))), torch.ones((1, 3, 10, 10))) + ref_model = make_ref_pt_model_with_tuple_input(([3, 3, 3, 3], [3, 3, 3, 3], [3, 3, 3, 3], [3, 3, 3, 3])) + return net, ref_model, {"example_input": example_input, "input": [[3, 3, 3, 3], [3, 3, 3, 3], [3, 3, 3, 3], [3, 3, 3, 3]]} + + +def create_pytorch_module_with_dict_case1(tmp_dir): + net = make_pt_model_with_dict_input() + example_input = ({"x": torch.zeros((1, 3, 10, 10)), "y": torch.ones((1, 3, 10, 10)), "z": torch.ones((1, 3, 10, 10))},) + ref_model = make_ref_pt_model_with_dict_input(([-1, -1, -1, -1], [-1, -1, -1, -1], [-1, -1, -1, -1])) + return net, ref_model, {"example_input": example_input} + + +def create_pytorch_module_with_dict_case2(tmp_dir): + net = make_pt_model_with_dict_input() + example_input = ({"x": torch.zeros((1, 3, 10, 10)), "y": torch.ones((1, 3, 10, 10)), "z": torch.ones((1, 3, 10, 10))},) + ref_model = make_ref_pt_model_with_dict_input(([3, 3, 3, 3], [3, 3, 3, 3], [3, 3, 3, 3])) + return net, ref_model, {"example_input": example_input, "input": {"x": [3, 3, 3, 3], "y": [3, 3, 3, 3], "z": [3, 3, 3, 3]}} + + def create_pytorch_module_with_compressed_int8_constant_compress_to_fp16_default(tmp_dir): import torch import torch.nn.functional as F @@ -798,8 +922,10 @@ def forward(self, z: Tuple[torch.Tensor, torch.Tensor]): return torch.cat([z1, zeros1], 1), torch.cat([z2, zeros2], 2) net = PTModel() - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float32) - constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float32) + constant_zeros2 = ov.opset10.constant( + np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) shape1 = PartialShape([1, -1]) shape2 = PartialShape([1, 5, -1]) param1 = ov.opset10.parameter(shape1, dtype=np.float32) @@ -821,8 +947,10 @@ def forward(self, z: Tuple[torch.Tensor, torch.Tensor]): return torch.cat([z1, zeros1], 1), torch.cat([z2, zeros2], 2) net = PTModel() - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float16) - constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float16) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float16) + constant_zeros2 = ov.opset10.constant( + np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float16) const1_decompress = ov.opset10.convert(constant_zeros1, np.float32) const2_decompress = ov.opset10.convert(constant_zeros2, np.float32) shape1 = PartialShape([1, -1]) @@ -845,8 +973,10 @@ def forward(self, x: torch.Tensor, z: Tuple[torch.Tensor, torch.Tensor]): return torch.cat([z1, zeros1], 1) + x, torch.cat([z2, zeros2], 2) net = PTModel() - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float32) - constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float32) + constant_zeros2 = ov.opset10.constant( + np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) shape1 = PartialShape([1, -1]) shape2 = PartialShape([1, 5, -1]) param0 = ov.opset10.parameter(PartialShape([-1, -1]), dtype=np.float32) @@ -873,8 +1003,10 @@ def forward(self, z: Tuple[torch.Tensor, torch.Tensor], x: torch.Tensor): net = PTModel() shape1 = PartialShape([1, -1]) shape2 = PartialShape([1, 5, -1]) - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float32) - constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float32) + constant_zeros2 = ov.opset10.constant( + np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) param1 = ov.opset10.parameter(shape1, dtype=np.float32) param2 = ov.opset10.parameter(shape2, dtype=np.float32) param3 = ov.opset10.parameter(PartialShape([-1, -1]), dtype=np.float32) @@ -897,8 +1029,10 @@ def forward(self, x: torch.Tensor, z: Tuple[torch.Tensor, torch.Tensor], y: torc return torch.cat([z1, zeros1], 1) + x, torch.cat([z2, zeros2], 2) * y net = PTModel() - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float32) - constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float32) + constant_zeros2 = ov.opset10.constant( + np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) shape1 = PartialShape([1, -1]) shape2 = PartialShape([1, 5, -1]) param1 = ov.opset10.parameter(shape1, dtype=np.float32) @@ -926,8 +1060,10 @@ def forward(self, x: torch.Tensor, z: Tuple[torch.Tensor, torch.Tensor], y: torc return torch.cat([z1, zeros1], 1) + x, torch.cat([z2, zeros2], 2) * y net = PTModel() - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float32) - constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float32) + constant_zeros2 = ov.opset10.constant( + np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) shape1 = PartialShape([1, -1]) shape2 = PartialShape([1, 5, -1]) param0 = ov.opset10.parameter(PartialShape([-1, -1]), dtype=np.float32) @@ -956,8 +1092,10 @@ def forward(self, x: torch.Tensor, y: torch.Tensor = None, z: Tuple[torch.Tensor return torch.cat([z1, zeros1], 1) + x, torch.cat([z2, zeros2], 2) net = PTModel() - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float32) - constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float32) + constant_zeros2 = ov.opset10.constant( + np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float32) shape1 = PartialShape([1, -1]) shape2 = PartialShape([1, 5, -1]) param0 = ov.opset10.parameter(PartialShape([-1, -1]), dtype=np.float32) @@ -982,7 +1120,8 @@ def forward(self, x: List[torch.Tensor]): net = PTModel() constant_one = ov.opset10.constant(np.ones((1, 1)), dtype=np.float32) const_zero = ov.opset10.constant(0, dtype=np.int64) - constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1, 1), dtype=np.float32), dtype=np.float32) param = ov.opset10.parameter(PartialShape([-1, -1, -1]), dtype=np.float32) gather = ov.opset10.gather(param, const_zero, const_zero) @@ -1004,7 +1143,8 @@ def forward(self, x): net = PTModel() constant_one = ov.opset10.constant(np.ones((1,)), dtype=np.float32) const_zero = ov.opset10.constant(0, dtype=np.int64) - constant_zeros1 = ov.opset10.constant(np.zeros((1,), dtype=np.float32), dtype=np.float32) + constant_zeros1 = ov.opset10.constant( + np.zeros((1,), dtype=np.float32), dtype=np.float32) param = ov.opset10.parameter(PartialShape([-1, -1]), dtype=np.float32) gather = ov.opset10.gather(param, const_zero, const_zero) @@ -1031,9 +1171,9 @@ def forward(self, a, b): return net, ref_model, { "example_input": ( { - "1": torch.tensor([1, 2], dtype=torch.float32), - "2": torch.tensor([3, 4], dtype=torch.float32) - }, + "1": torch.tensor([1, 2], dtype=torch.float32), + "2": torch.tensor([3, 4], dtype=torch.float32) + }, torch.tensor([5, 6], dtype=torch.float32) )} @@ -1063,6 +1203,7 @@ class TestMoConvertPyTorch(CommonMOConvertTest): create_pytorch_nn_module_with_compressed_constants, create_pytorch_nn_module_shapes_list_static, create_pytorch_nn_module_shapes_list_static_via_input, + create_pytorch_nn_module_reorder_inputs, create_pytorch_nn_module_shapes_list_dynamic, create_pytorch_nn_module_shapes_list_dynamic_via_input, create_pytorch_nn_module_shapes_list_dynamic_single_input, @@ -1079,6 +1220,11 @@ class TestMoConvertPyTorch(CommonMOConvertTest): create_pytorch_module_with_optional_inputs_case1, create_pytorch_module_with_optional_inputs_case2, create_pytorch_module_with_optional_inputs_case3, + create_pytorch_module_with_tuple_case1, + create_pytorch_module_with_tuple_case2, + create_pytorch_module_with_tuple_case3, + create_pytorch_module_with_dict_case1, + create_pytorch_module_with_dict_case2, create_pytorch_nn_module_with_scalar_input, create_pytorch_module_with_compressed_int8_constant, create_pytorch_module_with_compressed_int8_constant_compress_to_fp16_default, @@ -1137,7 +1283,7 @@ def test_incorrect_inputs_1(self): from openvino.tools.ovc import convert_model pytorch_model, _, _ = create_pytorch_nn_module_case1('') - with self.assertRaisesRegex(Exception, ".*No node with name.*"): + with self.assertRaisesRegex(Exception, "Input for tensor name \'input1\[1, 10\]\' is not found."): convert_model(pytorch_model, input='input1[1, 10]') def test_incorrect_inputs_2(self): @@ -1145,7 +1291,7 @@ def test_incorrect_inputs_2(self): pytorch_model, _, _ = create_pytorch_nn_module_case1('') # check that it accepts specified names as is, without parsing into 2 different inputs - with self.assertRaisesRegex(Exception, 'No node with name input1,input2'): + with self.assertRaisesRegex(Exception, 'Input for tensor name \'input1,input2\' is not found.'): convert_model(pytorch_model, input='input1,input2') def test_incorrect_inputs_3(self): @@ -1153,7 +1299,7 @@ def test_incorrect_inputs_3(self): pytorch_model, _, _ = create_pytorch_nn_module_case1('') # check that it accepts specified names as is, without parsing into 2 different inputs - with self.assertRaisesRegex(Exception, 'No node with name input1\[1, 10\],input2\[2, 100\]'): + with self.assertRaisesRegex(Exception, 'Input for tensor name \'input1\[1, 10\],input2\[2, 100\]\' is not found.'): convert_model(pytorch_model, input='input1[1, 10],input2[2, 100]') def test_incorrect_inputs_4(self): @@ -1161,8 +1307,9 @@ def test_incorrect_inputs_4(self): pytorch_model, _, _ = create_pytorch_nn_module_case1('') # check that it accepts specified names as is, without parsing into 2 different inputs - with self.assertRaisesRegex(Exception, 'No node with name input1\[1, 10\]'): - convert_model(pytorch_model, input=['input1[1, 10]', 'input2[2, 100]']) + with self.assertRaisesRegex(Exception, 'Input for tensor name \'input1\[1, 10\]\' is not found.'): + convert_model(pytorch_model, input=[ + 'input1[1, 10]', 'input2[2, 100]']) def test_failed_extension(self): from openvino.tools.ovc import convert_model @@ -1257,4 +1404,5 @@ def test_conversion_params(self, params, ie_device, precision, ir_version, ref_model = params['ref_model'] test_params.update({'input_model': fw_model}) - self._test_by_ref_graph(temp_dir, test_params, ref_model, compare_tensor_names=False) + self._test_by_ref_graph(temp_dir, test_params, + ref_model, compare_tensor_names=False) diff --git a/tools/ovc/openvino/tools/ovc/convert_impl.py b/tools/ovc/openvino/tools/ovc/convert_impl.py index c7cafcb4ea2385..2eb2f2adc133f1 100644 --- a/tools/ovc/openvino/tools/ovc/convert_impl.py +++ b/tools/ovc/openvino/tools/ovc/convert_impl.py @@ -328,6 +328,7 @@ def normalize_inputs(argv: argparse.Namespace): """ # Parse input to list of InputCutInfo inputs = input_to_input_cut_info(argv.input) + argv.input = inputs # Make list of input names input_names_list = [] @@ -338,8 +339,6 @@ def normalize_inputs(argv: argparse.Namespace): assert len(input_names_list) == len(inputs), "\"input\" parameter has unnamed inputs and named inputs. " \ "Please either set names for all inputs, " \ "or do not set names for all inputs." - argv.inputs_list = input_names_list - argv.input = ','.join(input_names_list) if len(input_names_list) > 0: # Named inputs case diff --git a/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py b/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py index 8a068f4dc987f5..12722b5d771b75 100644 --- a/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py +++ b/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py @@ -72,12 +72,14 @@ def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd): outputs = fe_output_user_data_repack(input_model, argv.output, moc_front_end.get_name()) input_model.override_all_outputs([x['node'] for x in outputs]) ''' - argv.placeholder_shapes, argv.placeholder_data_types = convert_params_lists_to_dicts( - input_model, argv.placeholder_shapes, argv.placeholder_data_types) - user_shapes, outputs, freeze_placeholder = fe_user_data_repack( - input_model, argv.placeholder_shapes, argv.placeholder_data_types, - argv.output, {}, moc_front_end.get_name()) + enabled_transforms, disabled_transforms = get_enabled_and_disabled_transforms() + if 'ANALYSIS_JSON_PRINT' in enabled_transforms: + # NOTE that model analysis is performed before applying user's settings (inputs's shapes etc.) + framework_model = moc_front_end.decode(input_model) + json_model_analysis_dump(framework_model) + # a model is not processed further in json analysis mode + sys.exit(0) def check_places_are_same(places_original: List[Place], places_new: List[Place]): """ @@ -90,6 +92,67 @@ def check_places_are_same(places_original: List[Place], places_new: List[Place]) [item for item in places_original if any( [item.is_equal(item2['node']) for item2 in places_new])]) == len(places_original) + if getattr(argv, "framework", None) == "pytorch": + iplaces = [] + for idx, input_info in enumerate(argv.input): + if getattr(input_info, "name", None): + place = input_model.get_place_by_tensor_name(input_info.name) + if not input_info.shape and not input_info.type: + # If we received place by name, we need to use it for FE to verify + # that such name exist, otherwise we silently ignore it. + # Using dynamic shape should be safe, because FE will not overwrite + # the shape that was produced after conversion, but merge it, so + # dynamic shape will not change anything. + input_model.set_partial_shape(place, PartialShape.dynamic()) + else: + place = input_model.get_place_by_input_index(idx) + iplaces.append(place) + if input_info.shape is not None: + input_model.set_partial_shape(place, input_info.shape) + if input_info.type is not None: + input_model.set_element_type(place, input_info.type) + model_inputs = input_model.get_inputs() + def merge_inputs(inputs, to_set_list): + # use input places instead of obtained by index if they are the same + res = [] + for p in to_set_list: + found = False + for i in inputs: + if p.is_equal(i): + res.append(i) + found = True + break + if not found: + res.append(p) + return res + iplaces = merge_inputs(model_inputs, iplaces) + # Currently this only work to reorder inputs/outputs + to_override_all_inputs = check_places_are_same(model_inputs, [{"node": p} for p in iplaces]) + to_override_all_outputs = False + if argv.output: + oplaces = [] + _outputs = fe_output_user_data_repack(input_model, argv.output, moc_front_end.get_name()) + for out_desc in _outputs: + oplaces.append(out_desc["name"]) + model_outputs = input_model.get_outputs() + to_override_all_outputs = check_places_are_same(model_outputs, [{"node": p} for p in oplaces]) + if to_override_all_inputs and to_override_all_outputs: + input_model.extract_subgraph(iplaces, oplaces) + elif to_override_all_inputs: + input_model.override_all_inputs(iplaces) + elif to_override_all_outputs: + input_model.override_all_outputs(oplaces) + + ov_model = moc_front_end.convert(input_model) + return ov_model + + argv.placeholder_shapes, argv.placeholder_data_types = convert_params_lists_to_dicts( + input_model, argv.placeholder_shapes, argv.placeholder_data_types) + + user_shapes, outputs, freeze_placeholder = fe_user_data_repack( + input_model, argv.placeholder_shapes, argv.placeholder_data_types, + argv.output, {}, moc_front_end.get_name()) + def add_names_to_tensors(model: InputModel, places: List[Place]): """ Adds additional names to some model input tensors. This helper should be used @@ -107,14 +170,6 @@ def add_names_to_tensors(model: InputModel, places: List[Place]): log.warning('Could not add an additional name to a tensor pointed to by \'{}\'. Details: {}'.format( new_input['input_name'], str(e))) - enabled_transforms, disabled_transforms = get_enabled_and_disabled_transforms() - if 'ANALYSIS_JSON_PRINT' in enabled_transforms: - # NOTE that model analysis is performed before applying user's settings (inputs's shapes etc.) - framework_model = moc_front_end.decode(input_model) - json_model_analysis_dump(framework_model) - # a model is not processed further in json analysis mode - sys.exit(0) - model_inputs = input_model.get_inputs() inputs_equal = True if user_shapes: @@ -238,9 +293,6 @@ def create_target_input_shapes(new_input_places): input_model.set_tensor_value(place, value) - def shape_to_array(shape: PartialShape): - return [shape.get_dimension(i) for i in range(shape.rank.get_length())] - ov_model = moc_front_end.convert(input_model) return ov_model diff --git a/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py b/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py index 6a7dad594faf08..03662dacd613ee 100644 --- a/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py +++ b/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py @@ -8,6 +8,7 @@ # pylint: disable=no-name-in-module,import-error from openvino.runtime import Tensor, PartialShape from openvino.tools.ovc.error import Error +from openvino.tools.ovc.cli_parser import single_input_to_input_cut_info, _InputCutInfo @@ -90,76 +91,85 @@ def get_value_from_list_or_dict(container, name, idx): return None +def flatten_inputs(inputs, names=None): + flattened = [] + if isinstance(inputs, dict): + # if names are provided we need to unpack in the same order + if names: + for name in names: + if isinstance(inputs[name], (list, tuple, dict)): + flattened.extend(flatten_inputs(inputs[name])) + else: + flattened.append((name, inputs[name])) + else: + for name, input_data in inputs.items(): + if isinstance(input_data, (list, tuple, dict)): + flattened.extend(flatten_inputs(input_data)) + else: + flattened.append((name, input_data)) + else: + for input_data in inputs: + if isinstance(input_data, (list, tuple, dict)): + flattened.extend(flatten_inputs(input_data)) + else: + flattened.append(input_data) + return flattened + + def extract_input_info_from_example(args, inputs): try: from openvino.frontend.pytorch.utils import pt_to_ov_type_map # pylint: disable=no-name-in-module,import-error except Exception as e: log.error("PyTorch frontend loading failed") raise e - example_inputs = args.example_input - data_types = args.placeholder_data_types or {} - input_shapes = args.placeholder_shapes or {} + example_inputs = args.input_model._example_input if args.input_model._example_input is not None else args.example_input + if example_inputs is None: + return is_dict_input = isinstance(example_inputs, dict) - list_inputs = list(example_inputs.values()) if is_dict_input else example_inputs - input_names = None if not isinstance(example_inputs, (list, tuple, dict)): - list_inputs = [list_inputs] - if args.input_model._input_is_list: - list_inputs[0] = list_inputs[0].unsqueeze(0) - if args.input_model._input_signature is not None and not is_dict_input: + example_inputs = [example_inputs] + input_names = None + if args.input_model._input_signature is not None: input_names = args.input_model._input_signature[1:] if args.input_model._input_signature[ 0] == "self" else args.input_model._input_signature - if not is_dict_input: - example_inputs = dict(zip(input_names, list_inputs)) - is_dict_input = True - elif is_dict_input: - input_names = list(example_inputs) - if not data_types and input_names is None: - data_types = [] - if not input_shapes and input_names is None: - input_shapes = [] - if inputs: - for input_id, input_info in enumerate(inputs): - input_name = input_info.name - if is_dict_input and input_name in example_inputs: - example_input = example_inputs[input_name] - else: - example_input = list_inputs[input_id] - if is_dict_input and input_name is None: - input_name = input_names[input_id] - dtype = getattr(example_input, "dtype", type(example_input)) - example_dtype = pt_to_ov_type_map.get(str(dtype)) - user_dtype = get_value_from_list_or_dict(data_types, input_name, input_id) - if user_dtype is not None and example_dtype is not None and example_dtype != user_dtype: - raise Error( - f"Defined input type {user_dtype} is not equal to provided example_input type {example_dtype}") - - data_rank = getattr(example_input, "ndim", 0) - user_input_shape = get_value_from_list_or_dict(input_shapes, input_name, input_id) - if user_input_shape.rank.is_static and user_input_shape.rank.get_length() != data_rank: - raise Error( - f"Requested input shape {user_input_shape.rank.get_length()} rank" - f" is not equal to provided example_input rank {data_rank}") - - input_shape = user_input_shape if user_input_shape is not None else PartialShape([-1] * data_rank) - update_list_or_dict(data_types, input_name, input_id, - example_dtype if example_dtype is not None else None) - update_list_or_dict(input_shapes, input_name, input_id, input_shape) - else: - for input_id, example_input in enumerate(list_inputs): - dtype = getattr(example_input, "dtype", type(example_input)) - ov_dtype = pt_to_ov_type_map.get(str(dtype)) - data_rank = getattr(example_input, "ndim", 0) - input_shape = PartialShape([-1] * data_rank) - input_name = input_names[input_id] if input_names else None - update_list_or_dict(input_shapes, input_name, input_id, input_shape) - update_list_or_dict(data_types, input_name, input_id, ov_dtype if ov_dtype is not None else None) - - args.placeholder_data_types = data_types - args.placeholder_shapes = input_shapes - if not args.input and input_names: - args.input_list = input_names - args.input = ",".join(input_names) + if input_names and not is_dict_input: + example_inputs = dict(zip(input_names, example_inputs)) + print(input_names) + example_inputs = flatten_inputs(example_inputs, input_names) + input_arg = [] + for example in example_inputs: + name = None + if isinstance(example, tuple) and len(example) == 2: + name = example[0] + example = example[1] + shape = PartialShape([-1] * example.ndim) if hasattr(example, "ndim") else PartialShape.dynamic() + dtype = getattr(example, "dtype", type(example)) + dtype = pt_to_ov_type_map.get(str(dtype)) + if name: + input_arg.append(single_input_to_input_cut_info((name, shape, dtype))) + else: + input_arg.append(single_input_to_input_cut_info((shape, dtype))) + if inputs is not None and len(inputs) != 0: + if len(inputs) == len(input_arg): + # we can update input argument with info from examples + new_input = [] + for i in range(len(input_arg)): + input_desc = args.input[i] + name = input_desc.name + dtype = input_desc.type + shape = input_desc.shape + if name is None: + name = input_arg[i].name + if dtype is None: + dtype = input_arg[i].type + if shape is None: + shape = input_arg[i].shape + new_input.append(_InputCutInfo(name, shape, dtype, input_desc.value)) + input_arg = new_input + else: + # we can't update args.input + return + args.input = input_arg # pylint: disable=no-member From 29183228c18b591fc5e4e6d70620e10639183f51 Mon Sep 17 00:00:00 2001 From: Vladimir Paramuzov Date: Mon, 1 Jul 2024 18:29:21 +0400 Subject: [PATCH 08/15] [GPU] Micro sdpa (#24656) ### Details: - Added SDPA impl based on microkernels using internal onednn API and related infra - Current limitations: - fused transpose shouldn't change order of innermost dim (head size). - is_causal = true is not supported - fp16 only - num heads dimension must be static - no indirect kv support - Initial version of KV Cache + SDPA func test - Enabled Transpose+SDPA fusion for static shape too ### Tickets: - CVS-141761 --- .../include/intel_gpu/plugin/plugin.hpp | 6 +- .../plugin/transformations_pipeline.hpp | 6 +- .../intel_gpu/runtime/debug_configuration.hpp | 6 +- .../include/intel_gpu/runtime/device_info.hpp | 14 +- .../include/intel_gpu/runtime/kernel_args.hpp | 3 +- .../impls/ocl/kernel_selector_helper.cpp | 49 +- .../graph/impls/ocl/kernel_selector_helper.h | 2 + .../ocl/scaled_dot_product_attention.cpp | 2 +- src/plugins/intel_gpu/src/graph/program.cpp | 2 +- .../src/kernel_selector/CMakeLists.txt | 4 + .../batch_headers/generic_vector_ops.cl | 75 +++ .../include/batch_headers/sdpa_utils.cl | 35 + .../include/batch_headers/tile_ops.cl | 628 +++++++++++++++++ .../include/batch_headers/vec_typedefs.cl | 16 +- .../kernel_selector/cl_kernels/sdpa_micro.cl | 461 +++++++++++++ .../kernel_selector/kernel_selector_common.h | 5 + .../kernel_selector/kernel_selector_params.h | 18 + .../kernels/sdpa/sdpa_kernel_micro.cpp | 630 ++++++++++++++++++ .../kernels/sdpa/sdpa_kernel_micro.h | 47 ++ .../kernels/sdpa/sdpa_kernel_opt.cpp | 4 +- .../kernels/sdpa/sdpa_kernel_selector.cpp | 4 + .../src/kernel_selector/micro_utils.hpp | 58 ++ .../src/kernel_selector/primitive_db.cpp | 2 +- .../src/kernel_selector/primitive_db.h | 4 +- .../src/kernel_selector/primitive_db_gen.py | 7 +- .../ops/scaled_dot_product_attention.cpp | 9 +- src/plugins/intel_gpu/src/plugin/plugin.cpp | 23 +- .../transformations/transpose_fusion.cpp | 14 +- .../src/plugin/transformations_pipeline.cpp | 11 +- .../intel_gpu/src/runtime/kernels_cache.cpp | 58 +- .../intel_gpu/src/runtime/kernels_cache.hpp | 26 +- .../intel_gpu/src/runtime/ocl/ocl_device.cpp | 43 +- .../intel_gpu/src/runtime/ocl/ocl_wrapper.hpp | 2 + .../tests/common/subgraphs_builders.hpp | 2 +- .../intel_gpu/thirdparty/CMakeLists.txt | 19 +- 35 files changed, 2216 insertions(+), 79 deletions(-) create mode 100644 src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/generic_vector_ops.cl create mode 100644 src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sdpa_utils.cl create mode 100644 src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/tile_ops.cl create mode 100644 src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_micro.cl create mode 100644 src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp create mode 100644 src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.h create mode 100644 src/plugins/intel_gpu/src/kernel_selector/micro_utils.hpp diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp index a5ea28b95f78ba..28a20fa737da76 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/plugin.hpp @@ -26,8 +26,10 @@ class Plugin : public ov::IPlugin { std::map> get_default_contexts() const; - std::shared_ptr clone_and_transform_model(const std::shared_ptr& network, const ExecutionConfig& config) const; - void transform_model(std::shared_ptr& model, const ExecutionConfig& config) const; + std::shared_ptr clone_and_transform_model(const std::shared_ptr& network, + const ExecutionConfig& config, + const std::shared_ptr& context) const; + void transform_model(std::shared_ptr& model, const ExecutionConfig& config, const std::shared_ptr& context) const; void register_primitives() const; std::string get_device_id_from_config(const ov::AnyMap& config) const; std::string get_device_id(const ov::AnyMap& config) const; diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/transformations_pipeline.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/transformations_pipeline.hpp index f0c0361ce72077..365deb66ad2a4d 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/transformations_pipeline.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/transformations_pipeline.hpp @@ -6,6 +6,7 @@ #include +#include "intel_gpu/plugin/remote_context.hpp" #include "openvino/core/model.hpp" #include "intel_gpu/runtime/execution_config.hpp" @@ -16,12 +17,13 @@ namespace intel_gpu { class TransformationsPipeline { public: - explicit TransformationsPipeline(const ExecutionConfig &conf, const cldnn::device_info &device_info) - : config(conf), device_info(device_info) {} + explicit TransformationsPipeline(const ExecutionConfig &conf, const std::shared_ptr& context) + : config(conf), m_context(context), device_info(context->get_engine().get_device_info()) {} void apply(std::shared_ptr func); private: const ExecutionConfig& config; + std::shared_ptr m_context; cldnn::device_info device_info; }; diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp index b3220136c6c8a8..d9a30e3244b4ae 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp @@ -48,7 +48,7 @@ enum class LogLevel : int8_t { #else #define SEPARATE '/' #endif -#define __FILENAME__ (strrchr(__FILE__, SEPARATE) ? strrchr(__FILE__, SEPARATE) + 1 : __FILE__) +#define GPU_FILENAME (strrchr(__FILE__, SEPARATE) ? strrchr(__FILE__, SEPARATE) + 1 : __FILE__) #define GPU_DEBUG_IF(cond) if (cond) #define GPU_DEBUG_CODE(...) __VA_ARGS__ #define GPU_DEBUG_DEFINE_MEM_LOGGER(stage) \ @@ -62,9 +62,9 @@ enum class LogLevel : int8_t { #define GPU_DEBUG_LOG_RAW_INT(min_verbose_level) if (cldnn::debug_configuration::get_instance()->verbose >= min_verbose_level) \ ((cldnn::debug_configuration::get_instance()->verbose_color == 0) ? GPU_DEBUG_LOG_PREFIX : GPU_DEBUG_LOG_COLOR_PREFIX) #define GPU_DEBUG_LOG_RAW(min_verbose_level) GPU_DEBUG_LOG_RAW_INT(static_cast::type>(min_verbose_level)) -#define GPU_DEBUG_LOG_PREFIX std::cout << cldnn::debug_configuration::prefix << __FILENAME__ << ":" <<__LINE__ << ":" << __func__ << ": " +#define GPU_DEBUG_LOG_PREFIX std::cout << cldnn::debug_configuration::prefix << GPU_FILENAME << ":" <<__LINE__ << ":" << __func__ << ": " #define GPU_DEBUG_LOG_COLOR_PREFIX std::cout << DARK_GRAY << cldnn::debug_configuration::prefix << \ - BLUE << __FILENAME__ << ":" << PURPLE << __LINE__ << ":" << CYAN << __func__ << ": " << RESET + BLUE << GPU_FILENAME << ":" << PURPLE << __LINE__ << ":" << CYAN << __func__ << ": " << RESET #define DARK_GRAY "\033[1;30m" #define BLUE "\033[1;34m" #define PURPLE "\033[1;35m" diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp index 7a18cd311e1fd4..0c83877851b48b 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/device_info.hpp @@ -10,7 +10,6 @@ #include #include #include -#include namespace cldnn { /// @addtogroup cpp_api C++ API @@ -25,6 +24,17 @@ enum class device_type { discrete_gpu = 1 }; +enum class gpu_arch { + unknown = 0, + gen9 = 1, + gen11 = 2, + xe_lp = 3, + xe_hp = 4, + xe_hpg = 5, + xe_hpc = 6, + xe2 = 7, +}; + /// @brief Defines version of GFX IP struct gfx_version { uint16_t major; @@ -77,6 +87,8 @@ struct device_info { device_type dev_type; ///< Defines type of current GPU device (integrated or discrete) gfx_version gfx_ver; ///< Defines GFX IP version + gpu_arch arch; ///< Defines arch human readable name + uint32_t ip_version; ///< Defines raw GFX IP version uint32_t device_id; ///< ID of current GPU uint32_t num_slices; ///< Number of slices uint32_t num_sub_slices_per_slice; ///< Number of subslices in a slice diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/kernel_args.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/kernel_args.hpp index 6bce66e64ef808..09dfcf68f05725 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/kernel_args.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/kernel_args.hpp @@ -121,8 +121,9 @@ struct kernel_string { std::string options; std::string entry_point; bool batch_compilation; + bool has_microkernels; - kernel_string() : str(""), jit(""), undefs(""), options(""), entry_point(""), batch_compilation(false) {} + kernel_string() : str(""), jit(""), undefs(""), options(""), entry_point(""), batch_compilation(false), has_microkernels(false) {} std::string get_str() const { return str + jit + undefs + options + entry_point; } size_t get_hash() const { return std::hash()(get_str()); } diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp index 12e20536407d45..1f492e14c9fc7e 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp @@ -5,6 +5,7 @@ #include "intel_gpu/graph/program.hpp" #include "kernel_selector_helper.h" +#include "intel_gpu/runtime/device_info.hpp" #include "kernel_selector_params.h" #include "to_string_utils.h" #include "program_node.h" @@ -32,7 +33,6 @@ #include "intel_gpu/primitives/extract_image_patches.hpp" #include "activation_inst.h" -#include "depth_to_space_inst.h" #include "eltwise_inst.h" #include "quantize_inst.h" #include "reorder_inst.h" @@ -44,9 +44,9 @@ #include "kernel_selector/kernels/reorder/reorder_kernel_base.h" #include "runtime/kernels_cache.hpp" -#include "kernel_base.h" #include +#include #include namespace { @@ -119,6 +119,48 @@ bool query_local_block_io_supported(engine& e, const ExecutionConfig& config) { namespace cldnn { +bool query_microkernels_supported(cldnn::engine& e, const cldnn::ExecutionConfig& config) { + auto device = e.get_device().get(); + + static std::mutex m; + std::lock_guard lock(m); + static std::map cache; + if (cache.find(device) != cache.end()) { + return cache.at(device); + } + + std::shared_ptr kernel_string = std::make_shared(); + // This program check that all required vISA features are supported by current IGC version + const char* kernel_code = R""""( + kernel void igc_check() { + __asm__ volatile( + ".decl AA0 v_type=G type=ud num_elts=1\n" + ".decl AA1 v_type=G type=ud num_elts=1\n" + ".implicit_PSEUDO_INPUT AA0 offset=256 size=4\n" + ".implicit_PSEUDO_INPUT AA1 offset=256 size=4\n" + "mov (M1_NM,1) AA0(0,0)<1> AA1(0,0)<0;1,0>\n" + ); + } + )""""; + + kernel_string->str = kernel_code; + kernel_string->options = ""; + kernel_string->entry_point = "igc_check"; + kernel_string->batch_compilation = true; + + try { + cldnn::kernel_impl_params dummy_params; + auto _kernels_cache_device_query = std::unique_ptr(new cldnn::kernels_cache(e, config, 0)); + _kernels_cache_device_query->add_kernels_source(dummy_params, {kernel_string}, false); + _kernels_cache_device_query->build_all(); + cache[device] = true; + } catch (std::exception&) { + cache[device] = false; + } + + return cache.at(device); +} + kernel_selector::data_type to_data_type(data_types dt) { switch (dt) { case cldnn::data_types::i4: @@ -1081,6 +1123,7 @@ void set_params(const kernel_impl_params& param_info, kernel_selector::params& p params.engineInfo.bOptHintsSupport = false; params.engineInfo.bLocalBlockIOSupport = query_local_block_io_supported(engine, config); + params.engineInfo.supports_microkernels = query_microkernels_supported(engine, config); params.engineInfo.deviceType = get_device_type(device_info.dev_type); params.engineInfo.maxWorkGroupSize = device_info.max_work_group_size; params.engineInfo.maxLocalMemSize = device_info.max_local_mem_size; @@ -1092,6 +1135,8 @@ void set_params(const kernel_impl_params& param_info, kernel_selector::params& p params.engineInfo.driverVersion = device_info.driver_version; params.engineInfo.supportedSimdSizes = device_info.supported_simd_sizes; params.engineInfo.vendor_id = device_info.vendor_id; + params.engineInfo.ip_version = device_info.ip_version; + params.engineInfo.arch = kernel_selector::gpu_arch(static_cast::type>(device_info.arch)); auto impl_forcing = config.get_property(ov::intel_gpu::force_implementations); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h index de746e94661ff7..67e2127a064001 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h @@ -294,4 +294,6 @@ inline void update_shapes(kernel_selector::Params& p, const kernel_impl_params& } } +bool query_microkernels_supported(cldnn::engine& e, const cldnn::ExecutionConfig& config); + } // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/scaled_dot_product_attention.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/scaled_dot_product_attention.cpp index 681237b2bcd02b..4bc6e16ce55fa9 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/scaled_dot_product_attention.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/scaled_dot_product_attention.cpp @@ -58,7 +58,7 @@ struct scaled_dot_product_attention_impl : multi_stage_primitive layouts; - if (_kernels_data.size() > 0) { + if (_kernels_data.size() > 0 && !_kernels_data[0].internalBufferSizes.empty()) { auto dtype = from_data_type(_kernels_data[0].internalBufferDataType); const auto bpp = data_type_traits::size_of(dtype); for (auto size : _kernels_data[0].internalBufferSizes) { diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 4b0cc049013ada..1c526ea38188d2 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -223,7 +223,7 @@ void program::init_program() { if (_task_executor == nullptr) _task_executor = program::make_task_executor(_config); _kernels_cache = std::unique_ptr(new kernels_cache(_engine, _config, prog_id, _task_executor, - kernel_selector::KernelBase::get_db().get_batch_header_str())); + kernel_selector::KernelBase::get_db().get_batch_headers())); if (!_compilation_context) _compilation_context = program::make_compilation_context(_config); diff --git a/src/plugins/intel_gpu/src/kernel_selector/CMakeLists.txt b/src/plugins/intel_gpu/src/kernel_selector/CMakeLists.txt index 52c3fb052ff093..993ef596a21256 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/CMakeLists.txt +++ b/src/plugins/intel_gpu/src/kernel_selector/CMakeLists.txt @@ -67,6 +67,10 @@ target_include_directories(${TARGET_NAME} PUBLIC $:$,/Os,-Os>>) +if (ENABLE_ONEDNN_FOR_GPU) + target_link_libraries(${TARGET_NAME} PRIVATE onednn_gpu_tgt) +endif() + if(COMMAND add_cpplint_target) add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME}) endif() diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/generic_vector_ops.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/generic_vector_ops.cl new file mode 100644 index 00000000000000..b9a3df7cc863ea --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/generic_vector_ops.cl @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright 2024 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#ifndef GPU_INTEL_OCL_GENERIC_VECTOR_OPS_H +#define GPU_INTEL_OCL_GENERIC_VECTOR_OPS_H + +typedef half __attribute__((ext_vector_type(1))) half1; +typedef uint __attribute__((ext_vector_type(1))) uint1; +typedef float __attribute__((ext_vector_type(1))) float1; + +float1 __attribute__((overloadable)) vmad(float1 a, float1 b, float1 c) { + c[0] = mad(a[0], b[0], c[0]); + return c; +} +float2 __attribute__((overloadable)) vmad(float2 a, float2 b, float2 c) { + return mad(a, b, c); +} +float4 __attribute__((overloadable)) vmad(float4 a, float4 b, float4 c) { + return mad(a, b, c); +} +float8 __attribute__((overloadable)) vmad(float8 a, float8 b, float8 c) { + return mad(a, b, c); +} +float16 __attribute__((overloadable)) vmad(float16 a, float16 b, float16 c) { + return mad(a, b, c); +} + +float1 __attribute__((overloadable)) native_vrecip(float1 x) { + x[0] = native_recip(x[0]); + return x; +} +float2 __attribute__((overloadable)) native_vrecip(float2 x) { + return native_recip(x); +} +float4 __attribute__((overloadable)) native_vrecip(float4 x) { + return native_recip(x); +} +float8 __attribute__((overloadable)) native_vrecip(float8 x) { + return native_recip(x); +} +float16 __attribute__((overloadable)) native_vrecip(float16 x) { + return native_recip(x); +} + +float1 __attribute__((overloadable)) native_vexp2(float1 x) { + x[0] = native_exp2(x[0]); + return x; +} +float2 __attribute__((overloadable)) native_vexp2(float2 x) { + return native_exp2(x); +} +float4 __attribute__((overloadable)) native_vexp2(float4 x) { + return native_exp2(x); +} +float8 __attribute__((overloadable)) native_vexp2(float8 x) { + return native_exp2(x); +} +float16 __attribute__((overloadable)) native_vexp2(float16 x) { + return native_exp2(x); +} + +#endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sdpa_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sdpa_utils.cl new file mode 100644 index 00000000000000..5943f23251bb7a --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/sdpa_utils.cl @@ -0,0 +1,35 @@ +/******************************************************************************* + * Copyright 2024 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#ifndef GPU_OCL_SDPA_UTILS_H +#define GPU_OCL_SDPA_UTILS_H + +#define _4D_OFF(tag, x0, x1, x2, x3) \ + (((x0) % tag##_B0) * tag##_SB0 + ((x0) / tag##_B0) * tag##_S0 \ + + ((x1) % tag##_B1) * tag##_SB1 + ((x1) / tag##_B1) * tag##_S1 \ + + ((x2) % tag##_B2) * tag##_SB2 + ((x2) / tag##_B2) * tag##_S2 \ + + ((x3) % tag##_B3) * tag##_SB3 + ((x3) / tag##_B3) * tag##_S3) + +#define QRY_OFF(x0, x1, x2, x3) _4D_OFF(QRY, x0, x1, x2, x3) +#define KEY_OFF(x0, x1, x2, x3) _4D_OFF(KEY, x0, x1, x2, x3) +#define VAL_OFF(x0, x1, x2, x3) _4D_OFF(VAL, x0, x1, x2, x3) +#define MSK_OFF(x0, x1, x2, x3) _4D_OFF(MSK, x0, x1, x2, x3) + +#define DST_OFF(x0, x1, d, h, w) \ +(((x0) % DST_B0) * DST_SB0 + ((x0) / DST_B0) * DST_S0 \ ++ ((x1) % DST_B1) * DST_SB1 + ((x1) / DST_B1) * DST_S1) + +#endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/tile_ops.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/tile_ops.cl new file mode 100644 index 00000000000000..b2ce72c33fc31c --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/tile_ops.cl @@ -0,0 +1,628 @@ +/******************************************************************************* + * Copyright 2024 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#ifndef GPU_OCL_TILE_OPS_H +#define GPU_OCL_TILE_OPS_H + +float __builtin_IB_atomic_max_local_f32(__local float *, float); + +__attribute__((overloadable)) float local_atomic_max(local float *p, float v) { + return __builtin_IB_atomic_max_local_f32(p, v); +} + +__attribute__((overloadable)) half local_atomic_max( + local half *p, half v) { /* not implemented */ + return v; +} + +__attribute__((overloadable)) uint local_atomic_max(local uint *p, uint v) { + return atomic_max(p, v); +} + +__attribute__((overloadable)) int local_atomic_max(local int *p, int v) { + return atomic_max(p, v); +} + +#define DEF_BLOCK_LOAD_STORE(type, itype, suffix, n) \ + __attribute__((overloadable)) type##n block_load( \ + const global type *p, int vlen) \ + __attribute__((enable_if(vlen == n, "wrong vector length"))) { \ + return as_##type##n( \ + intel_sub_group_block_read##suffix##n((global void *)p)); \ + } \ + __attribute__((overloadable)) void block_store( \ + global type *p, type##n v) { \ + intel_sub_group_block_write##suffix##n( \ + (global itype *)p, as_##itype##n(v)); \ + } + +#define DEF_BLOCK_LOAD_STORE1(type, itype, suffix) \ + __attribute__((overloadable)) \ + type##1 block_load(const global type *p, int vlen) __attribute__( \ + (enable_if(vlen == 1, "wrong vector length"))) { \ + type##1 x; \ + x[0] = as_##type( \ + intel_sub_group_block_read##suffix((global void *)p)); \ + return x; \ + } \ + __attribute__((overloadable)) void block_store( \ + global type *p, type##1 v) { \ + intel_sub_group_block_write##suffix( \ + (global itype *)p, as_##itype(v[0])); \ + } + +DEF_BLOCK_LOAD_STORE1(half, ushort, _us) +DEF_BLOCK_LOAD_STORE(half, ushort, _us, 2) +DEF_BLOCK_LOAD_STORE(half, ushort, _us, 4) +DEF_BLOCK_LOAD_STORE(half, ushort, _us, 8) +DEF_BLOCK_LOAD_STORE(half, ushort, _us, 16) +DEF_BLOCK_LOAD_STORE1(uint, uint, ) +DEF_BLOCK_LOAD_STORE(uint, uint, , 2) +DEF_BLOCK_LOAD_STORE(uint, uint, , 4) +DEF_BLOCK_LOAD_STORE(uint, uint, , 8) + +#define DEF_BLOCK2D_LOAD_STORE(type, itype, vl, SG, suffix, BR, BC) \ + itype##vl __builtin_IB_subgroup_block_read_flat_##suffix( \ + long, int, int, int, int2); \ + void __builtin_IB_subgroup_block_write_flat_##suffix( \ + long, int, int, int, int2, itype##vl); \ + __attribute__((overloadable)) type##vl block2d_load(const global type *p, \ + int w, int h, int ld, int x, int y, int br, int bc, \ + int sg) __attribute__((enable_if(br == BR, "wrong #rows"))) \ + __attribute__((enable_if(bc == BC, "wrong #columns"))) \ + __attribute__( \ + (enable_if(sg == SG, "wrong subgroup size"))) { \ + ulong pp = as_long(p); \ + ulong prem = pp & 0x3F; \ + pp &= ~0x3F; \ + x += (prem / sizeof(type)); \ + w += prem; \ + int2 coord = {x, y}; \ + return as_##type##vl(__builtin_IB_subgroup_block_read_flat_##suffix( \ + pp, w - 1, h - 1, ld - 1, coord)); \ + } \ + __attribute__((overloadable)) void block2d_store(type##vl v, \ + global type *p, int w, int h, int ld, int x, int y, int br, \ + int bc, \ + int sg) __attribute__((enable_if(br == BR, "wrong #rows"))) \ + __attribute__((enable_if(bc == BC, "wrong #columns"))) \ + __attribute__( \ + (enable_if(sg == SG, "wrong subgroup size"))) { \ + ulong pp = as_long(p); \ + ulong prem = pp & 0x3F; \ + pp &= ~0x3F; \ + x += (prem / sizeof(type)); \ + w += prem; \ + int2 coord = {x, y}; \ + __builtin_IB_subgroup_block_write_flat_##suffix( \ + pp, w - 1, h - 1, ld - 1, coord, as_##itype##vl(v)); \ + } + +DEF_BLOCK2D_LOAD_STORE(half, ushort, 8, 16, u16_m8k16v1, 16, 8) +DEF_BLOCK2D_LOAD_STORE(half, ushort, 8, 16, u16_m4k32v1, 32, 4) +DEF_BLOCK2D_LOAD_STORE(half, ushort, 16, 16, u16_m8k32v1, 32, 8) + +#define tile_fill(t, v) \ + do { \ + _Pragma("unroll") for (int i = 0; i < sizeof(t.x) / sizeof(t.x[0]); \ + i++) t.x[i] \ + = v; \ + } while (0) + +#define tile_elementwise(t, f) \ + do { \ + _Pragma("unroll") for (int i = 0; i < sizeof(t.x) / sizeof(t.x[0]); \ + i++) t.x[i] \ + = f(t.x[i]); \ + } while (0) + +#define tile_elementwise_s(t, f) \ + do { \ + _Pragma("unroll") for (int i = 0; i < sizeof(t.x) / sizeof(t.x[0]); \ + i++) { \ + _Pragma("unroll") for (int s = 0; \ + s < sizeof(t.x[0]) / sizeof(t.x[0][0]); \ + s++) t.x[i][s] \ + = f(t.x[i][s]); \ + } \ + } while (0) + +#define tile_binary(t, t2, f) \ + do { \ + _Pragma("unroll") for (int i = 0; i < sizeof(t.x) / sizeof(t.x[0]); \ + i++) t.x[i] \ + = f(t.x[i], t2.x[i]); \ + } while (0) + +#define tile_copy(t, t_new) \ + do { \ + _Pragma("unroll") for (int i = 0; i < sizeof(t.x) / sizeof(t.x[0]); \ + i++) t_new.x[i] \ + = __builtin_convertvector(t.x[i], __typeof__(t_new.x[i])); \ + } while (0) + +#define tile_copy_to_half2(t, t_new) \ + do { \ + _Pragma("unroll") for (int i = 0; i < sizeof(t.x) / sizeof(t.x[0]); \ + i++) { \ + _Pragma("unroll") for (int s = 0; \ + s < sizeof(t.x[0]) / sizeof(t.x[0][0]) / 2; \ + s++) { \ + half2 v = {t.x[i][2 * s], t.x[i][2 * s + 1]}; \ + t_new.x[i][s] = as_uint(v); \ + } \ + } \ + } while (0) + +#define tile_access(t, i0, j, sg, br, bc, nbr) \ + (t).x[(i0) / (br) + (nbr) * ((j) / (bc))] \ + [((i0) % (br)) / (sg) + ((j) % (bc)) * ((br) / (sg))] + +#define xlane_tile_access(t, i, j, sg, br, bc, nbr) \ + sub_group_broadcast(tile_access(t, i, j, sg, br, bc, nbr), i % sg) + +#define DECLARE_2D_TILE_OPS(tile_type, element_type, sg, br, bc, nbr, nbc) \ + __attribute__((overloadable)) void tile_load_full(tile_type *t, \ + const global element_type *ptr, int ld, int offset_r, \ + int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = i0 + get_sub_group_local_id(); \ + tile_access(*t, i0, j, sg, br, bc, nbr) = ptr[i]; \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_load_full(tile_type *t, \ + const local element_type *ptr, int ld, int offset_r, \ + int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = i0 + get_sub_group_local_id(); \ + tile_access(*t, i0, j, sg, br, bc, nbr) = ptr[i]; \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_load(tile_type *t, \ + const global element_type *ptr, int m, int n, int ld, \ + int offset_r, int offset_c) { \ + if (m >= offset_r + br * nbr && n >= offset_c + bc * nbc) { \ + tile_load_full(t, ptr, ld, offset_r, offset_c); \ + return; \ + } \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + if (offset_c + j < n) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = i0 + get_sub_group_local_id(); \ + if (offset_r + i < m) \ + tile_access(*t, i0, j, sg, br, bc, nbr) = ptr[i]; \ + } \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_load(tile_type *t, \ + const global element_type *ptr, int m, int n, int offset_r, \ + int offset_c) { \ + tile_load(t, ptr, m, n, m, offset_r, offset_c); \ + } \ + __attribute__((overloadable)) void tile_load_t_full(tile_type *t, \ + const global element_type *ptr, int ld, int offset_r, \ + int offset_c) { \ + ptr += ld * offset_r + offset_c; \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg, ptr += ld*sg) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + tile_access(*t, i0, j, sg, br, bc, nbr) = ptr[get_sub_group_local_id() * ld + j]; \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_load_t(tile_type *t, \ + const global element_type *ptr, int m, int n, int ld, \ + int offset_r, int offset_c) { \ + if (m >= offset_r + br * nbr && n >= offset_c + bc * nbc) { \ + tile_load_t_full(t, ptr, ld, offset_r, offset_c); \ + return; \ + } \ + ptr += ld * offset_r + offset_c; \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg, ptr += ld*sg) { \ + int i = i0 + get_sub_group_local_id(); \ + if (offset_r + i < m) \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + if (offset_c + j < n) { \ + tile_access(*t, i0, j, sg, br, bc, nbr) = ptr[get_sub_group_local_id() * ld + j]; \ + } \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_load_t(tile_type *t, \ + const global element_type *ptr, int m, int n, int offset_r, \ + int offset_c) { \ + tile_load(t, ptr, m, n, n, offset_r, offset_c); \ + } \ + __attribute__((overloadable)) void tile_store_full(tile_type t, \ + local element_type *ptr, int ld, int offset_r, int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = i0 + get_sub_group_local_id(); \ + ptr[i] = tile_access(t, i0, j, sg, br, bc, nbr); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_store_full(tile_type t, \ + global element_type *ptr, int ld, int offset_r, int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = i0 + get_sub_group_local_id(); \ + ptr[i] = tile_access(t, i0, j, sg, br, bc, nbr); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_store(tile_type t, \ + global element_type *ptr, int m, int n, int ld, int offset_r, \ + int offset_c) { \ + if (m >= offset_r + br * nbr && n >= offset_c + bc * nbc) { \ + tile_store_full(t, ptr, ld, offset_r, offset_c); \ + return; \ + } \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + if (offset_c + j < n) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = i0 + get_sub_group_local_id(); \ + if (offset_r + i < m) \ + ptr[i] = tile_access(t, i0, j, sg, br, bc, nbr); \ + } \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_store(tile_type t, \ + global element_type *ptr, int m, int n, int offset_r, \ + int offset_c) { \ + tile_store(t, ptr, m, n, m, offset_r, offset_c); \ + } \ + __attribute__((overloadable)) void tile_store_t_sys_src1(tile_type t, \ + local element_type *ptr, int ld, int offset_r, int offset_c) { \ + offset_c += get_sub_group_local_id(); \ + int offset_r0 = offset_r & (sg - 1); \ + int offset_r1 = offset_r & ~(sg - 1); \ + ptr += offset_r0 + sg * offset_c + ld * offset_r1; \ + _Pragma("unroll") for (int j0 = 0; j0 < br * nbr; \ + j0 += sg, ptr += sg * sg) { \ + _Pragma("unroll") for (int i = 0; i < bc * nbc; i++) ptr[i] \ + = tile_access(t, j0, i, sg, br, bc, nbr); \ + } \ + } \ + __attribute__((overloadable)) void tile_store_t_sys_src2(tile_type t, \ + local element_type *ptr, int tile_n, int ld, int offset_r, \ + int offset_c) { \ + const int cp = 32 / sizeof(element_type); \ + offset_c += get_sub_group_local_id(); \ + int offset_r0 = offset_r & (cp - 1); \ + int offset_r1 = offset_r & ~(cp - 1); \ + ptr += offset_r0 + tile_n * offset_r1; \ + _Pragma("unroll") for (int j0 = 0; j0 < br * nbr; \ + j0 += sg, offset_c += sg) { \ + int offset_c0 = offset_c & (tile_n - 1); \ + int offset_c1 = offset_c & ~(tile_n - 1); \ + local element_type *ptr_j = ptr + cp * offset_c0 + ld * offset_c1; \ + _Pragma("unroll") for (int i = 0; i < bc * nbc; i++) { \ + *ptr_j = tile_access(t, j0, i, sg, br, bc, nbr); \ + ptr_j++; \ + if ((~i & (cp - 1)) == 0) ptr_j += cp * (tile_n - 1); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_atomic_max_full(tile_type t, \ + local element_type *ptr, int ld, int offset_r, int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = i0 + get_sub_group_local_id(); \ + (void)local_atomic_max( \ + ptr + i, tile_access(t, i0, j, sg, br, bc, nbr)); \ + } \ + } \ + } + +#define DECLARE_2D_TILE_VREDUCE(tile_type, sg, br, bc, nbr, nbc, rtile_type, \ + rsg, rbr, rbc, rnbr, rnbc) \ + __attribute__((overloadable)) void tile_vreduce_add( \ + tile_type t, rtile_type *tr) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + tile_access(*tr, i0, 0, rsg, rbr, rbc, rnbr) \ + += tile_access(t, i0, j, sg, br, bc, nbr); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_vreduce_max( \ + tile_type t, rtile_type *tr) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + tile_access(*tr, i0, 0, rsg, rbr, rbc, rnbr) \ + = max(tile_access(t, i0, j, sg, br, bc, nbr), \ + tile_access(*tr, i0, 0, rsg, rbr, rbc, rnbr)); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_vbroadcast_sub( \ + tile_type *t, rtile_type tr) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + tile_access(*t, i0, j, sg, br, bc, nbr) \ + -= tile_access(tr, i0, 0, rsg, rbr, rbc, rnbr); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_vbroadcast_mul( \ + tile_type *t, rtile_type tr) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + tile_access(*t, i0, j, sg, br, bc, nbr) \ + *= tile_access(tr, i0, 0, rsg, rbr, rbc, rnbr); \ + } \ + } \ + } + +#define DECLARE_2D_TILE_HREDUCE(tile_type, sg, br, bc, nbr, nbc, rtile_type, \ + rsg, rbr, rbc, rnbr, rnbc) \ + __attribute__((overloadable)) void tile_hbroadcast_add( \ + tile_type *t, rtile_type tr) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + tile_access(*t, i0, j, sg, br, bc, nbr) \ + += xlane_tile_access(tr, j, 0, rsg, rbr, rbc, rnbr); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_hbroadcast_mul( \ + tile_type *t, rtile_type tr) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + tile_access(*t, i0, j, sg, br, bc, nbr) \ + *= xlane_tile_access(tr, j, 0, rsg, rbr, rbc, rnbr); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_hbroadcast_min( \ + tile_type *t, rtile_type tr) { \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + tile_access(*t, i0, j, sg, br, bc, nbr) = min( \ + tile_access(*t, i0, j, sg, br, bc, nbr), \ + xlane_tile_access(tr, j, 0, rsg, rbr, rbc, rnbr)); \ + } \ + } \ + } + +#define DECLARE_2D_TILE_RSELECT(tile_type0, sg0, br0, bc0, nbr0, nbc0, \ + tile_type1, sg1, br1, bc1, nbr1, nbc1) \ + __attribute__((overloadable)) void tile_rselect( \ + tile_type0 *t0, tile_type1 t1, int idx) { \ + _Pragma("unroll") for (int j = 0; j < bc0 * nbc0; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br0 * nbr0; i0 += sg0) { \ + tile_access(*t0, i0, j, sg0, br0, bc0, nbr0) \ + = tile_access(t1, i0, j, sg1, br1, bc1, nbr1); \ + _Pragma("unroll") for (int z = 1; \ + z < (br1 * nbr1 / br0 * nbr0); \ + z++) if (z == idx) { \ + tile_access(*t0, i0, j, sg0, br0, bc0, nbr0) \ + = tile_access(t1, i0 + z * br0 * nbr0, j, sg1, \ + br1, bc1, nbr1); \ + } \ + } \ + } \ + } + +#define DECLARE_2D_TILE_COPY_REBLOCK(tile_type0, sg0, br0, bc0, nbr0, nbc0, \ + tile_type1, sg1, br1, bc1, nbr1, nbc1) \ + __attribute__((overloadable)) void tile_copy_reblock( \ + tile_type0 t0, tile_type1 *t1) { \ + _Pragma("unroll") for (int j = 0; j < bc0 * nbc0; j++) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br0 * nbr0; i0 += sg0) { \ + tile_access(*t1, i0, j, sg1, br1, bc1, nbr1) \ + = tile_access(t0, i0, j, sg0, br0, bc0, nbr0); \ + } \ + } \ + } + +#define DECLARE_2D_TILE(tile_type, element_type, sg, br, bc, nbr, nbc) \ + typedef element_type __attribute__((ext_vector_type(br * bc / sg))) \ + _e_##tile_type; \ + typedef struct { \ + _e_##tile_type x[nbr * nbc]; \ + } tile_type; \ + DECLARE_2D_TILE_OPS(tile_type, element_type, sg, br, bc, nbr, nbc) + +/* Requires bc == 1 currently */ +#define DECLARE_2D_TILE_BLOCK_OPS( \ + tile_type, element_type, sg, br, bc, nbr, nbc) \ + __attribute__((overloadable)) void tile_load_block(tile_type *t, \ + const global element_type *ptr, int ld, int offset_r, \ + int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int jj = 0; jj < nbc; jj++, ptr += ld * bc) { \ + _Pragma("unroll") for (int ii = 0; ii < nbr; ii++)(t) \ + ->x[ii + nbr * jj] \ + = block_load(ptr + ii * br, br / SUBGROUP_SIZE); \ + } \ + } \ + __attribute__((overloadable)) void tile_store_block(tile_type t, \ + global element_type *ptr, int ld, int offset_r, int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int jj = 0; jj < nbc; jj++, ptr += ld * bc) { \ + _Pragma("unroll") for (int ii = 0; ii < nbr; ii++) \ + block_store(ptr + ii * br, (t).x[ii + nbr * jj]); \ + } \ + } \ + __attribute__((overloadable)) void tile_load_block(tile_type *t, \ + const global element_type *ptr, int n, int ld, int offset_r, \ + int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + n -= offset_c; \ + _Pragma("unroll") for (int jj = 0; jj < nbc; jj++, ptr += ld * bc) { \ + if (jj < n) { \ + _Pragma("unroll") for (int ii = 0; ii < nbr; ii++)(t) \ + ->x[ii + nbr * jj] \ + = block_load(ptr + ii * br, br / SUBGROUP_SIZE); \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_store_block(tile_type t, \ + global element_type *ptr, int n, int ld, int offset_r, \ + int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + n -= offset_c; \ + _Pragma("unroll") for (int jj = 0; jj < nbc; jj++, ptr += ld * bc) { \ + if (jj < n) { \ + _Pragma("unroll") for (int ii = 0; ii < nbr; ii++) \ + block_store(ptr + ii * br, (t).x[ii + nbr * jj]); \ + } \ + } \ + } + +#define DECLARE_2D_TILE_BLOCK2D_OPS( \ + tile_type, element_type, sg, br, bc, nbr, nbc) \ + __attribute__((overloadable)) void tile_load_block2d(tile_type *t, \ + const global element_type *ptr, int m, int n, int ld, \ + int offset_r, int offset_c) { \ + const int e = sizeof(element_type); \ + _Pragma("unroll") for (int jj = 0; jj < nbc; jj++) { \ + _Pragma("unroll") for (int ii = 0; ii < nbr; ii++)(t) \ + ->x[ii + nbr * jj] \ + = block2d_load(ptr, m * e, n, ld * e, offset_r + ii * br, \ + offset_c + jj * bc, br, bc, sg); \ + } \ + } \ + __attribute__((overloadable)) void tile_load_block2d(tile_type *t, \ + const global element_type *ptr, int m, int n, int offset_r, \ + int offset_c) { \ + tile_load_block2d(t, ptr, m, n, m, offset_r, offset_c); \ + } \ + __attribute__((overloadable)) void tile_store_block2d(tile_type t, \ + global element_type *ptr, int m, int n, int ld, int offset_r, \ + int offset_c) { \ + const int e = sizeof(element_type); \ + _Pragma("unroll") for (int jj = 0; jj < nbc; jj++) { \ + _Pragma("unroll") for (int ii = 0; ii < nbr; ii++) block2d_store( \ + (t).x[ii + nbr * jj], ptr, m *e, n, ld *e, \ + offset_r + ii * br, offset_c + jj * bc, br, bc, sg); \ + } \ + } \ + __attribute__((overloadable)) void tile_store_block2d(tile_type t, \ + const global element_type *ptr, int m, int n, int offset_r, \ + int offset_c) { \ + tile_store_block2d(t, ptr, m, n, m, offset_r, offset_c); \ + } + +#define DECLARE_2D_TILE_LOAD_PACKED_HALF(tile_type, sg, br, bc, nbr, nbc) \ + __attribute__((overloadable)) void tile_load_packed_half(tile_type *t, \ + const global half *ptr, int m, int n, int ld, int offset_r, \ + int offset_c) { \ + ptr += ld * offset_c + offset_r; \ + _Pragma("unroll") for (int j = 0; j < bc * nbc; j++, ptr += ld) { \ + if (offset_c + j < n) { \ + _Pragma("unroll") for (int i0 = 0; i0 < br * nbr; i0 += sg) { \ + int i = 2 * (i0 + get_sub_group_local_id()); \ + half2 loaded = 0; \ + if (offset_r + i < m) loaded.s0 = ptr[i]; \ + if (offset_r + i + 1 < m) loaded.s1 = ptr[i + 1]; \ + tile_access(*t, i0, j, sg, br, bc, nbr) = as_uint(loaded); \ + } \ + } \ + } \ + } \ + __attribute__((overloadable)) void tile_load_packed_half(tile_type *t, \ + const global half *ptr, int m, int n, int offset_r, \ + int offset_c) { \ + tile_load_packed_half(t, ptr, m, n, m, offset_r, offset_c); \ + } + +#define cooperative_prefetch_2d(ptr, r, c, ld, sg_id, n_sg, sg_size, caching) \ + cooperative_prefetch_2d_internal((const global char *)ptr, \ + (r) * sizeof(*(ptr)), c, (ld) * sizeof(*(ptr)), sg_id, n_sg, \ + sg_size, caching) + +#define cooperative_prefetch_2d_rem( \ + ptr, r, c, rmax, cmax, ld, sg_id, n_sg, sg_size, caching) \ + cooperative_prefetch_2d_internal((const global char *)ptr, \ + (r) * sizeof(*(ptr)), c, (rmax) * sizeof(*(ptr)), cmax, \ + (ld) * sizeof(*(ptr)), sg_id, n_sg, sg_size, caching) + +/* IGC prefetch intrinsics */ +enum LSC_LDCC { + LSC_LDCC_DEFAULT = 0, + LSC_LDCC_L1UC_L3UC = 1, + LSC_LDCC_L1UC_L3C = 2, + LSC_LDCC_L1C_L3UC = 3, + LSC_LDCC_L1C_L3C = 4, + LSC_LDCC_L1S_L3UC = 5, + LSC_LDCC_L1S_L3C = 6, + LSC_LDCC_L1IAR_L3C = 7, +}; + +extern void __builtin_IB_lsc_prefetch_global_uchar( + const __global uchar *base, int immElemOff, enum LSC_LDCC cacheOpt); + +extern void __builtin_IB_lsc_prefetch_global_uint( + const __global uint *base, int immElemOff, enum LSC_LDCC cacheOpt); + +__attribute__((overloadable)) void cooperative_prefetch_2d_internal( + const global char *ptr, uint rbytes, uint c, uint ld_bytes, uint sg_id, + uint n_sg, uint sg_size, enum LSC_LDCC caching) { + const uint cl_per_col = (rbytes + 63) >> 6; + const uint cl = cl_per_col * c; + const uint cl_per_sg = (cl + n_sg - 1) / n_sg; + const uint cl_iters = (cl_per_sg + sg_size - 1) / sg_size; +#pragma unroll + for (uint ii_cl = 0; ii_cl < cl_iters; ii_cl++) { + uint i_cl = ii_cl + (sg_id * cl_per_sg) + get_sub_group_local_id(); + uint r_cl = i_cl % cl_per_col; + uint c_cl = i_cl / cl_per_col; + if (i_cl < cl) { + __builtin_IB_lsc_prefetch_global_uint( + (const global uint *)(ptr + r_cl * 64 + c_cl * ld_bytes), 0, + caching); + } + } +} + +__attribute__((overloadable)) void cooperative_prefetch_2d_internal( + const global char *ptr, uint rbytes, uint c, uint rbytes_max, + uint c_max, uint ld_bytes, uint sg_id, uint n_sg, uint sg_size, + enum LSC_LDCC caching) { + const uint cl_per_col = (rbytes_max + 63) >> 6; + const uint cl = cl_per_col * c_max; + const uint cl_per_sg = (cl + n_sg - 1) / n_sg; + const uint cl_iters = (cl_per_sg + sg_size - 1) / sg_size; + const uint max_off = rbytes - 1 + (c - 1) * ld_bytes; +#pragma unroll + for (uint ii_cl = 0; ii_cl < cl_iters; ii_cl++) { + uint i_cl = ii_cl + (sg_id * cl_per_sg) + get_sub_group_local_id(); + uint r_cl = i_cl % cl_per_col; + uint c_cl = i_cl / cl_per_col; + uint pf_off = min(r_cl * 64 + c_cl * ld_bytes, max_off); + if (i_cl < cl) { + __builtin_IB_lsc_prefetch_global_uchar( + (const global uchar *)(ptr + pf_off), 0, caching); + } + } +} + +#endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/vec_typedefs.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/vec_typedefs.cl index ae4adfaf717fbd..29ca3ddb7185f7 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/vec_typedefs.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/vec_typedefs.cl @@ -2,27 +2,25 @@ // SPDX-License-Identifier: Apache-2.0 // -typedef struct half1 { half s0; } half1; typedef struct half5 { half s0; half s1; half s2; half s3; half s4; } half5; typedef struct half6 { half s0; half s1; half s2; half s3; half s4; half s5; } half6; typedef struct half7 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; } half7; -typedef struct half9 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; +typedef struct half9 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; half s8; } half9; -typedef struct half10 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; +typedef struct half10 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; half s8; half s9; } half10; -typedef struct half11 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; +typedef struct half11 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; half s8; half s9; half sa; } half11; -typedef struct half12 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; +typedef struct half12 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; half s8; half s9; half sa; half sb;} half12; -typedef struct half13 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; +typedef struct half13 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; half s8; half s9; half sa; half sb; half sc;} half13; -typedef struct half14 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; +typedef struct half14 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; half s8; half s9; half sa; half sb; half sc; half se;} half14; -typedef struct half15 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; +typedef struct half15 { half s0; half s1; half s2; half s3; half s4; half s5; half s6; half s7; half s8; half s9; half sa; half sb; half sc; half se; half sf;} half15; typedef struct half0 { half s0; } half0; //never used but makes compiler happy. -typedef struct float1 { float s0; } float1; typedef struct float5 { float s0; float s1; float s2; float s3; float s4; } float5; typedef struct float6 { float s0; float s1; float s2; float s3; float s4; float s5; } float6; typedef struct float7 { float s0; float s1; float s2; float s3; float s4; float s5; float s6; } float7; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_micro.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_micro.cl new file mode 100644 index 00000000000000..575d9fdff5bd32 --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_micro.cl @@ -0,0 +1,461 @@ +/******************************************************************************* +* Copyright 2024 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#include "include/batch_headers/generic_vector_ops.cl" +#include "include/batch_headers/sdpa_utils.cl" +#include "include/batch_headers/tile_ops.cl" + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define DIV_UP(x, y) (((x) + (y)-1) / (y)) + +#define sg_per_wg (ugemm_kq_sg_per_wg_m * ugemm_kq_sg_per_wg_n) +#define q_tile_sg_n DIV_UP(ugemm_kq_wg_tile_n, sg_per_wg) + +/* Instantiate tile types and operations */ +typedef ugemm_kq_c_type s_tile_type; +typedef ugemm_vs_c_type a_tile_type; + +DECLARE_2D_TILE(q_tile_type, uint, SUBGROUP_SIZE, D_MAX / 2, 1, 1, q_tile_sg_n) + +#ifdef BLOCK_Q +DECLARE_2D_TILE_BLOCK_OPS( + q_tile_type, uint, SUBGROUP_SIZE, D_MAX / 2, 1, 1, q_tile_sg_n) +#elif Q_ALIGN < 4 +DECLARE_2D_TILE_LOAD_PACKED_HALF( + q_tile_type, SUBGROUP_SIZE, D_MAX / 2, 1, 1, q_tile_sg_n) +#endif + +#ifdef BLOCK_A +DECLARE_2D_TILE(a_tile_type_half, half, SUBGROUP_SIZE, ugemm_vs_sg_tile_m, 1, 1, + ugemm_vs_sg_tile_n) +#else +DECLARE_2D_TILE(a_tile_type_half, half, SUBGROUP_SIZE, ugemm_vs_sg_tile_m, 8, 1, + ugemm_vs_sg_tile_n / 8) +#endif + +DECLARE_2D_TILE(s_tile_type_half2, uint, SUBGROUP_SIZE, ugemm_kq_c_type_block0, + ugemm_kq_c_type_block1 / 2, ugemm_kq_c_type_nblock0, + ugemm_kq_c_type_nblock1) + +DECLARE_2D_TILE( + s_sum_tile_type, float, SUBGROUP_SIZE, ugemm_kq_sg_tile_n, 1, 1, 1) + +DECLARE_2D_TILE( + a_scale_tile_type, float, SUBGROUP_SIZE, ugemm_vs_sg_tile_n, 1, 1, 1) + + +DECLARE_2D_TILE(mask_tile_type, half, SUBGROUP_SIZE, ugemm_kq_c_type_block0, ugemm_kq_c_type_block1, ugemm_kq_c_type_nblock0, ugemm_kq_c_type_nblock1) +DECLARE_2D_TILE(mask_tile_type_float, float, SUBGROUP_SIZE, ugemm_kq_c_type_block0, ugemm_kq_c_type_block1, ugemm_kq_c_type_nblock0, ugemm_kq_c_type_nblock1) + +#ifdef BLOCK_A +DECLARE_2D_TILE_BLOCK_OPS(a_tile_type_half, half, SUBGROUP_SIZE, + ugemm_vs_sg_tile_m, 1, 1, ugemm_vs_sg_tile_n) +#endif +#ifdef BLOCK_2D_A +DECLARE_2D_TILE_BLOCK2D_OPS(a_tile_type_half, half, SUBGROUP_SIZE, + ugemm_vs_sg_tile_m, 8, 1, ugemm_vs_sg_tile_n / 8) +#endif + +#ifdef BLOCK_A +DECLARE_2D_TILE_COPY_REBLOCK(a_tile_type, SUBGROUP_SIZE, ugemm_vs_c_type_block0, + ugemm_vs_c_type_block1, ugemm_vs_c_type_nblock0, + ugemm_vs_c_type_nblock1, a_tile_type_half, SUBGROUP_SIZE, + ugemm_vs_sg_tile_m, 1, 1, ugemm_vs_sg_tile_n) +#else +DECLARE_2D_TILE_COPY_REBLOCK(a_tile_type, SUBGROUP_SIZE, ugemm_vs_c_type_block0, + ugemm_vs_c_type_block1, ugemm_vs_c_type_nblock0, + ugemm_vs_c_type_nblock1, a_tile_type_half, SUBGROUP_SIZE, + ugemm_vs_sg_tile_m, 8, 1, ugemm_vs_sg_tile_n / 8) +#endif + +DECLARE_2D_TILE_VREDUCE(s_tile_type, SUBGROUP_SIZE, ugemm_kq_c_type_block0, + ugemm_kq_c_type_block1, ugemm_kq_c_type_nblock0, + ugemm_kq_c_type_nblock1, s_sum_tile_type, SUBGROUP_SIZE, + ugemm_kq_sg_tile_n, 1, 1, 1) + +DECLARE_2D_TILE_HREDUCE(s_tile_type, SUBGROUP_SIZE, ugemm_kq_c_type_block0, + ugemm_kq_c_type_block1, ugemm_kq_c_type_nblock0, + ugemm_kq_c_type_nblock1, mask_tile_type_float, SUBGROUP_SIZE, + ugemm_kq_sg_tile_m, 1, 1, 1) + +DECLARE_2D_TILE_HREDUCE(a_tile_type, SUBGROUP_SIZE, ugemm_vs_c_type_block0, + ugemm_vs_c_type_block1, ugemm_vs_c_type_nblock0, + ugemm_vs_c_type_nblock1, a_scale_tile_type, SUBGROUP_SIZE, + ugemm_vs_sg_tile_n, 1, 1, 1) + +#if ugemm_kq_wg_tile_n == ugemm_vs_wg_tile_n \ + && (ugemm_kq_sg_tile_n % ugemm_vs_sg_tile_n) == 0 +DECLARE_2D_TILE_RSELECT(a_scale_tile_type, SUBGROUP_SIZE, ugemm_vs_sg_tile_n, 1, + 1, 1, s_sum_tile_type, SUBGROUP_SIZE, ugemm_kq_sg_tile_n, 1, 1, 1) +#endif + +#if PREFETCH_REMAINDER +#define cooperative_prefetch_2d_maybe_rem cooperative_prefetch_2d_rem +#else +#define cooperative_prefetch_2d_maybe_rem( \ + ptr, r, c, rmax, cmax, ld, sg_id, n_sg, sg_size, caching) \ + cooperative_prefetch_2d(ptr, rmax, cmax, ld, sg_id, n_sg, sg_size, caching) +#endif + +#if TRANSPOSE_K +#define cooperative_prefetch_2d_k( \ + ptr, r, c, rmax, cmax, ld, sg_id, n_sg, sg_size, caching) \ + cooperative_prefetch_2d_maybe_rem( \ + ptr, c, r, cmax, rmax, ld, sg_id, n_sg, sg_size, caching) +#else +#define cooperative_prefetch_2d_k cooperative_prefetch_2d_maybe_rem +#endif + +#if REMAINDER_Q +#define tile_load_block_rem_q tile_load_block +#define tile_store_block_rem_q tile_store_block +#else +#define tile_load_block_rem_q(t, ptr, n, ld, off_r, off_c) \ + tile_load_block(t, ptr, ld, off_r, off_c) +#define tile_store_block_rem_q(t, ptr, n, ld, off_r, off_c) \ + tile_store_block(t, ptr, ld, off_r, off_c) +#endif + +#define binary_add(x, y) ((x) + (y)) + +__attribute__((intel_reqd_sub_group_size(SUBGROUP_SIZE))) +KERNEL(micro_sdpa)(OPTIONAL_SHAPE_INFO_ARG + const global half *K, const global half *Q, const global half *V, + global half *A, +#if WITH_ATTN_MASK + const global half *msk, +#endif +#if WITH_SCALE + global SCALE_DATA_T *scale_ptr, +#endif + int d, int k, int q) { + uint sg_ij = sub_group_broadcast(get_local_id(1), 0); + uint b0 = get_group_id(1); + uint b1 = get_group_id(2); + + uint wg_j0 = get_group_id(0) * ugemm_kq_wg_tile_n; + + /* Leading dimension for matrices */ + uint ldk = TRANSPOSE_K ? KEY_S3 : KEY_S2; + uint ldq = QRY_S2; + uint ldv = VAL_S2; + uint lda = DST_S2; + + /* Subgroup IDs for each GEMM */ + uint sg_i_kq = sg_ij % ugemm_kq_sg_per_wg_m; + uint sg_j_kq = sg_ij / ugemm_kq_sg_per_wg_m; + + uint sg_i_vs = sg_ij % ugemm_vs_sg_per_wg_m; + uint sg_j_vs = sg_ij / ugemm_vs_sg_per_wg_m; + + /* SLM allocations -- place in one array to work around compiler bug */ +#define Q_slm_size (D_MAX * ugemm_kq_wg_tile_n * sizeof(half)) +#define S_slm_size (ugemm_kq_wg_tile_m * ugemm_kq_wg_tile_n * sizeof(half)) +#define S_sum_slm_size \ + (ugemm_kq_wg_tile_n * ugemm_kq_sg_per_wg_m * sizeof(float)) +#define S_max_slm_size (ugemm_kq_wg_tile_n * sizeof(float)) +#define ugemm_slm_size MAX(ugemm_kq_slm_size, ugemm_vs_slm_size) + + local char slm[Q_slm_size + S_slm_size + S_sum_slm_size + S_max_slm_size + + ugemm_slm_size]; + + local half *Q_slm = (local half *)&slm[0]; + local half *S_slm = (local half *)&slm[Q_slm_size]; + local float *S_sum_slm = (local float *)&slm[Q_slm_size + S_slm_size]; + local float *S_max_slm + = (local float *)&slm[Q_slm_size + S_slm_size + S_sum_slm_size]; + local uint *ugemm_slm = (local uint *)&slm[Q_slm_size + S_slm_size + + S_sum_slm_size + S_max_slm_size]; + + const bool need_sum_barrier = (ugemm_vs_barrier_count == 0); + + /* Locate K/Q/V/A matrices within batch */ + K += KEY_OFF(b1, (b0 / KV_GROUP_SIZE), 0, 0); + Q += QRY_OFF(b1, b0, 0, 0); + V += VAL_OFF(b1, (b0 / KV_GROUP_SIZE), 0, 0); + A += DST_OFF(b1, b0, 0, 0, 0); + + __builtin_assume_aligned(K, K_ALIGN); + __builtin_assume_aligned(Q, Q_ALIGN); + __builtin_assume_aligned(V, V_ALIGN); + __builtin_assume_aligned(A, A_ALIGN); + + /* Load Q tile, destined for SLM */ + q_tile_type Q_tile; + uint q0_copy = q_tile_sg_n * sg_ij; +#ifdef BLOCK_Q + tile_load_block_rem_q( + &Q_tile, (global uint *)Q, q, ldq >> 1, 0, wg_j0 + q0_copy); +#elif Q_ALIGN >= 4 + tile_load(&Q_tile, (global uint *)Q, (d + 1) >> 1, q, ldq >> 1, 0, + wg_j0 + q0_copy); +#else + tile_load_packed_half(&Q_tile, Q, d, q, ldq, 0, wg_j0 + q0_copy); +#endif + +#if WITH_SCALE + /* Load scale */ + #if INVERT_SCALE + float iscale = convert_float(*scale_ptr); + float scale = native_recip(iscale); + #else + float scale = convert_float(*scale_ptr); + float iscale = native_recip(scale); + #endif +#else + float iscale = sqrt(convert_float(INPUT1_SIZE_X)); + float scale = native_recip(iscale); +#endif + scale *= 1.442695f; // log2(e) + +#ifdef PREFETCH_K0 + /* Prefetch first K tile. */ + cooperative_prefetch_2d_k(K, k, d, ugemm_kq_wg_tile_m, PREFETCH_D_MAX, ldk, + sg_ij, sg_per_wg, SUBGROUP_SIZE, LSC_LDCC_L1C_L3C); +#endif + + /* Initialize S column sums in SLM to -inf */ + const uint n_col_sg = DIV_UP(ugemm_kq_wg_tile_n, SUBGROUP_SIZE * sg_per_wg); + const float neg_inf = -INFINITY; + +#pragma unroll + for (int q = 0; q < n_col_sg; q++) + intel_sub_group_block_write( + (local uint *)&S_max_slm[(q + sg_ij * n_col_sg) + * SUBGROUP_SIZE], + as_uint(neg_inf)); + + /* Clear accumulator */ + a_tile_type A_tile; + tile_fill(A_tile, 0.0f); + + /* Store Q tile to SLM */ + tile_store_t_sys_src1( + Q_tile, (local uint *)&Q_slm[0], D_MAX / 2, q0_copy, 0); + + /* Clear S column sums/maxes */ + s_sum_tile_type S_sum_tile; + s_sum_tile_type S_max_tile, S_max_tile_old; + tile_fill(S_sum_tile, 0.0f); + tile_fill(S_max_tile, -INFINITY); + + /* Wait for Q data to reach SLM */ + barrier(CLK_LOCAL_MEM_FENCE); + + /* Main loop over k blocks */ + for (int k0 = 0; k0 < k; k0 += ugemm_kq_wg_tile_m) { + bool first = (k0 == 0); + bool last = (k0 + ugemm_kq_wg_tile_m >= k); + + uint sg_i0_kq = sg_i_kq * ugemm_kq_sg_tile_m; + uint sg_j0_kq = sg_j_kq * ugemm_kq_sg_tile_n; + +#if WITH_ATTN_MASK + mask_tile_type mask_tile; + tile_load_t(&mask_tile, msk, q, k, q, sg_j0_kq + wg_j0, k0 + sg_i0_kq); +#endif + +#if REMAINDER_K + /* Prepare k mask: NaN in bounds, -inf out of bounds */ + mask_tile_type_float k_mask; +#pragma unroll + for (int ii = 0; ii < ugemm_kq_sg_tile_m / SUBGROUP_SIZE; ii++) + k_mask.x[0][ii] = (k0 + sg_i0_kq + ii * SUBGROUP_SIZE + + get_sub_group_local_id() + < k) + ? nan(0u) + : -INFINITY; +#endif + + /* Calculate S = (K^T) * Q */ + s_tile_type S_tile + = ugemm_kq(K, ldk, Q_slm, D_MAX, k, ugemm_kq_wg_tile_n, d, k0, + 0, 0, sg_i_kq, sg_j_kq, (local char *)ugemm_slm); + + /* Apply attention mask */ +#if WITH_ATTN_MASK +#define unscale(x) ((x)*iscale) + mask_tile_type_float mask_tile_float; + tile_copy(mask_tile, mask_tile_float); + tile_elementwise(mask_tile_float, unscale); + tile_binary(S_tile, mask_tile_float, binary_add); +#endif + + /* Apply k mask */ +#if REMAINDER_K + tile_hbroadcast_min(&S_tile, k_mask); +#endif + + /* Before softmax, we will need to scale columns by maximum values to avoid overflow. */ + + /* Compute our maxima and reduce across SLM */ + tile_vreduce_max(S_tile, &S_max_tile); + tile_atomic_max_full( + S_max_tile, S_max_slm, ugemm_kq_wg_tile_n, sg_j0_kq, 0); + intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE); + +#ifdef PREFETCH_V + /* Prefetch V tile. */ + cooperative_prefetch_2d_maybe_rem(V, d, k - k0, D_MAX, + (ugemm_kq_wg_tile_m * PREFETCH_D_MAX) / D_MAX, ldv, sg_ij, + sg_per_wg, SUBGROUP_SIZE, LSC_LDCC_L1C_L3C); +#endif + +#ifndef ALT_MAX + /* Read back WG-wide maxima */ + intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE); + tile_load_full(&S_max_tile, S_max_slm, ugemm_kq_wg_tile_n, sg_j0_kq, 0); +#endif + + tile_vbroadcast_sub(&S_tile, S_max_tile); + +/* Scale + exponentiate */ +#define scaled_exp(x) native_vexp2(x *scale) + tile_elementwise(S_tile, scaled_exp); + +#ifdef ALT_MAX + /* Read back WG-wide maxima and adjust S to match */ + intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE); + s_sum_tile_type S_max_tile1; + tile_copy(S_max_tile, S_max_tile1); + tile_load_full(&S_max_tile, S_max_slm, ugemm_kq_wg_tile_n, sg_j0_kq, 0); + +#define binary_exp_neg(x, y) native_vexp2(scale *((x) - (y))) + tile_binary(S_max_tile1, S_max_tile, binary_exp_neg); + tile_vbroadcast_mul(&S_tile, S_max_tile1); +#endif + + /* Accumulate sums. S tile is transposed for easy summation. */ + s_sum_tile_type S_sum_tile1; + tile_fill(S_sum_tile1, 0.0f); + tile_vreduce_add(S_tile, &S_sum_tile1); + + /* Convert to half, VNNI format */ + s_tile_type_half2 S_tile_half2; + tile_copy_to_half2(S_tile, S_tile_half2); + + /* Store to SLM, in packed format */ + tile_store_t_sys_src2(S_tile_half2, (local uint *)S_slm, + ugemm_vs_sg_tile_n, ugemm_kq_wg_tile_m / 2, sg_i0_kq / 2, + sg_j0_kq); + intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE); + + /* Rescale existing accumulator and sums to match new maxima */ + if (!first) { +#define binary_exp_sub(x, y) native_vexp2(scale *((x) - (y))) +#define binary_mul(x, y) ((x) * (y)) + tile_binary(S_max_tile_old, S_max_tile, binary_exp_sub); + tile_binary(S_sum_tile, S_max_tile_old, binary_mul); + + /* Find the subset of sums that applies to the accumulation tile */ + a_scale_tile_type A_scale_tile; +#if ugemm_kq_wg_tile_n == ugemm_vs_wg_tile_n \ + && ugemm_kq_sg_tile_n == ugemm_vs_sg_tile_n + tile_copy(S_max_tile_old, A_scale_tile); +#elif ugemm_kq_wg_tile_n == ugemm_vs_wg_tile_n \ + && (ugemm_kq_sg_tile_n % ugemm_vs_sg_tile_n) == 0 + tile_rselect(&A_scale_tile, S_max_tile_old, + sg_j_vs % (ugemm_kq_sg_tile_n / ugemm_vs_sg_tile_n)); +#else +#error unimplemented +#endif + tile_hbroadcast_mul(&A_tile, A_scale_tile); + } + +/* Accumulate sums */ + tile_binary(S_sum_tile, S_sum_tile1, binary_add); + + /* Save maxima */ + tile_copy(S_max_tile, S_max_tile_old); + + /* Last iteration: store column sums in SLM */ + if (last) { + tile_store_full(S_sum_tile, S_sum_slm, ugemm_kq_wg_tile_n, sg_j0_kq, + sg_i_kq); + } + +#ifdef PREFETCH_K + /* Prefetch next K tile. */ + if (!last) { +#if TRANSPOSE_K + const uint stride_k = ldk; +#else + const uint stride_k = 1; +#endif + cooperative_prefetch_2d_k(K + (k0 + ugemm_kq_wg_tile_m) * stride_k, + k - k0 - ugemm_kq_wg_tile_m, d, ugemm_kq_wg_tile_m, + PREFETCH_D_MAX, ldk, sg_ij, sg_per_wg, SUBGROUP_SIZE, + LSC_LDCC_L1C_L3C); + } +#endif +#if WITH_ATTN_MASK && defined(PREFETCH_MASK) + /* Prefetch next mask tile. */ + if (!last) { + cooperative_prefetch_2d(msk + k0 + ugemm_kq_wg_tile_m + sg_i0_kq + (sg_j0_kq + wg_j0) * q, + ugemm_kq_sg_tile_m, ugemm_kq_sg_tile_n, 0, 0, 1, SUBGROUP_SIZE, + LSC_LDCC_L1UC_L3C); + } +#endif + + /* Wait for S stores */ + intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE); + + /* Last iteration: signal column sums are ready */ + if (last && need_sum_barrier) + intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE); + + /* Accumulate A += V * S */ + int k_chunk = min(k - k0, ugemm_kq_wg_tile_m); + a_tile_type A_tile1 = ugemm_vs(V, ldv, S_slm, ugemm_kq_wg_tile_m, d, + ugemm_kq_wg_tile_n, k_chunk, 0, 0, 0, sg_i_vs, sg_j_vs, + (local char *)ugemm_slm); + V += ldv * ugemm_kq_wg_tile_m; + tile_binary(A_tile, A_tile1, binary_add); + } + + /* Wait for column sums to be ready */ + if (need_sum_barrier) intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE); + + /* Load column sums from SLM + reduce in registers */ + a_scale_tile_type A_scale_tile, A_scale_tile_load; + tile_fill(A_scale_tile, 0.0f); + +#pragma unroll + for (uint sg1 = 0; sg1 < ugemm_kq_sg_per_wg_m; sg1++) { + tile_load_full(&A_scale_tile_load, S_sum_slm, ugemm_kq_wg_tile_n, + ugemm_vs_sg_tile_n * sg_j_vs, sg1); + tile_binary(A_scale_tile, A_scale_tile_load, binary_add); + } + + /* Rescale by 1 / (column sums) */ + tile_elementwise(A_scale_tile, native_vrecip); + tile_hbroadcast_mul(&A_tile, A_scale_tile); + + /* Convert to half precision and store */ + a_tile_type_half A_tile_half; + tile_copy_reblock(A_tile, &A_tile_half); + + uint sg_i0_vs = sg_i_vs * ugemm_vs_sg_tile_m; + uint sg_j0_vs = sg_j_vs * ugemm_vs_sg_tile_n + wg_j0; + +#ifdef BLOCK_2D_A + tile_store_block2d(A_tile_half, A, d, q, lda, sg_i0_vs, sg_j0_vs); +#elif defined(BLOCK_A) + tile_store_block_rem_q(A_tile_half, A, q, lda, sg_i0_vs, sg_j0_vs); +#else + tile_store(A_tile_half, A, d, q, lda, sg_i0_vs, sg_j0_vs); +#endif +} diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.h b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.h index 627fdf758586a3..40ac211b1d1026 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.h @@ -19,6 +19,10 @@ #define EXE_MODE_AGE_BASED "-cl-no-subgroup-ifp" #define EXE_MODE_NO_PRERA_SCH "-cl-intel-no-prera-scheduling" +namespace micro { +struct MicroKernelPackage; +} // namspace + namespace kernel_selector { #ifndef UNUSED @@ -64,6 +68,7 @@ struct KernelCode { struct clKernelData { KernelCode code; KernelParams params; + std::vector> micro_kernels; bool skip_execution = false; }; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h index 2154a6c9a51a60..4d776bf41f2439 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_params.h @@ -357,6 +357,21 @@ enum class dev_type { discrete_gpu = 1 }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Arch type +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +enum class gpu_arch { + unknown = 0, + gen9 = 1, + gen11 = 2, + xe_lp = 3, + xe_hp = 4, + xe_hpg = 5, + xe_hpc = 6, + xe2 = 7, +}; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // EngineInfo //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -377,9 +392,12 @@ struct EngineInfo { bool enable_sub_groups_emulation = false; bool bOptHintsSupport = false; bool bLocalBlockIOSupport = false; + bool supports_microkernels = false; uint32_t vendor_id = 0x0; dev_type deviceType = dev_type::integrated_gpu; uint32_t computeUnitsCount = 0; + uint32_t ip_version = 0; + gpu_arch arch = gpu_arch::unknown; uint32_t maxThreadsPerExecutionUnit = 0; uint32_t maxThreadsPerDevice = 0; uint64_t maxWorkGroupSize = 0; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp new file mode 100644 index 00000000000000..e3604a481a8f09 --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp @@ -0,0 +1,630 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#ifdef ENABLE_ONEDNN_FOR_GPU + +#include "sdpa_kernel_micro.h" +#include "common_tools.h" +#include "common_types.h" +#include "jitter.h" +#include "kernel_selector_common.h" +#include "kernel_selector_params.h" +#include "micro_utils.hpp" +#include "tensor_type.h" + +#include +#include +#include +#include + +namespace kernel_selector { + +namespace { + +size_t subgroup_size(gpu_arch arch) { + switch (arch) { + case gpu_arch::gen9: + case gpu_arch::gen11: + case gpu_arch::xe_lp: + case gpu_arch::xe_hp: + case gpu_arch::xe_hpg: return 8; + case gpu_arch::xe_hpc: + case gpu_arch::xe2: return 16; + default: return 0; + } +} + +inline int64_t get_d_max(int64_t head_size) { + for (int64_t i = 32; i <= 1024; i *= 2) + if (head_size <= i) + return i; + return head_size; +} + +micro::Type convert_type(Datatype t) { + switch (t) { + case Datatype::F32: return micro::Type::f32; + case Datatype::F16: return micro::Type::f16; + default: break; + } + OPENVINO_THROW("Unsupported dt: ", toString(t)); +} + +Tensor::NDims normalize_dims(const DataTensor& qkv) { + auto dims = qkv.GetDims(); // xyfb + std::reverse(dims.begin(), dims.end()); // bfyx + return dims; +} + +Tensor::Dim get_num_heads(const DataTensor& qkv, const std::vector& order) { + return normalize_dims(qkv)[order[1]]; +} + +Tensor::Dim get_seq_length(const DataTensor& qkv, const std::vector& order) { + return normalize_dims(qkv)[order[2]]; +} + +struct sdpa_config_t { + int unroll_m_kq, unroll_n_kq; // Subgroup tile sizes for K*Q GEMM + int unroll_m_vs, unroll_n_vs; // Subgroup tile sizes for V*S GEMM + int wg_m_kq, wg_n_kq; // Workgroup configuration for K*Q GEMM + int wg_m_vs, wg_n_vs; // Workgroup configuration for V*S GEMM +}; + +// Kernel configurations: +// h -- maximum head size = N +// s -- target sequence length = M +// 2nd -- second token (thin Q) +sdpa_config_t xehpg_h32 = {32, 16, 16, 16, 2, 16, 2, 16}; +sdpa_config_t xehpg_h32_s256 = {16, 16, 16, 16, 2, 8, 2, 8}; +sdpa_config_t xehpg_h32_s64 = {16, 16, 16, 8, 4, 4, 2, 8}; +sdpa_config_t xehpg_h32_s32 = {8, 8, 8, 8, 4, 4, 4, 4}; +sdpa_config_t xehpg_h32_2nd = {8, 32, 16, 8, 8, 1, 2, 4}; + +sdpa_config_t xehpg_h64 = {32, 16, 16, 16, 4, 8, 4, 8}; +sdpa_config_t xehpg_h64_s128 = {16, 16, 16, 16, 4, 8, 4, 8}; +sdpa_config_t xehpg_h64_s64 = {32, 16, 16, 8, 8, 4, 4, 8}; +sdpa_config_t xehpg_h64_2nd = {8, 16, 16, 8, 8, 1, 4, 2}; + +sdpa_config_t xehpg_h128 = {16, 16, 32, 8, 8, 4, 4, 8}; +sdpa_config_t xehpg_h128_s32 = {16, 16, 16, 8, 16, 2, 8, 4}; +sdpa_config_t xehpg_h128_2nd = {8, 16, 16, 8, 16, 1, 8, 2}; +sdpa_config_t xehpg_h128_s256_2nd = {8, 16, 32, 8, 8, 1, 4, 2}; + +sdpa_config_t xehpg_h256 = {16, 16, 32, 8, 16, 2, 8, 4}; +sdpa_config_t xehpg_h256_s128 = {8, 16, 32, 16, 8, 4, 8, 4}; +sdpa_config_t xehpg_h256_s32 = {8, 16, 32, 8, 16, 2, 8, 4}; +sdpa_config_t xehpg_h256_2nd = {8, 8, 16, 8, 16, 1, 16, 1}; +sdpa_config_t xehpg_h256_s64_2nd = {16, 8, 16, 8, 16, 1, 16, 1}; +sdpa_config_t xehpg_h256_s32_2nd = {16, 16, 32, 8, 16, 1, 8, 2}; + +sdpa_config_t xehpc_h32 = {16, 64, 32, 16, 4, 2, 1, 8}; +sdpa_config_t xehpc_h32_s32 = {16, 16, 16, 16, 2, 4, 2, 4}; +sdpa_config_t xehpc_h32_2nd = {16, 64, 16, 16, 8, 1, 2, 4}; + +sdpa_config_t xehpc_h64 = {16, 64, 32, 16, 8, 2, 2, 8}; +sdpa_config_t xehpc_h64_s64 = {32, 32, 32, 16, 4, 2, 2, 4}; +sdpa_config_t xehpc_h64_s32 = {16, 16, 16, 16, 4, 2, 4, 2}; +sdpa_config_t xehpc_h64_2nd = {32, 32, 32, 16, 4, 1, 2, 2}; +sdpa_config_t xehpc_h64_s64_2nd = {16, 16, 16, 16, 4, 1, 4, 1}; + +sdpa_config_t xehpc_h128 = {16, 64, 32, 16, 16, 2, 4, 8}; +sdpa_config_t xehpc_h128_s64 = {16, 32, 32, 32, 4, 2, 4, 2}; +sdpa_config_t xehpc_h128_s32 = {16, 16, 16, 16, 8, 2, 8, 2}; +sdpa_config_t xehpc_h128_2nd = {32, 32, 32, 16, 8, 1, 4, 2}; + +sdpa_config_t xehpc_h256 = {16, 32, 32, 32, 8, 4, 8, 4}; +sdpa_config_t xehpc_h256_s64 = {16, 32, 32, 32, 8, 1, 8, 1}; +sdpa_config_t xehpc_h256_2nd = {16, 16, 16, 16, 16, 1, 16, 1}; + +sdpa_config_t *choose_config_xehpg(int head_size, int seq, bool thin_q) { + if (head_size <= 32) { + if (thin_q) return &xehpg_h32_2nd; + if (seq <= 32) return &xehpg_h32_s32; + if (seq <= 64) return &xehpg_h32_s64; + if (seq <= 256) return &xehpg_h32_s256; + return &xehpg_h32; + } else if (head_size <= 64) { + if (thin_q) return &xehpg_h64_2nd; + if (seq <= 64) return &xehpg_h64_s64; + if (seq <= 128) return &xehpg_h64_s128; + return &xehpg_h64; + } else if (head_size <= 128) { + if (thin_q) { + if (seq <= 256) return &xehpg_h128_s256_2nd; + return &xehpg_h128_2nd; + } + if (seq <= 32) return &xehpg_h128_s32; + return &xehpg_h128; + } else if (head_size <= 256) { + if (thin_q) { + if (seq <= 32) return &xehpg_h256_s32_2nd; + if (seq <= 64) return &xehpg_h256_s64_2nd; + return &xehpg_h256_2nd; + } + if (seq <= 32) return &xehpg_h256_s32; + if (seq <= 128) return &xehpg_h256_s128; + return &xehpg_h256; + } + return nullptr; +} + +sdpa_config_t *choose_config_xehpc(int head_size, int seq, bool thin_q) { + if (head_size <= 32) { + if (thin_q) return &xehpc_h32_2nd; + if (seq <= 32) return &xehpc_h32_s32; + return &xehpc_h32; + } else if (head_size <= 64) { + if (thin_q) { + if (seq <= 64) return &xehpc_h64_s64_2nd; + return &xehpc_h64_2nd; + } + if (seq <= 32) return &xehpc_h64_s32; + if (seq <= 64) return &xehpc_h64_s64; + return &xehpc_h64; + } else if (head_size <= 128) { + if (thin_q) return &xehpc_h128_2nd; + if (seq <= 32) return &xehpc_h128_s32; + if (seq <= 64) return &xehpc_h128_s64; + return &xehpc_h128; + } else if (head_size <= 256) { + if (thin_q) return &xehpc_h256_2nd; + if (seq <= 64) return &xehpc_h256_s64; + return &xehpc_h256; + } + return nullptr; +} + +} // namespace + +std::mutex SDPAKernelMicro::m; + +void SDPAKernelMicro::init_microkernels(const sdpa_params& params, micro::Package& gemm_kq, micro::Package& gemm_vs, bool is_prefill) const { + // TODO: Remove once micro API is thread safe + std::lock_guard l(m); + const auto& Q = params.inputs[0]; + const auto& K = params.inputs[1]; + const auto& V = params.inputs[2]; + + auto& out = params.outputs[0]; + const auto head_size = params.conf.head_size; + const auto d_max = get_d_max(head_size); + const Tensor::Dim n_keys = get_seq_length(K, params.input1_order); + const Tensor::Dim n_queries = get_seq_length(Q, params.input0_order); + const Tensor::Dim n_values = V.X(); + const auto batch = out.Batch().v * out.Feature().v; + + /* Retrieve pre-tuned kernel configuration */ + sdpa_config_t *config = nullptr; + bool thin_q = (!n_queries.is_dynamic && (n_queries.v <= 16)) || !is_prefill; + + switch (params.engineInfo.arch) { + case gpu_arch::xe_hpg: { + config = choose_config_xehpg(static_cast(head_size), static_cast(n_keys.v), thin_q); + break; + } + case gpu_arch::xe_hpc: + case gpu_arch::xe2: { + config = choose_config_xehpc(static_cast(head_size), static_cast(n_keys.v), thin_q); + break; + } + default: break; + } + + /* Get device information */ + micro::HWInformation hw_info; + hw_info.euCount = params.engineInfo.computeUnitsCount; + hw_info.gmdid = params.engineInfo.ip_version; + hw_info.systolicAvailable = params.engineInfo.supports_immad; + + /* Set up GEMMProblem structure for first GEMM: K^T * Q */ + micro::GEMMProblem problem; + problem.Ta = problem.Ta_ext = convert_type(K.GetDType()); + problem.Tb = problem.Tb_ext = convert_type(Q.GetDType()); + problem.Tc = problem.Tc_ext = micro::Type::f32; + problem.Ts = problem.Tc; + + auto problem_kq = problem; + problem_kq.A.layout = micro::MatrixLayout::T; + problem_kq.B.layout = micro::MatrixLayout::Pr; + problem_kq.C.layout = micro::MatrixLayout::T; + problem_kq.A.setAlignment(micro::alignment_for_ld(head_size * problem.Ta)); + problem_kq.B.setAlignment(64); // Q is packed in VNNI format in SLM + problem_kq.B.crosspack = 2; + problem_kq.B.tileR = d_max; + problem_kq.B.tileC = static_cast(subgroup_size(params.engineInfo.arch)); + + /* Set up problem size information */ + micro::SizeParams sizes; + sizes.m = static_cast(n_keys.v); + sizes.n = static_cast(n_queries.v); + sizes.k = static_cast(head_size); + sizes.batch = static_cast(batch); + + /* Set up microkernel requirements */ + std::vector reqs_kq; + reqs_kq.push_back(micro::StrategyRequirement::UnrollM == config->unroll_m_kq); + reqs_kq.push_back(micro::StrategyRequirement::UnrollN == config->unroll_n_kq); + reqs_kq.push_back(micro::StrategyRequirement::WGM == config->wg_m_kq); + reqs_kq.push_back(micro::StrategyRequirement::WGN == config->wg_n_kq); + + /* Set up microkernel options */ + micro::GEMMProtocol::Options opts_kq; + opts_kq.localB = true; + opts_kq.slmPtr = true; + + /* Ask microkernel provider for microkernel */ + gemm_kq = micro::select_gemm_microkernel(opts_kq, hw_info, sizes, problem_kq, reqs_kq); + + /* Update for second GEMM: V*S */ + auto problem_vs = problem; + problem_vs.Ta = problem_vs.Ta_ext = convert_type(V.GetDType()); + problem_vs.A.layout = micro::MatrixLayout::N; + problem_vs.B.layout = micro::MatrixLayout::Pr; + problem_vs.C.layout = micro::MatrixLayout::N; + problem_vs.A.setAlignment(micro::alignment_for_ld(head_size * problem.Ta)); + problem_vs.B.setAlignment(64); // S is packed in SLM + problem_vs.B.crosspack = 16; + sizes.m = static_cast(n_values.v); + sizes.n = gemm_kq.getSetting("wg_tile_n"); + sizes.k = gemm_kq.getSetting("wg_tile_m"); + + /* Set up special kernel requirements */ + std::vector reqs_vs; + reqs_vs.push_back(micro::StrategyRequirement::UnrollM == config->unroll_m_vs); + reqs_vs.push_back(micro::StrategyRequirement::UnrollN == config->unroll_n_vs); + reqs_vs.push_back(micro::StrategyRequirement::WGM == config->wg_m_vs); + reqs_vs.push_back(micro::StrategyRequirement::WGN == config->wg_n_vs); + + micro::GEMMProtocol::Options opts_vs; + opts_vs.localB = true; + opts_vs.slmPtr = true; + + auto adjust_vs = [](micro::GEMMStrategy &strategy) { + /* Enable dpasw */ + strategy.dpasw |= strategy.fused; + }; + /* Ask microkernel provider for microkernel */ + gemm_vs = micro::select_gemm_microkernel(opts_vs, hw_info, sizes, problem_vs, reqs_vs, adjust_vs); +} + +ParamsKey SDPAKernelMicro::GetSupportedKey() const { + ParamsKey k; + k.EnableInputDataType(Datatype::F16); + k.EnableOutputDataType(Datatype::F16); + + k.EnableInputLayout(DataLayout::bfyx); + k.EnableOutputLayout(DataLayout::bfyx); + + k.EnableDifferentTypes(); + k.EnableTensorOffset(); + k.EnableTensorPitches(); + k.EnableBatching(); + k.EnableDynamicShapesSupport(); + + return k; +} + +bool SDPAKernelMicro::Validate(const Params& p) const { + if (!Parent::Validate(p)) + return false; + + const sdpa_params& params = static_cast(p); + + if (params.engineInfo.arch < gpu_arch::xe_hpg || !params.engineInfo.supports_microkernels) + return false; + + if (params.conf.is_causal) + return false; + + if (params.indirect_axis != -1) + return false; + + auto Q_num_heads_dim = get_num_heads(params.inputs[0], params.input0_order); + auto K_num_heads_dim = get_num_heads(params.inputs[1], params.input1_order); + auto V_num_heads_dim = get_num_heads(params.inputs[2], params.input2_order); + + if (params.input0_order != params.input1_order || params.input0_order != params.input2_order) + return false; + + if (params.input0_order[3] != 3) + return false; + + if (Q_num_heads_dim.is_dynamic || K_num_heads_dim.is_dynamic || V_num_heads_dim.is_dynamic || K_num_heads_dim.v != V_num_heads_dim.v) + return false; + + return true; +} + +JitConstants SDPAKernelMicro::GetJitConstants(const sdpa_params& params, const micro::Package& gemm_kq, const micro::Package& gemm_vs) const { + auto jit = MakeBaseParamsJitConstants(params); + const auto& prim_params = dynamic_cast(params); + + const auto& Q = prim_params.inputs[0]; + const auto& K = prim_params.inputs[1]; + const auto& V = prim_params.inputs[2]; + + const auto head_size = prim_params.conf.head_size; + + auto ldq = head_size * Q.ElementSize(); + auto ldk = head_size * K.ElementSize(); + auto ldv = head_size * V.ElementSize(); + auto lda = head_size * prim_params.outputs[0].ElementSize(); + + const auto d_max = get_d_max(head_size); + const auto n_keys = get_seq_length(K, prim_params.input1_order); + const auto n_queries = get_seq_length(Q, prim_params.input0_order); + const auto n_values = V.X(); + + jit.AddConstant(MakeJitConstant("D_MAX", d_max)); + jit.AddConstant(MakeJitConstant("SUBGROUP_SIZE", subgroup_size(prim_params.engineInfo.arch))); + jit.AddConstant(MakeJitConstant("INVERT_SCALE", false)); + jit.AddConstant(MakeJitConstant("SCALE_DATA_T", "half")); + + jit.AddConstant(MakeJitConstant("WITH_ATTN_MASK", params.inputs.size() > 3)); + jit.AddConstant(MakeJitConstant("WITH_SCALE", params.inputs.size() > 4)); + jit.AddConstant(MakeJitConstant("Q_ALIGN", micro::alignment_for_ld(ldq))); + jit.AddConstant(MakeJitConstant("K_ALIGN", micro::alignment_for_ld(ldk))); + jit.AddConstant(MakeJitConstant("V_ALIGN", micro::alignment_for_ld(ldv))); + jit.AddConstant(MakeJitConstant("A_ALIGN", micro::alignment_for_ld(lda))); + + jit.AddConstant(MakeJitConstant("TRANSPOSE_K", false)); + + int tile_k = gemm_kq.getSetting("wg_tile_m"); + int tile_q = gemm_kq.getSetting("wg_tile_n"); + int tile_v = gemm_vs.getSetting("wg_tile_m"); + + bool d_full = (head_size == d_max); + bool v_full = (head_size == tile_v); + bool k_full = !n_keys.is_dynamic && (n_keys.v % tile_k) == 0; + bool q_full = !n_queries.is_dynamic && (n_queries.v % tile_q) != 0; + + auto Q_num_heads_dim = get_num_heads(Q, params.input0_order); + auto K_num_heads_dim = get_num_heads(K, params.input1_order); + + jit.AddConstant(MakeJitConstant("REMAINDER_K", !k_full)); + jit.AddConstant(MakeJitConstant("KV_GROUP_SIZE", Q_num_heads_dim.v / K_num_heads_dim.v)); + + if (d_full) { + if (ldq % 4 == 0) + jit.AddConstant(MakeJitConstant("BLOCK_Q", 1)); + if (lda % 4 == 0 && v_full) + jit.AddConstant(MakeJitConstant("BLOCK_A", 1)); + jit.AddConstant(MakeJitConstant("REMAINDER_Q", !q_full)); + } else if (params.engineInfo.arch >= gpu_arch::xe_hpc) { + auto vbytes = n_values.v * V.ElementSize(); + if (lda % 16 == 0 && vbytes % 4 == 0) + jit.AddConstant(MakeJitConstant("BLOCK_2D_A", 1)); + } + + if (params.engineInfo.arch >= gpu_arch::xe_hpc) { + jit.AddConstant(MakeJitConstant("PREFETCH_MASK", 1)); + jit.AddConstant(MakeJitConstant("PREFETCH_K0", 1)); + jit.AddConstant(MakeJitConstant("PREFETCH_K", 1)); + jit.AddConstant(MakeJitConstant("PREFETCH_V", 1)); + bool no_rem = d_full && v_full && k_full; + jit.AddConstant(MakeJitConstant("PREFETCH_REMAINDER", !no_rem)); + jit.AddConstant(MakeJitConstant("PREFETCH_D_MAX", std::min(d_max, 64))); + } + + auto unit_parameters = [](std::string prefix) { + JitConstants definitions({}); + for (size_t i = 0; i < 4; i++) { + definitions.AddConstant(MakeJitConstant(prefix + "_B" + std::to_string(i), 1)); + definitions.AddConstant(MakeJitConstant(prefix + "_SB" + std::to_string(i), 1)); + } + + return definitions; + }; + + auto convert_strides = [](std::string target_prefix, std::string source_prefix, const std::vector order) { + JitConstants definitions({}); + + std::vector target_definitions = { + target_prefix + "_S0", + target_prefix + "_S1", + target_prefix + "_S2", + target_prefix + "_S3", + }; + + std::vector source_definitions = { + source_prefix + "_BATCH_PITCH", + source_prefix + "_FEATURE_PITCH", + source_prefix + "_Y_PITCH", + source_prefix + "_X_PITCH", + }; + + for (size_t i = 0; i < target_definitions.size(); i++) { + definitions.AddConstant(MakeJitConstant(target_definitions[order[i]], source_definitions[i])); + } + + return definitions; + }; + + jit.Merge(convert_strides("QRY", "INPUT0", prim_params.input0_order)); + jit.Merge(convert_strides("KEY", "INPUT1", prim_params.input1_order)); + jit.Merge(convert_strides("VAL", "INPUT2", prim_params.input2_order)); + jit.Merge(convert_strides("DST", "OUTPUT", prim_params.output_order)); + + jit.Merge(unit_parameters("QRY")); + jit.Merge(unit_parameters("KEY")); + jit.Merge(unit_parameters("VAL")); + jit.Merge(unit_parameters("DST")); + + return jit; +} + +CommonDispatchData SDPAKernelMicro::SetDefault(const sdpa_params& params, const micro::Package& gemm_kq, const micro::Package& gemm_vs) const { + CommonDispatchData dispatch_data; + + auto wg_tile_q = gemm_kq.getSetting("wg_tile_n"); + auto sg_per_wg = gemm_kq.getSetting("sg_per_wg_m") * gemm_kq.getSetting("sg_per_wg_n"); + + dispatch_data.lws = {subgroup_size(params.engineInfo.arch), (size_t)sg_per_wg, 1}; + dispatch_data.gws = dispatch_data.lws; + + dispatch_data.gws[0] *= CeilDiv(get_seq_length(params.inputs[0], params.input0_order).v, wg_tile_q); + dispatch_data.gws[1] *= params.outputs[0].Feature().v; + dispatch_data.gws[2] *= params.outputs[0].Batch().v; + + return dispatch_data; +} + +clKernelData SDPAKernelMicro::get_kernel_data(const sdpa_params& params, bool is_prefill) const { + auto name = kernelName + (is_prefill ? "_prefill" : "_generate"); + + std::vector gemms(2); // KQ and VS + init_microkernels(params, gemms[kq_id], gemms[vs_id], is_prefill); + auto dispatch_data = SetDefault(params, gemms[kq_id], gemms[vs_id]); + auto entry_point = GetEntryPoint(name, params.layerID, params); + auto jit = CreateJit(name, GetJitConstants(params, gemms[kq_id], gemms[vs_id]), entry_point); + clKernelData kernel; + + FillCLKernelData(kernel, dispatch_data, params.engineInfo, kernelName, jit, entry_point, + "", false, false, static_cast(params.inputs.size()), + GetFusedPrimitiveInputsCount(params), 1, params.is_shape_agnostic); + + kernel.params.arguments.clear(); + if (params.is_shape_agnostic ) + kernel.params.arguments.push_back({ArgumentDescriptor::Types::SHAPE_INFO, 0}); + + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, 1}); // K + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, 0}); // Q + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, 2}); // V + kernel.params.arguments.push_back({ArgumentDescriptor::Types::OUTPUT, 0}); // A + + if (params.inputs.size() >= 4) + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, 3}); // mask + if (params.inputs.size() >= 5) + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, 4}); // Scale + + kernel.params.arguments.push_back({ArgumentDescriptor::Types::SCALAR, 0}); // D + kernel.params.arguments.push_back({ArgumentDescriptor::Types::SCALAR, 1}); // K + kernel.params.arguments.push_back({ArgumentDescriptor::Types::SCALAR, 2}); // Q + + const auto& Q = params.inputs[0]; + const auto& K = params.inputs[1]; + + const auto n_queries = get_seq_length(Q, params.input0_order); + const auto n_keys = get_seq_length(K, params.input1_order); + + auto head_size = params.conf.head_size; + + ScalarDescriptor s_d; + s_d.t = ScalarDescriptor::Types::INT32; + s_d.v.s32 = static_cast(head_size); + + ScalarDescriptor s_k; + s_k.t = ScalarDescriptor::Types::INT32; + s_k.v.s32 = static_cast(n_keys.v); + + ScalarDescriptor s_q; + s_q.t = ScalarDescriptor::Types::INT32; + s_q.v.s32 = static_cast(n_queries.v); + + kernel.params.scalars.push_back(s_d); + kernel.params.scalars.push_back(s_k); + kernel.params.scalars.push_back(s_q); + + /* Generate microkernel shims */ + micro::ShimOptions shim_options; + shim_options.subgroupSize = static_cast(subgroup_size(params.engineInfo.arch)); + shim_options.useTileOps = true; + shim_options.decorator = "kq"; + + kernel.code.kernelString->jit += generateShim(gemms[kq_id], micro::HostLanguage::OpenCL_C, shim_options); + + shim_options.microkernelID++; + shim_options.decorator = "vs"; + kernel.code.kernelString->jit += generateShim(gemms[vs_id], micro::HostLanguage::OpenCL_C, shim_options); + + if (gemms[kq_id].grfMin > 128 || gemms[vs_id].grfMin > 128) + kernel.code.kernelString->options += " -cl-intel-256-GRF-per-thread"; + + std::string extra_options = " -Dcl_intel_dot_accumulate"; + extra_options += " -Dcl_intel_global_float_atomic"; + extra_options += " -Dcl_intel_subgroup_matrix_multiply_accumulate"; + extra_options += " -Dcl_intel_subgroup_split_matrix_multiply_accumulate"; + kernel.code.kernelString->options += extra_options; + + kernel.code.kernelString->batch_compilation = false; + kernel.code.kernelString->has_microkernels = true; + + for (auto& p : gemms) { + kernel.micro_kernels.push_back(std::make_shared(p)); + } + + return kernel; +} + +KernelsData SDPAKernelMicro::GetKernelsData(const Params& params) const { + const size_t num_kernels = 2; + KernelData kd = KernelData::Default(params, num_kernels); + const auto& prim_params = dynamic_cast(params); + + if (!Validate(params)) { + return {}; + } + + for (size_t i = 0; i < num_kernels; i++) { + kd.kernels[i] = get_kernel_data(prim_params, i == prefill_id); + } + + GetUpdateDispatchDataFunc(kd); + + return { kd }; +} + +void SDPAKernelMicro::GetUpdateDispatchDataFunc(KernelData& kd) const { + kd.update_dispatch_data_func = [this](const Params& params, KernelData& kernel_data) { + const auto& prim_params = static_cast(params); + const auto& Q = prim_params.inputs[0]; + const auto& K = prim_params.inputs[1]; + + const auto n_queries = get_seq_length(Q, prim_params.input0_order); + const auto n_keys = get_seq_length(K, prim_params.input1_order); + + auto head_size = prim_params.conf.head_size; + + ScalarDescriptor s_d; + s_d.t = ScalarDescriptor::Types::INT32; + s_d.v.s32 = static_cast(head_size); + + ScalarDescriptor s_k; + s_k.t = ScalarDescriptor::Types::INT32; + s_k.v.s32 = static_cast(n_keys.v); + + ScalarDescriptor s_q; + s_q.t = ScalarDescriptor::Types::INT32; + s_q.v.s32 = static_cast(n_queries.v); + + // TODO: Currently 2nd token version works slower than prefill version + const bool is_prefill = true;//n_queries.v > 1; + + OPENVINO_ASSERT(kernel_data.kernels.size() == 2, "[GPU] Invalid kernels size for update dispatch data func"); + + size_t target_kernel = is_prefill ? prefill_id : generate_id; + + kernel_data.kernels[prefill_id].skip_execution = true; + kernel_data.kernels[generate_id].skip_execution = true; + + const auto& gemms = kernel_data.kernels[target_kernel].micro_kernels; + auto dispatchData = SetDefault(prim_params, gemms[kq_id]->p, gemms[vs_id]->p); + kernel_data.kernels[target_kernel].params.workGroups.global = dispatchData.gws; + kernel_data.kernels[target_kernel].params.workGroups.local = dispatchData.lws; + kernel_data.kernels[target_kernel].skip_execution = KernelData::SkipKernelExecution(prim_params); + + kernel_data.kernels[target_kernel].params.scalars.clear(); + kernel_data.kernels[target_kernel].params.scalars.push_back(s_d); + kernel_data.kernels[target_kernel].params.scalars.push_back(s_k); + kernel_data.kernels[target_kernel].params.scalars.push_back(s_q); + }; +} + +KernelsPriority SDPAKernelMicro::GetKernelsPriority(const Params& /*params*/) const { + return FORCE_PRIORITY_1; +} +} // namespace kernel_selector + +#endif // ENABLE_ONEDNN_FOR_GPU diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.h new file mode 100644 index 00000000000000..f282c52f8437c9 --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.h @@ -0,0 +1,47 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#ifdef ENABLE_ONEDNN_FOR_GPU + +#pragma once + +#include "sdpa_kernel_base.h" +#include "micro_utils.hpp" + +namespace kernel_selector { +class SDPAKernelMicro : public SDPAKernelBase { +public: + using Parent = SDPAKernelBase; + SDPAKernelMicro() : SDPAKernelBase("sdpa_micro") {} + virtual ~SDPAKernelMicro() {} + + KernelsData GetKernelsData(const Params& params) const override; + KernelsPriority GetKernelsPriority(const Params& params) const override; + ParamsKey GetSupportedKey() const override; + +protected: + bool Validate(const Params& p) const override; + void GetUpdateDispatchDataFunc(KernelData& kd) const override; + CommonDispatchData SetDefault(const sdpa_params& params, const micro::Package& gemm_kq, const micro::Package& gemm_vs) const; + JitConstants GetJitConstants(const sdpa_params& params, const micro::Package& gemm_kq, const micro::Package& gemm_vs) const; + std::vector GetSupportedFusedOps() const override { + return {}; + } + + void init_microkernels(const sdpa_params& params, micro::Package& gemm_kq, micro::Package& gemm_vs, bool is_prefill) const; + clKernelData get_kernel_data(const sdpa_params& params, bool is_prefill) const; + +private: + static constexpr size_t prefill_id = 0; + static constexpr size_t generate_id = 1; + + static constexpr size_t kq_id = 0; + static constexpr size_t vs_id = 1; + + static std::mutex m; +}; +} // namespace kernel_selector + + +#endif // ENABLE_ONEDNN_FOR_GPU diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp index 23dc964eb18eea..e53338e7c4522d 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp @@ -273,7 +273,7 @@ void SDPAKernelOpt::GetUpdateDispatchDataFunc(KernelData& kd) const { }; } -KernelsPriority SDPAKernelOpt::GetKernelsPriority(const Params& /*params*/) const { - return FORCE_PRIORITY_1; +KernelsPriority SDPAKernelOpt::GetKernelsPriority(const Params& params) const { + return params.engineInfo.supports_immad ? FORCE_PRIORITY_2 : FORCE_PRIORITY_1; } } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_selector.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_selector.cpp index b58f04f23e2643..7e88d0ad71fed1 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_selector.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_selector.cpp @@ -5,12 +5,16 @@ #include "sdpa_kernel_selector.h" #include "sdpa_kernel_ref.h" #include "sdpa_kernel_opt.h" +#include "sdpa_kernel_micro.h" namespace kernel_selector { sdpa_kernel_selector::sdpa_kernel_selector() { Attach(); Attach(); +#ifdef ENABLE_ONEDNN_FOR_GPU + Attach(); +#endif } KernelsData sdpa_kernel_selector::GetBestKernels(const Params& params) const { diff --git a/src/plugins/intel_gpu/src/kernel_selector/micro_utils.hpp b/src/plugins/intel_gpu/src/kernel_selector/micro_utils.hpp new file mode 100644 index 00000000000000..828c9016d8669e --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/micro_utils.hpp @@ -0,0 +1,58 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#ifdef ENABLE_ONEDNN_FOR_GPU + +#ifdef UNUSED +# undef UNUSED +#endif + +#ifndef NOMINMAX +# define NOMINMAX +#endif + +#include "gpu/intel/microkernels/package.hpp" +#include "gpu/intel/jit/gemm/microkernel_provider.hpp" +#include "gpu/intel/jit/gemm/gen_gemm_kernel_generator.hpp" +#include "gpu/intel/microkernels/shim.hpp" + +namespace micro { + +using Package = dnnl::impl::gpu::intel::micro::Package; +using HWInformation = dnnl::impl::gpu::intel::jit::HWInformation; +using GEMMProblem = dnnl::impl::gpu::intel::jit::GEMMProblem; +using GEMMStrategy = dnnl::impl::gpu::intel::jit::GEMMStrategy; +using GEMMProtocol = dnnl::impl::gpu::intel::micro::GEMMProtocol; +using MatrixLayout = dnnl::impl::gpu::intel::jit::MatrixLayout; +using Type = dnnl::impl::gpu::intel::jit::Type; +using SizeParams = dnnl::impl::gpu::intel::jit::SizeParams; +using StrategyRequirement = dnnl::impl::gpu::intel::jit::StrategyRequirement; +using ShimOptions = dnnl::impl::gpu::intel::micro::ShimOptions; +using HostLanguage = dnnl::impl::gpu::intel::micro::HostLanguage; + +// Wrapper for Package which is used in clKernelData with forward declaration +// to avoid including this header in many places in plugin +// which may cause symbols conflicts with oneDNN +struct MicroKernelPackage { + explicit MicroKernelPackage(Package _p) : p(_p) {} + Package p; +}; + +inline Package select_gemm_microkernel(GEMMProtocol protocol, HWInformation hw_info, SizeParams sizes, const GEMMProblem &problem, + const std::vector &reqs = std::vector(), + void (*strategyAdjuster)(GEMMStrategy &strategy) = nullptr) { + return dnnl::impl::gpu::intel::jit::selectGEMMMicrokernel(protocol, hw_info, sizes, problem, reqs, strategyAdjuster); +} + +static inline int alignment_for_ld(int ld) { + return dnnl::impl::gpu::intel::jit::alignmentForLD(ld); +} + +} // namespace micro + +#undef UNUSED + +#endif // ENABLE_ONEDNN_FOR_GPU diff --git a/src/plugins/intel_gpu/src/kernel_selector/primitive_db.cpp b/src/plugins/intel_gpu/src/kernel_selector/primitive_db.cpp index 8863b47fe55bd2..cd8128baff37c9 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/primitive_db.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/primitive_db.cpp @@ -22,7 +22,7 @@ primitive_db::primitive_db() : primitives({ #include "ks_primitive_db.inc" }), - batch_header_str({ + batch_headers({ #include "ks_primitive_db_batch_headers.inc" }) { } diff --git a/src/plugins/intel_gpu/src/kernel_selector/primitive_db.h b/src/plugins/intel_gpu/src/kernel_selector/primitive_db.h index 73858754d35e39..e384f6c9879fb5 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/primitive_db.h +++ b/src/plugins/intel_gpu/src/kernel_selector/primitive_db.h @@ -22,7 +22,7 @@ struct primitive_db { primitive_db(); std::vector get(const primitive_id& id) const; - std::vector get_batch_header_str() const { return std::move(batch_header_str); } + std::map get_batch_headers() const { return std::move(batch_headers); } private: struct case_insensitive_compare { @@ -35,7 +35,7 @@ struct primitive_db { } }; std::multimap primitives; - std::vector batch_header_str; + std::map batch_headers; }; } // namespace cache diff --git a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py index 4f746d8cc526f6..116844f3bccfc7 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py +++ b/src/plugins/intel_gpu/src/kernel_selector/primitive_db_gen.py @@ -198,7 +198,8 @@ def batch_headers_to_str(self): characters = 1 # Newline character above res = "" for h in self.batch_headers: - res += '(std::string) R"(\n' + header_name = h[:h.find('.cl')] + res += '{{"{}",\n(std::string) R"(\n'.format(header_name) header_file = os.path.abspath(os.path.join(os.path.dirname(self.kernels_folder + "/include/batch_headers"), "batch_headers/" + h)) content = [] with open(header_file) as f: @@ -207,11 +208,11 @@ def batch_headers_to_str(self): if line.startswith('#include'): continue if (i + 1) % max_lines == 0 or characters + len(line) + 1 > max_characters: - res += ')",' + ' (std::string) R"(' + res += ')"\n + (std::string) R"(' characters = 0 res += '{}\n'.format(line.rstrip()) characters += len(line) + 1 - res += ')",\n\n' + res += ')"},\n\n' return self.post_process_sources(res) def post_process_sources(self, content): diff --git a/src/plugins/intel_gpu/src/plugin/ops/scaled_dot_product_attention.cpp b/src/plugins/intel_gpu/src/plugin/ops/scaled_dot_product_attention.cpp index d002c868ffd225..0ba780183b186d 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/scaled_dot_product_attention.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/scaled_dot_product_attention.cpp @@ -3,7 +3,6 @@ // #include "intel_gpu/plugin/program_builder.hpp" -#include "intel_gpu/plugin/common_utils.hpp" #include "intel_gpu/op/sdpa.hpp" #include "intel_gpu/op/indirect_sdpa.hpp" @@ -30,9 +29,15 @@ static void CreateScaledDotProductAttentionOp(ProgramBuilder& p, const std::shar auto layerName = layer_type_name_ID(op); bool is_causal = op->get_causal(); + auto order = ov::op::internal::SDPA::default_order(op->get_output_partial_shape(0).size()); auto sdpa_prim = cldnn::scaled_dot_product_attention(layerName, inputs, - is_causal); + is_causal, + -1, + order, + order, + order, + order); p.add_primitive(*op, sdpa_prim); } diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index b6db0233ee9093..e3f7c2cfe2f6b5 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -83,17 +83,18 @@ std::string Plugin::get_device_id(const ov::AnyMap& config) const { return id; } -void Plugin::transform_model(std::shared_ptr& model, const ExecutionConfig& config) const { +void Plugin::transform_model(std::shared_ptr& model, const ExecutionConfig& config, const std::shared_ptr& context) const { OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::transform_model"); - auto deviceInfo = m_device_map.at(config.get_property(ov::device::id))->get_info(); - TransformationsPipeline transformations(config, deviceInfo); + TransformationsPipeline transformations(config, context); auto start = Time::now(); transformations.apply(model); GPU_DEBUG_LOG << "Transformations time: " << std::chrono::duration_cast(Time::now() - start).count() << " ms" << std::endl; } -std::shared_ptr Plugin::clone_and_transform_model(const std::shared_ptr& model, const ExecutionConfig& config) const { +std::shared_ptr Plugin::clone_and_transform_model(const std::shared_ptr& model, + const ExecutionConfig& config, + const std::shared_ptr& context) const { OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::clone_and_transform_model"); GPU_DEBUG_GET_INSTANCE(debug_config); GPU_DEBUG_DEFINE_MEM_LOGGER("Plugin::clone_and_transform_model"); @@ -106,7 +107,7 @@ std::shared_ptr Plugin::clone_and_transform_model(const std::shared_p ov::pass::VisualizeTree(path_base + ".svg").run_on_model(cloned_model); } - transform_model(cloned_model, config); + transform_model(cloned_model, config, context); // Transformations for some reason may drop output tensor names, so here we copy those from the original model auto new_results = cloned_model->get_results(); @@ -169,7 +170,7 @@ std::shared_ptr Plugin::compile_model(const std::shared_ptr< config.set_user_property(orig_config); config.apply_user_properties(context->get_engine().get_device_info()); - auto transformed_model = clone_and_transform_model(model, config); + auto transformed_model = clone_and_transform_model(model, config, context); { OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::compile_model::CreateCompiledModel"); return std::make_shared(transformed_model, shared_from_this(), context, config); @@ -188,7 +189,7 @@ std::shared_ptr Plugin::compile_model(const std::shared_ptr< config.set_user_property(orig_config); config.apply_user_properties(context_impl->get_engine().get_device_info()); - auto transformed_model = clone_and_transform_model(model, config); + auto transformed_model = clone_and_transform_model(model, config, context_impl); return std::make_shared(transformed_model, shared_from_this(), context_impl, config); } @@ -259,10 +260,10 @@ ov::SupportedOpsMap Plugin::query_model(const std::shared_ptr& float query_model_ratio = config.get_property(ov::internal::query_model_ratio.name()).as(); auto supported = ov::get_supported_nodes(model, - [&config,this](std::shared_ptr& model) { + [&config,&ctx,this](std::shared_ptr& model) { std::map shapes; std::map> batch_dim; - transform_model(model, config); + transform_model(model, config, ctx); }, [&prog](std::shared_ptr node) { return prog.is_op_supported(node); @@ -709,7 +710,7 @@ uint32_t Plugin::get_max_batch_size(const ov::AnyMap& options) const { return static_cast(max_batch_size); } - TransformationsPipeline transformations(config, device_info); + TransformationsPipeline transformations(config, context); transformations.apply(cloned_model); program = std::make_shared(cloned_model, engine, config, true); std::pair device_memory_usage = program->get_compiled_program()->get_estimated_device_mem_usage(); @@ -778,7 +779,7 @@ uint32_t Plugin::get_optimal_batch_size(const ov::AnyMap& options) const { << ", L3_cache_size is (MB): " << float(L3_cache_size) / 1024 / 1024 << std::endl; } auto config = m_configs_map.at(device_id); - auto cloned_model = clone_and_transform_model(model, config); + auto cloned_model = clone_and_transform_model(model, config, context); ov::MemBandwidthPressure memPressure = ov::mem_bandwidth_pressure_tolerance(cloned_model, L3_cache_size); uint32_t batch = 1; if (memPressure.max_mem_tolerance != ov::MemBandwidthPressure::UNKNOWN) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/transpose_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/transpose_fusion.cpp index 3a6315978577c8..d26e45e8fea9ba 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/transpose_fusion.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/transpose_fusion.cpp @@ -45,14 +45,6 @@ TransposeSDPAMatcher::TransposeSDPAMatcher() { return std::dynamic_pointer_cast(output.get_node_shared_ptr()) == nullptr && is_fp_type(output); }; - auto is_dynamic = [](const ov::Output& output) -> bool { - bool is_dynamic = output.get_node_shared_ptr()->get_output_partial_shape(0).is_dynamic(); - size_t num_inputs = output.get_node_shared_ptr()->get_input_size(); - for (size_t idx = 0; idx < num_inputs; idx++) { - is_dynamic |= output.get_node_shared_ptr()->get_input_partial_shape(idx).is_dynamic(); - } - return is_dynamic; - }; auto input_q_m = any_input(not_transpose); auto input_k_m = any_input(not_transpose); @@ -70,10 +62,10 @@ TransposeSDPAMatcher::TransposeSDPAMatcher() { auto sdpa_in_k = std::make_shared(OutputVector{input_k_m, transpose_k_m}); auto sdpa_in_v = std::make_shared(OutputVector{input_v_m, transpose_v_m}); - auto sdpa_without_attn_mask_m = wrap_type({ sdpa_in_q, sdpa_in_k, sdpa_in_v }, is_dynamic); - auto sdpa_with_attn_mask_m = wrap_type({ sdpa_in_q, sdpa_in_k, sdpa_in_v, input_attn_mask }, is_dynamic); + auto sdpa_without_attn_mask_m = wrap_type({ sdpa_in_q, sdpa_in_k, sdpa_in_v }); + auto sdpa_with_attn_mask_m = wrap_type({ sdpa_in_q, sdpa_in_k, sdpa_in_v, input_attn_mask }); auto sdpa_with_attn_mask_and_scale_m = - wrap_type({ sdpa_in_q, sdpa_in_k, sdpa_in_v, input_attn_mask, input_scale }, is_dynamic); + wrap_type({ sdpa_in_q, sdpa_in_k, sdpa_in_v, input_attn_mask, input_scale }); auto sdpa_m = std::make_shared(OutputVector{sdpa_without_attn_mask_m, sdpa_with_attn_mask_m, sdpa_with_attn_mask_and_scale_m}); diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 4b18dbf3fa4b71..bb7385cbe5dbb1 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -188,6 +188,10 @@ static bool is_non_supported_decompression_op(const std::shared_ptr func) { return false; } - // - Head size should be static dim - const auto head_size_dim = query_ps[query_ps.size() - 1]; - if (head_size_dim.is_dynamic()) - return false; + // For platforms with DPAS support we don't have any other shape-based limitations besides head_size being static and equal for QKV + if (device_info.supports_immad && cldnn::query_microkernels_supported(m_context->get_engine(), config)) + return true; // - Head size should be 128 for any model type; or should be in the range of 64 to 256 for stateful LLMs because of performance reasons. // This limitations is recommended to prevent performance drop in models with small head size, such as SD, diff --git a/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp b/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp index 47483c6a1a0192..ac8d716882ef82 100644 --- a/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp +++ b/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp @@ -37,9 +37,32 @@ #include #endif +#ifdef ENABLE_ONEDNN_FOR_GPU +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include "gpu/intel/microkernels/fuser.hpp" +#endif + namespace { std::mutex cacheAccessMutex; +#ifdef ENABLE_ONEDNN_FOR_GPU +cl::Program fuse_microkernels(const cl::Context& context, const cl::Device& device, cl::Program& program, const std::string& code) { + using namespace dnnl::impl::gpu::intel; + std::vector> binaries = program.getInfo(); + OPENVINO_ASSERT(binaries.size() == 1); + std::vector binary = binaries[0]; + micro::fuseMicrokernels(binary, code.c_str()); + + cl::Program::Binaries fused_binary = { binary }; + cl::Program fused_program(context, {device}, fused_binary); + fused_program.build({device}); + + return fused_program; +} +#endif // ENABLE_ONEDNN_FOR_GPU + std::string reorder_options(const std::string& org_options) { std::stringstream ss(org_options); std::set sorted_options; @@ -128,7 +151,7 @@ void kernels_cache::get_program_source(const kernels_code& kernels_source_code, const auto& batch_id = 0; // increase bucket id if and only if new bucket comes bucket_id = static_cast(program_buckets.size() - 1); - current_bucket.push_back(batch_program(bucket_id, batch_id, options, batch_header_str)); + current_bucket.push_back(batch_program(bucket_id, batch_id, options, batch_headers)); } // This is a temporary walk-around to avoid severe performance drop. @@ -159,13 +182,22 @@ void kernels_cache::get_program_source(const kernels_code& kernels_source_code, || current_bucket.back().entry_point_to_id.find(entry_point) != current_bucket.back().entry_point_to_id.end() || need_separate_batch(entry_point)) { const auto& batch_id = static_cast(current_bucket.size()); - current_bucket.push_back(batch_program(bucket_id, batch_id, options, batch_header_str)); + current_bucket.push_back(batch_program(bucket_id, batch_id, options, batch_headers)); } auto& current_batch = current_bucket.back(); current_batch.dump_custom_program = dump_custom_program; current_batch.entry_point_to_id.emplace(entry_point, std::make_pair(code.params, kernel_part_idx)); + current_batch.has_microkernels |= kernel_string->has_microkernels; + + // TODO: Technically, microkernels doesn't require specific headers, but we don't want to include + // some headers to all batches as it may lead to compilation error on some driver versions. + // Need to generalize work with headers to include only necessary parts + if (current_batch.has_microkernels) { + current_batch.source.insert(current_batch.source.begin(), current_batch.micro_headers.begin(), current_batch.micro_headers.end()); + } + current_batch.source.push_back(std::move(full_code)); current_batch.kernels_counter++; } @@ -195,12 +227,12 @@ kernels_cache::kernels_cache(engine& engine, const ExecutionConfig& config, uint32_t prog_id, std::shared_ptr task_executor, - const std::vector& batch_header_str) + const std::map& batch_headers) : _engine(engine) , _task_executor(task_executor) , _config(config) , _prog_id(prog_id) - , batch_header_str(std::move(batch_header_str)) { } + , batch_headers(std::move(batch_headers)) { } static std::vector getProgramBinaries(cl::Program program) { // Get the size of the program binary in bytes. @@ -288,6 +320,17 @@ void kernels_cache::build_batch(const engine& build_engine, const batch_program& dump_file << "*/\n"; } + if (batch.has_microkernels) { +#ifdef ENABLE_ONEDNN_FOR_GPU + OPENVINO_ASSERT(batch.kernels_counter == 1); + // Do we need full source code here (with batch headers)? + program = fuse_microkernels(cl_build_engine.get_cl_context(), cl_build_engine.get_cl_device(), program, batch.source.back()); +#else // ENABLE_ONEDNN_FOR_GPU + OPENVINO_THROW("[GPU] Can't compile kernel w/ microkernels as onednn is not available"); +#endif // ENABLE_ONEDNN_FOR_GPU + } + + program.createKernels(&kernels); if (is_cache_enabled()) { @@ -348,6 +391,7 @@ void kernels_cache::build_batch(const engine& build_engine, const batch_program& GPU_DEBUG_INFO << "-------- End of OpenCL build error" << std::endl; std::stringstream err_ss(err_log); std::string line; + std::stringstream err; int cnt = 0; while (std::getline(err_ss, line, '\n')) { @@ -355,14 +399,14 @@ void kernels_cache::build_batch(const engine& build_engine, const batch_program& cnt = 5; cnt--; if (cnt > 0) - std::cout << line << std::endl; + err << line << std::endl; else if (cnt == 0) - std::cout << "...." << std::endl; + err << "...." << std::endl; } throw std::runtime_error("Program build failed(" + std::to_string(batch.bucket_id) + + "_part_" + std::to_string(batch.batch_id) - + "): You may enable OCL source dump to see the error log.\n"); + + "):\n" + err.str()); } } diff --git a/src/plugins/intel_gpu/src/runtime/kernels_cache.hpp b/src/plugins/intel_gpu/src/runtime/kernels_cache.hpp index ca389718a49d23..d3775731bb01a5 100644 --- a/src/plugins/intel_gpu/src/runtime/kernels_cache.hpp +++ b/src/plugins/intel_gpu/src/runtime/kernels_cache.hpp @@ -54,19 +54,34 @@ class kernels_cache { size_t hash_value; uint32_t kernels_counter; source_code source; + source_code micro_headers; std::string options; bool dump_custom_program; + bool has_microkernels; std::map> entry_point_to_id; - explicit batch_program(int32_t _bucket_id, int32_t _batch_id, std::string _options, const std::vector& batch_header_str) + explicit batch_program(int32_t _bucket_id, int32_t _batch_id, std::string _options, const std::map& batch_headers) : bucket_id(_bucket_id), batch_id(_batch_id), hash_value(0), kernels_counter(0), - source(std::move(batch_header_str)), + source({}), options(_options), dump_custom_program(false), + has_microkernels(false), entry_point_to_id({}) { + static const std::vector micro_kernel_include_names { + "generic_vector_ops", + "tile_ops", + "sdpa_utils" + }; + for (const auto& kv : batch_headers) { + if (std::find(micro_kernel_include_names.begin(), micro_kernel_include_names.end(), kv.first) == micro_kernel_include_names.end()) { + source.push_back(kv.second); + } else { + micro_headers.push_back(kv.second); + } + } } }; @@ -83,7 +98,7 @@ class kernels_cache { compiled_kernels _kernels; std::map, uint32_t> _cached_binaries; std::unordered_map _cached_kernels; - std::vector batch_header_str; + std::map batch_headers; std::unordered_map _kernel_batch_hash; void get_program_source(const kernels_code& kernels_source_code, std::vector*) const; void build_batch(const engine& build_engine, const batch_program& batch, compiled_kernels& compiled_kernels); @@ -97,12 +112,9 @@ class kernels_cache { const ExecutionConfig& config, uint32_t prog_id, std::shared_ptr task_executor = nullptr, - const std::vector& batch_header_str = {}); + const std::map& batch_headers = {}); kernel::ptr get_kernel_from_cached_kernels(std::string id) const; std::vector get_kernels(kernel_impl_params params) const; - void set_batch_header_str(const std::vector &batch_headers) { - batch_header_str = std::move(batch_headers); - } bool validate_simple_kernel_execution(kernel::ptr kernel); diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp index 15e379d5ad557b..74d9b033c64ca2 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp @@ -2,6 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#ifdef ENABLE_ONEDNN_FOR_GPU +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include "gpu/intel/jit/jit_generator.hpp" +#endif // ENABLE_ONEDNN_FOR_GPU + #include "ocl_device.hpp" #include "ocl_common.hpp" #include "intel_gpu/runtime/debug_configuration.hpp" @@ -43,6 +50,24 @@ namespace cldnn { namespace ocl { namespace { + +#ifdef ENABLE_ONEDNN_FOR_GPU +gpu_arch convert_ngen_arch(ngen::HW gpu_arch) { + switch (gpu_arch) { + case ngen::HW::Gen9: return gpu_arch::gen9; + case ngen::HW::Gen11: return gpu_arch::gen11; + case ngen::HW::XeLP: return gpu_arch::xe_lp; + case ngen::HW::XeHP: return gpu_arch::xe_hp; + case ngen::HW::XeHPG: return gpu_arch::xe_hpg; + case ngen::HW::XeHPC: return gpu_arch::xe_hpc; + case ngen::HW::Xe2: return gpu_arch::xe2; + case ngen::HW::Gen10: + case ngen::HW::Unknown: return gpu_arch::unknown; + } + return gpu_arch::unknown; +} +#endif + int driver_dev_id() { const std::vector unused_ids = { 0x4905, 0x4906, 0x4907, 0x4908 @@ -172,7 +197,7 @@ bool get_imad_support(const cl::Device& device) { return false; } -device_info init_device_info(const cl::Device& device) { +device_info init_device_info(const cl::Device& device, const cl::Context& context) { device_info info = {}; info.vendor_id = static_cast(device.getInfo()); info.dev_name = device.getInfo(); @@ -250,7 +275,8 @@ device_info init_device_info(const cl::Device& device) { bool nv_device_attr_supported = extensions.find("cl_nv_device_attribute_query") != std::string::npos; info.has_separate_cache = false; if (device_attr_supported) { - info.gfx_ver = parse_version(device.getInfo()); + info.ip_version = device.getInfo(); + info.gfx_ver = parse_version(info.ip_version); info.device_id = device.getInfo(); info.num_slices = device.getInfo(); info.num_sub_slices_per_slice = device.getInfo(); @@ -296,6 +322,17 @@ device_info init_device_info(const cl::Device& device) { info.num_ccs = std::max(num_queues, info.num_ccs); } + +#ifdef ENABLE_ONEDNN_FOR_GPU + using namespace dnnl::impl::gpu::intel::jit; + ngen::HW hw = ngen::HW::Unknown; + ngen::Product product = {ngen::ProductFamily::Unknown, 0}; + jit_generator::detectHWInfo(context.get(), device.get(), hw, product); + info.arch = convert_ngen_arch(hw); +#else // ENABLE_ONEDNN_FOR_GPU + info.arch = gpu_arch::unknown; +#endif // ENABLE_ONEDNN_FOR_GPU + return info; } @@ -331,7 +368,7 @@ ocl_device::ocl_device(const cl::Device dev, const cl::Context& ctx, const cl::P : _context(ctx) , _device(dev) , _platform(platform) -, _info(init_device_info(dev)) +, _info(init_device_info(dev, ctx)) , _mem_caps(init_memory_caps(dev, _info)) { } bool ocl_device::is_same(const device::ptr other) { diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_wrapper.hpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_wrapper.hpp index 57da8d5c748672..91a4d767ceaebe 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_wrapper.hpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_wrapper.hpp @@ -50,6 +50,7 @@ #pragma clang diagnostic ignored "-Wunused-variable" #pragma clang diagnostic ignored "-Wunused-function" #pragma clang diagnostic ignored "-Wignored-qualifiers" + #pragma clang diagnostic ignored "-Wdeprecated-declarations" #elif defined __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-compare" @@ -57,6 +58,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wignored-qualifiers" + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #if __GNUC__ >= 8 #pragma GCC diagnostic ignored "-Wcatch-value" #endif diff --git a/src/plugins/intel_gpu/tests/common/subgraphs_builders.hpp b/src/plugins/intel_gpu/tests/common/subgraphs_builders.hpp index 3ea2a654bb5416..8dd82c7f6f1bd8 100644 --- a/src/plugins/intel_gpu/tests/common/subgraphs_builders.hpp +++ b/src/plugins/intel_gpu/tests/common/subgraphs_builders.hpp @@ -14,6 +14,7 @@ #include "openvino/op/gather.hpp" #include "openvino/op/read_value.hpp" #include "openvino/op/reshape.hpp" +#include "openvino/op/scaled_dot_product_attention.hpp" #include "openvino/op/shape_of.hpp" #include "openvino/op/transpose.hpp" #include "openvino/op/result.hpp" @@ -119,7 +120,6 @@ inline std::shared_ptr make_llm_kv_cache_pattern(ov::Dimension batch ov::ResultVector results{kv_present, matmul_out}; auto model = std::make_shared(results, params, "LLM-KV-Cache"); - ov::pass::VisualizeTree("model.svg").run_on_model(model); if (stateful) { ov::pass::MakeStateful({{in_kv_prev, kv_present}}).run_on_model(model); } diff --git a/src/plugins/intel_gpu/thirdparty/CMakeLists.txt b/src/plugins/intel_gpu/thirdparty/CMakeLists.txt index 8e835c371a6123..3f0a4d60a9dedf 100644 --- a/src/plugins/intel_gpu/thirdparty/CMakeLists.txt +++ b/src/plugins/intel_gpu/thirdparty/CMakeLists.txt @@ -12,7 +12,7 @@ if(ENABLE_ONEDNN_FOR_GPU) set(ONEDNN_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/onednn_gpu_build") set(ONEDNN_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/onednn_gpu_install" CACHE PATH "Installation path for oneDNN GPU library") set(ONEDNN_PREFIX_DIR "${CMAKE_CURRENT_BINARY_DIR}/onednn_gpu_root") - set(ONEDNN_ENABLED_PRIMITIVES "CONCAT;CONVOLUTION;DECONVOLUTION;INNER_PRODUCT;MATMUL;REORDER;POOLING;REDUCTION") + set(ONEDNN_ENABLED_PRIMITIVES "CONCAT;CONVOLUTION;DECONVOLUTION;INNER_PRODUCT;MATMUL;REORDER;POOLING;REDUCTION;SDPA") set(ONEDNN_ENABLED_ISA "XEHPG;XEHPC;XE2") set(DNNL_GPU_LIBRARY_NAME "openvino_onednn_gpu" CACHE STRING "Name of oneDNN library for Intel GPU Plugin") @@ -141,11 +141,24 @@ if(ENABLE_ONEDNN_FOR_GPU) EXCLUDE_FROM_ALL ON ) + list(APPEND LIB_INCLUDE_DIRS ${ONEDNN_INSTALL_DIR}/include) + list(APPEND LIB_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/onednn_gpu/src) + set(LIB_DEFINITIONS ENABLE_ONEDNN_FOR_GPU + DNNL_DLL + DNNL_DLL_EXPORTS + DNNL_ENABLE_CPU_ISA_HINTS + DNNL_ENABLE_MAX_CPU_ISA + DNNL_X64=1 + NGEN_CPP11 + NGEN_NEO_INTERFACE + NGEN_NO_OP_NAMES + NGEN_SAFE + NGEN_WINDOWS_COMPAT) add_library(onednn_gpu_tgt INTERFACE) set_target_properties(onednn_gpu_tgt PROPERTIES INTERFACE_LINK_LIBRARIES $ - INTERFACE_INCLUDE_DIRECTORIES $ - INTERFACE_COMPILE_DEFINITIONS ENABLE_ONEDNN_FOR_GPU + INTERFACE_INCLUDE_DIRECTORIES "$" + INTERFACE_COMPILE_DEFINITIONS "${LIB_DEFINITIONS}" ) add_dependencies(onednn_gpu_tgt onednn_gpu_build) From e2c10ad23546f526de934da891ada5fd1f609e97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:11:27 +0200 Subject: [PATCH 09/15] Update flake8-comprehensions requirement from <=3.14.0 to <=3.15.0 in /src/bindings/python (#25300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the requirements on [flake8-comprehensions](https://github.com/adamchainz/flake8-comprehensions) to permit the latest version.
    Commits

    Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
    Dependabot commands and options
    You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
    Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- src/bindings/python/requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings/python/requirements_test.txt b/src/bindings/python/requirements_test.txt index c6d928dbfe155f..3bac1845592ea3 100644 --- a/src/bindings/python/requirements_test.txt +++ b/src/bindings/python/requirements_test.txt @@ -7,7 +7,7 @@ flake8-annotations-complexity<=0.0.8 flake8-broken-line<=1.0.0 flake8-bugbear<=24.4.26 flake8-class-attributes-order<=0.1.3 -flake8-comprehensions<=3.14.0 +flake8-comprehensions<=3.15.0 flake8-debugger<=4.1.2 flake8-docstrings<=1.7.0 flake8-eradicate<=1.5.0 From bc505ba8ddafe938e01fa5530ac5d6cc492efe17 Mon Sep 17 00:00:00 2001 From: hyunback kim Date: Tue, 2 Jul 2024 00:43:17 +0900 Subject: [PATCH 10/15] [GPU] Fix SD1.5 2.1 dynamic failure. (#25307) It's the regression from SA Conv fsv16 enable. ### Tickets: - *ticket-id* Signed-off-by: hyunback --- .../convolution/convolution_kernel_b_fs_yx_fsv16_1x1.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_yx_fsv16_1x1.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_yx_fsv16_1x1.cpp index 688e09c8010030..f91c5b870e53f0 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_yx_fsv16_1x1.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_yx_fsv16_1x1.cpp @@ -64,7 +64,7 @@ float ConvolutionKernel_b_fs_yx_fsv16_1x1::EstimateOccupancy(const convolution_p ConvolutionKernel_b_fs_yx_fsv16_1x1::ConvolutionTuningData ConvolutionKernel_b_fs_yx_fsv16_1x1::GetTuningParams(const convolution_params& params) const { ConvolutionTuningData tuning_data; - if (!params.has_dynamic_tensors()) { + if (!params.is_shape_agnostic) { const auto& input = params.inputs[0]; bool block_size_one_is_better = params.outputs[0].X().v == 1 && params.outputs[0].Y().v == 1 && input.Feature().v >= 2048; From be656b3a5101dcf90a03e3f9a947fb6d9301ff6c Mon Sep 17 00:00:00 2001 From: Alina Kladieva Date: Tue, 2 Jul 2024 12:03:45 +0200 Subject: [PATCH 11/15] [GHA] Temporarily remove GPU tests from mandatory (#25324) All GPU hosts are unexpectedly offline, so we have to wait for lab maintainers --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4358af5707b077..e75cc2a1867e55 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -687,7 +687,7 @@ jobs: Overall_Status: name: ci/gha_overall_status needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests, TensorFlow_Layer_Tests, - CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers, iGPU] + CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers] if: ${{ always() }} runs-on: ubuntu-latest steps: From 029d2006703da02ea5099bf66aaa695113fddc9b Mon Sep 17 00:00:00 2001 From: Andrzej Kopytko Date: Tue, 2 Jul 2024 10:39:31 +0200 Subject: [PATCH 12/15] [DOCS] Fixed footer file no3 (#25313) ### Details: - *item1* - *...* ### Tickets: - *ticket-id* --- .../about-openvino/additional-resources.rst | 34 ------------------- docs/sphinx_setup/_static/html/footer.html | 27 ++++++++------- 2 files changed, 14 insertions(+), 47 deletions(-) delete mode 100644 docs/articles_en/about-openvino/additional-resources.rst diff --git a/docs/articles_en/about-openvino/additional-resources.rst b/docs/articles_en/about-openvino/additional-resources.rst deleted file mode 100644 index cb8d0fc62f244a..00000000000000 --- a/docs/articles_en/about-openvino/additional-resources.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. {#resources} - -Additional Resources -==================== - - - -.. meta:: - :description: Learn more about OpenVINO from benchmark results, case studies - and lists of supported models, operations and devices. - -.. toctree:: - :maxdepth: 1 - :hidden: - - additional-resources/glossary - Legal and Responsible AI Information <./additional-resources/legal-information> - additional-resources/telemetry - Case Studies - - -:doc:`Performance Benchmarks ` contain results from benchmarking models with OpenVINO on Intel hardware. - -:doc:`Glossary ` contains terms used in OpenVINO. - -:doc:`Legal and Responsible AI Information ` provides trademark information and other legal statements. - -:doc:`OpenVINO™ Telemetry ` has detailed information on the telemetry data collection. - -`Case Studies `__ are articles about real-world examples of OpenVINO™ usage. - - - - diff --git a/docs/sphinx_setup/_static/html/footer.html b/docs/sphinx_setup/_static/html/footer.html index 2194cf3cec66b0..1fb833c54a42ec 100644 --- a/docs/sphinx_setup/_static/html/footer.html +++ b/docs/sphinx_setup/_static/html/footer.html @@ -3,11 +3,12 @@ margin: 0; } - .container { + .footer .container { padding-right: 30px; padding-left: 30px; margin-right: auto; margin-left: auto; + background-color: rgb(240, 240, 240); } .footer-link-list a { @@ -18,12 +19,12 @@ } .footer-link-list a:nth-child(1) { - margin-right: 20px; + margin-right: 80px; } .footer-contents { border-top: 1px solid lightgray; - font-size: 11px; + font-size: 12px; box-sizing: border-box; font-family: sans-serif; padding-top: 10px; @@ -34,10 +35,12 @@ margin-right: 0px; } - .footer-contents p { + .footer-item p { margin-top: 0; margin-bottom: 0; line-height: 1.65; + padding: 10px 10px 0 30px !important; + font-size: 12px !important; } .footer-contents .footer-link-list a { @@ -46,17 +49,15 @@ .newsletter-btn, .newsletter-btn:active { - background-color: #0054AE; - border: 1px solid #0054AE; + background-color: white; + border: 1px solid lightgray !important; border-radius: none; - color: white !important; - margin-top: 5px; - cursor: pointer; + color: #0054AE !important; display: inline-block; font-weight: 400; font-size: .8rem; line-height: 2.0; - padding: .2rem .5rem; + padding: 0.5rem 0.65625rem 0.5rem 0.96875rem; text-align: center; text-decoration: none; transition: color .15s ease-in-out, background-color .15s ease-in-out, border-color .15s ease-in-out, box-shadow .15s ease-in-out; @@ -73,8 +74,8 @@ } .newsletter-btn:hover { - background-color: rgb(240, 240, 240); - color: hsl(211, 100%, 34%) !important; + background-color: rgba(var(--ost-color-sst-dropdown-background), 1) !important; + color: rgb(240, 240, 240) !important; } .footer-list { @@ -134,7 +135,7 @@
    Changelog

    Sourced from flake8-comprehensions's changelog.

    3.15.0 (2024-06-29)

    • Add rule C420 to check for dict comprehensions with constant values, encouraging replacement with dict.fromkeys().

      Thanks to Tom Kuson in PR [#553](https://github.com/adamchainz/flake8-comprehensions/issues/553) <https://github.com/adamchainz/flake8-comprehensions/pull/553>__.

    3.14.0 (2023-07-10)

    • Drop Python 3.7 support.

    3.13.0 (2023-06-15)

    • Support Python 3.12.

    3.12.0 (2023-04-13)

    • Add rule C418 to check for calls passing a dict literal or dict comprehension to dict().

    • Add rule C419 to check for calls passing a list comprehension to any()/all().

    3.11.1 (2023-03-21)

    • Fix false positives in C406 “unnecessary dict literal”.

      Fixes Issue [#260](https://github.com/adamchainz/flake8-comprehensions/issues/260) <https://github.com/adamchainz/flake8-comprehensions/issues/260>__.

    3.11.0 (2023-03-18)

    • Expand C416 to dict comprehensions.

      Thanks to Aaron Gokaslan in PR [#490](https://github.com/adamchainz/flake8-comprehensions/issues/490) <https://github.com/adamchainz/flake8-comprehensions/pull/490>__.

    3.10.1 (2022-10-29)

    • Fix false positive in rules C402 and C404 for dict() calls with keyword arguments.

      Thanks to Anders Kaseorg for the report in Issue [#457](https://github.com/adamchainz/flake8-comprehensions/issues/457) <https://github.com/adamchainz/flake8-comprehensions/issues/457>__.

    3.10.0 (2022-05-19)

    • Add rule C417 which recommends rewriting use of map() with lambda to an equivalent generator expression or comprehension.

    ... (truncated)