rework auto test cases (openvinotoolkit#19862)

* initial commit Signed-off-by: fishbell <[email protected]> * clean up Signed-off-by: fishbell <[email protected]> * fix windows build failure Signed-off-by: fishbell <[email protected]> * enable auto func tests Signed-off-by: fishbell <[email protected]> * enable auto_func_test to ci Signed-off-by: fishbell <[email protected]> * some clean up in gpu case Signed-off-by: fishbell <[email protected]> * clang Signed-off-by: fishbell <[email protected]> * fix build warning Signed-off-by: fishbell <[email protected]> * enable new tests Signed-off-by: fishbell <[email protected]> * fix build warning Signed-off-by: fishbell <[email protected]> * enable consistency test Signed-off-by: fishbell <[email protected]> * try fix build error on manylinux Signed-off-by: fishbell <[email protected]> * enable cpplint Signed-off-by: fishbell <[email protected]> * enable clang-format Signed-off-by: fishbell <[email protected]> enable some tests Signed-off-by: fishbell <[email protected]> * fix typo Signed-off-by: fishbell <[email protected]> * clang for unit tests Signed-off-by: fishbell <[email protected]> * fix merge conflict Signed-off-by: fishbell <[email protected]> --------- Signed-off-by: fishbell <[email protected]>
iefode · Oct 7, 2023 · ad41d0f · ad41d0f
1 parent 7e7b648
commit ad41d0f
Show file tree

Hide file tree

Showing 143 changed files with 5,065 additions and 3,760 deletions.
diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml
@@ -411,6 +411,9 @@ jobs:
   - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_auto_unit_tests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ov_auto_unit_tests.xml
     displayName: 'AUTO UT'
 
+  - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_auto_func_tests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ov_auto_func_tests.xml
+    displayName: 'AUTO FuncTests'
+
   - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_auto_batch_unit_tests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ov_auto_batch_unit_tests.xml
     displayName: 'AutoBatch UT'
 

diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml
@@ -305,6 +305,9 @@ jobs:
   - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_auto_unit_tests --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-ov_auto_unit_tests.xml
     displayName: 'AUTO UT'
 
+  - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_auto_func_tests --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-ov_auto_func_tests.xml
+    displayName: 'AUTO FuncTests'
+
   - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_auto_batch_unit_tests --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-ov_auto_batch_unit_tests.xml
     displayName: 'AutoBatch UT'
 

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -733,6 +733,12 @@ jobs:
           ${INSTALL_TEST_DIR}/ov_auto_unit_tests --gtest_print_time=1 \
                 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_unit_tests.xml
 
+      - name: AUTO func Tests
+        run: |
+          source ${{ env.INSTALL_DIR }}/setupvars.sh
+          ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \
+                --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml
+
       - name: Template plugin func tests
         run: |
           source ${INSTALL_DIR}/setupvars.sh

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -595,6 +595,11 @@ jobs:
         run: |
           call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_auto_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_unit_tests.xml
 
+      - name: AUTO FuncTests
+        shell: cmd
+        run: |
+          call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml
+
       - name: Template plugin tests
         shell: cmd
         run: |

diff --git a/src/inference/src/infer_request.cpp b/src/inference/src/infer_request.cpp
@@ -30,6 +30,8 @@ OPENVINO_SUPPRESS_DEPRECATED_START
         __VA_ARGS__;                                                        \
     } catch (const ::InferenceEngine::RequestBusy& ex) {                    \
         ov::Busy::create(ex.what());                                        \
+    } catch (const ov::Busy&) {                                             \
+        throw;                                                              \
     } catch (const std::exception& ex) {                                    \
         OPENVINO_THROW(ex.what());                                          \
     } catch (...) {                                                         \

diff --git a/src/plugins/auto/src/plugin.cpp b/src/plugins/auto/src/plugin.cpp
@@ -415,8 +415,9 @@ std::shared_ptr<ov::ICompiledModel> Plugin::compile_model_impl(const std::string
     load_config.set_user_property(pre_process_config(properties));
     load_config.apply_user_properties();
     if (!work_mode_auto) {
-        if (iter_config != properties.end() && iter_config->second != "THROUGHPUT") {
-            LOG_WARNING_TAG("User set perf_hint:%s, but MULTI supports THROUGHPUT only", iter_config->second.as<std::string>().c_str());
+        if (iter_config != properties.end() && iter_config->second.as<std::string>() != "THROUGHPUT") {
+            LOG_WARNING_TAG("User set perf_hint:%s, but MULTI supports THROUGHPUT only",
+                            iter_config->second.as<std::string>().c_str());
         }
         load_config.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
     }

diff --git a/src/plugins/auto/tests/CMakeLists.txt b/src/plugins/auto/tests/CMakeLists.txt
@@ -8,4 +8,5 @@ if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL
     set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
 endif()
 
+add_subdirectory(functional)
 add_subdirectory(unit)
diff --git a/src/plugins/auto/tests/functional/CMakeLists.txt b/src/plugins/auto/tests/functional/CMakeLists.txt
@@ -0,0 +1,34 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set(TARGET_NAME ov_auto_func_tests)
+
+if(ENABLE_AUTO_BATCH)
+    list(APPEND DEPENDENCIES openvino_auto_batch_plugin)
+    list(APPEND COMPILE_DEFINITIONS ENABLE_AUTO_BATCH)
+endif()
+
+if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    ie_add_compiler_flags(/wd4305)
+endif()
+
+ov_add_test_target(
+        NAME ${TARGET_NAME}
+        ROOT ${CMAKE_CURRENT_SOURCE_DIR}
+        LINK_LIBRARIES
+            openvino::runtime::dev
+            gtest
+            gtest_main
+            openvino::funcSharedTests
+        INCLUDES
+            ${CMAKE_CURRENT_SOURCE_DIR}
+            ${TEST_COMMON_INCLUDE_DIR}
+        ADD_CLANG_FORMAT
+        LABELS
+            Multi
+            Auto
+)
+
+target_compile_definitions(${TARGET_NAME} PRIVATE ${COMPILE_DEFINITIONS})
+set_ie_threading_interface_for(${TARGET_NAME})
diff --git a/src/plugins/auto/tests/functional/behavior/async_compiled_for_multiple_device_test.cpp b/src/plugins/auto/tests/functional/behavior/async_compiled_for_multiple_device_test.cpp
@@ -0,0 +1,97 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "auto_func_test.hpp"
+#ifdef __GLIBC__
+#    include <gnu/libc-version.h>
+#    if __GLIBC_MINOR__ >= 34
+#        define ENABLETESTTHREADING
+#    endif
+#endif
+
+using namespace ov::auto_plugin::tests;
+
+#ifdef ENABLETESTTHREADING
+TEST_F(AutoFuncTests, can_compile_with_multiple_devices) {
+    ov::CompiledModel compiled_model;
+    ASSERT_NO_THROW(compiled_model =
+                        core.compile_model(model_can_batch, "AUTO", {ov::device::priorities("MOCK_GPU", "MOCK_CPU")}));
+    compiled_model = core.compile_model(model_can_batch,
+                                        "AUTO",
+                                        {ov::device::priorities("MOCK_GPU", "MOCK_CPU"),
+                                         ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)});
+}
+
+TEST_F(AutoFuncTests, threading_test) {
+    ThreadingTest::runParallel(
+        [&]() {
+            (void)core.compile_model(model_can_batch, "AUTO", {ov::device::priorities("MOCK_GPU", "MOCK_CPU")});
+        },
+        10,
+        10);
+    ThreadingTest::runParallel(
+        [&]() {
+            (void)core.compile_model(model_can_batch,
+                                     "AUTO",
+                                     {ov::device::priorities("MOCK_GPU", "MOCK_CPU"),
+                                      ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)});
+        },
+        10,
+        10);
+}
+
+TEST_F(AutoFuncTests, threading_test_cache_enabled) {
+    core.set_property(ov::cache_dir(cache_path));
+    ThreadingTest::runParallel(
+        [&]() {
+            (void)core.compile_model(model_can_batch,
+                                     "AUTO",
+                                     {ov::device::priorities("MOCK_GPU", "MOCK_CPU"),
+                                      ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)});
+        },
+        10,
+        10);
+    core.set_property(ov::cache_dir(""));
+}
+
+TEST_F(AutoFuncTests, threading_test_get_version) {
+    ThreadingTest::runParallel([&]() {
+        auto versions = core.get_versions("AUTO");
+        ASSERT_LE(1u, versions.size());
+    });
+}
+
+TEST_F(AutoFuncTests, theading_compiled_with_cpu_help) {
+    ThreadingTest::runParallel(
+        [&]() {
+            (void)core.compile_model(model_can_batch, "AUTO", {ov::device::priorities("MOCK_GPU", "MOCK_CPU")});
+        },
+        10,
+        10);
+}
+
+TEST_F(AutoFuncTests, threading_test_hardware_slower) {
+    core.compile_model(model_cannot_batch, "MOCK_CPU");
+    core.compile_model(model_cannot_batch, "MOCK_GPU");  // need to initialize the order of plugins in mock_engine
+    register_plugin_mock_gpu_compile_slower(core, "MOCK_GPU_SLOWER", {});
+    ThreadingTest::runParallel(
+        [&]() {
+            (void)core.compile_model(model_can_batch, "AUTO", {ov::device::priorities("MOCK_GPU_SLOWER", "MOCK_CPU")});
+        },
+        10,
+        10);
+}
+
+TEST_F(AutoFuncTests, threading_test_cpu_help_slower) {
+    core.compile_model(model_cannot_batch, "MOCK_CPU");
+    core.compile_model(model_cannot_batch, "MOCK_GPU");  // need to initialize the order of plugins in mock_engine
+    register_plugin_mock_cpu_compile_slower(core, "MOCK_CPU_SLOWER", {});
+    ThreadingTest::runParallel(
+        [&]() {
+            (void)core.compile_model(model_can_batch, "AUTO", {ov::device::priorities("MOCK_GPU", "MOCK_CPU_SLOWER")});
+        },
+        10,
+        10);
+}
+#endif