Mark mandatory conformance tests for HW plugins (#27197)

### Details: - *Mark OVHoldersTestOnImportedNetwork and OVClassQueryModelTest mandatory for HW plugin and optional for meta-plugin* - *Remove the meta-plugin skipping in test cases* ### Tickets: - *CVS-130212* --------- Signed-off-by: Chen, Peter <[email protected]> Co-authored-by: panhaiqi <[email protected]>
openvinotoolkit · Oct 26, 2024 · f3e3f75 · f3e3f75
1 parent 0c07136
commit f3e3f75
Show file tree

Hide file tree

Showing 8 changed files with 35 additions and 39 deletions.
diff --git a/src/plugins/hetero/tests/functional/skip_tests_config.cpp b/src/plugins/hetero/tests/functional/skip_tests_config.cpp
@@ -14,6 +14,6 @@ std::vector<std::string> disabledTestPatterns() {
         R"(.*smoke_(Multi|Auto|Hetero)_BehaviorTests.*OVPropertiesTests.*SetCorrectProperties.*)",
         R"(.*smoke_(Multi|Auto|Hetero)_BehaviorTests.*OVPropertiesTests.*canSetPropertyAndCheckGetProperty.*)",
         R"(.*OVInferRequestCheckTensorPrecision.*get(Input|Output|Inputs|Outputs)From.*FunctionWith(Single|Several).*type=(u4|u1|i4|boolean).*)",
-    };
+        R"(.*OVGetMetricPropsTest.*OVGetMetricPropsTest.*GetMetricAndPrintNoThrow_AVAILABLE_DEVICES.*)"};
     return retVector;
 }
diff --git a/.../plugin/conformance/test_runner/api_conformance_runner/src/ov_plugin/core_integration.cpp b/.../plugin/conformance/test_runner/api_conformance_runner/src/ov_plugin/core_integration.cpp
@@ -6,6 +6,8 @@
 #include "behavior/ov_plugin/query_model.hpp"
 #include "openvino/runtime/core.hpp"
 #include "ov_api_conformance_helpers.hpp"
+#include "behavior/ov_plugin/properties_tests.hpp"
+#include "base/ov_behavior_test_utils.hpp"
 
 using namespace ov::test::behavior;
 using namespace ov::test::conformance;
@@ -15,17 +17,17 @@ namespace {
 // OV Class Common tests with <pluginName, deviceName params>
 //
 
-INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory,
-                         OVClassModelTestP,
-                         ::testing::Values(ov::test::utils::target_device));
+INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVClassModelTestP, ::testing::Values(ov::test::utils::target_device));
 
 INSTANTIATE_TEST_SUITE_P(ov_plugin,
                          OVClassModelOptionalTestP,
                          ::testing::Values(ov::test::utils::target_device));
 
 // OV Class Query network
 
-INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory,
+static std::string (*NULL_getTestCaseName)(const testing::TestParamInfo<std::string>& info) = NULL;
+INSTANTIATE_TEST_SUITE_P(ov_plugin,
                          OVClassQueryModelTest,
-                         ::testing::Values(ov::test::utils::target_device));
+                         ::testing::Values(ov::test::utils::target_device),
+                         MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM());
 }  // namespace
diff --git a/...ctional/plugin/conformance/test_runner/api_conformance_runner/src/ov_plugin/life_time.cpp b/...ctional/plugin/conformance/test_runner/api_conformance_runner/src/ov_plugin/life_time.cpp
@@ -4,6 +4,7 @@
 
 #include "behavior/ov_plugin/life_time.hpp"
 #include "ov_api_conformance_helpers.hpp"
+#include "base/ov_behavior_test_utils.hpp"
 
 using namespace ov::test::behavior;
 using namespace ov::test::conformance;
@@ -14,7 +15,8 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVHoldersTest,
         ::testing::Values(ov::test::utils::target_device),
         OVHoldersTest::getTestCaseName);
 
-INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVHoldersTestOnImportedNetwork,
-        ::testing::Values(ov::test::utils::target_device),
-        OVHoldersTestOnImportedNetwork::getTestCaseName);
+INSTANTIATE_TEST_SUITE_P(ov_plugin,
+                         OVHoldersTestOnImportedNetwork,
+                         ::testing::Values(ov::test::utils::target_device),
+                         MARK_MANDATORY_API_FOR_HW_DEVICE_WITH_PARAM(OVHoldersTestOnImportedNetwork::getTestCaseName));
 }  // namespace
diff --git a/...tional/plugin/conformance/test_runner/api_conformance_runner/src/ov_plugin/properties.cpp b/...tional/plugin/conformance/test_runner/api_conformance_runner/src/ov_plugin/properties.cpp
@@ -105,17 +105,19 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckMetricsPropsTests_ModelDependceProps,
 // OV Class GetMetric
 //
 
-INSTANTIATE_TEST_SUITE_P(
-        ov_plugin_mandatory, OVGetMetricPropsTest,
-        ::testing::Values(ov::test::utils::target_device));
+INSTANTIATE_TEST_SUITE_P(ov_plugin,
+                         OVGetMetricPropsTest,
+                         ::testing::Values(ov::test::utils::target_device),
+                         MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM());
 
 INSTANTIATE_TEST_SUITE_P(
         ov_plugin, OVGetMetricPropsOptionalTest,
         ::testing::Values(ov::test::utils::target_device));
 
-INSTANTIATE_TEST_SUITE_P(
-        ov_plugin_mandatory, OVGetAvailableDevicesPropsTest,
-        ::testing::Values(ov::test::utils::target_device));
+INSTANTIATE_TEST_SUITE_P(ov_plugin,
+                         OVGetAvailableDevicesPropsTest,
+                         ::testing::Values(ov::test::utils::target_device),
+                         MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM());
 
 //
 // OV Class GetConfig

diff --git a/src/tests/functional/plugin/shared/include/base/ov_behavior_test_utils.hpp b/src/tests/functional/plugin/shared/include/base/ov_behavior_test_utils.hpp
@@ -30,12 +30,23 @@
 #include "common_test_utils/subgraph_builders/concat_with_params.hpp"
 #include "common_test_utils/subgraph_builders/split_concat.hpp"
 
-#define MARK_MANDATORY_PROPERTY_FOR_HW_DEVICE(GET_TEST_NAME)                                          \
-    [](const testing::TestParamInfo<PropertiesParams>& info) {                                        \
-        std::string name = GET_TEST_NAME(info);                                                       \
+#define MARK_MANDATORY_PROPERTY_FOR_HW_DEVICE(GET_TEST_NAME)                                            \
+    [](const testing::TestParamInfo<PropertiesParams>& info) {                                          \
+        std::string name = GET_TEST_NAME(info);                                                         \
         return (sw_plugin_in_target_device(ov::test::utils::target_device) ? "" : "mandatory_") + name; \
     }
 
+#define MARK_MANDATORY_API_FOR_HW_DEVICE_WITH_PARAM(GET_TEST_NAME)                                      \
+    [](const testing::TestParamInfo<std::string>& info) {                                               \
+        std::string name = GET_TEST_NAME(info);                                                         \
+        return (sw_plugin_in_target_device(ov::test::utils::target_device) ? "" : "mandatory_") + name; \
+    }
+
+#define MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM()                                                \
+    [](const testing::TestParamInfo<std::string>& info) {                                               \
+        return sw_plugin_in_target_device(ov::test::utils::target_device) ? "" : "mandatory_";          \
+    }
+
 namespace ov {
 namespace test {
 namespace behavior {

diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/query_model.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/query_model.hpp
@@ -76,9 +76,6 @@ TEST_P(OVClassQueryModelTest, QueryModelWithMatMul) {
 }
 
 TEST_P(OVClassQueryModelTest, QueryModelHETEROWithDeviceIDNoThrow) {
-    if (sw_plugin_in_target_device(target_device)) {
-        return;
-    }
     ov::Core ie = ov::test::utils::create_core();
 
     auto deviceIDs = ie.get_property(target_device, ov::available_devices);

diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/life_time.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/life_time.cpp
@@ -148,9 +148,6 @@ void OVHoldersTestOnImportedNetwork::TearDown() {
 }
 
 TEST_P(OVHoldersTestOnImportedNetwork, LoadedTensor) {
-    if (sw_plugin_in_target_device(target_device)) {
-        return;
-    }
     ov::Core core = ov::test::utils::create_core();
     std::stringstream stream;
     {
@@ -163,9 +160,6 @@ TEST_P(OVHoldersTestOnImportedNetwork, LoadedTensor) {
 }
 
 TEST_P(OVHoldersTestOnImportedNetwork, CreateRequestWithCoreRemoved) {
-    if (sw_plugin_in_target_device(target_device)) {
-        return;
-    }
     ov::Core core = ov::test::utils::create_core();
     std::stringstream stream;
     {

diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp
@@ -588,9 +588,6 @@ TEST_P(OVCheckMetricsPropsTests_ModelDependceProps, ChangeCorrectDevicePropertie
 TEST_P(OVClassSetDefaultDeviceIDPropTest, SetDefaultDeviceIDNoThrow) {
     ov::Core ie = ov::test::utils::create_core();
     // sw plugins are not requested to support `ov::available_devices` and ` ov::device::id` property
-    if (sw_plugin_in_target_device(target_device)) {
-        return;
-    }
     auto deviceIDs = ie.get_property(target_device, ov::available_devices);
     if (std::find(deviceIDs.begin(), deviceIDs.end(), deviceID) == deviceIDs.end()) {
         GTEST_FAIL();
@@ -613,9 +610,6 @@ TEST_P(OVSpecificDeviceSetConfigTest, GetConfigSpecificDeviceNoThrow) {
         deviceID =  target_device.substr(pos + 1,  target_device.size());
     }
     // sw plugins are not requested to support `ov::available_devices`, `ov::device::id` and `ov::num_streams` property
-    if (sw_plugin_in_target_device(target_device)) {
-        return;
-    }
     auto deviceIDs = ie.get_property(clear_target_device, ov::available_devices);
     if (std::find(deviceIDs.begin(), deviceIDs.end(), deviceID) == deviceIDs.end()) {
         GTEST_FAIL() << "No DeviceID" << std::endl;
@@ -672,9 +666,6 @@ TEST_P(OVSpecificDeviceGetConfigTest, GetConfigSpecificDeviceNoThrow) {
 }
 
 TEST_P(OVGetAvailableDevicesPropsTest, GetAvailableDevicesNoThrow) {
-    if (sw_plugin_in_target_device(target_device)) {
-        return;
-    }
     ov::Core ie = ov::test::utils::create_core();
     std::vector<std::string> devices;
 
@@ -727,9 +718,6 @@ TEST_P(OVClassSetDevicePriorityConfigPropsTest, SetConfigAndCheckGetConfigNoThro
 }
 
 TEST_P(OVGetMetricPropsTest, GetMetricAndPrintNoThrow_AVAILABLE_DEVICES) {
-    if (sw_plugin_in_target_device(target_device)) {
-        return;
-    }
     ov::Core ie = ov::test::utils::create_core();
     std::vector<std::string> device_ids;