[CONFORMANCE] Provide correct numbers for rel_passed && rel_all (open…

…vinotoolkit#20035)
iefode · Sep 26, 2023 · 47dec42 · 47dec42
1 parent b6427ad
commit 47dec42
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 34 deletions.
diff --git a/...s/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp b/...s/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp
@@ -214,6 +214,9 @@ uint64_t clip(uint64_t n, uint64_t lower, uint64_t upper) {
 }
 
 void ReadIRTest::SetUp() {
+    // todo: find the optimal way to find TEST_P instances
+    // inference + query_model + import_export
+    summary.setDowngradeCoefficient(3);
     std::pair<std::string, std::string> model_pair;
     std::tie(model_pair, targetDevice, configuration) = this->GetParam();
     std::tie(path_to_model, path_to_cache) = model_pair;

diff --git a/...sts/test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp b/...sts/test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp
@@ -30,16 +30,19 @@ class OpSummary : public virtual Summary {
     static OpSummary* p_instance;
     static bool extractBody;
     std::map<ov::NodeTypeInfo, PassRate> opsStats = {};
+    unsigned short int downgrade_coefficient;
 
     std::string getOpVersion(const std::string& version);
 
 protected:
-    OpSummary();
+    OpSummary(unsigned short int downgrade_coefficient = 1);
+    static OpSummary& createInstance(unsigned short int downgrade_coefficient = 1);
     static OpSummaryDestroyer destroyer;
     friend class OpSummaryDestroyer;
 
 public:
     static OpSummary& getInstance();
+    static void setDowngradeCoefficient(unsigned short int downgrade_coefficient = 1);
 
     std::map<ov::NodeTypeInfo, PassRate> getOPsStats() {
         return opsStats;

diff --git a/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/summary.hpp b/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/summary.hpp
@@ -66,18 +66,6 @@ class Summary {
 
     std::string getDeviceName() const;
 
-    // #define IE_TEST_DEBUG
-
-#ifdef IE_TEST_DEBUG
-    void saveDebugReport(const char* className,
-                         const char* opName,
-                         unsigned long passed,
-                         unsigned long failed,
-                         unsigned long skipped,
-                         unsigned long crashed,
-                         unsigned long hanged);
-#endif  // IE_TEST_DEBUG
-
     virtual void saveReport() {}
 
     void setReportFilename(const std::string& val);

diff --git a/src/tests/test_utils/functional_test_utils/src/summary/op_summary.cpp b/src/tests/test_utils/functional_test_utils/src/summary/op_summary.cpp
@@ -27,18 +27,30 @@ void OpSummaryDestroyer::initialize(OpSummary* p) {
     p_instance = p;
 }
 
-OpSummary::OpSummary() {
+OpSummary::OpSummary(unsigned short int in_downgrade_coefficient) {
     reportFilename = ov::test::utils::OP_REPORT_FILENAME;
+    downgrade_coefficient = in_downgrade_coefficient;
 }
 
-OpSummary& OpSummary::getInstance() {
+OpSummary& OpSummary::createInstance(unsigned short int in_downgrade_coefficient) {
     if (!p_instance) {
-        p_instance = new OpSummary();
+        p_instance = new OpSummary(in_downgrade_coefficient);
         destroyer.initialize(p_instance);
     }
     return *p_instance;
 }
 
+OpSummary& OpSummary::getInstance() {
+    return createInstance();
+}
+
+void OpSummary::setDowngradeCoefficient(unsigned short int in_downgrade_coefficient) {
+    if (p_instance && p_instance->downgrade_coefficient != in_downgrade_coefficient) {
+        p_instance->downgrade_coefficient = in_downgrade_coefficient;
+    }
+    auto& summary_instance = createInstance(in_downgrade_coefficient);
+}
+
 void OpSummary::updateOPsStats(const ov::NodeTypeInfo& op,
                                const PassRate::Statuses& status,
                                double rel_influence_coef) {
@@ -218,23 +230,6 @@ void OpSummary::updateOPsImplStatus(const std::shared_ptr<ov::Model>& model, con
     }
 }
 
-#ifdef IE_TEST_DEBUG
-void Summary::saveDebugReport(const char* className,
-                              const char* opName,
-                              unsigned long passed,
-                              unsigned long failed,
-                              unsigned long skipped,
-                              unsigned long crashed,
-                              unsigned long hanged) {
-    std::string outputFilePath = "./part_report.txt";
-    std::ofstream file;
-    file.open(outputFilePath, std::ios_base::app);
-    file << className << ' ' << opName << ' ' << passed << ' ' << failed << ' ' << skipped << ' ' << crashed << ' '
-         << hanged << '\n';
-    file.close();
-}
-#endif  // IE_TEST_DEBUG
-
 void OpSummary::saveReport() {
     if (isReported) {
         return;
@@ -316,7 +311,10 @@ void OpSummary::saveReport() {
     pugi::xml_node resultsNode = root.child("results");
     pugi::xml_node currentDeviceNode = resultsNode.append_child(summary.deviceName.c_str());
     std::unordered_set<std::string> opList;
-    for (const auto& it : stats) {
+    for (auto& it : stats) {
+        it.second.rel_passed /= downgrade_coefficient;
+        it.second.rel_all /= downgrade_coefficient;
+
         std::string name = std::string(it.first.name) + "-" + getOpVersion(it.first.version_id);
         opList.insert(name);
         pugi::xml_node entry = currentDeviceNode.append_child(name.c_str());