From c355d57799fd2767a024d3bb8130c2699a0c9382 Mon Sep 17 00:00:00 2001 From: sbalandi Date: Wed, 17 Apr 2024 12:03:43 +0100 Subject: [PATCH] change random generator, separate structure for ranges --- .../shared_test_classes/base/utils/ranges.hpp | 22 +++- .../src/base/ov_subgraph.cpp | 23 ++-- .../src/base/utils/generate_inputs.cpp | 55 --------- .../src/base/utils/ranges.cpp | 113 ++++++++++++------ .../include/common_test_utils/data_utils.hpp | 41 +++++-- .../common_test_utils/ov_tensor_utils.hpp | 62 +++++----- .../tests/generate_intepus.cpp | 56 +++++---- 7 files changed, 196 insertions(+), 176 deletions(-) diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/utils/ranges.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/utils/ranges.hpp index a13fe95c330162..9b92965e043898 100644 --- a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/utils/ranges.hpp +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/utils/ranges.hpp @@ -100,6 +100,7 @@ #include "openvino/op/convert.hpp" #include "openvino/op/range.hpp" #include "openvino/op/roi_align.hpp" +#include "openvino/op/fake_quantize.hpp" namespace ov { namespace test { @@ -189,8 +190,9 @@ static std::map TYPE_ALIAS {"integral", "real"}; -std::string get_range_id(const std::shared_ptr& node, size_t port, bool spectial = false); + std::shared_ptr general_real; + std::shared_ptr general_integral; + // key for map calculated in get_range_id and contais [Op Type Name]_[integral/real]_[port] + std::map> node_ranges; +public: + void collect_ranges(const std::shared_ptr& function, uint64_t kMaxRange); + void find_general_ranges(); + std::string get_range_id(const std::shared_ptr& node, size_t port); + ov::Tensor generate_input(std::shared_ptr node, size_t port, const ov::Shape& targetShape); +}; -std::map> collect_ranges(const std::shared_ptr& function, uint64_t kMaxRange); } // namespace utils } // namespace test diff --git a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp index a275386d9668ff..4de0862b2e8d91 100644 --- a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp +++ b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp @@ -29,10 +29,12 @@ #include "functional_test_utils/crash_handler.hpp" #include "shared_test_classes/base/ov_subgraph.hpp" -#include "shared_test_classes/base/utils/generate_inputs.hpp" +// #include "shared_test_classes/base/utils/generate_inputs.hpp" #include "shared_test_classes/base/utils/compare_results.hpp" #include "shared_test_classes/base/utils/calculate_thresholds.hpp" +#include "shared_test_classes/base/utils/ranges.hpp" + namespace ov { namespace test { @@ -320,28 +322,19 @@ void SubgraphBaseTest::compile_model() { void SubgraphBaseTest::generate_inputs(const std::vector& targetInputStaticShapes) { inputs.clear(); - auto inputDataMap = ov::test::utils::collect_ranges(function, testing::internal::Random::kMaxRange); - auto inputMap = utils::getInputMap(); + ov::test::utils::ModelRange modelRange; + modelRange.collect_ranges(function, testing::internal::Random::kMaxRange); + modelRange.find_general_ranges(); + auto itTargetShape = targetInputStaticShapes.begin(); for (const auto ¶m : function->get_parameters()) { std::shared_ptr inputNode = param; for (size_t i = 0; i < param->get_output_size(); i++) { for (const auto &node : param->get_output_target_inputs(i)) { std::shared_ptr nodePtr = node.get_node()->shared_from_this(); - auto it = inputMap.find(nodePtr->get_type_info()); - ASSERT_NE(it, inputMap.end()); for (size_t port = 0; port < nodePtr->get_input_size(); ++port) { if (nodePtr->get_input_node_ptr(port)->shared_from_this() == inputNode->shared_from_this()) { - if (!inputDataMap.empty()) { - std::string spetial_range_id = ov::test::utils::get_range_id(nodePtr, port, true); - if (inputDataMap.find(spetial_range_id) == inputDataMap.end()) { - spetial_range_id = ov::test::utils::get_range_id(nodePtr, port, false); - } - inputs.insert({param, it->second(nodePtr, port, param->get_element_type(), *itTargetShape, - inputDataMap[spetial_range_id])}); - } else { - inputs.insert({param, it->second(nodePtr, port, param->get_element_type(), *itTargetShape, nullptr)}); - } + inputs.insert({param, modelRange.generate_input(nodePtr, port, *itTargetShape)}); break; } } diff --git a/src/tests/functional/shared_test_classes/src/base/utils/generate_inputs.cpp b/src/tests/functional/shared_test_classes/src/base/utils/generate_inputs.cpp index 03de92c51b41a0..2430eb442481c1 100644 --- a/src/tests/functional/shared_test_classes/src/base/utils/generate_inputs.cpp +++ b/src/tests/functional/shared_test_classes/src/base/utils/generate_inputs.cpp @@ -748,20 +748,6 @@ ov::Tensor generate(const std::shared_ptr(node), port, elemType, targetShape, inGenRangeData); } -ov::Tensor generate(const std::shared_ptr& node, - size_t port, - const ov::element::Type& elemType, - const ov::Shape& targetShape, - std::shared_ptr inGenRangeData = nullptr) { - if (port == 2) { - ov::test::utils::InputGenerateData in_data; - in_data.start_from = 0; - in_data.range = 10; // max_seq_len - return ov::test::utils::create_and_fill_tensor(elemType, targetShape, in_data); - } - return generate(std::dynamic_pointer_cast(node), port, elemType, targetShape, inGenRangeData); -} - ov::Tensor generate(const std::shared_ptr& node, size_t port, const ov::element::Type& elemType, @@ -1149,14 +1135,6 @@ ov::Tensor generate(const std::shared_ptr& node, - size_t port, - const ov::element::Type& elemType, - const ov::Shape& targetShape) { - InputGenerateData inGenData(1, 0, 1, 1); - return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData); -} - ov::Tensor generate(const std::shared_ptr& node, size_t port, const ov::element::Type& elemType, @@ -1171,39 +1149,6 @@ ov::Tensor generate(const std::shared_ptr& node, - size_t port, - const ov::element::Type& elemType, - const ov::Shape& targetShape) { - if (port == 1) { - InputGenerateData inGenData(0, 1, 1000, 1); - return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData); - } - return generate(std::dynamic_pointer_cast(node), port, elemType, targetShape); -} - -ov::Tensor generate(const std::shared_ptr& node, - size_t port, - const ov::element::Type& elemType, - const ov::Shape& targetShape) { - if (port == 1) { - InputGenerateData inGenData(0, 1, 1000, 1); - return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData); - } - return generate(std::dynamic_pointer_cast(node), port, elemType, targetShape); -} - -ov::Tensor generate(const std::shared_ptr& node, - size_t port, - const ov::element::Type& elemType, - const ov::Shape& targetShape) { - if (port == 1) { - InputGenerateData inGenData(0, 1, 1000, 1); - return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData); - } - return generate(std::dynamic_pointer_cast(node), port, elemType, targetShape); -} - template ov::Tensor generateInput(const std::shared_ptr& node, size_t port, diff --git a/src/tests/functional/shared_test_classes/src/base/utils/ranges.cpp b/src/tests/functional/shared_test_classes/src/base/utils/ranges.cpp index 237af599051976..b0c718230bea7f 100644 --- a/src/tests/functional/shared_test_classes/src/base/utils/ranges.cpp +++ b/src/tests/functional/shared_test_classes/src/base/utils/ranges.cpp @@ -16,6 +16,8 @@ #include "shared_test_classes/base/utils/ranges.hpp" +#include "shared_test_classes/base/utils/generate_inputs.hpp" + namespace ov { namespace test { namespace utils { @@ -106,21 +108,17 @@ ov::test::utils::InputGenerateData get_range_by_type(ov::element::Type temp_type return inData; } -std::string get_range_id(const std::shared_ptr& node, size_t port, bool spectial) { - std::string range_id = std::to_string(node->input(port).get_element_type().is_real()); - if (spectial) { - range_id += '_' + std::string(node->get_type_name()) + '_' + std::to_string(port); - } +std::string ModelRange::get_range_id(const std::shared_ptr& node, size_t port) { + std::string range_id = std::string(node->get_type_name()); + range_id += "_" + TYPE_ALIAS[node->input(port).get_element_type().is_real()]; + range_id += "_" + std::to_string(port); return range_id; } -std::map> collect_ranges(const std::shared_ptr& function, uint64_t kMaxRange) { - bool success = true; - std::map> inputDataMap; - inputDataMap["0"] = std::make_shared(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined, - kMaxRange)); - inputDataMap["1"] = std::make_shared(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined, - kMaxRange)); +void ModelRange::collect_ranges(const std::shared_ptr& function, uint64_t kMaxRange) { + general_real = std::make_shared(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined, kMaxRange)); + general_integral = std::make_shared(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined, kMaxRange)); + for (const auto& node : function->get_ordered_ops()) { if (ov::op::util::is_output(node) || ov::op::util::is_constant(node) || @@ -144,47 +142,88 @@ std::map> colle } std::string range_id = get_range_id(node, port); - ov::test::utils::InputGenerateData temp_range; + ov::test::utils::InputGenerateData port_range; if (ranges.empty()) { - temp_range = ov::test::utils::get_range_by_type(node->input(port).get_element_type(), kMaxRange); - if (temp_range.start_from == inputDataMap[range_id]->start_from && temp_range.range == inputDataMap[range_id]->range) { - continue; - } + port_range = ov::test::utils::get_range_by_type(node->input(port).get_element_type(), kMaxRange); } else { auto op_range = ranges.at(node->input(port).get_element_type().is_real()); - temp_range = op_range.size() <= port ? op_range.front() : op_range.at(port); - if (temp_range.spetial) { - range_id = get_range_id(node, port, true); - inputDataMap[range_id] = std::make_shared(temp_range); - continue; - } + port_range = op_range.size() <= port ? op_range.front() : op_range.at(port); } + node_ranges[range_id] = std::make_shared(port_range); + } + } +} - success = inputDataMap[range_id]->correct_range(node, port, temp_range); - if (!success) - break; +void ModelRange::find_general_ranges() { + bool find_integral = true; + bool find_real = true; + + for (auto& node_range : node_ranges) { + if (node_range.second->spetial) { + continue; } - if (!success) - break; + if (node_range.first.find(TYPE_ALIAS[0]) != std::string::npos) { + find_integral = general_integral->correct_range(node_range.second); + } else if (node_range.first.find(TYPE_ALIAS[1]) != std::string::npos) { + find_real = general_real->correct_range(node_range.second); + } } + if (find_integral) { #ifndef NDEBUG - if (success) { - for (auto& range : inputDataMap) { - std::cout << "RANGE " << range.first << " :\n" << - " start from: " << std::to_string(range.second->start_from) << - " range: " << std::to_string(range.second->range) << - " resolution: " << std::to_string(range.second->resolution) << - " seed: " << std::to_string(range.second->resolution) << std::endl; - } + std::cout << "INEGRAL RANGE FOUND \n" << + " start from: " << std::to_string(general_integral->start_from) << + " range: " << std::to_string(general_integral->range) << + " resolution: " << std::to_string(general_integral->resolution) << + " seed: " << std::to_string(general_integral->seed) << std::endl; +#endif } else { + general_integral = nullptr; +#ifndef NDEBUG std::cout << " RANGE NOT FOUND \n"; +#endif } + + if (find_real) { +#ifndef NDEBUG + std::cout << "REAL RANGE FOUND \n" << + " start from: " << std::to_string(general_real->start_from) << + " range: " << std::to_string(general_real->range) << + " resolution: " << std::to_string(general_real->resolution) << + " seed: " << std::to_string(general_real->seed) << std::endl; #endif - return success ? inputDataMap : std::map>{}; + } else { + general_real = nullptr; +#ifndef NDEBUG + std::cout << " RANGE NOT FOUND \n"; +#endif + } } +ov::Tensor ModelRange::generate_input(std::shared_ptr node, size_t port, const ov::Shape& targetShape) { + std::string range_id = get_range_id(node, port); + if (node_ranges.find(range_id) == node_ranges.end()) { + throw std::runtime_error("Node info not in rages: " + node->get_friendly_name()); + } + + auto inputMap = ov::test::utils::getInputMap(); + auto it = inputMap.find(node->get_type_info()); + if (it == inputMap.end()) { + throw std::runtime_error("Couln't find Operation in inputMap: " + std::string(node->get_type_name())); + } + + std::shared_ptr range = node_ranges[range_id]; + if (!node_ranges[range_id]->spetial) { + if (node->get_input_element_type(port).is_real() && general_real) { + range = general_real; + } else { + range = general_integral; + } + } + + return it->second(node, port, node->get_input_element_type(port), targetShape, range); +} } // namespace utils } // namespace test diff --git a/src/tests/test_utils/common_test_utils/include/common_test_utils/data_utils.hpp b/src/tests/test_utils/common_test_utils/include/common_test_utils/data_utils.hpp index 9faf9ce215954c..a27d7a1ae35a1a 100644 --- a/src/tests/test_utils/common_test_utils/include/common_test_utils/data_utils.hpp +++ b/src/tests/test_utils/common_test_utils/include/common_test_utils/data_utils.hpp @@ -229,15 +229,40 @@ void inline fill_data_random(T* pointer, return; } - testing::internal::Random random(seed); - const uint32_t k_range = k * range; // range with respect to k - random.Generate(k_range); + if (k != 1) { + testing::internal::Random random(seed); + const uint32_t k_range = k * range; // range with respect to k + random.Generate(k_range); - if (start_from < 0 && !std::numeric_limits::is_signed) { - start_from = 0; - } - for (std::size_t i = 0; i < size; i++) { - pointer[i] = static_cast(start_from + (random.Generate(k_range) / k)); + if (start_from < 0 && !std::numeric_limits::is_signed) { + start_from = 0; + } + for (std::size_t i = 0; i < size; i++) { + pointer[i] = static_cast(start_from + (random.Generate(k_range) / k)); + } + } else { + if (std::numeric_limits::is_integer) { + std::mt19937 gen(seed); + std::uniform_int_distribution dist(0, range); + + for (std::size_t i = 0; i < size; i++) { + pointer[i] = static_cast(start_from + dist(gen)); + } + } else { + std::mt19937 gen(seed); + std::uniform_real_distribution dist(0, range); + + for (std::size_t i = 0; i < size; i++) { + auto value = dist(gen); + if (std::is_same::value) { + pointer[i] = static_cast(start_from + ov::float16(value)); + } else if (std::is_same::value) { + pointer[i] = static_cast(start_from + ov::bfloat16(value)); + } else { + pointer[i] = static_cast(start_from + value); + } + } + } } } diff --git a/src/tests/test_utils/common_test_utils/include/common_test_utils/ov_tensor_utils.hpp b/src/tests/test_utils/common_test_utils/include/common_test_utils/ov_tensor_utils.hpp index c473fcf21fd82d..eb2652b9cd3c27 100644 --- a/src/tests/test_utils/common_test_utils/include/common_test_utils/ov_tensor_utils.hpp +++ b/src/tests/test_utils/common_test_utils/include/common_test_utils/ov_tensor_utils.hpp @@ -57,67 +57,59 @@ struct InputGenerateData { } }; - bool correct_range(const std::shared_ptr& node, size_t port, const InputGenerateData& new_range) { + bool correct_range(const std::shared_ptr new_range) { bool success = true; - double new_max = new_range.start_from + new_range.range; + double new_max = new_range->start_from + new_range->range; double current_max = start_from + range; - if (start_from == new_range.start_from) { + if (start_from == new_range->start_from) { // nothing to do - -----start_curr/new+++++++++++++++range*res curr/new----------------------- // nothing to do - -----start_curr/new+++++++++++++++range*res curr----------range*res new---- // reduce range - -----start_curr/new+++++++++++++++range*res new-----------range*res curr--- if (current_max > new_max) { - range = new_range.range; - resolution = resolution != 1 && resolution <= new_range.resolution ? resolution : new_range.resolution; + range = new_range->range; + resolution = new_range->resolution > resolution ? new_range->resolution : resolution; } - } else if (start_from > new_range.start_from) { + } else if (start_from > new_range->start_from) { // nothing to do - -----start_new-----start_curr++++++++++range*res curr/new------------------- // nothing to do - -----start_new-----start_curr++++++++++range*res curr------range*res new---- // reduce range - -----start_new-----start_curr++++++++++range*res new-------range*res curr--- // could not find range - -----start_new---range*res new-----start_curr-----range*res curr--- if (start_from > new_max) { success = false; -#ifndef NDEBUG - std::cout << " FAIL TO FIND RANGE: start_from > new_range.start_from + new_range.range " << - // " new range node: " << node->get_friendly_name() << " CURRENT start_from: " << - // std::to_string(start_from) << - " new range node: " - << "node" - << " CURRENT start_from: " << std::to_string(start_from) - << " NEW start_from: " << std::to_string(new_range.start_from) - << " NEW max: " << std::to_string(new_max) << std::endl; -#endif +// #ifndef NDEBUG + std::cout << " FAIL TO FIND RANGE: current->start_from > new_range->start_from + new_range->range " + << " current->start_from: " << std::to_string(start_from) + << " new_range->start_from: " << std::to_string(new_range->start_from) + << " new_range max: " << std::to_string(new_max) << std::endl; +// #endif } else if (current_max > new_max) { range = (uint32_t)round(new_max - start_from); - resolution = resolution != 1 && resolution <= new_range.resolution ? resolution : new_range.resolution; + resolution = new_range->resolution > resolution ? new_range->resolution : resolution; } - } else if (start_from < new_range.start_from) { + } else if (start_from < new_range->start_from) { // reset to new - -----start_curr-----start_new++++++++++range*res curr/new------------------- // reset to new - -----start_curr-----start_new++++++++++range*res new-------range*res curr--- // recalculate range - -----start_curr-----start_new++++++++++range*res curr------range*res new---- // could not find range - -----start_curr---range*res curr-----start_new-----range*res new--- - if (current_max < new_range.start_from) { + if (current_max < new_range->start_from) { success = false; -#ifndef NDEBUG - std::cout << " FAIL TO FIND RANGE: current.start_from + current.range < new_range.start_from " << - // " new range node: " << node->get_friendly_name() << " NODE start_from: " << - // std::to_string(new_range.start_from) << - " new range node: " - << "node" - << " NODE start_from: " << std::to_string(new_range.start_from) - << " CURRENT start_from: " << std::to_string(start_from) - << " CURRENT max: " << std::to_string(current_max) << std::endl; -#endif +// #ifndef NDEBUG + std::cout << " FAIL TO FIND RANGE: current->start_from + current->range < new_range->start_from " + << " new_range start_from: " << std::to_string(new_range->start_from) + << " current->start_from: " << std::to_string(start_from) + << " current max: " << std::to_string(current_max) << std::endl; +// #endif } else if (current_max >= new_max) { - start_from = new_range.start_from; - range = new_range.range; - resolution = resolution != 1 && resolution <= new_range.resolution ? resolution : new_range.resolution; + start_from = new_range->start_from; + range = new_range->range; + resolution = new_range->resolution > resolution ? new_range->resolution : resolution; ; } else { - range = (uint32_t)round(current_max - new_range.start_from); - resolution = resolution != 1 && resolution <= new_range.resolution ? resolution : new_range.resolution; - start_from = new_range.start_from; + range = (uint32_t)round(current_max - new_range->start_from); + resolution = new_range->resolution > resolution ? new_range->resolution : resolution; + start_from = new_range->start_from; } } diff --git a/src/tests/test_utils/common_test_utils/tests/generate_intepus.cpp b/src/tests/test_utils/common_test_utils/tests/generate_intepus.cpp index dc1ce0f456d0fd..2a33cb09c7302f 100644 --- a/src/tests/test_utils/common_test_utils/tests/generate_intepus.cpp +++ b/src/tests/test_utils/common_test_utils/tests/generate_intepus.cpp @@ -30,13 +30,10 @@ using ov::op::v1::FloorMod; using ov::op::v1::Reshape; -TEST(Ranges, ranges_by_type) { +TEST(Ranges, ranges_by_type_real) { auto p0 = std::make_shared(ov::element::f16, Shape{3}); - auto p1 = std::make_shared(ov::element::i8, Shape{3}); - - auto convert = std::make_shared(p1, ov::element::f16); - auto add = std::make_shared(p0, convert); - + auto p1 = std::make_shared(ov::element::f16, Shape{3}); + auto add = std::make_shared(p0, p1); auto func = std::make_shared(add, ov::ParameterVector{p0, p1}); auto ranges = ov::test::utils::collect_ranges(func, testing::internal::Random::kMaxRange); @@ -55,27 +52,44 @@ TEST(Ranges, ranges_by_type) { auto inputMap = ov::test::utils::getInputMap(); auto it = inputMap.find(add->get_type_info()); - ov::Tensor tensor1 = it->second(add, 0, add->input(0).get_element_type(), Shape{3}, ranges[range_id]); - auto data1 = tensor1.data(); - for (size_t i = 0; i < shape_size(tensor1.get_shape()); ++i) { - double value = data1[i]; - ASSERT_GE(value, lowest); - ASSERT_LE(value, range); + + for (size_t port = 0; port < add->get_input_size(); ++port) { + ov::Tensor tensor1 = it->second(add, port, add->input(port).get_element_type(), Shape{3}, ranges[range_id]); + auto data1 = tensor1.data(); + for (size_t i = 0; i < shape_size(tensor1.get_shape()); ++i) { + double value = data1[i]; + ASSERT_GE(value, lowest); + ASSERT_LE(value, range); + } } +} + +TEST(Ranges, ranges_by_type_int) { + auto p0 = std::make_shared(ov::element::i8, Shape{3}); + auto p1 = std::make_shared(ov::element::i8, Shape{3}); + auto add = std::make_shared(p0, p1); + auto func = std::make_shared(add, ov::ParameterVector{p0, p1}); + + auto ranges = ov::test::utils::collect_ranges(func, testing::internal::Random::kMaxRange); + + ASSERT_EQ(ranges.size(), 2); - range_id = ov::test::utils::get_range_id(convert, 0); + auto range_id = ov::test::utils::get_range_id(add, 0); ASSERT_EQ(ranges[range_id]->start_from, std::numeric_limits::lowest()); - range = static_cast(std::numeric_limits::max()) - static_cast(std::numeric_limits::lowest()); + uint32_t range = static_cast(std::numeric_limits::max()) - static_cast(std::numeric_limits::lowest()); ASSERT_EQ(ranges[range_id]->range, range); ASSERT_EQ(ranges[range_id]->resolution, 1); - it = inputMap.find(convert->get_type_info()); - ov::Tensor tensor2 = it->second(convert, 0, convert->input(0).get_element_type(), Shape{3}, ranges[range_id]); - auto data2 = tensor2.data(); - for (size_t i = 0; i < shape_size(tensor2.get_shape()); ++i) { - double value = data2[i]; - ASSERT_GE(value, std::numeric_limits::lowest()); - ASSERT_LE(value, std::numeric_limits::max()); + auto inputMap = ov::test::utils::getInputMap(); + auto it = inputMap.find(add->get_type_info()); + for (size_t port = 0; port < add->get_input_size(); ++port) { + ov::Tensor tensor1 = it->second(add, port, add->input(port).get_element_type(), Shape{3}, ranges[range_id]); + auto data1 = tensor1.data(); + for (size_t i = 0; i < shape_size(tensor1.get_shape()); ++i) { + double value = data1[i]; + ASSERT_GE(value, std::numeric_limits::lowest()); + ASSERT_LE(value, std::numeric_limits::max()); + } } }