Skip to content

Commit

Permalink
change random generator, separate structure for ranges
Browse files Browse the repository at this point in the history
  • Loading branch information
@sbalandi
sbalandi committed Apr 17, 2024
1 parent a415155 commit c355d57
Show file tree
Hide file tree
Showing 7 changed files with 196 additions and 176 deletions.
22 changes: 17 additions & 5 deletions src/tests/functional/shared_test_classes/include/shared_test_classes/base/utils/ranges.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
#include "openvino/op/convert.hpp"
#include "openvino/op/range.hpp"
#include "openvino/op/roi_align.hpp"
#include "openvino/op/fake_quantize.hpp"

namespace ov {
namespace test {
Expand Down Expand Up @@ -189,8 +190,9 @@ static std::map<ov::NodeTypeInfo, std::vector<std::vector<ov::test::utils::Input
{ ov::op::v0::LRN::get_type_info_static(), {{{0, 15}}, {{0, 8, 32}}} },
{ ov::op::v1::Pad::get_type_info_static(), {{{0, 15}}, {{0, 8, 32}}} },
{ ov::op::v3::Broadcast::get_type_info_static(), {{{0, 2000}}, {{0, 2000, 32768}}} },
{ ov::op::v9::NonMaxSuppression::get_type_info_static(), {{{0, 15}}, {{0, 8, 32}, {0, 1, 1000, 1, true}}} },
{ ov::op::v8::MatrixNms::get_type_info_static(), {{{0, 15}}, {{0, 8, 32}, {0, 1, 1000, 1, true}}} },
{ ov::op::v5::NonMaxSuppression::get_type_info_static(), {{{0, 15}, {0, 1, 1000, 1, true}}, {{0, 8, 32}, {0, 1, 1000, 1, true}}} },
{ ov::op::v9::NonMaxSuppression::get_type_info_static(), {{{0, 15}, {0, 1, 1000, 1, true}}, {{0, 8, 32}, {0, 1, 1000, 1, true}}} },
{ ov::op::v8::MatrixNms::get_type_info_static(), {{{0, 15}, {0, 1, 1000, 1, true}}, {{0, 8, 32}, {0, 1, 1000, 1, true}}} },
{ ov::op::v6::ExperimentalDetectronGenerateProposalsSingleImage::get_type_info_static(), {{{1, 0, 1, 1}}, {{1, 0, 1, 1}}}},
{ ov::op::v6::ExperimentalDetectronPriorGridGenerator::get_type_info_static(), {{{0, 0, 1, 1}}, {{-100, 200, 2, 1}, {0, 0, 1, 1}, {0, 0, 1, 1}}}},
{ ov::op::v5::RNNSequence::get_type_info_static(), {{{0, 15}, {0, 15}, {0, 10, 1, 1, true}}, {{0, 8, 32}, {0, 8, 32}, {0, 10, 1, 1, true}}} },
Expand All @@ -208,13 +210,23 @@ static std::map<ov::NodeTypeInfo, std::vector<std::vector<ov::test::utils::Input
{ ov::op::v9::ROIAlign::get_type_info_static(), {{{0, 15}, {0, 1000, 1, 1, true}, {0, 1000, 1, 1, true}},
{{-1000, 2000, 32768}, {0, 1000, 1, 1, true}, {0, 1000, 1, 1, true}}} },
{ ov::op::v0::Convert::get_type_info_static(), {{{0, 1000}}, {{-100, 200, 32768}}} },
{ ov::op::v0::FakeQuantize::get_type_info_static(), {{{0, 1000}}, {{0, 8, 32}}} },
};

ov::test::utils::InputGenerateData get_range_by_type(ov::element::Type temp_type, uint64_t kMaxRange);
class ModelRange {
std::vector<std::string> TYPE_ALIAS {"integral", "real"};

std::string get_range_id(const std::shared_ptr<ov::Node>& node, size_t port, bool spectial = false);
std::shared_ptr<ov::test::utils::InputGenerateData> general_real;
std::shared_ptr<ov::test::utils::InputGenerateData> general_integral;
// key for map calculated in get_range_id and contais [Op Type Name]_[integral/real]_[port]
std::map<std::string, std::shared_ptr<ov::test::utils::InputGenerateData>> node_ranges;
public:
void collect_ranges(const std::shared_ptr<ov::Model>& function, uint64_t kMaxRange);
void find_general_ranges();
std::string get_range_id(const std::shared_ptr<ov::Node>& node, size_t port);
ov::Tensor generate_input(std::shared_ptr<ov::Node> node, size_t port, const ov::Shape& targetShape);
};

std::map<std::string, std::shared_ptr<ov::test::utils::InputGenerateData>> collect_ranges(const std::shared_ptr<ov::Model>& function, uint64_t kMaxRange);

} // namespace utils
} // namespace test
Expand Down
23 changes: 8 additions & 15 deletions src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@
#include "functional_test_utils/crash_handler.hpp"

#include "shared_test_classes/base/ov_subgraph.hpp"
#include "shared_test_classes/base/utils/generate_inputs.hpp"
// #include "shared_test_classes/base/utils/generate_inputs.hpp"
#include "shared_test_classes/base/utils/compare_results.hpp"
#include "shared_test_classes/base/utils/calculate_thresholds.hpp"

#include "shared_test_classes/base/utils/ranges.hpp"


namespace ov {
namespace test {
Expand Down Expand Up @@ -320,28 +322,19 @@ void SubgraphBaseTest::compile_model() {

void SubgraphBaseTest::generate_inputs(const std::vector<ov::Shape>& targetInputStaticShapes) {
inputs.clear();
auto inputDataMap = ov::test::utils::collect_ranges(function, testing::internal::Random::kMaxRange);
auto inputMap = utils::getInputMap();
ov::test::utils::ModelRange modelRange;
modelRange.collect_ranges(function, testing::internal::Random::kMaxRange);
modelRange.find_general_ranges();

auto itTargetShape = targetInputStaticShapes.begin();
for (const auto &param : function->get_parameters()) {
std::shared_ptr<ov::Node> inputNode = param;
for (size_t i = 0; i < param->get_output_size(); i++) {
for (const auto &node : param->get_output_target_inputs(i)) {
std::shared_ptr<ov::Node> nodePtr = node.get_node()->shared_from_this();
auto it = inputMap.find(nodePtr->get_type_info());
ASSERT_NE(it, inputMap.end());
for (size_t port = 0; port < nodePtr->get_input_size(); ++port) {
if (nodePtr->get_input_node_ptr(port)->shared_from_this() == inputNode->shared_from_this()) {
if (!inputDataMap.empty()) {
std::string spetial_range_id = ov::test::utils::get_range_id(nodePtr, port, true);
if (inputDataMap.find(spetial_range_id) == inputDataMap.end()) {
spetial_range_id = ov::test::utils::get_range_id(nodePtr, port, false);
}
inputs.insert({param, it->second(nodePtr, port, param->get_element_type(), *itTargetShape,
inputDataMap[spetial_range_id])});
} else {
inputs.insert({param, it->second(nodePtr, port, param->get_element_type(), *itTargetShape, nullptr)});
}
inputs.insert({param, modelRange.generate_input(nodePtr, port, *itTargetShape)});
break;
}
}
Expand Down
55 changes: 0 additions & 55 deletions src/tests/functional/shared_test_classes/src/base/utils/generate_inputs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -748,20 +748,6 @@ ov::Tensor generate(const std::shared_ptr<ov::op::v6::ExperimentalDetectronTopKR
return generate(std::dynamic_pointer_cast<ov::Node>(node), port, elemType, targetShape, inGenRangeData);
}

ov::Tensor generate(const std::shared_ptr<ov::op::v5::RNNSequence>& node,
size_t port,
const ov::element::Type& elemType,
const ov::Shape& targetShape,
std::shared_ptr<InputGenerateData> inGenRangeData = nullptr) {
if (port == 2) {
ov::test::utils::InputGenerateData in_data;
in_data.start_from = 0;
in_data.range = 10; // max_seq_len
return ov::test::utils::create_and_fill_tensor(elemType, targetShape, in_data);
}
return generate(std::dynamic_pointer_cast<ov::Node>(node), port, elemType, targetShape, inGenRangeData);
}

ov::Tensor generate(const std::shared_ptr<ov::op::v8::Softmax>& node,
size_t port,
const ov::element::Type& elemType,
Expand Down Expand Up @@ -1149,14 +1135,6 @@ ov::Tensor generate(const std::shared_ptr<ov::op::v6::ExperimentalDetectronDetec
return tensor;
}

ov::Tensor generate(const std::shared_ptr<ov::op::v6::ExperimentalDetectronGenerateProposalsSingleImage>& node,
size_t port,
const ov::element::Type& elemType,
const ov::Shape& targetShape) {
InputGenerateData inGenData(1, 0, 1, 1);
return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData);
}

ov::Tensor generate(const std::shared_ptr<ov::op::v6::ExperimentalDetectronPriorGridGenerator>& node,
size_t port,
const ov::element::Type& elemType,
Expand All @@ -1171,39 +1149,6 @@ ov::Tensor generate(const std::shared_ptr<ov::op::v6::ExperimentalDetectronPrior
return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData);
}

ov::Tensor generate(const std::shared_ptr<ov::op::v8::MatrixNms>& node,
size_t port,
const ov::element::Type& elemType,
const ov::Shape& targetShape) {
if (port == 1) {
InputGenerateData inGenData(0, 1, 1000, 1);
return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData);
}
return generate(std::dynamic_pointer_cast<ov::Node>(node), port, elemType, targetShape);
}

ov::Tensor generate(const std::shared_ptr<ov::op::v5::NonMaxSuppression>& node,
size_t port,
const ov::element::Type& elemType,
const ov::Shape& targetShape) {
if (port == 1) {
InputGenerateData inGenData(0, 1, 1000, 1);
return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData);
}
return generate(std::dynamic_pointer_cast<ov::Node>(node), port, elemType, targetShape);
}

ov::Tensor generate(const std::shared_ptr<ov::op::v9::NonMaxSuppression>& node,
size_t port,
const ov::element::Type& elemType,
const ov::Shape& targetShape) {
if (port == 1) {
InputGenerateData inGenData(0, 1, 1000, 1);
return ov::test::utils::create_and_fill_tensor(elemType, targetShape, inGenData);
}
return generate(std::dynamic_pointer_cast<ov::Node>(node), port, elemType, targetShape);
}

template<typename T>
ov::Tensor generateInput(const std::shared_ptr<ov::Node>& node,
size_t port,
Expand Down
113 changes: 76 additions & 37 deletions src/tests/functional/shared_test_classes/src/base/utils/ranges.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

#include "shared_test_classes/base/utils/ranges.hpp"

#include "shared_test_classes/base/utils/generate_inputs.hpp"

namespace ov {
namespace test {
namespace utils {
Expand Down Expand Up @@ -106,21 +108,17 @@ ov::test::utils::InputGenerateData get_range_by_type(ov::element::Type temp_type
return inData;
}

std::string get_range_id(const std::shared_ptr<ov::Node>& node, size_t port, bool spectial) {
std::string range_id = std::to_string(node->input(port).get_element_type().is_real());
if (spectial) {
range_id += '_' + std::string(node->get_type_name()) + '_' + std::to_string(port);
}
std::string ModelRange::get_range_id(const std::shared_ptr<ov::Node>& node, size_t port) {
std::string range_id = std::string(node->get_type_name());
range_id += "_" + TYPE_ALIAS[node->input(port).get_element_type().is_real()];
range_id += "_" + std::to_string(port);
return range_id;
}

std::map<std::string, std::shared_ptr<ov::test::utils::InputGenerateData>> collect_ranges(const std::shared_ptr<ov::Model>& function, uint64_t kMaxRange) {
bool success = true;
std::map<std::string, std::shared_ptr<ov::test::utils::InputGenerateData>> inputDataMap;
inputDataMap["0"] = std::make_shared<ov::test::utils::InputGenerateData>(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined,
kMaxRange));
inputDataMap["1"] = std::make_shared<ov::test::utils::InputGenerateData>(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined,
kMaxRange));
void ModelRange::collect_ranges(const std::shared_ptr<ov::Model>& function, uint64_t kMaxRange) {
general_real = std::make_shared<ov::test::utils::InputGenerateData>(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined, kMaxRange));
general_integral = std::make_shared<ov::test::utils::InputGenerateData>(ov::test::utils::get_range_by_type(ov::element::Type_t::undefined, kMaxRange));

for (const auto& node : function->get_ordered_ops()) {
if (ov::op::util::is_output(node) ||
ov::op::util::is_constant(node) ||
Expand All @@ -144,47 +142,88 @@ std::map<std::string, std::shared_ptr<ov::test::utils::InputGenerateData>> colle
}

std::string range_id = get_range_id(node, port);
ov::test::utils::InputGenerateData temp_range;
ov::test::utils::InputGenerateData port_range;
if (ranges.empty()) {
temp_range = ov::test::utils::get_range_by_type(node->input(port).get_element_type(), kMaxRange);
if (temp_range.start_from == inputDataMap[range_id]->start_from && temp_range.range == inputDataMap[range_id]->range) {
continue;
}
port_range = ov::test::utils::get_range_by_type(node->input(port).get_element_type(), kMaxRange);
} else {
auto op_range = ranges.at(node->input(port).get_element_type().is_real());
temp_range = op_range.size() <= port ? op_range.front() : op_range.at(port);
if (temp_range.spetial) {
range_id = get_range_id(node, port, true);
inputDataMap[range_id] = std::make_shared<ov::test::utils::InputGenerateData>(temp_range);
continue;
}
port_range = op_range.size() <= port ? op_range.front() : op_range.at(port);
}
node_ranges[range_id] = std::make_shared<ov::test::utils::InputGenerateData>(port_range);
}
}
}

success = inputDataMap[range_id]->correct_range(node, port, temp_range);
if (!success)
break;
void ModelRange::find_general_ranges() {
bool find_integral = true;
bool find_real = true;

for (auto& node_range : node_ranges) {
if (node_range.second->spetial) {
continue;
}

if (!success)
break;
if (node_range.first.find(TYPE_ALIAS[0]) != std::string::npos) {
find_integral = general_integral->correct_range(node_range.second);
} else if (node_range.first.find(TYPE_ALIAS[1]) != std::string::npos) {
find_real = general_real->correct_range(node_range.second);
}
}

if (find_integral) {
#ifndef NDEBUG
if (success) {
for (auto& range : inputDataMap) {
std::cout << "RANGE " << range.first << " :\n" <<
" start from: " << std::to_string(range.second->start_from) <<
" range: " << std::to_string(range.second->range) <<
" resolution: " << std::to_string(range.second->resolution) <<
" seed: " << std::to_string(range.second->resolution) << std::endl;
}
std::cout << "INEGRAL RANGE FOUND \n" <<
" start from: " << std::to_string(general_integral->start_from) <<
" range: " << std::to_string(general_integral->range) <<
" resolution: " << std::to_string(general_integral->resolution) <<
" seed: " << std::to_string(general_integral->seed) << std::endl;
#endif
} else {
general_integral = nullptr;
#ifndef NDEBUG
std::cout << " RANGE NOT FOUND \n";
#endif
}

if (find_real) {
#ifndef NDEBUG
std::cout << "REAL RANGE FOUND \n" <<
" start from: " << std::to_string(general_real->start_from) <<
" range: " << std::to_string(general_real->range) <<
" resolution: " << std::to_string(general_real->resolution) <<
" seed: " << std::to_string(general_real->seed) << std::endl;
#endif
return success ? inputDataMap : std::map<std::string, std::shared_ptr<ov::test::utils::InputGenerateData>>{};
} else {
general_real = nullptr;
#ifndef NDEBUG
std::cout << " RANGE NOT FOUND \n";
#endif
}
}

ov::Tensor ModelRange::generate_input(std::shared_ptr<ov::Node> node, size_t port, const ov::Shape& targetShape) {
std::string range_id = get_range_id(node, port);
if (node_ranges.find(range_id) == node_ranges.end()) {
throw std::runtime_error("Node info not in rages: " + node->get_friendly_name());
}

auto inputMap = ov::test::utils::getInputMap();
auto it = inputMap.find(node->get_type_info());
if (it == inputMap.end()) {
throw std::runtime_error("Couln't find Operation in inputMap: " + std::string(node->get_type_name()));
}

std::shared_ptr<ov::test::utils::InputGenerateData> range = node_ranges[range_id];
if (!node_ranges[range_id]->spetial) {
if (node->get_input_element_type(port).is_real() && general_real) {
range = general_real;
} else {
range = general_integral;
}
}

return it->second(node, port, node->get_input_element_type(port), targetShape, range);
}

} // namespace utils
} // namespace test
Expand Down
41 changes: 33 additions & 8 deletions src/tests/test_utils/common_test_utils/include/common_test_utils/data_utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,15 +229,40 @@ void inline fill_data_random(T* pointer,
return;
}

testing::internal::Random random(seed);
const uint32_t k_range = k * range; // range with respect to k
random.Generate(k_range);
if (k != 1) {
testing::internal::Random random(seed);
const uint32_t k_range = k * range; // range with respect to k
random.Generate(k_range);

if (start_from < 0 && !std::numeric_limits<T>::is_signed) {
start_from = 0;
}
for (std::size_t i = 0; i < size; i++) {
pointer[i] = static_cast<T>(start_from + (random.Generate(k_range) / k));
if (start_from < 0 && !std::numeric_limits<T>::is_signed) {
start_from = 0;
}
for (std::size_t i = 0; i < size; i++) {
pointer[i] = static_cast<T>(start_from + (random.Generate(k_range) / k));
}
} else {
if (std::numeric_limits<T>::is_integer) {
std::mt19937 gen(seed);
std::uniform_int_distribution<int64_t> dist(0, range);

for (std::size_t i = 0; i < size; i++) {
pointer[i] = static_cast<T>(start_from + dist(gen));
}
} else {
std::mt19937 gen(seed);
std::uniform_real_distribution<double> dist(0, range);

for (std::size_t i = 0; i < size; i++) {
auto value = dist(gen);
if (std::is_same<ov::float16, T>::value) {
pointer[i] = static_cast<T>(start_from + ov::float16(value));
} else if (std::is_same<ov::bfloat16, T>::value) {
pointer[i] = static_cast<T>(start_from + ov::bfloat16(value));
} else {
pointer[i] = static_cast<T>(start_from + value);
}
}
}
}
}

Expand Down
Loading

0 comments on commit c355d57

Please sign in to comment.