diff --git a/src/common/transformations/include/transformations/utils/utils.hpp b/src/common/transformations/include/transformations/utils/utils.hpp index 9897663b247cf1..c24a1e46c962e8 100644 --- a/src/common/transformations/include/transformations/utils/utils.hpp +++ b/src/common/transformations/include/transformations/utils/utils.hpp @@ -22,7 +22,7 @@ namespace op { namespace util { template -bool normalize_single_value(std::vector vec, float& value) { +bool normalize_single_value(std::vector vec, float& value, bool check_value_range = true) { for (const auto& val : vec) { if (val != *vec.begin()) return false; @@ -30,7 +30,8 @@ bool normalize_single_value(std::vector vec, float& value) { float ref_val = static_cast(*vec.begin()); - if (ref_val < std::numeric_limits::lowest() || ref_val > std::numeric_limits::max()) { + if (check_value_range && + (ref_val < std::numeric_limits::lowest() || ref_val > std::numeric_limits::max())) { return false; } @@ -159,7 +160,9 @@ bool has_constant_value(const std::shared_ptr& node, return const_values == values; } -TRANSFORMATIONS_API bool get_single_value(const std::shared_ptr& const_node, float& value); +TRANSFORMATIONS_API bool get_single_value(const std::shared_ptr& const_node, + float& value, + bool check_value_range = true); TRANSFORMATIONS_API std::shared_ptr normalize_constant(const std::shared_ptr& constant, const PartialShape& shape); diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp index e59440cf17a68a..d7609c935f1d61 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp @@ -240,12 +240,14 @@ TSGatherBackward::TSGatherBackward() { if (success) { size_t j = 0; for (size_t i = 0; i < shape.size(); ++i) { - if (shape[i] != new_shape[j] && shape[i] == 1) { - axes_val.push_back(i); - continue; - } else if (shape[i] != new_shape[j]) { - success = false; - break; + if (j >= new_shape.size() || shape[i] != new_shape[j]) { + if (shape[i] == 1) { + axes_val.push_back(i); + continue; + } else { + success = false; + break; + } } j++; } diff --git a/src/common/transformations/src/transformations/utils/utils.cpp b/src/common/transformations/src/transformations/utils/utils.cpp index c2754fb0ca5e58..c06ff0fefd394b 100644 --- a/src/common/transformations/src/transformations/utils/utils.cpp +++ b/src/common/transformations/src/transformations/utils/utils.cpp @@ -21,32 +21,32 @@ namespace ov { namespace op { namespace util { -bool get_single_value(const std::shared_ptr& const_node, float& value) { +bool get_single_value(const std::shared_ptr& const_node, float& value, bool check_value_range) { switch (const_node->get_element_type()) { case element::Type_t::f16: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::f32: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::bf16: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::f64: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::i8: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::i16: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::i32: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::i64: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::u8: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::u16: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::u32: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); case element::Type_t::u64: - return util::normalize_single_value(const_node->get_vector(), value); + return util::normalize_single_value(const_node->get_vector(), value, check_value_range); default: OPENVINO_THROW("Unsupported precision for const operation: ", const_node->get_friendly_name()); } diff --git a/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp index 003c61c373cfc5..9c26fb6ad646c9 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp @@ -125,8 +125,9 @@ INSTANTIATE_TEST_SUITE_P(TSCommonGatherForward_3, TSTestFixture, test_forward_ga struct GatherBackwardArguments { OutputVector inputs_to_main; - Output new_Gather_first_input; - AxisVector new_transpose_order; + Output ref_Gather_axis_input; + AxisVector ref_transpose_order; + AxisVector ref_unsqueeze_axes; }; auto test_backward_gather = [](const GatherBackwardArguments& test_arguments) { @@ -147,14 +148,14 @@ auto test_backward_gather = [](const GatherBackwardArguments& test_arguments) { OutputVector new_out_vec(out_vec.size()); new_out_vec[0] = out_vec[0]; new_out_vec[1] = out_vec[1]; - new_out_vec[2] = test_arguments.new_Gather_first_input; + new_out_vec[2] = test_arguments.ref_Gather_axis_input; return new_out_vec; }; auto new_transpose = [&test_arguments](const vector& idxs, const OutputVector& out_vec) -> OutputVector { OutputVector new_out_vec = out_vec; auto order = make_shared(i32, - Shape{test_arguments.new_transpose_order.size()}, - test_arguments.new_transpose_order); + Shape{test_arguments.ref_transpose_order.size()}, + test_arguments.ref_transpose_order); new_out_vec[0] = make_shared(out_vec[0], order); return new_out_vec; }; @@ -197,13 +198,14 @@ auto test_backward_gather_optimization = [](const GatherBackwardArguments& test_ OutputVector new_out_vec(out_vec.size()); new_out_vec[0] = out_vec[0]; new_out_vec[1] = make_shared(out_vec[1]); - new_out_vec[2] = test_arguments.new_Gather_first_input; + new_out_vec[2] = test_arguments.ref_Gather_axis_input; return new_out_vec; }; auto unsqueeze_for = [&](const vector& idxs, const OutputVector& out_vec) -> OutputVector { - auto axis = constant(i32, {1}, {0}); - return {make_shared(out_vec[0], axis)}; + const auto& axes_val = test_arguments.ref_unsqueeze_axes; + auto axes = constant(i32, {axes_val.size()}, axes_val); + return {make_shared(out_vec[0], axes)}; }; test_case.model_ref.preprocess_inputs_to_main = {{set_transpose_for, update_gather_inputs}, {{0}, {1, 2}}}; @@ -215,13 +217,29 @@ auto test_backward_gather_optimization = [](const GatherBackwardArguments& test_ }; vector tests_arguments_bw_optimization{ - {{{parameter(f32, {257, 8}), constant(i32, {1, 2}, {0}), constant(i32, {1}, {0})}, - constant(i32, {1}, {1}), - AxisVector{}}}}; + {{parameter(f32, {257, 8}), constant(i32, {1, 2}, {0}), constant(i32, {1}, {0})}, + constant(i32, {1}, {1}), + AxisVector{}, + AxisVector{0}}, + {{parameter(f32, {4}), constant(i32, {1}, {0}), constant(i32, {1}, {0})}, + constant(i32, {1}, {0}), + AxisVector{}, + AxisVector{0}}, + {{parameter(f32, {4}), constant(i32, {1, 1, 1}, {0}), constant(i32, {1}, {0})}, + constant(i32, {1}, {0}), + AxisVector{}, + AxisVector{0, 1, 2}}, +}; INSTANTIATE_TEST_SUITE_P(TSCommonGatherBackwardOptimization_0, TSTestFixture, test_backward_gather_optimization(tests_arguments_bw_optimization[0])); +INSTANTIATE_TEST_SUITE_P(TSCommonGatherBackwardOptimization_1, + TSTestFixture, + test_backward_gather_optimization(tests_arguments_bw_optimization[1])); +INSTANTIATE_TEST_SUITE_P(TSCommonGatherBackwardOptimization_2, + TSTestFixture, + test_backward_gather_optimization(tests_arguments_bw_optimization[2])); } // namespace gather } // namespace testing } // namespace transpose_sinking diff --git a/src/frontends/ir/src/ir_deserializer.cpp b/src/frontends/ir/src/ir_deserializer.cpp index 22443a779cc179..56aa587ee44adf 100644 --- a/src/frontends/ir/src/ir_deserializer.cpp +++ b/src/frontends/ir/src/ir_deserializer.cpp @@ -614,18 +614,15 @@ class MetaDataParser : public ov::Meta { } void parse() const { - // Thread safety is implemented on ov::Model level - if (m_parsed) - return; - const pugi::xml_node& node = m_meta.child(m_name.c_str()); - m_parsed_map = parse_node(node); - - m_parsed = true; + std::call_once(m_oc, [this]() { + const pugi::xml_node& node = m_meta.child(m_name.c_str()); + m_parsed_map = parse_node(node); + }); } pugi::xml_document m_meta; const std::string m_name; mutable ov::AnyMap m_parsed_map; - mutable bool m_parsed{false}; + mutable std::once_flag m_oc; }; void XmlDeserializer::read_meta_data(const std::shared_ptr& model, const pugi::xml_node& meta_section) { diff --git a/src/frontends/tensorflow/src/frontend.cpp b/src/frontends/tensorflow/src/frontend.cpp index d5c3c4e2bcbf59..09a3b3fbb30359 100644 --- a/src/frontends/tensorflow/src/frontend.cpp +++ b/src/frontends/tensorflow/src/frontend.cpp @@ -189,6 +189,8 @@ bool FrontEnd::supported_impl(const std::vector& variants) const { ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { // Last boolean flag in `variants` (if presented) is reserved for FE configuration size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; + // Enable mmap by default + bool mmap_enabled = variants[variants.size() - 1].is() ? variants[variants.size() - 1].as() : true; // For TF1 models it can be a case of two input variants: input model and v1 checkpoints FRONT_END_GENERAL_CHECK( @@ -203,7 +205,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va return std::make_shared(std::make_shared(model_path), m_telemetry); } else if (GraphIteratorSavedModel::is_supported(model_path)) { std::shared_ptr graph_iterator; - graph_iterator = std::make_shared(model_path, std::string("serve")); + graph_iterator = std::make_shared(model_path, std::string("serve"), mmap_enabled); return std::make_shared(graph_iterator, m_telemetry, graph_iterator->get_variables_index(), @@ -212,7 +214,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va nullptr, true); } else if (GraphIteratorMeta::is_supported(model_path)) { - auto graph_iterator = std::make_shared(model_path); + auto graph_iterator = std::make_shared(model_path, mmap_enabled); return std::make_shared(graph_iterator, m_telemetry, graph_iterator->get_variables_index(), @@ -257,7 +259,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va auto saved_model_tags = paths[1]; if (GraphIteratorSavedModel::is_supported(model_path)) { std::shared_ptr graph_iterator; - graph_iterator = std::make_shared(model_path, saved_model_tags); + graph_iterator = std::make_shared(model_path, saved_model_tags, mmap_enabled); return std::make_shared(graph_iterator, m_telemetry, graph_iterator->get_variables_index(), @@ -275,7 +277,9 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va return std::make_shared(std::make_shared(model_path), m_telemetry); } else if (GraphIteratorSavedModel::is_supported(model_path)) { std::shared_ptr graph_iterator; - graph_iterator = std::make_shared(model_path, std::string(META_GRAPH_DEFAULT_TAG)); + graph_iterator = std::make_shared(model_path, + std::string(META_GRAPH_DEFAULT_TAG), + mmap_enabled); return std::make_shared(graph_iterator, m_telemetry, graph_iterator->get_variables_index(), @@ -284,7 +288,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va nullptr, true); } else if (GraphIteratorMeta::is_supported(model_path)) { - auto graph_iterator = std::make_shared(model_path); + auto graph_iterator = std::make_shared(model_path, mmap_enabled); return std::make_shared(graph_iterator, m_telemetry, graph_iterator->get_variables_index(), @@ -329,7 +333,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va auto saved_model_tags = ov::util::wstring_to_string(paths[1]); if (GraphIteratorSavedModel::is_supported(model_path)) { std::shared_ptr graph_iterator; - graph_iterator = std::make_shared(model_path, saved_model_tags); + graph_iterator = std::make_shared(model_path, saved_model_tags, mmap_enabled); return std::make_shared(graph_iterator, m_telemetry, graph_iterator->get_variables_index(), diff --git a/src/frontends/tensorflow/src/graph_iterator_meta.hpp b/src/frontends/tensorflow/src/graph_iterator_meta.hpp index 0d1689ecf6befa..1e2789227260fb 100644 --- a/src/frontends/tensorflow/src/graph_iterator_meta.hpp +++ b/src/frontends/tensorflow/src/graph_iterator_meta.hpp @@ -31,11 +31,13 @@ class GraphIteratorMeta : public GraphIteratorProto { std::shared_ptr m_variables_index; std::shared_ptr> m_inputs_map; std::shared_ptr> m_outputs_map; + bool m_mmap_enabled; public: template - GraphIteratorMeta(const std::basic_string& path) - : m_metagraph_def(std::make_shared<::tensorflow::MetaGraphDef>()) { + GraphIteratorMeta(const std::basic_string& path, const bool mmap_enabled) + : m_metagraph_def(std::make_shared<::tensorflow::MetaGraphDef>()), + m_mmap_enabled(mmap_enabled) { this->read_meta(path); } @@ -75,7 +77,7 @@ class GraphIteratorMeta : public GraphIteratorProto { std::basic_string varIndexPath = get_variables_index_name(model_path); if (ov::util::file_exists(varIndexPath)) { - m_variables_index = std::make_shared(); + m_variables_index = std::make_shared(m_mmap_enabled); std::ifstream vi_stream{varIndexPath.c_str(), std::ifstream::in | std::ifstream::binary}; FRONT_END_GENERAL_CHECK(vi_stream && vi_stream.is_open(), "MetaGraph's variable index file does not exist"); FRONT_END_GENERAL_CHECK(m_variables_index->read_variables(vi_stream, model_path, false), diff --git a/src/frontends/tensorflow/src/graph_iterator_saved_model.hpp b/src/frontends/tensorflow/src/graph_iterator_saved_model.hpp index 377e17c947e79c..511f2a0a5bc307 100644 --- a/src/frontends/tensorflow/src/graph_iterator_saved_model.hpp +++ b/src/frontends/tensorflow/src/graph_iterator_saved_model.hpp @@ -38,11 +38,13 @@ class GraphIteratorSavedModel : public GraphIteratorProto { std::shared_ptr m_variables_index; std::shared_ptr> m_inputs_map; std::shared_ptr> m_outputs_map; + bool m_mmap_enabled; public: template - GraphIteratorSavedModel(const std::basic_string& path, const std::string& tags) - : m_saved_model(std::make_shared<::tensorflow::SavedModel>()) { + GraphIteratorSavedModel(const std::basic_string& path, const std::string& tags, const bool mmap_enabled) + : m_saved_model(std::make_shared<::tensorflow::SavedModel>()), + m_mmap_enabled(mmap_enabled) { this->read_saved_model(path, tags); } @@ -74,7 +76,7 @@ class GraphIteratorSavedModel : public GraphIteratorProto { std::basic_string varIndexPath = path + get_variables_index_name(); if (ov::util::file_exists(varIndexPath)) { - m_variables_index = std::make_shared(); + m_variables_index = std::make_shared(m_mmap_enabled); std::ifstream vi_stream{varIndexPath.c_str(), std::ifstream::in | std::ifstream::binary}; FRONT_END_GENERAL_CHECK(vi_stream && vi_stream.is_open(), "[TensorFlow Frontend] Saved Model's variable index file does not exist"); diff --git a/src/frontends/tensorflow/src/op/var_handle.cpp b/src/frontends/tensorflow/src/op/var_handle.cpp index 9f6819ccba40c2..50a5b73c449f8f 100644 --- a/src/frontends/tensorflow/src/op/var_handle.cpp +++ b/src/frontends/tensorflow/src/op/var_handle.cpp @@ -7,7 +7,9 @@ #include "helper_ops/string_constant.hpp" #include "helper_ops/unsupported_constant.hpp" #include "input_model.hpp" +#include "ngraph/runtime/shared_buffer.hpp" #include "openvino/opsets/opset8.hpp" +#include "openvino/util/mmap_object.hpp" #include "tensor_bundle.pb.h" using namespace std; @@ -26,22 +28,42 @@ static std::shared_ptr read_variable(std::shared_ptr v const ov::Shape shape, const ::tensorflow::BundleEntryProto& entry, const NodeContext& node) { - std::vector var_data; google::protobuf::int64 size = 1; for (uint64_t i = 0; i < shape.size(); ++i) { size *= static_cast(shape[i]); } - var_data.resize(size); TENSORFLOW_OP_VALIDATION(node, size == static_cast(entry.size() / sizeof(T)), "[TensorFlow Frontend] Internal error: Available data size isn't equal to calculated."); - auto fs = var_index->get_data_file(entry.shard_id()); - if (!fs.get()) { - TENSORFLOW_OP_VALIDATION(node, var_index, "[TensorFlow Frontend] Internal error: Cannot get shard file."); + if (var_index->is_mmap_enabled()) { + auto mapped_memory = var_index->get_data_mmap(entry.shard_id()); + if (!mapped_memory.get()) { + TENSORFLOW_OP_VALIDATION(node, var_index, "[TensorFlow Frontend] Internal error: Cannot get shard file."); + } + TENSORFLOW_OP_VALIDATION( + node, + static_cast(mapped_memory->size()) >= entry.offset() + entry.size(), + "[TensorFlow Frontend] Internal error: Variable entry size is out of bounds of mapped memory size."); + OPENVINO_SUPPRESS_DEPRECATED_START + return std::make_shared( + ov_type, + shape, + std::make_shared>>( + mapped_memory->data() + entry.offset(), + entry.size(), + mapped_memory)); + OPENVINO_SUPPRESS_DEPRECATED_END + } else { + std::vector var_data; + var_data.resize(size); + auto fs = var_index->get_data_file(entry.shard_id()); + if (!fs.get()) { + TENSORFLOW_OP_VALIDATION(node, var_index, "[TensorFlow Frontend] Internal error: Cannot get shard file."); + } + fs->seekg(entry.offset(), std::ios::beg); + fs->read(reinterpret_cast(var_data.data()), entry.size()); + return std::make_shared(ov_type, shape, var_data); } - fs->seekg(entry.offset(), std::ios::beg); - fs->read(reinterpret_cast(var_data.data()), entry.size()); - return std::make_shared(ov_type, shape, var_data); } OutputVector translate_varhandle_op(const NodeContext& node) { diff --git a/src/frontends/tensorflow/src/variables_index.cpp b/src/frontends/tensorflow/src/variables_index.cpp index 7534a6dcbd07c3..c24ffd8112bd09 100644 --- a/src/frontends/tensorflow/src/variables_index.cpp +++ b/src/frontends/tensorflow/src/variables_index.cpp @@ -10,6 +10,7 @@ #include "checkpoint_utils.hpp" #include "graph_iterator_saved_model.hpp" #include "openvino/core/type/element_type.hpp" +#include "openvino/util/mmap_object.hpp" #include "tensor_bundle.pb.h" #include "trackable_object_graph.pb.h" @@ -160,8 +161,13 @@ void VariablesIndex::read_checkpointable_object_graph() { // It looks like reinterpret_cast artifact // https://github.com/tensorflow/tensorflow/blob/d90f1947ebcf510b23c238f43c2191e5b3817cb3/tensorflow/cc/experimental/libexport/load.cc#L70 int chg = 6; - shard->second->seekg(entry.offset() + chg); - shard->second->read(data.data(), entry.size() - chg); + if (m_mmap_enabled) { + auto srcPtr = static_cast(shard->second.mmap->data() + entry.offset() + chg); + std::copy(srcPtr, srcPtr + entry.size() - chg, data.data()); + } else { + shard->second.stream->seekg(entry.offset() + chg); + shard->second.stream->read(data.data(), entry.size() - chg); + } // Might be need to remove this verification: // https://github.com/tensorflow/tensorflow/blob/d90f1947ebcf510b23c238f43c2191e5b3817cb3/tensorflow/cc/experimental/libexport/load.cc#L73 @@ -191,9 +197,14 @@ bool VariablesIndex::read_variables(std::ifstream& vi_stream, const std::string& } else { fullPath = path + "." + suffix.data(); } - m_data_files[shard] = std::shared_ptr( - new std::ifstream(fullPath.c_str(), std::ifstream::in | std::ifstream::binary)); - FRONT_END_GENERAL_CHECK(m_data_files[shard]->is_open(), "Variable index data file does not exist"); + if (m_mmap_enabled) { + m_data_files[shard].mmap = load_mmap_object(fullPath); + FRONT_END_GENERAL_CHECK(m_data_files[shard].mmap->data(), "Variable index data cannot be mapped"); + } else { + m_data_files[shard].stream = std::shared_ptr( + new std::ifstream(fullPath.c_str(), std::ifstream::in | std::ifstream::binary)); + FRONT_END_GENERAL_CHECK(m_data_files[shard].stream->is_open(), "Variable index data file does not exist"); + } } read_checkpointable_object_graph(); @@ -215,9 +226,14 @@ bool VariablesIndex::read_variables(std::ifstream& vi_stream, const std::wstring } else { fullPath = path + L"." + suffix.data(); } - m_data_files[shard] = std::shared_ptr( - new std::ifstream(fullPath.c_str(), std::ifstream::in | std::ifstream::binary)); - FRONT_END_GENERAL_CHECK(m_data_files[shard]->is_open(), "Variable index data file does not exist"); + if (m_mmap_enabled) { + m_data_files[shard].mmap = load_mmap_object(fullPath); + FRONT_END_GENERAL_CHECK(m_data_files[shard].mmap->data(), L"Variable index data cannot be mapped"); + } else { + m_data_files[shard].stream = std::shared_ptr( + new std::ifstream(fullPath.c_str(), std::ifstream::in | std::ifstream::binary)); + FRONT_END_GENERAL_CHECK(m_data_files[shard].stream->is_open(), L"Variable index data file does not exist"); + } } read_checkpointable_object_graph(); diff --git a/src/frontends/tensorflow/src/variables_index.hpp b/src/frontends/tensorflow/src/variables_index.hpp index 0c2f5dfc899c92..df852a627994e7 100644 --- a/src/frontends/tensorflow/src/variables_index.hpp +++ b/src/frontends/tensorflow/src/variables_index.hpp @@ -8,6 +8,7 @@ #include "graph_iterator_proto.hpp" #include "openvino/util/file_util.hpp" +#include "openvino/util/mmap_object.hpp" #include "saved_model.pb.h" namespace ov { @@ -16,6 +17,11 @@ namespace tensorflow { struct VIBlock; +struct VariableStorage { + std::shared_ptr stream; + std::shared_ptr mmap; +}; + // Stores information about variables index class VariablesIndex { // Contains file size for internal checks @@ -25,11 +31,19 @@ class VariablesIndex { // Contains BundleEntryProto variables list, readed from .index file std::map> m_variables_index; // List of opened data files for using with BundleEntryProto - std::map> m_data_files; + std::map m_data_files; // List of mapped variables which could be read using TrackableObjectGraph std::map m_variables_map; + // Flag shows which file storage is using + bool m_mmap_enabled; public: + VariablesIndex(bool mmap_enabled = false) : m_mmap_enabled(mmap_enabled) {} + /// \brief Returns mmap_enabled state. + /// \returns True if mmap is enabled, false otherwise + bool is_mmap_enabled(void) const { + return m_mmap_enabled; + } /// \brief Reads variables from opened variable index file. Can cause an asserts in case of issues. /// \param vi_stream Opened stream file, file pointer doesn't matter, it will be rewind internally. /// \param path A path to file with variables data @@ -89,8 +103,20 @@ class VariablesIndex { /// \param shard_id Requested shard_id /// \returns Valid shared_ptr with ifstream or with nullptr if shard isn't found std::shared_ptr get_data_file(const int32_t shard_id) const { + FRONT_END_GENERAL_CHECK(m_mmap_enabled == false, + "[TensorFlow Frontend] Requested ifstream, but mmap is enabled"); + auto result = m_data_files.find(shard_id); + return result != m_data_files.end() ? result->second.stream : nullptr; + } + + /// \brief Returns shared pointer to a requested shard_id, or nullptr in case of shard_id isn't found + /// \param shard_id Requested shard_id + /// \returns Valid shared_ptr with MappedMemory or with nullptr if shard isn't found + std::shared_ptr get_data_mmap(const int32_t shard_id) const { + FRONT_END_GENERAL_CHECK(m_mmap_enabled == true, + "[TensorFlow Frontend] Requested MappedMemory, but mmap is disabled"); auto result = m_data_files.find(shard_id); - return result != m_data_files.end() ? result->second : nullptr; + return result != m_data_files.end() ? result->second.mmap : nullptr; } /// \brief Adds variable mapping to the variables map diff --git a/src/frontends/tensorflow/tests/convert_saved_model.cpp b/src/frontends/tensorflow/tests/convert_saved_model.cpp index 19979229d14dae..76a5caf1fe54b1 100644 --- a/src/frontends/tensorflow/tests/convert_saved_model.cpp +++ b/src/frontends/tensorflow/tests/convert_saved_model.cpp @@ -158,6 +158,11 @@ TEST_F(FrontEndConversionWithReferenceTestsF, SavedModelWithIntermediateOutput) } } +TEST_F(FrontEndConversionWithReferenceTestsF, SavedModelMMAPCompare) { + { model = convert_model("saved_model_variables"); } + { model_ref = convert_model("saved_model_variables", nullptr, {}, {}, {}, {}, {}, true); } +} + TEST_F(FrontEndConversionWithReferenceTestsF, SavedModelWithNumericalNames) { comparator.enable(FunctionsComparator::CmpValues::TENSOR_NAMES); // The test aims to check that model with only numerical names for operation diff --git a/src/frontends/tensorflow/tests/convert_tricky_models.cpp b/src/frontends/tensorflow/tests/convert_tricky_models.cpp index 4c72557554348c..ec493a6876b89d 100644 --- a/src/frontends/tensorflow/tests/convert_tricky_models.cpp +++ b/src/frontends/tensorflow/tests/convert_tricky_models.cpp @@ -550,6 +550,11 @@ TEST_F(FrontEndConversionWithReferenceTestsF, MetaGraphCutIdentity) { } } +TEST_F(FrontEndConversionWithReferenceTestsF, MetaGraphMMAPCompare) { + { model = convert_model("metagraph_variables/graph.meta"); } + { model_ref = convert_model("metagraph_variables/graph.meta", nullptr, {}, {}, {}, {}, {}, true); } +} + TEST_F(FrontEndConversionWithReferenceTestsF, SplitInFunction) { { // create FAKE conversion extension for Split using named ports, this is not required for Split, but it tests diff --git a/src/frontends/tensorflow/tests/tf_utils.cpp b/src/frontends/tensorflow/tests/tf_utils.cpp index 09c4fdf18bde04..120b8ffab8659d 100644 --- a/src/frontends/tensorflow/tests/tf_utils.cpp +++ b/src/frontends/tensorflow/tests/tf_utils.cpp @@ -22,7 +22,8 @@ shared_ptr convert_model(const string& model_path, const vector& input_types, const vector& input_shapes, const std::vector& input_names_to_freeze, - const std::vector& freeze_values) { + const std::vector& freeze_values, + const bool disable_mmap) { FrontEndManager fem; auto front_end = fem.load_by_framework(TF_FE); if (!front_end) { @@ -32,7 +33,13 @@ shared_ptr convert_model(const string& model_path, front_end->add_extension(conv_ext); } auto model_filename = FrontEndTestUtils::make_model_path(string(TEST_TENSORFLOW_MODELS_DIRNAME) + model_path); - auto input_model = front_end->load(model_filename); + ov::frontend::InputModel::Ptr input_model; + if (!disable_mmap) { + input_model = front_end->load(model_filename); + } else { + input_model = front_end->load({model_filename, false}); + } + if (!input_model) { throw "Input model is not read"; } diff --git a/src/frontends/tensorflow/tests/tf_utils.hpp b/src/frontends/tensorflow/tests/tf_utils.hpp index d039a9f9edb007..1c48a95c85fee8 100644 --- a/src/frontends/tensorflow/tests/tf_utils.hpp +++ b/src/frontends/tensorflow/tests/tf_utils.hpp @@ -24,7 +24,8 @@ std::shared_ptr convert_model(const std::string& model_path, const std::vector& input_types = {}, const std::vector& input_shapes = {}, const std::vector& input_names_to_freeze = {}, - const std::vector& freeze_values = {}); + const std::vector& freeze_values = {}, + const bool disable_mmap = false); } // namespace tests } // namespace tensorflow diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp new file mode 100644 index 00000000000000..512d73518d026e --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp @@ -0,0 +1,17 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +namespace { +const Params paramsStreams[] = { + std::tuple{ ov::test::utils::DEVICE_CPU, {{ov::num_streams(ov::streams::AUTO)}}}, +}; +} // namespace + +INSTANTIATE_TEST_SUITE_P(CPU_Streams, CoreThreadingTestsWithCacheEnabled, + testing::Combine(testing::ValuesIn(paramsStreams), + testing::Values(20), + testing::Values(10)), + CoreThreadingTestsWithCacheEnabled::getTestCaseName); diff --git a/src/plugins/intel_gpu/src/plugin/ops/pad.cpp b/src/plugins/intel_gpu/src/plugin/ops/pad.cpp index 02481c339d145b..1dc5a2abd56d8d 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/pad.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/pad.cpp @@ -44,7 +44,8 @@ static void CreatePadOp(Program& p, const std::shared_ptr& if (op->get_pad_mode() == ov::op::PadMode::CONSTANT && op->get_input_size() == 4) { auto const_node = std::dynamic_pointer_cast(op->get_input_node_shared_ptr(3)); if (const_node) { - OPENVINO_ASSERT(ov::op::util::get_single_value(const_node, pad_value), + const bool check_value_range = false; // Allows the usage of infinity value as pad_value + OPENVINO_ASSERT(ov::op::util::get_single_value(const_node, pad_value, check_value_range), "Invalid parameter size in ", op->get_friendly_name(), " (", op->get_type_name(), ")"); is_value_const = true; } diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_threading.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_threading.hpp new file mode 100644 index 00000000000000..ca4f157c2f5cd7 --- /dev/null +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_threading.hpp @@ -0,0 +1,295 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "openvino/runtime/properties.hpp" +#include +#include +#include +#include +#include +#include +#include "base/ov_behavior_test_utils.hpp" + +using Device = std::string; +using Config = ov::AnyMap; +using Params = std::tuple; + +class CoreThreadingTestsBase { +public: + static void runParallel(std::function func, + const unsigned int iterations = 100, + const unsigned int threadsNum = 8) { + std::vector threads(threadsNum); + + for (auto & thread : threads) { + thread = std::thread([&](){ + for (unsigned int i = 0; i < iterations; ++i) { + func(); + } + }); + } + + for (auto & thread : threads) { + if (thread.joinable()) + thread.join(); + } + } + + void safePluginUnregister(ov::Core & core, const std::string& deviceName) { + try { + core.unload_plugin(deviceName); + } catch (const ov::Exception & ex) { + // if several threads unload plugin at once, the first thread does this + // while all others will throw an exception that plugin is not registered + ASSERT_STR_CONTAINS(ex.what(), "name is not registered in the"); + } + } + + Config config; +}; + +// +// Parameterized tests with number of parallel threads, iterations +// + +using Threads = unsigned int; +using Iterations = unsigned int; + +using CoreThreadingParams = std::tuple; + +class CoreThreadingTestsWithCacheEnabled : public testing::WithParamInterface, + public ov::test::behavior::OVPluginTestBase, + public CoreThreadingTestsBase { +public: + void SetUp() override { + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + std::tie(target_device, config) = std::get<0>(GetParam()); + numThreads = std::get<1>(GetParam()); + numIterations = std::get<2>(GetParam()); + auto hash = std::hash()(::testing::UnitTest::GetInstance()->current_test_info()->name()); + std::stringstream ss; + ss << std::this_thread::get_id(); + cache_path = "threading_test" + std::to_string(hash) + "_" + + ss.str() + "_" + GetTimestamp() + "_cache"; + } + + void TearDown() override { + ov::test::utils::removeFilesWithExt(cache_path, "blob"); + std::remove(cache_path.c_str()); + } + + static std::string getTestCaseName(testing::TestParamInfo obj) { + unsigned int numThreads, numIterations; + std::string deviceName; + Config config; + std::tie(deviceName, config) = std::get<0>(obj.param); + std::replace(deviceName.begin(), deviceName.end(), ':', '.'); + numThreads = std::get<1>(obj.param); + numIterations = std::get<2>(obj.param); + char separator('_'); + std::ostringstream result; + result << "targetDevice=" << deviceName << separator; + result << "config="; + for (auto& confItem : config) { + result << confItem.first << "=" << confItem.second.as() << separator; + } + result << "numThreads=" << numThreads << separator; + result << "numIter=" << numIterations; + return result.str(); + } + + +protected: + unsigned int numIterations; + unsigned int numThreads; + std::string cache_path; + std::shared_ptr model; + void SetupModel() { + ov::Core core; + std::string ir_with_meta = R"V0G0N( + + + + + + + 1 + 3 + 22 + 22 + + + + + + + 1 + 3 + 22 + 22 + + + + + 1 + 3 + 22 + 22 + + + + + + + 1 + 3 + 22 + 22 + + + + + + + + + + + + + + + + + + + + + + + { + 'compression': { + 'algorithms': [ + { + 'name': 'DefaultQuantization', + 'params': { + 'num_samples_for_tuning': 2000, + 'preset': 'performance', + 'stat_subset_size': 300, + 'use_layerwise_tuning': false + } + } + ], + 'dump_intermediate_model': true, + 'target_device': 'ANY' + }, + 'engine': { + 'models': [ + { + 'name': 'bert-small-uncased-whole-word-masking-squad-0001', + 'launchers': [ + { + 'framework': 'openvino', + 'adapter': { + 'type': 'bert_question_answering', + 'start_token_logits_output': 'output_s', + 'end_token_logits_output': 'output_e' + }, + 'inputs': [ + { + 'name': 'input_ids', + 'type': 'INPUT', + 'value': 'input_ids' + }, + { + 'name': 'attention_mask', + 'type': 'INPUT', + 'value': 'input_mask' + }, + { + 'name': 'token_type_ids', + 'type': 'INPUT', + 'value': 'segment_ids' + } + ], + 'device': 'cpu' + } + ], + 'datasets': [ + { + 'name': 'squad_v1_1_msl384_mql64_ds128_lowercase', + 'annotation_conversion': { + 'converter': 'squad', + 'testing_file': 'PATH', + 'max_seq_length': 384, + 'max_query_length': 64, + 'doc_stride': 128, + 'lower_case': true, + 'vocab_file': 'PATH' + }, + 'reader': { + 'type': 'annotation_features_extractor', + 'features': [ + 'input_ids', + 'input_mask', + 'segment_ids' + ] + }, + 'postprocessing': [ + { + 'type': 'extract_answers_tokens', + 'max_answer': 30, + 'n_best_size': 20 + } + ], + 'metrics': [ + { + 'name': 'F1', + 'type': 'f1', + 'reference': 0.9157 + }, + { + 'name': 'EM', + 'type': 'exact_match', + 'reference': 0.8504 + } + ], + '_command_line_mapping': { + 'testing_file': 'PATH', + 'vocab_file': [ + 'PATH' + ] + } + } + ] + } + ], + 'stat_requests_number': null, + 'eval_requests_number': null, + 'type': 'accuracy_checker' + } + } + + + + + )V0G0N"; + ov::Tensor weights = {}; + model = core.read_model(ir_with_meta, weights); + OPENVINO_ASSERT(model); + } +}; + +// tested function: set_property, compile_model +TEST_P(CoreThreadingTestsWithCacheEnabled, smoke_compilemodel_cache_enabled) { + ov::Core core; + SetupModel(); + core.set_property(target_device, config); + core.set_property(ov::cache_dir(cache_path)); + runParallel([&] () { + (void)core.compile_model(model, target_device); + }, numIterations, numThreads); + core.set_property(ov::cache_dir("")); +} \ No newline at end of file diff --git a/tests/layer_tests/tensorflow_tests/test_tf_MaxPoolWithArgmax.py b/tests/layer_tests/tensorflow_tests/test_tf_MaxPoolWithArgmax.py index bb39a94594b91a..4d1fed5747ba11 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_MaxPoolWithArgmax.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_MaxPoolWithArgmax.py @@ -55,11 +55,7 @@ def create_max_pool_with_argmax_net(self, input_shape, ksize, strides, input_typ True, False ]) @pytest.mark.parametrize("with_second_output", [ - pytest.param( - True, - marks=pytest.mark.skip(reason="117415: TransposeSinking crash") - ), - False + True, False ]) @pytest.mark.precommit_tf_fe @pytest.mark.nightly