Skip to content

Commit

Permalink
Merge branch 'master' into fast_fp16_compression
Browse files Browse the repository at this point in the history
  • Loading branch information
@slyalin
slyalin authored Aug 15, 2023
2 parents 2ac6027 + 8509737 commit 1379778
Show file tree
Hide file tree
Showing 19 changed files with 499 additions and 80 deletions.
9 changes: 6 additions & 3 deletions src/common/transformations/include/transformations/utils/utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,16 @@ namespace op {
namespace util {

template <class T>
bool normalize_single_value(std::vector<T> vec, float& value) {
bool normalize_single_value(std::vector<T> vec, float& value, bool check_value_range = true) {
for (const auto& val : vec) {
if (val != *vec.begin())
return false;
}

float ref_val = static_cast<float>(*vec.begin());

if (ref_val < std::numeric_limits<float>::lowest() || ref_val > std::numeric_limits<float>::max()) {
if (check_value_range &&
(ref_val < std::numeric_limits<float>::lowest() || ref_val > std::numeric_limits<float>::max())) {
return false;
}

Expand Down Expand Up @@ -159,7 +160,9 @@ bool has_constant_value(const std::shared_ptr<Node>& node,
return const_values == values;
}

TRANSFORMATIONS_API bool get_single_value(const std::shared_ptr<opset4::Constant>& const_node, float& value);
TRANSFORMATIONS_API bool get_single_value(const std::shared_ptr<opset4::Constant>& const_node,
float& value,
bool check_value_range = true);

TRANSFORMATIONS_API std::shared_ptr<Node> normalize_constant(const std::shared_ptr<opset4::Constant>& constant,
const PartialShape& shape);
Expand Down
14 changes: 8 additions & 6 deletions src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,12 +240,14 @@ TSGatherBackward::TSGatherBackward() {
if (success) {
size_t j = 0;
for (size_t i = 0; i < shape.size(); ++i) {
if (shape[i] != new_shape[j] && shape[i] == 1) {
axes_val.push_back(i);
continue;
} else if (shape[i] != new_shape[j]) {
success = false;
break;
if (j >= new_shape.size() || shape[i] != new_shape[j]) {
if (shape[i] == 1) {
axes_val.push_back(i);
continue;
} else {
success = false;
break;
}
}
j++;
}
Expand Down
26 changes: 13 additions & 13 deletions src/common/transformations/src/transformations/utils/utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,32 +21,32 @@ namespace ov {
namespace op {
namespace util {

bool get_single_value(const std::shared_ptr<op::v0::Constant>& const_node, float& value) {
bool get_single_value(const std::shared_ptr<op::v0::Constant>& const_node, float& value, bool check_value_range) {
switch (const_node->get_element_type()) {
case element::Type_t::f16:
return util::normalize_single_value(const_node->get_vector<float16>(), value);
return util::normalize_single_value(const_node->get_vector<float16>(), value, check_value_range);
case element::Type_t::f32:
return util::normalize_single_value(const_node->get_vector<float>(), value);
return util::normalize_single_value(const_node->get_vector<float>(), value, check_value_range);
case element::Type_t::bf16:
return util::normalize_single_value(const_node->get_vector<bfloat16>(), value);
return util::normalize_single_value(const_node->get_vector<bfloat16>(), value, check_value_range);
case element::Type_t::f64:
return util::normalize_single_value(const_node->get_vector<double>(), value);
return util::normalize_single_value(const_node->get_vector<double>(), value, check_value_range);
case element::Type_t::i8:
return util::normalize_single_value(const_node->get_vector<int8_t>(), value);
return util::normalize_single_value(const_node->get_vector<int8_t>(), value, check_value_range);
case element::Type_t::i16:
return util::normalize_single_value(const_node->get_vector<int16_t>(), value);
return util::normalize_single_value(const_node->get_vector<int16_t>(), value, check_value_range);
case element::Type_t::i32:
return util::normalize_single_value(const_node->get_vector<int32_t>(), value);
return util::normalize_single_value(const_node->get_vector<int32_t>(), value, check_value_range);
case element::Type_t::i64:
return util::normalize_single_value(const_node->get_vector<int64_t>(), value);
return util::normalize_single_value(const_node->get_vector<int64_t>(), value, check_value_range);
case element::Type_t::u8:
return util::normalize_single_value(const_node->get_vector<uint8_t>(), value);
return util::normalize_single_value(const_node->get_vector<uint8_t>(), value, check_value_range);
case element::Type_t::u16:
return util::normalize_single_value(const_node->get_vector<uint16_t>(), value);
return util::normalize_single_value(const_node->get_vector<uint16_t>(), value, check_value_range);
case element::Type_t::u32:
return util::normalize_single_value(const_node->get_vector<uint32_t>(), value);
return util::normalize_single_value(const_node->get_vector<uint32_t>(), value, check_value_range);
case element::Type_t::u64:
return util::normalize_single_value(const_node->get_vector<uint64_t>(), value);
return util::normalize_single_value(const_node->get_vector<uint64_t>(), value, check_value_range);
default:
OPENVINO_THROW("Unsupported precision for const operation: ", const_node->get_friendly_name());
}
Expand Down
40 changes: 29 additions & 11 deletions src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,9 @@ INSTANTIATE_TEST_SUITE_P(TSCommonGatherForward_3, TSTestFixture, test_forward_ga

struct GatherBackwardArguments {
OutputVector inputs_to_main;
Output<Node> new_Gather_first_input;
AxisVector new_transpose_order;
Output<Node> ref_Gather_axis_input;
AxisVector ref_transpose_order;
AxisVector ref_unsqueeze_axes;
};

auto test_backward_gather = [](const GatherBackwardArguments& test_arguments) {
Expand All @@ -147,14 +148,14 @@ auto test_backward_gather = [](const GatherBackwardArguments& test_arguments) {
OutputVector new_out_vec(out_vec.size());
new_out_vec[0] = out_vec[0];
new_out_vec[1] = out_vec[1];
new_out_vec[2] = test_arguments.new_Gather_first_input;
new_out_vec[2] = test_arguments.ref_Gather_axis_input;
return new_out_vec;
};
auto new_transpose = [&test_arguments](const vector<size_t>& idxs, const OutputVector& out_vec) -> OutputVector {
OutputVector new_out_vec = out_vec;
auto order = make_shared<Constant>(i32,
Shape{test_arguments.new_transpose_order.size()},
test_arguments.new_transpose_order);
Shape{test_arguments.ref_transpose_order.size()},
test_arguments.ref_transpose_order);
new_out_vec[0] = make_shared<Transpose>(out_vec[0], order);
return new_out_vec;
};
Expand Down Expand Up @@ -197,13 +198,14 @@ auto test_backward_gather_optimization = [](const GatherBackwardArguments& test_
OutputVector new_out_vec(out_vec.size());
new_out_vec[0] = out_vec[0];
new_out_vec[1] = make_shared<Squeeze>(out_vec[1]);
new_out_vec[2] = test_arguments.new_Gather_first_input;
new_out_vec[2] = test_arguments.ref_Gather_axis_input;
return new_out_vec;
};

auto unsqueeze_for = [&](const vector<size_t>& idxs, const OutputVector& out_vec) -> OutputVector {
auto axis = constant<int>(i32, {1}, {0});
return {make_shared<Unsqueeze>(out_vec[0], axis)};
const auto& axes_val = test_arguments.ref_unsqueeze_axes;
auto axes = constant<size_t>(i32, {axes_val.size()}, axes_val);
return {make_shared<Unsqueeze>(out_vec[0], axes)};
};

test_case.model_ref.preprocess_inputs_to_main = {{set_transpose_for, update_gather_inputs}, {{0}, {1, 2}}};
Expand All @@ -215,13 +217,29 @@ auto test_backward_gather_optimization = [](const GatherBackwardArguments& test_
};

vector<GatherBackwardArguments> tests_arguments_bw_optimization{
{{{parameter(f32, {257, 8}), constant<int>(i32, {1, 2}, {0}), constant<int>(i32, {1}, {0})},
constant<int>(i32, {1}, {1}),
AxisVector{}}}};
{{parameter(f32, {257, 8}), constant<int>(i32, {1, 2}, {0}), constant<int>(i32, {1}, {0})},
constant<int>(i32, {1}, {1}),
AxisVector{},
AxisVector{0}},
{{parameter(f32, {4}), constant<int>(i32, {1}, {0}), constant<int>(i32, {1}, {0})},
constant<int>(i32, {1}, {0}),
AxisVector{},
AxisVector{0}},
{{parameter(f32, {4}), constant<int>(i32, {1, 1, 1}, {0}), constant<int>(i32, {1}, {0})},
constant<int>(i32, {1}, {0}),
AxisVector{},
AxisVector{0, 1, 2}},
};

INSTANTIATE_TEST_SUITE_P(TSCommonGatherBackwardOptimization_0,
TSTestFixture,
test_backward_gather_optimization(tests_arguments_bw_optimization[0]));
INSTANTIATE_TEST_SUITE_P(TSCommonGatherBackwardOptimization_1,
TSTestFixture,
test_backward_gather_optimization(tests_arguments_bw_optimization[1]));
INSTANTIATE_TEST_SUITE_P(TSCommonGatherBackwardOptimization_2,
TSTestFixture,
test_backward_gather_optimization(tests_arguments_bw_optimization[2]));
} // namespace gather
} // namespace testing
} // namespace transpose_sinking
13 changes: 5 additions & 8 deletions src/frontends/ir/src/ir_deserializer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,18 +614,15 @@ class MetaDataParser : public ov::Meta {
}

void parse() const {
// Thread safety is implemented on ov::Model level
if (m_parsed)
return;
const pugi::xml_node& node = m_meta.child(m_name.c_str());
m_parsed_map = parse_node(node);

m_parsed = true;
std::call_once(m_oc, [this]() {
const pugi::xml_node& node = m_meta.child(m_name.c_str());
m_parsed_map = parse_node(node);
});
}
pugi::xml_document m_meta;
const std::string m_name;
mutable ov::AnyMap m_parsed_map;
mutable bool m_parsed{false};
mutable std::once_flag m_oc;
};

void XmlDeserializer::read_meta_data(const std::shared_ptr<ov::Model>& model, const pugi::xml_node& meta_section) {
Expand Down
16 changes: 10 additions & 6 deletions src/frontends/tensorflow/src/frontend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const {
// Last boolean flag in `variants` (if presented) is reserved for FE configuration
size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
// Enable mmap by default
bool mmap_enabled = variants[variants.size() - 1].is<bool>() ? variants[variants.size() - 1].as<bool>() : true;

// For TF1 models it can be a case of two input variants: input model and v1 checkpoints
FRONT_END_GENERAL_CHECK(
Expand All @@ -203,7 +205,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& va
return std::make_shared<InputModel>(std::make_shared<GraphIteratorProto>(model_path), m_telemetry);
} else if (GraphIteratorSavedModel::is_supported(model_path)) {
std::shared_ptr<GraphIteratorSavedModel> graph_iterator;
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path, std::string("serve"));
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path, std::string("serve"), mmap_enabled);
return std::make_shared<InputModel>(graph_iterator,
m_telemetry,
graph_iterator->get_variables_index(),
Expand All @@ -212,7 +214,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& va
nullptr,
true);
} else if (GraphIteratorMeta::is_supported(model_path)) {
auto graph_iterator = std::make_shared<GraphIteratorMeta>(model_path);
auto graph_iterator = std::make_shared<GraphIteratorMeta>(model_path, mmap_enabled);
return std::make_shared<InputModel>(graph_iterator,
m_telemetry,
graph_iterator->get_variables_index(),
Expand Down Expand Up @@ -257,7 +259,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& va
auto saved_model_tags = paths[1];
if (GraphIteratorSavedModel::is_supported(model_path)) {
std::shared_ptr<GraphIteratorSavedModel> graph_iterator;
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path, saved_model_tags);
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path, saved_model_tags, mmap_enabled);
return std::make_shared<InputModel>(graph_iterator,
m_telemetry,
graph_iterator->get_variables_index(),
Expand All @@ -275,7 +277,9 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& va
return std::make_shared<InputModel>(std::make_shared<GraphIteratorProto>(model_path), m_telemetry);
} else if (GraphIteratorSavedModel::is_supported(model_path)) {
std::shared_ptr<GraphIteratorSavedModel> graph_iterator;
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path, std::string(META_GRAPH_DEFAULT_TAG));
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path,
std::string(META_GRAPH_DEFAULT_TAG),
mmap_enabled);
return std::make_shared<InputModel>(graph_iterator,
m_telemetry,
graph_iterator->get_variables_index(),
Expand All @@ -284,7 +288,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& va
nullptr,
true);
} else if (GraphIteratorMeta::is_supported(model_path)) {
auto graph_iterator = std::make_shared<GraphIteratorMeta>(model_path);
auto graph_iterator = std::make_shared<GraphIteratorMeta>(model_path, mmap_enabled);
return std::make_shared<InputModel>(graph_iterator,
m_telemetry,
graph_iterator->get_variables_index(),
Expand Down Expand Up @@ -329,7 +333,7 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& va
auto saved_model_tags = ov::util::wstring_to_string(paths[1]);
if (GraphIteratorSavedModel::is_supported(model_path)) {
std::shared_ptr<GraphIteratorSavedModel> graph_iterator;
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path, saved_model_tags);
graph_iterator = std::make_shared<GraphIteratorSavedModel>(model_path, saved_model_tags, mmap_enabled);
return std::make_shared<InputModel>(graph_iterator,
m_telemetry,
graph_iterator->get_variables_index(),
Expand Down
8 changes: 5 additions & 3 deletions src/frontends/tensorflow/src/graph_iterator_meta.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,13 @@ class GraphIteratorMeta : public GraphIteratorProto {
std::shared_ptr<VariablesIndex> m_variables_index;
std::shared_ptr<std::map<std::string, std::string>> m_inputs_map;
std::shared_ptr<std::map<std::string, std::string>> m_outputs_map;
bool m_mmap_enabled;

public:
template <typename T>
GraphIteratorMeta(const std::basic_string<T>& path)
: m_metagraph_def(std::make_shared<::tensorflow::MetaGraphDef>()) {
GraphIteratorMeta(const std::basic_string<T>& path, const bool mmap_enabled)
: m_metagraph_def(std::make_shared<::tensorflow::MetaGraphDef>()),
m_mmap_enabled(mmap_enabled) {
this->read_meta(path);
}

Expand Down Expand Up @@ -75,7 +77,7 @@ class GraphIteratorMeta : public GraphIteratorProto {

std::basic_string<T> varIndexPath = get_variables_index_name<T>(model_path);
if (ov::util::file_exists(varIndexPath)) {
m_variables_index = std::make_shared<VariablesIndex>();
m_variables_index = std::make_shared<VariablesIndex>(m_mmap_enabled);
std::ifstream vi_stream{varIndexPath.c_str(), std::ifstream::in | std::ifstream::binary};
FRONT_END_GENERAL_CHECK(vi_stream && vi_stream.is_open(), "MetaGraph's variable index file does not exist");
FRONT_END_GENERAL_CHECK(m_variables_index->read_variables(vi_stream, model_path, false),
Expand Down
8 changes: 5 additions & 3 deletions src/frontends/tensorflow/src/graph_iterator_saved_model.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,13 @@ class GraphIteratorSavedModel : public GraphIteratorProto {
std::shared_ptr<VariablesIndex> m_variables_index;
std::shared_ptr<std::map<std::string, std::string>> m_inputs_map;
std::shared_ptr<std::map<std::string, std::string>> m_outputs_map;
bool m_mmap_enabled;

public:
template <typename T>
GraphIteratorSavedModel(const std::basic_string<T>& path, const std::string& tags)
: m_saved_model(std::make_shared<::tensorflow::SavedModel>()) {
GraphIteratorSavedModel(const std::basic_string<T>& path, const std::string& tags, const bool mmap_enabled)
: m_saved_model(std::make_shared<::tensorflow::SavedModel>()),
m_mmap_enabled(mmap_enabled) {
this->read_saved_model(path, tags);
}

Expand Down Expand Up @@ -74,7 +76,7 @@ class GraphIteratorSavedModel : public GraphIteratorProto {

std::basic_string<T> varIndexPath = path + get_variables_index_name<T>();
if (ov::util::file_exists(varIndexPath)) {
m_variables_index = std::make_shared<VariablesIndex>();
m_variables_index = std::make_shared<VariablesIndex>(m_mmap_enabled);
std::ifstream vi_stream{varIndexPath.c_str(), std::ifstream::in | std::ifstream::binary};
FRONT_END_GENERAL_CHECK(vi_stream && vi_stream.is_open(),
"[TensorFlow Frontend] Saved Model's variable index file does not exist");
Expand Down
Loading

0 comments on commit 1379778

Please sign in to comment.