diff --git a/be/src/olap/inverted_index_profile.h b/be/src/olap/inverted_index_profile.h new file mode 100644 index 000000000000000..6ec0c6bbc1f2675 --- /dev/null +++ b/be/src/olap/inverted_index_profile.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "olap/inverted_index_stats.h" +#include "olap/tablet_schema.h" +#include "util/runtime_profile.h" + +namespace doris { + +class InvertedIndexProfileReporter { +public: + InvertedIndexProfileReporter(RuntimeProfile* profile) : _profile(profile) {} + + void update(const InvertedIndexStatistics* statistics, const TabletSchemaSPtr& tablet_schema) { + if (_profile == nullptr) { + return; + } + + std::string inverted_index_filter = "["; + for (auto stats : statistics->stats) { + inverted_index_filter += "("; + inverted_index_filter += + tablet_schema->column_by_uid(stats.column_unique_id).name() + ", "; + inverted_index_filter += PrettyPrinter::print(stats.filter_count, TUnit::UNIT) + ", "; + inverted_index_filter += PrettyPrinter::print(stats.filter_time, TUnit::TIME_NS); + inverted_index_filter += "), "; + } + if (inverted_index_filter.length() > 1) { + inverted_index_filter = + inverted_index_filter.substr(0, inverted_index_filter.length() - 2); + } + inverted_index_filter += "]"; + + auto* child = _profile->create_child("IndexFilter"); + child->add_info_string("InvertedIndexFilter", inverted_index_filter); + } + +private: + RuntimeProfile* _profile = nullptr; +}; + +} // namespace doris diff --git a/be/src/olap/inverted_index_stats.h b/be/src/olap/inverted_index_stats.h new file mode 100644 index 000000000000000..8c6562224b3b87d --- /dev/null +++ b/be/src/olap/inverted_index_stats.h @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + +struct InvertedIndexQueryStatistics { + int32_t column_unique_id; + int64_t filter_count = 0; + int64_t filter_time = 0; +}; + +struct InvertedIndexStatistics { + std::vector stats; +}; + +} // namespace doris diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 623d2c83e49a9fe..24477b9b66baef7 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -38,6 +38,7 @@ #include "common/config.h" #include "common/exception.h" #include "io/io_common.h" +#include "olap/inverted_index_stats.h" #include "olap/olap_define.h" #include "olap/rowset/rowset_fwd.h" #include "util/hash_util.hpp" @@ -378,6 +379,7 @@ struct OlapReaderStatistics { int64_t inverted_index_searcher_cache_hit = 0; int64_t inverted_index_searcher_cache_miss = 0; int64_t inverted_index_downgrade_count = 0; + InvertedIndexStatistics inverted_index_stats; int64_t output_index_result_column_timer = 0; // number of segment filtered by column stat when creating seg iterator diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 5fd1223e0fa101d..40e9e376a32550a 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -1160,8 +1160,24 @@ Status InvertedIndexIterator::read_from_inverted_index( } } - RETURN_IF_ERROR(_reader->query(&_io_ctx, _stats, _runtime_state, column_name, query_value, - query_type, bit_map)); + auto execute_query = [&]() { + return _reader->query(&_io_ctx, _stats, _runtime_state, column_name, query_value, + query_type, bit_map); + }; + + if (_runtime_state->query_options().enable_profile) { + InvertedIndexQueryStatistics query_stats; + { + SCOPED_RAW_TIMER(&query_stats.filter_time); + RETURN_IF_ERROR(execute_query()); + } + query_stats.column_unique_id = std::stoi(column_name); + query_stats.filter_count = bit_map->cardinality(); + _stats->inverted_index_stats.stats.emplace_back(query_stats); + } else { + RETURN_IF_ERROR(execute_query()); + } + return Status::OK(); } diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index 17dedc82368511b..3e6cfdd95a8e82e 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -41,6 +41,7 @@ #include "exprs/function_filter.h" #include "io/cache/block_file_cache_profile.h" #include "io/io_common.h" +#include "olap/inverted_index_profile.h" #include "olap/olap_common.h" #include "olap/olap_tuple.h" #include "olap/rowset/rowset.h" @@ -634,6 +635,10 @@ void NewOlapScanner::_collect_profile_before_close() { stats.inverted_index_searcher_cache_miss); COUNTER_UPDATE(local_state->_inverted_index_downgrade_count_counter, stats.inverted_index_downgrade_count); + + InvertedIndexProfileReporter inverted_index_profile(local_state->_segment_profile.get()); + inverted_index_profile.update(&stats.inverted_index_stats, _tablet_reader_params.tablet_schema); + if (config::enable_file_cache) { io::FileCacheProfileReporter cache_profile(local_state->_segment_profile.get()); cache_profile.update(&stats.file_cache_stats); diff --git a/be/test/olap/inverted_index_profile_test.cpp b/be/test/olap/inverted_index_profile_test.cpp new file mode 100644 index 000000000000000..35cc84baec2029e --- /dev/null +++ b/be/test/olap/inverted_index_profile_test.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/inverted_index_profile.h" + +#include + +#include + +#include "olap/inverted_index_stats.h" + +namespace doris { + +class InvertedIndexProfileReporterTest : public ::testing::Test { +protected: + void SetUp() override { + _runtime_profile = new RuntimeProfile("test_profile"); + _reporter = new InvertedIndexProfileReporter(_runtime_profile); + } + + void TearDown() override { + delete _reporter; + delete _runtime_profile; + } + + RuntimeProfile* _runtime_profile = nullptr; + InvertedIndexProfileReporter* _reporter = nullptr; +}; + +class MockInvertedIndexStatistics1 : public InvertedIndexStatistics { +public: + MockInvertedIndexStatistics1() { stats.push_back({123, 100, 200}); } +}; + +class MockInvertedIndexStatistics2 : public InvertedIndexStatistics { +public: + MockInvertedIndexStatistics2() { + stats.push_back({123, 100, 200}); + stats.push_back({124, 150, 250}); + stats.push_back({125, 200, 300}); + } +}; + +TEST_F(InvertedIndexProfileReporterTest, UpdateTest) { + MockInvertedIndexStatistics1 statistics; + + TabletColumn tablet_column; + tablet_column.set_unique_id(123); + tablet_column.set_name("test_column"); + + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->append_column(tablet_column); + + _reporter->update(&statistics, tablet_schema); + + std::vector children; + _runtime_profile->get_children(&children); + + ASSERT_EQ(children.size(), 1); + ASSERT_EQ(children[0]->name(), "IndexFilter"); + + const std::string* index_filter = children[0]->get_info_string("InvertedIndexFilter"); + ASSERT_NE(index_filter, nullptr); + + // 验证输出格式是否正确 + ASSERT_TRUE(index_filter->find("test_column") != std::string::npos); + ASSERT_TRUE(index_filter->find("100") != std::string::npos); + ASSERT_TRUE(index_filter->find("200") != std::string::npos); +} + +TEST_F(InvertedIndexProfileReporterTest, UpdateTestWithMultipleStats) { + MockInvertedIndexStatistics2 statistics; + + TabletColumn tablet_column1, tablet_column2, tablet_column3; + tablet_column1.set_unique_id(123); + tablet_column1.set_name("test_column1"); + tablet_column2.set_unique_id(124); + tablet_column2.set_name("test_column2"); + tablet_column3.set_unique_id(125); + tablet_column3.set_name("test_column3"); + + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->append_column(tablet_column1); + tablet_schema->append_column(tablet_column2); + tablet_schema->append_column(tablet_column3); + + _reporter->update(&statistics, tablet_schema); + + std::vector children; + _runtime_profile->get_children(&children); + + ASSERT_EQ(children.size(), 1); + ASSERT_EQ(children[0]->name(), "IndexFilter"); + + const std::string* index_filter = children[0]->get_info_string("InvertedIndexFilter"); + ASSERT_NE(index_filter, nullptr); + + // 验证输出格式是否正确 + ASSERT_TRUE(index_filter->find("test_column1") != std::string::npos); + ASSERT_TRUE(index_filter->find("test_column2") != std::string::npos); + ASSERT_TRUE(index_filter->find("test_column3") != std::string::npos); + ASSERT_TRUE(index_filter->find("100") != std::string::npos); + ASSERT_TRUE(index_filter->find("200") != std::string::npos); + ASSERT_TRUE(index_filter->find("150") != std::string::npos); + ASSERT_TRUE(index_filter->find("250") != std::string::npos); + ASSERT_TRUE(index_filter->find("200") != std::string::npos); + ASSERT_TRUE(index_filter->find("300") != std::string::npos); +} + +} // namespace doris \ No newline at end of file