From 115ad8060d14a4fc4c92cba058ec956126416e04 Mon Sep 17 00:00:00 2001 From: Matias Romeo Date: Wed, 20 Sep 2023 23:44:43 -0300 Subject: [PATCH] Revert "Properly read address/topic index bitmap from db to serve eth_getLogs calls" This reverts commit ff8047c90402189ccb66a84976905a9668d685ac. --- silkworm/silkrpc/commands/eth_api.cpp | 3 +- silkworm/silkrpc/ethdb/bitmap.cpp | 49 ++++++++++++++++----------- silkworm/silkrpc/ethdb/bitmap.hpp | 8 ++--- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/silkworm/silkrpc/commands/eth_api.cpp b/silkworm/silkrpc/commands/eth_api.cpp index 5332776b5..2a46e949d 100644 --- a/silkworm/silkrpc/commands/eth_api.cpp +++ b/silkworm/silkrpc/commands/eth_api.cpp @@ -2093,7 +2093,8 @@ awaitable EthereumRpcApi::get_logs(ethdb::TransactionDatabase& tx_database FilterAddresses& addresses, FilterTopics& topics, std::vector& logs) { SILK_INFO << "start block: " << start << " end block: " << end; - roaring::Roaring64Map block_numbers(roaring::api::roaring_bitmap_from_range(start, end+1, 1)); + roaring::Roaring block_numbers; + block_numbers.addRange(start, end + 1); // [min, max) SILK_DEBUG << "block_numbers.cardinality(): " << block_numbers.cardinality(); diff --git a/silkworm/silkrpc/ethdb/bitmap.cpp b/silkworm/silkrpc/ethdb/bitmap.cpp index 2b0af6a80..ee17ffc71 100644 --- a/silkworm/silkrpc/ethdb/bitmap.cpp +++ b/silkworm/silkrpc/ethdb/bitmap.cpp @@ -25,37 +25,46 @@ #include #include -#include namespace silkworm::rpc::ethdb::bitmap { using roaring_bitmap_t = roaring::api::roaring_bitmap_t; -using Roaring64Map = roaring::Roaring64Map; +using Roaring = roaring::Roaring; -static Roaring64Map fast_or(size_t n, const std::vector>& inputs) { - Roaring64Map result; +static Roaring fast_or(size_t n, const std::vector>& inputs) { + const auto** x = static_cast(malloc(n * sizeof(roaring_bitmap_t*))); + if (x == nullptr) { + throw std::runtime_error("failed memory alloc in fast_or"); + } for (size_t k = 0; k < n; ++k) { - result |= *inputs[k]; + x[k] = &inputs[k]->roaring; + } + + roaring_bitmap_t* c_ans = roaring_bitmap_or_many(n, x); + if (c_ans == nullptr) { + free(x); + throw std::runtime_error("failed memory alloc in fast_or"); } - return result; + Roaring ans(c_ans); + free(x); + return ans; } -awaitable get(core::rawdb::DatabaseReader& db_reader, const std::string& table, silkworm::Bytes& key, +awaitable get(core::rawdb::DatabaseReader& db_reader, const std::string& table, silkworm::Bytes& key, uint32_t from_block, uint32_t to_block) { - std::vector> chunks; + std::vector> chunks; silkworm::Bytes from_key{key.begin(), key.end()}; - from_key.resize(key.size() + sizeof(uint16_t)); - boost::endian::store_big_u16(&from_key[key.size()], 0); + from_key.resize(key.size() + sizeof(uint32_t)); + boost::endian::store_big_u32(&from_key[key.size()], from_block); SILK_DEBUG << "table: " << table << " key: " << key << " from_key: " << from_key; core::rawdb::Walker walker = [&](const silkworm::Bytes& k, const silkworm::Bytes& v) { SILK_TRACE << "k: " << k << " v: " << v; - auto chunck = std::make_unique(silkworm::db::bitmap::parse(v)); - auto block = chunck->maximum(); - if (block >= from_block && chunck->minimum() <= to_block) { - chunks.push_back(std::move(chunck)); - } + auto chunk = std::make_unique(Roaring::readSafe(reinterpret_cast(v.data()), v.size())); + SILK_TRACE << "chunk: " << chunk->toString(); + chunks.push_back(std::move(chunk)); + auto block = boost::endian::load_big_u32(&k[k.size() - sizeof(uint32_t)]); return block < to_block; }; co_await db_reader.walk(table, from_key, gsl::narrow(key.size() * CHAR_BIT), walker); @@ -65,13 +74,13 @@ awaitable get(core::rawdb::DatabaseReader& db_reader, const std::s co_return result; } -awaitable from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterTopics& topics, +awaitable from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterTopics& topics, uint64_t start, uint64_t end) { SILK_DEBUG << "#topics: " << topics.size() << " start: " << start << " end: " << end; - roaring::Roaring64Map result_bitmap; + roaring::Roaring result_bitmap; for (const auto& subtopics : topics) { SILK_DEBUG << "#subtopics: " << subtopics.size(); - roaring::Roaring64Map subtopic_bitmap; + roaring::Roaring subtopic_bitmap; for (auto topic : subtopics) { silkworm::Bytes topic_key{std::begin(topic.bytes), std::end(topic.bytes)}; SILK_TRACE << "topic: " << topic << " topic_key: " << silkworm::to_hex(topic); @@ -92,10 +101,10 @@ awaitable from_topics(core::rawdb::DatabaseReader& db_reader, cons co_return result_bitmap; } -awaitable from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterAddresses& addresses, +awaitable from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterAddresses& addresses, uint64_t start, uint64_t end) { SILK_TRACE << "#addresses: " << addresses.size() << " start: " << start << " end: " << end; - roaring::Roaring64Map result_bitmap; + roaring::Roaring result_bitmap; for (auto address : addresses) { silkworm::Bytes address_key{std::begin(address.bytes), std::end(address.bytes)}; auto bitmap = co_await ethdb::bitmap::get(db_reader, table, address_key, gsl::narrow(start), gsl::narrow(end)); diff --git a/silkworm/silkrpc/ethdb/bitmap.hpp b/silkworm/silkrpc/ethdb/bitmap.hpp index d04d83786..efcda135c 100644 --- a/silkworm/silkrpc/ethdb/bitmap.hpp +++ b/silkworm/silkrpc/ethdb/bitmap.hpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wold-style-cast" #pragma GCC diagnostic ignored "-Wconversion" #pragma GCC diagnostic ignored "-Wsign-conversion" -#include +#include #pragma GCC diagnostic pop #include @@ -36,13 +36,13 @@ namespace silkworm::rpc::ethdb::bitmap { using boost::asio::awaitable; -awaitable get(core::rawdb::DatabaseReader& db_reader, const std::string& table, +awaitable get(core::rawdb::DatabaseReader& db_reader, const std::string& table, silkworm::Bytes& key, uint32_t from_block, uint32_t to_block); -awaitable from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table, +awaitable from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterTopics& topics, uint64_t start, uint64_t end); -awaitable from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table, +awaitable from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterAddresses& addresses, uint64_t start, uint64_t end); } // namespace silkworm::rpc::ethdb::bitmap