CachingPageTracer Optimization

src/llfs/api_types.hpp

@@ -81,6 +81,10 @@ BATT_STRONG_TYPEDEF(usize, BufferCount);
  */
 BATT_STRONG_TYPEDEF(usize, BufferSize);
 
+/** \brief True if a page contains outgoing references to other pages.
+ */
+BATT_STRONG_TYPEDEF(bool, HasOutgoingRefs);
+
 }  // namespace llfs
 
 #endif  // LLFS_API_TYPES_HPP

src/llfs/committable_page_cache_job.cpp

@@ -88,11 +88,8 @@ usize CommittablePageCacheJob::new_page_count() const noexcept
 //
 BoxedSeq<PageId> CommittablePageCacheJob::deleted_page_ids() const
 {
-  return as_seq(this->job_->get_deleted_pages().begin(), this->job_->get_deleted_pages().end())  //
-         | seq::map([](const auto& kv_pair) -> PageId {
-             return kv_pair.first;
-           })  //
-         | seq::boxed();
+  return BoxedSeq<PageId>{
+      as_seq(this->job_->get_deleted_pages().begin(), this->job_->get_deleted_pages().end())};
 }
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
@@ -482,7 +479,7 @@ Status CommittablePageCacheJob::await_ref_count_updates(const PageRefCountUpdate
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-auto CommittablePageCacheJob::get_page_ref_count_updates(u64 /*callers*/) const
+auto CommittablePageCacheJob::get_page_ref_count_updates(u64 /*callers*/)
     -> StatusOr<PageRefCountUpdates>
 {
   std::unordered_map<PageId, i32, PageId::Hash> ref_count_delta = this->job_->get_root_set_delta();
@@ -512,19 +509,22 @@ auto CommittablePageCacheJob::get_page_ref_count_updates(u64 /*callers*/) const
   // Trace deleted pages non-recursively, decrementing the ref counts of all pages they directly
   // reference.
   //
-  for (const auto& p : this->job_->get_deleted_pages()) {
-    // Sanity check; deleted pages should have a ref_count_delta of kRefCount_1_to_0.
+  LoadingPageTracer loading_tracer{loader, /*ok_if_not_found=*/true};
+  CachingPageTracer caching_tracer{this->job_->cache().devices_by_id(), loading_tracer};
+  for (const PageId& deleted_page_id : this->job_->get_deleted_pages()) {
+    // Decrement ref counts.
     //
-    const PageId deleted_page_id = p.first;
-    {
-      auto iter = ref_count_delta.find(deleted_page_id);
-      BATT_CHECK_NE(iter, ref_count_delta.end());
-      BATT_CHECK_EQ(iter->second, kRefCount_1_to_0);
+    batt::StatusOr<batt::BoxedSeq<PageId>> outgoing_refs =
+        caching_tracer.trace_page_refs(deleted_page_id);
+    if (outgoing_refs.status() == batt::StatusCode::kNotFound) {
+      this->not_found_deleted_pages_.insert(deleted_page_id);
+      continue;
     }
+    BATT_REQUIRE_OK(outgoing_refs);
 
-    // Decrement ref counts.
-    //
-    p.second->trace_refs() | seq::for_each([&ref_count_delta, deleted_page_id](PageId id) {
+    ref_count_delta[deleted_page_id] = kRefCount_1_to_0;
+
+    *outgoing_refs | seq::for_each([&ref_count_delta, deleted_page_id](PageId id) {
       if (id) {
         LLFS_VLOG(1) << " decrementing ref count for page " << id
                      << " (because it was referenced from deleted page " << deleted_page_id << ")";
@@ -596,13 +596,13 @@ Status CommittablePageCacheJob::drop_deleted_pages(u64 callers)
 {
   LLFS_VLOG(1) << "commit(PageCacheJob): dropping deleted pages";
 
-  const auto& deleted_pages = this->job_->get_deleted_pages();
-
-  return parallel_drop_pages(as_seq(deleted_pages.begin(), deleted_pages.end())  //
-                                 | seq::map([](const auto& kv_pair) -> PageId {
-                                     return kv_pair.first;
-                                   })  //
-                                 | seq::collect_vec(),
+  // From the set of all deleted pages, filter out those that were not found during the attempt to
+  // trace their outgoing refs.
+  //
+  return parallel_drop_pages(this->deleted_page_ids() | seq::filter([this](const PageId& id) {
+                               auto iter = this->not_found_deleted_pages_.find(id);
+                               return iter == this->not_found_deleted_pages_.end();
+                             }) | seq::collect_vec(),
                              this->job_->cache(), this->job_->job_id, callers);
 }
 

src/llfs/committable_page_cache_job.hpp

@@ -193,7 +193,7 @@ class CommittablePageCacheJob
 
   Status write_new_pages();
 
-  StatusOr<PageRefCountUpdates> get_page_ref_count_updates(u64 callers) const;
+  StatusOr<PageRefCountUpdates> get_page_ref_count_updates(u64 callers);
 
   StatusOr<DeadPages> start_ref_count_updates(const JobCommitParams& params,
                                               PageRefCountUpdates& updates, u64 callers);
@@ -212,6 +212,7 @@ class CommittablePageCacheJob
   boost::intrusive_ptr<FinalizedJobTracker> tracker_;
   PageRefCountUpdates ref_count_updates_;
   std::unique_ptr<WriteNewPagesContext> write_new_pages_context_;
+  std::unordered_set<PageId, PageId::Hash> not_found_deleted_pages_;
 };
 
 /** \brief Write all changes in `job` to durable storage.  This is guaranteed to be atomic.

src/llfs/no_outgoing_refs_cache.cpp

@@ -0,0 +1,99 @@
+//#=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++
+//
+// Part of the LLFS Project, under Apache License v2.0.
+// See https://www.apache.org/licenses/LICENSE-2.0 for license information.
+// SPDX short identifier: Apache-2.0
+//
+//+++++++++++-+-+--+----- --- -- -  -  -   -
+#include <llfs/no_outgoing_refs_cache.hpp>
+
+namespace llfs {
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+NoOutgoingRefsCache::NoOutgoingRefsCache(const PageIdFactory& page_ids) noexcept
+    : page_ids_{page_ids}
+    , cache_(this->page_ids_.get_physical_page_count().value())
+{
+}
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+void NoOutgoingRefsCache::set_page_state(PageId page_id,
+                                         HasOutgoingRefs new_has_outgoing_refs_state) noexcept
+{
+  BATT_CHECK_EQ(PageIdFactory::get_device_id(page_id), this->page_ids_.get_device_id());
+  const u64 physical_page = this->page_ids_.get_physical_page(page_id);
+  const page_generation_int generation = this->page_ids_.get_generation(page_id);
+  BATT_CHECK_LT((usize)physical_page, this->cache_.size());
+
+  u64 new_cache_entry = generation << this->kGenerationShift;
+
+  // Set the "valid" bit to 1.
+  //
+  new_cache_entry |= this->kValidBitMask;
+
+  // If new_has_outgoing_refs_state has value false, page_id has no outgoing references.
+  //
+  if (!new_has_outgoing_refs_state) {
+    // Set the "has no outgoing references" bit to 1.
+    //
+    new_cache_entry |= this->kHasNoOutgoingRefsBitMask;
+  }
+
+  u64 old_cache_entry = this->cache_[physical_page].exchange(new_cache_entry);
+
+  // Two sanity checks:
+  //  1) We are not going backwards in generation.
+  //  2) If the cache entry is set of the same generation multiple times, the same value should be
+  //  set.
+  //
+  page_generation_int old_generation = old_cache_entry >> this->kGenerationShift;
+  BATT_CHECK_GE(generation, old_generation);
+  if (generation == old_generation) {
+    BATT_CHECK_EQ(new_cache_entry, old_cache_entry);
+  }
+}
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+batt::BoolStatus NoOutgoingRefsCache::has_outgoing_refs(PageId page_id) const noexcept
+{
+  BATT_CHECK_EQ(PageIdFactory::get_device_id(page_id), this->page_ids_.get_device_id());
+  const u64 physical_page = this->page_ids_.get_physical_page(page_id);
+  const page_generation_int generation = this->page_ids_.get_generation(page_id);
+  BATT_CHECK_LT((usize)physical_page, this->cache_.size());
+
+  u64 current_cache_entry = this->cache_[physical_page].load();
+  page_generation_int stored_generation = current_cache_entry >> this->kGenerationShift;
+  u64 outgoing_refs_status = current_cache_entry & this->kOutgoingRefsStatusBitsMask;
+
+  // If the generation that is currently stored in the cache is not the same as the generation we
+  // are querying for, this cache entry is invalid. Thus, we return a "unknown" status.
+  //
+  if (stored_generation != generation) {
+    return batt::BoolStatus::kUnknown;
+  }
+
+  switch (outgoing_refs_status) {
+    case 0:
+      // Bit status 00, not traced yet.
+      //
+      return batt::BoolStatus::kUnknown;
+    case 1:
+      BATT_PANIC() << "The lower two outgoing refs bits in a cache entry can never be 01!";
+      BATT_UNREACHABLE();
+    case 2:
+      // Bit status 10, has outgoing refs.
+      //
+      return batt::BoolStatus::kTrue;
+    case 3:
+      // Bit status 11, no outgoing refs.
+      //
+      return batt::BoolStatus::kFalse;
+    default:
+      BATT_PANIC() << "Impossible outgoing refs bits state: " << outgoing_refs_status;
+      BATT_UNREACHABLE();
+  }
+}
+}  // namespace llfs

src/llfs/no_outgoing_refs_cache.hpp

@@ -0,0 +1,102 @@
+//#=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++
+//
+// Part of the LLFS Project, under Apache License v2.0.
+// See https://www.apache.org/licenses/LICENSE-2.0 for license information.
+// SPDX short identifier: Apache-2.0
+//
+//+++++++++++-+-+--+----- --- -- -  -  -   -
+
+#pragma once
+#ifndef LLFS_NO_OUTGOING_REFS_CACHE_HPP
+#define LLFS_NO_OUTGOING_REFS_CACHE_HPP
+
+#include <llfs/api_types.hpp>
+#include <llfs/page_id_factory.hpp>
+
+#include <batteries/bool_status.hpp>
+
+#include <atomic>
+#include <vector>
+
+namespace llfs {
+
+//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+---------------
+/** \brief A cache to store information about a page's outgoing references to other pages. Can be
+ * used by an implementer of PageTracer as a way to organize and look up this information on the
+ * PageDevice level. This cache is implemented as a vector of unsigned 64-bit integers, where every
+ * element of the vector represents the outgoing refs "state" of a physical page in a PageDevice.
+ * The lowest bit in an element represents if the page has no outgoing refs. The second lowest bit
+ * represents the validity of the page's state to help determine if a page's outgoing refs have ever
+ * been traced. The remaining upper 62 bits are used to store the generation of the physical page.
+ *
+ * Example cache entry: 0000000000000000000000000000000000000000000000000000000000000110
+ * The upper 62 bits represent generation, which in this case is 1. The second lowest bit is the
+ * validity bit, which is 1 here. This indicates that the page has been traced and therefore the
+ * cache entry contains valid information for the page of this generation. The lowest bit is 0,
+ * which indicates that the page has outgoing references to other pages.
+ */
+class NoOutgoingRefsCache
+{
+ public:
+  explicit NoOutgoingRefsCache(const PageIdFactory& page_ids) noexcept;
+
+  NoOutgoingRefsCache(const NoOutgoingRefsCache&) = delete;
+  NoOutgoingRefsCache& operator=(const NoOutgoingRefsCache&) = delete;
+
+  //----- --- -- -  -  -   -
+  /** \brief Sets the two outgoing refs state bits for the given `page_id` based on
+   * whether the page has outgoing refs or not, as indicated by `new_has_outgoing_refs_state`. This
+   * function also updates the generation stored in the cache for the physical page associated with
+   * `page_id`.
+   *
+   * @param page_id The id of the page whose outgoing refs information we are storing.
+   *
+   * @param new_has_outgoing_refs_state The status to store, indicating whether or not the page has
+   * outgoing refs. A value of `false` indicates that the page does not have any outgoing refs, and
+   * a value of `true` indicates that a page does have outgoing refs.
+   */
+  void set_page_state(PageId page_id, HasOutgoingRefs new_has_outgoing_refs_state) noexcept;
+
+  //----- --- -- -  -  -   -
+  /** \brief Queries for whether or not the page with id `page_id` contains outgoing references to
+   * other pages.
+   *
+   * \return Returns a `batt::BoolStatus` type, where `kFalse` indicates that the page has no
+   * outgoing refs, `kTrue` indicates that the page does have outgoing refs, and `kUnknown`
+   * indicates that the cache entry for the given page does not have any valid information stored in
+   * it currently.
+   */
+  batt::BoolStatus has_outgoing_refs(PageId page_id) const noexcept;
+
+ private:
+  /** \brief A mask to retrieve the lowest two outgoing refs state bits.
+   */
+  static constexpr u64 kOutgoingRefsStatusBitsMask = 0b11;
+
+  /** \brief A mask to access the "valid" bit of the two outgoing refs state bits. This is the
+   * higher of the two bits.
+   */
+  static constexpr u64 kValidBitMask = 0b10;
+
+  /** \brief A mask to access the "has no outgoing references" bit of the two outgoing refs state
+   * bits. This is the lower of the two bits.
+   */
+  static constexpr u64 kHasNoOutgoingRefsBitMask = 0b01;
+
+  /** \brief A constant representing the number of bits to shift a cache entry in order retrieve the
+   * generation stored.
+   */
+  static constexpr u8 kGenerationShift = 2;
+
+  /** \brief A PageIdFactory object to help resolve physical page and generation values from a
+   * PageId object.
+   */
+  const PageIdFactory page_ids_;
+
+  /** \brief The vector storing the cache entries, indexed by physical page id.
+   */
+  std::vector<std::atomic<u64>> cache_;
+};
+}  // namespace llfs
+
+#endif  // LLFS_NO_OUTGOING_REFS_CACHE_HPP

src/llfs/page_cache.cpp

@@ -22,7 +22,7 @@ namespace llfs {
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-usize get_page_size(const PageCache::PageDeviceEntry* entry)
+usize get_page_size(const PageDeviceEntry* entry)
 {
   return entry ? get_page_size(entry->arena) : 0;
 }
@@ -420,14 +420,21 @@ void PageCache::prefetch_hint(PageId page_id)
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-Slice<PageCache::PageDeviceEntry* const> PageCache::all_devices() const
+const std::vector<std::unique_ptr<PageDeviceEntry>>& PageCache::devices_by_id() const
+{
+  return this->page_devices_;
+}
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+Slice<PageDeviceEntry* const> PageCache::all_devices() const
 {
   return as_slice(this->page_devices_by_page_size_);
 }
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-Slice<PageCache::PageDeviceEntry* const> PageCache::devices_with_page_size(usize size) const
+Slice<PageDeviceEntry* const> PageCache::devices_with_page_size(usize size) const
 {
   const usize size_log2 = batt::log2_ceil(size);
   BATT_CHECK_EQ(size, usize{1} << size_log2) << "page size must be a power of 2";
@@ -437,8 +444,7 @@ Slice<PageCache::PageDeviceEntry* const> PageCache::devices_with_page_size(usize
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-Slice<PageCache::PageDeviceEntry* const> PageCache::devices_with_page_size_log2(
-    usize size_log2) const
+Slice<PageDeviceEntry* const> PageCache::devices_with_page_size_log2(usize size_log2) const
 {
   BATT_CHECK_LT(size_log2, kMaxPageSizeLog2);
 
@@ -533,7 +539,7 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id)
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-PageCache::PageDeviceEntry* PageCache::get_device_for_page(PageId page_id)
+PageDeviceEntry* PageCache::get_device_for_page(PageId page_id)
 {
   const page_device_id_int device_id = PageIdFactory::get_device_id(page_id);
   if (BATT_HINT_FALSE(device_id >= this->page_devices_.size())) {