CachingPageTracer Optimization

src/llfs/api_types.hpp

@@ -81,6 +81,8 @@ BATT_STRONG_TYPEDEF(usize, BufferCount);
  */
 BATT_STRONG_TYPEDEF(usize, BufferSize);
 
+BATT_STRONG_TYPEDEF(bool, HasNoOutgoingRefs);
+
 }  // namespace llfs
 
 #endif  // LLFS_API_TYPES_HPP

src/llfs/committable_page_cache_job.cpp

@@ -88,11 +88,8 @@ usize CommittablePageCacheJob::new_page_count() const noexcept
 //
 BoxedSeq<PageId> CommittablePageCacheJob::deleted_page_ids() const
 {
-  return as_seq(this->job_->get_deleted_pages().begin(), this->job_->get_deleted_pages().end())  //
-         | seq::map([](const auto& kv_pair) -> PageId {
-             return kv_pair.first;
-           })  //
-         | seq::boxed();
+  return BoxedSeq<PageId>{
+      as_seq(this->job_->get_deleted_pages().begin(), this->job_->get_deleted_pages().end())};
 }
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
@@ -482,7 +479,7 @@ Status CommittablePageCacheJob::await_ref_count_updates(const PageRefCountUpdate
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-auto CommittablePageCacheJob::get_page_ref_count_updates(u64 /*callers*/) const
+auto CommittablePageCacheJob::get_page_ref_count_updates(u64 /*callers*/)
     -> StatusOr<PageRefCountUpdates>
 {
   std::unordered_map<PageId, i32, PageId::Hash> ref_count_delta = this->job_->get_root_set_delta();
@@ -512,19 +509,33 @@ auto CommittablePageCacheJob::get_page_ref_count_updates(u64 /*callers*/) const
   // Trace deleted pages non-recursively, decrementing the ref counts of all pages they directly
   // reference.
   //
+  LoadingPageTracer loading_tracer{loader, true};
+  CachingPageTracer caching_tracer{this->job_->cache().devices_by_id(), loading_tracer};
   for (const auto& p : this->job_->get_deleted_pages()) {
-    // Sanity check; deleted pages should have a ref_count_delta of kRefCount_1_to_0.
-    //
-    const PageId deleted_page_id = p.first;
-    {
-      auto iter = ref_count_delta.find(deleted_page_id);
-      BATT_CHECK_NE(iter, ref_count_delta.end());
-      BATT_CHECK_EQ(iter->second, kRefCount_1_to_0);
-    }
+    const PageId deleted_page_id = p;
 
     // Decrement ref counts.
     //
-    p.second->trace_refs() | seq::for_each([&ref_count_delta, deleted_page_id](PageId id) {
+    batt::StatusOr<batt::BoxedSeq<PageId>> outgoing_refs =
+        caching_tracer.trace_page_refs(deleted_page_id);
+    if (outgoing_refs.status() == batt::StatusCode::kNotFound) {
+      continue;
+    }
+    BATT_REQUIRE_OK(outgoing_refs);
+
+    ref_count_delta[p] = kRefCount_1_to_0;
+    // TODO [vsilai 2024-10-28] not sure if finalized_deleted_pages_ is needed. Given how we
+    // restructured this function and `delete_page` in PageCacheJob, I think that we have to
+    // ensure that we don't accidentally drop a page that wasn't found when `trace_page_refs` was
+    // called above, because we keep this page id in the job's deleted_pages_ set and simply call
+    // continue in the loop. Before, `drop_pages` looked into the job's deleted_pages_ set to create
+    // a vector of all the ids to drop, which may cause this accidental drop. However, since drops
+    // should be idempotent (as described by the comment on line 275 of this file), we may not need
+    // this...
+    //
+    this->finalized_deleted_pages_.insert(p);
+
+    *outgoing_refs | seq::for_each([&ref_count_delta, deleted_page_id](PageId id) {
       if (id) {
         LLFS_VLOG(1) << " decrementing ref count for page " << id
                      << " (because it was referenced from deleted page " << deleted_page_id << ")";
@@ -590,19 +601,21 @@ Status CommittablePageCacheJob::recycle_dead_pages(const JobCommitParams& params
   // the page recycler's log.
 }
 
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+BoxedSeq<PageId> CommittablePageCacheJob::finalized_deleted_page_ids() const
+{
+  return BoxedSeq<PageId>{
+      as_seq(this->finalized_deleted_pages_.begin(), this->finalized_deleted_pages_.end())};
+}
+
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
 Status CommittablePageCacheJob::drop_deleted_pages(u64 callers)
 {
   LLFS_VLOG(1) << "commit(PageCacheJob): dropping deleted pages";
 
-  const auto& deleted_pages = this->job_->get_deleted_pages();
-
-  return parallel_drop_pages(as_seq(deleted_pages.begin(), deleted_pages.end())  //
-                                 | seq::map([](const auto& kv_pair) -> PageId {
-                                     return kv_pair.first;
-                                   })  //
-                                 | seq::collect_vec(),
+  return parallel_drop_pages(this->finalized_deleted_page_ids() | seq::collect_vec(),
                              this->job_->cache(), this->job_->job_id, callers);
 }
 

src/llfs/committable_page_cache_job.hpp

@@ -193,7 +193,7 @@ class CommittablePageCacheJob
 
   Status write_new_pages();
 
-  StatusOr<PageRefCountUpdates> get_page_ref_count_updates(u64 callers) const;
+  StatusOr<PageRefCountUpdates> get_page_ref_count_updates(u64 callers);
 
   StatusOr<DeadPages> start_ref_count_updates(const JobCommitParams& params,
                                               PageRefCountUpdates& updates, u64 callers);
@@ -206,12 +206,15 @@ class CommittablePageCacheJob
 
   Status drop_deleted_pages(u64 callers);
 
+  BoxedSeq<PageId> finalized_deleted_page_ids() const;
+
   //+++++++++++-+-+--+----- --- -- -  -  -   -
 
   std::shared_ptr<const PageCacheJob> job_;
   boost::intrusive_ptr<FinalizedJobTracker> tracker_;
   PageRefCountUpdates ref_count_updates_;
   std::unique_ptr<WriteNewPagesContext> write_new_pages_context_;
+  std::unordered_set<PageId, PageId::Hash> finalized_deleted_pages_;
 };
 
 /** \brief Write all changes in `job` to durable storage.  This is guaranteed to be atomic.

src/llfs/no_outgoing_refs_cache.cpp

@@ -0,0 +1,59 @@
+//#=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++
+//
+// Part of the LLFS Project, under Apache License v2.0.
+// See https://www.apache.org/licenses/LICENSE-2.0 for license information.
+// SPDX short identifier: Apache-2.0
+//
+//+++++++++++-+-+--+----- --- -- -  -  -   -
+#include <llfs/no_outgoing_refs_cache.hpp>
+
+namespace llfs {
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+NoOutgoingRefsCache::NoOutgoingRefsCache(u64 physical_page_count) noexcept
+    : cache_(physical_page_count)
+{
+}
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+void NoOutgoingRefsCache::set_page_bits(i64 physical_page_id, page_generation_int generation,
+                                        HasNoOutgoingRefs has_no_out_going_refs)
+{
+  BATT_CHECK_LT((usize)physical_page_id, this->cache_.size());
+
+  u64 new_state = generation << this->generation_shift_;
+  // Set the "valid" bit to 1.
+  //
+  new_state |= (u64{1} << 1);
+  if (has_no_out_going_refs) {
+    // Set the "has no outgoing references" bit to 1.
+    //
+    new_state |= u64{1};
+  }
+
+  u64 old_state = this->cache_[physical_page_id].exchange(new_state, std::memory_order_acq_rel);
+  // Sanity check: we are not setting the bits for the same generation more than once.
+  //
+  page_generation_int old_generation = old_state >> this->generation_shift_;
+  BATT_CHECK_NE(generation, old_generation);
+}
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+u64 NoOutgoingRefsCache::get_page_bits(i64 physical_page_id, page_generation_int generation) const
+{
+  BATT_CHECK_LT((usize)physical_page_id, this->cache_.size());
+
+  u64 current_state = this->cache_[physical_page_id].load(std::memory_order_acquire);
+  page_generation_int stored_generation = current_state >> this->generation_shift_;
+  // If the generation that is currently stored in the cache is not the same as the generation we
+  // are querying for, this cache entry is invalid. Thus, we return a "not traced" status.
+  //
+  if (stored_generation != generation) {
+    return u64{0};
+  }
+  return current_state & this->bit_mask_;
+}
+}  // namespace llfs

src/llfs/no_outgoing_refs_cache.hpp

@@ -0,0 +1,58 @@
+//#=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++
+//
+// Part of the LLFS Project, under Apache License v2.0.
+// See https://www.apache.org/licenses/LICENSE-2.0 for license information.
+// SPDX short identifier: Apache-2.0
+//
+//+++++++++++-+-+--+----- --- -- -  -  -   -
+
+#pragma once
+#ifndef LLFS_NO_OUTGOING_REFS_CACHE_HPP
+#define LLFS_NO_OUTGOING_REFS_CACHE_HPP
+
+#include <llfs/api_types.hpp>
+#include <llfs/page_id_factory.hpp>
+
+#include <atomic>
+#include <vector>
+
+namespace llfs {
+//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+---------------
+/** \brief A cache to store information about a page's outgoing references to other pages. Can be
+ * used by an implementer of PageTracer as a way to organize and look up this information on the
+ * PageDevice level. This cache is implemented as a vector of unsigned 64-bit integers, where every
+ * element of the vector represents the outgoing refs "state" of a physical page in a PageDevice.
+ * The lowest bit in an element represents if the page has no outgoing refs. The second lowest bit
+ * represents the validity of the page's state to help determine if a page's outgoing refs have ever
+ * been traced. The remaining upper 62 bits are used to store the generation of the physical page.
+ */
+class NoOutgoingRefsCache
+{
+ public:
+  explicit NoOutgoingRefsCache(u64 physical_page_count) noexcept;
+
+  NoOutgoingRefsCache(const NoOutgoingRefsCache&) = delete;
+  NoOutgoingRefsCache& operator=(const NoOutgoingRefsCache&) = delete;
+
+  //----- --- -- -  -  -   -
+  /** \brief Sets the two outgoing refs state bits for the given `physical_page_id` based on
+   * whether the page has outgoing refs or not, as indicated by `has_outgoing_refs`.
+   */
+  void set_page_bits(i64 physical_page_id, page_generation_int generation,
+                     HasNoOutgoingRefs has_no_outgoing_refs);
+
+  //----- --- -- -  -  -   -
+  /** \brief Retrieves the two outgoing refs state bits for the given `physical_page_id`.
+   *
+   * \return Can return a value of 0 (00), 2 (10), or 3 (11).
+   */
+  u64 get_page_bits(i64 physical_page_id, page_generation_int generation) const;
+
+ private:
+  std::vector<std::atomic<u64>> cache_;
+  static constexpr u64 bit_mask_ = 0b11;
+  static constexpr u8 generation_shift_ = 2;
+};
+}  // namespace llfs
+
+#endif  // LLFS_NO_OUTGOING_REFS_CACHE_HPP

src/llfs/page_cache.cpp

@@ -22,7 +22,7 @@ namespace llfs {
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-usize get_page_size(const PageCache::PageDeviceEntry* entry)
+usize get_page_size(const PageDeviceEntry* entry)
 {
   return entry ? get_page_size(entry->arena) : 0;
 }
@@ -420,14 +420,21 @@ void PageCache::prefetch_hint(PageId page_id)
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-Slice<PageCache::PageDeviceEntry* const> PageCache::all_devices() const
+const std::vector<std::unique_ptr<PageDeviceEntry>>& PageCache::devices_by_id() const
+{
+  return this->page_devices_;
+}
+
+//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
+//
+Slice<PageDeviceEntry* const> PageCache::all_devices() const
 {
   return as_slice(this->page_devices_by_page_size_);
 }
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-Slice<PageCache::PageDeviceEntry* const> PageCache::devices_with_page_size(usize size) const
+Slice<PageDeviceEntry* const> PageCache::devices_with_page_size(usize size) const
 {
   const usize size_log2 = batt::log2_ceil(size);
   BATT_CHECK_EQ(size, usize{1} << size_log2) << "page size must be a power of 2";
@@ -437,8 +444,7 @@ Slice<PageCache::PageDeviceEntry* const> PageCache::devices_with_page_size(usize
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-Slice<PageCache::PageDeviceEntry* const> PageCache::devices_with_page_size_log2(
-    usize size_log2) const
+Slice<PageDeviceEntry* const> PageCache::devices_with_page_size_log2(usize size_log2) const
 {
   BATT_CHECK_LT(size_log2, kMaxPageSizeLog2);
 
@@ -533,7 +539,7 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id)
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -
 //
-PageCache::PageDeviceEntry* PageCache::get_device_for_page(PageId page_id)
+PageDeviceEntry* PageCache::get_device_for_page(PageId page_id)
 {
   const page_device_id_int device_id = PageIdFactory::get_device_id(page_id);
   if (BATT_HINT_FALSE(device_id >= this->page_devices_.size())) {

src/llfs/page_cache.hpp

@@ -19,12 +19,11 @@
 #include <llfs/metrics.hpp>
 #include <llfs/optional.hpp>
 #include <llfs/page_allocator.hpp>
-#include <llfs/page_arena.hpp>
 #include <llfs/page_buffer.hpp>
 #include <llfs/page_cache_metrics.hpp>
 #include <llfs/page_cache_options.hpp>
 #include <llfs/page_device.hpp>
-#include <llfs/page_device_cache.hpp>
+#include <llfs/page_device_entry.hpp>
 #include <llfs/page_filter.hpp>
 #include <llfs/page_id_slot.hpp>
 #include <llfs/page_loader.hpp>
@@ -103,26 +102,6 @@ class PageCache : public PageLoader
     int line;
   };
 
-  /** \brief All the per-PageDevice state for a single device in the storage pool.
-   */
-  struct PageDeviceEntry {
-    explicit PageDeviceEntry(PageArena&& arena,
-                             boost::intrusive_ptr<PageCacheSlot::Pool>&& slot_pool) noexcept
-        : arena{std::move(arena)}
-        , cache{this->arena.device().page_ids(), std::move(slot_pool)}
-    {
-    }
-
-    /** \brief The PageDevice and PageAllocator.
-     */
-    PageArena arena;
-
-    /** \brief A per-device page cache; shares a PageCacheSlot::Pool with all other PageDeviceEntry
-     * objects that have the same page size.
-     */
-    PageDeviceCache cache;
-  };
-
   class PageDeleterImpl : public PageDeleter
   {
    public:
@@ -195,6 +174,8 @@ class PageCache : public PageLoader
 
   Slice<PageDeviceEntry* const> all_devices() const;
 
+  const std::vector<std::unique_ptr<PageDeviceEntry>>& devices_by_id() const;
+
   const PageArena& arena_for_page_id(PageId id_val) const;
 
   const PageArena& arena_for_device_id(page_device_id_int device_id_val) const;

src/llfs/page_cache_job.cpp

@@ -356,17 +356,10 @@ Status PageCacheJob::delete_page(PageId page_id)
   if (!page_id) {
     return OkStatus();
   }
-  StatusOr<PinnedPage> page_view = this->get_page(page_id, OkIfNotFound{true});
-  if (page_view.ok()) {
-    this->pruned_ = false;
-    this->deleted_pages_.emplace(page_id, *page_view);
-    this->root_set_delta_[page_id] = kRefCount_1_to_0;
-    return OkStatus();
-  }
-  if (page_view.status() == batt::StatusCode::kNotFound) {
-    return OkStatus();
-  }
-  return page_view.status();
+
+  this->deleted_pages_.insert(page_id);
+  this->pruned_ = false;
+  return OkStatus();
 }
 
 //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- -  -  -   -