Skip to content

Commit

Permalink
Merge pull request #53062 from xxhdx1985126/wip-seastore-list-objects
Browse files Browse the repository at this point in the history
crimson/os/seastore: return ghobject_t::max as the end when list_objects reaches the end of the listing

Reviewed-by: Yingxin Cheng <[email protected]>
Reviewed-by: Chunmei Liu <[email protected]>
  • Loading branch information
cyx1231st authored Oct 12, 2023
2 parents 3a9a5e8 + e989538 commit b044a96
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ FLTreeOnodeManager::list_onodes_ret FLTreeOnodeManager::list_onodes(
const ghobject_t& end,
uint64_t limit)
{
LOG_PREFIX(FLTreeOnodeManager::list_onodes);
DEBUGT("start {}, end {}, limit {}", trans, start, end, limit);
return tree.lower_bound(trans, start
).si_then([this, &trans, end, limit] (auto&& cursor) {
using crimson::os::seastore::onode::full_key_t;
Expand All @@ -145,21 +147,28 @@ FLTreeOnodeManager::list_onodes_ret FLTreeOnodeManager::list_onodes(
return trans_intr::repeat(
[this, &trans, end, &to_list, &current_cursor, &ret] ()
-> eagain_ifuture<seastar::stop_iteration> {
LOG_PREFIX(FLTreeOnodeManager::list_onodes);
if (current_cursor.is_end()) {
DEBUGT("reached the onode tree end", trans);
std::get<1>(ret) = ghobject_t::get_max();
return seastar::make_ready_future<seastar::stop_iteration>(
seastar::stop_iteration::yes);
} else if (current_cursor.get_ghobj() >= end) {
DEBUGT("reached the end {} > {}",
trans, current_cursor.get_ghobj(), end);
std::get<1>(ret) = end;
return seastar::make_ready_future<seastar::stop_iteration>(
seastar::stop_iteration::yes);
}
if (to_list == 0) {
DEBUGT("reached the limit", trans);
std::get<1>(ret) = current_cursor.get_ghobj();
return seastar::make_ready_future<seastar::stop_iteration>(
seastar::stop_iteration::yes);
}
std::get<0>(ret).emplace_back(current_cursor.get_ghobj());
auto ghobj = current_cursor.get_ghobj();
DEBUGT("found onode for {}", trans, ghobj);
std::get<0>(ret).emplace_back(std::move(ghobj));
return tree.get_next(trans, current_cursor
).si_then([&to_list, &current_cursor] (auto&& next_cursor) mutable {
// we intentionally hold the current_cursor during get_next() to
Expand Down
20 changes: 16 additions & 4 deletions src/crimson/os/seastore/seastore.cc
Original file line number Diff line number Diff line change
Expand Up @@ -688,11 +688,14 @@ SeaStore::Shard::list_objects(CollectionRef ch,
std::vector<ghobject_t>(),
ghobject_t::get_max()));
} else {
LOG_PREFIX(SeaStore::list_objects);
DEBUGT("start {}, end {}, limit {}, bits {}",
t, start, end, limit, *bits);
auto filter = SeaStore::get_objs_range(ch, *bits);
using list_iertr = OnodeManager::list_onodes_iertr;
using repeat_ret = list_iertr::future<seastar::stop_iteration>;
return trans_intr::repeat(
[this, &t, &ret, &limit,
[this, &t, &ret, &limit, end,
filter, ranges = get_ranges(ch, start, end, filter)
]() mutable -> repeat_ret {
if (limit == 0 || ranges.empty()) {
Expand All @@ -704,9 +707,12 @@ SeaStore::Shard::list_objects(CollectionRef ch,
auto pstart = ite->first;
auto pend = ite->second;
ranges.pop_front();
LOG_PREFIX(SeaStore::list_objects);
DEBUGT("pstart {}, pend {}, limit {}", t, pstart, pend, limit);
return onode_manager->list_onodes(
t, pstart, pend, limit
).si_then([&limit, &ret, pend](auto &&_ret) mutable {
).si_then([&limit, &ret, pend, &t, last=ranges.empty(), end]
(auto &&_ret) mutable {
auto &next_objects = std::get<0>(_ret);
auto &ret_objects = std::get<0>(ret);
ret_objects.insert(
Expand All @@ -716,9 +722,15 @@ SeaStore::Shard::list_objects(CollectionRef ch,
std::get<1>(ret) = std::get<1>(_ret);
assert(limit >= next_objects.size());
limit -= next_objects.size();
LOG_PREFIX(SeaStore::list_objects);
DEBUGT("got {} objects, left limit {}",
t, next_objects.size(), limit);
if (last && std::get<1>(ret) == pend) {
std::get<1>(ret) = end;
}
assert(limit == 0 ||
std::get<1>(_ret) == pend ||
std::get<1>(_ret) == ghobject_t::get_max());
std::get<1>(ret) == pend ||
std::get<1>(ret) == ghobject_t::get_max());
return list_iertr::make_ready_future<
seastar::stop_iteration
>(seastar::stop_iteration::no);
Expand Down
2 changes: 2 additions & 0 deletions src/crimson/os/seastore/seastore.h
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ class SeaStore final : public FuturizedStore {
CollectionRef c,
const ghobject_t& oid) final;

/// std::get<1>(ret) returns end if and only if the listing has listed all
/// the items within the range, otherwise it returns the next key to be listed.
seastar::future<std::tuple<std::vector<ghobject_t>, ghobject_t>> list_objects(
CollectionRef c,
const ghobject_t& start,
Expand Down
2 changes: 1 addition & 1 deletion src/test/crimson/seastore/test_seastore.cc
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,7 @@ struct seastore_test_t :
EXPECT_GE(next, right_bound);
} else {
// next <= *correct_end since *correct_end is the next object to list
EXPECT_LE(next, *correct_end);
EXPECT_LE(listed.back(), *correct_end);
// next > *(correct_end - 1) since we already listed it
EXPECT_GT(next, *(correct_end - 1));
}
Expand Down

0 comments on commit b044a96

Please sign in to comment.