Skip to content

Commit

Permalink
blockstore: Update flaky compaction test (#5131)
Browse files Browse the repository at this point in the history 
Call compact_range_cf() with the begin/end keys set to None in order to
compact the entire range

(cherry picked from commit 760b6fc)
  • Loading branch information
@steviez @mergify
steviez authored and mergify[bot] committed Mar 5, 2025
1 parent f8de613 commit 1adfbfc
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 73 deletions.
58 changes: 6 additions & 52 deletions ledger/src/blockstore/blockstore_purge.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,38 +803,15 @@ pub mod tests {
assert_eq!(status_entry_iterator.next(), None);
}

fn get_index_bounds(
blockstore: &Blockstore,
) -> (
<cf::TransactionStatus as Column>::Key,
<cf::TransactionStatus as Column>::Key,
) {
let (first_index, _value) = blockstore
.transaction_status_cf
.iterator_cf_raw_key(IteratorMode::Start)
.next()
.unwrap();
let (last_index, _value) = blockstore
.transaction_status_cf
.iterator_cf_raw_key(IteratorMode::End)
.next()
.unwrap();

(first_index, last_index)
}

fn purge_exact(blockstore: &Blockstore, oldest_slot: Slot) {
blockstore
.run_purge(0, oldest_slot - 1, PurgeType::Exact)
.unwrap();
}

fn purge_compaction_filter(blockstore: &Blockstore, oldest_slot: Slot) {
let (first_index, last_index) = get_index_bounds(blockstore);
blockstore.db.set_oldest_slot(oldest_slot);
blockstore
.transaction_status_cf
.compact_range_raw_key(&first_index, &last_index);
blockstore.transaction_status_cf.compact();
}

#[test_case(purge_exact; "exact")]
Expand Down Expand Up @@ -994,13 +971,10 @@ pub mod tests {
let max_slot = 19;

clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot);
let (first_index, last_index) = get_index_bounds(&blockstore);

let oldest_slot = 3;
blockstore.db.set_oldest_slot(oldest_slot);
blockstore
.transaction_status_cf
.compact_range_raw_key(&first_index, &last_index);
blockstore.transaction_status_cf.compact();

let status_entry_iterator = blockstore
.transaction_status_cf
Expand All @@ -1014,13 +988,10 @@ pub mod tests {
assert_eq!(count, max_slot - (oldest_slot - 1));

clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot);
let (first_index, last_index) = get_index_bounds(&blockstore);

let oldest_slot = 12;
blockstore.db.set_oldest_slot(oldest_slot);
blockstore
.transaction_status_cf
.compact_range_raw_key(&first_index, &last_index);
blockstore.transaction_status_cf.compact();

let status_entry_iterator = blockstore
.transaction_status_cf
Expand Down Expand Up @@ -1076,22 +1047,9 @@ pub mod tests {
)
.unwrap();

let (first_index, _value) = blockstore
.transaction_memos_cf
.iterator_cf_raw_key(IteratorMode::Start)
.next()
.unwrap();
let (last_index, _value) = blockstore
.transaction_memos_cf
.iterator_cf_raw_key(IteratorMode::End)
.next()
.unwrap();

// Purge at slot 0 should not affect any memos
blockstore.db.set_oldest_slot(0);
blockstore
.transaction_memos_cf
.compact_range_raw_key(&first_index, &last_index);
blockstore.transaction_memos_cf.compact();
let num_memos = blockstore
.transaction_memos_cf
.iter(IteratorMode::Start)
Expand All @@ -1101,9 +1059,7 @@ pub mod tests {

// Purge at oldest_slot without clean_slot_0 only purges the current memo at slot 4
blockstore.db.set_oldest_slot(oldest_slot);
blockstore
.transaction_memos_cf
.compact_range_raw_key(&first_index, &last_index);
blockstore.transaction_memos_cf.compact();
let memos_iterator = blockstore
.transaction_memos_cf
.iter(IteratorMode::Start)
Expand All @@ -1117,9 +1073,7 @@ pub mod tests {

// Purge at oldest_slot with clean_slot_0 purges deprecated memos
blockstore.db.set_clean_slot_0(true);
blockstore
.transaction_memos_cf
.compact_range_raw_key(&first_index, &last_index);
blockstore.transaction_memos_cf.compact();
let memos_iterator = blockstore
.transaction_memos_cf
.iter(IteratorMode::Start)
Expand Down
37 changes: 16 additions & 21 deletions ledger/src/blockstore_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -659,10 +659,22 @@ where
}

#[cfg(test)]
pub fn compact_range_raw_key(&self, from: &[u8], to: &[u8]) {
self.backend
.db
.compact_range_cf(self.handle(), Some(from), Some(to));
// The validator performs compactions asynchronously, this method is
// provided to force a synchronous compaction to test our compaction filter
pub fn compact(&self) {
// compact_range_cf() optionally takes a start and end key to limit
// compaction. Providing values will result in a different method
// getting called in the rocksdb code, even if the specified keys span
// the entire key range of the column
//
// Internally, rocksdb will do some checks to figure out if it should
// run a compaction. Empirically, it has been found that passing the
// keys leads to more variability in whether rocksdb runs a compaction
// or not. For the sake of our unit tests, we want the compaction to
// run everytime. So, set the keys as None which will result in rocksdb
// using the heavier method to determine if a compaction should run
let (start, end) = (None::<&[u8]>, None::<&[u8]>);
self.backend.db.compact_range_cf(self.handle(), start, end);
}

#[inline]
Expand Down Expand Up @@ -1386,21 +1398,4 @@ pub mod tests {
.put_cf(self.handle(), C::deprecated_key(index), &serialized_value)
}
}

impl<C> LedgerColumn<C>
where
C: ColumnIndexDeprecation + ColumnName,
{
pub(crate) fn iterator_cf_raw_key(
&self,
iterator_mode: IteratorMode<C::Index>,
) -> impl Iterator<Item = (C::Key, Box<[u8]>)> + '_ {
// The conversion of key back into Box<[u8]> incurs an extra
// allocation. However, this is test code and the goal is to
// maximize code reuse over efficiency
self.iter(iterator_mode)
.unwrap()
.map(|(key, value)| (C::key(&key), value))
}
}
}

0 comments on commit 1adfbfc

Please sign in to comment.