Skip to content

Commit

Permalink
More tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@mm304321141
mm304321141 committed Feb 25, 2022
1 parent db1fe8c commit b382632
Show file tree
Hide file tree
Showing 5 changed files with 197 additions and 137 deletions.
271 changes: 187 additions & 84 deletions db/db_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5744,30 +5744,44 @@ TEST_F(DBTest, SkipValueGet) {

TEST_F(DBTest, ForceEvict) {
Options options = CurrentOptions();
CompressionType compressions[] = {kZlibCompression, kBZip2Compression,
kLZ4Compression, kLZ4HCCompression,
kXpressCompression};

BlockBasedTableOptions bbto;
bbto.cache_index_and_filter_blocks = false;
bbto.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
bbto.filter_policy.reset(TERARKDB_NAMESPACE::NewBloomFilterPolicy(10));
bbto.partition_filters = true;
bbto.block_cache = TERARKDB_NAMESPACE::NewLRUCache(64ull << 20, 4);
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
options.statistics = TERARKDB_NAMESPACE::CreateDBStatistics();
options.optimize_filters_for_hits = false;
options.target_file_size_base = 1;
options.disable_auto_compactions = true;
options.compression = kNoCompression;

int evict = 0;
int force_evict = 0;
int found = 0;
int open = 0;
int run = 0;
int failures = 0;
int TC_evict = 0;
int TC_force_evict = 0;
int TC_found = 0;
int TC_open = 0;
int TC_run = 0;
int BC_erase = 0;
int BC_failures = 0;

int test_sst = 40;
int max_open_files = 20;

SyncPoint::GetInstance()->SetCallBack("TableCache::Evict",
[&](void* /* arg */) { ++evict; });
[&](void* /* arg */) { ++TC_evict; });
SyncPoint::GetInstance()->SetCallBack(
"TableCache::ForceEvict", [&](void* /* arg */) { ++force_evict; });
"TableCache::ForceEvict", [&](void* /* arg */) { ++TC_force_evict; });
SyncPoint::GetInstance()->SetCallBack("TableCache::ForceEvict:Found",
[&](void* /* arg */) { ++found; });
[&](void* /* arg */) { ++TC_found; });
SyncPoint::GetInstance()->SetCallBack("TableCache::ForceEvict:Open",
[&](void* /* arg */) { ++open; });
[&](void* /* arg */) { ++TC_open; });
SyncPoint::GetInstance()->SetCallBack("TableCache::ForceEvict:Run",
[&](void* /* arg */) { ++run; });
[&](void* /* arg */) { ++TC_run; });
SyncPoint::GetInstance()->EnableProcessing();

auto do_something = [&] {
Expand All @@ -5778,106 +5792,195 @@ TEST_F(DBTest, ForceEvict) {
for (int i = 0; i < 40; ++i) {
ASSERT_OK(db_->Get(ReadOptions{}, db_->DefaultColumnFamily(), Key(i)));
}
evict = 0;
force_evict = 0;
found = 0;
open = 0;
run = 0;
TC_evict = 0;
TC_force_evict = 0;
TC_found = 0;
TC_open = 0;
TC_run = 0;
TestGetAndResetTickerCount(options, TERARKDB_NAMESPACE::BLOCK_CACHE_ERASE);
TestGetAndResetTickerCount(options,
TERARKDB_NAMESPACE::BLOCK_CACHE_ERASE_FAILURES);

ASSERT_OK(db_->CompactRange(TERARKDB_NAMESPACE::CompactRangeOptions(),
db_->DefaultColumnFamily(), nullptr, nullptr));
for (int i = 0; i < 40; ++i) {
ASSERT_OK(db_->Get(ReadOptions{}, db_->DefaultColumnFamily(), Key(i)));
}

failures = TestGetAndResetTickerCount(
options, TERARKDB_NAMESPACE::BLOCK_CACHE_ERASE) -
TestGetAndResetTickerCount(
options, TERARKDB_NAMESPACE::BLOCK_CACHE_ERASE_FAILURES);

// fprintf(stderr,
// "evict = %d;"
// "force_evict = %d;"
// "found = %d;"
// "open = %d;"
// "run = %d;"
// "failures = %d;\n",
// evict, force_evict, found, open, run, failures);
BC_erase = TestGetAndResetTickerCount(
options, TERARKDB_NAMESPACE::BLOCK_CACHE_ERASE);
BC_failures = TestGetAndResetTickerCount(
options, TERARKDB_NAMESPACE::BLOCK_CACHE_ERASE_FAILURES);

fprintf(stderr,
"TC_evict = %d;"
"TC_force_evict = %d;"
"TC_found = %d;"
"TC_open = %d;"
"TC_run = %d;"
"BC_erase = %d;"
"BC_failures = %d;\n",
TC_evict, TC_force_evict, TC_found, TC_open, TC_run, BC_erase,
BC_failures);
};

options.table_evict_type = TERARKDB_NAMESPACE::kSkipForceEvict;
options.max_open_files = -1;
DestroyAndReopen(options);
do_something();
// always evict all
ASSERT_EQ(evict, 40);
// no try force_evict
ASSERT_EQ(force_evict, 0);
// no foound
ASSERT_EQ(found, 0);
// no open
ASSERT_EQ(open, 0);
// no run force_evict
ASSERT_EQ(run, 0);
// no cache evict failures
ASSERT_EQ(failures, 0);
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, 0);
ASSERT_EQ(TC_found, 0);
ASSERT_EQ(TC_open, 0);
ASSERT_EQ(TC_run, 0);
// block_cache erase:
// index_block(N)
// filter_block(N)
// filter_block_partitions(N)
ASSERT_EQ(BC_erase, test_sst * 3);
// block_cache failures:
// index_block(N)
// filter_block(N)
ASSERT_EQ(BC_failures, test_sst * 2);

options.table_evict_type = TERARKDB_NAMESPACE::kForceEvictIfOpen;
options.max_open_files = 20;
options.max_open_files = max_open_files;
DestroyAndReopen(options);
do_something();
// always evict all
ASSERT_EQ(evict, 40);
// try force_evict all
ASSERT_EQ(force_evict, 40);
// found 20+ (for max_open_files)
ASSERT_GE(found, 20);
ASSERT_LT(found, 40);
// no open
ASSERT_EQ(open, 0);
// run force_evict 20+ (for max_open_files)
ASSERT_GE(run, 20);
ASSERT_LT(run, 40);
// cache evict failures 20+ (for max_open_files)
ASSERT_GE(failures, 20);
ASSERT_LT(failures, 40);
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, test_sst);
ASSERT_GE(TC_found, max_open_files);
ASSERT_LT(TC_found, test_sst);
ASSERT_EQ(TC_open, 0);
ASSERT_EQ(TC_run, TC_found);
// block_cache erase:
// index_block(N+F)
// index_block_partitions(F)
// filter_block(N)
// filter_block_partitions(N)
// data_blocks(F)
ASSERT_EQ(BC_erase, test_sst * 3 + TC_found * 3);
// block_cache failures:
// index_block(N+F)
// filter_block(N)
ASSERT_EQ(BC_failures, test_sst * 2 + TC_found);

options.table_evict_type = TERARKDB_NAMESPACE::kForceEvictIfOpen;
options.max_open_files = -1;
DestroyAndReopen(options);
do_something();
// always evict all
ASSERT_EQ(evict, 40);
// try force_evict all
ASSERT_EQ(force_evict, 40);
// found all
ASSERT_EQ(found, 40);
// no open
ASSERT_EQ(open, 0);
// run all force_evict
ASSERT_EQ(run, 40);
// cache evict failures all
ASSERT_EQ(failures, 40);
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, test_sst);
ASSERT_EQ(TC_found, test_sst);
ASSERT_EQ(TC_open, 0);
ASSERT_EQ(TC_run, test_sst);
// block_cache erase:
// index_block(2N)
// index_block_partitions(N)
// filter_block(N)
// filter_block_partitions(N)
// data_blocks(N)
ASSERT_EQ(BC_erase, test_sst * 6);
// block_cache failures:
// index_block(2N)
// filter_block(N)
ASSERT_EQ(BC_failures, test_sst * 3);

options.table_evict_type = TERARKDB_NAMESPACE::kAlwaysForceEvict;
options.max_open_files = 20;
options.max_open_files = max_open_files;
DestroyAndReopen(options);
do_something();
// always evict all
ASSERT_EQ(evict, 40);
// try force_evict all
ASSERT_EQ(force_evict, 40);
// found 20+ (for max_open_files)
ASSERT_GE(found, 20);
ASSERT_LT(found, 40);
// open 20- (for max_open_files)
ASSERT_GT(open, 0);
ASSERT_LE(open, 20);
// run all force_evict
ASSERT_EQ(run, 40);
// cache evict failures all
ASSERT_EQ(failures, 40);
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, test_sst);
ASSERT_GE(TC_found, max_open_files);
ASSERT_LT(TC_found, test_sst);
ASSERT_EQ(TC_open, test_sst - TC_found);
ASSERT_EQ(TC_run, test_sst);
// block_cache erase:
// index_block(2N+O)
// index_block_partitions(N)
// filter_block(N+O)
// filter_block_partitions(N+O)
// data_blocks(N)
ASSERT_EQ(BC_erase, test_sst * 6 + TC_open * 3);
// block_cache failures:
// index_block(2N+O)
// filter_block(N+O)
ASSERT_EQ(BC_failures, test_sst * 3 + TC_open * 2);

for (auto comp : compressions) {
if (CompressionTypeSupported(comp)) {
options.compression = comp;
break;
}
}
if (options.compression == kNoCompression) {
return;
}
bbto.no_block_cache = true;
bbto.block_cache_compressed.swap(bbto.block_cache);
options.table_factory.reset(NewBlockBasedTableFactory(bbto));

options.table_evict_type = TERARKDB_NAMESPACE::kSkipForceEvict;
options.max_open_files = -1;
DestroyAndReopen(options);
do_something();
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, 0);
ASSERT_EQ(TC_found, 0);
ASSERT_EQ(TC_open, 0);
ASSERT_EQ(TC_run, 0);
ASSERT_EQ(BC_erase, 0);
ASSERT_EQ(BC_failures, 0);

options.table_evict_type = TERARKDB_NAMESPACE::kForceEvictIfOpen;
options.max_open_files = max_open_files;
DestroyAndReopen(options);
do_something();
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, test_sst);
ASSERT_GE(TC_found, max_open_files);
ASSERT_LT(TC_found, test_sst);
ASSERT_EQ(TC_open, 0);
ASSERT_EQ(TC_run, TC_found);
// block_cache_compressed erase:
// index_block_partitions(F)
// data_blocks(F)
ASSERT_EQ(BC_erase, TC_found * 2);
ASSERT_EQ(BC_failures, 0);

options.table_evict_type = TERARKDB_NAMESPACE::kForceEvictIfOpen;
options.max_open_files = -1;
DestroyAndReopen(options);
do_something();
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, test_sst);
ASSERT_EQ(TC_found, test_sst);
ASSERT_EQ(TC_open, 0);
ASSERT_EQ(TC_run, test_sst);
// block_cache_compressed erase:
// index_block_partitions(N)
// data_blocks(N)
ASSERT_EQ(BC_erase, test_sst * 2);
ASSERT_EQ(BC_failures, 0);

options.table_evict_type = TERARKDB_NAMESPACE::kAlwaysForceEvict;
options.max_open_files = max_open_files;
DestroyAndReopen(options);
do_something();
ASSERT_EQ(TC_evict, test_sst);
ASSERT_EQ(TC_force_evict, test_sst);
ASSERT_GE(TC_found, max_open_files);
ASSERT_LT(TC_found, test_sst);
ASSERT_EQ(TC_open, test_sst - TC_found);
ASSERT_EQ(TC_run, test_sst);
// block_cache_compressed erase:
// index_block_partitions(N)
// data_blocks(N)
ASSERT_EQ(BC_erase, test_sst * 2);
ASSERT_EQ(BC_failures, 0);
}

} // namespace TERARKDB_NAMESPACE

int main(int argc, char** argv) {
Expand Down
26 changes: 10 additions & 16 deletions table/block_based_table_reader.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ class PartitionIndexReader : public IndexReader, public Cleanable {
rep->compressed_cache_key_prefix_size,
handle, compressed_cache_key);

bool erased = block_cache->Erase(key);
bool erased = block_cache_compressed->Erase(key);
++block_cache_erase_count;
block_cache_erase_failures_count += !erased;
}
Expand Down Expand Up @@ -2634,16 +2634,16 @@ Status BlockBasedTable::ForceEvict() {
uint64_t block_cache_erase_failures_count = 0;

auto do_evict = [=, &cache_key, &compressed_cache_key,
&block_cache_erase_count,
&block_cache_erase_failures_count](uint64_t offset) {
if (block_cache) {
&block_cache_erase_count, &block_cache_erase_failures_count](
uint64_t offset, bool with_block_cache_compressed) {
if (block_cache != nullptr) {
char* end = EncodeVarint64(cache_key + cache_key_prefix_size, offset);
bool erased = block_cache->Erase(
Slice(cache_key, static_cast<size_t>(end - cache_key)));
++block_cache_erase_count;
block_cache_erase_failures_count += !erased;
}
if (block_cache_compressed) {
if (with_block_cache_compressed && block_cache_compressed != nullptr) {
char* end = EncodeVarint64(
compressed_cache_key + compressed_cache_key_prefix_size, offset);
bool erased = block_cache_compressed->Erase(
Expand All @@ -2654,31 +2654,25 @@ Status BlockBasedTable::ForceEvict() {
}
};

// FilterBlock
if (rep_->filter_entry.value != nullptr) {
rep_->filter_entry.value->ForceEvict();
}
if (rep_->filter_policy != nullptr &&
!rep_->table_options.cache_index_and_filter_blocks) {
do_evict(rep_->filter_handle.offset());
}

ReadOptions read_options;
read_options.fill_cache = false;
std::unique_ptr<InternalIteratorBase<BlockHandle>> iiter(
NewIndexIterator(read_options));

// DataBlock
for (iiter->SeekToFirst(); iiter->Valid(); iiter->Next()) {
do_evict(iiter->value().offset());
do_evict(iiter->value().offset(), true);
}

iiter.reset();

// FilterBlock is evicted in BlockBasedTable::Close()

// IndexBlock
if (rep_->index_reader != nullptr) {
rep_->index_reader->ForceEvict();
}
do_evict(rep_->dummy_index_reader_offset);
do_evict(rep_->dummy_index_reader_offset, false);

RecordTick(rep_->ioptions.statistics, BLOCK_CACHE_ERASE,
block_cache_erase_count);
Expand Down
Loading

0 comments on commit b382632

Please sign in to comment.