diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36850fa0..11004e46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,14 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Check Java codestyle + - name: Check Persistent Java codestyle run: | - cd java + cd persistent/java + mvn spotless:check + + - name: Check Volatile Java codestyle + run: | + cd volatile/java mvn spotless:check - name: Get cmake @@ -36,8 +41,16 @@ jobs: - name: Init submodules run: git submodule update --init --recursive - - name: Check codestyle & Build + - name: Check Persistent codestyle & Build + run: | + cd persistent + mkdir -p build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release -DCHECK_CPP_STYLE=ON -DWITH_JNI=ON + make -j + + - name: Check Volatile codestyle & Build run: | + cd volatile mkdir -p build && cd build cmake .. -DCMAKE_BUILD_TYPE=Release -DCHECK_CPP_STYLE=ON -DWITH_JNI=ON make -j diff --git a/.gitmodules b/.gitmodules index 667caa69..971d1998 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,12 @@ -[submodule "extern/gtest"] - path = extern/gtest +[submodule "volatile/extern/jemalloc"] + path = volatile/extern/jemalloc + url = https://github.com/jemalloc/jemalloc +[submodule "volatile/extern/numactl"] + path = volatile/extern/numactl + url = https://github.com/numactl/numactl +[submodule "persistent/extern/gtest"] + path = persistent/extern/gtest + url = https://github.com/google/googletest.git +[submodule "volatile/extern/gtest"] + path = volatile/extern/gtest url = https://github.com/google/googletest.git diff --git a/pic/kvdk-qrcode.png b/.pic/kvdk-qrcode.png similarity index 100% rename from pic/kvdk-qrcode.png rename to .pic/kvdk-qrcode.png diff --git a/pic/kvdk_logo.png b/.pic/kvdk_logo.png similarity index 100% rename from pic/kvdk_logo.png rename to .pic/kvdk_logo.png diff --git a/README.md b/README.md index decdead5..27e2f8c5 100644 --- a/README.md +++ b/README.md @@ -1,124 +1,19 @@
-


+


-`KVDK` (Key-Value Development Kit) is a key-value store library implemented in C++ language. It is designed for supporting DRAM, Optane persistent memory and CXL memory pool. It also demonstrates several optimization methods for high performance with tiered memory. Besides providing the basic APIs of key-value store, it offers several advanced features, like read-modify-write, checkpoint, etc. +`KVDK` (Key-Value Development Kit) is a key-value store library implemented in C++ language. It is designed for supporting Optane persistent memory, DRAM and CXL memory pool. It also demonstrates several optimization methods for high performance with tiered memory. Besides providing the basic APIs of key-value store, it offers several advanced features, like read-modify-write, checkpoint, etc. -## Features -* Rich data types - * string, sorted, hash, list, hash -* Basic KV operations - * get/put/update/delete/scan -* Read-Modify-Write -* Support TTL -* Atomic Batch Write -* Snapshot based Scan -* Consistent Dump & Restore to/from storage -* Consistent Checkpoint -* Transaction -* C/C++/Java APIs - -# Limitations -* The maximum supported key-value size is 64KB-4GB. -* No approach to key-value compression. -* Users can't expand the persistent memory space on the fly. +**Notice: The DRAM engine and CXL memory pool are in development, you can checkout to v1.0 for a release version of PMEM based persistent engine.** ## Getting the Source ```bash git clone --recurse-submodules https://github.com/pmem/kvdk.git ``` -## Building -### Install dependent tools and libraries on Ubuntu 18.04 -```bash -sudo apt install make clang-format-9 pkg-config g++ autoconf libtool asciidoctor libkmod-dev libudev-dev uuid-dev libjson-c-dev libkeyutils-dev pandoc libhwloc-dev libgflags-dev libtext-diff-perl bash-completion systemd wget git curl - -git clone https://github.com/pmem/ndctl.git -cd ndctl -git checkout v70.1 -./autogen.sh -./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib -make -sudo make install - -git clone https://github.com/pmem/pmdk.git -cd pmdk -git checkout 1.11.1 -make -sudo make install - -wget https://github.com/Kitware/CMake/releases/download/v3.12.4/cmake-3.12.4.tar.gz -tar vzxf cmake-3.12.4.tar.gz -cd cmake-3.12.4 -./bootstrap -make -sudo make install - -``` - -### Install dependent tools and libraries on CentOS 8 -```bash -yum config-manager --add-repo /etc/yum.repos.d/CentOS-Linux-PowerTools.repo -yum config-manager --set-enabled PowerTools - -yum install -y git gcc gcc-c++ autoconf automake asciidoc bash-completion xmlto libtool pkgconfig glib2 glib2-devel libfabric libfabric-devel doxygen graphviz pandoc ncurses kmod kmod-devel libudev-devel libuuid-devel json-c-devel keyutils-libs-devel gem make cmake libarchive clang-tools-extra hwloc-devel perl-Text-Diff gflags-devel curl - -git clone https://github.com/pmem/ndctl.git -cd ndctl -git checkout v70.1 -./autogen.sh -./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib -make -sudo make install - -git clone https://github.com/pmem/pmdk.git -cd pmdk -git checkout 1.11.1 -make -sudo make install - -wget https://github.com/Kitware/CMake/releases/download/v3.12.4/cmake-3.12.4.tar.gz -tar vzxf cmake-3.12.4.tar.gz -cd cmake-3.12.4 -./bootstrap -make -sudo make install -``` - -### Compile KVDK -```bash -mkdir -p build && cd build -cmake .. -DCMAKE_BUILD_TYPE=Release -DCHECK_CPP_STYLE=ON && make -j -``` - -### How to test it on a system without PMEM -```bash -# set the correct path for pmdk library -export LD_LIBRARY_PATH=/usr/local/lib64 - -# setup a tmpfs for test -mkdir /mnt/pmem0 -mount -t tmpfs -o size=2G tmpfs /mnt/pmem0 - -# force the program work on non-pmem directory -export PMEM_IS_PMEM_FORCE=1 - -cd kvdk/build/examples -# Note: this requires CPU supporting AVX512 -./cpp_api_tutorial - -``` - -## Benchmarks -[Here](./doc/benchmark.md) are the examples of how to benchmark the performance of KVDK on your systems. - -## Documentations - -### User Guide - -Please refer to [User guide](./doc/user_doc.md) for API introductions of KVDK. +## User guide -### Architecture +KVDK of different storage type are independently modules, please refer to [persistent](./persistent/README.md) and [volatile](./volatile/README.md) for building, test and docs. # Support Welcome to join the wechat group or slack channel for KVDK tech discussion. diff --git a/persistent/.gitignore b/persistent/.gitignore new file mode 100644 index 00000000..47a99d9c --- /dev/null +++ b/persistent/.gitignore @@ -0,0 +1,53 @@ +# IDE related +.idea/ +cmake-build-* +.vscode/ +.vs/ + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +build/ +.cache/ + +scripts/result* +scripts/__pycache__ + +java/out +java/target +java/benchmark/target +java/examples/target +java/test-libs +java/*.log +java/include/io_pmem_*.h +java/src/main/resources diff --git a/CMakeLists.txt b/persistent/CMakeLists.txt similarity index 99% rename from CMakeLists.txt rename to persistent/CMakeLists.txt index b8f194c8..8cf1b68f 100644 --- a/CMakeLists.txt +++ b/persistent/CMakeLists.txt @@ -156,7 +156,7 @@ add_cppstyle(src ${CMAKE_CURRENT_SOURCE_DIR}/engine/*.c* ${CMAKE_CURRENT_SOURCE_DIR}/engine/utils/*.h* ${CMAKE_CURRENT_SOURCE_DIR}/examples/tutorial/*.c* ${CMAKE_CURRENT_SOURCE_DIR}/benchmark/*.c* - ${CMAKE_CURRENT_SOURCE_DIR}/include/kvdk/*.h* + ${CMAKE_CURRENT_SOURCE_DIR}/include/kvdk/persistent/*.h* ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.c* ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.h*) diff --git a/persistent/README.md b/persistent/README.md new file mode 100644 index 00000000..51c00fa6 --- /dev/null +++ b/persistent/README.md @@ -0,0 +1,126 @@ +
+


+
+ +This is the persistent module of `kvdk`, based on Intel Optane Persistent Memory. + +## Features +* Rich data types + * string, sorted, hash, list, hash +* Basic KV operations + * get/put/update/delete/scan +* Read-Modify-Write +* Support TTL +* Atomic Batch Write +* Snapshot based Scan +* Consistent Dump & Restore to/from storage +* Consistent Checkpoint +* Transaction +* C/C++/Java APIs + +# Limitations +* The maximum supported key-value size is 64KB-4GB. +* No approach to key-value compression. +* Users can't expand the persistent memory space on the fly. + +## Getting the Source +```bash +git clone --recurse-submodules https://github.com/pmem/kvdk.git +``` + +## Building +### Install dependent tools and libraries on Ubuntu 18.04 +```bash +sudo apt install make clang-format-9 pkg-config g++ autoconf libtool asciidoctor libkmod-dev libudev-dev uuid-dev libjson-c-dev libkeyutils-dev pandoc libhwloc-dev libgflags-dev libtext-diff-perl bash-completion systemd wget git curl + +git clone https://github.com/pmem/ndctl.git +cd ndctl +git checkout v70.1 +./autogen.sh +./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib +make +sudo make install + +git clone https://github.com/pmem/pmdk.git +cd pmdk +git checkout 1.11.1 +make +sudo make install + +wget https://github.com/Kitware/CMake/releases/download/v3.12.4/cmake-3.12.4.tar.gz +tar vzxf cmake-3.12.4.tar.gz +cd cmake-3.12.4 +./bootstrap +make +sudo make install + +``` + +### Install dependent tools and libraries on CentOS 8 +```bash +yum config-manager --add-repo /etc/yum.repos.d/CentOS-Linux-PowerTools.repo +yum config-manager --set-enabled PowerTools + +yum install -y git gcc gcc-c++ autoconf automake asciidoc bash-completion xmlto libtool pkgconfig glib2 glib2-devel libfabric libfabric-devel doxygen graphviz pandoc ncurses kmod kmod-devel libudev-devel libuuid-devel json-c-devel keyutils-libs-devel gem make cmake libarchive clang-tools-extra hwloc-devel perl-Text-Diff gflags-devel curl + +git clone https://github.com/pmem/ndctl.git +cd ndctl +git checkout v70.1 +./autogen.sh +./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib +make +sudo make install + +git clone https://github.com/pmem/pmdk.git +cd pmdk +git checkout 1.11.1 +make +sudo make install + +wget https://github.com/Kitware/CMake/releases/download/v3.12.4/cmake-3.12.4.tar.gz +tar vzxf cmake-3.12.4.tar.gz +cd cmake-3.12.4 +./bootstrap +make +sudo make install +``` + +### Compile KVDK +```bash +mkdir -p build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Release -DCHECK_CPP_STYLE=ON && make -j +``` + +### How to test it on a system without PMEM +```bash +# set the correct path for pmdk library +export LD_LIBRARY_PATH=/usr/local/lib64 + +# setup a tmpfs for test +mkdir /mnt/pmem0 +mount -t tmpfs -o size=2G tmpfs /mnt/pmem0 + +# force the program work on non-pmem directory +export PMEM_IS_PMEM_FORCE=1 + +cd kvdk/build/examples +# Note: this requires CPU supporting AVX512 +./cpp_api_tutorial + +``` + +## Benchmarks +[Here](./doc/benchmark.md) are the examples of how to benchmark the performance of KVDK on your systems. + +## Documentations + +### User Guide + +Please refer to [User guide](./doc/user_doc.md) for API introductions of KVDK. + +### Architecture + +# Support +Welcome to join the wechat group or slack channel for KVDK tech discussion. +- [Wechat](https://github.com/pmem/kvdk/issues/143) +- [Slack Channel](https://join.slack.com/t/kvdksupportcommunity/shared_invite/zt-12b66vg1c-4FGb~Ri4w8K2_msau6v86Q) diff --git a/benchmark/bench.cpp b/persistent/benchmark/bench.cpp similarity index 99% rename from benchmark/bench.cpp rename to persistent/benchmark/bench.cpp index 6593bdb6..c1e00124 100644 --- a/benchmark/bench.cpp +++ b/persistent/benchmark/bench.cpp @@ -11,8 +11,8 @@ #include #include "generator.hpp" -#include "kvdk/engine.hpp" -#include "kvdk/types.hpp" +#include "kvdk/persistent/engine.hpp" +#include "kvdk/persistent/types.hpp" using namespace google; using namespace KVDK_NAMESPACE; diff --git a/benchmark/generator.hpp b/persistent/benchmark/generator.hpp similarity index 100% rename from benchmark/generator.hpp rename to persistent/benchmark/generator.hpp diff --git a/benchmark/utils/rand64.hpp b/persistent/benchmark/utils/rand64.hpp similarity index 100% rename from benchmark/utils/rand64.hpp rename to persistent/benchmark/utils/rand64.hpp diff --git a/benchmark/utils/range_iterator.hpp b/persistent/benchmark/utils/range_iterator.hpp similarity index 100% rename from benchmark/utils/range_iterator.hpp rename to persistent/benchmark/utils/range_iterator.hpp diff --git a/benchmark/utils/zipf.hpp b/persistent/benchmark/utils/zipf.hpp similarity index 100% rename from benchmark/utils/zipf.hpp rename to persistent/benchmark/utils/zipf.hpp diff --git a/cmake/functions.cmake b/persistent/cmake/functions.cmake similarity index 100% rename from cmake/functions.cmake rename to persistent/cmake/functions.cmake diff --git a/doc/benchmark.md b/persistent/doc/benchmark.md similarity index 100% rename from doc/benchmark.md rename to persistent/doc/benchmark.md diff --git a/doc/figs/pmem_allocator.png b/persistent/doc/figs/pmem_allocator.png similarity index 100% rename from doc/figs/pmem_allocator.png rename to persistent/doc/figs/pmem_allocator.png diff --git a/doc/pmem_allocator.md b/persistent/doc/pmem_allocator.md similarity index 100% rename from doc/pmem_allocator.md rename to persistent/doc/pmem_allocator.md diff --git a/doc/user_doc.md b/persistent/doc/user_doc.md similarity index 99% rename from doc/user_doc.md rename to persistent/doc/user_doc.md index c982c499..c573f288 100644 --- a/doc/user_doc.md +++ b/persistent/doc/user_doc.md @@ -18,8 +18,8 @@ or how to reopen an existing KVDK instance at the path supplied. (In the following example, PMem is mounted as /mnt/pmem0/ and the KVDK instance is named tutorial_kvdk_example.) ```c++ -#include "kvdk/engine.hpp" -#include "kvdk/namespace.hpp" +#include "kvdk/persistent/engine.hpp" +#include "kvdk/persistent/namespace.hpp" #include #include #include diff --git a/engine/alias.hpp b/persistent/engine/alias.hpp similarity index 90% rename from engine/alias.hpp rename to persistent/engine/alias.hpp index ce93123f..3540e5dd 100644 --- a/engine/alias.hpp +++ b/persistent/engine/alias.hpp @@ -6,7 +6,7 @@ #include -#include "kvdk/types.hpp" +#include "kvdk/persistent/types.hpp" namespace KVDK_NAMESPACE { enum class WriteOp { Put, Delete }; diff --git a/engine/allocator.hpp b/persistent/engine/allocator.hpp similarity index 100% rename from engine/allocator.hpp rename to persistent/engine/allocator.hpp diff --git a/engine/backup_log.hpp b/persistent/engine/backup_log.hpp similarity index 99% rename from engine/backup_log.hpp rename to persistent/engine/backup_log.hpp index bea59fd3..e0acc8fd 100644 --- a/engine/backup_log.hpp +++ b/persistent/engine/backup_log.hpp @@ -10,8 +10,8 @@ #include "configs.hpp" #include "data_record.hpp" -#include "kvdk/configs.hpp" -#include "kvdk/types.hpp" +#include "kvdk/persistent/configs.hpp" +#include "kvdk/persistent/types.hpp" #include "logger.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/c/kvdk_basic_op.cpp b/persistent/engine/c/kvdk_basic_op.cpp similarity index 100% rename from engine/c/kvdk_basic_op.cpp rename to persistent/engine/c/kvdk_basic_op.cpp diff --git a/engine/c/kvdk_batch.cpp b/persistent/engine/c/kvdk_batch.cpp similarity index 100% rename from engine/c/kvdk_batch.cpp rename to persistent/engine/c/kvdk_batch.cpp diff --git a/engine/c/kvdk_c.hpp b/persistent/engine/c/kvdk_c.hpp similarity index 84% rename from engine/c/kvdk_c.hpp rename to persistent/engine/c/kvdk_c.hpp index 318cb623..2686c990 100644 --- a/engine/c/kvdk_c.hpp +++ b/persistent/engine/c/kvdk_c.hpp @@ -8,12 +8,12 @@ #include #include "../alias.hpp" -#include "kvdk/configs.hpp" -#include "kvdk/engine.h" -#include "kvdk/engine.hpp" -#include "kvdk/iterator.hpp" -#include "kvdk/transaction.hpp" -#include "kvdk/write_batch.hpp" +#include "kvdk/persistent/configs.hpp" +#include "kvdk/persistent/engine.h" +#include "kvdk/persistent/engine.hpp" +#include "kvdk/persistent/iterator.hpp" +#include "kvdk/persistent/transaction.hpp" +#include "kvdk/persistent/write_batch.hpp" using kvdk::StringView; diff --git a/engine/c/kvdk_hash.cpp b/persistent/engine/c/kvdk_hash.cpp similarity index 100% rename from engine/c/kvdk_hash.cpp rename to persistent/engine/c/kvdk_hash.cpp diff --git a/engine/c/kvdk_list.cpp b/persistent/engine/c/kvdk_list.cpp similarity index 100% rename from engine/c/kvdk_list.cpp rename to persistent/engine/c/kvdk_list.cpp diff --git a/engine/c/kvdk_sorted.cpp b/persistent/engine/c/kvdk_sorted.cpp similarity index 100% rename from engine/c/kvdk_sorted.cpp rename to persistent/engine/c/kvdk_sorted.cpp diff --git a/engine/c/kvdk_string.cpp b/persistent/engine/c/kvdk_string.cpp similarity index 100% rename from engine/c/kvdk_string.cpp rename to persistent/engine/c/kvdk_string.cpp diff --git a/engine/c/kvdk_transaction.cpp b/persistent/engine/c/kvdk_transaction.cpp similarity index 100% rename from engine/c/kvdk_transaction.cpp rename to persistent/engine/c/kvdk_transaction.cpp diff --git a/engine/collection.hpp b/persistent/engine/collection.hpp similarity index 100% rename from engine/collection.hpp rename to persistent/engine/collection.hpp diff --git a/engine/configs.hpp b/persistent/engine/configs.hpp similarity index 96% rename from engine/configs.hpp rename to persistent/engine/configs.hpp index fa32d081..5b00fc19 100644 --- a/engine/configs.hpp +++ b/persistent/engine/configs.hpp @@ -5,7 +5,7 @@ #pragma once #include "alias.hpp" -#include "kvdk/configs.hpp" +#include "kvdk/persistent/configs.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/data_record.cpp b/persistent/engine/data_record.cpp similarity index 100% rename from engine/data_record.cpp rename to persistent/engine/data_record.cpp diff --git a/engine/data_record.hpp b/persistent/engine/data_record.hpp similarity index 99% rename from engine/data_record.hpp rename to persistent/engine/data_record.hpp index b8ea2283..036d4554 100644 --- a/engine/data_record.hpp +++ b/persistent/engine/data_record.hpp @@ -9,7 +9,7 @@ #include #include "alias.hpp" -#include "kvdk/configs.hpp" +#include "kvdk/persistent/configs.hpp" #include "utils/utils.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/dl_list.cpp b/persistent/engine/dl_list.cpp similarity index 100% rename from engine/dl_list.cpp rename to persistent/engine/dl_list.cpp diff --git a/engine/dl_list.hpp b/persistent/engine/dl_list.hpp similarity index 99% rename from engine/dl_list.hpp rename to persistent/engine/dl_list.hpp index f450fe35..0179d8e9 100644 --- a/engine/dl_list.hpp +++ b/persistent/engine/dl_list.hpp @@ -6,7 +6,7 @@ #include "collection.hpp" #include "data_record.hpp" -#include "kvdk/types.hpp" +#include "kvdk/persistent/types.hpp" #include "lock_table.hpp" #include "pmem_allocator/pmem_allocator.hpp" #include "utils/sync_point.hpp" diff --git a/engine/dram_allocator.cpp b/persistent/engine/dram_allocator.cpp similarity index 100% rename from engine/dram_allocator.cpp rename to persistent/engine/dram_allocator.cpp diff --git a/engine/dram_allocator.hpp b/persistent/engine/dram_allocator.hpp similarity index 97% rename from engine/dram_allocator.hpp rename to persistent/engine/dram_allocator.hpp index 46458f7b..88237767 100644 --- a/engine/dram_allocator.hpp +++ b/persistent/engine/dram_allocator.hpp @@ -12,7 +12,7 @@ #include "alias.hpp" #include "allocator.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "logger.hpp" #include "structures.hpp" diff --git a/persistent/engine/engine.cpp b/persistent/engine/engine.cpp new file mode 100644 index 00000000..f27dbd8d --- /dev/null +++ b/persistent/engine/engine.cpp @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "kvdk/persistent/engine.hpp" + +#include "kv_engine.hpp" + +namespace KVDK_NAMESPACE { +Status Engine::Open(const StringView name, Engine** engine_ptr, + const Configs& configs, FILE* log_file) { + GlobalLogger.Init(log_file, configs.log_level); + Status s = KVEngine::Open(name, engine_ptr, configs); + return s; +} + +Status Engine::Restore(const StringView engine_path, + const StringView backup_file, Engine** engine_ptr, + const Configs& configs, FILE* log_file) { + GlobalLogger.Init(log_file, configs.log_level); + Status s = KVEngine::Restore(engine_path, backup_file, engine_ptr, configs); + return s; +} + +Engine::~Engine() {} +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/engine/hash_collection/hash_list.cpp b/persistent/engine/hash_collection/hash_list.cpp similarity index 100% rename from engine/hash_collection/hash_list.cpp rename to persistent/engine/hash_collection/hash_list.cpp diff --git a/engine/hash_collection/hash_list.hpp b/persistent/engine/hash_collection/hash_list.hpp similarity index 99% rename from engine/hash_collection/hash_list.hpp rename to persistent/engine/hash_collection/hash_list.hpp index 033bf691..f96a1581 100644 --- a/engine/hash_collection/hash_list.hpp +++ b/persistent/engine/hash_collection/hash_list.hpp @@ -2,7 +2,7 @@ #include "../dl_list.hpp" #include "../hash_table.hpp" -#include "kvdk/types.hpp" +#include "kvdk/persistent/types.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/hash_collection/iterator.hpp b/persistent/engine/hash_collection/iterator.hpp similarity index 95% rename from engine/hash_collection/iterator.hpp rename to persistent/engine/hash_collection/iterator.hpp index 39d57fcf..eb1b06b0 100644 --- a/engine/hash_collection/iterator.hpp +++ b/persistent/engine/hash_collection/iterator.hpp @@ -2,8 +2,8 @@ #include "../version/version_controller.hpp" #include "hash_list.hpp" -#include "kvdk/engine.hpp" -#include "kvdk/iterator.hpp" +#include "kvdk/persistent/engine.hpp" +#include "kvdk/persistent/iterator.hpp" namespace KVDK_NAMESPACE { class KVEngine; diff --git a/engine/hash_collection/rebuilder.hpp b/persistent/engine/hash_collection/rebuilder.hpp similarity index 100% rename from engine/hash_collection/rebuilder.hpp rename to persistent/engine/hash_collection/rebuilder.hpp diff --git a/engine/hash_table.cpp b/persistent/engine/hash_table.cpp similarity index 100% rename from engine/hash_table.cpp rename to persistent/engine/hash_table.cpp diff --git a/engine/hash_table.hpp b/persistent/engine/hash_table.hpp similarity index 99% rename from engine/hash_table.hpp rename to persistent/engine/hash_table.hpp index a0d6d6fd..ff5a27cb 100644 --- a/engine/hash_table.hpp +++ b/persistent/engine/hash_table.hpp @@ -12,7 +12,7 @@ #include "alias.hpp" #include "data_record.hpp" #include "dram_allocator.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "pmem_allocator/pmem_allocator.hpp" #include "structures.hpp" diff --git a/engine/kv_engine.cpp b/persistent/engine/kv_engine.cpp similarity index 99% rename from engine/kv_engine.cpp rename to persistent/engine/kv_engine.cpp index cb7cc794..effa20b2 100644 --- a/engine/kv_engine.cpp +++ b/persistent/engine/kv_engine.cpp @@ -21,7 +21,7 @@ #include "configs.hpp" #include "dram_allocator.hpp" #include "hash_collection/iterator.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "list_collection/iterator.hpp" #include "sorted_collection/iterator.hpp" #include "structures.hpp" diff --git a/engine/kv_engine.hpp b/persistent/engine/kv_engine.hpp similarity index 99% rename from engine/kv_engine.hpp rename to persistent/engine/kv_engine.hpp index f577e96b..fe68596f 100644 --- a/engine/kv_engine.hpp +++ b/persistent/engine/kv_engine.hpp @@ -25,7 +25,7 @@ #include "hash_collection/hash_list.hpp" #include "hash_collection/rebuilder.hpp" #include "hash_table.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "list_collection/list.hpp" #include "list_collection/rebuilder.hpp" #include "lock_table.hpp" diff --git a/engine/kv_engine_cleaner.cpp b/persistent/engine/kv_engine_cleaner.cpp similarity index 100% rename from engine/kv_engine_cleaner.cpp rename to persistent/engine/kv_engine_cleaner.cpp diff --git a/engine/kv_engine_cleaner.hpp b/persistent/engine/kv_engine_cleaner.hpp similarity index 100% rename from engine/kv_engine_cleaner.hpp rename to persistent/engine/kv_engine_cleaner.hpp diff --git a/engine/kv_engine_hash.cpp b/persistent/engine/kv_engine_hash.cpp similarity index 100% rename from engine/kv_engine_hash.cpp rename to persistent/engine/kv_engine_hash.cpp diff --git a/engine/kv_engine_list.cpp b/persistent/engine/kv_engine_list.cpp similarity index 100% rename from engine/kv_engine_list.cpp rename to persistent/engine/kv_engine_list.cpp diff --git a/engine/kv_engine_sorted.cpp b/persistent/engine/kv_engine_sorted.cpp similarity index 100% rename from engine/kv_engine_sorted.cpp rename to persistent/engine/kv_engine_sorted.cpp diff --git a/engine/kv_engine_string.cpp b/persistent/engine/kv_engine_string.cpp similarity index 100% rename from engine/kv_engine_string.cpp rename to persistent/engine/kv_engine_string.cpp diff --git a/engine/list_collection/iterator.hpp b/persistent/engine/list_collection/iterator.hpp similarity index 95% rename from engine/list_collection/iterator.hpp rename to persistent/engine/list_collection/iterator.hpp index 09d77499..bbb224c5 100644 --- a/engine/list_collection/iterator.hpp +++ b/persistent/engine/list_collection/iterator.hpp @@ -1,8 +1,8 @@ #pragma once #include "../version/version_controller.hpp" -#include "kvdk/engine.hpp" -#include "kvdk/iterator.hpp" +#include "kvdk/persistent/engine.hpp" +#include "kvdk/persistent/iterator.hpp" #include "list.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/list_collection/list.cpp b/persistent/engine/list_collection/list.cpp similarity index 100% rename from engine/list_collection/list.cpp rename to persistent/engine/list_collection/list.cpp diff --git a/engine/list_collection/list.hpp b/persistent/engine/list_collection/list.hpp similarity index 99% rename from engine/list_collection/list.hpp rename to persistent/engine/list_collection/list.hpp index 999c660b..4212e1de 100644 --- a/engine/list_collection/list.hpp +++ b/persistent/engine/list_collection/list.hpp @@ -5,7 +5,7 @@ #pragma once #include "../dl_list.hpp" -#include "kvdk/types.hpp" +#include "kvdk/persistent/types.hpp" namespace KVDK_NAMESPACE { class ListIteratorImpl; diff --git a/engine/list_collection/rebuilder.hpp b/persistent/engine/list_collection/rebuilder.hpp similarity index 100% rename from engine/list_collection/rebuilder.hpp rename to persistent/engine/list_collection/rebuilder.hpp diff --git a/engine/lock_table.hpp b/persistent/engine/lock_table.hpp similarity index 100% rename from engine/lock_table.hpp rename to persistent/engine/lock_table.hpp diff --git a/engine/logger.cpp b/persistent/engine/logger.cpp similarity index 100% rename from engine/logger.cpp rename to persistent/engine/logger.cpp diff --git a/persistent/engine/logger.hpp b/persistent/engine/logger.hpp new file mode 100644 index 00000000..6713928b --- /dev/null +++ b/persistent/engine/logger.hpp @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include + +#include +#include +#include + +#include "alias.hpp" +#include "kvdk/persistent/configs.hpp" + +#define DO_LOG 1 + +namespace KVDK_NAMESPACE { + +class Logger { + public: + void Info(const char* format, ...); + void Error(const char* format, ...); + void Debug(const char* format, ...); + void Init(FILE* fp, LogLevel level); + + private: + void Log(const char* log_type, const char* format, va_list& args); + + FILE* log_file_ = NULL; + LogLevel level_; + std::mutex mut_; + + std::chrono::time_point start_ts_; +}; + +extern Logger GlobalLogger; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/engine/macros.hpp b/persistent/engine/macros.hpp similarity index 100% rename from engine/macros.hpp rename to persistent/engine/macros.hpp diff --git a/engine/pmem_allocator/free_list.cpp b/persistent/engine/pmem_allocator/free_list.cpp similarity index 100% rename from engine/pmem_allocator/free_list.cpp rename to persistent/engine/pmem_allocator/free_list.cpp diff --git a/engine/pmem_allocator/free_list.hpp b/persistent/engine/pmem_allocator/free_list.hpp similarity index 100% rename from engine/pmem_allocator/free_list.hpp rename to persistent/engine/pmem_allocator/free_list.hpp diff --git a/engine/pmem_allocator/pmem_allocator.cpp b/persistent/engine/pmem_allocator/pmem_allocator.cpp similarity index 100% rename from engine/pmem_allocator/pmem_allocator.cpp rename to persistent/engine/pmem_allocator/pmem_allocator.cpp diff --git a/engine/pmem_allocator/pmem_allocator.hpp b/persistent/engine/pmem_allocator/pmem_allocator.hpp similarity index 100% rename from engine/pmem_allocator/pmem_allocator.hpp rename to persistent/engine/pmem_allocator/pmem_allocator.hpp diff --git a/engine/snapshot.hpp b/persistent/engine/snapshot.hpp similarity index 100% rename from engine/snapshot.hpp rename to persistent/engine/snapshot.hpp diff --git a/engine/sorted_collection/iterator.hpp b/persistent/engine/sorted_collection/iterator.hpp similarity index 100% rename from engine/sorted_collection/iterator.hpp rename to persistent/engine/sorted_collection/iterator.hpp diff --git a/engine/sorted_collection/rebuilder.cpp b/persistent/engine/sorted_collection/rebuilder.cpp similarity index 100% rename from engine/sorted_collection/rebuilder.cpp rename to persistent/engine/sorted_collection/rebuilder.cpp diff --git a/engine/sorted_collection/rebuilder.hpp b/persistent/engine/sorted_collection/rebuilder.hpp similarity index 100% rename from engine/sorted_collection/rebuilder.hpp rename to persistent/engine/sorted_collection/rebuilder.hpp diff --git a/engine/sorted_collection/skiplist.cpp b/persistent/engine/sorted_collection/skiplist.cpp similarity index 100% rename from engine/sorted_collection/skiplist.cpp rename to persistent/engine/sorted_collection/skiplist.cpp diff --git a/engine/sorted_collection/skiplist.hpp b/persistent/engine/sorted_collection/skiplist.hpp similarity index 99% rename from engine/sorted_collection/skiplist.hpp rename to persistent/engine/sorted_collection/skiplist.hpp index a069ca48..9afa1472 100644 --- a/engine/sorted_collection/skiplist.hpp +++ b/persistent/engine/sorted_collection/skiplist.hpp @@ -19,7 +19,7 @@ #include "../structures.hpp" #include "../utils/utils.hpp" #include "../write_batch_impl.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "rebuilder.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/structures.hpp b/persistent/engine/structures.hpp similarity index 100% rename from engine/structures.hpp rename to persistent/engine/structures.hpp diff --git a/engine/thread_manager.cpp b/persistent/engine/thread_manager.cpp similarity index 100% rename from engine/thread_manager.cpp rename to persistent/engine/thread_manager.cpp diff --git a/persistent/engine/thread_manager.hpp b/persistent/engine/thread_manager.hpp new file mode 100644 index 00000000..418bb4ac --- /dev/null +++ b/persistent/engine/thread_manager.hpp @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include + +#include "alias.hpp" +#include "kvdk/persistent/engine.hpp" +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { + +class ThreadManager; + +struct Thread { + public: + Thread() : id(-1), manager(nullptr) {} + int64_t id; + std::shared_ptr manager; + + ~Thread(); +}; + +extern thread_local Thread this_thread; + +class ThreadManager : public std::enable_shared_from_this { + public: + static ThreadManager* Get() { return manager_.get(); } + static int64_t ThreadID() { + Get()->MaybeInitThread(this_thread); + return this_thread.id; + } + void MaybeInitThread(Thread& t); + void Release(Thread& t); + + private: + ThreadManager() : ids_(0), recycle_id_(), spin_() {} + + static std::shared_ptr manager_; + std::atomic ids_; + std::unordered_set recycle_id_; + SpinMutex spin_; +}; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/engine/transaction_impl.cpp b/persistent/engine/transaction_impl.cpp similarity index 100% rename from engine/transaction_impl.cpp rename to persistent/engine/transaction_impl.cpp diff --git a/engine/transaction_impl.hpp b/persistent/engine/transaction_impl.hpp similarity index 99% rename from engine/transaction_impl.hpp rename to persistent/engine/transaction_impl.hpp index 00ef3bf6..8b646e03 100644 --- a/engine/transaction_impl.hpp +++ b/persistent/engine/transaction_impl.hpp @@ -9,7 +9,7 @@ #include #include "alias.hpp" -#include "kvdk/transaction.hpp" +#include "kvdk/persistent/transaction.hpp" #include "write_batch_impl.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/utils/codec.hpp b/persistent/engine/utils/codec.hpp similarity index 100% rename from engine/utils/codec.hpp rename to persistent/engine/utils/codec.hpp diff --git a/engine/utils/sync_impl.hpp b/persistent/engine/utils/sync_impl.hpp similarity index 100% rename from engine/utils/sync_impl.hpp rename to persistent/engine/utils/sync_impl.hpp diff --git a/engine/utils/sync_point.cpp b/persistent/engine/utils/sync_point.cpp similarity index 100% rename from engine/utils/sync_point.cpp rename to persistent/engine/utils/sync_point.cpp diff --git a/engine/utils/sync_point.hpp b/persistent/engine/utils/sync_point.hpp similarity index 100% rename from engine/utils/sync_point.hpp rename to persistent/engine/utils/sync_point.hpp diff --git a/engine/utils/utils.cpp b/persistent/engine/utils/utils.cpp similarity index 100% rename from engine/utils/utils.cpp rename to persistent/engine/utils/utils.cpp diff --git a/engine/utils/utils.hpp b/persistent/engine/utils/utils.hpp similarity index 100% rename from engine/utils/utils.hpp rename to persistent/engine/utils/utils.hpp diff --git a/engine/version/old_records_cleaner.cpp b/persistent/engine/version/old_records_cleaner.cpp similarity index 100% rename from engine/version/old_records_cleaner.cpp rename to persistent/engine/version/old_records_cleaner.cpp diff --git a/engine/version/old_records_cleaner.hpp b/persistent/engine/version/old_records_cleaner.hpp similarity index 97% rename from engine/version/old_records_cleaner.hpp rename to persistent/engine/version/old_records_cleaner.hpp index bb75edc9..a92cb1fb 100644 --- a/engine/version/old_records_cleaner.hpp +++ b/persistent/engine/version/old_records_cleaner.hpp @@ -13,7 +13,7 @@ #include "../kv_engine_cleaner.hpp" #include "../thread_manager.hpp" #include "../utils/utils.hpp" -#include "kvdk/configs.hpp" +#include "kvdk/persistent/configs.hpp" #include "version_controller.hpp" namespace KVDK_NAMESPACE { diff --git a/engine/version/version_controller.hpp b/persistent/engine/version/version_controller.hpp similarity index 99% rename from engine/version/version_controller.hpp rename to persistent/engine/version/version_controller.hpp index 97f1b50f..0b39c44c 100644 --- a/engine/version/version_controller.hpp +++ b/persistent/engine/version/version_controller.hpp @@ -6,7 +6,7 @@ #include "../alias.hpp" #include "../thread_manager.hpp" #include "../utils/utils.hpp" -#include "kvdk/configs.hpp" +#include "kvdk/persistent/configs.hpp" namespace KVDK_NAMESPACE { constexpr TimestampType kMaxTimestamp = UINT64_MAX; diff --git a/engine/write_batch_impl.cpp b/persistent/engine/write_batch_impl.cpp similarity index 100% rename from engine/write_batch_impl.cpp rename to persistent/engine/write_batch_impl.cpp diff --git a/engine/write_batch_impl.hpp b/persistent/engine/write_batch_impl.hpp similarity index 99% rename from engine/write_batch_impl.hpp rename to persistent/engine/write_batch_impl.hpp index 7ce49045..b19ae9ad 100644 --- a/engine/write_batch_impl.hpp +++ b/persistent/engine/write_batch_impl.hpp @@ -8,7 +8,7 @@ #include "alias.hpp" #include "hash_table.hpp" -#include "kvdk/write_batch.hpp" +#include "kvdk/persistent/write_batch.hpp" #include "utils/codec.hpp" #include "utils/utils.hpp" diff --git a/examples/graph_sim/CMakeLists.txt b/persistent/examples/graph_sim/CMakeLists.txt similarity index 100% rename from examples/graph_sim/CMakeLists.txt rename to persistent/examples/graph_sim/CMakeLists.txt diff --git a/examples/graph_sim/README.md b/persistent/examples/graph_sim/README.md similarity index 100% rename from examples/graph_sim/README.md rename to persistent/examples/graph_sim/README.md diff --git a/examples/graph_sim/bench/graph_bench.cpp b/persistent/examples/graph_sim/bench/graph_bench.cpp similarity index 100% rename from examples/graph_sim/bench/graph_bench.cpp rename to persistent/examples/graph_sim/bench/graph_bench.cpp diff --git a/examples/graph_sim/scripts/build_rocksdb.sh b/persistent/examples/graph_sim/scripts/build_rocksdb.sh similarity index 100% rename from examples/graph_sim/scripts/build_rocksdb.sh rename to persistent/examples/graph_sim/scripts/build_rocksdb.sh diff --git a/examples/graph_sim/src/coding.hpp b/persistent/examples/graph_sim/src/coding.hpp similarity index 100% rename from examples/graph_sim/src/coding.hpp rename to persistent/examples/graph_sim/src/coding.hpp diff --git a/examples/graph_sim/src/graph_algorithm/top_n.hpp b/persistent/examples/graph_sim/src/graph_algorithm/top_n.hpp similarity index 100% rename from examples/graph_sim/src/graph_algorithm/top_n.hpp rename to persistent/examples/graph_sim/src/graph_algorithm/top_n.hpp diff --git a/examples/graph_sim/src/graph_impl.cpp b/persistent/examples/graph_sim/src/graph_impl.cpp similarity index 100% rename from examples/graph_sim/src/graph_impl.cpp rename to persistent/examples/graph_sim/src/graph_impl.cpp diff --git a/examples/graph_sim/src/graph_impl.hpp b/persistent/examples/graph_sim/src/graph_impl.hpp similarity index 100% rename from examples/graph_sim/src/graph_impl.hpp rename to persistent/examples/graph_sim/src/graph_impl.hpp diff --git a/examples/graph_sim/src/graph_impl_test.cpp b/persistent/examples/graph_sim/src/graph_impl_test.cpp similarity index 100% rename from examples/graph_sim/src/graph_impl_test.cpp rename to persistent/examples/graph_sim/src/graph_impl_test.cpp diff --git a/examples/graph_sim/src/kv_engines/KVEngine.hpp b/persistent/examples/graph_sim/src/kv_engines/KVEngine.hpp similarity index 99% rename from examples/graph_sim/src/kv_engines/KVEngine.hpp rename to persistent/examples/graph_sim/src/kv_engines/KVEngine.hpp index 9e5d2935..f3168501 100644 --- a/examples/graph_sim/src/kv_engines/KVEngine.hpp +++ b/persistent/examples/graph_sim/src/kv_engines/KVEngine.hpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include diff --git a/examples/graph_sim/src/kv_engines/engine_factory.hpp b/persistent/examples/graph_sim/src/kv_engines/engine_factory.hpp similarity index 100% rename from examples/graph_sim/src/kv_engines/engine_factory.hpp rename to persistent/examples/graph_sim/src/kv_engines/engine_factory.hpp diff --git a/examples/graph_sim/src/kv_engines/kvdk.cpp b/persistent/examples/graph_sim/src/kv_engines/kvdk.cpp similarity index 100% rename from examples/graph_sim/src/kv_engines/kvdk.cpp rename to persistent/examples/graph_sim/src/kv_engines/kvdk.cpp diff --git a/examples/graph_sim/src/kv_engines/rocksdb.cpp b/persistent/examples/graph_sim/src/kv_engines/rocksdb.cpp similarity index 100% rename from examples/graph_sim/src/kv_engines/rocksdb.cpp rename to persistent/examples/graph_sim/src/kv_engines/rocksdb.cpp diff --git a/examples/graph_sim/src/options.hpp b/persistent/examples/graph_sim/src/options.hpp similarity index 100% rename from examples/graph_sim/src/options.hpp rename to persistent/examples/graph_sim/src/options.hpp diff --git a/examples/kvredis/0001-redis-with-kvdk.patch b/persistent/examples/kvredis/0001-redis-with-kvdk.patch similarity index 100% rename from examples/kvredis/0001-redis-with-kvdk.patch rename to persistent/examples/kvredis/0001-redis-with-kvdk.patch diff --git a/examples/kvredis/README.md b/persistent/examples/kvredis/README.md similarity index 100% rename from examples/kvredis/README.md rename to persistent/examples/kvredis/README.md diff --git a/examples/kvredis/bench_redis_kvdk.sh b/persistent/examples/kvredis/bench_redis_kvdk.sh similarity index 100% rename from examples/kvredis/bench_redis_kvdk.sh rename to persistent/examples/kvredis/bench_redis_kvdk.sh diff --git a/examples/kvredis/build_redis_kvdk.sh b/persistent/examples/kvredis/build_redis_kvdk.sh similarity index 100% rename from examples/kvredis/build_redis_kvdk.sh rename to persistent/examples/kvredis/build_redis_kvdk.sh diff --git a/examples/kvrocks/0001-use-KVDK-for-set-get-operations.patch b/persistent/examples/kvrocks/0001-use-KVDK-for-set-get-operations.patch similarity index 100% rename from examples/kvrocks/0001-use-KVDK-for-set-get-operations.patch rename to persistent/examples/kvrocks/0001-use-KVDK-for-set-get-operations.patch diff --git a/examples/kvrocks/README.md b/persistent/examples/kvrocks/README.md similarity index 100% rename from examples/kvrocks/README.md rename to persistent/examples/kvrocks/README.md diff --git a/examples/tutorial/CMakeLists.txt b/persistent/examples/tutorial/CMakeLists.txt similarity index 100% rename from examples/tutorial/CMakeLists.txt rename to persistent/examples/tutorial/CMakeLists.txt diff --git a/examples/tutorial/c_api_tutorial.c b/persistent/examples/tutorial/c_api_tutorial.c similarity index 99% rename from examples/tutorial/c_api_tutorial.c rename to persistent/examples/tutorial/c_api_tutorial.c index df5d7fc5..8b792b2e 100644 --- a/examples/tutorial/c_api_tutorial.c +++ b/persistent/examples/tutorial/c_api_tutorial.c @@ -7,7 +7,7 @@ #include #include -#include "kvdk/engine.h" +#include "kvdk/persistent/engine.h" // The KVDK instance is mounted as a directory // /mnt/pmem0/tutorial_kvdk_example. diff --git a/examples/tutorial/cpp_api_tutorial.cpp b/persistent/examples/tutorial/cpp_api_tutorial.cpp similarity index 99% rename from examples/tutorial/cpp_api_tutorial.cpp rename to persistent/examples/tutorial/cpp_api_tutorial.cpp index 63522f7d..caae6a17 100644 --- a/examples/tutorial/cpp_api_tutorial.cpp +++ b/persistent/examples/tutorial/cpp_api_tutorial.cpp @@ -11,7 +11,7 @@ #include #include -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #define DEBUG // For assert diff --git a/examples/tutorial/zset.c b/persistent/examples/tutorial/zset.c similarity index 99% rename from examples/tutorial/zset.c rename to persistent/examples/tutorial/zset.c index 646e408f..2e1935f7 100644 --- a/examples/tutorial/zset.c +++ b/persistent/examples/tutorial/zset.c @@ -5,7 +5,7 @@ // This is a example of single-thread redis zset implementation with kvdk c api #include "assert.h" -#include "kvdk/engine.h" +#include "kvdk/persistent/engine.h" #include "malloc.h" #include "string.h" diff --git a/extern/gtest b/persistent/extern/gtest similarity index 100% rename from extern/gtest rename to persistent/extern/gtest diff --git a/extern/libpmemobj++/string_view.hpp b/persistent/extern/libpmemobj++/string_view.hpp similarity index 100% rename from extern/libpmemobj++/string_view.hpp rename to persistent/extern/libpmemobj++/string_view.hpp diff --git a/extern/xxhash.h b/persistent/extern/xxhash.h similarity index 100% rename from extern/xxhash.h rename to persistent/extern/xxhash.h diff --git a/include/kvdk/comparator.hpp b/persistent/include/kvdk/persistent/comparator.hpp similarity index 100% rename from include/kvdk/comparator.hpp rename to persistent/include/kvdk/persistent/comparator.hpp diff --git a/include/kvdk/configs.hpp b/persistent/include/kvdk/persistent/configs.hpp similarity index 100% rename from include/kvdk/configs.hpp rename to persistent/include/kvdk/persistent/configs.hpp diff --git a/include/kvdk/engine.h b/persistent/include/kvdk/persistent/engine.h similarity index 100% rename from include/kvdk/engine.h rename to persistent/include/kvdk/persistent/engine.h diff --git a/include/kvdk/engine.hpp b/persistent/include/kvdk/persistent/engine.hpp similarity index 100% rename from include/kvdk/engine.hpp rename to persistent/include/kvdk/persistent/engine.hpp diff --git a/include/kvdk/iterator.hpp b/persistent/include/kvdk/persistent/iterator.hpp similarity index 100% rename from include/kvdk/iterator.hpp rename to persistent/include/kvdk/persistent/iterator.hpp diff --git a/include/kvdk/snapshot.hpp b/persistent/include/kvdk/persistent/snapshot.hpp similarity index 100% rename from include/kvdk/snapshot.hpp rename to persistent/include/kvdk/persistent/snapshot.hpp diff --git a/include/kvdk/transaction.hpp b/persistent/include/kvdk/persistent/transaction.hpp similarity index 100% rename from include/kvdk/transaction.hpp rename to persistent/include/kvdk/persistent/transaction.hpp diff --git a/include/kvdk/types.h b/persistent/include/kvdk/persistent/types.h similarity index 100% rename from include/kvdk/types.h rename to persistent/include/kvdk/persistent/types.h diff --git a/include/kvdk/types.hpp b/persistent/include/kvdk/persistent/types.hpp similarity index 100% rename from include/kvdk/types.hpp rename to persistent/include/kvdk/persistent/types.hpp diff --git a/include/kvdk/write_batch.hpp b/persistent/include/kvdk/persistent/write_batch.hpp similarity index 100% rename from include/kvdk/write_batch.hpp rename to persistent/include/kvdk/persistent/write_batch.hpp diff --git a/java/CMakeLists.txt b/persistent/java/CMakeLists.txt similarity index 100% rename from java/CMakeLists.txt rename to persistent/java/CMakeLists.txt diff --git a/java/README.md b/persistent/java/README.md similarity index 100% rename from java/README.md rename to persistent/java/README.md diff --git a/java/benchmark/pom.xml b/persistent/java/benchmark/pom.xml similarity index 100% rename from java/benchmark/pom.xml rename to persistent/java/benchmark/pom.xml diff --git a/java/benchmark/scripts/benchmark_all.sh b/persistent/java/benchmark/scripts/benchmark_all.sh similarity index 100% rename from java/benchmark/scripts/benchmark_all.sh rename to persistent/java/benchmark/scripts/benchmark_all.sh diff --git a/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/KVDKBenchmark.java b/persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/KVDKBenchmark.java similarity index 100% rename from java/benchmark/src/main/java/io/pmem/kvdk/benchmark/KVDKBenchmark.java rename to persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/KVDKBenchmark.java diff --git a/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/ConstantLongGenerator.java b/persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/ConstantLongGenerator.java similarity index 100% rename from java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/ConstantLongGenerator.java rename to persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/ConstantLongGenerator.java diff --git a/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/LongGenerator.java b/persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/LongGenerator.java similarity index 100% rename from java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/LongGenerator.java rename to persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/LongGenerator.java diff --git a/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RandomLongGenerator.java b/persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RandomLongGenerator.java similarity index 100% rename from java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RandomLongGenerator.java rename to persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RandomLongGenerator.java diff --git a/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RangeLongGenerator.java b/persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RangeLongGenerator.java similarity index 100% rename from java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RangeLongGenerator.java rename to persistent/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RangeLongGenerator.java diff --git a/java/examples/pom.xml b/persistent/java/examples/pom.xml similarity index 100% rename from java/examples/pom.xml rename to persistent/java/examples/pom.xml diff --git a/java/examples/src/main/java/io/pmem/kvdk/exmaples/KVDKExamples.java b/persistent/java/examples/src/main/java/io/pmem/kvdk/exmaples/KVDKExamples.java similarity index 100% rename from java/examples/src/main/java/io/pmem/kvdk/exmaples/KVDKExamples.java rename to persistent/java/examples/src/main/java/io/pmem/kvdk/exmaples/KVDKExamples.java diff --git a/java/kvdkjni/configs.cc b/persistent/java/kvdkjni/configs.cc similarity index 100% rename from java/kvdkjni/configs.cc rename to persistent/java/kvdkjni/configs.cc diff --git a/java/kvdkjni/cplusplus_to_java_convert.h b/persistent/java/kvdkjni/cplusplus_to_java_convert.h similarity index 100% rename from java/kvdkjni/cplusplus_to_java_convert.h rename to persistent/java/kvdkjni/cplusplus_to_java_convert.h diff --git a/java/kvdkjni/engine.cc b/persistent/java/kvdkjni/engine.cc similarity index 100% rename from java/kvdkjni/engine.cc rename to persistent/java/kvdkjni/engine.cc diff --git a/java/kvdkjni/iterator.cc b/persistent/java/kvdkjni/iterator.cc similarity index 100% rename from java/kvdkjni/iterator.cc rename to persistent/java/kvdkjni/iterator.cc diff --git a/java/kvdkjni/kvdkjni.h b/persistent/java/kvdkjni/kvdkjni.h similarity index 99% rename from java/kvdkjni/kvdkjni.h rename to persistent/java/kvdkjni/kvdkjni.h index fa6b9ee7..9424193b 100644 --- a/java/kvdkjni/kvdkjni.h +++ b/persistent/java/kvdkjni/kvdkjni.h @@ -10,7 +10,7 @@ #include -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "kvdkjni/cplusplus_to_java_convert.h" namespace KVDK_NAMESPACE { diff --git a/java/kvdkjni/native_bytes_handle.cc b/persistent/java/kvdkjni/native_bytes_handle.cc similarity index 100% rename from java/kvdkjni/native_bytes_handle.cc rename to persistent/java/kvdkjni/native_bytes_handle.cc diff --git a/java/kvdkjni/write_batch.cc b/persistent/java/kvdkjni/write_batch.cc similarity index 100% rename from java/kvdkjni/write_batch.cc rename to persistent/java/kvdkjni/write_batch.cc diff --git a/java/pom.xml b/persistent/java/pom.xml similarity index 100% rename from java/pom.xml rename to persistent/java/pom.xml diff --git a/java/src/main/java/io/pmem/kvdk/AbstractNativeReference.java b/persistent/java/src/main/java/io/pmem/kvdk/AbstractNativeReference.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/AbstractNativeReference.java rename to persistent/java/src/main/java/io/pmem/kvdk/AbstractNativeReference.java diff --git a/java/src/main/java/io/pmem/kvdk/Configs.java b/persistent/java/src/main/java/io/pmem/kvdk/Configs.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/Configs.java rename to persistent/java/src/main/java/io/pmem/kvdk/Configs.java diff --git a/java/src/main/java/io/pmem/kvdk/Engine.java b/persistent/java/src/main/java/io/pmem/kvdk/Engine.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/Engine.java rename to persistent/java/src/main/java/io/pmem/kvdk/Engine.java diff --git a/java/src/main/java/io/pmem/kvdk/Iterator.java b/persistent/java/src/main/java/io/pmem/kvdk/Iterator.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/Iterator.java rename to persistent/java/src/main/java/io/pmem/kvdk/Iterator.java diff --git a/java/src/main/java/io/pmem/kvdk/KVDKException.java b/persistent/java/src/main/java/io/pmem/kvdk/KVDKException.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/KVDKException.java rename to persistent/java/src/main/java/io/pmem/kvdk/KVDKException.java diff --git a/java/src/main/java/io/pmem/kvdk/KVDKObject.java b/persistent/java/src/main/java/io/pmem/kvdk/KVDKObject.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/KVDKObject.java rename to persistent/java/src/main/java/io/pmem/kvdk/KVDKObject.java diff --git a/java/src/main/java/io/pmem/kvdk/NativeBytesHandle.java b/persistent/java/src/main/java/io/pmem/kvdk/NativeBytesHandle.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/NativeBytesHandle.java rename to persistent/java/src/main/java/io/pmem/kvdk/NativeBytesHandle.java diff --git a/java/src/main/java/io/pmem/kvdk/NativeLibraryLoader.java b/persistent/java/src/main/java/io/pmem/kvdk/NativeLibraryLoader.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/NativeLibraryLoader.java rename to persistent/java/src/main/java/io/pmem/kvdk/NativeLibraryLoader.java diff --git a/java/src/main/java/io/pmem/kvdk/Status.java b/persistent/java/src/main/java/io/pmem/kvdk/Status.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/Status.java rename to persistent/java/src/main/java/io/pmem/kvdk/Status.java diff --git a/java/src/main/java/io/pmem/kvdk/WriteBatch.java b/persistent/java/src/main/java/io/pmem/kvdk/WriteBatch.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/WriteBatch.java rename to persistent/java/src/main/java/io/pmem/kvdk/WriteBatch.java diff --git a/java/src/main/java/io/pmem/kvdk/WriteOptions.java b/persistent/java/src/main/java/io/pmem/kvdk/WriteOptions.java similarity index 100% rename from java/src/main/java/io/pmem/kvdk/WriteOptions.java rename to persistent/java/src/main/java/io/pmem/kvdk/WriteOptions.java diff --git a/persistent/java/src/main/resources/libatomic.so.1 b/persistent/java/src/main/resources/libatomic.so.1 new file mode 100644 index 00000000..15f244d7 Binary files /dev/null and b/persistent/java/src/main/resources/libatomic.so.1 differ diff --git a/persistent/java/src/main/resources/libstdc++.so.6 b/persistent/java/src/main/resources/libstdc++.so.6 new file mode 100644 index 00000000..a82db502 Binary files /dev/null and b/persistent/java/src/main/resources/libstdc++.so.6 differ diff --git a/java/src/test/java/io/pmem/kvdk/ConfigsTest.java b/persistent/java/src/test/java/io/pmem/kvdk/ConfigsTest.java similarity index 100% rename from java/src/test/java/io/pmem/kvdk/ConfigsTest.java rename to persistent/java/src/test/java/io/pmem/kvdk/ConfigsTest.java diff --git a/java/src/test/java/io/pmem/kvdk/EngineTest.java b/persistent/java/src/test/java/io/pmem/kvdk/EngineTest.java similarity index 100% rename from java/src/test/java/io/pmem/kvdk/EngineTest.java rename to persistent/java/src/test/java/io/pmem/kvdk/EngineTest.java diff --git a/java/src/test/java/io/pmem/kvdk/EngineTestBase.java b/persistent/java/src/test/java/io/pmem/kvdk/EngineTestBase.java similarity index 100% rename from java/src/test/java/io/pmem/kvdk/EngineTestBase.java rename to persistent/java/src/test/java/io/pmem/kvdk/EngineTestBase.java diff --git a/java/src/test/java/io/pmem/kvdk/IteratorTest.java b/persistent/java/src/test/java/io/pmem/kvdk/IteratorTest.java similarity index 100% rename from java/src/test/java/io/pmem/kvdk/IteratorTest.java rename to persistent/java/src/test/java/io/pmem/kvdk/IteratorTest.java diff --git a/java/src/test/java/io/pmem/kvdk/WriteBatchTest.java b/persistent/java/src/test/java/io/pmem/kvdk/WriteBatchTest.java similarity index 100% rename from java/src/test/java/io/pmem/kvdk/WriteBatchTest.java rename to persistent/java/src/test/java/io/pmem/kvdk/WriteBatchTest.java diff --git a/kvdk.pc.in b/persistent/kvdk.pc.in similarity index 100% rename from kvdk.pc.in rename to persistent/kvdk.pc.in diff --git a/scripts/benchmark_impl.py b/persistent/scripts/benchmark_impl.py similarity index 100% rename from scripts/benchmark_impl.py rename to persistent/scripts/benchmark_impl.py diff --git a/scripts/clang_format.sh b/persistent/scripts/clang_format.sh similarity index 100% rename from scripts/clang_format.sh rename to persistent/scripts/clang_format.sh diff --git a/scripts/clang_tidy.sh b/persistent/scripts/clang_tidy.sh similarity index 100% rename from scripts/clang_tidy.sh rename to persistent/scripts/clang_tidy.sh diff --git a/scripts/cppstyle b/persistent/scripts/cppstyle similarity index 100% rename from scripts/cppstyle rename to persistent/scripts/cppstyle diff --git a/scripts/init_devdax.sh b/persistent/scripts/init_devdax.sh similarity index 100% rename from scripts/init_devdax.sh rename to persistent/scripts/init_devdax.sh diff --git a/scripts/run_benchmark.py b/persistent/scripts/run_benchmark.py similarity index 100% rename from scripts/run_benchmark.py rename to persistent/scripts/run_benchmark.py diff --git a/scripts/test_coverage.sh b/persistent/scripts/test_coverage.sh similarity index 100% rename from scripts/test_coverage.sh rename to persistent/scripts/test_coverage.sh diff --git a/tests/CMakeLists.txt b/persistent/tests/CMakeLists.txt similarity index 100% rename from tests/CMakeLists.txt rename to persistent/tests/CMakeLists.txt diff --git a/tests/allocator.hpp b/persistent/tests/allocator.hpp similarity index 100% rename from tests/allocator.hpp rename to persistent/tests/allocator.hpp diff --git a/tests/c_api_test.hpp b/persistent/tests/c_api_test.hpp similarity index 97% rename from tests/c_api_test.hpp rename to persistent/tests/c_api_test.hpp index 569d1982..6035a534 100644 --- a/tests/c_api_test.hpp +++ b/persistent/tests/c_api_test.hpp @@ -10,7 +10,7 @@ #include #include "gtest/gtest.h" -#include "kvdk/engine.h" +#include "kvdk/persistent/engine.h" #include "test_util.h" class EngineCAPITestBase : public testing::Test { diff --git a/tests/c_api_test_hash.cpp b/persistent/tests/c_api_test_hash.cpp similarity index 100% rename from tests/c_api_test_hash.cpp rename to persistent/tests/c_api_test_hash.cpp diff --git a/tests/c_api_test_list.cpp b/persistent/tests/c_api_test_list.cpp similarity index 100% rename from tests/c_api_test_list.cpp rename to persistent/tests/c_api_test_list.cpp diff --git a/tests/pmem_allocator_bench.cpp b/persistent/tests/pmem_allocator_bench.cpp similarity index 99% rename from tests/pmem_allocator_bench.cpp rename to persistent/tests/pmem_allocator_bench.cpp index 4457eecd..2ca8e101 100644 --- a/tests/pmem_allocator_bench.cpp +++ b/persistent/tests/pmem_allocator_bench.cpp @@ -16,7 +16,7 @@ #include "../engine/kv_engine.hpp" #include "../engine/thread_manager.hpp" #include "allocator.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "test_util.h" using namespace KVDK_NAMESPACE; diff --git a/tests/stress_test.cpp b/persistent/tests/stress_test.cpp similarity index 99% rename from tests/stress_test.cpp rename to persistent/tests/stress_test.cpp index 391ad68f..38819130 100644 --- a/tests/stress_test.cpp +++ b/persistent/tests/stress_test.cpp @@ -17,8 +17,8 @@ #include #include -#include "kvdk/engine.hpp" -#include "kvdk/types.hpp" +#include "kvdk/persistent/engine.hpp" +#include "kvdk/persistent/types.hpp" #include "test_util.h" DEFINE_bool( diff --git a/tests/test_pmem_allocator.cpp b/persistent/tests/test_pmem_allocator.cpp similarity index 99% rename from tests/test_pmem_allocator.cpp rename to persistent/tests/test_pmem_allocator.cpp index 4f644c09..bef57d40 100644 --- a/tests/test_pmem_allocator.cpp +++ b/persistent/tests/test_pmem_allocator.cpp @@ -16,7 +16,7 @@ #include "../engine/logger.hpp" #include "../engine/thread_manager.hpp" #include "gtest/gtest.h" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "pmem_allocator/free_list.hpp" #include "pmem_allocator/pmem_allocator.hpp" #include "test_util.h" diff --git a/tests/test_util.h b/persistent/tests/test_util.h similarity index 100% rename from tests/test_util.h rename to persistent/tests/test_util.h diff --git a/tests/tests.cpp b/persistent/tests/tests.cpp similarity index 99% rename from tests/tests.cpp rename to persistent/tests/tests.cpp index 7f25c086..94856785 100644 --- a/tests/tests.cpp +++ b/persistent/tests/tests.cpp @@ -14,7 +14,7 @@ #include "../engine/kv_engine.hpp" #include "../engine/pmem_allocator/pmem_allocator.hpp" #include "../engine/utils/sync_point.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/persistent/engine.hpp" #include "test_util.h" DEFINE_string(path, "/mnt/pmem0/kvdk_unit_test", diff --git a/volatile/.clang-format b/volatile/.clang-format new file mode 100644 index 00000000..45dafec9 --- /dev/null +++ b/volatile/.clang-format @@ -0,0 +1,5 @@ +# Run manually to reformat a file: +# clang-format-9 -i --style=file +# find . -iname '*.cpp' -o -iname '*.hpp' | xargs clang-format-9 -i --style=file +BasedOnStyle: Google +DerivePointerAlignment: false diff --git a/volatile/.gitignore b/volatile/.gitignore new file mode 100644 index 00000000..47a99d9c --- /dev/null +++ b/volatile/.gitignore @@ -0,0 +1,53 @@ +# IDE related +.idea/ +cmake-build-* +.vscode/ +.vs/ + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +build/ +.cache/ + +scripts/result* +scripts/__pycache__ + +java/out +java/target +java/benchmark/target +java/examples/target +java/test-libs +java/*.log +java/include/io_pmem_*.h +java/src/main/resources diff --git a/volatile/CMakeLists.txt b/volatile/CMakeLists.txt new file mode 100644 index 00000000..39546299 --- /dev/null +++ b/volatile/CMakeLists.txt @@ -0,0 +1,197 @@ +cmake_minimum_required(VERSION 3.12.4) + +project(KVDK VERSION 1.0 + DESCRIPTION "A fast persistent KV engine for Persistent Memory" + LANGUAGES CXX C) + +set(KVDK_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +include(${KVDK_ROOT_DIR}/cmake/functions.cmake) +include(GNUInstallDirs) + +set(CMAKE_CXX_STANDARD 11) + +option(COVERAGE "code coverage" OFF) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mrdseed -mrdrnd -mclwb -mclflushopt") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + +if (CMAKE_BUILD_TYPE STREQUAL "Release") +elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") +elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel") +elseif (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -fsanitize=address -fsanitize-address-use-after-scope -fno-omit-frame-pointer") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -fsanitize=address -fsanitize-address-use-after-scope -fno-omit-frame-pointer") +elseif (CMAKE_BUILD_TYPE STREQUAL "FastDebug") + set(CMAKE_CXX_FLAGS "-Wall -Wextra -g -O2 -march=native") + set(CMAKE_C_FLAGS "-Wall -Wextra -g -O2 -march=native") +else () + message(FATAL_ERROR "Invalid build type!") +endif () + +if (CMAKE_BUILD_TYPE STREQUAL "Release") + add_compile_definitions(KVDK_DEBUG_LEVEL=0) +else () + add_compile_definitions(KVDK_DEBUG_LEVEL=1) + add_compile_definitions(KVDK_ENABLE_CRASHPOINT) +endif () + +# code coverage +if (COVERAGE) + if(NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) + message(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading" ) + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage -fprofile-arcs -ftest-coverage") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -fprofile-arcs -ftest-coverage") +endif() + +set (KVDK_SHARED_LIB_LINK_LIBS pthread gflags hwloc atomic) +set (KVDK_STATIC_LIB_LINK_LIBS "") + +# jemalloc +option(WITH_JEMALLOC "Build with jemalloc to support memory management with jemalloc" ON) +if (WITH_JEMALLOC) + message(STATUS "jemalloc support is enabled") + # libnuma static library, to support memory binding on numa nodes + add_subdirectory(extern/numactl-cmake) + + list(APPEND KVDK_SHARED_LIB_LINK_LIBS kvdk_extern_lib::numa) + list(APPEND KVDK_STATIC_LIB_LINK_LIBS kvdk_extern_lib::numa) + + # jemalloc + add_subdirectory(extern/jemalloc-cmake) + add_compile_definitions(KVDK_WITH_JEMALLOC) + + list(APPEND KVDK_SHARED_LIB_LINK_LIBS kvdk_extern_lib::jemalloc) + list(APPEND KVDK_STATIC_LIB_LINK_LIBS kvdk_extern_lib::jemalloc) +else () + message(STATUS "jemalloc support is disabled") +endif() + +# source files +set(SOURCES + engine/c/kvdk_basic_op.cpp + engine/c/kvdk_batch.cpp + engine/c/kvdk_hash.cpp + engine/c/kvdk_list.cpp + engine/c/kvdk_sorted.cpp + engine/c/kvdk_string.cpp + engine/utils/utils.cpp + engine/utils/sync_point.cpp + engine/engine.cpp + engine/kv_engine.cpp + engine/kv_engine_cleaner.cpp + engine/kv_engine_hash.cpp + engine/kv_engine_list.cpp + engine/kv_engine_sorted.cpp + engine/kv_engine_string.cpp + engine/logger.cpp + engine/hash_table.cpp + engine/sorted_collection/skiplist.cpp + engine/hash_collection/hash_list.cpp + engine/list_collection/list.cpp + engine/write_batch_impl.cpp + engine/dram_allocator.cpp + engine/thread_manager.cpp + engine/data_record.cpp + engine/dl_list.cpp + engine/version/old_records_cleaner.cpp + ) + +# .so library +add_library(engine SHARED ${SOURCES}) +target_include_directories(engine PUBLIC ./include ./extern) +target_link_libraries(engine PUBLIC ${KVDK_SHARED_LIB_LINK_LIBS}) + +configure_file(kvdk.pc.in kvdk.pc @ONLY) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/kvdk.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +set(KVDK_PUBLIC_HEADERS + ${PROJECT_SOURCE_DIR}/include/kvdk/comparator.hpp + ${PROJECT_SOURCE_DIR}/include/kvdk/configs.hpp + ${PROJECT_SOURCE_DIR}/include/kvdk/engine.h + ${PROJECT_SOURCE_DIR}/include/kvdk/engine.hpp + ${PROJECT_SOURCE_DIR}/include/kvdk/iterator.hpp + ${PROJECT_SOURCE_DIR}/include/kvdk/write_batch.hpp + ${PROJECT_SOURCE_DIR}/extern/libpmemobj++/string_view.hpp +) + +set_target_properties(engine PROPERTIES PUBLIC_HEADER "${KVDK_PUBLIC_HEADERS}") + +install(TARGETS engine + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/kvdk + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +# executables +add_executable(bench benchmark/bench.cpp) +target_link_libraries(bench PUBLIC engine) +target_include_directories(bench PUBLIC ./include ./extern ./) + +option(BUILD_TESTING "Build the tests" ON) +if (BUILD_TESTING) + add_subdirectory(${CMAKE_SOURCE_DIR}/extern/gtest) + add_subdirectory(tests) +endif () + + +option(BUILD_TUTORIAL "Build the tutorial" ON) +add_subdirectory(examples/tutorial) + +option(BUILD_GRAPH_SIMULATOR "Build the graph simulator" ON) + +add_custom_target(checkers ALL) +add_custom_target(cppstyle) +add_custom_target(cppformat) + +find_program(CLANG_FORMAT NAMES clang-format-9 clang-format-9.0 clang-format) +set(CLANG_FORMAT_MINIMUM_REQUIRED "9.0") +if (CLANG_FORMAT) + get_program_version_major_minor(${CLANG_FORMAT} CLANG_FORMAT_VERSION) + message(STATUS "Found clang-format: ${CLANG_FORMAT} (version: ${CLANG_FORMAT_VERSION})") +endif () + +if (CHECK_CPP_STYLE) + if (CLANG_FORMAT) + if (NOT (CLANG_FORMAT_VERSION VERSION_GREATER_EQUAL CLANG_FORMAT_MINIMUM_REQUIRED)) + message(FATAL_ERROR "minimum required clang-format version is ${CCLANG_FORMAT_MINIMUM_REQUIRED}") + endif () + else () + message(FATAL_ERROR "CHECK_CPP_STYLE=ON, but clang-format not found (required version: ${CLANG_FORMAT_MINIMUM_REQUIRED})") + endif () + + add_dependencies(checkers cppstyle) +endif () + +add_cppstyle(src ${CMAKE_CURRENT_SOURCE_DIR}/engine/*.c* + ${CMAKE_CURRENT_SOURCE_DIR}/engine/*.h* + ${CMAKE_CURRENT_SOURCE_DIR}/engine/utils/*.c* + ${CMAKE_CURRENT_SOURCE_DIR}/engine/utils/*.h* + ${CMAKE_CURRENT_SOURCE_DIR}/examples/tutorial/*.c* + ${CMAKE_CURRENT_SOURCE_DIR}/benchmark/*.c* + ${CMAKE_CURRENT_SOURCE_DIR}/include/kvdk/volatile/*.h* + ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.c* + ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.h*) + +option(WITH_JNI "Build with JNI" OFF) +if(WITH_JNI OR JNI) + message(STATUS "JNI library is enabled") + set(KVDK_SHARED_LIB engine) + set(KVDK_STATIC_LIB engine-static) + + include(FindPackageHandleStandardArgs) + + add_library(${KVDK_STATIC_LIB} STATIC ${SOURCES}) + target_include_directories(${KVDK_STATIC_LIB} PUBLIC ./include ./extern) + target_link_libraries(${KVDK_STATIC_LIB} PRIVATE ${KVDK_STATIC_LIB_LINK_LIBS} ${KVDK_SHARED_LIB_LINK_LIBS}) + + set_property(TARGET ${KVDK_STATIC_LIB} PROPERTY POSITION_INDEPENDENT_CODE ON) + + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java) + + add_cppstyle(jni_src + ${CMAKE_CURRENT_SOURCE_DIR}/java/kvdkjni/*.c* + ${CMAKE_CURRENT_SOURCE_DIR}/java/kvdkjni/*.h*) +else() + message(STATUS "JNI library is disabled") +endif() diff --git a/volatile/LICENSE b/volatile/LICENSE new file mode 100644 index 00000000..e2335064 --- /dev/null +++ b/volatile/LICENSE @@ -0,0 +1,30 @@ +BSD 3-Clause License +Copyright 2021, Intel Corporation + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/volatile/README.md b/volatile/README.md new file mode 100644 index 00000000..bc67cb87 --- /dev/null +++ b/volatile/README.md @@ -0,0 +1,122 @@ +
+


+
+ +This is the volatile module of `kvdk`, data stored on local DRAM. **This module is in development**. + +## Features +* Rich data types + * string, sorted, hash, list, hash +* Basic KV operations + * get/put/update/delete/scan +* Read-Modify-Write +* Support TTL +* Snapshot based Scan +* Consistent Dump & Restore to/from storage +* C/C++/Java APIs + +# Limitations +* The maximum supported key-value size is 64KB-4GB. +* No approach to key-value compression. + +## Getting the Source +```bash +git clone --recurse-submodules https://github.com/pmem/kvdk.git +``` + +## Building +### Install dependent tools and libraries on Ubuntu 18.04 +```bash +sudo apt install make clang-format-9 pkg-config g++ autoconf libtool asciidoctor libkmod-dev libudev-dev uuid-dev libjson-c-dev libkeyutils-dev pandoc libhwloc-dev libgflags-dev libtext-diff-perl bash-completion systemd wget git curl + +git clone https://github.com/pmem/ndctl.git +cd ndctl +git checkout v70.1 +./autogen.sh +./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib +make +sudo make install + +git clone https://github.com/pmem/pmdk.git +cd pmdk +git checkout 1.11.1 +make +sudo make install + +wget https://github.com/Kitware/CMake/releases/download/v3.12.4/cmake-3.12.4.tar.gz +tar vzxf cmake-3.12.4.tar.gz +cd cmake-3.12.4 +./bootstrap +make +sudo make install + +``` + +### Install dependent tools and libraries on CentOS 8 +```bash +yum config-manager --add-repo /etc/yum.repos.d/CentOS-Linux-PowerTools.repo +yum config-manager --set-enabled PowerTools + +yum install -y git gcc gcc-c++ autoconf automake asciidoc bash-completion xmlto libtool pkgconfig glib2 glib2-devel libfabric libfabric-devel doxygen graphviz pandoc ncurses kmod kmod-devel libudev-devel libuuid-devel json-c-devel keyutils-libs-devel gem make cmake libarchive clang-tools-extra hwloc-devel perl-Text-Diff gflags-devel curl + +git clone https://github.com/pmem/ndctl.git +cd ndctl +git checkout v70.1 +./autogen.sh +./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib +make +sudo make install + +git clone https://github.com/pmem/pmdk.git +cd pmdk +git checkout 1.11.1 +make +sudo make install + +wget https://github.com/Kitware/CMake/releases/download/v3.12.4/cmake-3.12.4.tar.gz +tar vzxf cmake-3.12.4.tar.gz +cd cmake-3.12.4 +./bootstrap +make +sudo make install +``` + +### Compile KVDK +```bash +mkdir -p build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Release -DCHECK_CPP_STYLE=ON && make -j +``` + +### How to test it on a system without PMEM +```bash +# set the correct path for pmdk library +export LD_LIBRARY_PATH=/usr/local/lib64 + +# setup a tmpfs for test +mkdir /mnt/pmem0 +mount -t tmpfs -o size=2G tmpfs /mnt/pmem0 + +# force the program work on non-pmem directory +export PMEM_IS_PMEM_FORCE=1 + +cd kvdk/build/examples +# Note: this requires CPU supporting AVX512 +./cpp_api_tutorial + +``` + +## Benchmarks +[Here](./doc/benchmark.md) are the examples of how to benchmark the performance of KVDK on your systems. + +## Documentations + +### User Guide + +Please refer to [User guide](./doc/user_doc.md) for API introductions of KVDK. + +### Architecture + +# Support +Welcome to join the wechat group or slack channel for KVDK tech discussion. +- [Wechat](https://github.com/pmem/kvdk/issues/143) +- [Slack Channel](https://join.slack.com/t/kvdksupportcommunity/shared_invite/zt-12b66vg1c-4FGb~Ri4w8K2_msau6v86Q) diff --git a/volatile/benchmark/bench.cpp b/volatile/benchmark/bench.cpp new file mode 100644 index 00000000..44592181 --- /dev/null +++ b/volatile/benchmark/bench.cpp @@ -0,0 +1,846 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "generator.hpp" +#include "kvdk/volatile/engine.hpp" +#include "kvdk/volatile/types.hpp" + +using namespace google; +using namespace KVDK_NAMESPACE; + +#define MAX_LAT (10000000) + +// Benchmark configs +DEFINE_string(path, "/mnt/pmem0/kvdk", "Instance path"); + +DEFINE_uint64(num_kv, (1 << 23), "Number of KVs to place"); + +DEFINE_uint64(num_operations, (1 << 20), + "Number of total operations. " + "num_kv will override this when benchmarking fill/insert"); + +DEFINE_int64(timeout, 30, + "Time to benchmark, this is valid only if fill=false"); + +DEFINE_uint64(value_size, 120, "Value size of KV"); + +DEFINE_string(value_size_distribution, "constant", + "Distribution of value size to write, can be constant/random, " + "default is constant. If set to random, the max value size " + "will be FLAGS_value_size."); + +DEFINE_uint64(threads, 10, + "Number of concurrent threads to run benchmark. " + "max_access_threads will override this when benchmarking fill."); + +DEFINE_bool(latency, false, "Stat operation latencies"); + +DEFINE_string(type, "string", + "Storage engine to benchmark, can be string, sorted, hash, list " + "or blackhole"); + +DEFINE_uint64(num_collection, 1, + "Number of collections in the instance to benchmark. " + "This will be ignored when benchmarking data type " + "string/blackhole."); + +DEFINE_uint64( + batch_size, 0, + "Size of write batch. If batch>0, write string type kv with atomic batch " + "write, this is valid only if we benchmark string engine"); + +DEFINE_string(key_distribution, "random", + "Distribution of benchmark keys, if fill is true, this para will " + "be ignored and only uniform distribution will be used"); + +// Engine configs +DEFINE_uint64(max_access_threads, 64, "Max access threads of the instance"); + +DEFINE_uint64(hash_bucket_num, (1 << 20), + "The number of initial buckets in hash table"); + +DEFINE_bool(opt_large_sorted_collection_restore, true, + " Optional optimization strategy which Multi-thread recovery a " + "skiplist. When having few large skiplists, the optimization can " + "get better performance"); + +DEFINE_bool(use_devdax_mode, false, "Use devdax device for kvdk"); + +DEFINE_string(dest_memory_nodes, "", + "Set the memory nodes where volatile KV " + "memory allocator binds to"); + +class Timer { + public: + void Start() { clock_gettime(CLOCK_REALTIME, &start); } + + std::uint64_t End() { + struct timespec end; + clock_gettime(CLOCK_REALTIME, &end); + return (end.tv_sec - start.tv_sec) * 1000000000 + + (end.tv_nsec - start.tv_nsec); + } + + private: + struct timespec start; +}; + +std::atomic_uint64_t read_ops{0}; +std::atomic_uint64_t write_ops{0}; +std::atomic_uint64_t read_not_found{0}; +std::atomic_uint64_t read_cnt{UINT64_MAX}; +std::vector collections; +Engine* engine{nullptr}; +std::string value_pool; +size_t operations_per_thread; +bool has_timed_out; +std::vector has_finished; // std::vector is a trap! +// Record operation latencies of access threads. Latencies of write threads +// stored in first part of the vector, latencies of read threads stored in +// second part +std::vector> latencies; + +std::vector random_engines; +std::vector ranges; + +enum class DataType { String, Sorted, Hashes, List, Blackhole } bench_data_type; + +enum class KeyDistribution { Range, Uniform, Zipf } key_dist; + +enum class ValueSizeDistribution { Constant, Uniform } vsz_dist; + +// Define variables those differ across benchmarks +bool fill = false; +double read_ratio = 0; +double existing_keys_ratio = 0; +std::uint64_t batch_size = 0; +bool scan = false; +std::uint64_t num_operations = 0; +std::uint64_t benchmark_threads = 0; + +std::uint64_t max_key = UINT64_MAX; +extd::zipfian_distribution* zipf = nullptr; +std::uniform_int_distribution uniform{0, UINT64_MAX}; + +std::uint64_t generate_key(size_t tid) { + switch (key_dist) { + case KeyDistribution::Range: { + return ranges[tid].gen(); + } + case KeyDistribution::Uniform: { + return uniform(random_engines[tid].gen); + } + case KeyDistribution::Zipf: { + return (*zipf)(random_engines[tid].gen); + } + default: { + throw; + } + } +} + +size_t generate_value_size(size_t tid) { + switch (vsz_dist) { + case ValueSizeDistribution::Constant: { + return FLAGS_value_size; + } + case ValueSizeDistribution::Uniform: { + return random_engines[tid].gen() % FLAGS_value_size + 1; + } + default: { + throw; + } + } +} + +void DBWrite(int tid) { + std::string key(8, ' '); + std::unique_ptr batch; + if (engine != nullptr) { + batch = engine->WriteBatchCreate(); + } + + for (size_t operations = 0; operations < operations_per_thread; + ++operations) { + if (has_timed_out) { + break; + } + + // generate key + std::uint64_t num = generate_key(tid); + std::uint64_t cid = num % FLAGS_num_collection; + memcpy(&key[0], &num, 8); + StringView value = StringView(value_pool.data(), generate_value_size(tid)); + + Timer timer; + if (FLAGS_latency) timer.Start(); + + Status s; + switch (bench_data_type) { + case DataType::String: { + if (batch_size == 0) { + s = engine->Put(key, value, WriteOptions()); + } else { + batch->StringPut(key, std::string{value.data(), value.size()}); + if ((operations + 1) % batch_size == 0) { + s = engine->BatchWrite(batch); + batch->Clear(); + } + } + break; + } + case DataType::Sorted: { + if (batch_size == 0) { + s = engine->SortedPut(collections[cid], key, value); + } else { + batch->SortedPut(collections[cid], key, + std::string{value.data(), value.size()}); + if ((operations + 1) % batch_size == 0) { + s = engine->BatchWrite(batch); + batch->Clear(); + } + } + break; + } + case DataType::Hashes: { + if (batch_size == 0) { + s = engine->HashPut(collections[cid], key, value); + } else { + batch->HashPut(collections[cid], key, + std::string{value.data(), value.size()}); + if ((operations + 1) % batch_size == 0) { + s = engine->BatchWrite(batch); + batch->Clear(); + } + } + break; + } + case DataType::List: { + s = engine->ListPushFront(collections[cid], value); + break; + } + case DataType::Blackhole: { + s = Status::Ok; + break; + } + default: { + throw std::runtime_error{"Unsupported data type!"}; + } + } + + if (FLAGS_latency) { + std::uint64_t lat = timer.End(); + if (lat / 100 >= MAX_LAT) { + throw std::runtime_error{"Write latency overflow"}; + } + latencies[tid][lat / 100]++; + } + + if (s != Status::Ok) { + throw std::runtime_error{"Put error"}; + } + + if ((operations + 1) % 1000 == 0) { + write_ops.fetch_add(1000); + }; + } + has_finished[tid] = 1; + return; +} + +void DBScan(int tid) { + std::string key(8, ' '); + std::string value_sink; + + for (size_t operations = 0, operations_counted = 0; + operations < operations_per_thread;) { + if (has_timed_out) { + break; + } + + std::uint64_t num = generate_key(tid); + std::uint64_t cid = num % FLAGS_num_collection; + memcpy(&key[0], &num, 8); + + switch (bench_data_type) { + case DataType::Sorted: { + size_t const scan_length = 100; + auto iter = engine->SortedIteratorCreate(collections[cid]); + if (iter) { + iter->Seek(key); + for (size_t i = 0; (i < scan_length) && (iter->Valid()); + i++, iter->Next()) { + key = iter->Key(); + value_sink = iter->Value(); + ++operations; + if (operations > operations_counted + 1000) { + read_ops.fetch_add(operations - operations_counted); + operations_counted = operations; + } + } + } else { + throw std::runtime_error{"Error creating SortedIterator"}; + } + engine->SortedIteratorRelease(iter); + break; + } + case DataType::Hashes: { + auto iter = engine->HashIteratorCreate(collections[cid]); + if (iter) { + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + key = iter->Key(); + value_sink = iter->Value(); + ++operations; + if (operations > operations_counted + 1000) { + read_ops += (operations - operations_counted); + operations_counted = operations; + } + } + } else { + throw std::runtime_error{"Error creating HashIterator"}; + } + engine->HashIteratorRelease(iter); + break; + } + case DataType::Blackhole: { + operations += 1024; + read_ops.fetch_add(1024); + break; + } + case DataType::String: + case DataType::List: + default: { + throw std::runtime_error{"Unsupported data type!"}; + } + } + } + + has_finished[tid] = 1; + return; +} + +void DBRead(int tid) { + std::string key(8, ' '); + std::string value_sink; + + std::uint64_t not_found = 0; + for (size_t operations = 0; operations < operations_per_thread; + ++operations) { + if (has_timed_out) { + break; + } + + std::uint64_t num = generate_key(tid); + std::uint64_t cid = num % FLAGS_num_collection; + memcpy(&key[0], &num, 8); + + Timer timer; + if (FLAGS_latency) timer.Start(); + + Status s; + switch (bench_data_type) { + case DataType::String: { + s = engine->Get(key, &value_sink); + break; + } + case DataType::Sorted: { + s = engine->SortedGet(collections[cid], key, &value_sink); + break; + } + case DataType::Hashes: { + s = engine->HashGet(collections[cid], key, &value_sink); + break; + } + case DataType::List: { + s = engine->ListPopBack(collections[cid], &value_sink); + break; + } + case DataType::Blackhole: { + s = Status::Ok; + break; + } + default: { + throw std::runtime_error{"Unsupported!"}; + } + } + + if (FLAGS_latency) { + auto lat = timer.End(); + if (lat / 100 >= MAX_LAT) { + fprintf(stderr, "Read latency overflow: %ld us\n", lat / 100); + std::abort(); + } + latencies[tid][lat / 100]++; + } + + if (s != Status::Ok) { + if (s != Status::NotFound) { + throw std::runtime_error{"Fail to Read"}; + } else { + if (++not_found % 1000 == 0) { + read_not_found.fetch_add(1000); + } + } + } + + if ((operations + 1) % 1000 == 0) { + read_ops.fetch_add(1000); + } + } + + has_finished[tid] = 1; + return; +} + +void InitializeBenchmark() { + if (FLAGS_type == "sorted") { + bench_data_type = DataType::Sorted; + } else if (FLAGS_type == "string") { + bench_data_type = DataType::String; + } else if (FLAGS_type == "hash") { + bench_data_type = DataType::Hashes; + } else if (FLAGS_type == "list") { + bench_data_type = DataType::List; + } else if (FLAGS_type == "blackhole") { + bench_data_type = DataType::Blackhole; + } else { + throw std::invalid_argument{"Unsupported data type"}; + } + + if (bench_data_type != DataType::Blackhole) { + Configs configs; + configs.max_access_threads = FLAGS_max_access_threads; + configs.hash_bucket_num = FLAGS_hash_bucket_num; + configs.opt_large_sorted_collection_recovery = + FLAGS_opt_large_sorted_collection_restore; + configs.dest_memory_nodes = FLAGS_dest_memory_nodes; + Status s = Engine::Open(FLAGS_path, &engine, configs, stdout); + if (s != Status::Ok) { + throw std::runtime_error{ + std::string{"Fail to open KVDK instance. Status: "} + + KVDKStatusStrings[static_cast(s)]}; + } + } + + { + value_pool.clear(); + value_pool.reserve(FLAGS_value_size); + std::default_random_engine rand_engine{42}; + for (size_t i = 0; i < FLAGS_value_size; i++) { + value_pool.push_back('a' + rand_engine() % 26); + } + } +} + +void ProcessBenchmarkConfigs() { + // Initialize collections and batch parameters + switch (bench_data_type) { + case DataType::String: + case DataType::Blackhole: { + break; + } + case DataType::Hashes: + case DataType::List: + case DataType::Sorted: { + collections.resize(FLAGS_num_collection); + for (size_t i = 0; i < FLAGS_num_collection; i++) { + collections[i] = "Collection_" + std::to_string(i); + } + break; + } + } + + if (batch_size > 0 && (bench_data_type == DataType::List)) { + throw std::invalid_argument{R"(List does not support batch write.)"}; + } + + // Check for scan flag + switch (bench_data_type) { + case DataType::String: + case DataType::List: { + if (scan) { + throw std::invalid_argument{ + R"(Scan is not supported for "String" and "List" type data.)"}; + } + } + default: { + break; + } + } + + if (FLAGS_value_size > 102400) { + throw std::invalid_argument{"value size too large"}; + } + + benchmark_threads = fill ? FLAGS_max_access_threads : FLAGS_threads; + random_engines.resize(benchmark_threads); + if (fill) { + assert(read_ratio == 0); + key_dist = KeyDistribution::Range; + operations_per_thread = FLAGS_num_kv / benchmark_threads + 1; + ranges.clear(); + for (size_t i = 0; i < benchmark_threads; i++) { + ranges.emplace_back(i * operations_per_thread, + (i + 1) * operations_per_thread); + } + } else { + operations_per_thread = num_operations / benchmark_threads; + if (FLAGS_key_distribution == "random") { + key_dist = KeyDistribution::Uniform; + } else if (FLAGS_key_distribution == "zipf") { + key_dist = KeyDistribution::Zipf; + } else { + throw std::invalid_argument{"Invalid key distribution"}; + } + } + + if (FLAGS_value_size_distribution == "constant") { + vsz_dist = ValueSizeDistribution::Constant; + } else if (FLAGS_value_size_distribution == "random") { + vsz_dist = ValueSizeDistribution::Uniform; + } else { + throw std::runtime_error{"Invalid value size distribution"}; + } + + max_key = existing_keys_ratio == 0 ? UINT64_MAX + : FLAGS_num_kv / existing_keys_ratio; + if (zipf) { + free(zipf); + } + zipf = new extd::zipfian_distribution(max_key, 0.99); + uniform = std::uniform_int_distribution(0, max_key); +} + +void ResetBenchmarkData() { + read_ops = 0; + write_ops = 0; + read_not_found = 0; + has_timed_out = false; + has_finished.clear(); + has_finished.resize(benchmark_threads, 0); + + if (FLAGS_latency) { + printf("calculate latencies\n"); + latencies.clear(); + latencies.resize(benchmark_threads, std::vector(MAX_LAT, 0)); + } +} + +void RunBenchmark() { + ProcessBenchmarkConfigs(); + ResetBenchmarkData(); + + size_t write_threads = + fill ? benchmark_threads + : benchmark_threads - read_ratio * 100 * benchmark_threads / 100; + int read_threads = fill ? 0 : benchmark_threads - write_threads; + std::vector ts; + + switch (bench_data_type) { + case DataType::Sorted: { + printf("Create %ld Sorted Collections\n", FLAGS_num_collection); + for (auto col : collections) { + SortedCollectionConfigs s_configs; + Status s = engine->SortedCreate(col, s_configs); + if (s != Status::Ok && s != Status::Existed) { + throw std::runtime_error{"Fail to create Sorted collection"}; + } + } + break; + } + case DataType::Hashes: { + for (auto col : collections) { + Status s = engine->HashCreate(col); + if (s != Status::Ok && s != Status::Existed) { + throw std::runtime_error{"Fail to create Hashset"}; + } + } + break; + } + case DataType::List: { + for (auto col : collections) { + Status s = engine->ListCreate(col); + if (s != Status::Ok && s != Status::Existed) { + throw std::runtime_error{"Fail to create List"}; + } + } + break; + } + default: { + break; + } + } + + std::cout << "Init " << read_threads << " readers " + << "and " << write_threads << " writers." << std::endl; + + for (size_t i = 0; i < write_threads; i++) { + ts.emplace_back(DBWrite, i); + } + for (size_t i = write_threads; i < benchmark_threads; i++) { + ts.emplace_back(scan ? DBScan : DBRead, i); + } + + size_t const field_width = 15; + std::cout << "----------------------------------------------------------\n" + << std::setw(field_width) << "Time(ms)" << std::setw(field_width) + << "Read Ops" << std::setw(field_width) << "Write Ops" + << std::setw(field_width) << "Not Found" << std::setw(field_width) + << "Total Read" << std::setw(field_width) << "Total Write" + << std::endl; + + std::vector read_cnt{0}; + std::vector write_cnt{0}; + std::vector notfound_cnt{0}; + size_t last_effective_idx = read_cnt.size(); + auto start_ts = std::chrono::system_clock::now(); + while (true) { + std::this_thread::sleep_for(std::chrono::seconds{1}); + auto duration = std::chrono::duration_cast( + std::chrono::system_clock::now() - start_ts); + + read_cnt.push_back(read_ops.load()); + write_cnt.push_back(write_ops.load()); + notfound_cnt.push_back(read_not_found.load()); + + size_t idx = read_cnt.size() - 1; + std::cout << std::setw(field_width) << duration.count() + << std::setw(field_width) << read_cnt[idx] - read_cnt[idx - 1] + << std::setw(field_width) << write_cnt[idx] - write_cnt[idx - 1] + << std::setw(field_width) + << notfound_cnt[idx] - notfound_cnt[idx - 1] + << std::setw(field_width) << read_cnt[idx] + << std::setw(field_width) << write_cnt[idx] << std::endl; + + size_t num_finished = + std::accumulate(has_finished.begin(), has_finished.end(), 0UL); + + if (num_finished == 0 || idx < 2) { + last_effective_idx = idx; + } + if (num_finished == benchmark_threads) { + break; + } + if (!fill && (duration.count() >= FLAGS_timeout * 1000)) { + // Signal a timeout for read, scan, update and insert + // Fill will never timeout + has_timed_out = true; + break; + } + } + + std::cout << "Benchmark finished." << std::endl; + printf("finish bench\n"); + + for (size_t i = 0; i < ts.size(); i++) { + ts[i].join(); + } + + size_t time_elapsed; + size_t total_effective_read; + size_t total_effective_write; + size_t const warmup_time = 2; + if (last_effective_idx <= warmup_time) { + time_elapsed = last_effective_idx; + total_effective_read = read_cnt[last_effective_idx]; + total_effective_write = write_cnt[last_effective_idx]; + } else { + time_elapsed = last_effective_idx - warmup_time; + total_effective_read = read_cnt[last_effective_idx] - read_cnt[warmup_time]; + total_effective_write = + write_cnt[last_effective_idx] - write_cnt[warmup_time]; + } + + std::cout << "----------------------------------------------------------\n" + << "Average Read Ops:\t" << total_effective_read / time_elapsed + << ". " + << "Average Write Ops:\t" << total_effective_write / time_elapsed + << std::endl; + + if (FLAGS_latency) { + auto ro = read_ops.load(); + if (ro > 0 && read_threads > 0) { + double total = 0; + double avg = 0; + double cur = 0; + double l50 = 0; + double l99 = 0; + double l995 = 0; + double l999 = 0; + double l9999 = 0; + for (std::uint64_t i = 1; i < MAX_LAT; i++) { + for (auto j = 0; j < read_threads; j++) { + cur += latencies[write_threads + j][i]; + total += latencies[write_threads + j][i] * i; + if (l50 == 0 && (double)cur / ro > 0.5) { + l50 = (double)i / 10; + } else if (l99 == 0 && (double)cur / ro > 0.99) { + l99 = (double)i / 10; + } else if (l995 == 0 && (double)cur / ro > 0.995) { + l995 = (double)i / 10; + } else if (l999 == 0 && (double)cur / ro > 0.999) { + l999 = (double)i / 10; + } else if (l9999 == 0 && (double)cur / ro > 0.9999) { + l9999 = (double)i / 10; + } + } + } + avg = total / ro / 10; + + printf( + "read lantencies (us): Avg: %.2f, P50: %.2f, P99: %.2f, P99.5: " + "%.2f, " + "P99.9: %.2f, P99.99: %.2f\n", + avg, l50, l99, l995, l999, l9999); + } + + auto wo = write_ops.load(); + if (wo > 0 && write_threads > 0) { + double total = 0; + double avg = 0; + double cur = 0; + double l50 = 0; + double l99 = 0; + double l995 = 0; + double l999 = 0; + double l9999 = 0; + for (std::uint64_t i = 1; i < MAX_LAT; i++) { + for (size_t j = 0; j < write_threads; j++) { + cur += latencies[j][i]; + total += latencies[j][i] * i; + if (l50 == 0 && (double)cur / wo > 0.5) { + l50 = (double)i / 10; + } else if (l99 == 0 && (double)cur / wo > 0.99) { + l99 = (double)i / 10; + } else if (l995 == 0 && (double)cur / wo > 0.995) { + l995 = (double)i / 10; + } else if (l999 == 0 && (double)cur / wo > 0.999) { + l999 = (double)i / 10; + } else if (l9999 == 0 && (double)cur / wo > 0.9999) { + l9999 = (double)i / 10; + } + } + } + avg = total / wo / 10; + + printf( + "write lantencies (us): Avg: %.2f, P50: %.2f, P99: %.2f, P99.5: " + "%.2f, " + "P99.9: %.2f, P99.99: %.2f\n", + avg, l50, l99, l995, l999, l9999); + } + } +} + +void FinalizeBenchmark() { + if (bench_data_type != DataType::Blackhole) delete engine; + + if (zipf) { + free(zipf); + } +} + +int main(int argc, char** argv) { + // Gflags function + ParseCommandLineFlags(&argc, &argv, true); + + InitializeBenchmark(); + + // fill + std::cout << "##########################################################\n" + << "Benchmark started: fill" << std::endl; + fill = true; + read_ratio = 0; + existing_keys_ratio = 0; + batch_size = 0; + scan = false; + num_operations = FLAGS_num_operations; + RunBenchmark(); + + // random batch insert + std::cout << "##########################################################\n" + << "Benchmark started: random batch insert" << std::endl; + fill = false; + read_ratio = 0; + existing_keys_ratio = 0; + batch_size = 100; + scan = false; + if (bench_data_type != DataType::Blackhole) { + num_operations = FLAGS_num_kv; + } + RunBenchmark(); + + // random insert + std::cout << "##########################################################\n" + << "Benchmark started: random insert" << std::endl; + fill = false; + read_ratio = 0; + existing_keys_ratio = 0; + batch_size = 0; + scan = false; + if (bench_data_type != DataType::Blackhole) { + num_operations = FLAGS_num_kv; + } + RunBenchmark(); + + // range scan + if (bench_data_type != DataType::String && + bench_data_type != DataType::List) { + std::cout << "##########################################################\n" + << "Benchmark started: range scan" << std::endl; + fill = false; + read_ratio = 1; + existing_keys_ratio = 1; + batch_size = 0; + scan = true; + num_operations = FLAGS_num_operations; + RunBenchmark(); + } + + // random read + + std::cout << "##########################################################\n" + << "Benchmark started: random read" << std::endl; + fill = false; + read_ratio = 1; + existing_keys_ratio = 1; + batch_size = 0; + scan = false; + num_operations = FLAGS_num_operations; + RunBenchmark(); + + // random read write (9R:1W) + std::cout << "##########################################################\n" + << "Benchmark started: random read write (9R:1W)" << std::endl; + fill = false; + read_ratio = 0.9; + existing_keys_ratio = 1; + batch_size = 0; + scan = false; + num_operations = FLAGS_num_operations; + RunBenchmark(); + + // random update + std::cout << "##########################################################\n" + << "Benchmark started: random update" << std::endl; + fill = false; + read_ratio = 0; + existing_keys_ratio = 1; + batch_size = 0; + scan = false; + num_operations = FLAGS_num_operations; + RunBenchmark(); + + FinalizeBenchmark(); + + return 0; +} diff --git a/volatile/benchmark/generator.hpp b/volatile/benchmark/generator.hpp new file mode 100644 index 00000000..4db8c06a --- /dev/null +++ b/volatile/benchmark/generator.hpp @@ -0,0 +1,27 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/rand64.hpp" +#include "utils/range_iterator.hpp" +#include "utils/zipf.hpp" + +struct PaddedEngine { + extd::xorshift_engine gen; + char padding[64]; +}; + +struct PaddedRangeIterators { + extd::range_iterator gen; + PaddedRangeIterators(std::uint64_t lo, std::uint64_t hi) : gen{lo, hi} {} + char padding[64]; +}; diff --git a/volatile/benchmark/utils/rand64.hpp b/volatile/benchmark/utils/rand64.hpp new file mode 100644 index 00000000..5b922f89 --- /dev/null +++ b/volatile/benchmark/utils/rand64.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace extd { +class xorshift_engine { + public: + using result_type = std::uint64_t; + + inline explicit xorshift_engine() { reseed(); } + + inline explicit xorshift_engine(std::uint64_t seed) : s{seed} { + assert(s != 0); + } + + inline result_type operator()() { + update_state(); + return s * magic; + } + + constexpr static result_type min() { + return std::numeric_limits::min(); + } + + constexpr static result_type max() { + return std::numeric_limits::max(); + } + + private: + void reseed() { + size_t failures = 0; + unsigned long long sink; + while (_rdseed64_step(&sink) != 1 || sink == 0) { + ++failures; + if (failures > 10000) throw std::runtime_error{"Fail to seed the engine"}; + } + s = sink; + } + + void update_state() { + if (s == 0) { + reseed(); + } + s ^= (s >> 12); + s ^= (s << 25); + s ^= (s >> 27); + } + + private: + result_type s; + static constexpr result_type magic = 0x2545F4914F6CDD1D; +}; + +} // namespace extd diff --git a/volatile/benchmark/utils/range_iterator.hpp b/volatile/benchmark/utils/range_iterator.hpp new file mode 100644 index 00000000..91ce5170 --- /dev/null +++ b/volatile/benchmark/utils/range_iterator.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +namespace extd { +template ::value, + bool>::type = true> +// Yield Number in range [lower, upper), by step. +class range_iterator { + public: + range_iterator(IntType lo, IntType hi, IntType s = 1) + : lower{lo}, upper{hi}, step{s}, curr{lower} {} + + IntType operator()() { + IntType old = curr; + curr += step; + assert(old < upper); + return old; + } + + private: + IntType const lower; + IntType const upper; + IntType const step; + IntType curr; +}; + +} // namespace extd diff --git a/volatile/benchmark/utils/zipf.hpp b/volatile/benchmark/utils/zipf.hpp new file mode 100644 index 00000000..e041dea0 --- /dev/null +++ b/volatile/benchmark/utils/zipf.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include + +namespace extd { + +// Param: s, N +// P(X=k) = k^{-s}/\sum_{i=1}^N i^{-s} = k^{-s}/zeta(N;s) +template +class zipfian_distribution { + public: + using result_type = IntType; + using float_type = double; + + static constexpr float_type default_s = 0.99; + static constexpr size_t precalc_n = 8; + static constexpr float_type half = 0.5; + + private: + float_type zeta_k[precalc_n]{}; + + result_type const N; + float_type const s; + + float_type const r = 1 - s; + float_type const inv_r = 1 / r; + float_type const a = precalc_n + 0.5; + float_type const a_exp_r = std::pow(a, r); + float_type const zeta_n{zeta(N)}; // Last to initialize! + + public: + inline zipfian_distribution(result_type n, float_type e = default_s) + : N{n}, s{e} { + for (size_t i = 0; i < precalc_n; i++) { + zeta_k[i] = zeta(i + 1); + } + } + + template + inline result_type operator()(UniformBitRandomGenerator& g) const { + using GeneratorIntType = typename UniformBitRandomGenerator::result_type; + static constexpr GeneratorIntType min = UniformBitRandomGenerator::min(); + static constexpr GeneratorIntType max = UniformBitRandomGenerator::max(); + GeneratorIntType r = g(); + float_type sum = (r - min) * zeta_n / (max - min); + assert(0 <= sum && sum <= zeta_n); + return inv_zeta(sum); + } + + inline float_type probability(result_type k) const { + if (k < 1 || k > N) { + return 0.0; + } + return std::pow(k, -s) / zeta_n; + } + + private: + // zeta(k) = sum_{i=1}^{k} i^{-s} + inline constexpr float_type zeta(result_type k) const { + constexpr result_type acc_n = 100; + if (k <= acc_n) { + float_type sum = 0; + for (result_type i = 1; i <= k; i++) { + sum += std::pow(i, -s); + } + return sum; + } else { + float_type a = acc_n + half; + float_type b = k + half; + if (s != 1.0) { + return zeta(acc_n) + (std::pow(b, r) - std::pow(a, r)) / r; + } else { + return zeta(acc_n) + (std::log(b) - std::log(a)); + } + } + } + + // (k = inv_zeta(sum)) <=> (zeta(k) <= sum < zeta(k+1)) + inline result_type inv_zeta(float_type sum) const { + for (size_t i = 0; i < precalc_n; i++) { + if (sum <= zeta_k[i]) { + return i + 1; + } + } + // integral == \int_a^b zeta(x)dx + float_type integral = sum - zeta_k[precalc_n - 1]; + assert(integral > 0); + + if (s != 1.0) { + float_type b = std::pow(integral * r + a_exp_r, inv_r); + assert(a < b && b <= N + half); + return std::floor(b + half); + } else { + float_type b = a * std::exp(integral); + assert(a < b && b <= N + half); + return std::floor(b + half); + } + } +}; + +} // namespace extd diff --git a/volatile/cmake/functions.cmake b/volatile/cmake/functions.cmake new file mode 100644 index 00000000..4849dfc0 --- /dev/null +++ b/volatile/cmake/functions.cmake @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018-2021, Intel Corporation + +# +# functions.cmake - helper functions for top-level CMakeLists.txt +# + +# Sets ${ret} to version of program specified by ${name} in major.minor format +function(get_program_version_major_minor name ret) + execute_process(COMMAND ${name} --version + OUTPUT_VARIABLE cmd_ret + ERROR_QUIET) + STRING(REGEX MATCH "([0-9]+.)([0-9]+)" VERSION ${cmd_ret}) + SET(${ret} ${VERSION} PARENT_SCOPE) +endfunction() + +# Generates cppstyle-$name and cppformat-$name targets and attaches them +# as dependencies of global "cppformat" target. +# cppstyle-$name target verifies C++ style of files in current source dir. +# cppformat-$name target reformats files in current source dir. +# If more arguments are used, then they are used as files to be checked +# instead. +# ${name} must be unique. +function(add_cppstyle name) + if(NOT CLANG_FORMAT OR NOT (CLANG_FORMAT_VERSION VERSION_GREATER_EQUAL CLANG_FORMAT_MINIMUM_REQUIRED)) + return() + endif() + + if(${ARGC} EQUAL 1) + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cppstyle-${name}-status + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp + COMMAND ${PERL_EXECUTABLE} + ${KVDK_ROOT_DIR}/scripts/cppstyle + ${CLANG_FORMAT} + check + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/cppstyle-${name}-status + ) + + add_custom_target(cppformat-${name} + COMMAND ${PERL_EXECUTABLE} + ${KVDK_ROOT_DIR}/scripts/cppstyle + ${CLANG_FORMAT} + format + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp + ) + else() + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cppstyle-${name}-status + DEPENDS ${ARGN} + COMMAND ${PERL_EXECUTABLE} + ${KVDK_ROOT_DIR}/scripts/cppstyle + ${CLANG_FORMAT} + check + ${ARGN} + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/cppstyle-${name}-status + ) + + add_custom_target(cppformat-${name} + COMMAND ${PERL_EXECUTABLE} + ${KVDK_ROOT_DIR}/scripts/cppstyle + ${CLANG_FORMAT} + format + ${ARGN} + ) + endif() + + add_custom_target(cppstyle-${name} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cppstyle-${name}-status) + + add_dependencies(cppstyle cppstyle-${name}) + add_dependencies(cppformat cppformat-${name}) +endfunction() diff --git a/volatile/doc/benchmark.md b/volatile/doc/benchmark.md new file mode 100644 index 00000000..9156c5a9 --- /dev/null +++ b/volatile/doc/benchmark.md @@ -0,0 +1,39 @@ +# Benchmark tool + +To test performance of KVDK, you can run our benchmark tool "bench", the tool is auto-built along with KVDK library in the build dir. + +Here is an example to run benchmarks on `string` type: +```bash +./bench -path=./kvdk_bench_dir -type=string -num_kv=8388608 -num_operations=1048576 -threads=10 -max_access_threads=64 -value_size=120 -latency=0 +``` + +To benchmark performance when KVs are stored on separted memory nodes, we can use `numactl`: +```bash +numactl --cpunodebind=0 --membind=0 ./bench -path=./kvdk_bench_dir -type=string -num_kv=8388608 -num_operations=1048576 -threads=10 -max_access_threads=64 -value_size=120 -latency=0 -dest_memory_nodes=1 +``` + +The above configurations will consume ~5.4 GB memory. Specifically, ~4.5 GB memory is used by KVDK. ~0.9 GB memory is used by the benchmark itself. + +Explanation of arguments: + + -path: KVDK initialized here + + -type: Type of key-value pairs to benchmark, it can be string/sorted/hash/list/blackhole. + + -num_kv: Number of KV when benchmarking fill/insert. + + -num_operations: Number of operations running benchmarks other than fill/insert. + + -threads: Number of threads of benchmark. `max_access_threads` will override this when benchmarking `fill`. + + -max_access_threads: Max concurrent access threads in the KVDK instance, set it to the number of the hyper-threads for performance consideration. You can call KVDK API with any number of threads, but if your parallel threads more than max_access_threads, the performance will be degraded due to synchronization cost. + + -value_size: Value length of values in Byte. + + -latency: Print latencies of operations or not. Enabling this makes benchmark use more memory. + + -dest_memory_nodes: The memory nodes to store KV data. + +## More configurations + +For more configurations of the benchmark tool, please reference to "benchmark/bench.cpp". diff --git a/volatile/doc/user_doc.md b/volatile/doc/user_doc.md new file mode 100644 index 00000000..84f70fe3 --- /dev/null +++ b/volatile/doc/user_doc.md @@ -0,0 +1,437 @@ +KVDK +======= + +KVDK(Key-Value Development Kit) is a Key-Value store for Persistent Memory (PMem). + +KVDK supports basic read and write operations on both sorted and unsorted KV-Pairs, it also support some advanced features, such as **backup**, **checkpoint**, **expire key**, **atomic batch write** and **transactions**. + +Code snippets in this user documents are from `./examples/tutorial/cpp_api_tutorial.cpp`, which is built as `./build/examples/tutorial/cpp_api_tutorial`. + +## Open a KVDK instance + +A KVDK instance is associated with a PMem directory mounted under linux system. Keys and values +are stored in PMem under this path and are indexed by HashTable which is stored in DRAM. +HashTable is not persisted on PMem and is reconstructed with information on PMem. + +The following code shows how to create a KVDK instance if none exist +or how to reopen an existing KVDK instance at the path supplied. +(In the following example, PMem is mounted as /mnt/pmem0/ and the KVDK instance is named tutorial_kvdk_example.) + +```c++ +#include "kvdk/volatile/engine.hpp" +#include "kvdk/volatile/namespace.hpp" +#include +#include +#include +#include +#include +#include + +#define Debug // For assert + +int main() +{ + kvdk::Status status; + kvdk::Engine *engine = nullptr; + + // Initialize a KVDK instance. + { + kvdk::Configs engine_configs; + { + // Configure for a tiny KVDK instance. + // Please refer to "Configuration" section in user documentation for + // details. + engine_configs.hash_bucket_num = (1ull << 10); + } + // The KVDK instance is mounted as a directory + // /mnt/pmem0/tutorial_kvdk_example. + // Modify this path if necessary. + std::string engine_path{"/mnt/pmem0/tutorial_kvdk_example"}; + + // Purge old KVDK instance + int sink = system(std::string{"rm -rf " + engine_path + "\n"}.c_str()); + + status = kvdk::Engine::Open(engine_path, &engine, engine_configs, stdout); + assert(status == kvdk::Status::Ok); + printf("Successfully opened a KVDK instance.\n"); + } + + ... Do something with KVDK instance ... + + ... Close KVDK instance and exit ... +} +``` + +## Status + +`kvdk::Status` indicates status of KVDK function calls. +Functions return `kvdk::Status::Ok` if such a function call is a success. +If exceptions are raised during function calls, other `kvdk::Status` is returned, +such as `kvdk::Status::MemoryOverflow` while no enough memory to allocate. + +## Close a KVDK instance + +To close a KVDK instance, just delete the instance. + +When a KVDK instance is closed, Key-Value pairs are still persisted on PMem. +KV-Pair indexing stored on DRAM are purged. +Follow "Open a KVDK instance" to reopen the KVDK instance will reconstruct indexing on DRAM with information persisted on PMem. + +To purge contents on PMem, just delete its directory under mounted PMem. + +```c++ +int main() +{ + ... Open a KVDK instance as described in "Open a KVDK instance" ... + ... Do something with KVDK instance ... + + // Close KVDK instance. + delete engine; + + // Remove persisted contents on PMem + return system(std::string{"rm -rf " + engine_path + "\n"}.c_str()); +} +``` + +## Data types +KVDK currently supports raw string, sorted collection, hash collection and list data type. + +### Raw String + +All keys and values in a KVDK instance are strings. You can directly store or read key-value pairs in global namespace, which is accessible via Get, Put, Delete and Modify operations, we call them string type data in kvdk. + +Keys are limited to have a maximum size of 64KB. + +A value can be at max 64MB in length by default. The maximum length can be configured when initializing a KVDK instance. + +### Collections + +Instead of raw string, you can organize key-value pairs to a collection, each collection has its own namespace. + +Currently we have three types of collection: + +#### Sorted Collection + +KV pairs are stored with some kind of order (lexicographical order by default) in Sorted Collection, they can be iterated forward or backward starting from an arbitrary point(at a key or between two keys) by an iterator. They can also be directly accessed via SortedGet, SortedPut, SortedDelete operations. + +#### Hash Collection + +Hash Collection is like Raw String with a name space, you can access KV pairs via HashGet, HashPut, HashDelete and HashModify operations. + +In current version, performance of operations on hash collection is similar to sorted collection, which much slower than raw-string, so we recomend use raw-string or sorted collection as high priority. + +#### List + +List is a list of string elements, you can access elems at the front or back via ListPushFront, ListPushBack, ListPopFron, ListPopBack, or operation elems with index via ListInsertAt, ListInsertBefore, ListInsertAfter and ListErase. Notice that operation with index take O(n) time, while operation on front and back only takes O(1). + +### Namespace + +Each collection has its own namespace, so you can store same key in every collection. Howevery, collection name and raw string key are in a same namespace, so you can't assign same name for a collection and a string key, otherwise a error status (Status::WrongType) will be returned. + +## API Examples + +### Reads and Writes with String type + +A KVDK instance provides Get, Put, Delete methods to query/modify/delete raw string kvs. + +The following code performs a series of Get, Put and Delete operations. + +```c++ +int main() +{ + ... Open a KVDK instance as described in "Open a KVDK instance" ... + + // Reads and Writes String KV + { + std::string key1{"key1"}; + std::string key2{"key2"}; + std::string value1{"value1"}; + std::string value2{"value2"}; + std::string v; + + // Insert key1-value1 + status = engine->Put(key1, value1); + assert(status == kvdk::Status::Ok); + + // Get value1 by key1 + status = engine->Get(key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value1); + + // Update key1-value1 to key1-value2 + status = engine->Put(key1, value2); + assert(status == kvdk::Status::Ok); + + // Get value2 by key1 + status = engine->Get(key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value2); + + // Insert key2-value2 + status = engine->Put(key2, value2); + assert(status == kvdk::Status::Ok); + + // Delete key1-value2 + status = engine->Delete(key1); + assert(status == kvdk::Status::Ok); + + // Delete key2-value2 + status = engine->Delete(key2); + assert(status == kvdk::Status::Ok); + + printf("Successfully performed Get, Put, Delete operations on anonymous " + "global collection.\n"); + } + + ... Do something else with KVDK instance ... + + ... Close KVDK instance and exit ... +} +``` + +### Reads and Writes in a Sorted Collection + +A KVDK instance provides SortedGet, SortedPut, SortedDelete methods to query/modify/delete sorted entries. + +The following code performs a series of SortedGet, SortedPut and SortedDelete operations on a sorted collection. + +```c++ +int main() +{ + ... Open a KVDK instance as described in "Open a KVDK instance" ... + + // Reads and Writes on Named Collection + { + std::string collection1{"my_collection_1"}; + std::string collection2{"my_collection_2"}; + std::string key1{"key1"}; + std::string key2{"key2"}; + std::string value1{"value1"}; + std::string value2{"value2"}; + std::string v; + + // You must create sorted collections before you do any operations on them + status = engine->SortedCreate(collection1); + assert(status == kvdk::Status::Ok); + status = engine->SortedCreate(collection2); + assert(status == kvdk::Status::Ok); + + // Insert key1-value1 into "my_collection_1". + status = engine->SortedPut(collection1, key1, value1); + assert(status == kvdk::Status::Ok); + + // Get value1 by key1 in collection "my_collection_1" + status = engine->SortedGet(collection1, key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value1); + + // Insert key1-value2 into "my_collection_2". + status = engine->SortedPut(collection2, key1, value2); + assert(status == kvdk::Status::Ok); + + // Get value2 by key1 in collection "my_collection_2" + status = engine->SortedGet(collection2, key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value2); + + // Get value1 by key1 in collection "my_collection_1" + // key1-value2 is stored in "my_collection_2" + // Thus key1-value1 stored in "my_collection_1" is unaffected by operation + // engine->SortedPut(collection2, key1, value2). + status = engine->SortedGet(collection1, key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value1); + + // Insert key2-value2 into collection "my_collection_2" + // Collection "my_collection_2" already exists and no implicit collection + // creation occurs. + status = engine->SortedPut(collection2, key2, value2); + assert(status == kvdk::Status::Ok); + + // Delete key1-value1 in collection "my_collection_1" + // Although "my_collection_1" has no elements now, the collection itself is + // not deleted though. + status = engine->SortedDelete(collection1, key1); + assert(status == kvdk::Status::Ok); + + // Destroy sorted collections + status = engine->SortedDestroy(collection1); + assert(status == kvdk::Status::Ok); + status = engine->SrotedDestroy(collection2); + assert(status == kvdk::Status::Ok); + + printf("Successfully performed SortedGet, SortedPut, SortedDelete operations.\n"); + } + + ... Do something else with KVDK instance ... + + ... Close KVDK instance and exit ... +} +``` + +### Iterating a Sorted Collection +The following example demonstrates how to iterate through a sorted collection at a consistent view of data. It also demonstrates how to iterate through a range defined by Key. + +```c++ +int main() +{ + ... Open a KVDK instance as described in "Open a KVDK instance" ... + + // Iterating a Sorted Sorted Collection + { + std::string sorted_collection{"my_sorted_collection"}; + engine->SortedCreate(sorted_collection); + // Create toy keys and values. + std::vector> kv_pairs; + for (int i = 0; i < 10; ++i) { + kv_pairs.emplace_back(std::make_pair("key" + std::to_string(i), + "value" + std::to_string(i))); + } + std::shuffle(kv_pairs.begin(), kv_pairs.end(), std::mt19937{42}); + // Print out kv_pairs to check if they are really shuffled. + printf("The shuffled kv-pairs are:\n"); + for (const auto &kv : kv_pairs) + printf("%s\t%s\n", kv.first.c_str(), kv.second.c_str()); + + // Populate collection "my_sorted_collection" with keys and values. + // kv_pairs are not necessarily sorted, but kv-pairs in collection + // "my_sorted_collection" are sorted. + for (int i = 0; i < 10; ++i) { + // Collection "my_sorted_collection" is implicitly created in first + // iteration + status = engine->SortedPut(sorted_collection, kv_pairs[i].first, + kv_pairs[i].second); + assert(status == kvdk::Status::Ok); + } + // Sort kv_pairs for checking the order of "my_sorted_collection". + std::sort(kv_pairs.begin(), kv_pairs.end()); + + // Iterate through collection "my_sorted_collection", the iter is + // created on a consistent view while you create it, e.g. all + // modifications after you create the iter won't be observed + auto iter = engine->SortedIteratorCreate(sorted_collection); + iter->SeekToFirst(); + { + int i = 0; + while (iter->Valid()) { + assert(iter->Key() == kv_pairs[i].first); + assert(iter->Value() == kv_pairs[i].second); + iter->Next(); + ++i; + } + } + + // Iterate through range ["key1", "key8"). + std::string beg{"key1"}; + std::string end{"key8"}; + { + int i = 1; + iter->Seek(beg); + for (iter->Seek(beg); iter->Valid() && iter->Key() < end; iter->Next()) { + assert(iter->Key() == kv_pairs[i].first); + assert(iter->Value() == kv_pairs[i].second); + ++i; + } + } + + // Reversely iterate through range ["key8", "key1"). + beg = "key8"; + end = "key1"; + { + int i = 8; + for (iter->Seek(beg); iter->Valid() && iter->Key() > end; iter->Prev()) { + assert(iter->Key() == kv_pairs[i].first); + assert(iter->Value() == kv_pairs[i].second); + --i; + } + } + + printf("Successfully iterated through a sorted collections.\n"); + engine->SortedIteratorRelease(iter); + } + + ... Do something else with KVDK instance ... + + ... Close KVDK instance and exit ... +} +``` + +### Atomic Updates +KVDK supports organizing a series of Put, Delete operations into a `kvdk::WriteBatch` object as an atomic operation. If KVDK fail to apply the `kvdk::WriteBatch` object as a whole, i.e. the system shuts down during applying the batch, it will roll back to the status right before applying the `kvdk::WriteBatch`. + +```c++ +int main() +{ + ... Open a KVDK instance as described in "Open a KVDK instance" ... + + // BatchWrite on Anonymous Global Collection + { + std::string key1{"key1"}; + std::string key2{"key2"}; + std::string value1{"value1"}; + std::string value2{"value2"}; + std::string v; + + kvdk::WriteBatch batch; + batch.Put(key1, value1); + batch.Put(key1, value2); + batch.Put(key2, value2); + batch.Delete(key2); + + // If the batch is successfully written, there should be only key1-value2 in + // anonymous global collection. + status = engine->BatchWrite(batch); + assert(status == kvdk::Status::Ok); + + // Get value2 by key1 + status = engine->Get(key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value2); + + // Get value2 by key1 + status = engine->Get(key2, &v); + assert(status == kvdk::Status::NotFound); + // v is unchanged, but it is invalid. Always Check kvdk::Status before + // perform further operations! + assert(v == value2); + + printf( + "Successfully performed BatchWrite on anonymous global collection.\n"); + } + + ... Do something else with KVDK instance ... + + ... Close KVDK instance and exit ... +} +``` + +## Concurrency +A KVDK instance can be accessed by multiple read and write threads safely. Synchronization is handled by KVDK implementation. + +## Configuration + +Users can configure KVDK to adapt to their system environment by setting up a `kvdk::Configs` object and passing it to 'kvdk::Engine::Open' when initializing a KVDK instance. + +### Max Access Threads +Maximum number of internal access threads in kvdk is specified by `kvdk::Configs::max_access_threads`. Defaulted to 64. It's recommended to set this number to the number of threads provided by CPU. + +You can call KVDK API with any number of threads, but if your parallel threads more than max_access_threads, the performance will be degraded due to synchronization cost + +### Clean Threads +KVDK reclaim space of updated/deleted data in background with dynamic number of clean threads, you can specify max clean thread number with `kvdk::Configs::clean_threads`. Defaulted to 8, you can config more clean threads in delete intensive workloads to avoid space be exhausted. + +**This parameter is immutable after initialization of the KVDK instance.** + +### HashBucket Size +Specified by `kvdk::Configs::hash_bucket_size`. Defaulted to 128(Bytes). +Larger HashBucket Size will slightly improve performance but will occupy larger space. Please read Architecture Documentation for details before tuning this parameter. + +### Number of HashBucket Chains +Specified by `kvdk::Configs::hash_bucket_num`. Greater number will improve performance by reducing hash conflicts at the cost of greater DRAM space. Please read Architecture Documentation for details before tuning this parameter. + +### Buckets per Slot +Specified by `kvdk::Configs::num_buckets_per_slot`. Smaller number will improve performance by reducing lock contentions and improving caching at the cost of greater DRAM space. Please read Architecture Documentation for details before tuning this parameter. + +## Advanced features and more API + +Please read examples/tutorial for more API and advanced features in KVDK. diff --git a/volatile/engine/alias.hpp b/volatile/engine/alias.hpp new file mode 100644 index 00000000..968de154 --- /dev/null +++ b/volatile/engine/alias.hpp @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include + +#include "kvdk/volatile/types.hpp" + +namespace KVDK_NAMESPACE { +enum class WriteOp { Put, Delete }; + +// Internal types +using MemoryOffsetType = std::uint64_t; +using TimestampType = std::uint64_t; + +const uint64_t kMaxCachedOldRecords = 1024; +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/allocator.hpp b/volatile/engine/allocator.hpp new file mode 100644 index 00000000..bcf13c67 --- /dev/null +++ b/volatile/engine/allocator.hpp @@ -0,0 +1,335 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include + +#include "alias.hpp" +#include "logger.hpp" +#include "macros.hpp" +#include "thread_manager.hpp" + +namespace KVDK_NAMESPACE { + +struct DLRecord; +struct StringRecord; + +constexpr MemoryOffsetType kNullMemoryOffset = UINT64_MAX; + +struct SpaceEntry { + class SpaceCmp { + public: + bool operator()(const SpaceEntry& s1, const SpaceEntry& s2) const { + if (s1.size > s2.size) return true; + if (s1.size == s2.size && s1.offset < s2.offset) return true; + return false; + } + }; + SpaceEntry() = default; + + SpaceEntry(uint64_t _offset, uint64_t _size) : offset(_offset), size(_size) {} + uint64_t offset; + uint64_t size = 0; +}; + +class Allocator { + public: + virtual SpaceEntry Allocate(uint64_t size) = 0; + virtual SpaceEntry AllocateAligned(size_t alignment, uint64_t size) = 0; + virtual void Free(const SpaceEntry& entry) = 0; + virtual std::string AllocatorName() = 0; + + virtual void BatchFree(const std::vector& entries) { + for (const SpaceEntry& entry : entries) { + this->Free(entry); + } + } + + // Set up this allocator for multi-threads access. + // Resource initialization may be involved. + virtual bool SetMaxAccessThreads(uint32_t max_access_threads) { + auto ul = AcquireLock(); + return EnableThreadLocalCounters(max_access_threads); + } + + // Set the backend memory nodes for this allocator. + // Some allocators may not support this feature. + virtual void SetDestMemoryNodes(std::string) {} + + Allocator(char* base_addr, uint64_t max_offset) + : base_addr_(base_addr), max_offset_(max_offset) {} + + virtual ~Allocator() {} + + // Purge a kvdk data record and free it + template + void PurgeAndFree(T* data_record) { + static_assert(std::is_same::value || + std::is_same::value, + ""); + data_record->Destroy(); + Free(SpaceEntry(addr2offset_checked(data_record), + data_record->GetRecordSize())); + } + + inline void LogAllocation(int tid, size_t sz) { + if (KVDK_UNLIKELY(tid == -1 || !thread_local_counter_enabled_)) { + global_allocated_size_.fetch_add(sz); + } else { + assert(tid >= 0); + assert(allocator_thread_cache_.size() > 0); + allocator_thread_cache_[tid % allocator_thread_cache_.size()] + .allocated_size += sz; + } + } + + inline void LogDeallocation(int tid, size_t sz) { + if (KVDK_UNLIKELY(tid == -1 || !thread_local_counter_enabled_)) { + global_allocated_size_.fetch_sub(sz); + } else { + assert(tid >= 0); + assert(allocator_thread_cache_.size() > 0); + allocator_thread_cache_[tid % allocator_thread_cache_.size()] + .allocated_size -= sz; + } + } + + std::int64_t BytesAllocated() const { + std::int64_t total = 0; + for (auto& tc : allocator_thread_cache_) { + total += tc.allocated_size; + } + total += global_allocated_size_.load(); + return total; + } + + std::unique_lock AcquireLock() { + return std::unique_lock{allocator_spin_}; + } + + // translate offset of allocated space to address + inline void* offset2addr_checked(MemoryOffsetType offset) const { + assert(validate_offset(offset) && "Trying to access invalid offset"); + return base_addr_ + offset; + } + + inline void* offset2addr(MemoryOffsetType offset) const { + if (validate_offset(offset)) { + return base_addr_ + offset; + } + return nullptr; + } + + template + inline T* offset2addr_checked(MemoryOffsetType offset) const { + return static_cast(offset2addr_checked(offset)); + } + + template + inline T* offset2addr(MemoryOffsetType offset) const { + return static_cast(offset2addr(offset)); + } + + // translate address of allocated space to offset + inline MemoryOffsetType addr2offset_checked(const void* addr) const { + assert((char*)addr >= base_addr_); + MemoryOffsetType offset = (char*)addr - base_addr_; + assert(validate_offset(offset) && "Trying to create invalid offset"); + return offset; + } + + inline MemoryOffsetType addr2offset(const void* addr) const { + if (addr) { + MemoryOffsetType offset = (char*)addr - base_addr_; + if (validate_offset(offset)) { + return offset; + } + } + return kNullMemoryOffset; + } + + inline bool validate_offset(uint64_t offset) const { + return offset < max_offset_; + } + + protected: + /** + * @brief Enable thread local memory usage counters. + * This helps to avoid possible bottleneck when multiple threads + * access the std::atomic `global_allocated_size_`. + * + * @param max_access_threads + * @return true Success + * @return false Failure + */ + bool EnableThreadLocalCounters(uint32_t max_access_threads) { + if (max_access_threads < 1) { + GlobalLogger.Error( + "Invalid number of thread local counters for Allocator: %u.\n", + max_access_threads); + return false; + } + + thread_local_counter_enabled_ = true; + allocator_thread_cache_.resize(max_access_threads); + + return true; + } + + private: + struct alignas(64) AllocatorThreadCache { + int64_t allocated_size; + // align to cache line to avoid cache conherence + char padding[64 - sizeof(int64_t)]; + }; + static_assert(sizeof(AllocatorThreadCache) % 64 == 0); + + char* base_addr_; + uint64_t max_offset_; + + bool thread_local_counter_enabled_{}; + std::vector allocator_thread_cache_; + std::atomic global_allocated_size_{0}; + + SpinMutex allocator_spin_; +}; + +// Global default memory allocator +Allocator* global_memory_allocator(); + +/// Caution: AlignedPoolAllocator is not thread-safe +template +class AlignedPoolAllocator { + static_assert(alignof(T) <= 1024, + "Alignment greater than 1024B not supported"); + + public: + using value_type = T; + + explicit inline AlignedPoolAllocator() : pools_{}, pos_{TrunkSize} {} + + inline AlignedPoolAllocator(AlignedPoolAllocator const&) + : AlignedPoolAllocator{} {} + AlignedPoolAllocator(AlignedPoolAllocator&&) = delete; + ~AlignedPoolAllocator() { + for (auto p : pools_) { + free(p); + } + } + + inline T* allocate(size_t n) { + if (pools_.capacity() < 64) { + pools_.reserve(64); + } + + if (pos_ + n <= TrunkSize) { + size_t old_pos = pos_; + pos_ += n; + return &pools_.back()[old_pos]; + } else if (n <= TrunkSize) { + allocate_trunk(); + pos_ = n; + return &pools_.back()[0]; + } else { + allocate_trunk(n); + pos_ = TrunkSize; + return &pools_.back()[0]; + } + } + + inline void deallocate(T*, size_t) noexcept { return; } + + private: + inline void allocate_trunk(size_t sz = TrunkSize) { + pools_.emplace_back( + static_cast(aligned_alloc(alignof(T), sizeof(T) * sz))); + if (pools_.back() == nullptr || alignof(pools_.back()[0]) != alignof(T)) { + throw std::bad_alloc{}; + } + } + + static constexpr size_t TrunkSize = 1024; + std::vector pools_; + size_t pos_; +}; + +// Thread safety guaranteed by aligned_alloc +template +class AlignedAllocator { + public: + using value_type = T; + + inline T* allocate(size_t n) { + static_assert(sizeof(T) % alignof(T) == 0); + T* p = static_cast(aligned_alloc(alignof(T), n * sizeof(T))); + if (p == nullptr) { + throw std::bad_alloc{}; + } + return p; + } + + inline void deallocate(T* p, size_t) noexcept { free(p); } +}; + +template +class Array { + public: + template + explicit Array(uint64_t size, Args&&... args) : size_(size) { + static_assert(sizeof(T) % alignof(T) == 0); + allocated = alloc_->AllocateAligned(alignof(T), size * sizeof(T)); + if (allocated.size == 0) { + throw std::bad_alloc{}; + } + + data_ = alloc_->offset2addr(allocated.offset); + for (uint64_t i = 0; i < size; i++) { + new (data_ + i) T{std::forward(args)...}; + } + } + + Array(const Array&) = delete; + Array& operator=(const Array&) = delete; + Array(Array&&) = delete; + + Array() : size_(0), data_(nullptr){}; + + ~Array() { + if (data_ != nullptr) { + for (uint64_t i = 0; i < size_; i++) { + data_[i].~T(); + } + alloc_->Free(allocated); + } + } + + T& back() { + assert(size_ > 0); + return data_[size_ - 1]; + } + + T& front() { + assert(size_ > 0); + return data_[0]; + } + + T& operator[](uint64_t index) { + if (index >= size_) { + throw std::out_of_range("array out of range"); + } + return data_[index]; + } + + uint64_t size() { return size_; } + + private: + uint64_t const size_; + T* data_{nullptr}; + Allocator* alloc_ = global_memory_allocator(); + SpaceEntry allocated; +}; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/backup_log.hpp b/volatile/engine/backup_log.hpp new file mode 100644 index 00000000..6013aa82 --- /dev/null +++ b/volatile/engine/backup_log.hpp @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include +#include +#include + +#include + +#include "data_record.hpp" +#include "kvdk/volatile/configs.hpp" +#include "kvdk/volatile/types.hpp" +#include "logger.hpp" + +namespace KVDK_NAMESPACE { + +constexpr size_t kMaxBackupLogBufferSize = 128 << 20; + +enum class BackupStage { + NotFinished = 0, + Finished, +}; + +// Persist/Read backup of a kvdk instance as log-like manner +// Format: +// stage | string record 1 | string record 2 | sorted header record 1 | sorted +// elems of header 1 | ... sorted header n | sorted elems of header n | ... | +// string record n| ... +class BackupLog { + public: + struct LogRecord { + RecordType type; + std::string key; + std::string val; + ExpireTimeType expire_time; + + bool DecodeFrom(StringView* str) { + return FetchUint32(str, (uint32_t*)&type) && + FetchFixedString(str, &key) & FetchFixedString(str, &val) && + FetchUint64(str, (uint64_t*)&expire_time); + } + + void EncodeTo(std::string* str) { + assert(str != nullptr); + AppendUint32(str, type); + AppendFixedString(str, key); + AppendFixedString(str, val); + AppendUint64(str, expire_time); + } + }; + + // Iterate log records on a backup log + class LogIterator { + public: + LogIterator(StringView records_on_file) + : records_on_file_(records_on_file) { + valid_ = curr_.DecodeFrom(&records_on_file_); + } + + const LogRecord& Record() { return curr_; } + void Next() { + if (Valid()) { + valid_ = curr_.DecodeFrom(&records_on_file_); + } + } + + bool Valid() { return valid_; } + + private: + StringView records_on_file_; + LogRecord curr_; + bool valid_ = true; + }; + + ~BackupLog() { Close(); } + + BackupLog() = default; + BackupLog(const BackupLog&) = delete; + + // Init a new backup log + Status Init(const std::string& backup_log) { + Status s = Status::Ok; + if (file_exist(backup_log)) { + GlobalLogger.Error("Init backup log %s error: file already exist\n", + backup_log.c_str()); + s = Status::Abort; + } + if (s == Status::Ok) { + file_name_ = backup_log; + fd_ = open(backup_log.c_str(), O_CREAT | O_RDWR, 0666); + if (fd_ < 0) { + GlobalLogger.Error("Init bakcup log %s error while opening file: %s\n", + backup_log.c_str(), strerror(errno)); + s = Status::IOError; + } + } + + if (s == Status::Ok) { + stage_ = BackupStage::NotFinished; + file_size_ = write(fd_, &stage_, sizeof(BackupStage)); + if (file_size_ != sizeof(BackupStage)) { + GlobalLogger.Error( + "Init bakcup log %s error while writing backup stage: %s\n", + backup_log.c_str(), strerror(errno)); + s = Status::IOError; + } + } + + if (s == Status::Ok) { + log_file_ = mmap(nullptr, sizeof(BackupStage), PROT_WRITE | PROT_READ, + MAP_SHARED, fd_, 0); + if (log_file_ == MAP_FAILED) { + GlobalLogger.Error( + "Init bakcup log %s error while mapping log file: %s\n", + backup_log.c_str(), strerror(errno)); + log_file_ = nullptr; + s = Status::IOError; + } + } + + if (s != Status::Ok) { + Destroy(); + } + return s; + } + + // Open a existing backup log + Status Open(const std::string& backup_log) { + Status s = Status::Ok; + file_name_ = backup_log; + fd_ = open(backup_log.c_str(), O_RDWR, 0666); + if (fd_ < 0) { + GlobalLogger.Error("Open bakcup log %s error while opening file: %s\n", + backup_log.c_str(), strerror(errno)); + s = Status::IOError; + } + + if (s == Status::Ok) { + file_size_ = lseek(fd_, 0, SEEK_END); + if (file_size_ < sizeof(BackupStage)) { + GlobalLogger.Error( + "Open backup log file %s error: file size %lu smaller than " + "persisted " + "stage flag", + backup_log.size(), file_size_); + s = Status::Abort; + } + } + + if (s == Status::Ok) { + log_file_ = + mmap(nullptr, file_size_, PROT_WRITE | PROT_READ, MAP_SHARED, fd_, 0); + if (log_file_ == MAP_FAILED) { + GlobalLogger.Error( + "Open bakcup log %s error while mapping log file: %s\n", + backup_log.c_str(), strerror(errno)); + log_file_ = nullptr; + s = Status::IOError; + } else { + stage_ = *persistedStage(); + } + } + + if (s != Status::Ok) { + Close(); + } + + return s; + } + + // Append a record to backup log + Status Append(RecordType type, const StringView& key, const StringView& val, + ExpireTimeType expire_time) { + if (finished()) { + changeStage(BackupStage::NotFinished); + } + // we do not encapsulate LogRecord here to avoid a memory copy + // TODO: use pinable string view in LogRecord + AppendUint32(&delta_, type); + AppendFixedString(&delta_, key); + AppendFixedString(&delta_, val); + AppendUint64(&delta_, expire_time); + if (delta_.size() >= kMaxBackupLogBufferSize) { + return persistDelta(); + } + return Status::Ok; + } + + Status Finish() { + Status s = persistDelta(); + if (s != Status::Ok) { + return s; + } + changeStage(BackupStage::Finished); + return Status::Ok; + } + + // Get iterator of log records + // Notice: the iterator will be corrupted if append new records to log + std::unique_ptr GetIterator() { + return finished() + ? std::unique_ptr(new LogIterator(logRecordsView())) + : nullptr; + } + + // Close backup log file + void Close() { + if (log_file_ != nullptr) { + munmap(log_file_, file_size_); + } + if (fd_ >= 0) { + close(fd_); + } + delta_.clear(); + file_name_.clear(); + fd_ = -1; + file_size_ = 0; + stage_ = BackupStage::NotFinished; + log_file_ = nullptr; + } + + // Destroy backup log file + void Destroy() { + Close(); + remove(file_name_.c_str()); + } + + private: + Status persistDelta() { + if (ftruncate64(fd_, file_size_ + delta_.size())) { + GlobalLogger.Error("Allocate space for backup log file error: %s\n", + strerror(errno)); + return Status::IOError; + } + log_file_ = mremap(log_file_, file_size_, file_size_ + delta_.size(), + MREMAP_MAYMOVE); + + if (log_file_ == MAP_FAILED) { + GlobalLogger.Error("Map backup log file error: %s\n", strerror(errno)); + log_file_ = nullptr; + return Status::IOError; + } + memcpy((char*)log_file_ + file_size_, delta_.data(), delta_.size()); + msync((char*)log_file_ + file_size_, delta_.size(), MS_SYNC); + file_size_ += delta_.size(); + delta_.clear(); + return Status::Ok; + } + + StringView logRecordsView() { + kvdk_assert(log_file_ != MAP_FAILED && file_size_ >= sizeof(BackupStage), + ""); + return StringView((char*)log_file_ + sizeof(BackupStage), + file_size_ - sizeof(BackupStage)); + } + + BackupStage* persistedStage() { + kvdk_assert(log_file_ != MAP_FAILED && file_size_ >= sizeof(BackupStage), + ""); + return (BackupStage*)log_file_; + } + + void changeStage(BackupStage stage) { + memcpy(persistedStage(), &stage, sizeof(BackupStage)); + msync(persistedStage(), sizeof(BackupStage), MS_SYNC); + } + + bool finished() { return stage_ == BackupStage::Finished; } + + std::string file_name_{}; + std::string delta_{}; + void* log_file_{nullptr}; + size_t file_size_{0}; + int fd_{-1}; + BackupStage stage_{BackupStage::NotFinished}; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/c/kvdk_basic_op.cpp b/volatile/engine/c/kvdk_basic_op.cpp new file mode 100644 index 00000000..c239f0db --- /dev/null +++ b/volatile/engine/c/kvdk_basic_op.cpp @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include +#include +#include + +#include "kvdk_c.hpp" + +extern "C" { +KVDKRegex* KVDKRegexCreate(char const* data, size_t len) { + KVDKRegex* re = new KVDKRegex; + re->rep = std::regex{data, len}; + return re; +} + +void KVDKRegexDestroy(KVDKRegex* re) { delete re; } + +KVDKConfigs* KVDKCreateConfigs() { return new KVDKConfigs; } + +void KVDKSetConfigs(KVDKConfigs* kv_config, uint64_t max_access_threads, + uint64_t hash_bucket_num, uint32_t num_buckets_per_slot) { + kv_config->rep.max_access_threads = max_access_threads; + kv_config->rep.hash_bucket_num = hash_bucket_num; + kv_config->rep.num_buckets_per_slot = num_buckets_per_slot; +} + +void KVDKConfigRegisterCompFunc(KVDKConfigs* kv_config, + const char* compara_name, size_t compara_len, + int (*compare)(const char* src, size_t src_len, + const char* target, + size_t target_len)) { + auto comp_func = [compare](const StringView& src, + const StringView& target) -> int { + return compare(src.data(), src.size(), target.data(), target.size()); + }; + kv_config->rep.comparator.RegisterComparator( + StringView(compara_name, compara_len), comp_func); +} + +void KVDKDestroyConfigs(KVDKConfigs* kv_config) { delete kv_config; } + +KVDKWriteOptions* KVDKCreateWriteOptions(void) { return new KVDKWriteOptions; } + +void KVDKDestroyWriteOptions(KVDKWriteOptions* kv_options) { + delete kv_options; +} + +void KVDKWriteOptionsSetTTLTime(KVDKWriteOptions* kv_options, + int64_t ttl_time) { + kv_options->rep.ttl_time = ttl_time; +} + +void KVDKWriteOptionsSetUpdateTTL(KVDKWriteOptions* kv_options, + int update_ttl) { + kv_options->rep.update_ttl = update_ttl; +} + +KVDKStatus KVDKOpen(const char* name, const KVDKConfigs* config, FILE* log_file, + KVDKEngine** kv_engine) { + Engine* engine; + KVDKStatus s = + Engine::Open(std::string(name), &engine, config->rep, log_file); + *kv_engine = nullptr; + if (s != KVDKStatus::Ok) { + return s; + } + *kv_engine = new KVDKEngine; + (*kv_engine)->rep.reset(engine); + return s; +} + +KVDKStatus KVDKBackup(KVDKEngine* engine, const char* backup_path, + size_t backup_path_len, KVDKSnapshot* snapshot) { + return engine->rep->Backup(StringView(backup_path, backup_path_len), + snapshot->rep); +} + +KVDKStatus KVDKRestore(const char* name, const char* backup_log, + const KVDKConfigs* config, FILE* log_file, + KVDKEngine** kv_engine) { + Engine* engine; + KVDKStatus s = Engine::Restore(std::string(name), std::string(backup_log), + &engine, config->rep, log_file); + if (s == KVDKStatus::Ok) { + *kv_engine = new KVDKEngine; + (*kv_engine)->rep.reset(engine); + } + return s; +} + +KVDKSnapshot* KVDKGetSnapshot(KVDKEngine* engine, int make_checkpoint) { + KVDKSnapshot* snapshot = new KVDKSnapshot; + snapshot->rep = engine->rep->GetSnapshot(make_checkpoint); + return snapshot; +} + +void KVDKReleaseSnapshot(KVDKEngine* engine, KVDKSnapshot* snapshot) { + engine->rep->ReleaseSnapshot(snapshot->rep); + delete snapshot; +} + +void KVDKCloseEngine(KVDKEngine* engine) { delete engine; } + +int KVDKRegisterCompFunc(KVDKEngine* engine, const char* compara_name, + size_t compara_len, + int (*compare)(const char* src, size_t src_len, + const char* target, + size_t target_len)) { + auto comp_func = [compare](const StringView& src, + const StringView& target) -> int { + return compare(src.data(), src.size(), target.data(), target.size()); + }; + return engine->rep->registerComparator(StringView(compara_name, compara_len), + comp_func); +} + +KVDKStatus KVDKExpire(KVDKEngine* engine, const char* str, size_t str_len, + int64_t ttl_time) { + return engine->rep->Expire(StringView{str, str_len}, ttl_time); +} + +KVDKStatus KVDKGetTTL(KVDKEngine* engine, const char* str, size_t str_len, + int64_t* ttl_time) { + return engine->rep->GetTTL(StringView{str, str_len}, ttl_time); +} + +KVDKStatus KVDKTypeOf(KVDKEngine* engine, char const* key_data, size_t key_len, + KVDKValueType* type) { + return engine->rep->TypeOf(StringView{key_data, key_len}, type); +} +} + +// List diff --git a/volatile/engine/c/kvdk_batch.cpp b/volatile/engine/c/kvdk_batch.cpp new file mode 100644 index 00000000..5127bf89 --- /dev/null +++ b/volatile/engine/c/kvdk_batch.cpp @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "kvdk_c.hpp" + +extern "C" { +KVDKWriteBatch* KVDKWriteBatchCreate(KVDKEngine* engine) { + KVDKWriteBatch* batch = new KVDKWriteBatch{}; + batch->rep = engine->rep->WriteBatchCreate(); + return batch; +} + +void KVDKWriteBatchDestory(KVDKWriteBatch* batch) { delete batch; } + +void KVDKWRiteBatchClear(KVDKWriteBatch* batch) { batch->rep->Clear(); } + +void KVDKWriteBatchStringPut(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* val_data, + size_t val_len) { + batch->rep->StringPut(std::string{key_data, key_len}, + std::string{val_data, val_len}); +} + +void KVDKWriteBatchStringDelete(KVDKWriteBatch* batch, char const* key_data, + size_t key_len) { + batch->rep->StringDelete(std::string{key_data, key_len}); +} + +void KVDKWriteBatchSortedPut(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* field_data, + size_t field_len, char const* val_data, + size_t val_len) { + batch->rep->SortedPut(std::string{key_data, key_len}, + std::string{field_data, field_len}, + std::string{val_data, val_len}); +} + +void KVDKWriteBatchSortedDelete(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* field_data, + size_t field_len) { + batch->rep->SortedDelete(std::string{key_data, key_len}, + std::string{field_data, field_len}); +} + +void KVDKWriteBatchHashPut(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* field_data, + size_t field_len, char const* val_data, + size_t val_len) { + batch->rep->HashPut(std::string{key_data, key_len}, + std::string{field_data, field_len}, + std::string{val_data, val_len}); +} + +void KVDKWriteBatchHashDelete(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* field_data, + size_t field_len) { + batch->rep->HashDelete(std::string{key_data, key_len}, + std::string{field_data, field_len}); +} + +KVDKStatus KVDKBatchWrite(KVDKEngine* engine, KVDKWriteBatch const* batch) { + return engine->rep->BatchWrite(batch->rep); +} + +} // extern "C" diff --git a/volatile/engine/c/kvdk_c.hpp b/volatile/engine/c/kvdk_c.hpp new file mode 100644 index 00000000..ee9af260 --- /dev/null +++ b/volatile/engine/c/kvdk_c.hpp @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include +#include +#include +#include + +#include "../alias.hpp" +#include "kvdk/volatile/configs.hpp" +#include "kvdk/volatile/engine.h" +#include "kvdk/volatile/engine.hpp" +#include "kvdk/volatile/iterator.hpp" +#include "kvdk/volatile/write_batch.hpp" + +using kvdk::StringView; + +using kvdk::Configs; +using kvdk::Engine; +using kvdk::HashIterator; +using kvdk::ListIterator; +using kvdk::Snapshot; +using kvdk::SortedCollectionConfigs; +using kvdk::SortedIterator; +using kvdk::WriteBatch; +using kvdk::WriteOptions; + +extern "C" { + +struct KVDKConfigs { + Configs rep; +}; + +struct KVDKEngine { + std::unique_ptr rep; +}; + +struct KVDKWriteBatch { + std::unique_ptr rep; +}; + +struct KVDKSortedIterator { + SortedIterator* rep; +}; + +struct KVDKListIterator { + ListIterator* rep; +}; + +struct KVDKHashIterator { + HashIterator* rep; +}; + +struct KVDKSnapshot { + Snapshot* rep; +}; + +struct KVDKWriteOptions { + WriteOptions rep; +}; + +struct KVDKSortedCollectionConfigs { + SortedCollectionConfigs rep; +}; + +struct KVDKRegex { + std::regex rep; +}; + +inline char* CopyStringToChar(const std::string& str) { + char* result = static_cast(malloc(str.size())); + memcpy(result, str.data(), str.size()); + return result; +} + +} // extern "C" \ No newline at end of file diff --git a/volatile/engine/c/kvdk_hash.cpp b/volatile/engine/c/kvdk_hash.cpp new file mode 100644 index 00000000..b3fb04d8 --- /dev/null +++ b/volatile/engine/c/kvdk_hash.cpp @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include +#include + +#include "kvdk_c.hpp" + +extern "C" { +KVDKStatus KVDKHashCreate(KVDKEngine* engine, char const* key_data, + size_t key_len) { + return engine->rep->HashCreate(StringView{key_data, key_len}); +} +KVDKStatus KVDKHashDestroy(KVDKEngine* engine, char const* key_data, + size_t key_len) { + return engine->rep->HashDestroy(StringView{key_data, key_len}); +} + +KVDKStatus KVDKHashLength(KVDKEngine* engine, char const* key_data, + size_t key_len, size_t* len) { + return engine->rep->HashSize(StringView{key_data, key_len}, len); +} + +KVDKStatus KVDKHashGet(KVDKEngine* engine, const char* key_data, size_t key_len, + const char* field_data, size_t field_len, + char** val_data, size_t* val_len) { + std::string val_str; + *val_data = nullptr; + KVDKStatus s = + engine->rep->HashGet(StringView(key_data, key_len), + StringView(field_data, field_len), &val_str); + if (s != KVDKStatus::Ok) { + *val_len = 0; + return s; + } + *val_len = val_str.size(); + *val_data = CopyStringToChar(val_str); + return s; +} + +KVDKStatus KVDKHashPut(KVDKEngine* engine, const char* key_data, size_t key_len, + const char* field_data, size_t field_len, + const char* val_data, size_t val_len) { + return engine->rep->HashPut(StringView(key_data, key_len), + StringView(field_data, field_len), + StringView(val_data, val_len)); +} + +KVDKStatus KVDKHashDelete(KVDKEngine* engine, const char* key_data, + size_t key_len, const char* field_data, + size_t field_len) { + return engine->rep->HashDelete(StringView(key_data, key_len), + StringView(field_data, field_len)); +} + +extern KVDKStatus KVDKHashModify(KVDKEngine* engine, const char* key_data, + size_t key_len, const char* field_data, + size_t field_len, KVDKModifyFunc modify_func, + void* args, KVDKFreeFunc free_func) { + auto ModifyFunc = [&](const std::string* old_value, std::string* new_value, + void* arg) { + int op; + char* new_val_data = nullptr; + size_t new_val_len = 0; + if (old_value != nullptr) { + op = modify_func(old_value->data(), old_value->size(), &new_val_data, + &new_val_len, arg); + } else { + op = modify_func(nullptr, 0, &new_val_data, &new_val_len, arg); + } + if (op == KVDK_MODIFY_WRITE) { + assert(new_val_data != nullptr); + new_value->assign(new_val_data, new_val_len); + } + if (free_func != nullptr && new_val_data != nullptr) { + free_func(new_val_data); + } + return static_cast(op); + }; + KVDKStatus s = engine->rep->HashModify(StringView{key_data, key_len}, + StringView{field_data, field_len}, + ModifyFunc, args); + return s; +} + +KVDKHashIterator* KVDKHashIteratorCreate(KVDKEngine* engine, + char const* key_data, size_t key_len, + KVDKSnapshot* snapshot, + KVDKStatus* s) { + auto iter = engine->rep->HashIteratorCreate( + StringView{key_data, key_len}, snapshot ? snapshot->rep : nullptr, s); + if (iter == nullptr) { + return nullptr; + } + KVDKHashIterator* hash_iter = new KVDKHashIterator; + hash_iter->rep = iter; + return hash_iter; +} + +void KVDKHashIteratorDestroy(KVDKEngine* engine, KVDKHashIterator* iter) { + if (iter->rep) { + engine->rep->HashIteratorRelease(iter->rep); + } + delete iter; +} + +void KVDKHashIteratorPrev(KVDKHashIterator* iter) { iter->rep->Prev(); } + +void KVDKHashIteratorNext(KVDKHashIterator* iter) { iter->rep->Next(); } + +void KVDKHashIteratorSeekToFirst(KVDKHashIterator* iter) { + iter->rep->SeekToFirst(); +} + +void KVDKHashIteratorSeekToLast(KVDKHashIterator* iter) { + iter->rep->SeekToLast(); +} + +int KVDKHashIteratorIsValid(KVDKHashIterator* iter) { + bool valid = iter->rep->Valid(); + return (valid ? 1 : 0); +} + +void KVDKHashIteratorGetValue(KVDKHashIterator* iter, char** value_data, + size_t* value_len) { + *value_data = nullptr; + *value_len = 0; + std::string buffer = iter->rep->Value(); + *value_data = CopyStringToChar(buffer); + *value_len = buffer.size(); +} + +void KVDKHashIteratorGetKey(KVDKHashIterator* iter, char** field_data, + size_t* field_len) { + *field_data = nullptr; + *field_len = 0; + std::string buffer = iter->rep->Key(); + *field_data = CopyStringToChar(buffer); + *field_len = buffer.size(); +} + +int KVDKHashIteratorMatchKey(KVDKHashIterator* iter, KVDKRegex const* re) { + return iter->rep->MatchKey(re->rep) ? 1 : 0; +} + +} // extern "C" diff --git a/volatile/engine/c/kvdk_list.cpp b/volatile/engine/c/kvdk_list.cpp new file mode 100644 index 00000000..b3ceb138 --- /dev/null +++ b/volatile/engine/c/kvdk_list.cpp @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "kvdk_c.hpp" +extern "C" { +KVDKStatus KVDKListCreate(KVDKEngine* engine, char const* key_data, + size_t key_len) { + return engine->rep->ListCreate(StringView{key_data, key_len}); +} +KVDKStatus KVDKListDestroy(KVDKEngine* engine, char const* key_data, + size_t key_len) { + return engine->rep->ListDestroy(StringView{key_data, key_len}); +} +KVDKStatus KVDKListSize(KVDKEngine* engine, char const* key_data, + size_t key_len, size_t* len) { + return engine->rep->ListSize(StringView{key_data, key_len}, len); +} + +KVDKStatus KVDKListPushFront(KVDKEngine* engine, char const* key_data, + size_t key_len, char const* elem_data, + size_t elem_len) { + return engine->rep->ListPushFront(StringView{key_data, key_len}, + StringView{elem_data, elem_len}); +} + +KVDKStatus KVDKListPushBack(KVDKEngine* engine, char const* key_data, + size_t key_len, char const* elem_data, + size_t elem_len) { + return engine->rep->ListPushBack(StringView{key_data, key_len}, + StringView{elem_data, elem_len}); +} + +KVDKStatus KVDKListPopFront(KVDKEngine* engine, char const* key_data, + size_t key_len, char** elem_data, + size_t* elem_len) { + *elem_data = nullptr; + *elem_len = 0; + std::string buffer; + KVDKStatus s = + engine->rep->ListPopFront(StringView{key_data, key_len}, &buffer); + if (s == KVDKStatus::Ok) { + *elem_data = CopyStringToChar(buffer); + *elem_len = buffer.size(); + } + return s; +} + +KVDKStatus KVDKListPopBack(KVDKEngine* engine, char const* key_data, + size_t key_len, char** elem_data, size_t* elem_len) { + *elem_data = nullptr; + *elem_len = 0; + std::string buffer; + KVDKStatus s = + engine->rep->ListPopBack(StringView{key_data, key_len}, &buffer); + if (s == KVDKStatus::Ok) { + *elem_data = CopyStringToChar(buffer); + *elem_len = buffer.size(); + } + return s; +} + +KVDKStatus KVDKListInsertAt(KVDKEngine* engine, char const* list_name, + size_t list_name_len, char const* elem_data, + size_t elem_len, long index) { + return engine->rep->ListInsertAt(StringView(list_name, list_name_len), + StringView(elem_data, elem_len), index); +} + +KVDKStatus KVDKListInsertBefore(KVDKEngine* engine, char const* list_name, + size_t list_name_len, char const* elem_data, + size_t elem_len, char const* pos_elem, + size_t pos_elem_len) { + return engine->rep->ListInsertBefore(StringView(list_name, list_name_len), + StringView(elem_data, elem_len), + StringView(pos_elem, pos_elem_len)); +} + +KVDKStatus KVDKListInsertAfter(KVDKEngine* engine, char const* list_name, + size_t list_name_len, char const* elem_data, + size_t elem_len, char const* pos_elem, + size_t pos_elem_len) { + return engine->rep->ListInsertAfter(StringView(list_name, list_name_len), + StringView(elem_data, elem_len), + StringView(pos_elem, pos_elem_len)); +} + +KVDKStatus KVDKListErase(KVDKEngine* engine, char const* list_name, + size_t list_len, long index, char** elem_data, + size_t* elem_len) { + std::string buffer; + KVDKStatus s = + engine->rep->ListErase(StringView(list_name, list_len), index, &buffer); + if (s == KVDKStatus::Ok) { + *elem_data = CopyStringToChar(buffer); + *elem_len = buffer.size(); + } + return s; +} + +KVDKStatus KVDKListReplace(KVDKEngine* engine, char const* list_name, + size_t list_name_len, long index, char const* elem, + size_t elem_len) { + return engine->rep->ListReplace(StringView(list_name, list_name_len), index, + StringView(elem, elem_len)); +} + +KVDKStatus KVDKListBatchPushFront(KVDKEngine* engine, char const* key_data, + size_t key_len, char const* const* elems_data, + size_t const* elems_len, size_t elems_cnt) { + std::vector elems; + for (size_t i = 0; i < elems_cnt; i++) { + elems.emplace_back(elems_data[i], elems_len[i]); + } + return engine->rep->ListBatchPushFront(StringView{key_data, key_len}, elems); +} + +KVDKStatus KVDKListBatchPushBack(KVDKEngine* engine, char const* key_data, + size_t key_len, char const* const* elems_data, + size_t const* elems_len, size_t elems_cnt) { + std::vector elems; + for (size_t i = 0; i < elems_cnt; i++) { + elems.emplace_back(elems_data[i], elems_len[i]); + } + return engine->rep->ListBatchPushBack(StringView{key_data, key_len}, elems); +} + +KVDKStatus KVDKListBatchPopFront(KVDKEngine* engine, char const* key_data, + size_t key_len, size_t n, + void (*cb)(char const* elem_data, + size_t elem_len, void* args), + void* args) { + std::vector elems; + KVDKStatus s = + engine->rep->ListBatchPopFront(StringView{key_data, key_len}, n, &elems); + if (s != KVDKStatus::Ok) { + return s; + } + for (auto const& elem : elems) { + cb(elem.data(), elem.size(), args); + } + return s; +} + +KVDKStatus KVDKListBatchPopBack(KVDKEngine* engine, char const* key_data, + size_t key_len, size_t n, + void (*cb)(char const* elem_data, + size_t elem_len, void* args), + void* args) { + std::vector elems; + KVDKStatus s = + engine->rep->ListBatchPopBack(StringView{key_data, key_len}, n, &elems); + if (s != KVDKStatus::Ok) { + return s; + } + for (auto const& elem : elems) { + cb(elem.data(), elem.size(), args); + } + return s; +} + +KVDKStatus KVDKListMove(KVDKEngine* engine, char const* src_data, + size_t src_len, int src_pos, char const* dst_data, + size_t dst_len, int dst_pos, char** elem_data, + size_t* elem_len) { + if ((src_pos != KVDK_LIST_BACK && src_pos != KVDK_LIST_FRONT) || + (dst_pos != KVDK_LIST_BACK && dst_pos != KVDK_LIST_FRONT)) { + return InvalidArgument; + } + std::string elem; + KVDKStatus s = engine->rep->ListMove( + StringView{src_data, src_len}, + static_cast(src_pos), + StringView{dst_data, dst_len}, + static_cast(dst_pos), &elem); + *elem_data = CopyStringToChar(elem); + *elem_len = elem.size(); + return s; +} + +KVDKListIterator* KVDKListIteratorCreate(KVDKEngine* engine, + char const* key_data, size_t key_len, + KVDKStatus* s) { + auto rep = engine->rep->ListIteratorCreate(StringView{key_data, key_len}, + nullptr, s); + if (rep == nullptr) { + return nullptr; + } + KVDKListIterator* iter = new KVDKListIterator; + iter->rep = rep; + return iter; +} + +void KVDKListIteratorDestroy(KVDKEngine* engine, KVDKListIterator* iter) { + if (iter->rep) { + engine->rep->ListIteratorRelease(iter->rep); + } + delete iter; +} + +void KVDKListIteratorPrev(KVDKListIterator* iter) { iter->rep->Prev(); } + +void KVDKListIteratorNext(KVDKListIterator* iter) { iter->rep->Next(); } + +void KVDKListIteratorSeekToFirst(KVDKListIterator* iter) { + iter->rep->SeekToFirst(); +} + +void KVDKListIteratorSeekToLast(KVDKListIterator* iter) { + iter->rep->SeekToLast(); +} + +void KVDKListIteratorSeekPos(KVDKListIterator* iter, long pos) { + iter->rep->Seek(pos); +} + +void KVDKListIteratorPrevElem(KVDKListIterator* iter, char const* elem_data, + size_t elem_len) { + iter->rep->Prev(StringView{elem_data, elem_len}); +} + +void KVDKListIteratorNextElem(KVDKListIterator* iter, char const* elem_data, + size_t elem_len) { + iter->rep->Next(StringView{elem_data, elem_len}); +} + +void KVDKListIteratorSeekToFirstElem(KVDKListIterator* iter, + char const* elem_data, size_t elem_len) { + iter->rep->SeekToFirst(StringView{elem_data, elem_len}); +} + +void KVDKListIteratorSeekToLastElem(KVDKListIterator* iter, + char const* elem_data, size_t elem_len) { + iter->rep->SeekToLast(StringView{elem_data, elem_len}); +} + +int KVDKListIteratorIsValid(KVDKListIterator* iter) { + bool valid = iter->rep->Valid(); + return (valid ? 1 : 0); +} + +void KVDKListIteratorGetValue(KVDKListIterator* iter, char** elem_data, + size_t* elem_len) { + *elem_data = nullptr; + *elem_len = 0; + std::string buffer = iter->rep->Value(); + *elem_data = CopyStringToChar(buffer); + *elem_len = buffer.size(); +} +} diff --git a/volatile/engine/c/kvdk_sorted.cpp b/volatile/engine/c/kvdk_sorted.cpp new file mode 100644 index 00000000..66bb4c27 --- /dev/null +++ b/volatile/engine/c/kvdk_sorted.cpp @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "kvdk_c.hpp" + +extern "C" { +KVDKSortedCollectionConfigs* KVDKCreateSortedCollectionConfigs() { + return new KVDKSortedCollectionConfigs; +} + +void KVDKSetSortedCollectionConfigs(KVDKSortedCollectionConfigs* configs, + const char* comp_func_name, + size_t comp_func_len, + int index_with_hashtable) { + configs->rep.comparator_name = std::string(comp_func_name, comp_func_len); + configs->rep.index_with_hashtable = index_with_hashtable; +} + +void KVDKDestroySortedCollectionConfigs(KVDKSortedCollectionConfigs* configs) { + delete configs; +} + +KVDKStatus KVDKSortedCreate(KVDKEngine* engine, const char* collection_name, + size_t collection_len, + KVDKSortedCollectionConfigs* configs) { + KVDKStatus s = engine->rep->SortedCreate( + StringView(collection_name, collection_len), configs->rep); + if (s != KVDKStatus::Ok) { + return s; + } + return s; +} + +KVDKStatus KVDKSortedDestroy(KVDKEngine* engine, const char* collection_name, + size_t collection_len) { + return engine->rep->SortedDestroy( + StringView(collection_name, collection_len)); +} + +KVDKStatus KVDKSortedSize(KVDKEngine* engine, const char* collection, + size_t collection_len, size_t* size) { + return engine->rep->SortedSize(StringView(collection, collection_len), size); +} + +KVDKStatus KVDKSortedPut(KVDKEngine* engine, const char* collection, + size_t collection_len, const char* key, size_t key_len, + const char* val, size_t val_len) { + return engine->rep->SortedPut(StringView(collection, collection_len), + StringView(key, key_len), + StringView(val, val_len)); +} + +KVDKStatus KVDKSortedGet(KVDKEngine* engine, const char* collection, + size_t collection_len, const char* key, size_t key_len, + size_t* val_len, char** val) { + std::string val_str; + + *val = nullptr; + KVDKStatus s = engine->rep->SortedGet(StringView(collection, collection_len), + StringView(key, key_len), &val_str); + if (s != KVDKStatus::Ok) { + *val_len = 0; + return s; + } + *val_len = val_str.size(); + *val = CopyStringToChar(val_str); + return s; +} + +KVDKStatus KVDKSortedDelete(KVDKEngine* engine, const char* collection, + size_t collection_len, const char* key, + size_t key_len) { + return engine->rep->SortedDelete(StringView(collection, collection_len), + StringView(key, key_len)); +} + +KVDKSortedIterator* KVDKSortedIteratorCreate(KVDKEngine* engine, + const char* collection, + size_t collection_len, + KVDKSnapshot* snapshot, + KVDKStatus* s) { + KVDKSortedIterator* result = new KVDKSortedIterator; + result->rep = (engine->rep->SortedIteratorCreate( + StringView{collection, collection_len}, + snapshot ? snapshot->rep : nullptr, s)); + if (!result->rep) { + delete result; + return nullptr; + } + return result; +} + +void KVDKSortedIteratorDestroy(KVDKEngine* engine, + KVDKSortedIterator* iterator) { + if (iterator != nullptr) { + engine->rep->SortedIteratorRelease(iterator->rep); + } + delete iterator; +} + +void KVDKSortedIteratorSeekToFirst(KVDKSortedIterator* iter) { + iter->rep->SeekToFirst(); +} + +void KVDKSortedIteratorSeekToLast(KVDKSortedIterator* iter) { + iter->rep->SeekToLast(); +} + +void KVDKSortedIteratorSeek(KVDKSortedIterator* iter, const char* str, + size_t str_len) { + iter->rep->Seek(std::string(str, str_len)); +} + +unsigned char KVDKSortedIteratorValid(KVDKSortedIterator* iter) { + return iter->rep->Valid(); +} + +void KVDKSortedIteratorNext(KVDKSortedIterator* iter) { iter->rep->Next(); } + +void KVDKSortedIteratorPrev(KVDKSortedIterator* iter) { iter->rep->Prev(); } + +void KVDKSortedIteratorKey(KVDKSortedIterator* iter, char** key, + size_t* key_len) { + std::string key_str = iter->rep->Key(); + *key_len = key_str.size(); + *key = CopyStringToChar(key_str); +} + +void KVDKSortedIteratorValue(KVDKSortedIterator* iter, char** value, + size_t* val_len) { + std::string val_str = iter->rep->Value(); + *val_len = val_str.size(); + *value = CopyStringToChar(val_str); +} + +} // extern "C" diff --git a/volatile/engine/c/kvdk_string.cpp b/volatile/engine/c/kvdk_string.cpp new file mode 100644 index 00000000..764db0ac --- /dev/null +++ b/volatile/engine/c/kvdk_string.cpp @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "kvdk_c.hpp" +extern "C" { +KVDKStatus KVDKGet(KVDKEngine* engine, const char* key, size_t key_len, + size_t* val_len, char** val) { + std::string val_str; + + *val = nullptr; + KVDKStatus s = engine->rep->Get(StringView(key, key_len), &val_str); + if (s != KVDKStatus::Ok) { + *val_len = 0; + return s; + } + *val_len = val_str.size(); + *val = CopyStringToChar(val_str); + return s; +} + +KVDKStatus KVDKPut(KVDKEngine* engine, const char* key, size_t key_len, + const char* val, size_t val_len, + const KVDKWriteOptions* write_option) { + return engine->rep->Put(StringView(key, key_len), StringView(val, val_len), + write_option->rep); +} + +KVDKStatus KVDKModify(KVDKEngine* engine, const char* key, size_t key_len, + KVDKModifyFunc modify_func, void* modify_args, + KVDKFreeFunc free_func, + const KVDKWriteOptions* write_option) { + auto cpp_modify_func = [&](const std::string* old_value, + std::string* new_value, void* args) { + char* nv = nullptr; + size_t nv_len = 0; + auto result = + modify_func(old_value ? old_value->data() : nullptr, + old_value ? old_value->size() : 0, &nv, &nv_len, args); + if (result == KVDK_MODIFY_WRITE) { + assert(nv != nullptr); + new_value->assign(nv, nv_len); + } + if (nv != nullptr && free_func != nullptr) { + free_func(nv); + } + return kvdk::ModifyOperation(result); + }; + KVDKStatus s = engine->rep->Modify(StringView(key, key_len), cpp_modify_func, + modify_args, write_option->rep); + return s; +} + +KVDKStatus KVDKDelete(KVDKEngine* engine, const char* key, size_t key_len) { + return engine->rep->Delete(StringView(key, key_len)); +} + +} // extern "C" diff --git a/volatile/engine/collection.hpp b/volatile/engine/collection.hpp new file mode 100644 index 00000000..2f18885a --- /dev/null +++ b/volatile/engine/collection.hpp @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include +#include + +#include "alias.hpp" +#include "macros.hpp" +#include "utils/codec.hpp" +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { +/// TODO: (ziyan) add expire_time field to Collection. +class Collection { + public: + Collection(const StringView& name, CollectionIDType id) + : collection_name_(string_view_2_string(name)), collection_id_(id) {} + virtual ~Collection() = default; + // Return unique ID of the collection + uint64_t ID() const { return collection_id_; } + + // Return name of the collection + const std::string& Name() const { return collection_name_; } + + virtual ExpireTimeType GetExpireTime() const = 0; + virtual bool HasExpired() const = 0; + + // Return internal representation of "key" in the collection + // By default, we concat key with the collection id + std::string InternalKey(const StringView& key) { + return makeInternalKey(key, ID()); + } + + inline static std::string EncodeID(CollectionIDType id) { + return EncodeUint64(id); + } + + inline static CollectionIDType DecodeID(const StringView& string_id) { + CollectionIDType id{}; + bool ret = DecodeUint64(string_id, &id); + kvdk_assert(ret, "size of string id does not match CollectionIDType size!"); + return id; + } + + inline static StringView ExtractUserKey(const StringView& internal_key) { + constexpr size_t sz_id = sizeof(CollectionIDType); + kvdk_assert(sz_id <= internal_key.size(), + "internal_key does not has space for key"); + return StringView(internal_key.data() + sz_id, internal_key.size() - sz_id); + } + + inline static uint64_t ExtractID(const StringView& internal_key) { + return DecodeID(internal_key); + } + + inline static std::string ID2String(CollectionIDType id) { + return std::string{reinterpret_cast(&id), sizeof(CollectionIDType)}; + } + + struct TTLCmp { + public: + bool operator()(const Collection* a, const Collection* b) const { + if (a->GetExpireTime() < b->GetExpireTime()) return true; + if (a->GetExpireTime() == b->GetExpireTime() && a->ID() < b->ID()) + return true; + return false; + } + }; + + protected: + inline static std::string makeInternalKey(const StringView& user_key, + uint64_t list_id) { + return ID2String(list_id).append(user_key.data(), user_key.size()); + } + + std::string collection_name_; + CollectionIDType collection_id_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/data_record.cpp b/volatile/engine/data_record.cpp new file mode 100644 index 00000000..c6cd3830 --- /dev/null +++ b/volatile/engine/data_record.cpp @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "data_record.hpp" + +namespace KVDK_NAMESPACE {} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/data_record.hpp b/volatile/engine/data_record.hpp new file mode 100644 index 00000000..4b40a810 --- /dev/null +++ b/volatile/engine/data_record.hpp @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include + +#include "alias.hpp" +#include "kvdk/volatile/configs.hpp" +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { + +enum RecordType : uint8_t { + Empty = 0, + String = (1 << 0), + SortedHeader = (1 << 1), + SortedElem = (1 << 2), + HashHeader = (1 << 3), + HashElem = (1 << 4), + ListHeader = (1 << 5), + ListElem = (1 << 6), +}; + +enum class RecordStatus : uint8_t { + // Indicate a up-to-date record + Normal = 0, + // Indicate key of the record is updated + Dirty, + // Indicate deleted or expired record + Outdated, +}; + +const uint8_t ExpirableRecordType = + (RecordType::String | RecordType::SortedHeader | RecordType::HashHeader | + RecordType::ListHeader); + +const uint8_t PrimaryRecordType = ExpirableRecordType; + +const uint8_t ElemType = + (RecordType::SortedElem | RecordType::HashElem | RecordType::ListElem); + +const uint8_t CollectionType = + (RecordType::SortedHeader | RecordType::HashHeader | + RecordType::ListHeader); + +struct DataHeader { + DataHeader() = default; + DataHeader(uint32_t c, uint32_t s) : checksum(c), record_size(s) {} + + uint32_t checksum; + uint32_t record_size; +}; + +struct DataMeta { + DataMeta() = default; + DataMeta(TimestampType _timestamp, RecordType _type, RecordStatus _status, + uint16_t _key_size, uint32_t _value_size) + : type(_type), + status(_status), + k_size(_key_size), + v_size(_value_size), + timestamp(_timestamp) {} + + RecordType type; + RecordStatus status; + uint16_t k_size; + uint32_t v_size; + TimestampType timestamp; +}; + +// Header and metadata of a data record +struct DataEntry { + DataEntry(uint32_t _checksum, uint32_t _record_size /* size in blocks */, + TimestampType _timestamp, RecordType _type, RecordStatus _status, + uint16_t _key_size, uint32_t _value_size) + : header(_checksum, _record_size), + meta(_timestamp, _type, _status, _key_size, _value_size) {} + + DataEntry() = default; + + void Destroy() { meta.type = RecordType::Empty; } + + // TODO jiayu: use function to access these + DataHeader header; + DataMeta meta; +}; +static_assert(sizeof(DataEntry) <= kMinMemoryBlockSize); + +struct StringRecord { + public: + DataEntry entry; + MemoryOffsetType old_version; + ExpireTimeType expired_time; + char data[0]; + + // Construct a StringRecord instance at target_address. As the record need + // additional space to store data, we need pre-allocate enough space for it. + // + // target_address: pre-allocated space to store constructed record, it + // should larger than sizeof(StringRecord) + key size + value size + static StringRecord* ConstructStringRecord( + void* target_address, uint32_t _record_size, TimestampType _timestamp, + RecordType _record_type, RecordStatus _record_status, + MemoryOffsetType _old_version, const StringView& _key, + const StringView& _value, ExpireTimeType _expired_time = kPersistTime) { + StringRecord* record = new (target_address) + StringRecord(_record_size, _timestamp, _record_type, _record_status, + _old_version, _key, _value, _expired_time); + return record; + } + + void Destroy() { entry.Destroy(); } + + // make sure there is data followed in data[0] + StringView Key() const { return StringView(data, entry.meta.k_size); } + + // make sure there is data followed in data[0] + StringView Value() const { + return StringView(data + entry.meta.k_size, entry.meta.v_size); + } + + // Check whether the record corrupted + bool Validate() { + if (ValidateRecordSize()) { + return Checksum() == entry.header.checksum; + } + return false; + } + + bool ValidOrDirty() { + bool valid = Validate(); + asm volatile("" ::: "memory"); + bool dirty = (GetRecordStatus() == RecordStatus::Dirty); + return (valid || dirty); + } + + // Check whether the record corrupted with expected checksum + bool Validate(uint32_t expected_checksum) { + if (ValidateRecordSize()) { + return Checksum() == expected_checksum; + } + return false; + } + + ExpireTimeType GetExpireTime() const { return expired_time; } + bool HasExpired() const { return TimeUtils::CheckIsExpired(GetExpireTime()); } + + void SetExpireTime(ExpireTimeType time) { expired_time = time; } + + void SetOldVersion(MemoryOffsetType offset) { old_version = offset; } + + void SetStatus(RecordStatus status) { entry.meta.status = status; } + + TimestampType GetTimestamp() const { return entry.meta.timestamp; } + + RecordType GetRecordType() const { return entry.meta.type; } + + RecordStatus GetRecordStatus() const { return entry.meta.status; } + + uint32_t GetRecordSize() const { return entry.header.record_size; } + + static uint32_t RecordSize(const StringView& key, const StringView& value) { + return key.size() + value.size() + sizeof(StringRecord); + } + + private: + StringRecord(uint32_t _record_size, TimestampType _timestamp, + RecordType _record_type, RecordStatus _record_status, + MemoryOffsetType _old_version, const StringView& _key, + const StringView& _value, ExpireTimeType _expired_time) + : entry(0, _record_size, _timestamp, _record_type, _record_status, + _key.size(), _value.size()), + old_version(_old_version), + expired_time(_expired_time) { + kvdk_assert(_record_type == RecordType::String, ""); + memcpy(data, _key.data(), _key.size()); + memcpy(data + _key.size(), _value.data(), _value.size()); + entry.header.checksum = Checksum(); + } + + // check validation of k_size and v_size, as record may be left corrupted + bool ValidateRecordSize() { + return entry.meta.k_size + entry.meta.v_size + sizeof(StringRecord) <= + entry.header.record_size; + } + + uint32_t Checksum() { + // we don't checksum expire time and old version + uint32_t meta_checksum_size = sizeof(DataMeta); + uint32_t data_checksum_size = entry.meta.k_size + entry.meta.v_size; + + return get_checksum((char*)&entry.meta, meta_checksum_size) + + get_checksum(data, data_checksum_size); + } +}; + +// doubly linked record +struct DLRecord { + public: + DataEntry entry; + MemoryOffsetType old_version; + MemoryOffsetType prev; + MemoryOffsetType next; + ExpireTimeType expired_time; + + char data[0]; + + // Construct a DLRecord instance at "target_address". As the record need + // additional space to store data, we need pre-allocate enough space for it. + // + // target_address: pre-allocated space to store constructed record, it + // should no smaller than sizeof(DLRecord) + key size + value size + static DLRecord* ConstructDLRecord( + void* target_address, uint32_t record_size, TimestampType timestamp, + RecordType record_type, RecordStatus record_status, + MemoryOffsetType old_version, uint64_t prev, uint64_t next, + const StringView& key, const StringView& value, + ExpireTimeType expired_time = kPersistTime) { + DLRecord* record = new (target_address) + DLRecord(record_size, timestamp, record_type, record_status, + old_version, prev, next, key, value, expired_time); + return record; + } + + void Destroy() { entry.Destroy(); } + + bool Validate() { + if (ValidateRecordSize()) { + return Checksum() == entry.header.checksum; + } + return false; + } + + bool Validate(uint32_t expected_checksum) { + if (ValidateRecordSize()) { + return Checksum() == expected_checksum; + } + return false; + } + + StringView Key() const { return StringView(data, entry.meta.k_size); } + + StringView Value() const { + return StringView(data + entry.meta.k_size, entry.meta.v_size); + } + + void SetNext(MemoryOffsetType offset) { next = offset; } + + void SetPrev(MemoryOffsetType offset) { prev = offset; } + + void SetPrevSetExpireTime(ExpireTimeType time) { + kvdk_assert(entry.meta.type & ExpirableRecordType, ""); + expired_time = time; + } + + void SetOldVersion(MemoryOffsetType offset) { old_version = offset; } + + void SetStatus(RecordStatus status) { entry.meta.status = status; } + + ExpireTimeType GetExpireTime() const { + kvdk_assert(entry.meta.type & ExpirableRecordType, + "Call DLRecord::GetExpireTime with an unexpirable type"); + return expired_time; + } + + RecordType GetRecordType() const { return entry.meta.type; } + + RecordStatus GetRecordStatus() const { return entry.meta.status; } + + bool HasExpired() const { return TimeUtils::CheckIsExpired(GetExpireTime()); } + TimestampType GetTimestamp() const { return entry.meta.timestamp; } + + uint32_t GetRecordSize() const { return entry.header.record_size; } + + static uint32_t RecordSize(const StringView& key, const StringView& value) { + return sizeof(DLRecord) + key.size() + value.size(); + } + + private: + DLRecord(uint32_t _record_size, TimestampType _timestamp, RecordType _type, + RecordStatus _status, MemoryOffsetType _old_version, + MemoryOffsetType _prev, MemoryOffsetType _next, + const StringView& _key, const StringView& _value, + ExpireTimeType _expired_time) + : entry(0, _record_size, _timestamp, _type, _status, _key.size(), + _value.size()), + old_version(_old_version), + prev(_prev), + next(_next), + expired_time(_expired_time) { + kvdk_assert(_type & (RecordType::SortedElem | RecordType::SortedHeader | + RecordType::HashElem | RecordType::HashHeader | + RecordType::ListElem | RecordType::ListHeader), + ""); + memcpy(data, _key.data(), _key.size()); + memcpy(data + _key.size(), _value.data(), _value.size()); + entry.header.checksum = Checksum(); + } + + // check validation of k_size and v_size, as record may be left corrupted + bool ValidateRecordSize() { + return entry.meta.k_size + entry.meta.v_size + sizeof(DLRecord) <= + entry.header.record_size; + } + + uint32_t Checksum() { + // we don't checksum next/prev pointers, expire time and old_version + uint32_t meta_checksum_size = sizeof(DataMeta); + uint32_t data_checksum_size = entry.meta.k_size + entry.meta.v_size; + + return get_checksum((char*)&entry.meta, meta_checksum_size) + + get_checksum(data, data_checksum_size); + } +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/dl_list.cpp b/volatile/engine/dl_list.cpp new file mode 100644 index 00000000..375c9a45 --- /dev/null +++ b/volatile/engine/dl_list.cpp @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "dl_list.hpp" + +namespace KVDK_NAMESPACE { +std::unique_ptr DLList::GetRecordIterator() { + return std::unique_ptr( + new DLListRecordIterator(this, kv_allocator_)); +} + +Status DLList::PushBack(const DLList::WriteArgs& args) { + kvdk_assert(header_ != nullptr, ""); + Status s; + do { + s = InsertBefore(args, header_); + } while (s == Status::Fail); + return s; +} + +Status DLList::PushFront(const DLList::WriteArgs& args) { + kvdk_assert(header_ != nullptr, ""); + Status s; + do { + s = InsertAfter(args, header_); + } while (s == Status::Fail); + return s; +} + +DLRecord* DLList::RemoveFront() { + kvdk_assert(header_ != nullptr, ""); + while (true) { + DLRecord* front = + kv_allocator_->offset2addr_checked(header_->next); + if (front == header_) { + return nullptr; + } + // Maybe removed by another thread + bool success = Remove(front); + if (success) { + return front; + } + } +} + +DLRecord* DLList::RemoveBack() { + kvdk_assert(header_ != nullptr, ""); + while (true) { + DLRecord* back = + kv_allocator_->offset2addr_checked(header_->prev); + if (back == header_) { + return nullptr; + } + // Maybe removed by another thread + bool success = Remove(back); + if (success) { + return back; + } + } +} + +Status DLList::InsertBetween(const DLList::WriteArgs& args, DLRecord* prev, + DLRecord* next) { + auto ul = acquireInsertLock(prev); + MemoryOffsetType next_offset = kv_allocator_->addr2offset_checked(next); + MemoryOffsetType prev_offset = kv_allocator_->addr2offset_checked(prev); + // Check if the linkage has changed before we successfully acquire lock. + bool check_linkage = prev->next == next_offset && next->prev == prev_offset; + if (!check_linkage) { + return Status::Fail; + } + + DLRecord* new_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(args.space.offset), args.space.size, + args.ts, args.type, args.status, kNullMemoryOffset, prev_offset, + next_offset, args.key, args.val); + linkRecord(prev, next, new_record); + + return Status::Ok; +} + +Status DLList::InsertAfter(const DLList::WriteArgs& args, DLRecord* prev) { + return InsertBetween( + args, prev, kv_allocator_->offset2addr_checked(prev->next)); +} + +Status DLList::InsertBefore(const DLList::WriteArgs& args, DLRecord* next) { + return InsertBetween( + args, kv_allocator_->offset2addr_checked(next->prev), next); +} + +Status DLList::Update(const DLList::WriteArgs& args, DLRecord* current) { + kvdk_assert(current != nullptr && equal_string_view(current->Key(), args.key), + ""); + auto guard = acquireRecordLock(current); + MemoryOffsetType current_offset = kv_allocator_->addr2offset_checked(current); + MemoryOffsetType prev_offset = current->prev; + MemoryOffsetType next_offset = current->next; + DLRecord* prev = kv_allocator_->offset2addr_checked(prev_offset); + DLRecord* next = kv_allocator_->offset2addr_checked(next_offset); + if (next->prev != current_offset || prev->next != current_offset) { + return Status::Fail; + } + DLRecord* new_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(args.space.offset), args.space.size, + args.ts, args.type, args.status, current_offset, prev_offset, next_offset, + args.key, args.val); + linkRecord(prev, next, new_record); + return Status::Ok; +} + +bool DLList::Replace(DLRecord* old_record, DLRecord* new_record) { + bool ret = Replace(old_record, new_record, kv_allocator_, lock_table_); + if (ret && old_record == header_) { + header_ = new_record; + } + return ret; +} + +bool DLList::Remove(DLRecord* removing_record) { + bool ret = Remove(removing_record, kv_allocator_, lock_table_); + return ret; +} + +bool DLList::Replace(DLRecord* old_record, DLRecord* new_record, + Allocator* kv_allocator, LockTable* lock_table) { + auto guard = acquireRecordLock(old_record, kv_allocator, lock_table); + MemoryOffsetType prev_offset = old_record->prev; + MemoryOffsetType next_offset = old_record->next; + auto old_record_offset = kv_allocator->addr2offset(old_record); + DLRecord* prev = kv_allocator->offset2addr_checked(prev_offset); + DLRecord* next = kv_allocator->offset2addr_checked(next_offset); + bool on_list = + prev != nullptr && next != nullptr && prev->next == old_record_offset; + if (on_list) { + if (prev_offset == old_record_offset && next_offset == old_record_offset) { + // old record is the only record (the header) in the list, so we + // make + // new record point to itself and break linkage of the old one for + // recovery + kvdk_assert((new_record->GetRecordType() & CollectionType) && + (old_record->GetRecordType() & CollectionType), + "Non-header record shouldn't be the only record in a list"); + linkRecord(new_record, new_record, new_record, kv_allocator); + auto new_record_offset = kv_allocator->addr2offset(new_record); + old_record->SetPrev(new_record_offset); + } else { + new_record->prev = prev_offset; + new_record->next = next_offset; + linkRecord(prev, next, new_record, kv_allocator); + } + } + return on_list; +} + +bool DLList::Remove(DLRecord* removing_record, Allocator* kv_allocator, + LockTable* lock_table) { + auto guard = acquireRecordLock(removing_record, kv_allocator, lock_table); + MemoryOffsetType removing_offset = kv_allocator->addr2offset(removing_record); + MemoryOffsetType prev_offset = removing_record->prev; + MemoryOffsetType next_offset = removing_record->next; + DLRecord* prev = kv_allocator->offset2addr_checked(prev_offset); + DLRecord* next = kv_allocator->offset2addr_checked(next_offset); + bool on_list = + prev != nullptr && next != nullptr && prev->next == removing_offset; + if (on_list) { + // For repair in recovery due to crashes during pointers changing, we + // should + // first unlink deleting entry from next's prev.(It is the reverse process + // of insertion) + next->prev = prev_offset; + TEST_SYNC_POINT("KVEngine::DLList::Remove::PersistNext'sPrev::After"); + prev->next = next_offset; + } + return on_list; +} +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/dl_list.hpp b/volatile/engine/dl_list.hpp new file mode 100644 index 00000000..d5a6831d --- /dev/null +++ b/volatile/engine/dl_list.hpp @@ -0,0 +1,344 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#pragma once + +#include "allocator.hpp" +#include "collection.hpp" +#include "data_record.hpp" +#include "kvdk/volatile/types.hpp" +#include "lock_table.hpp" +#include "utils/sync_point.hpp" +#include "version/version_controller.hpp" + +namespace KVDK_NAMESPACE { +class DLListRecordIterator; + +// Persistent doubly linked list +class DLList { + public: + DLList(DLRecord* header, Allocator* kv_allocator, LockTable* lock_table) + : header_(header), kv_allocator_(kv_allocator), lock_table_(lock_table) {} + + struct WriteArgs { + WriteArgs(const StringView& _key, const StringView& _val, RecordType _type, + RecordStatus _status, TimestampType _ts, const SpaceEntry& _space) + : key(_key), + val(_val), + type(_type), + status(_status), + ts(_ts), + space(_space) { + kvdk_assert(space.size >= DLRecord::RecordSize(_key, _val), + "space to write dl record too small"); + } + + WriteArgs(const WriteArgs&) = delete; + + WriteArgs(WriteArgs&& args) = delete; + + StringView key; + StringView val; + RecordType type; + RecordStatus status; + TimestampType ts; + SpaceEntry space; + }; + + const DLRecord* Header() const { return header_; } + + DLRecord* Header() { return header_; } + + Status PushBack(const WriteArgs& args); + + Status PushFront(const WriteArgs& args); + + DLRecord* RemoveFront(); + + DLRecord* RemoveBack(); + + Status InsertBetween(const WriteArgs& args, DLRecord* prev, DLRecord* next); + + Status InsertAfter(const WriteArgs& args, DLRecord* prev); + + Status InsertBefore(const WriteArgs& args, DLRecord* next); + + Status Update(const WriteArgs& args, DLRecord* current); + + bool Replace(DLRecord* old_record, DLRecord* new_record); + + bool Remove(DLRecord* removing_record); + + std::unique_ptr GetRecordIterator(); + + static bool Replace(DLRecord* old_record, DLRecord* new_record, + Allocator* kv_allocator, LockTable* lock_table); + + static bool Remove(DLRecord* removing_record, Allocator* kv_allocator, + LockTable* lock_table); + + private: + // lock position to insert a new record by locking the prev DLRecord + LockTable::ULockType acquireInsertLock(DLRecord* prev) { + return lock_table_->AcquireLock(recordHash(prev)); + } + + // lock position of "record" to replace or unlink it by locking its prev + // DLRecord and itself + LockTable::MultiGuardType acquireRecordLock(DLRecord* record) { + return acquireRecordLock(record, kv_allocator_, lock_table_); + } + + // lock position of "record" to replace or unlink it by locking its prev + // DLRecord and itself + static LockTable::MultiGuardType acquireRecordLock(DLRecord* record, + Allocator* kv_allocator, + LockTable* lock_table) { + while (1) { + MemoryOffsetType prev_offset = record->prev; + MemoryOffsetType next_offset = record->next; + DLRecord* prev = kv_allocator->offset2addr_checked(prev_offset); + auto guard = + lock_table->MultiGuard({recordHash(prev), recordHash(record)}); + // Check if the linkage has changed before we successfully acquire lock. + if (record->prev != prev_offset || record->next != next_offset) { + continue; + } + + return guard; + } + } + + void linkRecord(DLRecord* prev, DLRecord* next, DLRecord* linking_record) { + linkRecord(prev, next, linking_record, kv_allocator_); + } + + static LockTable::HashValueType recordHash(const DLRecord* record) { + kvdk_assert(record != nullptr, ""); + return XXH3_64bits(record, sizeof(const DLRecord*)); + } + + static void linkRecord(DLRecord* prev, DLRecord* next, + DLRecord* linking_record, Allocator* kv_allocator) { + auto linking_record_offset = + kv_allocator->addr2offset_checked(linking_record); + prev->SetNext(linking_record_offset); + TEST_SYNC_POINT("KVEngine::DLList::LinkDLRecord::HalfLink"); + next->SetPrev(linking_record_offset); + } + + DLRecord* header_; + Allocator* kv_allocator_; + LockTable* lock_table_; +}; + +// Iter valid data under a snapshot in a dl list +class DLListDataIterator { + public: + DLListDataIterator(DLList* dl_list, const Allocator* kv_allocator, + const SnapshotImpl* snapshot) + : dl_list_(dl_list), + kv_allocator_(kv_allocator), + current_(nullptr), + snapshot_(snapshot) {} + + void Locate(DLRecord* record, bool forward) { + kvdk_assert(record != nullptr, ""); + current_ = record; + skipInvalidRecords(forward); + } + + void SeekToFirst() { + auto first = dl_list_->Header()->next; + current_ = kv_allocator_->offset2addr_checked(first); + skipInvalidRecords(true); + } + + void SeekToLast() { + auto last = dl_list_->Header()->prev; + current_ = kv_allocator_->offset2addr(last); + skipInvalidRecords(false); + } + + bool Valid() const { + return current_ != nullptr && (current_->GetRecordType() & ElemType); + } + + virtual void Next() { + if (!Valid()) { + return; + } + current_ = kv_allocator_->offset2addr_checked(current_->next); + skipInvalidRecords(true); + } + + void Prev() { + if (!Valid()) { + return; + } + current_ = (kv_allocator_->offset2addr(current_->prev)); + skipInvalidRecords(false); + } + + StringView Key() const { + if (!Valid()) return ""; + return current_->Key(); + } + + StringView Value() const { + if (!Valid()) return ""; + return current_->Value(); + } + + private: + DLRecord* findValidVersion(DLRecord* data_record) { + DLRecord* curr = data_record; + TimestampType ts = snapshot_->GetTimestamp(); + while (curr != nullptr && curr->GetTimestamp() > ts) { + curr = kv_allocator_->offset2addr(curr->old_version); + kvdk_assert(curr == nullptr || curr->Validate(), + "Broken checkpoint: invalid older version sorted record"); + kvdk_assert( + curr == nullptr || equal_string_view(curr->Key(), data_record->Key()), + "Broken checkpoint: key of older version sorted data is " + "not same as new " + "version"); + } + return curr; + } + + // Move current_ to next/prev valid version data record + void skipInvalidRecords(bool forward) { + while (Valid()) { + DLRecord* valid_version_record = findValidVersion(current_); + if (valid_version_record == nullptr || + valid_version_record->GetRecordStatus() == RecordStatus::Outdated) { + current_ = + forward + ? kv_allocator_->offset2addr_checked(current_->next) + : kv_allocator_->offset2addr_checked(current_->prev); + } else { + current_ = valid_version_record; + break; + } + } + } + + DLList* dl_list_; + const Allocator* kv_allocator_; + DLRecord* current_; + const SnapshotImpl* snapshot_; +}; + +// Iter all records in a dl list +class DLListRecordIterator { + public: + DLListRecordIterator(DLList* dl_list, Allocator* kv_allocator) + : dl_list_(dl_list), + header_(dl_list->Header()), + current_(header_), + kv_allocator_(kv_allocator) {} + + void Locate(DLRecord* record) { current_ = record; } + + void Next() { + if (Valid()) { + current_ = kv_allocator_->offset2addr_checked(current_->next); + } + } + + void Prev() { + if (Valid()) { + current_ = kv_allocator_->offset2addr_checked(current_->prev); + } + } + + bool Valid() { return current_ && (current_->GetRecordType() & ElemType); } + + void SeekToFirst() { + kvdk_assert(header_ != nullptr, ""); + current_ = kv_allocator_->offset2addr_checked(header_->next); + } + + void SeekToLast() { + kvdk_assert(header_ != nullptr, ""); + current_ = kv_allocator_->offset2addr_checked(header_->prev); + } + + DLRecord* Record() { return Valid() ? current_ : nullptr; } + + private: + DLList* dl_list_; + DLRecord* header_; + DLRecord* current_; + const Allocator* kv_allocator_; +}; + +// Used in recovery of dl list based collections +template +class DLListRecoveryUtils { + public: + DLListRecoveryUtils(const Allocator* kv_allocator) + : kv_allocator_(kv_allocator) {} + + bool CheckAndRepairLinkage(DLRecord* record) { + // The next linkage is correct. If the prev linkage is correct too, the + // record linkage is ok. If the prev linkage is not correct, it will be + // repaired by the correct prodecessor soon, so directly return true here. + if (CheckNextLinkage(record)) { + return true; + } + // If only prev linkage is correct, then repair the next linkage + if (CheckPrevLinkage(record)) { + DLRecord* next = + kv_allocator_->offset2addr_checked(record->next); + next->SetPrev(kv_allocator_->addr2offset_checked(record)); + return true; + } + + return false; + } + + bool CheckNextLinkage(DLRecord* record) { + uint64_t offset = kv_allocator_->addr2offset_checked(record); + DLRecord* next = kv_allocator_->offset2addr_checked(record->next); + + auto check_linkage = [&]() { return next->prev == offset; }; + + auto check_type = [&]() { return CType::MatchType(record); }; + + auto check_id = [&]() { + auto next_id = CType::FetchID(next); + auto record_id = CType::FetchID(record); + return record_id == next_id; + }; + + return check_linkage() && check_type() && check_id(); + } + + bool CheckPrevLinkage(DLRecord* record) { + uint64_t offset = kv_allocator_->addr2offset_checked(record); + DLRecord* prev = kv_allocator_->offset2addr_checked(record->prev); + + auto check_linkage = [&]() { return prev->next == offset; }; + + auto check_type = [&]() { return CType::MatchType(record); }; + + auto check_id = [&]() { + auto prev_id = CType::FetchID(prev); + auto record_id = CType::FetchID(record); + return record_id == prev_id; + }; + + return check_linkage() && check_type() && check_id(); + } + + bool CheckLinkage(DLRecord* record) { + return CheckPrevLinkage(record) && CheckNextLinkage(record); + } + + private: + const Allocator* kv_allocator_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/dram_allocator.cpp b/volatile/engine/dram_allocator.cpp new file mode 100644 index 00000000..4798660a --- /dev/null +++ b/volatile/engine/dram_allocator.cpp @@ -0,0 +1,299 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "dram_allocator.hpp" + +#include "thread_manager.hpp" + +namespace KVDK_NAMESPACE { + +constexpr size_t kDefaultChunkAlignment = 64; + +void ChunkBasedAllocator::Free(const SpaceEntry&) { + // Not supported yet +} + +SpaceEntry ChunkBasedAllocator::Allocate(uint64_t size) { + kvdk_assert(ThreadManager::ThreadID() >= 0, ""); + SpaceEntry entry; + auto& tc = dalloc_thread_cache_[ThreadManager::ThreadID() % + dalloc_thread_cache_.size()]; + if (size > chunk_size_) { + entry = alloc_->AllocateAligned(kDefaultChunkAlignment, chunk_size_); + if (entry.size != 0) { + tc.allocated_chunks.push_back(entry); + } + return entry; + } + + if (tc.usable_bytes < size) { + entry = alloc_->AllocateAligned(kDefaultChunkAlignment, chunk_size_); + if (entry.size == 0) { + return entry; + } + void* addr = alloc_->offset2addr(entry.offset); + tc.chunk_addr = (char*)addr; + tc.usable_bytes = chunk_size_; + tc.allocated_chunks.push_back(entry); + } + + entry.size = size; + entry.offset = alloc_->addr2offset(tc.chunk_addr); + tc.chunk_addr += size; + tc.usable_bytes -= size; + return entry; +} + +static struct GlobalState { + SystemMemoryAllocator system_memory_allocator; +} global_state; + +Allocator* global_memory_allocator() { + return &global_state.system_memory_allocator; +} + +static unsigned integer_log2(unsigned v) { + return (sizeof(unsigned) * 8) - (__builtin_clz(v) + 1); +} + +static unsigned round_pow2_up(unsigned v) { + unsigned v_log2 = integer_log2(v); + + unsigned one = 1; + if (v != one << v_log2) { + v = one << (v_log2 + 1); + } + return v; +} + +// SplitMix64 hash +static uint64_t hash64(uint64_t x) { + x += 0x9e3779b97f4a7c15; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; + x = (x ^ (x >> 27)) * 0x94d049bb133111eb; + return x ^ (x >> 31); +} + +static uintptr_t get_fs_base() { return (uintptr_t)pthread_self(); } + +#ifdef KVDK_WITH_JEMALLOC +bool JemallocMemoryAllocator::SetMaxAccessThreads(uint32_t max_access_threads) { + if (max_access_threads == 0 || arenas_initialized_) { + return false; + } + + auto ul = Allocator::AcquireLock(); + + if (arenas_initialized_) { + return false; + } + + num_arenas_ = round_pow2_up(max_access_threads); + arena_mask_ = num_arenas_ - 1; + + return Allocator::EnableThreadLocalCounters(max_access_threads); +} + +static SpinMutex jemalloc_arena_spin; +static std::map arena_allocator_map; + +// Allocates size bytes aligned to alignment. Returns NULL if allocation fails. +void* alloc_aligned_slow(size_t size, size_t alignment, unsigned arena_index) { + JemallocMemoryAllocator* alloc = arena_allocator_map[arena_index]; + if (alloc == nullptr) { + return NULL; + } + + size_t extended_size = size + alignment; + void* ptr; + + ptr = mmap(NULL, extended_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (ptr == MAP_FAILED) { + return NULL; + } + + // mbind + bitmask* mask = alloc->GetMbindNodesMask(); + if (mask) { + int mode = alloc->GetMbindMode(); + int err = mbind(ptr, extended_size, mode, mask->maskp, mask->size, 0); + if (KVDK_UNLIKELY(err)) { + GlobalLogger.Error("syscall mbind() errno: %d.\n", errno); + munmap(ptr, extended_size); + return NULL; + } + } + + uintptr_t addr = (uintptr_t)ptr; + uintptr_t aligned_addr = (addr + alignment) & ~(alignment - 1); + + size_t head_len = aligned_addr - addr; + if (head_len > 0) { + munmap(ptr, head_len); + } + + uintptr_t tail = aligned_addr + size; + size_t tail_len = (addr + extended_size) - (aligned_addr + size); + if (tail_len > 0) { + munmap((void*)tail, tail_len); + } + + return (void*)aligned_addr; +} + +static void* arena_extent_alloc(extent_hooks_t*, void* new_addr, size_t size, + size_t alignment, bool* zero, bool* commit, + unsigned arena_index) { + JemallocMemoryAllocator* alloc = arena_allocator_map[arena_index]; + if (alloc == nullptr) { + return NULL; + } + + void* addr = mmap(new_addr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + return NULL; + } + + // mbind + bitmask* mask = alloc->GetMbindNodesMask(); + if (mask) { + int mode = alloc->GetMbindMode(); + int err = mbind(addr, size, mode, mask->maskp, mask->size, 0); + if (KVDK_UNLIKELY(err)) { + GlobalLogger.Error("syscall mbind() errno: %d.\n", errno); + munmap(addr, size); + return NULL; + } + } + + if (new_addr != NULL && addr != new_addr) { + /* wrong place */ + munmap(addr, size); + return NULL; + } + + if ((uintptr_t)addr & (alignment - 1)) { + munmap(addr, size); + addr = alloc_aligned_slow(size, alignment, arena_index); + if (addr == NULL) { + return NULL; + } + } + + *zero = true; + *commit = true; + + return addr; +} + +static bool arena_extent_dalloc(extent_hooks_t*, void*, size_t, bool, + unsigned) { + return true; +} + +static bool arena_extent_commit(extent_hooks_t*, void*, size_t, size_t, size_t, + unsigned) { + return false; +} + +static bool arena_extent_decommit(extent_hooks_t*, void*, size_t, size_t, + size_t, unsigned) { + return true; +} + +static bool arena_extent_purge(extent_hooks_t*, void* addr, size_t, + size_t offset, size_t length, unsigned) { + int err = madvise((char*)addr + offset, length, MADV_DONTNEED); + return (err != 0); +} + +static bool arena_extent_split(extent_hooks_t*, void*, size_t, size_t, size_t, + bool, unsigned) { + return false; +} + +static bool arena_extent_merge(extent_hooks_t*, void*, size_t, void*, size_t, + bool, unsigned) { + return false; +} + +// clang-format off +static extent_hooks_t arena_extent_hooks = { + arena_extent_alloc, + arena_extent_dalloc, + NULL, + arena_extent_commit, + arena_extent_decommit, + arena_extent_purge, + NULL, + arena_extent_split, + arena_extent_merge +}; +// clang-format on + +unsigned JemallocMemoryAllocator::getArenaForCurrentThread() { + unsigned int arena_idx; + arena_idx = hash64(get_fs_base()) & arena_mask_; + return arena_index_[arena_idx]; +} + +unsigned JemallocMemoryAllocator::createOneArena() { + std::unique_lock ul(jemalloc_arena_spin); + + unsigned arena_index; + size_t unsigned_size = sizeof(unsigned int); + je_kvdk_mallctl("arenas.create", (void*)&arena_index, &unsigned_size, NULL, + 0); + + // setup extent_hooks for newly created arena + char cmd[64]; + extent_hooks_t* hooks = &arena_extent_hooks; + snprintf(cmd, sizeof(cmd), "arena.%u.extent_hooks", arena_index); + je_kvdk_mallctl(cmd, NULL, NULL, (void*)&hooks, sizeof(extent_hooks_t*)); + + arena_allocator_map[arena_index] = this; + return arena_index; +} + +void JemallocMemoryAllocator::destroyArenas() { + if (!arenas_initialized_ || num_arenas_ == 0) { + return; + } + + std::unique_lock ul(jemalloc_arena_spin); + + for (uint32_t i = 0; i < num_arenas_; i++) { + char cmd[128]; + unsigned arena_index = arena_index_[i]; + snprintf(cmd, 128, "arena.%u.destroy", arena_index); + je_kvdk_mallctl(cmd, NULL, NULL, NULL, 0); + arena_allocator_map.erase(arena_index); + } + + arena_index_.clear(); + arenas_initialized_ = false; +} + +void* JemallocMemoryAllocator::allocateWithFlags(size_t size, int flags) { + if (KVDK_LIKELY(size)) { + void* ptr = je_kvdk_mallocx(size, flags); + return ptr; + } + return nullptr; +} + +int JemallocMemoryAllocator::checkAlignment(size_t alignment) { + int err = 0; + if ((alignment < sizeof(void*)) || (((alignment - 1) & alignment) != 0)) { + err = 1; + } + return err; +} + +#endif // #ifdef KVDK_WITH_JEMALLOC + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/dram_allocator.hpp b/volatile/engine/dram_allocator.hpp new file mode 100644 index 00000000..e32f19c0 --- /dev/null +++ b/volatile/engine/dram_allocator.hpp @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include + +#include +#include +#include + +#include "alias.hpp" +#include "allocator.hpp" +#include "kvdk/volatile/engine.hpp" +#include "logger.hpp" +#include "macros.hpp" +#include "structures.hpp" + +#ifdef KVDK_WITH_JEMALLOC +#include "jemalloc/jemalloc.h" +#include "numa.h" +#include "numaif.h" +#endif + +namespace KVDK_NAMESPACE { + +// System memory allocator used for default scenarios. +class SystemMemoryAllocator : public Allocator { + public: + SystemMemoryAllocator() : Allocator(0, UINT64_MAX) {} + + SpaceEntry Allocate(uint64_t size) override { + SpaceEntry entry; + void* addr = malloc(size); + if (addr != nullptr) { + LogAllocation(ThreadManager::ThreadID(), size); + entry.offset = reinterpret_cast(addr); + entry.size = size; + } + + return entry; + } + + SpaceEntry AllocateAligned(size_t alignment, uint64_t size) override { + SpaceEntry entry; + void* addr = aligned_alloc(alignment, size); + if (addr != nullptr) { + LogAllocation(ThreadManager::ThreadID(), size); + entry.offset = reinterpret_cast(addr); + entry.size = size; + } + + return entry; + } + + void Free(const SpaceEntry& entry) override { + if (entry.offset) { + free(reinterpret_cast(entry.offset)); + LogDeallocation(ThreadManager::ThreadID(), entry.size); + } + } + + std::string AllocatorName() override { return "System"; } +}; + +#ifdef KVDK_WITH_JEMALLOC +// jemalloc memory allocator used for KV. +class JemallocMemoryAllocator : public Allocator { + public: + JemallocMemoryAllocator() : Allocator(0, UINT64_MAX) {} + + SpaceEntry Allocate(uint64_t size) override { + SpaceEntry entry; + + void* addr = nullptr; + if (KVDK_UNLIKELY(num_arenas_ == 0)) { + addr = je_kvdk_malloc(size); + } else { + initializeArenas(); + unsigned arena = getArenaForCurrentThread(); + addr = + allocateWithFlags(size, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE); + } + + if (KVDK_LIKELY(addr != nullptr)) { + LogAllocation(ThreadManager::ThreadID(), size); + entry.offset = reinterpret_cast(addr); + entry.size = size; + } + + return entry; + } + + SpaceEntry AllocateAligned(size_t alignment, uint64_t size) override { + SpaceEntry entry; + + void* addr = nullptr; + if (KVDK_UNLIKELY(num_arenas_ == 0)) { + addr = je_kvdk_aligned_alloc(alignment, size); + } else { + if (KVDK_LIKELY(checkAlignment(alignment) == 0)) { + initializeArenas(); + unsigned arena = getArenaForCurrentThread(); + addr = allocateWithFlags(size, MALLOCX_ALIGN(alignment) | + MALLOCX_ARENA(arena) | + MALLOCX_TCACHE_NONE); + } + } + + if (KVDK_LIKELY(addr != nullptr)) { + LogAllocation(ThreadManager::ThreadID(), size); + entry.offset = reinterpret_cast(addr); + entry.size = size; + } + + return entry; + } + + void Free(const SpaceEntry& entry) override { + if (KVDK_UNLIKELY(entry.offset == 0)) { + return; + } + + void* ptr = reinterpret_cast(entry.offset); + if (KVDK_UNLIKELY(num_arenas_ == 0)) { + je_kvdk_free(ptr); + } else { + je_kvdk_dallocx(ptr, MALLOCX_TCACHE_NONE); + } + + LogDeallocation(ThreadManager::ThreadID(), entry.size); + } + + std::string AllocatorName() override { return "jemalloc"; } + + bool SetMaxAccessThreads(uint32_t max_access_threads) override; + + void SetDestMemoryNodes(std::string dest_memory_nodes) override { + auto ul = Allocator::AcquireLock(); + + dest_memory_nodes_ = dest_memory_nodes; + setDestMemoryNodesMask(); + } + + int GetMbindMode() { return MPOL_BIND; } + + bitmask* GetMbindNodesMask() { return dest_memory_nodes_mask_; } + + ~JemallocMemoryAllocator() { + destroyArenas(); + + if (dest_memory_nodes_mask_) { + numa_bitmask_free(dest_memory_nodes_mask_); + dest_memory_nodes_mask_ = nullptr; + } + } + + private: + uint32_t num_arenas_{0}; + uint32_t arena_mask_{0}; + + bool arenas_initialized_{false}; + std::vector arena_index_; + + std::string dest_memory_nodes_; + bitmask* dest_memory_nodes_mask_ = nullptr; + + unsigned createOneArena(); + unsigned getArenaForCurrentThread(); + void destroyArenas(); + + static void* allocateWithFlags(size_t size, int flags); + static int checkAlignment(size_t alignment); + + inline void initializeArenas() { + if (KVDK_LIKELY(arenas_initialized_ || num_arenas_ == 0)) { + return; + } + + auto ul = Allocator::AcquireLock(); + if (arenas_initialized_ || num_arenas_ == 0) { + return; + } + + for (uint32_t i = 0; i < num_arenas_; i++) { + unsigned arena_index = createOneArena(); + arena_index_.push_back(arena_index); + } + + arenas_initialized_ = true; + } + + void setDestMemoryNodesMask() { + if (dest_memory_nodes_mask_) { + numa_bitmask_free(dest_memory_nodes_mask_); + dest_memory_nodes_mask_ = nullptr; + } + + if (dest_memory_nodes_.size() == 0) { + return; + } + + dest_memory_nodes_mask_ = numa_parse_nodestring(dest_memory_nodes_.c_str()); + if (dest_memory_nodes_mask_ == 0) { + GlobalLogger.Error("Invalid value of `dest_numa_nodes`: %s.\n", + dest_memory_nodes_.c_str()); + } + } +}; +#endif // #ifdef KVDK_WITH_JEMALLOC + +// Chunk based simple implementation +// TODO: optimize, implement free +class ChunkBasedAllocator { + public: + SpaceEntry Allocate(uint64_t size); + void Free(const SpaceEntry& entry); + + ChunkBasedAllocator(uint32_t max_access_threads, Allocator* alloc) + : dalloc_thread_cache_(max_access_threads), alloc_(alloc) {} + ChunkBasedAllocator(ChunkBasedAllocator const&) = delete; + ChunkBasedAllocator(ChunkBasedAllocator&&) = delete; + ~ChunkBasedAllocator() { + for (uint64_t i = 0; i < dalloc_thread_cache_.size(); i++) { + auto& tc = dalloc_thread_cache_[i]; + for (auto chunk : tc.allocated_chunks) { + alloc_->Free(chunk); + } + } + } + + template + inline T* offset2addr(MemoryOffsetType offset) const { + return alloc_->offset2addr(offset); + } + + private: + struct alignas(64) DAllocThreadCache { + char* chunk_addr = nullptr; + uint64_t usable_bytes = 0; + std::vector allocated_chunks; + + DAllocThreadCache() = default; + DAllocThreadCache(const DAllocThreadCache&) = delete; + DAllocThreadCache(DAllocThreadCache&&) = delete; + }; + + const uint32_t chunk_size_ = (1 << 20); + Array dalloc_thread_cache_; + Allocator* alloc_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/engine/engine.cpp b/volatile/engine/engine.cpp similarity index 95% rename from engine/engine.cpp rename to volatile/engine/engine.cpp index 9ba3bf3a..c4169ddb 100644 --- a/engine/engine.cpp +++ b/volatile/engine/engine.cpp @@ -2,7 +2,7 @@ * Copyright(c) 2021 Intel Corporation */ -#include "kvdk/engine.hpp" +#include "kvdk/volatile/engine.hpp" #include "kv_engine.hpp" diff --git a/volatile/engine/hash_collection/hash_list.cpp b/volatile/engine/hash_collection/hash_list.cpp new file mode 100644 index 00000000..d85ede4b --- /dev/null +++ b/volatile/engine/hash_collection/hash_list.cpp @@ -0,0 +1,424 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "hash_list.hpp" + +namespace KVDK_NAMESPACE { +HashList::WriteResult HashList::Put(const StringView& key, + const StringView& value, + TimestampType timestamp) { + WriteResult ret; + HashWriteArgs args = InitWriteArgs(key, value, WriteOp::Put); + ret.s = PrepareWrite(args, timestamp); + if (ret.s == Status::Ok) { + ret = Write(args); + } + return ret; +} + +Status HashList::Get(const StringView& key, std::string* value) { + std::string internal_key(InternalKey(key)); + auto lookup_result = + hash_table_->Lookup(internal_key, RecordType::HashElem); + if (lookup_result.s != Status::Ok || + lookup_result.entry.GetRecordStatus() == RecordStatus::Outdated) { + return Status::NotFound; + } + + DLRecord* data_record = lookup_result.entry.GetIndex().dl_record; + kvdk_assert(data_record->GetRecordType() == RecordType::HashElem, ""); + // As get is lockless, skiplist node may point to a new elem delete record + // after we get it from hashtable + if (data_record->GetRecordStatus() == RecordStatus::Outdated) { + return Status::NotFound; + } else { + value->assign(data_record->Value().data(), data_record->Value().size()); + return Status::Ok; + } +} + +HashList::WriteResult HashList::Delete(const StringView& key, + TimestampType timestamp) { + WriteResult ret; + HashWriteArgs args = InitWriteArgs(key, "", WriteOp::Delete); + ret.s = PrepareWrite(args, timestamp); + if (ret.s == Status::Ok && args.space.size > 0) { + ret = Write(args); + } + return ret; +} + +HashList::WriteResult HashList::Modify(const StringView key, + ModifyFunc modify_func, + void* modify_args, TimestampType ts) { + WriteResult ret; + std::string internal_key(InternalKey(key)); + auto lookup_result = + hash_table_->Lookup(internal_key, RecordType::HashElem); + DLRecord* existing_record = nullptr; + std::string exisiting_value; + std::string new_value; + bool data_existing = false; + if (lookup_result.s == Status::Ok) { + existing_record = lookup_result.entry.GetIndex().dl_record; + ret.existing_record = existing_record; + if (existing_record->GetRecordStatus() != RecordStatus::Outdated) { + data_existing = true; + exisiting_value.assign(existing_record->Value().data(), + existing_record->Value().size()); + } + } else if (lookup_result.s == Status::NotFound) { + // nothing todo + } else { + ret.s = lookup_result.s; + return ret; + } + + auto modify_operation = modify_func( + data_existing ? &exisiting_value : nullptr, &new_value, modify_args); + switch (modify_operation) { + case ModifyOperation::Write: { + // TODO: check new value size + HashWriteArgs args = InitWriteArgs(key, new_value, WriteOp::Put); + args.ts = ts; + args.lookup_result = lookup_result; + args.space = kv_allocator_->Allocate( + DLRecord::RecordSize(internal_key, new_value)); + if (args.space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + return Write(args); + } + + case ModifyOperation::Delete: { + HashWriteArgs args = InitWriteArgs(key, "", WriteOp::Delete); + args.ts = ts; + args.lookup_result = lookup_result; + args.space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, "")); + if (args.space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + return Write(args); + } + + case ModifyOperation::Abort: { + ret.s = Status::Abort; + return ret; + } + + case ModifyOperation::Noop: { + ret.s = Status::Ok; + return ret; + } + + default: { + std::abort(); // non-reach area + } + } +} + +HashWriteArgs HashList::InitWriteArgs(const StringView& key, + const StringView& value, WriteOp op) { + HashWriteArgs args; + args.key = key; + args.value = value; + args.op = op; + args.collection = Name(); + args.hlist = this; + return args; +} + +Status HashList::PrepareWrite(HashWriteArgs& args, TimestampType ts) { + kvdk_assert(args.op == WriteOp::Put || args.value.size() == 0, + "value of delete operation should be empty"); + if (args.hlist != this) { + return Status::InvalidArgument; + } + + args.ts = ts; + bool op_delete = args.op == WriteOp::Delete; + std::string internal_key(InternalKey(args.key)); + bool allocate_space = true; + if (op_delete) { + args.lookup_result = + hash_table_->Lookup(internal_key, RecordType::HashElem); + } else { + args.lookup_result = + hash_table_->Lookup(internal_key, RecordType::HashElem); + } + + switch (args.lookup_result.s) { + case Status::Ok: { + if (op_delete && args.lookup_result.entry.GetRecordStatus() == + RecordStatus::Outdated) { + allocate_space = false; + } + break; + } + case Status::NotFound: { + if (op_delete) { + allocate_space = false; + } + break; + } + case Status::MemoryOverflow: { + return args.lookup_result.s; + } + default: + std::abort(); // never should reach + } + + if (allocate_space) { + auto request_size = DLRecord::RecordSize(internal_key, args.value); + args.space = kv_allocator_->Allocate(request_size); + if (args.space.size == 0) { + return Status::MemoryOverflow; + } + } + + return Status::Ok; +} + +HashList::WriteResult HashList::Write(HashWriteArgs& args) { + WriteResult ret; + if (args.hlist != this) { + ret.s = Status::InvalidArgument; + return ret; + } + if (args.op == WriteOp::Put) { + ret = putPrepared(args.lookup_result, args.key, args.value, args.ts, + args.space); + if (ret.existing_record == nullptr || + ret.existing_record->GetRecordStatus() == RecordStatus::Outdated) { + UpdateSize(1); + } + } else { + ret = deletePrepared(args.lookup_result, args.key, args.ts, args.space); + if (ret.existing_record != nullptr && + ret.existing_record->GetRecordStatus() == RecordStatus::Normal) { + UpdateSize(-1); + } + } + return ret; +} + +HashList::WriteResult HashList::SetExpireTime(ExpireTimeType expired_time, + TimestampType timestamp) { + WriteResult ret; + DLRecord* header = HeaderRecord(); + SpaceEntry space = kv_allocator_->Allocate( + DLRecord::RecordSize(header->Key(), header->Value())); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space.offset), space.size, timestamp, + RecordType::HashHeader, RecordStatus::Normal, + kv_allocator_->addr2offset_checked(header), header->prev, header->next, + header->Key(), header->Value(), expired_time); + bool success = dl_list_.Replace(header, data_record); + kvdk_assert(success, "existing header should be linked on its list"); + ret.existing_record = header; + ret.write_record = data_record; + return ret; +} + +Status HashList::CheckIndex() { + DLRecord* prev = HeaderRecord(); + size_t cnt = 0; + DLListRecoveryUtils recovery_utils(kv_allocator_); + while (true) { + DLRecord* curr = kv_allocator_->offset2addr_checked(prev->next); + if (curr == HeaderRecord()) { + break; + } + StringView key = curr->Key(); + auto ret = hash_table_->Lookup(key, curr->GetRecordType()); + if (ret.s != Status::Ok) { + GlobalLogger.Error( + "Check hash index error: record not exist in hash table\n"); + return Status::Abort; + } + if (ret.entry.GetIndex().dl_record != curr) { + GlobalLogger.Error( + "Check hash index error: Dlrecord miss-match with hash " + "table\n"); + return Status::Abort; + } + if (!recovery_utils.CheckLinkage(curr)) { + GlobalLogger.Error("Check hash index error: dl record linkage error\n"); + return Status::Abort; + } + cnt++; + prev = curr; + } + return Status::Ok; +} + +CollectionIDType HashList::FetchID(const DLRecord* record) { + assert(record != nullptr); + switch (record->GetRecordType()) { + case RecordType::HashElem: + return ExtractID(record->Key()); + case RecordType::HashHeader: + return DecodeID(record->Value()); + default: + GlobalLogger.Error("Wrong record type %u in HashListID", + record->GetRecordType()); + kvdk_assert(false, "Wrong type in HashListID"); + return 0; + } +} + +void HashList::Destroy() { + std::vector to_free; + DLRecord* header = HeaderRecord(); + if (header) { + DLRecord* to_destroy = nullptr; + do { + to_destroy = kv_allocator_->offset2addr_checked(header->next); + StringView key = to_destroy->Key(); + auto ul = hash_table_->AcquireLock(key); + if (dl_list_.Remove(to_destroy)) { + auto lookup_result = + hash_table_->Lookup(key, to_destroy->GetRecordType()); + if (lookup_result.s == Status::Ok) { + DLRecord* hash_indexed_record = nullptr; + auto hash_index = lookup_result.entry.GetIndex(); + switch (lookup_result.entry.GetIndexType()) { + case PointerType::HashList: + hash_indexed_record = hash_index.hlist->HeaderRecord(); + break; + case PointerType::DLRecord: + hash_indexed_record = hash_index.dl_record; + break; + default: + kvdk_assert(false, "Wrong hash index type of hash record"); + } + + if (hash_indexed_record == to_destroy) { + hash_table_->Erase(lookup_result.entry_ptr); + } + } + + to_destroy->Destroy(); + to_free.emplace_back(kv_allocator_->addr2offset_checked(to_destroy), + to_destroy->GetRecordSize()); + if (to_free.size() > kMaxCachedOldRecords) { + kv_allocator_->BatchFree(to_free); + to_free.clear(); + } + } + } while (to_destroy != header); + } + kv_allocator_->BatchFree(to_free); +} + +void HashList::DestroyAll() { + std::vector to_free; + DLRecord* header = HeaderRecord(); + DLRecord* to_destroy = nullptr; + kvdk_assert(header != nullptr, ""); + do { + to_destroy = kv_allocator_->offset2addr_checked(header->next); + StringView key = to_destroy->Key(); + auto ul = hash_table_->AcquireLock(key); + if (dl_list_.Remove(to_destroy)) { + auto lookup_result = + hash_table_->Lookup(key, to_destroy->GetRecordType()); + if (lookup_result.s == Status::Ok) { + DLRecord* hash_indexed_record = nullptr; + auto hash_index = lookup_result.entry.GetIndex(); + switch (lookup_result.entry.GetIndexType()) { + case PointerType::HashList: + hash_indexed_record = hash_index.hlist->HeaderRecord(); + break; + case PointerType::DLRecord: + hash_indexed_record = hash_index.dl_record; + break; + default: + kvdk_assert(false, "Wrong hash index type of hash record"); + } + + if (hash_indexed_record == to_destroy) { + hash_table_->Erase(lookup_result.entry_ptr); + } + } + auto old_record = + kv_allocator_->offset2addr(to_destroy->old_version); + while (old_record) { + auto old_version = old_record->old_version; + to_free.emplace_back(kv_allocator_->addr2offset_checked(old_record), + old_record->GetRecordSize()); + old_record->Destroy(); + old_record = kv_allocator_->offset2addr(old_version); + } + + to_free.emplace_back(kv_allocator_->addr2offset_checked(to_destroy), + to_destroy->GetRecordSize()); + to_destroy->Destroy(); + if (to_free.size() > kMaxCachedOldRecords) { + kv_allocator_->BatchFree(to_free); + to_free.clear(); + } + } + } while (to_destroy != header); + kv_allocator_->BatchFree(to_free); +} + +HashList::WriteResult HashList::putPrepared( + const HashTable::LookupResult& lookup_result, const StringView& key, + const StringView& value, TimestampType timestamp, const SpaceEntry& space) { + WriteResult ret; + std::string internal_key(InternalKey(key)); + DLList::WriteArgs args(internal_key, value, RecordType::HashElem, + RecordStatus::Normal, timestamp, space); + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + ret.hash_entry_ptr = lookup_result.entry_ptr; + if (lookup_result.s == Status::Ok) { + ret.existing_record = lookup_result.entry.GetIndex().dl_record; + kvdk_assert(timestamp > ret.existing_record->GetTimestamp(), ""); + while ((ret.s = dl_list_.Update(args, ret.existing_record)) != Status::Ok) { + kvdk_assert(ret.s == Status::Fail, ""); + } + } else { + kvdk_assert(lookup_result.s == Status::NotFound, ""); + bool push_back = fast_random_64() % 2 == 0; + Status s = push_back ? dl_list_.PushBack(args) : dl_list_.PushFront(args); + kvdk_assert(s == Status::Ok, ""); + } + hash_table_->Insert(lookup_result, RecordType::HashElem, RecordStatus::Normal, + ret.write_record, PointerType::DLRecord); + return ret; +} + +HashList::WriteResult HashList::deletePrepared( + const HashTable::LookupResult& lookup_result, const StringView& key, + TimestampType timestamp, const SpaceEntry& space) { + WriteResult ret; + std::string internal_key(InternalKey(key)); + kvdk_assert(lookup_result.s == Status::Ok && + lookup_result.entry.GetRecordType() == RecordType::HashElem && + lookup_result.entry.GetRecordStatus() == RecordStatus::Normal, + ""); + assert(space.size >= DLRecord::RecordSize(internal_key, "")); + ret.existing_record = lookup_result.entry.GetIndex().dl_record; + kvdk_assert(timestamp > ret.existing_record->GetTimestamp(), ""); + DLList::WriteArgs args(internal_key, "", RecordType::HashElem, + RecordStatus::Outdated, timestamp, space); + while ((ret.s = dl_list_.Update(args, ret.existing_record)) != Status::Ok) { + kvdk_assert(ret.s == Status::Fail, ""); + } + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + hash_table_->Insert(lookup_result, RecordType::HashElem, + RecordStatus::Outdated, ret.write_record, + PointerType::DLRecord); + return ret; +} + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/hash_collection/hash_list.hpp b/volatile/engine/hash_collection/hash_list.hpp new file mode 100644 index 00000000..b64f21f9 --- /dev/null +++ b/volatile/engine/hash_collection/hash_list.hpp @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#pragma once + +#include "../dl_list.hpp" +#include "../hash_table.hpp" +#include "kvdk/volatile/types.hpp" + +namespace KVDK_NAMESPACE { + +class HashIteratorImpl; + +struct HashWriteArgs { + StringView collection; + StringView key; + StringView value; + WriteOp op; + HashList* hlist; + SpaceEntry space; + TimestampType ts; + HashTable::LookupResult lookup_result; +}; + +class HashList : public Collection { + public: + struct WriteResult { + Status s = Status::Ok; + DLRecord* existing_record = nullptr; + DLRecord* write_record = nullptr; + HashEntry* hash_entry_ptr = nullptr; + }; + + HashList(DLRecord* header, const StringView& name, CollectionIDType id, + Allocator* kv_allocator, HashTable* hash_table, + LockTable* lock_table) + : Collection(name, id), + dl_list_(header, kv_allocator, lock_table), + size_(0), + kv_allocator_(kv_allocator), + hash_table_(hash_table) {} + + ~HashList() final = default; + + DLList* GetDLList() { return &dl_list_; } + + const DLRecord* HeaderRecord() const { return dl_list_.Header(); } + + DLRecord* HeaderRecord() { return dl_list_.Header(); } + + ExpireTimeType GetExpireTime() const final { + return HeaderRecord()->GetExpireTime(); + } + + TimestampType GetTimeStamp() const { return HeaderRecord()->GetTimestamp(); } + + bool HasExpired() const final { return HeaderRecord()->HasExpired(); } + + // Return number of valid data record in this hash list + size_t Size() { return size_; } + + // Put "key, value" to the hash list + // + // Args: + // * timestamp: kvdk engine timestamp of this operation + // + // Return Ok on success, with the writed data record, its dram node and + // updated data record if it exists + // + // Notice: the putting key should already been locked by engine + WriteResult Put(const StringView& key, const StringView& value, + TimestampType timestamp); + + // Get value of "key" from the hash list + Status Get(const StringView& key, std::string* value); + + // Delete "key" from the hash list by replace it with a delete record + // + // Args: + // * timestamp: kvdk engine timestamp of this operation + // + // Return Ok on success, with the writed delete record and deleted + // record if it exists + // + // Notice: the deleting key should already been locked by engine + WriteResult Delete(const StringView& key, TimestampType timestamp); + + // Modify value of "key" in the hash list + // + // Args: + // * modify_func: customized function to modify existing value of key. See + // definition of ModifyFunc (types.hpp) for more details. + // * modify_args: customized arguments of modify_func. + // + // Return: + // Status::Ok if modify success. + // Status::Abort if modify function abort modifying. + // Return other non-Ok status on any error. + WriteResult Modify(const StringView key, ModifyFunc modify_func, + void* modify_args, TimestampType timestamp); + + // Init args for put or delete operations + HashWriteArgs InitWriteArgs(const StringView& key, const StringView& value, + WriteOp op); + + // Prepare neccessary resources for write, store lookup result of key and + // required memory space to write new reocrd in args + // + // Args: + // * args: generated by InitWriteArgs() + // + // Return: + // Ok on success + // MemoryOverflow if no enough kv memory space + // MemoryOverflow if no enough dram space + // + // Notice: args.key should already been locked by engine + Status PrepareWrite(HashWriteArgs& args, TimestampType ts); + + // Do batch write according to args + // + // Args: + // * args: write args prepared by PrepareWrite() + // + // Return: + // Status Ok on success, with the writed delete record, its dram node and + // deleted record if existing + WriteResult Write(HashWriteArgs& args); + + // Set this hash list expire at expired_time + // + // Args: + // * expired_time: time to expire + // * timestamp: kvdk engine timestamp of calling this function + // + // Return Ok on success + WriteResult SetExpireTime(ExpireTimeType expired_time, + TimestampType timestamp); + + // Replace "old_record" from the hash list with "replacing_record" + // + // Args: + // * old_record: existing record to be replaced + // * new_record: new reocrd to replace the older one + // + // Return: + // * true on success + // * false if old_record not linked on a skiplist + // + // Notice: + // 1. key of the replacing record should already been locked by engine + // 2. hash table will not be modified + bool Replace(DLRecord* old_record, DLRecord* new_record) { + return dl_list_.Replace(old_record, new_record); + } + + // Destroy and free the whole hash list with old version list. + void DestroyAll(); + + // Destroy and free the whole hash list of newest version records + void Destroy(); + + void UpdateSize(int64_t delta) { + kvdk_assert(delta >= 0 || size_.load() >= static_cast(-delta), + "Update hash list size to negative"); + size_.fetch_add(delta, std::memory_order_relaxed); + } + + Status CheckIndex(); + + bool TryCleaningLock() { return cleaning_lock_.try_lock(); } + + void ReleaseCleaningLock() { cleaning_lock_.unlock(); } + + static CollectionIDType FetchID(const DLRecord* record); + + static bool MatchType(const DLRecord* record) { + RecordType type = record->GetRecordType(); + return type == RecordType::HashElem || type == RecordType::HashHeader; + } + + private: + friend HashIteratorImpl; + DLList dl_list_; + std::atomic size_; + Allocator* kv_allocator_; + HashTable* hash_table_; + // to avoid illegal access caused by cleaning skiplist by multi-thread + SpinMutex cleaning_lock_; + + WriteResult putPrepared(const HashTable::LookupResult& lookup_result, + const StringView& key, const StringView& value, + TimestampType timestamp, const SpaceEntry& space); + + WriteResult deletePrepared(const HashTable::LookupResult& lookup_result, + const StringView& key, TimestampType timestamp, + const SpaceEntry& space); +}; +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/hash_collection/iterator.hpp b/volatile/engine/hash_collection/iterator.hpp new file mode 100644 index 00000000..d1d2c21f --- /dev/null +++ b/volatile/engine/hash_collection/iterator.hpp @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#pragma once + +#include "../version/version_controller.hpp" +#include "hash_list.hpp" +#include "kvdk/volatile/engine.hpp" +#include "kvdk/volatile/iterator.hpp" + +namespace KVDK_NAMESPACE { +class KVEngine; + +class HashIteratorImpl final : public HashIterator { + public: + HashIteratorImpl(HashList* hlist, const SnapshotImpl* snapshot, + bool own_snapshot) + : hlist_(hlist), + snapshot_(snapshot), + own_snapshot_(own_snapshot), + dl_iter_(&hlist->dl_list_, hlist->kv_allocator_, snapshot) {} + void SeekToFirst() final { dl_iter_.SeekToFirst(); } + + void SeekToLast() final { dl_iter_.SeekToLast(); } + + bool Valid() const final { return dl_iter_.Valid(); } + + void Next() final { dl_iter_.Next(); } + + void Prev() final { dl_iter_.Prev(); } + + std::string Key() const final { + if (!Valid()) { + kvdk_assert(false, "Accessing data with invalid HashIterator!"); + return std::string{}; + } + return string_view_2_string(Collection::ExtractUserKey(dl_iter_.Key())); + } + + std::string Value() const final { + if (!Valid()) { + kvdk_assert(false, "Accessing data with invalid HashIterator!"); + return std::string{}; + } + return string_view_2_string(dl_iter_.Value()); + } + + bool MatchKey(std::regex const& re) final { + if (!Valid()) { + kvdk_assert(false, "Accessing data with invalid HashIterator!"); + return false; + } + return std::regex_match(Key(), re); + } + + private: + friend KVEngine; + + HashList* hlist_; + const SnapshotImpl* snapshot_; + bool own_snapshot_; + DLListDataIterator dl_iter_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/hash_table.cpp b/volatile/engine/hash_table.cpp new file mode 100644 index 00000000..3ffedbfe --- /dev/null +++ b/volatile/engine/hash_table.cpp @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "hash_table.hpp" + +#include "hash_collection/hash_list.hpp" +#include "list_collection/list.hpp" +#include "sorted_collection/skiplist.hpp" +#include "thread_manager.hpp" + +namespace KVDK_NAMESPACE { +HashTable* HashTable::NewHashTable(uint64_t hash_bucket_num, + uint32_t num_buckets_per_slot, + const Allocator* kv_allocator, + Allocator* new_bucket_allocator, + uint32_t max_access_threads) { + HashTable* table; + // We catch exception here as we may need to allocate large memory for hash + // table here + try { + table = new HashTable(hash_bucket_num, num_buckets_per_slot, kv_allocator, + new_bucket_allocator, max_access_threads); + } catch (std::bad_alloc& b) { + GlobalLogger.Error("No enough dram to create global hash table: b\n", + b.what()); + table = nullptr; + } + + return table; +} + +bool HashEntry::Match(const StringView& key, uint32_t hash_k_prefix, + uint8_t target_type, DataEntry* data_entry_metadata) { + if ((target_type & header_.record_type) && + hash_k_prefix == header_.key_prefix) { + void* data_record = nullptr; + StringView data_entry_key; + + switch (header_.index_type) { + case PointerType::Empty: + case PointerType::Allocated: { + return false; + } + case PointerType::StringRecord: { + data_record = index_.string_record; + data_entry_key = index_.string_record->Key(); + break; + } + case PointerType::HashElem: + case PointerType::DLRecord: { + data_record = index_.dl_record; + data_entry_key = index_.dl_record->Key(); + break; + } + case PointerType::List: { + data_entry_key = index_.list->Name(); + break; + } + case PointerType::HashList: { + data_entry_key = index_.hlist->Name(); + break; + } + case PointerType::SkiplistNode: { + SkiplistNode* dram_node = index_.skiplist_node; + data_record = dram_node->record; + data_entry_key = dram_node->record->Key(); + break; + } + case PointerType::Skiplist: { + Skiplist* skiplist = index_.skiplist; + data_record = skiplist->HeaderRecord(); + data_entry_key = skiplist->Name(); + break; + } + default: { + GlobalLogger.Error("Not supported hash index type: %u\n", + header_.index_type); + assert(false && "Trying to use invalid PointerType!"); + return false; + } + } + + if (data_entry_metadata != nullptr) { + memcpy(data_entry_metadata, data_record, sizeof(DataEntry)); + } + + if (equal_string_view(key, data_entry_key)) { + return true; + } + } + return false; +} + +template +HashTable::LookupResult HashTable::Lookup(const StringView& key, + uint8_t type_mask) { + LookupResult ret; + HashEntry* empty_entry = nullptr; + auto hint = getHint(key); + ret.key_hash_prefix = hint.key_hash_prefix; + + HashBucket* bucket_ptr = &hash_buckets_[hint.bucket]; + _mm_prefetch(bucket_ptr, _MM_HINT_T0); + + // search cache + ret.entry_ptr = slots_[hint.slot].hash_cache.entry_ptr; + if (ret.entry_ptr != nullptr) { + atomic_load_16(&ret.entry, ret.entry_ptr); + if (ret.entry.Match(key, hint.key_hash_prefix, type_mask, nullptr)) { + return ret; + } + } + + // iterate hash entries in the bucket + HashBucketIterator iter(this, hint.bucket); + while (iter.Valid()) { + ret.entry_ptr = &*iter; + atomic_load_16(&ret.entry, ret.entry_ptr); + if (ret.entry.Match(key, hint.key_hash_prefix, type_mask, nullptr)) { + slots_[hint.slot].hash_cache.entry_ptr = ret.entry_ptr; + return ret; + } + if (ret.entry_ptr->Empty()) { + empty_entry = ret.entry_ptr; + } + iter++; + } + + if (may_insert) { + if (empty_entry == nullptr) { + ret.s = allocateEntry(iter); + if (ret.s != Status::Ok) { + kvdk_assert(ret.s == Status::MemoryOverflow, ""); + return ret; + } + kvdk_assert( + iter.Valid(), + "HashBucketIterator should be valid after allocate new entry"); + kvdk_assert(iter->Empty(), "newly allocated hash entry should be empty"); + ret.entry_ptr = &(*iter); + } else { + ret.entry_ptr = empty_entry; + } + } + + ret.s = NotFound; + if (may_insert) { + ret.entry_ptr->MarkAsAllocated(); + } + return ret; +} + +template HashTable::LookupResult HashTable::Lookup(const StringView&, + uint8_t); +template HashTable::LookupResult HashTable::Lookup(const StringView&, + uint8_t); + +void HashTable::Insert(const LookupResult& insert_position, RecordType type, + RecordStatus status, void* index, + PointerType index_type) { + HashEntry new_hash_entry(insert_position.key_hash_prefix, type, status, index, + index_type); + atomic_store_16(insert_position.entry_ptr, &new_hash_entry); +} + +HashTable::LookupResult HashTable::Insert(const StringView& key, + RecordType type, RecordStatus status, + void* index, PointerType index_type) { + auto lookup_result = Lookup(key, type); + if (lookup_result.s == Status::Ok || lookup_result.s == Status::NotFound) { + Insert(lookup_result, type, status, index, index_type); + } + return lookup_result; +} + +Status HashTable::allocateEntry(HashBucketIterator& bucket_iter) { + kvdk_assert(bucket_iter.hash_table_ == this, ""); + kvdk_assert( + bucket_iter.entry_idx_ == hash_bucket_entries_[bucket_iter.bucket_idx_], + "Only allocate new hash entry at end of hash bucket"); + kvdk_assert(bucket_iter.bucket_ptr_ != nullptr, ""); + if (hash_bucket_entries_[bucket_iter.bucket_idx_] > 0 && + hash_bucket_entries_[bucket_iter.bucket_idx_] % kNumEntryPerBucket == 0) { + auto space = chunk_based_bucket_allocator_.Allocate(kHashBucketSize); + if (space.size == 0) { + GlobalLogger.Error("MemoryOverflow!\n"); + return Status::MemoryOverflow; + } + bucket_iter.bucket_ptr_->next = + chunk_based_bucket_allocator_.offset2addr(space.offset); + bucket_iter.bucket_ptr_ = bucket_iter.bucket_ptr_->next; + } + bucket_iter.entry_idx_ = hash_bucket_entries_[bucket_iter.bucket_idx_]++; + bucket_iter->Clear(); + kvdk_assert(bucket_iter.Valid(), ""); + return Status::Ok; +} + +HashTableIterator HashTable::GetIterator(uint64_t start_slot_idx, + uint64_t end_slot_idx) { + return HashTableIterator{this, start_slot_idx, end_slot_idx}; +} + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/hash_table.hpp b/volatile/engine/hash_table.hpp new file mode 100644 index 00000000..f5720109 --- /dev/null +++ b/volatile/engine/hash_table.hpp @@ -0,0 +1,420 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include +#include + +#include "alias.hpp" +#include "allocator.hpp" +#include "data_record.hpp" +#include "dram_allocator.hpp" +#include "kvdk/volatile/engine.hpp" +#include "structures.hpp" + +namespace KVDK_NAMESPACE { + +class Skiplist; +class SkiplistNode; +struct HashBucketIterator; + +struct List; +struct HashList; + +struct alignas(16) HashEntry { + public: + friend class HashTable; + HashEntry& operator=(const HashEntry&) = delete; + HashEntry(const HashEntry& hash_entry) { atomic_load_16(this, &hash_entry); } + union Index { + Index(void* _ptr) : ptr(_ptr) {} + Index() = default; + void* ptr; + SkiplistNode* skiplist_node; + StringRecord* string_record; + DLRecord* dl_record; + Skiplist* skiplist; + List* list; + HashList* hlist; + }; + static_assert(sizeof(Index) == 8); + + HashEntry() = default; + + HashEntry(uint32_t key_hash_prefix, RecordType record_type, + RecordStatus record_status, void* _index, PointerType index_type) + : index_(_index), + header_({key_hash_prefix, record_type, record_status, index_type}) {} + + bool Empty() { return header_.index_type == PointerType::Empty; } + + // Make this hash entry empty while its content been deleted + void Clear() { header_.index_type = PointerType::Empty; } + + bool Allocated() { return header_.index_type == PointerType::Allocated; } + + void MarkAsAllocated() { header_.index_type = PointerType::Allocated; } + + Index GetIndex() const { return index_; } + + PointerType GetIndexType() const { return header_.index_type; } + + RecordType GetRecordType() const { return header_.record_type; } + + RecordStatus GetRecordStatus() const { return header_.record_status; } + + // Check if "key" of data type "target_type" is indexed by "this". If + // matches, copy data entry of data record of "key" to "data_entry_metadata" + // and return true, otherwise return false. + // + // Args: + // * target_type: a mask of RecordType, search all masked types + bool Match(const StringView& key, uint32_t hash_k_prefix, uint8_t target_type, + DataEntry* data_entry_metadata); + + private: + struct EntryHeader { + uint32_t key_prefix; + RecordType record_type; + RecordStatus record_status; + PointerType index_type; + }; + + Index index_; + EntryHeader header_; +}; +static_assert(sizeof(HashEntry) == 16); + +// Size of each hash bucket +// +// It should be larger than hans entry size (which is 16) plus 8 (the pointer +// to next bucket). It is recommended to set it align to cache line +constexpr size_t kHashBucketSize = 128; +constexpr size_t kNumEntryPerBucket = + (kHashBucketSize - sizeof(void*)) / sizeof(HashEntry); +struct HashBucket { + HashBucket() { + memset(hash_entries, 0, sizeof(HashEntry) * kNumEntryPerBucket); + } + HashEntry hash_entries[kNumEntryPerBucket]; + HashBucket* next{nullptr}; + char padding[kHashBucketSize - sizeof(hash_entries) - sizeof(next)]; +}; +static_assert(sizeof(HashBucket) == kHashBucketSize); + +struct HashCache { + HashEntry* entry_ptr = nullptr; +}; + +struct Slot { + HashCache hash_cache; + SpinMutex spin; +}; + +struct HashTableIterator; + +class HashTable { + public: + friend class HashTableIterator; + friend class HashSlotIterator; + friend class HashBucketIterator; + struct LookupResult { + public: + Status s{Status::Ok}; + HashEntry entry{}; + HashEntry* entry_ptr{nullptr}; + + LookupResult& operator=(LookupResult const& other) { + s = other.s; + memcpy_16(&entry, &other.entry); + entry_ptr = other.entry_ptr; + key_hash_prefix = other.key_hash_prefix; + return *this; + } + + private: + friend class HashTable; + uint32_t key_hash_prefix; + }; + + static HashTable* NewHashTable(uint64_t hash_bucket_num, + uint32_t num_buckets_per_slot, + const Allocator* kv_allocator, + Allocator* new_bucket_allocator, + uint32_t max_access_threads); + + // Look up key in hashtable + // Store a copy of hash entry in LookupResult::entry, and a pointer to the + // hash entry on hash table in LookupResult::entry_ptr + // If may_insert is true and key not found, then store + // pointer of a free-to-write hash entry in LookupResult::entry_ptr. + // + // * type_mask: which data types to search + // + // return status: + // Status::NotFound is key is not found. + // Status::MemoryOverflow if may_insert is true but + // failed to allocate new hash entry + // Status::Ok on success. + // + // Notice: key should be locked if set may_insert to true + template + LookupResult Lookup(const StringView& key, uint8_t type_mask); + + // Insert a hash entry to hash table + // * insert_position: indicate the the postion to insert new entry, it should + // be return of Lookup of the inserting key + void Insert(const LookupResult& insert_position, RecordType type, + RecordStatus status, void* index, PointerType index_type); + + // Lookup and insert a hash entry of key to hash table, return lookup result + LookupResult Insert(const StringView& key, RecordType type, + RecordStatus status, void* index, PointerType index_type); + + // Erase a hash entry so it can be reused in future + void Erase(HashEntry* entry_ptr) { + assert(entry_ptr != nullptr); + entry_ptr->Clear(); + } + + std::unique_lock AcquireLock(StringView const& key) { + return std::unique_lock{*getHint(key).spin}; + } + + HashTableIterator GetIterator(uint64_t start_slot_idx, uint64_t end_slot_idx); + + size_t GetSlotsNum() { return slots_.size(); } + + // StringAlike is std::string or StringView + template + std::vector> RangeLock( + std::vector const& keys) { + std::vector spins; + for (auto const& key : keys) { + spins.push_back(getHint(key).spin); + } + std::sort(spins.begin(), spins.end()); + auto end = std::unique(spins.begin(), spins.end()); + + std::vector> guard; + for (auto iter = spins.begin(); iter != end; ++iter) { + guard.emplace_back(**iter); + } + return guard; + } + + private: + HashTable(uint64_t hash_bucket_num, uint32_t num_buckets_per_slot, + const Allocator* kv_allocator, Allocator* new_bucket_allocator, + uint32_t max_access_threads) + : num_hash_buckets_(hash_bucket_num), + num_buckets_per_slot_(num_buckets_per_slot), + kv_allocator_(kv_allocator), + chunk_based_bucket_allocator_{max_access_threads, new_bucket_allocator}, + slots_(hash_bucket_num / num_buckets_per_slot), + hash_bucket_entries_(hash_bucket_num, 0), + hash_buckets_(num_hash_buckets_) {} + + struct KeyHashHint { + uint32_t bucket; + uint32_t slot; + // hash value stored on hash entry + uint32_t key_hash_prefix; + SpinMutex* spin; + }; + + KeyHashHint getHint(const StringView& key) { + KeyHashHint hint; + uint64_t hash_val = hash_str(key.data(), key.size()); + hint.key_hash_prefix = hash_val >> 32; + hint.bucket = get_bucket_num(hash_val); + hint.slot = get_slot_num(hint.bucket); + hint.spin = &slots_[hint.slot].spin; + return hint; + } + + inline uint32_t get_bucket_num(uint64_t key_hash_value) { + return key_hash_value & (num_hash_buckets_ - 1); + } + + inline uint32_t get_slot_num(uint32_t bucket) { + return bucket / num_buckets_per_slot_; + } + + Status allocateEntry(HashBucketIterator& bucket_iter); + + const uint64_t num_hash_buckets_; + const uint32_t num_buckets_per_slot_; + const Allocator* kv_allocator_; + ChunkBasedAllocator chunk_based_bucket_allocator_; + Array slots_; + std::vector hash_bucket_entries_; + Array hash_buckets_; + void* main_buckets_; +}; + +// Iterator all hash entries in a hash table bucket +class HashBucketIterator { + public: + HashBucketIterator(HashTable* hash_table /* should be non-null */, + uint64_t bucket_idx) + : hash_table_(hash_table), + bucket_idx_(bucket_idx), + entry_idx_(0), + bucket_ptr_(nullptr) { + if (bucket_idx_ < hash_table_->hash_buckets_.size()) { + bucket_ptr_ = &hash_table_->hash_buckets_[bucket_idx_]; + _mm_prefetch(bucket_ptr_, _MM_HINT_T0); + } + } + + HashBucketIterator(const HashBucketIterator&) = default; + + bool Valid() { + return bucket_ptr_ != nullptr && + entry_idx_ < hash_table_->hash_bucket_entries_[bucket_idx_]; + } + + HashEntry& operator*() { + return bucket_ptr_->hash_entries[entry_idx_ % kNumEntryPerBucket]; + } + + HashEntry* operator->() { return &operator*(); } + + HashBucketIterator& operator++() { + next(); + return *this; + } + + HashBucketIterator operator++(int) { + HashBucketIterator tmp{*this}; + this->operator++(); + return tmp; + } + + private: + friend class HashTable; + + void next() { + if (Valid()) { + entry_idx_++; + if (entry_idx_ % kNumEntryPerBucket == 0 && Valid()) { + bucket_ptr_ = bucket_ptr_->next; + _mm_prefetch(bucket_ptr_, _MM_HINT_T0); + } + } + } + + HashTable* hash_table_; + uint64_t bucket_idx_; + uint64_t entry_idx_; + HashBucket* bucket_ptr_; +}; + +// Iterator all hash entries in a hash table slot +class HashSlotIterator { + public: + HashSlotIterator(HashTable* hash_table /* should be non null */, + uint64_t slot_idx) + : hash_table_(hash_table), + start_bucket_(hash_table_->num_buckets_per_slot_ * slot_idx), + end_bucket_(start_bucket_ + hash_table_->num_buckets_per_slot_), + current_bucket_(start_bucket_), + bucket_iter_(hash_table_, current_bucket_) { + getBucket(); + } + + HashEntry& operator*() { return *bucket_iter_; } + + HashEntry* operator->() { return &operator*(); } + + HashSlotIterator& operator++() { + next(); + return *this; + } + + HashSlotIterator operator++(int) { + auto tmp = *this; + this->operator++(); + return tmp; + } + + bool Valid() { + return bucket_iter_.Valid() && current_bucket_ < end_bucket_ && + current_bucket_ >= start_bucket_; + } + + private: + // Locate to bucket with hash entries from current + void getBucket() { + if (!bucket_iter_.Valid()) { + current_bucket_++; + while (current_bucket_ < end_bucket_) { + bucket_iter_ = HashBucketIterator(hash_table_, current_bucket_); + if (bucket_iter_.Valid()) { + return; + } + current_bucket_++; + } + } + } + + void next() { + if (Valid()) { + bucket_iter_++; + if (!bucket_iter_.Valid()) { + getBucket(); + } + } + } + + HashTable* hash_table_; + uint64_t start_bucket_; + uint64_t end_bucket_; + uint64_t current_bucket_; + HashBucketIterator bucket_iter_; +}; + +// Iterate all slots in a hashtable +struct HashTableIterator { + public: + HashTableIterator(HashTable* hash_table, uint64_t start_slot_idx, + uint64_t end_slot_idx) + : hash_table_(hash_table), + current_slot_idx_(start_slot_idx), + end_slot_idx_(end_slot_idx) {} + + std::unique_lock AcquireSlotLock() { + SpinMutex* slot_lock = GetSlotLock(); + return std::unique_lock(*slot_lock); + } + + void Next() { + if (Valid()) { + current_slot_idx_++; + } + } + + bool Valid() { return current_slot_idx_ < end_slot_idx_; } + + HashSlotIterator Slot() { + return HashSlotIterator{hash_table_, current_slot_idx_}; + } + + SpinMutex* GetSlotLock() { + return &hash_table_->slots_[current_slot_idx_].spin; + } + + private: + // lock current access slot + std::unique_lock iter_lock_slot_; + // current slot id + HashTable* hash_table_; + uint64_t current_slot_idx_; + uint64_t end_slot_idx_; +}; +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/kv_engine.cpp b/volatile/engine/kv_engine.cpp new file mode 100644 index 00000000..b57c06a4 --- /dev/null +++ b/volatile/engine/kv_engine.cpp @@ -0,0 +1,1064 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "kv_engine.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "backup_log.hpp" +#include "dram_allocator.hpp" +#include "hash_collection/iterator.hpp" +#include "kvdk/volatile/engine.hpp" +#include "list_collection/iterator.hpp" +#include "sorted_collection/iterator.hpp" +#include "structures.hpp" +#include "utils/sync_point.hpp" +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { + +// Set class name of memory allocator for volatile key values. +#ifndef KVDK_VOLATILE_KV_MEMORY_ALLOCATOR_CLASS +#define KVDK_VOLATILE_KV_MEMORY_ALLOCATOR_CLASS SystemMemoryAllocator +#endif +// Enable jemalloc memory allocator for volatile key values if available. +#ifdef KVDK_WITH_JEMALLOC +#undef KVDK_VOLATILE_KV_MEMORY_ALLOCATOR_CLASS +#define KVDK_VOLATILE_KV_MEMORY_ALLOCATOR_CLASS JemallocMemoryAllocator +#endif + +// Set class name of memory allocator for Skiplist nodes. +#ifndef KVDK_SKIPLIST_NODE_MEMORY_ALLOCATOR_CLASS +#define KVDK_SKIPLIST_NODE_MEMORY_ALLOCATOR_CLASS SystemMemoryAllocator +#endif + +// Set class name of memory allocator for Hash Table's new buckets. +// HashTable's initial buckets are allocated by `global_memory_allocator` +#ifndef KVDK_HASH_TABLE_NEW_BUCKET_MEMORY_ALLOCATOR_CLASS +#define KVDK_HASH_TABLE_NEW_BUCKET_MEMORY_ALLOCATOR_CLASS SystemMemoryAllocator +#endif + +KVEngine::~KVEngine() { + GlobalLogger.Info("Closing instance ... \n"); + GlobalLogger.Info("Waiting bg threads exit ... \n"); + closing_ = true; + terminateBackgroundWorks(); + // deleteCollections(); + ReportMemoryUsage(); + + GlobalLogger.Info("Instance closed\n"); +} + +Status KVEngine::Open(const StringView engine_path, Engine** engine_ptr, + const Configs& configs) { + std::string engine_path_str(string_view_2_string(engine_path)); + GlobalLogger.Info("Opening kvdk instance from %s ...\n", + engine_path_str.c_str()); + KVEngine* engine = new KVEngine(configs); + Status s = engine->init(engine_path_str, configs); + + if (s == Status::Ok) { + *engine_ptr = engine; + engine->startBackgroundWorks(); + engine->ReportMemoryUsage(); + } else { + GlobalLogger.Error("Init kvdk instance failed: %d\n", s); + delete engine; + } + return s; +} + +void KVEngine::ReportMemoryUsage() { + // Check KV allocator is initialized before use it. + // It may not be successfully initialized due to file operation errors. + if (kv_allocator_ == nullptr) { + return; + } + + auto bytes = global_memory_allocator()->BytesAllocated(); + auto total = bytes; + GlobalLogger.Info( + "[0] Global Memory Allocator Usage: %ld B, %ld KB, %ld MB, %ld GB\n", + bytes, (bytes / (1LL << 10)), (bytes / (1LL << 20)), + (bytes / (1LL << 30))); + + bytes = kv_allocator_->BytesAllocated(); + total += bytes; + GlobalLogger.Info( + "[1] KV Memory Allocator Usage: %ld B, %ld KB, %ld MB, %ld GB\n", bytes, + (bytes / (1LL << 10)), (bytes / (1LL << 20)), (bytes / (1LL << 30))); + + bytes = skiplist_node_allocator_->BytesAllocated(); + total += bytes; + GlobalLogger.Info( + "[2] Skiplist Node Memory Allocator Usage: %ld B, %ld KB, %ld MB, %ld " + "GB\n", + bytes, (bytes / (1LL << 10)), (bytes / (1LL << 20)), + (bytes / (1LL << 30))); + + bytes = hashtable_new_bucket_allocator_->BytesAllocated(); + total += bytes; + GlobalLogger.Info( + "[3] Hashtable New Bucket Memory Allocator Usage: %ld B, %ld KB, %ld MB, " + "%ld " + "GB\n", + bytes, (bytes / (1LL << 10)), (bytes / (1LL << 20)), + (bytes / (1LL << 30))); + + GlobalLogger.Info("[+] Total Memory Usage: %ld B, %ld KB, %ld MB, %ld GB\n", + total, (total / (1LL << 10)), (total / (1LL << 20)), + (total / (1LL << 30))); +} + +void KVEngine::startBackgroundWorks() { + std::unique_lock ul(bg_work_signals_.terminating_lock); + bg_work_signals_.terminating = false; + bg_threads_.emplace_back(&KVEngine::backgroundMemoryUsageReporter, this); + + bool close_reclaimer = false; + TEST_SYNC_POINT_CALLBACK("KVEngine::backgroundCleaner::NothingToDo", + &close_reclaimer); + if (!close_reclaimer) { + cleaner_.Start(); + } +} + +void KVEngine::terminateBackgroundWorks() { + cleaner_.Close(); + { + std::unique_lock ul(bg_work_signals_.terminating_lock); + bg_work_signals_.terminating = true; + bg_work_signals_.dram_cleaner_cv.notify_all(); + bg_work_signals_.memory_usage_reporter_cv.notify_all(); + } + for (auto& t : bg_threads_) { + t.join(); + } +} + +Status KVEngine::init(const std::string& name, const Configs& configs) { + Status s = Status::Ok; + configs_ = configs; + s = checkGeneralConfigs(configs); + if (s != Status::Ok) { + return s; + } + + (void)name; // To suppress compile warnings + Allocator* kv_allocator = new KVDK_VOLATILE_KV_MEMORY_ALLOCATOR_CLASS(); + kv_allocator->SetMaxAccessThreads(configs_.max_access_threads); + kv_allocator->SetDestMemoryNodes(configs_.dest_memory_nodes); + kv_allocator_.reset(kv_allocator); + + Allocator* skiplist_node_allocator = + new KVDK_SKIPLIST_NODE_MEMORY_ALLOCATOR_CLASS(); + skiplist_node_allocator->SetMaxAccessThreads(configs_.max_access_threads); + skiplist_node_allocator_.reset(skiplist_node_allocator); + + Allocator* hashtable_new_bucket_allocator = + new KVDK_HASH_TABLE_NEW_BUCKET_MEMORY_ALLOCATOR_CLASS(); + hashtable_new_bucket_allocator->SetMaxAccessThreads( + configs_.max_access_threads); + hashtable_new_bucket_allocator_.reset(hashtable_new_bucket_allocator); + + GlobalLogger.Info("Global memory allocator: %s\n", + global_memory_allocator()->AllocatorName().c_str()); + GlobalLogger.Info("KV memory allocator: %s\n", + kv_allocator_->AllocatorName().c_str()); + GlobalLogger.Info("Skiplist node memory allocator: %s\n", + skiplist_node_allocator_->AllocatorName().c_str()); + GlobalLogger.Info("Hashtable new bucket memory allocator: %s\n", + hashtable_new_bucket_allocator->AllocatorName().c_str()); + + hash_table_.reset(HashTable::NewHashTable( + configs_.hash_bucket_num, configs_.num_buckets_per_slot, + kv_allocator_.get(), hashtable_new_bucket_allocator, + configs_.max_access_threads)); + dllist_locks_.reset(new LockTable{1UL << 20}); + + if (kv_allocator_ == nullptr || hash_table_ == nullptr || + dllist_locks_ == nullptr) { + GlobalLogger.Error("Init kvdk basic components error\n"); + return Status::Abort; + } + + registerComparator("default", compare_string_view); + return s; +} + +Status KVEngine::Backup(const pmem::obj::string_view backup_log, + const Snapshot* snapshot) { + std::string backup_log_file = string_view_2_string(backup_log); + BackupLog backup; + Status s = backup.Init(backup_log_file); + GlobalLogger.Info("Backup instance to %s ...\n", backup_log_file.c_str()); + if (s != Status::Ok) { + return s; + } + TimestampType backup_ts = + static_cast(snapshot)->GetTimestamp(); + auto hashtable_iterator = + hash_table_->GetIterator(0, hash_table_->GetSlotsNum()); + while (hashtable_iterator.Valid()) { + auto ul = hashtable_iterator.AcquireSlotLock(); + auto slot_iter = hashtable_iterator.Slot(); + while (slot_iter.Valid()) { + switch (slot_iter->GetRecordType()) { + case RecordType::String: { + StringRecord* record = slot_iter->GetIndex().string_record; + while (record != nullptr && record->GetTimestamp() > backup_ts) { + record = + kv_allocator_->offset2addr(record->old_version); + } + if (record && record->GetRecordStatus() == RecordStatus::Normal && + !record->HasExpired()) { + s = backup.Append(RecordType::String, record->Key(), + record->Value(), record->GetExpireTime()); + } + break; + } + case RecordType::SortedHeader: { + DLRecord* header = slot_iter->GetIndex().skiplist->HeaderRecord(); + while (header != nullptr && header->GetTimestamp() > backup_ts) { + header = kv_allocator_->offset2addr(header->old_version); + } + if (header && header->GetRecordStatus() == RecordStatus::Normal && + !header->HasExpired()) { + s = backup.Append(RecordType::SortedHeader, header->Key(), + header->Value(), header->GetExpireTime()); + if (s == Status::Ok) { + // Append skiplist elems following the header + auto skiplist = getSkiplist(Skiplist::FetchID(header)); + kvdk_assert(skiplist != nullptr, + "Backup skiplist should exist in map"); + auto skiplist_iter = SortedIteratorImpl( + skiplist.get(), kv_allocator_.get(), + static_cast(snapshot), false); + for (skiplist_iter.SeekToFirst(); skiplist_iter.Valid(); + skiplist_iter.Next()) { + s = backup.Append(RecordType::SortedElem, skiplist_iter.Key(), + skiplist_iter.Value(), kPersistTime); + if (s != Status::Ok) { + break; + } + } + } + } + break; + } + case RecordType::HashHeader: { + DLRecord* header = slot_iter->GetIndex().hlist->HeaderRecord(); + while (header != nullptr && header->GetTimestamp() > backup_ts) { + header = kv_allocator_->offset2addr(header->old_version); + } + if (header && header->GetRecordStatus() == RecordStatus::Normal && + !header->HasExpired()) { + s = backup.Append(RecordType::HashHeader, header->Key(), + header->Value(), header->GetExpireTime()); + if (s == Status::Ok) { + // Append hlist elems following the header + auto hlist = getHashlist(HashList::FetchID(header)); + kvdk_assert(hlist != nullptr, "Backup hlist should exist in map"); + auto hlist_iter = HashIteratorImpl( + hlist.get(), static_cast(snapshot), + false); + for (hlist_iter.SeekToFirst(); hlist_iter.Valid(); + hlist_iter.Next()) { + s = backup.Append(RecordType::HashElem, hlist_iter.Key(), + hlist_iter.Value(), kPersistTime); + if (s != Status::Ok) { + break; + } + } + } + } + break; + } + case RecordType::ListHeader: { + DLRecord* header = slot_iter->GetIndex().list->HeaderRecord(); + while (header != nullptr && header->GetTimestamp() > backup_ts) { + header = kv_allocator_->offset2addr(header->old_version); + } + if (header && header->GetRecordStatus() == RecordStatus::Normal && + !header->HasExpired()) { + s = backup.Append(RecordType::ListHeader, header->Key(), + header->Value(), header->GetExpireTime()); + if (s == Status::Ok) { + // Append hlist elems following the header + auto list = getList(List::FetchID(header)); + kvdk_assert(list != nullptr, "Backup list should exist in map"); + auto list_iter = ListIteratorImpl( + list.get(), static_cast(snapshot), + false); + for (list_iter.SeekToFirst(); list_iter.Valid(); + list_iter.Next()) { + s = backup.Append(RecordType::ListElem, "", list_iter.Value(), + kPersistTime); + if (s != Status::Ok) { + break; + } + } + } + } + break; + } + default: + // Hash and list backup is not supported yet + break; + } + if (s != Status::Ok) { + backup.Destroy(); + return s; + } + slot_iter++; + } + hashtable_iterator.Next(); + } + backup.Finish(); + GlobalLogger.Info("Backup instance to %s Finished\n", + backup_log_file.c_str()); + return Status::Ok; +} + +Status KVEngine::Restore(const StringView engine_path, + const StringView backup_log, Engine** engine_ptr, + const Configs& configs) { + std::string engine_path_str(string_view_2_string(engine_path)); + std::string backup_log_str(string_view_2_string(backup_log)); + GlobalLogger.Info( + "Restoring kvdk instance from backup log %s to engine path %s\n", + backup_log_str.c_str(), engine_path_str.c_str()); + KVEngine* engine = new KVEngine(configs); + Status s = engine->init(engine_path_str, configs); + if (s == Status::Ok) { + s = engine->restoreDataFromBackup(backup_log_str); + } + + if (s == Status::Ok) { + *engine_ptr = engine; + engine->startBackgroundWorks(); + engine->ReportMemoryUsage(); + } else { + GlobalLogger.Error("Restore kvdk instance from backup log %s failed: %d\n", + backup_log_str.c_str(), s); + delete engine; + } + return s; +} + +Status KVEngine::restoreDataFromBackup(const std::string& backup_log) { + // TODO: make this multi-thread + BackupLog backup; + Status s = backup.Open(backup_log); + if (s != Status::Ok) { + return s; + } + auto wo = WriteOptions(); + auto iter = backup.GetIterator(); + if (iter == nullptr) { + GlobalLogger.Error("Restore from a not finished backup log %s\n", + backup_log.c_str()); + return Status::Abort; + } + uint64_t cnt = 0; + GlobalLogger.Info("Start iterating backup log\n"); + while (iter->Valid()) { + auto record = iter->Record(); + bool expired = TimeUtils::CheckIsExpired(record.expire_time); + wo.ttl_time = record.expire_time; + switch (record.type) { + case RecordType::String: { + if (!expired) { + cnt++; + s = Put(record.key, record.val, wo); + } + iter->Next(); + break; + } + case RecordType::SortedHeader: { + // Maybe reuse id? + std::shared_ptr skiplist = nullptr; + if (!expired) { + CollectionIDType id; + SortedCollectionConfigs s_configs; + s = Skiplist::DecodeSortedCollectionValue(record.val, id, s_configs); + if (s != Status::Ok) { + break; + } + s = buildSkiplist(record.key, s_configs, skiplist); + if (s == Status::Ok && wo.ttl_time != kPersistTime) { + skiplist->SetExpireTime(wo.ttl_time, + version_controller_.GetCurrentTimestamp()); + } + if (s != Status::Ok) { + break; + } + cnt++; + } + iter->Next(); + // the header is followed by all its elems in backup log + while (iter->Valid()) { + record = iter->Record(); + if (record.type != RecordType::SortedElem) { + break; + } + if (!expired) { + auto ret = skiplist->Put(record.key, record.val, + version_controller_.GetCurrentTimestamp()); + s = ret.s; + if (s != Status::Ok) { + break; + } + cnt++; + } + iter->Next(); + } + break; + } + case RecordType::HashHeader: { + std::shared_ptr hlist = nullptr; + if (!expired) { + s = buildHashlist(record.key, hlist); + if (s == Status::Ok && wo.ttl_time != kPersistTime) { + hlist->SetExpireTime(wo.ttl_time, + version_controller_.GetCurrentTimestamp()); + } + if (s != Status::Ok) { + break; + } + cnt++; + } + iter->Next(); + // the header is followed by all its elems in backup log + while (iter->Valid()) { + record = iter->Record(); + if (record.type != RecordType::HashElem) { + break; + } + if (!expired) { + auto ret = hlist->Put(record.key, record.val, + version_controller_.GetCurrentTimestamp()); + s = ret.s; + if (s != Status::Ok) { + break; + } + cnt++; + } + iter->Next(); + } + break; + } + case RecordType::ListHeader: { + std::shared_ptr list = nullptr; + if (!expired) { + s = buildList(record.key, list); + if (s == Status::Ok && wo.ttl_time != kPersistTime) { + list->SetExpireTime(wo.ttl_time, + version_controller_.GetCurrentTimestamp()); + } + if (s != Status::Ok) { + break; + } + cnt++; + } + iter->Next(); + // the header is followed by all its elems in backup log + while (iter->Valid()) { + record = iter->Record(); + if (record.type != RecordType::ListElem) { + break; + } + if (!expired) { + auto ret = list->PushBack( + record.val, version_controller_.GetCurrentTimestamp()); + s = ret.s; + if (s != Status::Ok) { + break; + } + cnt++; + } + iter->Next(); + } + break; + } + case RecordType::SortedElem: { + GlobalLogger.Error("sorted elems not lead by header in backup log %s\n", + backup_log.c_str()); + return Status::Abort; + } + case RecordType::HashElem: { + GlobalLogger.Error("hash elems not lead by header in backup log %s\n", + backup_log.c_str()); + return Status::Abort; + } + case RecordType::ListElem: { + GlobalLogger.Error("list elems not lead by header in backup log %s\n", + backup_log.c_str()); + return Status::Abort; + } + default: + GlobalLogger.Error("unsupported record type %u in backup log %s\n", + record.type, backup_log.c_str()); + return Status::Abort; + } + if (s != Status::Ok) { + return Status::Abort; + } + } + GlobalLogger.Info("Restore %lu records from backup log\n", cnt); + return Status::Ok; +} + +Status KVEngine::checkGeneralConfigs(const Configs& configs) { + auto is_2pown = [](uint64_t n) { return (n > 0) && (n & (n - 1)) == 0; }; + + if (!is_2pown(configs.hash_bucket_num) || + !is_2pown(configs.num_buckets_per_slot)) { + GlobalLogger.Error( + "hash_bucket_num and num_buckets_per_slot should be 2^n\n"); + return Status::InvalidConfiguration; + } + + if (configs.hash_bucket_num >= ((uint64_t)1 << 32)) { + GlobalLogger.Error("too many hash buckets\n"); + return Status::InvalidConfiguration; + } + + if (configs.num_buckets_per_slot > configs.hash_bucket_num) { + GlobalLogger.Error( + "num_buckets_per_slot should less than hash_bucket_num\n"); + return Status::InvalidConfiguration; + } + + return Status::Ok; +} + +Status KVEngine::BatchWrite(std::unique_ptr const& batch) { + WriteBatchImpl const* batch_impl = + dynamic_cast(batch.get()); + if (batch_impl == nullptr) { + return Status::InvalidArgument; + } + + return batchWriteImpl(*batch_impl); +} + +Status KVEngine::maybeInitBatchLogFile() { + // TODO Implement batch write without pmem + return Status::Ok; +} + +Status KVEngine::batchWriteImpl(WriteBatchImpl const& batch) { + if (batch.Size() > BatchWriteLog::Capacity()) { + return Status::InvalidBatchSize; + } + + auto thread_holder = AcquireAccessThread(); + + Status s = maybeInitBatchLogFile(); + if (s != Status::Ok) { + return s; + } + + // Prevent collection and nodes in double linked lists from being deleted + auto access_token = version_controller_.GetLocalSnapshotHolder(); + + std::vector string_args; + string_args.reserve(batch.StringOps().size()); + std::vector sorted_args; + sorted_args.reserve(batch.SortedOps().size()); + std::vector hash_args; + hash_args.reserve(batch.HashOps().size()); + + for (auto const& string_op : batch.StringOps()) { + string_args.emplace_back(); + string_args.back().Assign(string_op); + } + + // Lookup Skiplists and Hashes for further operations + for (auto const& sorted_op : batch.SortedOps()) { + auto res = lookupKey(sorted_op.collection, RecordType::SortedHeader); + /// TODO: this is a temporary work-around + /// We cannot lock both key and field, which may trigger deadlock. + /// However, if a collection is created and a field is inserted, + /// Delete operation in batch may skip this field, + /// which causes inconsistency. + if (res.s == Status::Outdated) { + return Status::NotFound; + } + if (res.s != Status::Ok) { + return res.s; + } + Skiplist* skiplist = res.entry.GetIndex().skiplist; + sorted_args.emplace_back( + skiplist->InitWriteArgs(sorted_op.key, sorted_op.value, sorted_op.op)); + } + + for (auto const& hash_op : batch.HashOps()) { + HashList* hlist; + Status s = hashListFind(hash_op.collection, &hlist); + if (s != Status::Ok) { + return s; + } + hash_args.emplace_back( + hlist->InitWriteArgs(hash_op.key, hash_op.value, hash_op.op)); + } + + // Keys/internal keys to be locked on HashTable + std::vector keys_to_lock; + for (auto const& string_op : batch.StringOps()) { + keys_to_lock.push_back(string_op.key); + } + for (auto const& arg : sorted_args) { + keys_to_lock.push_back(arg.skiplist->InternalKey(arg.key)); + } + for (auto const& arg : hash_args) { + keys_to_lock.push_back(arg.hlist->InternalKey(arg.key)); + } + + auto guard = hash_table_->RangeLock(keys_to_lock); + keys_to_lock.clear(); + + // Lookup keys, allocate space according to result. + auto ReleaseResources = [&]() { + // Don't Free() if we simulate a crash. +#ifndef KVDK_ENABLE_CRASHPOINT + for (auto iter = hash_args.rbegin(); iter != hash_args.rend(); ++iter) { + kv_allocator_->Free(iter->space); + if (iter->lookup_result.entry_ptr->Allocated()) { + kvdk_assert(iter->lookup_result.s == Status::NotFound, ""); + iter->lookup_result.entry_ptr->Clear(); + } + } + for (auto iter = sorted_args.rbegin(); iter != sorted_args.rend(); ++iter) { + kv_allocator_->Free(iter->space); + if (iter->lookup_result.entry_ptr->Allocated()) { + kvdk_assert(iter->lookup_result.s == Status::NotFound, ""); + iter->lookup_result.entry_ptr->Clear(); + } + } + for (auto iter = string_args.rbegin(); iter != string_args.rend(); ++iter) { + kv_allocator_->Free(iter->space); + if (iter->res.entry_ptr->Allocated()) { + kvdk_assert(iter->res.s == Status::NotFound, ""); + iter->res.entry_ptr->Clear(); + } + } +#endif + }; + + defer(ReleaseResources()); + + // Prevent generating snapshot newer than this WriteBatch + auto bw_token = version_controller_.GetBatchWriteToken(); + + // Prepare for Strings + for (auto& args : string_args) { + Status s = stringWritePrepare(args, bw_token.Timestamp()); + if (s != Status::Ok) { + return s; + } + } + + // Prepare for Sorted Elements + for (auto& args : sorted_args) { + Status s = sortedWritePrepare(args, bw_token.Timestamp()); + if (s != Status::Ok) { + return s; + } + } + + // Prepare for Hash Elements + for (auto& args : hash_args) { + Status s = hashWritePrepare(args, bw_token.Timestamp()); + if (s != Status::Ok) { + return s; + } + } + + // Preparation done. Persist BatchLog for rollback. + BatchWriteLog log; + log.SetTimestamp(bw_token.Timestamp()); + auto& tc = engine_thread_cache_[ThreadManager::ThreadID() % + configs_.max_access_threads]; + for (auto& args : string_args) { + if (args.space.size == 0) { + continue; + } + if (args.op == WriteOp::Put) { + log.StringPut(args.space.offset); + } else { + log.StringDelete(args.space.offset); + } + } + for (auto& args : sorted_args) { + if (args.space.size == 0) { + continue; + } + if (args.op == WriteOp::Put) { + log.SortedPut(args.space.offset); + } else { + log.SortedDelete(args.space.offset); + } + } + + for (auto& args : hash_args) { + if (args.space.size == 0) { + continue; + } + if (args.op == WriteOp::Put) { + log.HashPut(args.space.offset); + } else { + log.HashDelete(args.space.offset); + } + } + + log.EncodeTo(tc.batch_log); + + BatchWriteLog::MarkProcessing(tc.batch_log); + + // After preparation stage, no runtime error is allowed for now, + // otherwise we have to perform runtime rollback. + + // Write Strings + for (auto& args : string_args) { + if (args.space.size == 0) { + continue; + } + Status s = stringWrite(args); + kvdk_assert(s == Status::Ok, ""); + } + + // Write Sorted Elems + for (auto& args : sorted_args) { + if (args.space.size == 0) { + continue; + } + Status s = sortedWrite(args); + kvdk_assert(s == Status::Ok, ""); + } + + // Write Hash Elems + for (auto& args : hash_args) { + if (args.space.size == 0) { + continue; + } + Status s = hashListWrite(args); + kvdk_assert(s == Status::Ok, ""); + } + + TEST_CRASH_POINT("KVEngine::batchWriteImpl::BeforeCommit", ""); + + BatchWriteLog::MarkCommitted(tc.batch_log); + + // Publish stages is where Strings and Collections make BatchWrite + // visible to other threads. + // This stage allows no failure during runtime, + // otherwise dirty read may occur. + // Crash is tolerated as BatchWrite will be recovered. + + // Publish Strings to HashTable + for (auto const& args : string_args) { + if (args.space.size == 0) { + continue; + } + Status s = stringWritePublish(args); + kvdk_assert(s == Status::Ok, ""); + } + + // Publish Sorted Elements to HashTable + for (auto const& args : sorted_args) { + if (args.space.size == 0) { + continue; + } + Status s = sortedWritePublish(args); + kvdk_assert(s == Status::Ok, ""); + } + + // Publish Hash Elements to HashTable + for (auto& args : hash_args) { + if (args.space.size == 0) { + continue; + } + Status s = hashListPublish(args); + kvdk_assert(s == Status::Ok, ""); + } + + hash_args.clear(); + sorted_args.clear(); + string_args.clear(); + + return Status::Ok; +} + +Status KVEngine::GetTTL(const StringView key, TTLType* ttl_time) { + *ttl_time = kInvalidTTL; + auto ul = hash_table_->AcquireLock(key); + auto res = lookupKey(key, ExpirableRecordType); + + if (res.s == Status::Ok) { + ExpireTimeType expire_time; + switch (res.entry_ptr->GetIndexType()) { + case PointerType::Skiplist: { + expire_time = res.entry_ptr->GetIndex().skiplist->GetExpireTime(); + break; + } + case PointerType::List: { + expire_time = res.entry_ptr->GetIndex().list->GetExpireTime(); + break; + } + case PointerType::HashList: { + expire_time = res.entry_ptr->GetIndex().hlist->GetExpireTime(); + break; + } + case PointerType::StringRecord: { + expire_time = res.entry_ptr->GetIndex().string_record->GetExpireTime(); + break; + } + default: { + return Status::NotSupported; + } + } + // return ttl time + *ttl_time = TimeUtils::ExpireTimeToTTL(expire_time); + } + return res.s == Status::Outdated ? Status::NotFound : res.s; +} + +Status KVEngine::TypeOf(StringView key, ValueType* type) { + auto res = lookupKey(key, ExpirableRecordType); + + if (res.s == Status::Ok) { + switch (res.entry_ptr->GetIndexType()) { + case PointerType::Skiplist: { + *type = ValueType::SortedCollection; + break; + } + case PointerType::List: { + *type = ValueType::List; + break; + } + case PointerType::HashList: { + *type = ValueType::HashCollection; + break; + } + case PointerType::StringRecord: { + *type = ValueType::String; + break; + } + default: { + return Status::Abort; + } + } + } + return res.s == Status::Outdated ? Status::NotFound : res.s; +} + +Status KVEngine::Expire(const StringView key, TTLType ttl_time) { + auto thread_holder = AcquireAccessThread(); + + int64_t base_time = TimeUtils::millisecond_time(); + if (!TimeUtils::CheckTTL(ttl_time, base_time)) { + return Status::InvalidArgument; + } + + ExpireTimeType expired_time = TimeUtils::TTLToExpireTime(ttl_time, base_time); + auto ul = hash_table_->AcquireLock(key); + auto snapshot_holder = version_controller_.GetLocalSnapshotHolder(); + // TODO: maybe have a wrapper function(lookupKeyAndMayClean). + auto lookup_result = lookupKey(key, ExpirableRecordType); + if (lookup_result.s == Status::Outdated) { + return Status::NotFound; + } + + if (lookup_result.s == Status::Ok) { + WriteOptions write_option{ttl_time}; + switch (lookup_result.entry_ptr->GetIndexType()) { + case PointerType::StringRecord: { + ul.unlock(); + version_controller_.ReleaseLocalSnapshot(); + lookup_result.s = Modify( + key, + [](const std::string* old_val, std::string* new_val, void*) { + new_val->assign(*old_val); + return ModifyOperation::Write; + }, + nullptr, write_option); + break; + } + case PointerType::Skiplist: { + auto new_ts = snapshot_holder.Timestamp(); + Skiplist* skiplist = lookup_result.entry_ptr->GetIndex().skiplist; + std::unique_lock skiplist_lock(skiplists_mu_); + expirable_skiplists_.erase(skiplist); + auto ret = skiplist->SetExpireTime(expired_time, new_ts); + expirable_skiplists_.emplace(skiplist); + lookup_result.s = ret.s; + break; + } + case PointerType::HashList: { + auto new_ts = snapshot_holder.Timestamp(); + HashList* hlist = lookup_result.entry_ptr->GetIndex().hlist; + std::unique_lock hlist_lock(hlists_mu_); + expirable_hlists_.erase(hlist); + lookup_result.s = hlist->SetExpireTime(expired_time, new_ts).s; + expirable_hlists_.emplace(hlist); + break; + } + case PointerType::List: { + auto new_ts = snapshot_holder.Timestamp(); + List* list = lookup_result.entry_ptr->GetIndex().list; + lookup_result.s = list->SetExpireTime(expired_time, new_ts).s; + break; + } + default: { + return Status::NotSupported; + } + } + } + return lookup_result.s; +} +} // namespace KVDK_NAMESPACE + +// lookupKey +namespace KVDK_NAMESPACE { +template +HashTable::LookupResult KVEngine::lookupElem(StringView key, + uint8_t type_mask) { + kvdk_assert(type_mask & (RecordType::HashElem | RecordType::SortedElem), ""); + return hash_table_->Lookup(key, type_mask); +} + +template HashTable::LookupResult KVEngine::lookupElem(StringView, + uint8_t); +template HashTable::LookupResult KVEngine::lookupElem(StringView, + uint8_t); + +template +HashTable::LookupResult KVEngine::lookupKey(StringView key, uint8_t type_mask) { + auto result = hash_table_->Lookup(key, PrimaryRecordType); + + if (result.s != Status::Ok) { + kvdk_assert( + result.s == Status::NotFound || result.s == Status::MemoryOverflow, ""); + return result; + } + + RecordType type = result.entry.GetRecordType(); + RecordStatus record_status = result.entry.GetRecordStatus(); + bool type_match = type_mask & type; + + // TODO: fix mvcc of different type keys + if (!type_match) { + result.s = Status::WrongType; + } else if (record_status == RecordStatus::Outdated) { + result.s = Status::Outdated; + } else { + switch (type) { + case RecordType::String: { + result.s = result.entry.GetIndex().string_record->HasExpired() + ? Status::Outdated + : Status::Ok; + break; + } + case RecordType::SortedHeader: + result.s = result.entry.GetIndex().skiplist->HasExpired() + ? Status::Outdated + : Status::Ok; + + break; + case RecordType::ListHeader: + result.s = result.entry.GetIndex().list->HasExpired() ? Status::Outdated + : Status::Ok; + break; + case RecordType::HashHeader: { + result.s = result.entry.GetIndex().hlist->HasExpired() + ? Status::Outdated + : Status::Ok; + break; + } + default: { + kvdk_assert(false, "Unreachable branch!"); + std::abort(); + } + } + } + return result; +} + +template HashTable::LookupResult KVEngine::lookupKey(StringView, uint8_t); +template HashTable::LookupResult KVEngine::lookupKey(StringView, + uint8_t); + +template +T* KVEngine::removeOutDatedVersion(T* record, TimestampType min_snapshot_ts) { + static_assert( + std::is_same::value || std::is_same::value, + "Invalid record type, should be StringRecord or DLRecord."); + T* ret = nullptr; + auto old_record = record; + while (old_record && old_record->GetTimestamp() > min_snapshot_ts) { + old_record = + static_cast(kv_allocator_->offset2addr(old_record->old_version)); + } + + // the snapshot should access the old record, so we need to purge and free the + // older version of the old record + if (old_record && old_record->old_version != kNullMemoryOffset) { + T* remove_record = + kv_allocator_->offset2addr_checked(old_record->old_version); + ret = remove_record; + old_record->SetOldVersion(kNullMemoryOffset); + while (remove_record != nullptr) { + if (remove_record->GetRecordStatus() == RecordStatus::Normal) { + remove_record->SetStatus(RecordStatus::Dirty); + } + remove_record = kv_allocator_->offset2addr(remove_record->old_version); + } + } + return ret; +} + +template StringRecord* KVEngine::removeOutDatedVersion( + StringRecord*, TimestampType); +template DLRecord* KVEngine::removeOutDatedVersion(DLRecord*, + TimestampType); + +} // namespace KVDK_NAMESPACE + +// Snapshot, delayFree and background work +namespace KVDK_NAMESPACE { + +/// TODO: move this into VersionController. +Snapshot* KVEngine::GetSnapshot(bool) { + Snapshot* ret = version_controller_.NewGlobalSnapshot(); + return ret; +} + +void KVEngine::backgroundMemoryUsageReporter() { + auto interval = std::chrono::milliseconds{ + static_cast(configs_.report_memory_usage_interval * 1000)}; + while (!bg_work_signals_.terminating) { + { + std::unique_lock ul(bg_work_signals_.terminating_lock); + if (!bg_work_signals_.terminating) { + bg_work_signals_.memory_usage_reporter_cv.wait_for(ul, interval); + } + } + ReportMemoryUsage(); + GlobalLogger.Info("Cleaner Thread Num: %ld\n", cleaner_.ActiveThreadNum()); + } +} + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/kv_engine.hpp b/volatile/engine/kv_engine.hpp new file mode 100644 index 00000000..0fd7008e --- /dev/null +++ b/volatile/engine/kv_engine.hpp @@ -0,0 +1,609 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "alias.hpp" +#include "data_record.hpp" +#include "dram_allocator.hpp" +#include "hash_collection/hash_list.hpp" +#include "hash_table.hpp" +#include "kvdk/volatile/engine.hpp" +#include "list_collection/list.hpp" +#include "lock_table.hpp" +#include "logger.hpp" +#include "sorted_collection/skiplist.hpp" +#include "structures.hpp" +#include "thread_manager.hpp" +#include "utils/utils.hpp" +#include "version/old_records_cleaner.hpp" +#include "version/version_controller.hpp" +#include "write_batch_impl.hpp" + +namespace KVDK_NAMESPACE { +class KVEngine : public Engine { + public: + ~KVEngine(); + + static Status Open(const StringView engine_path, Engine** engine_ptr, + const Configs& configs); + + static Status Restore(const StringView engine_path, + const StringView backup_log, Engine** engine_ptr, + const Configs& configs); + + Snapshot* GetSnapshot(bool make_checkpoint) override; + + Status Backup(const pmem::obj::string_view backup_log, + const Snapshot* snapshot) override; + + void ReleaseSnapshot(const Snapshot* snapshot) override { + version_controller_.ReleaseSnapshot( + static_cast(snapshot)); + } + void ReportMemoryUsage(); + + // Expire str after ttl_time + // + // Notice: + // 1. Expire assumes that str is not duplicated among all types, which is not + // implemented yet + // 2. Expire is not compatible with checkpoint for now + Status Expire(const StringView key, TTLType ttl_time) final; + + // Get time to expire of str + // + // Notice: + // Expire assumes that str is not duplicated among all types, which is not + // implemented yet + Status GetTTL(const StringView key, TTLType* ttl_time) final; + + Status TypeOf(StringView key, ValueType* type) final; + + // String + Status Get(const StringView key, std::string* value) override; + Status Put(const StringView key, const StringView value, + const WriteOptions& write_options) override; + Status Delete(const StringView key) override; + Status Modify(const StringView key, ModifyFunc modify_func, void* modify_args, + const WriteOptions& options) override; + + // Sorted + Status SortedCreate(const StringView collection_name, + const SortedCollectionConfigs& configs) override; + Status SortedDestroy(const StringView collection_name) override; + Status SortedSize(const StringView collection, size_t* size) override; + Status SortedGet(const StringView collection, const StringView user_key, + std::string* value) override; + Status SortedPut(const StringView collection, const StringView user_key, + const StringView value) override; + Status SortedDelete(const StringView collection, + const StringView user_key) override; + SortedIterator* SortedIteratorCreate(const StringView collection, + Snapshot* snapshot, Status* s) override; + void SortedIteratorRelease(SortedIterator* sorted_iterator) override; + + // List + Status ListCreate(StringView key) final; + Status ListDestroy(StringView key) final; + Status ListSize(StringView key, size_t* sz) final; + Status ListPushFront(StringView key, StringView elem) final; + Status ListPushBack(StringView key, StringView elem) final; + Status ListPopFront(StringView key, std::string* elem) final; + Status ListPopBack(StringView key, std::string* elem) final; + Status ListBatchPushFront(StringView key, + std::vector const& elems) final; + Status ListBatchPushFront(StringView key, + std::vector const& elems) final; + Status ListBatchPushBack(StringView key, + std::vector const& elems) final; + Status ListBatchPushBack(StringView key, + std::vector const& elems) final; + Status ListBatchPopFront(StringView key, size_t n, + std::vector* elems) final; + Status ListBatchPopBack(StringView key, size_t n, + std::vector* elems) final; + Status ListMove(StringView src, ListPos src_pos, StringView dst, + ListPos dst_pos, std::string* elem) final; + Status ListInsertAt(StringView collection, StringView key, long index) final; + Status ListInsertBefore(StringView collection, StringView key, + StringView pos) final; + Status ListInsertAfter(StringView collection, StringView key, + StringView pos) final; + Status ListErase(StringView collection, long index, std::string* elem) final; + + Status ListReplace(StringView list_name, long index, StringView elem) final; + ListIterator* ListIteratorCreate(StringView collection, Snapshot* snapshot, + Status* status) final; + void ListIteratorRelease(ListIterator* iter) final; + + // Hash + Status HashCreate(StringView key) final; + Status HashDestroy(StringView key) final; + Status HashSize(StringView key, size_t* len) final; + Status HashGet(StringView key, StringView field, std::string* value) final; + Status HashPut(StringView key, StringView field, StringView value) final; + Status HashDelete(StringView key, StringView field) final; + Status HashModify(StringView key, StringView field, ModifyFunc modify_func, + void* cb_args) final; + HashIterator* HashIteratorCreate(StringView key, Snapshot* snapshot, + Status* s) final; + void HashIteratorRelease(HashIterator*) final; + + // BatchWrite + // It takes 3 stages + // Stage 1: Preparation + // BatchWrite() sort the keys and remove duplicants, + // lock the keys/fields in HashTable, + // and allocate spaces and persist BatchWriteLog + // Stage 2: Execution + // Batches are dispatched to different data types + // Each data type update keys/fields + // Outdated records are not purged in this stage. + // Stage 3: Publish + // Each data type commits its batch, clean up outdated data. + Status BatchWrite(std::unique_ptr const& batch) final; + std::unique_ptr WriteBatchCreate() final { + return std::unique_ptr{new WriteBatchImpl{}}; + } + + // For test cases + const std::unordered_map>& + GetSkiplists() { + return skiplists_; + }; + Cleaner* EngineCleaner() { return &cleaner_; } + HashTable* GetHashTable() { return hash_table_.get(); } + void TestCleanOutDated(size_t start_slot_idx, size_t end_slot_idx); + + private: + friend OldRecordsCleaner; + friend Cleaner; + + KVEngine(const Configs& configs) + : access_thread_cv_(configs.max_access_threads), + engine_thread_cache_(configs.max_access_threads), + cleaner_thread_cache_(configs.max_access_threads), + version_controller_(configs.max_access_threads), + old_records_cleaner_(this, configs.max_access_threads), + cleaner_(this, configs.clean_threads), + comparators_(configs.comparator){}; + + struct EngineThreadCache { + EngineThreadCache() = default; + + char* batch_log = nullptr; + + // Info used in recovery + uint64_t newest_restored_ts = 0; + std::unordered_map visited_skiplist_ids{}; + }; + + struct CleanerThreadCache { + template + struct OutdatedRecord { + OutdatedRecord(T* _record, TimestampType _release_time) + : record(_record), release_time(_release_time) {} + + T* record; + TimestampType release_time; + }; + + CleanerThreadCache() = default; + std::deque> outdated_string_records; + std::deque> outdated_dl_records; + SpinMutex mtx; + }; + + struct AccessThreadCV { + public: + struct Holder { + public: + Holder(AccessThreadCV* cv) : cv_(cv) { cv->Acquire(); } + ~Holder() { cv_->Release(); } + + private: + AccessThreadCV* cv_; + }; + + private: + void Acquire() { + std::unique_lock ul(spin_); + while (holder_id_ != ThreadManager::ThreadID() && holder_id_ != -1) { + cv_.wait(ul); + } + holder_id_ = ThreadManager::ThreadID(); + } + + void Release() { + std::unique_lock ul(spin_); + holder_id_ = -1; + cv_.notify_one(); + } + + int64_t holder_id_ = -1; + SpinMutex spin_; + std::condition_variable_any cv_; + }; + + AccessThreadCV::Holder AcquireAccessThread() { + return AccessThreadCV::Holder(&access_thread_cv_[ThreadManager::ThreadID() % + access_thread_cv_.size()]); + } + + bool checkKeySize(const StringView& key) { return key.size() <= UINT16_MAX; } + + bool checkValueSize(const StringView& value) { + return value.size() <= UINT32_MAX; + } + + // Init basic components of the engine + Status init(const std::string& name, const Configs& configs); + + Status hashGetImpl(const StringView& key, std::string* value, + uint16_t type_mask); + + bool registerComparator(const StringView& collection_name, + Comparator comp_func) { + return comparators_.RegisterComparator(collection_name, comp_func); + } + + // Look up a first level key in hash table(e.g. collections or string, not + // collection elems), the first level key should be unique among all types + // + // Store a copy of hash entry in LookupResult::entry, and a pointer to the + // hash entry in LookupResult::entry_ptr + // If may_insert is true and key not found, then store + // pointer of a free-to-write hash entry in LookupResult::entry_ptr. + // + // return status: + // Status::Ok if key exist and alive + // Status::NotFound is key is not found. + // Status::WrongType if type_mask does not match. + // Status::Outdated if key has been expired or deleted + // Status::MemoryOverflow if may_insert is true but failed to allocate new + // hash entry + // + // Notice: key should be locked if set may_insert to true + template + HashTable::LookupResult lookupKey(StringView key, uint8_t type_mask); + + // Look up a collection element in hash table + // + // Store a copy of hash entry in LookupResult::entry, and a pointer to the + // hash entry in LookupResult::entry_ptr + // If may_insert is true and key not found, then store + // pointer of a free-to-write hash entry in LookupResult::entry_ptr. + // + // return status: + // Status::Ok if key exist + // Status::NotFound is key is not found. + // Status::MemoryOverflow if may_insert is true but failed to allocate new + // hash entry + // + // Notice: elem should be locked if set may_insert to true + template + HashTable::LookupResult lookupElem(StringView key, uint8_t type_mask); + + // Remove a key or elem from hash table, ret should be return of + // lookupKey/lookupElem + void removeKeyOrElem(HashTable::LookupResult ret) { + kvdk_assert(ret.s == Status::Ok || ret.s == Status::Outdated, ""); + hash_table_->Erase(ret.entry_ptr); + } + + // insert/update key or elem to hashtable, ret must be return value of + // lookupElem or lookupKey + void insertKeyOrElem(HashTable::LookupResult ret, RecordType type, + RecordStatus status, void* addr) { + hash_table_->Insert(ret, type, status, addr, pointerType(type)); + } + + template + static constexpr RecordType collectionType() { + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value, + "Invalid type!"); + return std::is_same::value + ? RecordType::SortedHeader + : std::is_same::value + ? RecordType::ListHeader + : std::is_same::value + ? RecordType::HashHeader + : RecordType::Empty; + } + + static PointerType pointerType(RecordType rtype) { + switch (rtype) { + case RecordType::Empty: { + return PointerType::Empty; + } + case RecordType::String: { + return PointerType::StringRecord; + } + case RecordType::SortedElem: { + kvdk_assert(false, "Not supported!"); + return PointerType::Invalid; + } + case RecordType::SortedHeader: { + return PointerType::Skiplist; + } + case RecordType::ListHeader: { + return PointerType::List; + } + case RecordType::HashHeader: { + return PointerType::HashList; + } + case RecordType::HashElem: { + return PointerType::HashElem; + } + case RecordType::ListElem: + default: { + kvdk_assert(false, "Invalid type!"); + return PointerType::Invalid; + } + } + } + + // Lockless. It's up to caller to lock the HashTable + template + Status registerCollection(CollectionType* coll) { + auto type = collectionType(); + auto ret = lookupKey(coll->Name(), type); + if (ret.s == Status::Ok) { + kvdk_assert(false, "Collection already registered!"); + return Status::Abort; + } + if (ret.s != Status::NotFound && ret.s != Status::Outdated) { + return ret.s; + } + insertKeyOrElem(ret, type, RecordStatus::Normal, coll); + return Status::Ok; + } + + Status maybeInitBatchLogFile(); + + Status stringPutImpl(const StringView& key, const StringView& value, + const WriteOptions& write_options); + + Status stringDeleteImpl(const StringView& key); + + Status stringWritePrepare(StringWriteArgs& args, TimestampType ts); + Status stringWrite(StringWriteArgs& args); + Status stringWritePublish(StringWriteArgs const& args); + Status stringRollback(TimestampType ts, + BatchWriteLog::StringLogEntry const& entry); + + Status sortedPutImpl(Skiplist* skiplist, const StringView& collection_key, + const StringView& value); + + Status sortedDeleteImpl(Skiplist* skiplist, const StringView& user_key); + + Status restoreDataFromBackup(const std::string& backup_log); + Status sortedWritePrepare(SortedWriteArgs& args, TimestampType ts); + Status sortedWrite(SortedWriteArgs& args); + Status sortedWritePublish(SortedWriteArgs const& args); + + Status batchWriteImpl(WriteBatchImpl const& batch); + + /// List helper functions + // Find and lock the list. Initialize non-existing if required. + // Guarantees always return a valid List and lockes it if returns Status::Ok + Status listFind(StringView key, List** list); + + Status listBatchPushImpl(StringView key, ListPos pos, + std::vector const& elems); + Status listBatchPopImpl(StringView list_name, ListPos pos, size_t n, + std::vector* elems); + + /// Hash helper funtions + Status hashListFind(StringView key, HashList** hlist); + + Status hashListWrite(HashWriteArgs& args); + Status hashWritePrepare(HashWriteArgs& args, TimestampType ts); + Status hashListPublish(HashWriteArgs const& args); + + /// Other + Status checkGeneralConfigs(const Configs& configs); + + void purgeAndFreeStringRecords(const std::vector& old_offset); + + void purgeAndFreeDLRecords(const std::vector& old_offset); + + // remove outdated records which without snapshot hold. + template + T* removeOutDatedVersion(T* record, TimestampType min_snapshot_ts); + + template + void removeOutdatedCollection(T* collection); + + // find delete and old records in skiplist with no hash index + void cleanNoHashIndexedSkiplist(Skiplist* skiplist, + std::vector& purge_dl_records); + + void cleanList(List* list, std::vector& purge_dl_records); + + double cleanOutDated(PendingCleanRecords& pending_clean_records, + size_t start_slot_idx, size_t slot_block_size); + + void purgeAndFree(PendingCleanRecords& pending_clean_records); + + TimestampType getTimestamp() { + return version_controller_.GetCurrentTimestamp(); + } + + void removeSkiplist(CollectionIDType id) { + std::lock_guard lg(skiplists_mu_); + skiplists_.erase(id); + } + + void addSkiplistToMap(std::shared_ptr skiplist) { + std::lock_guard lg(skiplists_mu_); + skiplists_.emplace(skiplist->ID(), skiplist); + } + + std::shared_ptr getSkiplist(CollectionIDType id) { + std::lock_guard lg(skiplists_mu_); + return skiplists_[id]; + } + + void removeHashlist(CollectionIDType id) { + std::lock_guard lg(hlists_mu_); + hlists_.erase(id); + } + + void addHashlistToMap(std::shared_ptr hlist) { + std::lock_guard lg(hlists_mu_); + hlists_.emplace(hlist->ID(), hlist); + } + + std::shared_ptr getHashlist(CollectionIDType id) { + std::lock_guard lg(hlists_mu_); + return hlists_[id]; + } + + void removeList(CollectionIDType id) { + std::lock_guard lg(lists_mu_); + lists_.erase(id); + } + + void addListToMap(std::shared_ptr list) { + std::lock_guard lg(hlists_mu_); + lists_.emplace(list->ID(), list); + } + + std::shared_ptr getList(CollectionIDType id) { + std::lock_guard lg(lists_mu_); + return lists_[id]; + } + + Status buildSkiplist(const StringView& name, + const SortedCollectionConfigs& s_configs, + std::shared_ptr& skiplist); + + Status buildHashlist(const StringView& name, + std::shared_ptr& hlist); + + Status buildList(const StringView& name, std::shared_ptr& list); + + inline std::string data_file() { return data_file(dir_); } + + inline static std::string data_file(const std::string& instance_path) { + return format_dir_path(instance_path) + "data"; + } + + inline std::string checkpoint_file() { return checkpoint_file(dir_); } + + inline static std::string checkpoint_file(const std::string& instance_path) { + return format_dir_path(instance_path) + "checkpoint"; + } + + inline std::string config_file() { return config_file(dir_); } + + inline static std::string config_file(const std::string& instance_path) { + return format_dir_path(instance_path) + "configs"; + } + + // Run in background to report DRAM/PMem usage regularly + void backgroundMemoryUsageReporter(); + + /* functions for cleaner thread cache */ + // Remove old version records from version chain of new_record and cache it + template + void removeAndCacheOutdatedVersion(T* new_record); + // Clean a outdated record in cleaner_thread_cache_ + void tryCleanCachedOutdatedRecord(); + template + void cleanOutdatedRecordImpl(T* record); + // Cleaner thread fetches cached outdated records + void fetchCachedOutdatedVersion( + PendingCleanRecords& pending_clean_records, + std::vector& purge_string_records, + std::vector& purge_dl_records); + /* functions for cleaner thread cache */ + + void startBackgroundWorks(); + + void terminateBackgroundWorks(); + + std::unique_ptr kv_allocator_; + std::unique_ptr skiplist_node_allocator_; + std::unique_ptr hashtable_new_bucket_allocator_; + + Array access_thread_cv_; + Array engine_thread_cache_; + Array cleaner_thread_cache_; + + // restored kvs in reopen + std::atomic restored_{0}; + std::atomic collection_id_{0}; + + std::unique_ptr hash_table_; + + std::mutex skiplists_mu_; + std::unordered_map> skiplists_; + std::set expirable_skiplists_; + + std::mutex lists_mu_; + std::unordered_map> lists_; + std::set expirable_lists_; + + std::mutex hlists_mu_; + std::unordered_map> hlists_; + std::set expirable_hlists_; + + std::unique_ptr dllist_locks_; + + std::string dir_; + std::string batch_log_dir_; + std::string data_file_; + + Configs configs_; + bool closing_{false}; + std::vector bg_threads_; + + VersionController version_controller_; + OldRecordsCleaner old_records_cleaner_; + Cleaner cleaner_; + + ComparatorTable comparators_; + + struct BackgroundWorkSignals { + BackgroundWorkSignals() = default; + BackgroundWorkSignals(const BackgroundWorkSignals&) = delete; + + std::condition_variable_any memory_usage_reporter_cv; + std::condition_variable_any dram_cleaner_cv; + + SpinMutex terminating_lock; + bool terminating = false; + }; + + BackgroundWorkSignals bg_work_signals_; + + std::atomic round_robin_id_{0}; + + // We manually allocate recovery thread id for no conflict in multi-thread + // recovering + // Todo: do not hard code + std::atomic next_recovery_tid_{0}; +}; + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/kv_engine_cleaner.cpp b/volatile/engine/kv_engine_cleaner.cpp new file mode 100644 index 00000000..74416475 --- /dev/null +++ b/volatile/engine/kv_engine_cleaner.cpp @@ -0,0 +1,866 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ +#include + +#include "kv_engine.hpp" +#include "utils/sync_point.hpp" + +namespace KVDK_NAMESPACE { + +constexpr uint64_t kForegroundUpdateSnapshotInterval = 1000; + +template +void KVEngine::removeOutdatedCollection(T* collection) { + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value); + auto cur_id = collection->ID(); + auto cur_head_record = collection->HeaderRecord(); + while (cur_head_record) { + auto old_head_record = + kv_allocator_->offset2addr(cur_head_record->old_version); + if (old_head_record) { + auto old_collection_id = T::FetchID(old_head_record); + if (old_collection_id != cur_id) { + if (getSkiplist(old_collection_id) == nullptr && + getHashlist(old_collection_id) == nullptr && + getList(old_collection_id) == nullptr) { + GlobalLogger.Debug("Collection already been destroyed\n"); + } + kvdk_assert(getSkiplist(old_collection_id) != nullptr || + getHashlist(old_collection_id) != nullptr || + getList(old_collection_id) != nullptr, + "collection should not be destroyed yet!"); + cur_head_record->SetOldVersion(kNullMemoryOffset); + cur_id = old_collection_id; + } + } + cur_head_record = old_head_record; + } +} + +template +void KVEngine::removeAndCacheOutdatedVersion(T* record) { + static_assert(std::is_same::value || + std::is_same::value); + kvdk_assert(ThreadManager::ThreadID() >= 0, ""); + auto& tc = cleaner_thread_cache_[ThreadManager::ThreadID() % + configs_.max_access_threads]; + if (std::is_same::value) { + StringRecord* old_record = removeOutDatedVersion( + (StringRecord*)record, version_controller_.GlobalOldestSnapshotTs()); + if (old_record) { + std::lock_guard lg(tc.mtx); + tc.outdated_string_records.emplace_back( + old_record, version_controller_.GetCurrentTimestamp()); + } + } else { + DLRecord* old_record = removeOutDatedVersion( + (DLRecord*)record, version_controller_.GlobalOldestSnapshotTs()); + if (old_record) { + std::lock_guard lg(tc.mtx); + tc.outdated_dl_records.emplace_back( + old_record, version_controller_.GetCurrentTimestamp()); + } + } +} + +template void KVEngine::removeAndCacheOutdatedVersion( + StringRecord* record); +template void KVEngine::removeAndCacheOutdatedVersion( + DLRecord* record); + +template +void KVEngine::cleanOutdatedRecordImpl(T* old_record) { + static_assert(std::is_same::value || + std::is_same::value); + while (old_record) { + T* next = kv_allocator_->offset2addr(old_record->old_version); + auto record_size = old_record->GetRecordSize(); + if (old_record->GetRecordStatus() == RecordStatus::Normal) { + old_record->Destroy(); + } + kv_allocator_->Free(SpaceEntry( + kv_allocator_->addr2offset_checked(old_record), record_size)); + old_record = next; + } +} + +void KVEngine::tryCleanCachedOutdatedRecord() { + kvdk_assert(ThreadManager::ThreadID() >= 0, ""); + auto& tc = cleaner_thread_cache_[ThreadManager::ThreadID() % + configs_.max_access_threads]; + // Regularly update local oldest snapshot + thread_local uint64_t round = 0; + if (++round % kForegroundUpdateSnapshotInterval == 0) { + version_controller_.UpdateLocalOldestSnapshot(); + } + auto release_time = version_controller_.LocalOldestSnapshotTS(); + if (!tc.outdated_string_records.empty()) { + std::unique_lock ul(tc.mtx); + if (!tc.outdated_string_records.empty() && + tc.outdated_string_records.front().release_time < release_time) { + auto to_clean = tc.outdated_string_records.front(); + tc.outdated_string_records.pop_front(); + ul.unlock(); + cleanOutdatedRecordImpl(to_clean.record); + return; + } + } + + if (!tc.outdated_dl_records.empty()) { + std::unique_lock ul(tc.mtx); + if (!tc.outdated_dl_records.empty() && + tc.outdated_dl_records.front().release_time < release_time) { + auto to_clean = tc.outdated_dl_records.front(); + tc.outdated_dl_records.pop_front(); + ul.unlock(); + cleanOutdatedRecordImpl(to_clean.record); + } + } +} + +void KVEngine::purgeAndFreeStringRecords( + const std::vector& old_records) { + std::vector entries; + for (auto old_record : old_records) { + while (old_record) { + StringRecord* next = + kv_allocator_->offset2addr(old_record->old_version); + if (old_record->GetRecordStatus() == RecordStatus::Normal) { + old_record->Destroy(); + } + entries.emplace_back(kv_allocator_->addr2offset(old_record), + old_record->GetRecordSize()); + old_record = next; + } + } + kv_allocator_->BatchFree(entries); +} + +void KVEngine::purgeAndFreeDLRecords( + const std::vector& old_records) { + std::vector entries; + std::vector outdated_skiplists; + for (auto data_record : old_records) { + while (data_record) { + DLRecord* next_record = + kv_allocator_->offset2addr(data_record->old_version); + RecordType type = data_record->GetRecordType(); + RecordStatus record_status = data_record->GetRecordStatus(); + switch (type) { + case RecordType::HashElem: + case RecordType::SortedElem: + case RecordType::ListElem: { + entries.emplace_back(kv_allocator_->addr2offset(data_record), + data_record->GetRecordSize()); + if (record_status == RecordStatus::Normal) { + data_record->Destroy(); + } + break; + } + case RecordType::SortedHeader: { + if (record_status != RecordStatus::Outdated && + !data_record->HasExpired()) { + entries.emplace_back( + kv_allocator_->addr2offset_checked(data_record), + data_record->GetRecordSize()); + data_record->Destroy(); + } else { + auto skiplist_id = Skiplist::FetchID(data_record); + kvdk_assert(skiplists_.find(skiplist_id) != skiplists_.end(), + "Skiplist should not be removed."); + auto head_record = getSkiplist(skiplist_id)->HeaderRecord(); + if (head_record != data_record) { + entries.emplace_back( + kv_allocator_->addr2offset_checked(data_record), + data_record->GetRecordSize()); + data_record->Destroy(); + } else { + data_record->SetOldVersion(kNullMemoryOffset); + } + } + break; + } + case RecordType::HashHeader: { + if (record_status != RecordStatus::Outdated && + !data_record->HasExpired()) { + entries.emplace_back( + kv_allocator_->addr2offset_checked(data_record), + data_record->GetRecordSize()); + data_record->Destroy(); + } else { + auto hash_id = HashList::FetchID(data_record); + kvdk_assert(hlists_.find(hash_id) != hlists_.end(), + "Hashlist should not be removed."); + auto head_record = getHashlist(hash_id)->HeaderRecord(); + if (head_record != data_record) { + entries.emplace_back( + kv_allocator_->addr2offset_checked(data_record), + data_record->GetRecordSize()); + data_record->Destroy(); + } else { + data_record->SetOldVersion(kNullMemoryOffset); + } + } + break; + } + case RecordType::ListHeader: { + if (record_status != RecordStatus::Outdated && + !data_record->HasExpired()) { + entries.emplace_back( + kv_allocator_->addr2offset_checked(data_record), + data_record->GetRecordSize()); + data_record->Destroy(); + } else { + auto list_id = List::FetchID(data_record); + kvdk_assert(lists_.find(list_id) != lists_.end(), + "Hashlist should not be removed."); + auto header_record = getList(list_id)->HeaderRecord(); + if (header_record != data_record) { + entries.emplace_back( + kv_allocator_->addr2offset_checked(data_record), + data_record->GetRecordSize()); + data_record->Destroy(); + } else { + data_record->SetOldVersion(kNullMemoryOffset); + } + } + break; + } + default: + GlobalLogger.Error("Cleaner abort on record type %u\n", type); + std::abort(); + } + data_record = next_record; + } + } + kv_allocator_->BatchFree(entries); +} + +void KVEngine::cleanNoHashIndexedSkiplist( + Skiplist* skiplist, std::vector& purge_dl_records) { + auto prev_node = skiplist->HeaderNode(); + auto iter = skiplist->GetDLList()->GetRecordIterator(); + iter->SeekToFirst(); + while (iter->Valid() && !closing_) { + DLRecord* cur_record = iter->Record(); + iter->Next(); + auto min_snapshot_ts = version_controller_.GlobalOldestSnapshotTs(); + auto ul = hash_table_->AcquireLock(cur_record->Key()); + // iter old version list + auto old_record = + removeOutDatedVersion(cur_record, min_snapshot_ts); + if (old_record) { + purge_dl_records.emplace_back(old_record); + } + + // check record has dram skiplist node and update skiplist node; + SkiplistNode* dram_node = nullptr; + SkiplistNode* cur_node = prev_node->Next(1).RawPointer(); + while (cur_node) { + if (cur_node->Next(1).GetTag() == SkiplistNode::NodeStatus::Deleted) { + // cur_node already been deleted + cur_node = cur_node->Next(1).RawPointer(); + } else { + kvdk_assert(cur_node->record->GetRecordType() == RecordType::SortedElem, + ""); + if (skiplist->Compare(cur_node->UserKey(), + Skiplist::UserKey(cur_record)) < 0) { + prev_node = cur_node; + cur_node = cur_node->Next(1).RawPointer(); + } else { + break; + } + } + } + + if (cur_node && cur_node->record == cur_record) { + dram_node = cur_node; + } + + if (cur_record->GetRecordType() == RecordType::SortedElem && + cur_record->GetRecordStatus() == RecordStatus::Outdated && + cur_record->GetTimestamp() < min_snapshot_ts) { + TEST_SYNC_POINT( + "KVEngine::BackgroundCleaner::IterSkiplist::" + "UnlinkDeleteRecord"); + /* Notice: a user thread firstly update this key, its old version + * record is delete record(cur_record). So the cur_record is not in + * this skiplist, `Remove` function returns false. Nothing to do for + * this cur_record which will be purged and freed in the next + * iteration. + */ + if (Skiplist::Remove(cur_record, dram_node, kv_allocator_.get(), + dllist_locks_.get())) { + purge_dl_records.emplace_back(cur_record); + } + } + } +} + +void KVEngine::cleanList(List* list, std::vector& purge_dl_records) { + auto iter = list->GetDLList()->GetRecordIterator(); + iter->SeekToFirst(); + while (iter->Valid() && !closing_) { + DLRecord* cur_record = iter->Record(); + iter->Next(); + auto ul = list->AcquireLock(); + auto min_snapshot_ts = + std::min(version_controller_.GlobalOldestSnapshotTs(), + version_controller_.LocalOldestSnapshotTS()); + auto old_record = + removeOutDatedVersion(cur_record, min_snapshot_ts); + bool clean_cur = false; + if (cur_record->GetRecordType() == RecordType::ListElem && + cur_record->GetRecordStatus() == RecordStatus::Outdated && + cur_record->GetTimestamp() < min_snapshot_ts) { + TEST_SYNC_POINT( + "KVEngine::BackgroundCleaner::IterList::" + "UnlinkDeleteRecord"); + if (list->GetDLList()->Remove(cur_record)) { + clean_cur = true; + } + } + ul.unlock(); + if (old_record) { + purge_dl_records.emplace_back(old_record); + } + if (clean_cur) { + purge_dl_records.emplace_back(cur_record); + } + } +} + +void KVEngine::purgeAndFree(PendingCleanRecords& pending_clean_records) { + { // purge and free pending string records + while (!pending_clean_records.pending_purge_strings.empty()) { + auto& pending_strings = + pending_clean_records.pending_purge_strings.front(); + if (pending_strings.release_time < + version_controller_.LocalOldestSnapshotTS()) { + purgeAndFreeStringRecords(pending_strings.records); + pending_clean_records.pending_purge_strings.pop_front(); + } else { + break; + } + } + } + + { // Destroy skiplist + while (!pending_clean_records.outdated_skiplists.empty()) { + auto& ts_skiplist = pending_clean_records.outdated_skiplists.front(); + auto skiplist = ts_skiplist.second; + if (ts_skiplist.first < version_controller_.LocalOldestSnapshotTS() && + skiplist->TryCleaningLock()) { + skiplist->DestroyAll(); + removeSkiplist(skiplist->ID()); + skiplist->ReleaseCleaningLock(); + pending_clean_records.outdated_skiplists.pop_front(); + } else { + break; + } + } + } + + { // Destroy list + while (!pending_clean_records.outdated_lists.empty()) { + auto& ts_list = pending_clean_records.outdated_lists.front(); + auto list = ts_list.second; + if (ts_list.first < version_controller_.LocalOldestSnapshotTS() && + list->TryCleaningLock()) { + { + auto ul = list->AcquireLock(); + list->DestroyAll(); + } + removeList(list->ID()); + list->ReleaseCleaningLock(); + pending_clean_records.outdated_lists.pop_front(); + } else { + break; + } + } + } + + { // Destroy hash + while (!pending_clean_records.outdated_hlists.empty()) { + auto& ts_hlist = pending_clean_records.outdated_hlists.front(); + auto hlist = ts_hlist.second; + if (ts_hlist.first < version_controller_.LocalOldestSnapshotTS() && + hlist->TryCleaningLock()) { + hlist->DestroyAll(); + removeHashlist(hlist->ID()); + hlist->ReleaseCleaningLock(); + pending_clean_records.outdated_hlists.pop_front(); + } else { + break; + } + } + } + + { // purge and free pending old dl records + while (!pending_clean_records.pending_purge_dls.empty()) { + auto& pending_dls = pending_clean_records.pending_purge_dls.front(); + if (pending_dls.release_time < + version_controller_.LocalOldestSnapshotTS()) { + purgeAndFreeDLRecords(pending_dls.records); + pending_clean_records.pending_purge_dls.pop_front(); + } else { + break; + } + } + } +} + +void KVEngine::fetchCachedOutdatedVersion( + PendingCleanRecords& pending_clean_records, + std::vector& purge_string_records, + std::vector& purge_dl_records) { + size_t i = round_robin_id_.fetch_add(1) % configs_.max_access_threads; + auto& tc = cleaner_thread_cache_[i]; + std::deque> + outdated_string_records; + std::deque> outdated_dl_records; + if (tc.outdated_dl_records.size() > kMaxCachedOldRecords || + tc.outdated_string_records.size() > kMaxCachedOldRecords) { + std::lock_guard lg(tc.mtx); + if (tc.outdated_string_records.size() > kMaxCachedOldRecords) { + outdated_string_records.swap(tc.outdated_string_records); + } + if (tc.outdated_dl_records.size() > kMaxCachedOldRecords) { + outdated_dl_records.swap(tc.outdated_dl_records); + } + } + if (outdated_string_records.size() > 0) { + std::vector to_free_strings; + version_controller_.UpdateLocalOldestSnapshot(); + auto release_time = version_controller_.LocalOldestSnapshotTS(); + auto iter = outdated_string_records.begin(); + while (iter != outdated_string_records.end()) { + if (iter->release_time < release_time) { + to_free_strings.emplace_back(iter->record); + } else { + purge_string_records.push_back(iter->record); + } + iter++; + } + pending_clean_records.pending_purge_strings.emplace_front( + std::move(to_free_strings), release_time); + } + if (outdated_dl_records.size() > 0) { + std::vector to_free_dls; + version_controller_.UpdateLocalOldestSnapshot(); + auto release_time = version_controller_.LocalOldestSnapshotTS(); + auto iter = outdated_dl_records.begin(); + + while (iter != outdated_dl_records.end()) { + if (iter->release_time < release_time) { + to_free_dls.emplace_back(iter->record); + } else { + purge_dl_records.push_back(iter->record); + } + iter++; + } + pending_clean_records.pending_purge_dls.emplace_front( + std::move(to_free_dls), release_time); + } +} + +double KVEngine::cleanOutDated(PendingCleanRecords& pending_clean_records, + size_t start_slot_idx, size_t slot_block_size) { + size_t total_num = 0; + size_t need_purge_num = 0; + version_controller_.UpdateLocalOldestSnapshot(); + + std::vector purge_string_records; + std::vector purge_dl_records; + + fetchCachedOutdatedVersion(pending_clean_records, purge_string_records, + purge_dl_records); + double outdated_collection_ratio = cleaner_.SearchOutdatedCollections(); + cleaner_.FetchOutdatedCollections(pending_clean_records); + + // Iterate hash table + size_t end_slot_idx = start_slot_idx + slot_block_size; + if (end_slot_idx > hash_table_->GetSlotsNum()) { + end_slot_idx = hash_table_->GetSlotsNum(); + } + auto hashtable_iter = hash_table_->GetIterator(start_slot_idx, end_slot_idx); + while (hashtable_iter.Valid() && !closing_) { + { // Slot lock section + auto min_snapshot_ts = version_controller_.GlobalOldestSnapshotTs(); + auto now = TimeUtils::millisecond_time(); + + auto slot_lock(hashtable_iter.AcquireSlotLock()); + auto slot_iter = hashtable_iter.Slot(); + while (slot_iter.Valid()) { + if (!slot_iter->Empty()) { + switch (slot_iter->GetIndexType()) { + case PointerType::StringRecord: { + total_num++; + auto string_record = slot_iter->GetIndex().string_record; + auto old_record = removeOutDatedVersion( + string_record, min_snapshot_ts); + if (old_record) { + purge_string_records.emplace_back(old_record); + need_purge_num++; + } + if ((string_record->GetRecordStatus() == RecordStatus::Outdated || + string_record->GetExpireTime() <= now) && + string_record->GetTimestamp() < min_snapshot_ts) { + hash_table_->Erase(&(*slot_iter)); + purge_string_records.emplace_back(string_record); + need_purge_num++; + } + break; + } + case PointerType::SkiplistNode: { + total_num++; + auto node = slot_iter->GetIndex().skiplist_node; + auto dl_record = node->record; + auto old_record = + removeOutDatedVersion(dl_record, min_snapshot_ts); + if (old_record) { + purge_dl_records.emplace_back(old_record); + need_purge_num++; + } + if (slot_iter->GetRecordStatus() == RecordStatus::Outdated && + dl_record->GetTimestamp() < min_snapshot_ts) { + bool success = Skiplist::Remove( + dl_record, node, kv_allocator_.get(), dllist_locks_.get()); + kvdk_assert(success, ""); + hash_table_->Erase(&(*slot_iter)); + purge_dl_records.emplace_back(dl_record); + need_purge_num++; + } + break; + } + case PointerType::DLRecord: { + total_num++; + auto dl_record = slot_iter->GetIndex().dl_record; + auto old_record = + removeOutDatedVersion(dl_record, min_snapshot_ts); + if (old_record) { + purge_dl_records.emplace_back(old_record); + need_purge_num++; + } + if (slot_iter->GetRecordStatus() == RecordStatus::Outdated && + dl_record->GetTimestamp() < min_snapshot_ts) { + bool success = DLList::Remove(dl_record, kv_allocator_.get(), + dllist_locks_.get()); + kvdk_assert(success, ""); + hash_table_->Erase(&(*slot_iter)); + purge_dl_records.emplace_back(dl_record); + need_purge_num++; + } + break; + } + case PointerType::Skiplist: { + Skiplist* skiplist = slot_iter->GetIndex().skiplist; + total_num++; + if (!skiplist->IndexWithHashtable() && + slot_iter->GetRecordStatus() != RecordStatus::Outdated && + !skiplist->HasExpired()) { + need_purge_num++; + pending_clean_records.no_hash_skiplists.emplace_back(skiplist); + } + skiplist->CleanObsoletedNodes(); + break; + } + case PointerType::List: { + List* list = slot_iter->GetIndex().list; + total_num++; + if (slot_iter->GetRecordStatus() != RecordStatus::Outdated && + !list->HasExpired()) { + need_purge_num++; + pending_clean_records.valid_lists.emplace_back(list); + } + break; + } + default: + break; + } + } + slot_iter++; + } + hashtable_iter.Next(); + } // Finish a slot. + + if (!pending_clean_records.no_hash_skiplists.empty()) { + for (auto& skiplist : pending_clean_records.no_hash_skiplists) { + if (skiplist && skiplist->TryCleaningLock()) { + cleanNoHashIndexedSkiplist(skiplist, purge_dl_records); + skiplist->ReleaseCleaningLock(); + } + } + pending_clean_records.no_hash_skiplists.clear(); + } + + if (!pending_clean_records.valid_lists.empty()) { + for (auto& list : pending_clean_records.valid_lists) { + if (list && list->TryCleaningLock()) { + cleanList(list, purge_dl_records); + list->ReleaseCleaningLock(); + } + } + pending_clean_records.valid_lists.clear(); + } + + auto new_ts = version_controller_.GetCurrentTimestamp(); + if (purge_string_records.size() > kMaxCachedOldRecords) { + pending_clean_records.pending_purge_strings.emplace_back( + std::move(purge_string_records), new_ts); + purge_string_records = std::vector(); + } + + if (purge_dl_records.size() > kMaxCachedOldRecords) { + pending_clean_records.pending_purge_dls.emplace_back( + std::move(purge_dl_records), new_ts); + purge_dl_records = std::vector(); + } + + purgeAndFree(pending_clean_records); + + } // Finsh iterating hash table + + // Push the remaining need purged records to global pool. + auto new_ts = version_controller_.GetCurrentTimestamp(); + if (!purge_string_records.empty()) { + pending_clean_records.pending_purge_strings.emplace_back( + std::move(purge_string_records), new_ts); + purge_string_records = std::vector(); + } + + if (!purge_dl_records.empty()) { + pending_clean_records.pending_purge_dls.emplace_back( + std::move(purge_dl_records), new_ts); + purge_dl_records = std::vector(); + } + return total_num == 0 + ? 0.0f + outdated_collection_ratio + : (need_purge_num / (double)total_num) + outdated_collection_ratio; +} + +void KVEngine::TestCleanOutDated(size_t start_slot_idx, size_t end_slot_idx) { + PendingCleanRecords pending_clean_records; + while (!bg_work_signals_.terminating) { + auto cur_slot_idx = start_slot_idx; + while (cur_slot_idx < end_slot_idx && !bg_work_signals_.terminating) { + cleanOutDated(pending_clean_records, cur_slot_idx, 1024); + cur_slot_idx += 1024; + } + } +} +// Space Cleaner + +Cleaner::OutDatedCollections::~OutDatedCollections() {} + +void Cleaner::cleanWork() { + PendingCleanRecords pending_clean_records; + std::int64_t start_pos = start_slot_.fetch_add(kSlotBlockUnit) % + (kv_engine_->hash_table_->GetSlotsNum()); + kv_engine_->cleanOutDated(pending_clean_records, start_pos, kSlotBlockUnit); + + while (pending_clean_records.Size() != 0 && !close_.load()) { + kv_engine_->version_controller_.UpdateLocalOldestSnapshot(); + kv_engine_->purgeAndFree(pending_clean_records); + } +} + +void Cleaner::AdjustCleanWorkers(size_t advice_wokers_num) { + kvdk_assert(advice_wokers_num <= clean_workers_.size(), ""); + auto active_workers_num = active_clean_workers_.load(); + if (active_workers_num < advice_wokers_num) { + for (size_t i = active_workers_num; i < advice_wokers_num; ++i) { + clean_workers_[i].Run(); + } + } else if (active_workers_num > advice_wokers_num) { + for (size_t i = advice_wokers_num; i < active_workers_num; ++i) { + clean_workers_[i].Stop(); + } + } + active_clean_workers_ = advice_wokers_num; +} + +double Cleaner::SearchOutdatedCollections() { + size_t limited_fetch_num = 0; + std::unique_lock queue_lock(outdated_collections_.queue_mtx); + int64_t before_queue_size = + static_cast(outdated_collections_.hashlists.size() + + outdated_collections_.lists.size() + + outdated_collections_.skiplists.size()); + { + std::unique_lock skiplist_lock(kv_engine_->skiplists_mu_); + auto iter = kv_engine_->expirable_skiplists_.begin(); + while (!kv_engine_->expirable_skiplists_.empty() && (*iter)->HasExpired() && + limited_fetch_num < max_thread_num_) { + auto outdated_skiplist = *iter; + iter = kv_engine_->expirable_skiplists_.erase(iter); + outdated_collections_.skiplists.emplace( + outdated_skiplist, + kv_engine_->version_controller_.GetCurrentTimestamp()); + limited_fetch_num++; + } + } + + { + std::unique_lock list_lock(kv_engine_->lists_mu_); + auto iter = kv_engine_->expirable_lists_.begin(); + while (!kv_engine_->expirable_lists_.empty() && (*iter)->HasExpired() && + limited_fetch_num < max_thread_num_) { + auto outdated_list = *iter; + iter = kv_engine_->expirable_lists_.erase(iter); + outdated_collections_.lists.emplace( + outdated_list, kv_engine_->version_controller_.GetCurrentTimestamp()); + limited_fetch_num++; + } + } + + { + std::unique_lock hlist_lock(kv_engine_->hlists_mu_); + auto iter = kv_engine_->expirable_hlists_.begin(); + while (!kv_engine_->expirable_hlists_.empty() && (*iter)->HasExpired() && + limited_fetch_num < max_thread_num_) { + auto expired_hash_list = *iter; + iter = kv_engine_->expirable_hlists_.erase(iter); + outdated_collections_.hashlists.emplace( + expired_hash_list, + kv_engine_->version_controller_.GetCurrentTimestamp()); + limited_fetch_num++; + } + } + + int64_t after_queue_size = + static_cast(outdated_collections_.hashlists.size() + + outdated_collections_.lists.size() + + outdated_collections_.skiplists.size()); + + double diff_size_ratio = + (after_queue_size - before_queue_size) / max_thread_num_; + if (diff_size_ratio > 0) { + outdated_collections_.increase_ratio = + outdated_collections_.increase_ratio == 0 + ? diff_size_ratio + : diff_size_ratio / outdated_collections_.increase_ratio; + } + return outdated_collections_.increase_ratio; +} + +void Cleaner::FetchOutdatedCollections( + PendingCleanRecords& pending_clean_records) { + Collection* outdated_collection = nullptr; + RecordType record_type; + auto min_snapshot_ts = + kv_engine_->version_controller_.GlobalOldestSnapshotTs(); + { + std::unique_lock queue_lock(outdated_collections_.queue_mtx); + if (!outdated_collection && !outdated_collections_.skiplists.empty()) { + auto skiplist_iter = outdated_collections_.skiplists.begin(); + if (skiplist_iter->second < min_snapshot_ts) { + outdated_collection = skiplist_iter->first; + record_type = RecordType::SortedHeader; + outdated_collections_.skiplists.erase(skiplist_iter); + } + } + + if (!outdated_collection && !outdated_collections_.lists.empty()) { + auto list_iter = outdated_collections_.lists.begin(); + if (list_iter->second < min_snapshot_ts) { + outdated_collection = list_iter->first; + record_type = RecordType::ListHeader; + outdated_collections_.lists.erase(list_iter); + } + } + + if (!outdated_collection && !outdated_collections_.hashlists.empty()) { + auto hash_list_iter = outdated_collections_.hashlists.begin(); + if (hash_list_iter->second < min_snapshot_ts) { + outdated_collection = hash_list_iter->first; + record_type = RecordType::HashHeader; + outdated_collections_.hashlists.erase(hash_list_iter); + } + } + } + + if (outdated_collection) { + auto guard = + kv_engine_->hash_table_->AcquireLock(outdated_collection->Name()); + auto lookup_result = + kv_engine_->lookupKey(outdated_collection->Name(), record_type); + switch (lookup_result.entry_ptr->GetIndexType()) { + case PointerType::HashList: { + auto outdated_hlist = static_cast(outdated_collection); + if (lookup_result.entry_ptr->GetIndex().hlist->ID() == + outdated_collection->ID()) { + kv_engine_->hash_table_->Erase(lookup_result.entry_ptr); + } + kv_engine_->removeOutdatedCollection( + lookup_result.entry_ptr->GetIndex().hlist); + pending_clean_records.outdated_hlists.emplace_back(std::make_pair( + kv_engine_->version_controller_.GetCurrentTimestamp(), + outdated_hlist)); + break; + } + case PointerType::List: { + auto outdated_list = static_cast(outdated_collection); + if (lookup_result.entry_ptr->GetIndex().list->ID() == + outdated_collection->ID()) { + kv_engine_->hash_table_->Erase(lookup_result.entry_ptr); + } + kv_engine_->removeOutdatedCollection( + lookup_result.entry_ptr->GetIndex().list); + pending_clean_records.outdated_lists.emplace_back(std::make_pair( + kv_engine_->version_controller_.GetCurrentTimestamp(), + outdated_list)); + break; + } + case PointerType::Skiplist: { + auto outdated_skiplist = static_cast(outdated_collection); + if (lookup_result.entry_ptr->GetIndex().skiplist->ID() == + outdated_collection->ID()) { + kv_engine_->hash_table_->Erase(lookup_result.entry_ptr); + } + kv_engine_->removeOutdatedCollection( + lookup_result.entry_ptr->GetIndex().skiplist); + pending_clean_records.outdated_skiplists.emplace_back(std::make_pair( + kv_engine_->version_controller_.GetCurrentTimestamp(), + outdated_skiplist)); + break; + } + default: + break; + } + } +} + +void Cleaner::mainWork() { + PendingCleanRecords pending_clean_records; + std::int64_t start_pos = start_slot_.fetch_add(kSlotBlockUnit) % + (kv_engine_->hash_table_->GetSlotsNum()); + + double outdated_ratio = kv_engine_->cleanOutDated(pending_clean_records, + start_pos, kSlotBlockUnit); + + size_t advice_thread_num = min_thread_num_; + if (outdated_ratio >= kWakeUpThreshold) { + advice_thread_num = std::ceil(outdated_ratio * max_thread_num_); + advice_thread_num = + std::min(std::max(min_thread_num_, advice_thread_num), max_thread_num_); + } + TEST_SYNC_POINT_CALLBACK("KVEngine::Cleaner::AdjustCleanWorkers", + &advice_thread_num); + size_t advice_clean_workers = + advice_thread_num > 0 ? advice_thread_num - 1 : 0; + + AdjustCleanWorkers(advice_clean_workers); + if (outdated_ratio < kWakeUpThreshold) { + sleep(1); + } + + while (pending_clean_records.Size() != 0 && !close_.load()) { + kv_engine_->version_controller_.UpdateLocalOldestSnapshot(); + kv_engine_->purgeAndFree(pending_clean_records); + } +} +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/kv_engine_cleaner.hpp b/volatile/engine/kv_engine_cleaner.hpp new file mode 100644 index 00000000..0e236a29 --- /dev/null +++ b/volatile/engine/kv_engine_cleaner.hpp @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "alias.hpp" +#include "collection.hpp" +#include "hash_table.hpp" +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { + +struct PendingFreeSpaceEntries { + std::vector entries; + // Indicate timestamp of the oldest refered snapshot of kvdk instance while we + // could safely free these entries + TimestampType release_time; +}; + +struct PendingFreeSpaceEntry { + SpaceEntry entry; + // Indicate timestamp of the oldest refered snapshot of kvdk instance while we + // could safely free this entry + TimestampType release_time; +}; + +struct PendingPurgeStrRecords { + PendingPurgeStrRecords(std::vector&& _records, + TimestampType _release_time) + : records(_records), release_time(_release_time) {} + + std::vector records; + TimestampType release_time; +}; + +struct PendingPurgeDLRecords { + PendingPurgeDLRecords(std::vector&& _records, + TimestampType _release_time) + : records(_records), release_time(_release_time) {} + + std::vector records; + TimestampType release_time; +}; + +struct PendingCleanRecords { + std::deque> outdated_lists; + std::deque> outdated_hlists; + std::deque> outdated_skiplists; + std::deque pending_purge_strings; + std::deque pending_purge_dls; + std::deque no_hash_skiplists; + std::deque valid_lists; + size_t Size() { + return outdated_lists.size() + outdated_hlists.size() + + outdated_skiplists.size() + pending_purge_strings.size() + + pending_purge_dls.size() + no_hash_skiplists.size() + + valid_lists.size(); + } +}; + +class KVEngine; + +class Cleaner { + public: + static constexpr int64_t kSlotBlockUnit = 1024; + static constexpr double kWakeUpThreshold = 0.1; + + Cleaner(KVEngine* kv_engine, int64_t max_cleaner_threads) + : kv_engine_(kv_engine), + max_thread_num_(max_cleaner_threads), + min_thread_num_(1), + close_(false), + start_slot_(0), + active_clean_workers_(0), + main_worker_([this]() { this->mainWork(); }), + clean_workers_(max_thread_num_ - 1 /*1 for main worker*/) { + for (auto& w : clean_workers_) { + w.Init([this]() { this->cleanWork(); }); + } + } + + ~Cleaner() { Close(); } + + void Start() { + if (!close_ && min_thread_num_ > 0) { + main_worker_.Run(); + } + } + + void Close() { + close_ = true; + main_worker_.Join(); + for (auto& w : clean_workers_) { + w.Join(); + } + } + + void AdjustCleanWorkers(size_t advice_workers_num); + + size_t ActiveThreadNum() { return active_clean_workers_.load() + 1; } + + double SearchOutdatedCollections(); + void FetchOutdatedCollections(PendingCleanRecords& pending_clean_records); + + private: + class Worker { + public: + ~Worker() { Join(); } + + Worker(std::function f) { Init(f); } + + Worker() = default; + + void Init(std::function f) { + Join(); + func = f; + std::unique_lock ul(spin); + join_ = false; + keep_work_ = false; + + auto work = [&]() { + while (true) { + { + std::unique_lock ul(spin); + while (!keep_work_ && !join_) { + cv.wait(ul); + } + if (join_) { + return; + } + } + func(); + } + }; + worker_thread = std::thread(work); + } + + void Run() { + std::unique_lock ul(spin); + keep_work_ = true; + cv.notify_all(); + } + + void Stop() { + std::unique_lock ul(spin); + keep_work_ = false; + } + + void Join() { + { + std::unique_lock ul(spin); + keep_work_ = false; + join_ = true; + cv.notify_all(); + } + if (worker_thread.joinable()) { + worker_thread.join(); + } + } + + private: + bool keep_work_; + bool join_; + std::condition_variable_any cv; + SpinMutex spin; + std::thread worker_thread; + std::function func; + }; + + KVEngine* kv_engine_; + + size_t max_thread_num_; + size_t min_thread_num_; + std::atomic close_; + std::atomic start_slot_; + std::atomic active_clean_workers_; + Worker main_worker_; + std::vector clean_workers_; + + struct OutDatedCollections { + struct TimeStampCmp { + public: + bool operator()(const std::pair a, + const std::pair b) const { + if (a.second < b.second) return true; + if (a.second == b.second && a.first->ID() < b.first->ID()) return true; + return false; + } + }; + using ListQueue = std::set, TimeStampCmp>; + using HashListQueue = + std::set, TimeStampCmp>; + + using SkiplistQueue = + std::set, TimeStampCmp>; + + SpinMutex queue_mtx; + ListQueue lists; + HashListQueue hashlists; + SkiplistQueue skiplists; + double increase_ratio = 0; + ~OutDatedCollections(); + }; + + OutDatedCollections outdated_collections_; + + private: + void cleanWork(); + void mainWork(); +}; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/kv_engine_hash.cpp b/volatile/engine/kv_engine_hash.cpp new file mode 100644 index 00000000..c849bc56 --- /dev/null +++ b/volatile/engine/kv_engine_hash.cpp @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "hash_collection/iterator.hpp" +#include "kv_engine.hpp" + +namespace KVDK_NAMESPACE { +Status KVEngine::HashCreate(StringView collection) { + auto thread_holder = AcquireAccessThread(); + + if (!checkKeySize(collection)) { + return Status::InvalidDataSize; + } + + std::shared_ptr hlist = nullptr; + return buildHashlist(collection, hlist); +} + +Status KVEngine::buildHashlist(const StringView& collection, + std::shared_ptr& hlist) { + auto ul = hash_table_->AcquireLock(collection); + auto holder = version_controller_.GetLocalSnapshotHolder(); + TimestampType new_ts = holder.Timestamp(); + auto lookup_result = lookupKey(collection, RecordType::HashHeader); + if (lookup_result.s == Status::NotFound || + lookup_result.s == Status::Outdated) { + DLRecord* existing_header = + lookup_result.s == Outdated + ? lookup_result.entry.GetIndex().hlist->HeaderRecord() + : nullptr; + CollectionIDType id = collection_id_.fetch_add(1); + std::string value_str = HashList::EncodeID(id); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(collection, value_str)); + if (space.size == 0) { + return Status::MemoryOverflow; + } + // dl list is circular, so the next and prev pointers of + // header point to itself + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space.offset), space.size, new_ts, + RecordType::HashHeader, RecordStatus::Normal, + kv_allocator_->addr2offset(existing_header), space.offset, space.offset, + collection, value_str); + hlist = std::make_shared(data_record, collection, id, + kv_allocator_.get(), hash_table_.get(), + dllist_locks_.get()); + kvdk_assert(hlist != nullptr, ""); + addHashlistToMap(hlist); + insertKeyOrElem(lookup_result, RecordType::HashHeader, RecordStatus::Normal, + hlist.get()); + return Status::Ok; + } else { + return lookup_result.s == Status::Ok ? Status::Existed : lookup_result.s; + } +} + +Status KVEngine::HashDestroy(StringView collection) { + auto thread_holder = AcquireAccessThread(); + + if (!checkKeySize(collection)) { + return Status::InvalidDataSize; + } + + auto ul = hash_table_->AcquireLock(collection); + auto snapshot_holder = version_controller_.GetLocalSnapshotHolder(); + auto new_ts = snapshot_holder.Timestamp(); + HashList* hlist; + Status s = hashListFind(collection, &hlist); + if (s == Status::Ok) { + DLRecord* header = hlist->HeaderRecord(); + kvdk_assert(header->GetRecordType() == RecordType::HashHeader, ""); + StringView value = header->Value(); + auto request_size = DLRecord::RecordSize(collection, value); + SpaceEntry space = kv_allocator_->Allocate(request_size); + if (space.size == 0) { + return Status::MemoryOverflow; + } + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space.offset), space.size, new_ts, + RecordType::HashHeader, RecordStatus::Outdated, + kv_allocator_->addr2offset_checked(header), header->prev, header->next, + collection, value); + bool success = hlist->Replace(header, data_record); + kvdk_assert(success, "existing header should be linked on its hlist"); + hash_table_->Insert(collection, RecordType::HashHeader, + RecordStatus::Outdated, hlist, PointerType::HashList); + { + std::unique_lock hlist_lock(hlists_mu_); + expirable_hlists_.emplace(hlist); + } + } + return s; +} + +Status KVEngine::HashSize(StringView collection, size_t* len) { + if (!checkKeySize(collection)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + HashList* hlist; + Status s = hashListFind(collection, &hlist); + if (s != Status::Ok) { + return s; + } + *len = hlist->Size(); + return Status::Ok; +} + +Status KVEngine::HashGet(StringView collection, StringView key, + std::string* value) { + auto thread_holder = AcquireAccessThread(); + + // Hold current snapshot in this thread + auto holder = version_controller_.GetLocalSnapshotHolder(); + + HashList* hlist; + Status s = hashListFind(collection, &hlist); + if (s == Status::Ok) { + s = hlist->Get(key, value); + } + return s; +} + +Status KVEngine::HashPut(StringView collection, StringView key, + StringView value) { + auto thread_holder = AcquireAccessThread(); + + // Hold current snapshot in this thread + auto holder = version_controller_.GetLocalSnapshotHolder(); + + HashList* hlist; + Status s = hashListFind(collection, &hlist); + if (s == Status::Ok) { + std::string collection_key(hlist->InternalKey(key)); + if (!checkKeySize(collection_key) || !checkValueSize(value)) { + s = Status::InvalidDataSize; + } else { + auto ul = hash_table_->AcquireLock(collection_key); + auto ret = + hlist->Put(key, value, version_controller_.GetCurrentTimestamp()); + if (ret.s == Status::Ok && ret.existing_record && + hlist->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + hlist->ReleaseCleaningLock(); + } + tryCleanCachedOutdatedRecord(); + s = ret.s; + } + } + return s; +} + +Status KVEngine::HashDelete(StringView collection, StringView key) { + auto thread_holder = AcquireAccessThread(); + + // Hold current snapshot in this thread + auto holder = version_controller_.GetLocalSnapshotHolder(); + + HashList* hlist; + Status s = hashListFind(collection, &hlist); + if (s == Status::Ok) { + std::string collection_key(hlist->InternalKey(key)); + if (!checkKeySize(collection_key)) { + s = Status::InvalidDataSize; + } else { + auto ul = hash_table_->AcquireLock(collection_key); + auto ret = hlist->Delete(key, version_controller_.GetCurrentTimestamp()); + if (ret.s == Status::Ok && ret.existing_record && ret.write_record && + hlist->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + hlist->ReleaseCleaningLock(); + } + tryCleanCachedOutdatedRecord(); + s = ret.s; + } + } + return s; +} + +Status KVEngine::HashModify(StringView collection, StringView key, + ModifyFunc modify_func, void* cb_args) { + auto thread_holder = AcquireAccessThread(); + + // Hold current snapshot in this thread + auto holder = version_controller_.GetLocalSnapshotHolder(); + HashList* hlist; + Status s = hashListFind(collection, &hlist); + if (s == Status::Ok) { + std::string internal_key(hlist->InternalKey(key)); + auto ul = hash_table_->AcquireLock(internal_key); + auto ret = hlist->Modify(key, modify_func, cb_args, + version_controller_.GetCurrentTimestamp()); + s = ret.s; + if (s == Status::Ok && ret.existing_record && ret.write_record && + hlist->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + hlist->ReleaseCleaningLock(); + } + tryCleanCachedOutdatedRecord(); + } + return s; +} + +HashIterator* KVEngine::HashIteratorCreate(StringView collection, + Snapshot* snapshot, Status* status) { + Status s{Status::Ok}; + HashIterator* ret(nullptr); + if (!checkKeySize(collection)) { + s = Status::InvalidDataSize; + } + + if (s == Status::Ok) { + bool create_snapshot = snapshot == nullptr; + if (create_snapshot) { + snapshot = GetSnapshot(false); + } + HashList* hlist; + Status s = hashListFind(collection, &hlist); + if (s == Status::Ok) { + ret = new HashIteratorImpl(hlist, static_cast(snapshot), + create_snapshot); + } else if (create_snapshot) { + ReleaseSnapshot(snapshot); + } + } + if (status) { + *status = s; + } + return ret; +} + +void KVEngine::HashIteratorRelease(HashIterator* hash_iter) { + if (hash_iter == nullptr) { + GlobalLogger.Info("pass a nullptr in KVEngine::HashIteratorRelease!\n"); + return; + } + HashIteratorImpl* iter = static_cast(hash_iter); + if (iter->own_snapshot_) { + ReleaseSnapshot(iter->snapshot_); + } + delete iter; +} + +Status KVEngine::hashListFind(StringView collection, HashList** hlist) { + // Callers should acquire the access token or snapshot. + // Lockless lookup for the collection + auto result = lookupKey(collection, RecordType::HashHeader); + if (result.s == Status::Outdated) { + return Status::NotFound; + } + if (result.s != Status::Ok) { + return result.s; + } + (*hlist) = result.entry.GetIndex().hlist; + return Status::Ok; +} + +Status KVEngine::hashWritePrepare(HashWriteArgs& args, TimestampType ts) { + return args.hlist->PrepareWrite(args, ts); +} + +Status KVEngine::hashListWrite(HashWriteArgs& args) { + return args.hlist->Write(args).s; +} + +Status KVEngine::hashListPublish(HashWriteArgs const&) { return Status::Ok; } + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/kv_engine_list.cpp b/volatile/engine/kv_engine_list.cpp new file mode 100644 index 00000000..28e6d040 --- /dev/null +++ b/volatile/engine/kv_engine_list.cpp @@ -0,0 +1,607 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "kv_engine.hpp" +#include "list_collection/iterator.hpp" +#include "utils/sync_point.hpp" + +namespace KVDK_NAMESPACE { +Status KVEngine::ListCreate(StringView list_name) { + auto thread_holder = AcquireAccessThread(); + + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + + std::shared_ptr list = nullptr; + return buildList(list_name, list); +} + +Status KVEngine::buildList(const StringView& list_name, + std::shared_ptr& list) { + auto ul = hash_table_->AcquireLock(list_name); + auto holder = version_controller_.GetLocalSnapshotHolder(); + TimestampType new_ts = holder.Timestamp(); + auto lookup_result = lookupKey(list_name, RecordType::ListHeader); + if (lookup_result.s == Status::NotFound || + lookup_result.s == Status::Outdated) { + DLRecord* existing_header = + lookup_result.s == Outdated + ? lookup_result.entry.GetIndex().hlist->HeaderRecord() + : nullptr; + CollectionIDType id = collection_id_.fetch_add(1); + std::string value_str = List::EncodeID(id); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(list_name, value_str)); + if (space.size == 0) { + return Status::MemoryOverflow; + } + // dl list is circular, so the next and prev pointers of + // header point to itself + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space.offset), space.size, new_ts, + RecordType::ListHeader, RecordStatus::Normal, + kv_allocator_->addr2offset(existing_header), space.offset, space.offset, + list_name, value_str); + list = std::make_shared(data_record, list_name, id, + kv_allocator_.get(), dllist_locks_.get()); + kvdk_assert(list != nullptr, ""); + addListToMap(list); + insertKeyOrElem(lookup_result, RecordType::ListHeader, RecordStatus::Normal, + list.get()); + return Status::Ok; + } else { + return lookup_result.s == Status::Ok ? Status::Existed : lookup_result.s; + } +} + +Status KVEngine::ListDestroy(StringView collection) { + if (!checkKeySize(collection)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto ul = hash_table_->AcquireLock(collection); + auto snapshot_holder = version_controller_.GetLocalSnapshotHolder(); + auto new_ts = snapshot_holder.Timestamp(); + List* list; + Status s = listFind(collection, &list); + if (s == Status::Ok) { + DLRecord* header = list->HeaderRecord(); + kvdk_assert(header->GetRecordType() == RecordType::ListHeader, ""); + StringView value = header->Value(); + auto request_size = DLRecord::RecordSize(collection, value); + SpaceEntry space = kv_allocator_->Allocate(request_size); + if (space.size == 0) { + return Status::MemoryOverflow; + } + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space.offset), space.size, new_ts, + RecordType::ListHeader, RecordStatus::Outdated, + kv_allocator_->addr2offset_checked(header), header->prev, header->next, + collection, value); + bool success = list->Replace(header, data_record); + kvdk_assert(success, "existing header should be linked on its list"); + hash_table_->Insert(collection, RecordType::ListHeader, + RecordStatus::Outdated, list, PointerType::List); + { + std::unique_lock list_lock(lists_mu_); + expirable_lists_.emplace(list); + } + } + return s; +} + +Status KVEngine::ListSize(StringView list_name, size_t* sz) { + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + *sz = list->Size(); + return Status::Ok; +} + +Status KVEngine::ListPushFront(StringView collection, StringView elem) { + if (!checkKeySize(collection) || !checkValueSize(elem)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(collection, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + return list->PushFront(elem, version_controller_.GetCurrentTimestamp()).s; +} + +Status KVEngine::ListPushBack(StringView list_name, StringView elem) { + if (!checkKeySize(list_name) || !checkValueSize(elem)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + return list->PushBack(elem, version_controller_.GetCurrentTimestamp()).s; +} + +Status KVEngine::ListPopFront(StringView list_name, std::string* elem) { + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + auto ret = list->PopFront(version_controller_.GetCurrentTimestamp()); + + if (ret.s == Status::Ok) { + kvdk_assert(ret.existing_record && ret.write_record, ""); + if (elem) { + elem->assign(ret.existing_record->Value().data(), + ret.existing_record->Value().size()); + } + if (list->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + list->ReleaseCleaningLock(); + } + } + tryCleanCachedOutdatedRecord(); + return ret.s; +} + +Status KVEngine::ListPopBack(StringView list_name, std::string* elem) { + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + auto ret = list->PopBack(version_controller_.GetCurrentTimestamp()); + if (ret.existing_record == nullptr) { + /// TODO: NotFound does not properly describe the situation + return Status::NotFound; + } + + if (ret.s == Status::Ok) { + kvdk_assert(ret.existing_record && ret.write_record, ""); + if (elem) { + elem->assign(ret.existing_record->Value().data(), + ret.existing_record->Value().size()); + } + kvdk_assert(ret.existing_record && ret.write_record, ""); + + if (list->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + list->ReleaseCleaningLock(); + } + } + tryCleanCachedOutdatedRecord(); + return ret.s; +} + +Status KVEngine::ListBatchPushFront(StringView list_name, + std::vector const& elems) { + return ListBatchPushFront( + list_name, std::vector{elems.begin(), elems.end()}); +} + +Status KVEngine::ListBatchPushFront(StringView list_name, + std::vector const& elems) { + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + if (elems.size() > BatchWriteLog::Capacity()) { + return Status::InvalidBatchSize; + } + for (auto const& elem : elems) { + if (!checkValueSize(elem)) { + return Status::InvalidDataSize; + } + } + auto thread_holder = AcquireAccessThread(); + + Status s = maybeInitBatchLogFile(); + if (s != Status::Ok) { + return s; + } + return listBatchPushImpl(list_name, ListPos::Front, elems); +} + +Status KVEngine::ListBatchPushBack(StringView list_name, + std::vector const& elems) { + return ListBatchPushBack(list_name, + std::vector{elems.begin(), elems.end()}); +} + +Status KVEngine::ListBatchPushBack(StringView list_name, + std::vector const& elems) { + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + if (elems.size() > BatchWriteLog::Capacity()) { + return Status::InvalidBatchSize; + } + for (auto const& elem : elems) { + if (!checkValueSize(elem)) { + return Status::InvalidDataSize; + } + } + auto thread_holder = AcquireAccessThread(); + + Status s = maybeInitBatchLogFile(); + if (s != Status::Ok) { + return s; + } + return listBatchPushImpl(list_name, ListPos::Back, elems); +} + +Status KVEngine::ListBatchPopFront(StringView list_name, size_t n, + std::vector* elems) { + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + Status s = maybeInitBatchLogFile(); + if (s != Status::Ok) { + return s; + } + return listBatchPopImpl(list_name, ListPos::Front, n, elems); +} + +Status KVEngine::ListBatchPopBack(StringView list_name, size_t n, + std::vector* elems) { + if (!checkKeySize(list_name)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + Status s = maybeInitBatchLogFile(); + if (s != Status::Ok) { + return s; + } + + return listBatchPopImpl(list_name, ListPos::Back, n, elems); +} + +Status KVEngine::ListMove(StringView src, ListPos src_pos, StringView dst, + ListPos dst_pos, std::string* elem) { + if (!checkKeySize(src) || !checkKeySize(dst)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + Status s = maybeInitBatchLogFile(); + if (s != Status::Ok) { + return s; + } + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* src_list; + List* dst_list; + /// TODO: we must guarantee a consistent view for the List. + /// The same holds for other collections. + /// No collection should be expired, created or deleted during BatchWrite. + s = listFind(src, &src_list); + if (s != Status::Ok) { + return s; + } + if (src != dst) { + s = listFind(dst, &dst_list); + if (s != Status::Ok) { + return s; + } + } else { + dst_list = src_list; + } + + std::unique_lock guard1; + std::unique_lock guard2; + if (src_list < dst_list) { + guard1 = src_list->AcquireLock(); + guard2 = dst_list->AcquireLock(); + } else if (src_list > dst_list) { + guard1 = dst_list->AcquireLock(); + guard2 = src_list->AcquireLock(); + } else { + kvdk_assert(src == dst, ""); + guard1 = src_list->AcquireLock(); + } + + if (src == dst && src_pos == dst_pos) { + s = src_pos == ListPos::Front ? src_list->Front(elem) + : src_list->Back(elem); + return s; + } + + auto bw_token = version_controller_.GetBatchWriteToken(); + BatchWriteLog log; + log.SetTimestamp(bw_token.Timestamp()); + std::vector elems; + + auto pop_args = + src_list->PreparePopN((ListPos)src_pos, 1, bw_token.Timestamp(), &elems); + if (pop_args.s != Status::Ok) { + return pop_args.s; + } + kvdk_assert(elems.size() == 1, ""); + elem->swap(elems[0]); + std::vector elems_view{StringView(elem->data(), elem->size())}; + auto push_args = dst_list->PreparePushN((ListPos)dst_pos, elems_view, + bw_token.Timestamp()); + if (push_args.s != Status::Ok) { + return push_args.s; + } + + log.ListDelete(pop_args.spaces[0].offset); + log.ListEmplace(push_args.spaces[0].offset); + auto& tc = engine_thread_cache_[ThreadManager::ThreadID() % + configs_.max_access_threads]; + log.EncodeTo(tc.batch_log); + + BatchWriteLog::MarkProcessing(tc.batch_log); + + s = src_list->PopN(pop_args); + kvdk_assert(s == Status::Ok, "pop n always success"); + TEST_CRASH_POINT("KVEngine::ListMove", ""); + s = dst_list->PushN(push_args); + kvdk_assert(s == Status::Ok, "push n always success"); + + BatchWriteLog::MarkCommitted(tc.batch_log); + return Status::Ok; +} + +Status KVEngine::ListInsertAt(StringView list_name, StringView elem, + long index) { + if (!checkValueSize(elem)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + return list->InsertAt(elem, index, version_controller_.GetCurrentTimestamp()) + .s; +} + +Status KVEngine::ListInsertBefore(StringView list_name, StringView elem, + StringView pos) { + if (!checkValueSize(elem)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + return list + ->InsertBefore(elem, pos, version_controller_.GetCurrentTimestamp()) + .s; +} + +Status KVEngine::ListInsertAfter(StringView collection, StringView elem, + StringView dst) { + if (!checkValueSize(elem)) { + return Status::InvalidDataSize; + } + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(collection, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + return list->InsertAfter(elem, dst, version_controller_.GetCurrentTimestamp()) + .s; +} + +Status KVEngine::ListErase(StringView list_name, long index, + std::string* elem) { + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + auto ret = list->Erase(index, version_controller_.GetCurrentTimestamp()); + if (ret.s == Status::Ok) { + kvdk_assert(ret.existing_record && ret.write_record, ""); + if (elem) { + elem->assign(ret.existing_record->Value().data(), + ret.existing_record->Value().size()); + } + if (list->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + list->ReleaseCleaningLock(); + } + } + tryCleanCachedOutdatedRecord(); + return ret.s; +} + +// Replace the element at pos +Status KVEngine::ListReplace(StringView collection, long index, + StringView elem) { + auto thread_holder = AcquireAccessThread(); + + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(collection, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + return list->Update(index, elem, version_controller_.GetCurrentTimestamp()).s; +} + +ListIterator* KVEngine::ListIteratorCreate(StringView collection, + Snapshot* snapshot, Status* status) { + Status s{Status::Ok}; + ListIterator* ret(nullptr); + if (!checkKeySize(collection)) { + s = Status::InvalidDataSize; + } + + if (s == Status::Ok) { + bool create_snapshot = snapshot == nullptr; + if (create_snapshot) { + snapshot = GetSnapshot(false); + } + List* list; + Status s = listFind(collection, &list); + if (s == Status::Ok) { + ret = new ListIteratorImpl(list, static_cast(snapshot), + create_snapshot); + } else if (create_snapshot) { + ReleaseSnapshot(snapshot); + } + } + + if (status) { + *status = s; + } + return ret; +} + +void KVEngine::ListIteratorRelease(ListIterator* list) { + if (list == nullptr) { + GlobalLogger.Info("pass a nullptr in KVEngine::SortedIteratorRelease!\n"); + return; + } + ListIteratorImpl* iter = static_cast(list); + if (iter->own_snapshot_) { + ReleaseSnapshot(iter->snapshot_); + } + delete iter; +} + +Status KVEngine::listFind(StringView list_name, List** list) { + auto result = lookupKey(list_name, RecordType::ListHeader); + if (result.s == Status::Outdated) { + return Status::NotFound; + } + if (result.s != Status::Ok) { + return result.s; + } + (*list) = result.entry.GetIndex().list; + return Status::Ok; +} + +Status KVEngine::listBatchPushImpl(StringView list_name, ListPos pos, + std::vector const& elems) { + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + auto bw_token = version_controller_.GetBatchWriteToken(); + BatchWriteLog log; + log.SetTimestamp(bw_token.Timestamp()); + + auto push_n_args = list->PreparePushN(pos, elems, bw_token.Timestamp()); + if (push_n_args.s != Status::Ok) { + return push_n_args.s; + } + + for (auto& space : push_n_args.spaces) { + log.ListEmplace(space.offset); + } + + auto& tc = engine_thread_cache_[ThreadManager::ThreadID() % + configs_.max_access_threads]; + log.EncodeTo(tc.batch_log); + BatchWriteLog::MarkProcessing(tc.batch_log); + + s = list->PushN(push_n_args); + kvdk_assert(s == Status::Ok, "PushN always success"); + BatchWriteLog::MarkCommitted(tc.batch_log); + return s; +} + +Status KVEngine::listBatchPopImpl(StringView list_name, ListPos pos, size_t n, + std::vector* elems) { + auto token = version_controller_.GetLocalSnapshotHolder(); + List* list; + Status s = listFind(list_name, &list); + if (s != Status::Ok) { + return s; + } + auto guard = list->AcquireLock(); + + auto bw_token = version_controller_.GetBatchWriteToken(); + BatchWriteLog log; + log.SetTimestamp(bw_token.Timestamp()); + + auto pop_n_args = list->PreparePopN(pos, n, bw_token.Timestamp(), elems); + if (pop_n_args.s != Status::Ok) { + return pop_n_args.s; + } + + for (auto& space : pop_n_args.spaces) { + log.ListDelete(space.offset); + } + + auto& tc = engine_thread_cache_[ThreadManager::ThreadID() % + configs_.max_access_threads]; + log.EncodeTo(tc.batch_log); + BatchWriteLog::MarkProcessing(tc.batch_log); + + s = list->PopN(pop_n_args); + kvdk_assert(s == Status::Ok, "PopN always success with lock"); + BatchWriteLog::MarkCommitted(tc.batch_log); + return s; +} + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/kv_engine_sorted.cpp b/volatile/engine/kv_engine_sorted.cpp new file mode 100644 index 00000000..7bee661f --- /dev/null +++ b/volatile/engine/kv_engine_sorted.cpp @@ -0,0 +1,283 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "kv_engine.hpp" +#include "sorted_collection/iterator.hpp" + +namespace KVDK_NAMESPACE { +Status KVEngine::SortedCreate(const StringView collection_name, + const SortedCollectionConfigs& s_configs) { + auto thread_holder = AcquireAccessThread(); + + if (!checkKeySize(collection_name)) { + return Status::InvalidDataSize; + } + + std::shared_ptr skiplist = nullptr; + + return buildSkiplist(collection_name, s_configs, skiplist); +} + +Status KVEngine::buildSkiplist(const StringView& collection_name, + const SortedCollectionConfigs& s_configs, + std::shared_ptr& skiplist) { + auto ul = hash_table_->AcquireLock(collection_name); + auto holder = version_controller_.GetLocalSnapshotHolder(); + TimestampType new_ts = holder.Timestamp(); + auto lookup_result = + lookupKey(collection_name, RecordType::SortedHeader); + if (lookup_result.s == NotFound || lookup_result.s == Outdated) { + DLRecord* existing_header = + lookup_result.s == Outdated + ? lookup_result.entry.GetIndex().skiplist->HeaderRecord() + : nullptr; + auto comparator = comparators_.GetComparator(s_configs.comparator_name); + if (comparator == nullptr) { + GlobalLogger.Error("Compare function %s is not registered\n", + s_configs.comparator_name); + return Status::Abort; + } + CollectionIDType id = collection_id_.fetch_add(1); + std::string value_str = + Skiplist::EncodeSortedCollectionValue(id, s_configs); + uint32_t request_size = + sizeof(DLRecord) + collection_name.size() + value_str.size(); + SpaceEntry space_entry = kv_allocator_->Allocate(request_size); + if (space_entry.size == 0) { + return Status::MemoryOverflow; + } + + // Data level of dl list is circular, so the next and prev pointers of + // header point to itself + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr(space_entry.offset), space_entry.size, + new_ts, RecordType::SortedHeader, RecordStatus::Normal, + kv_allocator_->addr2offset(existing_header), space_entry.offset, + space_entry.offset, collection_name, value_str); + + skiplist = std::make_shared( + data_record, string_view_2_string(collection_name), id, comparator, + kv_allocator_.get(), skiplist_node_allocator_.get(), hash_table_.get(), + dllist_locks_.get(), s_configs.index_with_hashtable); + addSkiplistToMap(skiplist); + insertKeyOrElem(lookup_result, RecordType::SortedHeader, + RecordStatus::Normal, skiplist.get()); + } else { + return lookup_result.s == Status::Ok ? Status::Existed : lookup_result.s; + } + return Status::Ok; +} + +Status KVEngine::SortedDestroy(const StringView collection_name) { + auto thread_holder = AcquireAccessThread(); + + auto ul = hash_table_->AcquireLock(collection_name); + auto snapshot_holder = version_controller_.GetLocalSnapshotHolder(); + auto new_ts = snapshot_holder.Timestamp(); + auto lookup_result = + lookupKey(collection_name, RecordType::SortedHeader); + if (lookup_result.s == Status::Ok) { + Skiplist* skiplist = lookup_result.entry.GetIndex().skiplist; + DLRecord* header = skiplist->HeaderRecord(); + assert(header->GetRecordType() == RecordType::SortedHeader); + StringView value = header->Value(); + auto request_size = + sizeof(DLRecord) + collection_name.size() + value.size(); + SpaceEntry space_entry = kv_allocator_->Allocate(request_size); + if (space_entry.size == 0) { + return Status::MemoryOverflow; + } + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space_entry.offset), + space_entry.size, new_ts, RecordType::SortedHeader, + RecordStatus::Outdated, kv_allocator_->addr2offset_checked(header), + header->prev, header->next, collection_name, value, 0); + bool success = + Skiplist::Replace(header, data_record, skiplist->HeaderNode(), + kv_allocator_.get(), dllist_locks_.get()); + kvdk_assert(success, "existing header should be linked on its skiplist"); + insertKeyOrElem(lookup_result, RecordType::SortedHeader, + RecordStatus::Outdated, skiplist); + { + std::unique_lock skiplist_lock(skiplists_mu_); + expirable_skiplists_.emplace(skiplist); + } + } else if (lookup_result.s == Status::Outdated || + lookup_result.s == Status::NotFound) { + lookup_result.s = Status::Ok; + } + return lookup_result.s; +} + +Status KVEngine::SortedSize(const StringView collection, size_t* size) { + auto thread_holder = AcquireAccessThread(); + + auto holder = version_controller_.GetLocalSnapshotHolder(); + + Skiplist* skiplist = nullptr; + auto ret = lookupKey(collection, RecordType::SortedHeader); + if (ret.s != Status::Ok) { + return ret.s == Status::Outdated ? Status::NotFound : ret.s; + } + + kvdk_assert(ret.entry.GetIndexType() == PointerType::Skiplist, + "pointer type of skiplist in hash entry should be skiplist"); + skiplist = ret.entry.GetIndex().skiplist; + *size = skiplist->Size(); + return Status::Ok; +} + +Status KVEngine::SortedGet(const StringView collection, + const StringView user_key, std::string* value) { + auto thread_holder = AcquireAccessThread(); + + // Hold current snapshot in this thread + auto holder = version_controller_.GetLocalSnapshotHolder(); + + Skiplist* skiplist = nullptr; + auto ret = lookupKey(collection, RecordType::SortedHeader); + if (ret.s != Status::Ok) { + return ret.s == Status::Outdated ? Status::NotFound : ret.s; + } + + kvdk_assert(ret.entry.GetIndexType() == PointerType::Skiplist, + "pointer type of skiplist in hash entry should be skiplist"); + skiplist = ret.entry.GetIndex().skiplist; + + assert(skiplist); + return skiplist->Get(user_key, value); +} + +Status KVEngine::SortedPut(const StringView collection, + const StringView user_key, const StringView value) { + auto thread_holder = AcquireAccessThread(); + + auto snapshot_holder = version_controller_.GetLocalSnapshotHolder(); + + Skiplist* skiplist = nullptr; + + auto ret = lookupKey(collection, RecordType::SortedHeader); + if (ret.s != Status::Ok) { + return ret.s == Status::Outdated ? Status::NotFound : ret.s; + } + + kvdk_assert(ret.entry.GetIndexType() == PointerType::Skiplist, + "pointer type of skiplist in hash entry should be skiplist"); + skiplist = ret.entry.GetIndex().skiplist; + return sortedPutImpl(skiplist, user_key, value); +} + +Status KVEngine::SortedDelete(const StringView collection, + const StringView user_key) { + auto thread_holder = AcquireAccessThread(); + + // Hold current snapshot in this thread + auto holder = version_controller_.GetLocalSnapshotHolder(); + + Skiplist* skiplist = nullptr; + auto ret = lookupKey(collection, RecordType::SortedHeader); + if (ret.s != Status::Ok) { + return (ret.s == Status::Outdated || ret.s == Status::NotFound) ? Status::Ok + : ret.s; + } + + kvdk_assert(ret.entry.GetIndexType() == PointerType::Skiplist, + "pointer type of skiplist in hash entry should be skiplist"); + skiplist = ret.entry.GetIndex().skiplist; + + return sortedDeleteImpl(skiplist, user_key); +} + +SortedIterator* KVEngine::SortedIteratorCreate(const StringView collection, + Snapshot* snapshot, Status* s) { + Skiplist* skiplist; + bool create_snapshot = snapshot == nullptr; + if (create_snapshot) { + snapshot = GetSnapshot(false); + } + // find collection + auto res = lookupKey(collection, RecordType::SortedHeader); + if (s != nullptr) { + *s = (res.s == Status::Outdated) ? Status::NotFound : res.s; + } + if (res.s == Status::Ok) { + skiplist = res.entry_ptr->GetIndex().skiplist; + return new SortedIteratorImpl(skiplist, kv_allocator_.get(), + static_cast(snapshot), + create_snapshot); + } else { + if (create_snapshot) { + ReleaseSnapshot(snapshot); + } + return nullptr; + } +} + +void KVEngine::SortedIteratorRelease(SortedIterator* sorted_iterator) { + if (sorted_iterator == nullptr) { + GlobalLogger.Info("pass a nullptr in KVEngine::SortedIteratorRelease!\n"); + return; + } + SortedIteratorImpl* iter = static_cast(sorted_iterator); + if (iter->own_snapshot_) { + ReleaseSnapshot(iter->snapshot_); + } + delete iter; +} + +Status KVEngine::sortedDeleteImpl(Skiplist* skiplist, + const StringView& user_key) { + std::string collection_key(skiplist->InternalKey(user_key)); + if (!checkKeySize(collection_key)) { + return Status::InvalidDataSize; + } + + auto ul = hash_table_->AcquireLock(collection_key); + TimestampType new_ts = version_controller_.GetCurrentTimestamp(); + + auto ret = skiplist->Delete(user_key, new_ts); + + if (ret.existing_record && ret.write_record && skiplist->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + skiplist->ReleaseCleaningLock(); + } + tryCleanCachedOutdatedRecord(); + + return ret.s; +} + +Status KVEngine::sortedPutImpl(Skiplist* skiplist, const StringView& user_key, + const StringView& value) { + std::string collection_key(skiplist->InternalKey(user_key)); + if (!checkKeySize(collection_key) || !checkValueSize(value)) { + return Status::InvalidDataSize; + } + + auto ul = hash_table_->AcquireLock(collection_key); + TimestampType new_ts = version_controller_.GetCurrentTimestamp(); + auto ret = skiplist->Put(user_key, value, new_ts); + + // Collect outdated version records + if (ret.existing_record && skiplist->TryCleaningLock()) { + removeAndCacheOutdatedVersion(ret.write_record); + skiplist->ReleaseCleaningLock(); + } + tryCleanCachedOutdatedRecord(); + + return ret.s; +} + +Status KVEngine::sortedWritePrepare(SortedWriteArgs& args, TimestampType ts) { + return args.skiplist->PrepareWrite(args, ts); +} + +Status KVEngine::sortedWrite(SortedWriteArgs& args) { + // Notice Jiayu: we do not handle delay free here as is no longer need soon + return args.skiplist->Write(args).s; +} + +Status KVEngine::sortedWritePublish(SortedWriteArgs const&) { + return Status::Ok; +} +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/kv_engine_string.cpp b/volatile/engine/kv_engine_string.cpp new file mode 100644 index 00000000..25074985 --- /dev/null +++ b/volatile/engine/kv_engine_string.cpp @@ -0,0 +1,291 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "kv_engine.hpp" +#include "utils/sync_point.hpp" + +namespace KVDK_NAMESPACE { + +Status KVEngine::Modify(const StringView key, ModifyFunc modify_func, + void* modify_args, const WriteOptions& write_options) { + int64_t base_time = TimeUtils::millisecond_time(); + if (!TimeUtils::CheckTTL(write_options.ttl_time, base_time)) { + return Status::InvalidArgument; + } + + auto thread_holder = AcquireAccessThread(); + + auto ul = hash_table_->AcquireLock(key); + auto holder = version_controller_.GetLocalSnapshotHolder(); + TimestampType new_ts = holder.Timestamp(); + auto lookup_result = lookupKey(key, RecordType::String); + + StringRecord* existing_record = nullptr; + std::string existing_value; + std::string new_value; + // push it into cleaner + if (lookup_result.s == Status::Ok) { + existing_record = lookup_result.entry.GetIndex().string_record; + existing_value.assign(existing_record->Value().data(), + existing_record->Value().size()); + } else if (lookup_result.s == Status::Outdated) { + existing_record = lookup_result.entry.GetIndex().string_record; + } else if (lookup_result.s == Status::NotFound) { + // nothing todo + } else { + return lookup_result.s; + } + + auto modify_operation = + modify_func(lookup_result.s == Status::Ok ? &existing_value : nullptr, + &new_value, modify_args); + switch (modify_operation) { + case ModifyOperation::Write: { + if (!checkValueSize(new_value)) { + return Status::InvalidDataSize; + } + + ExpireTimeType expired_time = + lookup_result.s == Status::Ok && !write_options.update_ttl + ? existing_record->GetExpireTime() + : TimeUtils::TTLToExpireTime(write_options.ttl_time, base_time); + + SpaceEntry space_entry = + kv_allocator_->Allocate(StringRecord::RecordSize(key, new_value)); + if (space_entry.size == 0) { + return Status::MemoryOverflow; + } + + StringRecord* new_record = + kv_allocator_->offset2addr_checked(space_entry.offset); + StringRecord::ConstructStringRecord( + new_record, space_entry.size, new_ts, RecordType::String, + RecordStatus::Normal, + existing_record == nullptr + ? kNullMemoryOffset + : kv_allocator_->addr2offset_checked(existing_record), + key, new_value, expired_time); + insertKeyOrElem(lookup_result, RecordType::String, RecordStatus::Normal, + new_record); + break; + } + case ModifyOperation::Delete: { + if (lookup_result.s == Status::Ok) { + SpaceEntry space_entry = + kv_allocator_->Allocate(StringRecord::RecordSize(key, "")); + if (space_entry.size == 0) { + return Status::MemoryOverflow; + } + + void* data_ptr = kv_allocator_->offset2addr_checked(space_entry.offset); + StringRecord::ConstructStringRecord( + data_ptr, space_entry.size, new_ts, RecordType::String, + RecordStatus::Outdated, + kv_allocator_->addr2offset_checked(existing_record), key, ""); + insertKeyOrElem(lookup_result, RecordType::String, + RecordStatus::Outdated, data_ptr); + break; + } + case ModifyOperation::Abort: { + return Status::Abort; + } + case ModifyOperation::Noop: { + return Status::Ok; + } + } + } + + return Status::Ok; +} + +Status KVEngine::Put(const StringView key, const StringView value, + const WriteOptions& options) { + auto thread_holder = AcquireAccessThread(); + + if (!checkKeySize(key) || !checkValueSize(value)) { + return Status::InvalidDataSize; + } + + return stringPutImpl(key, value, options); +} + +Status KVEngine::Get(const StringView key, std::string* value) { + auto thread_holder = AcquireAccessThread(); + + if (!checkKeySize(key)) { + return Status::InvalidDataSize; + } + auto holder = version_controller_.GetLocalSnapshotHolder(); + auto ret = lookupKey(key, RecordType::String); + if (ret.s == Status::Ok) { + StringRecord* string_record = ret.entry.GetIndex().string_record; + kvdk_assert(string_record->GetRecordType() == RecordType::String && + string_record->GetRecordStatus() != RecordStatus::Outdated, + "Got wrong data type in string get"); + kvdk_assert(string_record->ValidOrDirty(), "Corrupted data in string get"); + value->assign(string_record->Value().data(), string_record->Value().size()); + return Status::Ok; + } else { + return ret.s == Status::Outdated ? Status::NotFound : ret.s; + } +} + +Status KVEngine::Delete(const StringView key) { + auto thread_holder = AcquireAccessThread(); + + if (!checkKeySize(key)) { + return Status::InvalidDataSize; + } + + return stringDeleteImpl(key); +} + +Status KVEngine::stringDeleteImpl(const StringView& key) { + auto ul = hash_table_->AcquireLock(key); + auto holder = version_controller_.GetLocalSnapshotHolder(); + TimestampType new_ts = holder.Timestamp(); + + auto lookup_result = lookupKey(key, RecordType::String); + if (lookup_result.s == Status::Ok) { + // We only write delete record if key exist + auto request_size = key.size() + sizeof(StringRecord); + SpaceEntry space_entry = kv_allocator_->Allocate(request_size); + if (space_entry.size == 0) { + return Status::MemoryOverflow; + } + + StringRecord* data_ptr = + kv_allocator_->offset2addr_checked(space_entry.offset); + StringRecord::ConstructStringRecord( + data_ptr, space_entry.size, new_ts, RecordType::String, + RecordStatus::Outdated, + kv_allocator_->addr2offset_checked( + lookup_result.entry.GetIndex().string_record), + key, ""); + insertKeyOrElem(lookup_result, RecordType::String, RecordStatus::Outdated, + data_ptr); + + removeAndCacheOutdatedVersion(data_ptr); + } + tryCleanCachedOutdatedRecord(); + + return (lookup_result.s == Status::NotFound || + lookup_result.s == Status::Outdated) + ? Status::Ok + : lookup_result.s; +} + +Status KVEngine::stringPutImpl(const StringView& key, const StringView& value, + const WriteOptions& write_options) { + int64_t base_time = TimeUtils::millisecond_time(); + if (!TimeUtils::CheckTTL(write_options.ttl_time, base_time)) { + return Status::InvalidArgument; + } + + TEST_SYNC_POINT("KVEngine::stringPutImpl::BeforeLock"); + auto ul = hash_table_->AcquireLock(key); + auto holder = version_controller_.GetLocalSnapshotHolder(); + TimestampType new_ts = holder.Timestamp(); + + // Lookup key in hashtable + auto lookup_result = lookupKey(key, RecordType::String); + if (lookup_result.s == Status::MemoryOverflow || + lookup_result.s == Status::WrongType) { + return lookup_result.s; + } + + kvdk_assert(lookup_result.s == Status::NotFound || + lookup_result.s == Status::Ok || + lookup_result.s == Status::Outdated, + "Wrong return status in lookupKey in stringPutImpl"); + StringRecord* existing_record = + lookup_result.s == Status::NotFound + ? nullptr + : lookup_result.entry.GetIndex().string_record; + kvdk_assert(!existing_record || new_ts > existing_record->GetTimestamp(), + "existing record has newer timestamp or wrong return status in " + "string set"); + ExpireTimeType expired_time = + lookup_result.s == Status::Ok && !write_options.update_ttl + ? existing_record->GetExpireTime() + : TimeUtils::TTLToExpireTime(write_options.ttl_time, base_time); + + // Save key-value pair + SpaceEntry space_entry = + kv_allocator_->Allocate(StringRecord::RecordSize(key, value)); + if (space_entry.size == 0) { + return Status::MemoryOverflow; + } + + StringRecord* new_record = + kv_allocator_->offset2addr_checked(space_entry.offset); + StringRecord::ConstructStringRecord( + new_record, space_entry.size, new_ts, RecordType::String, + RecordStatus::Normal, kv_allocator_->addr2offset(existing_record), key, + value, expired_time); + + insertKeyOrElem(lookup_result, RecordType::String, RecordStatus::Normal, + new_record); + + if (existing_record) { + removeAndCacheOutdatedVersion(new_record); + } + tryCleanCachedOutdatedRecord(); + + return Status::Ok; +} + +Status KVEngine::stringWritePrepare(StringWriteArgs& args, TimestampType ts) { + args.res = lookupKey(args.key, RecordType::String); + if (args.res.s != Status::Ok && args.res.s != Status::NotFound && + args.res.s != Status::Outdated) { + return args.res.s; + } + args.ts = ts; + if (args.op == WriteOp::Delete && args.res.s != Status::Ok) { + return Status::Ok; + } + args.space = + kv_allocator_->Allocate(StringRecord::RecordSize(args.key, args.value)); + if (args.space.size == 0) { + return Status::MemoryOverflow; + } + return Status::Ok; +} + +Status KVEngine::stringWrite(StringWriteArgs& args) { + RecordStatus record_status = + args.op == WriteOp::Put ? RecordStatus::Normal : RecordStatus::Outdated; + void* new_addr = kv_allocator_->offset2addr_checked(args.space.offset); + MemoryOffsetType old_off; + if (args.res.s == Status::NotFound) { + kvdk_assert(args.op == WriteOp::Put, ""); + old_off = kNullMemoryOffset; + } else { + kvdk_assert(args.res.s == Status::Ok || args.res.s == Status::Outdated, ""); + old_off = kv_allocator_->addr2offset_checked( + args.res.entry.GetIndex().string_record); + } + args.new_rec = StringRecord::ConstructStringRecord( + new_addr, args.space.size, args.ts, RecordType::String, record_status, + old_off, args.key, args.value); + return Status::Ok; +} + +Status KVEngine::stringWritePublish(StringWriteArgs const& args) { + RecordStatus record_status = + args.op == WriteOp::Put ? RecordStatus::Normal : RecordStatus::Outdated; + insertKeyOrElem(args.res, RecordType::String, record_status, + const_cast(args.new_rec)); + return Status::Ok; +} + +Status KVEngine::stringRollback(TimestampType, + BatchWriteLog::StringLogEntry const& log) { + static_cast(kv_allocator_->offset2addr_checked(log.offset)) + ->Destroy(); + return Status::Ok; +} + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/list_collection/iterator.hpp b/volatile/engine/list_collection/iterator.hpp new file mode 100644 index 00000000..98b4aa2b --- /dev/null +++ b/volatile/engine/list_collection/iterator.hpp @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#pragma once + +#include "../version/version_controller.hpp" +#include "kvdk/volatile/engine.hpp" +#include "kvdk/volatile/iterator.hpp" +#include "list.hpp" + +namespace KVDK_NAMESPACE { +class ListIteratorImpl final : public ListIterator { + public: + ListIteratorImpl(List* list, const SnapshotImpl* snapshot, bool own_snapshot) + : list_(list), + snapshot_(snapshot), + own_snapshot_(own_snapshot), + dl_iter_(&list->dl_list_, list->kv_allocator_, snapshot) {} + + void Seek(long index) final { + if (index < 0) { + SeekToLast(); + long cur = -1; + while (cur-- > index && Valid()) { + Prev(); + } + } else { + SeekToFirst(); + long cur = 0; + while (cur++ < index && Valid()) { + Next(); + } + } + } + + void SeekToFirst() final { dl_iter_.SeekToFirst(); } + + void SeekToLast() final { dl_iter_.SeekToLast(); } + + void SeekToFirst(StringView elem) final { + SeekToFirst(); + Next(elem); + } + + void SeekToLast(StringView elem) final { + SeekToLast(); + Prev(elem); + } + + bool Valid() const final { return dl_iter_.Valid(); } + + void Next() final { dl_iter_.Next(); } + + void Prev() final { dl_iter_.Prev(); } + + void Next(StringView elem) final { + while (Valid()) { + Next(); + if (!Valid() || equal_string_view(elem, dl_iter_.Value())) { + break; + } + } + } + + void Prev(StringView elem) final { + while (Valid()) { + Prev(); + if (!Valid() || equal_string_view(elem, dl_iter_.Value())) { + break; + } + } + } + + std::string Value() const final { + if (!Valid()) { + kvdk_assert(false, "Accessing data with invalid ListIterator!"); + return std::string{}; + } + auto sw = dl_iter_.Value(); + return std::string{sw.data(), sw.size()}; + } + + private: + friend KVEngine; + + List* list_; + const SnapshotImpl* snapshot_; + bool own_snapshot_; + DLListDataIterator dl_iter_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/list_collection/list.cpp b/volatile/engine/list_collection/list.cpp new file mode 100644 index 00000000..ffb7cd1d --- /dev/null +++ b/volatile/engine/list_collection/list.cpp @@ -0,0 +1,464 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "list.hpp" + +namespace KVDK_NAMESPACE { +List::WriteResult List::SetExpireTime(ExpireTimeType expired_time, + TimestampType timestamp) { + WriteResult ret; + DLRecord* header = HeaderRecord(); + SpaceEntry space = kv_allocator_->Allocate( + DLRecord::RecordSize(header->Key(), header->Value())); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space.offset), space.size, timestamp, + RecordType::ListHeader, RecordStatus::Normal, + kv_allocator_->addr2offset_checked(header), header->prev, header->next, + header->Key(), header->Value(), expired_time); + bool success = dl_list_.Replace(header, data_record); + kvdk_assert(success, "existing header should be linked on its list"); + ret.existing_record = header; + ret.write_record = data_record; + return ret; +} + +List::WriteResult List::PushFront(const StringView& elem, TimestampType ts) { + WriteResult ret; + std::string internal_key(InternalKey("")); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, elem)); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + + DLList::WriteArgs args(internal_key, elem, RecordType::ListElem, + RecordStatus::Normal, ts, space); + ret.s = dl_list_.PushFront(args); + kvdk_assert(ret.s == Status::Ok, "Push front should alwasy success"); + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + live_records_.push_front(ret.write_record); + return ret; +} + +List::WriteResult List::PushBack(const StringView& elem, TimestampType ts) { + WriteResult ret; + std::string internal_key(InternalKey("")); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, elem)); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + + DLList::WriteArgs args(internal_key, elem, RecordType::ListElem, + RecordStatus::Normal, ts, space); + ret.s = dl_list_.PushBack(args); + kvdk_assert(ret.s == Status::Ok, "Push front should alwasy success"); + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + live_records_.push_back(ret.write_record); + return ret; +} + +List::WriteResult List::PopFront(TimestampType ts) { + WriteResult ret; + if (Size() == 0) { + ret.s = Status::NotFound; + } else { + DLRecord* record = live_records_.front(); + kvdk_assert(record->GetRecordStatus() == RecordStatus::Normal, ""); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(record->Key(), "")); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLList::WriteArgs args(record->Key(), "", RecordType::ListElem, + RecordStatus::Outdated, ts, space); + while ((ret.s = dl_list_.Update(args, record)) != Status::Ok) { + kvdk_assert(ret.s == Status::Fail, ""); + }; + ret.write_record = + kv_allocator_->offset2addr_checked(space.offset); + ret.existing_record = record; + live_records_.pop_front(); + } + return ret; +} + +List::WriteResult List::PopBack(TimestampType ts) { + WriteResult ret; + if (Size() == 0) { + ret.s = Status::NotFound; + } else { + DLRecord* record = live_records_.back(); + kvdk_assert(record->GetRecordStatus() == RecordStatus::Normal, ""); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(record->Key(), "")); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLList::WriteArgs args(record->Key(), "", RecordType::ListElem, + RecordStatus::Outdated, ts, space); + while ((ret.s = dl_list_.Update(args, record)) != Status::Ok) { + kvdk_assert(ret.s == Status::Fail, ""); + }; + ret.write_record = + kv_allocator_->offset2addr_checked(space.offset); + ret.existing_record = record; + live_records_.pop_back(); + } + return ret; +} + +List::WriteResult List::InsertBefore(const StringView& elem, + const StringView& existing_elem, + TimestampType ts) { + WriteResult ret; + auto iter = findLiveRecord(existing_elem); + if (iter == live_records_.end()) { + ret.s = Status::NotFound; + } else { + std::string internal_key(InternalKey("")); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, elem)); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLList::WriteArgs args(internal_key, elem, RecordType::ListElem, + RecordStatus::Normal, ts, space); + ret.s = dl_list_.InsertBefore(args, *iter); + kvdk_assert(ret.s == Status::Ok, + "the whole list is locked, so the insertion must be success"); + ret.write_record = + kv_allocator_->offset2addr_checked(space.offset); + live_records_.insert(iter, ret.write_record); + } + return ret; +} + +List::WriteResult List::InsertAfter(const StringView& elem, + const StringView& existing_elem, + TimestampType ts) { + WriteResult ret; + auto iter = findLiveRecord(existing_elem); + if (iter == live_records_.end()) { + ret.s = Status::NotFound; + } else { + std::string internal_key(InternalKey("")); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, elem)); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLList::WriteArgs args(internal_key, elem, RecordType::ListElem, + RecordStatus::Normal, ts, space); + ret.s = dl_list_.InsertAfter(args, *iter); + kvdk_assert(ret.s == Status::Ok, + "the whole list is locked, so the insertion must be success"); + ret.write_record = + kv_allocator_->offset2addr_checked(space.offset); + live_records_.insert(iter + 1, ret.write_record); + } + return ret; +} + +List::WriteResult List::InsertAt(const StringView& elem, long index, + TimestampType ts) { + WriteResult ret; + size_t required_size = index < 0 ? std::abs(index) - 1 : index; + if (required_size > Size()) { + ret.s = Status::NotFound; + return ret; + } + if (index < 0) { + index = live_records_.size() + index; + } + auto iter = live_records_.begin() + index; + DLRecord* next = *iter; + std::string internal_key(InternalKey("")); + kvdk_assert(next->GetRecordType() == RecordType::ListElem && + next->GetRecordStatus() == RecordStatus::Normal, + ""); + + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, elem)); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLList::WriteArgs args(internal_key, elem, RecordType::ListElem, + RecordStatus::Normal, ts, space); + ret.s = dl_list_.InsertBefore(args, next); + kvdk_assert(ret.s == Status::Ok, + "the whole list is locked, so the insertion must be success"); + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + live_records_.insert(iter, ret.write_record); + return ret; +} + +List::WriteResult List::Erase(long index, TimestampType ts) { + WriteResult ret; + size_t required_size = index < 0 ? std::abs(index) - 1 : index; + if (required_size > Size()) { + ret.s = Status::NotFound; + return ret; + } + if (index < 0) { + index = live_records_.size() + index; + } + auto iter = live_records_.begin() + index; + DLRecord* record = *iter; + kvdk_assert(record->GetRecordType() == RecordType::ListElem && + record->GetRecordStatus() == RecordStatus::Normal, + ""); + + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(record->Key(), "")); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLList::WriteArgs args(record->Key(), "", RecordType::ListElem, + RecordStatus::Outdated, ts, space); + while ((ret.s = dl_list_.Update(args, record)) != Status::Ok) { + kvdk_assert(ret.s == Status::Fail, ""); + }; + ret.existing_record = record; + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + live_records_.erase(iter); + return ret; +} + +Status List::Front(std::string* elem) { + if (Size() == 0) { + return Status::NotFound; + } + DLRecord* record = live_records_.front(); + StringView sw = record->Value(); + elem->assign(sw.data(), sw.size()); + return Status::Ok; +} + +Status List::Back(std::string* elem) { + if (Size() == 0) { + return Status::NotFound; + } + DLRecord* record = live_records_.back(); + StringView sw = record->Value(); + elem->assign(sw.data(), sw.size()); + return Status::Ok; +} + +List::WriteResult List::Update(long index, const StringView& elem, + TimestampType ts) { + WriteResult ret; + size_t required_size = index < 0 ? std::abs(index) - 1 : index; + if (required_size > Size()) { + ret.s = Status::NotFound; + return ret; + } + if (index < 0) { + index = live_records_.size() + index; + } + auto iter = live_records_.begin() + index; + DLRecord* record = *iter; + kvdk_assert(record->GetRecordType() == RecordType::ListElem && + record->GetRecordStatus() == RecordStatus::Normal, + ""); + std::string internal_key(InternalKey("")); + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, elem)); + if (space.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLList::WriteArgs args(internal_key, elem, RecordType::ListElem, + RecordStatus::Normal, ts, space); + + while ((ret.s = dl_list_.Update(args, record)) != Status::Ok) { + kvdk_assert(ret.s == Status::Fail, ""); + } + ret.existing_record = record; + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + *iter = ret.write_record; + return ret; +} + +List::PushNArgs List::PreparePushN(ListPos pos, + const std::vector& elems, + TimestampType ts) { + PushNArgs args; + args.pos = pos; + args.ts = ts; + if (elems.size() > 0) { + std::string internal_key(InternalKey("")); + for (auto& elem : elems) { + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(internal_key, elem)); + if (space.size == 0) { + GlobalLogger.Error("Try allocate %lu error\n", + DLRecord::RecordSize(internal_key, elem)); + for (auto& sp : args.spaces) { + kv_allocator_->Free(sp); + } + args.s = Status::MemoryOverflow; + break; + } + args.spaces.emplace_back(space); + } + args.elems = elems; + } + args.s = Status::Ok; + return args; +} + +List::PopNArgs List::PreparePopN(ListPos pos, size_t n, TimestampType ts, + std::vector* elems) { + size_t nn = n; + PopNArgs args; + args.timestamp_ = ts; + if (Size() > 0) { + auto iter = + pos == ListPos::Front ? live_records_.begin() : live_records_.end() - 1; + while (nn > 0) { + DLRecord* record = *iter; + SpaceEntry space = + kv_allocator_->Allocate(DLRecord::RecordSize(record->Key(), "")); + if (space.size == 0) { + for (auto& sp : args.spaces) { + kv_allocator_->Free(sp); + } + args.s = Status::MemoryOverflow; + return args; + } + + if (elems) { + StringView sw = record->Value(); + elems->emplace_back(sw.data(), sw.size()); + } + args.spaces.emplace_back(space); + args.to_pop_.emplace_back(iter); + nn--; + if (pos == ListPos::Front) { + iter++; + if (iter == live_records_.end()) { + break; + } + } else { + if (iter == live_records_.begin()) { + break; + } + iter--; + } + } + args.s = Status::Ok; + } + return args; +} + +Status List::PushN(const List::PushNArgs& args) { + if (args.s != Status::Ok) { + return args.s; + } + std::string internal_key(InternalKey("")); + kvdk_assert(args.elems.size() == args.spaces.size(), ""); + for (size_t i = 0; i < args.elems.size(); i++) { + DLList::WriteArgs wa(internal_key, args.elems[i], RecordType::ListElem, + RecordStatus::Normal, args.ts, args.spaces[i]); + Status s; + if (args.pos == ListPos::Front) { + s = dl_list_.PushFront(wa); + live_records_.push_front( + kv_allocator_->offset2addr_checked(wa.space.offset)); + } else { + s = dl_list_.PushBack(wa); + live_records_.push_back( + kv_allocator_->offset2addr_checked(wa.space.offset)); + } + kvdk_assert(s == Status::Ok, "Push back/front should always success"); + TEST_CRASH_POINT("List::PushN", ""); + } + return Status::Ok; +} + +Status List::PopN(const List::PopNArgs& args) { + if (args.s != Status::Ok) { + return args.s; + } + std::string internal_key(InternalKey("")); + kvdk_assert(args.spaces.size() == args.to_pop_.size(), ""); + for (size_t i = 0; i < args.to_pop_.size(); i++) { + DLList::WriteArgs wa(internal_key, "", RecordType::ListElem, + RecordStatus::Outdated, args.timestamp_, + args.spaces[i]); + Status s; + while ((s = dl_list_.Update(wa, *args.to_pop_[i])) != Status::Ok) { + kvdk_assert(s == Status::Fail, ""); + } + live_records_.erase(args.to_pop_[i]); + TEST_CRASH_POINT("List::PopN", ""); + } + return Status::Ok; +} + +void List::Destroy() { + std::vector to_free; + DLRecord* header = HeaderRecord(); + if (header) { + DLRecord* to_destroy = nullptr; + do { + to_destroy = kv_allocator_->offset2addr_checked(header->next); + if (dl_list_.Remove(to_destroy)) { + to_destroy->Destroy(); + to_free.emplace_back(kv_allocator_->addr2offset_checked(to_destroy), + to_destroy->GetRecordSize()); + if (to_free.size() > kMaxCachedOldRecords) { + kv_allocator_->BatchFree(to_free); + to_free.clear(); + } + } + } while (to_destroy != header); + } + kv_allocator_->BatchFree(to_free); +} + +void List::DestroyAll() { + std::vector to_free; + DLRecord* header = HeaderRecord(); + if (header) { + DLRecord* to_destroy = nullptr; + do { + to_destroy = kv_allocator_->offset2addr_checked(header->next); + if (dl_list_.Remove(to_destroy)) { + auto old_record = + kv_allocator_->offset2addr(to_destroy->old_version); + while (old_record) { + auto old_version = old_record->old_version; + old_record->Destroy(); + to_free.emplace_back(kv_allocator_->addr2offset_checked(old_record), + old_record->GetRecordSize()); + old_record = kv_allocator_->offset2addr(old_version); + } + + to_destroy->Destroy(); + to_free.emplace_back(kv_allocator_->addr2offset_checked(to_destroy), + to_destroy->GetRecordSize()); + if (to_free.size() > kMaxCachedOldRecords) { + kv_allocator_->BatchFree(to_free); + to_free.clear(); + } + } + } while (to_destroy != header); + } + kv_allocator_->BatchFree(to_free); +} +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/list_collection/list.hpp b/volatile/engine/list_collection/list.hpp new file mode 100644 index 00000000..d3dd5402 --- /dev/null +++ b/volatile/engine/list_collection/list.hpp @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include "../dl_list.hpp" +#include "../logger.hpp" +#include "kvdk/volatile/types.hpp" + +namespace KVDK_NAMESPACE { +class ListIteratorImpl; + +class List : public Collection { + public: + List(DLRecord* header, const StringView& name, CollectionIDType id, + Allocator* kv_allocator, LockTable* lock_table) + : Collection(name, id), + list_lock_(), + dl_list_(header, kv_allocator, lock_table), + kv_allocator_(kv_allocator), + live_records_() {} + + struct WriteResult { + Status s = Status::Ok; + DLRecord* write_record = nullptr; + DLRecord* existing_record = nullptr; + }; + + struct PopNArgs { + public: + Status s{Status::InvalidArgument}; + std::vector spaces{}; + + private: + friend List; + std::vector::iterator> to_pop_{}; + TimestampType timestamp_; + }; + + struct PushNArgs { + public: + Status s{Status::InvalidArgument}; + std::vector spaces; + std::vector elems; + ListPos pos; + TimestampType ts; + }; + + const DLRecord* HeaderRecord() const { return dl_list_.Header(); } + + DLRecord* HeaderRecord() { return dl_list_.Header(); } + + ExpireTimeType GetExpireTime() const final { + return HeaderRecord()->GetExpireTime(); + } + + TimestampType GetTimeStamp() const { return HeaderRecord()->GetTimestamp(); } + + bool HasExpired() const final { return HeaderRecord()->HasExpired(); } + + WriteResult SetExpireTime(ExpireTimeType expired_time, + TimestampType timestamp); + + WriteResult PushFront(const StringView& elem, TimestampType ts); + + WriteResult PushBack(const StringView& elem, TimestampType ts); + + WriteResult PopFront(TimestampType ts); + + WriteResult PopBack(TimestampType ts); + + WriteResult InsertBefore(const StringView& elem, + const StringView& existing_elem, TimestampType ts); + + WriteResult InsertAfter(const StringView& elem, + const StringView& existing_elem, TimestampType ts); + + WriteResult InsertAt(const StringView& elem, long index, TimestampType ts); + + WriteResult Erase(long index, TimestampType ts); + + Status Front(std::string* elem); + + Status Back(std::string* elem); + + bool Replace(DLRecord* old_record, DLRecord* new_record) { + return dl_list_.Replace(old_record, new_record); + } + + WriteResult Update(long index, const StringView& elem, TimestampType ts); + + void AddLiveRecord(DLRecord* elem, ListPos pos) { + if (pos == ListPos::Front) { + live_records_.push_front(elem); + } else { + live_records_.push_back(elem); + } + } + + size_t Size() { return live_records_.size(); } + + std::unique_lock AcquireLock() { + return std::unique_lock(list_lock_); + } + + DLList* GetDLList() { return &dl_list_; } + + void DestroyAll(); + + void Destroy(); + + PushNArgs PreparePushN(ListPos pos, const std::vector& elems, + TimestampType ts); + + PopNArgs PreparePopN(ListPos pos, size_t n, TimestampType ts, + std::vector* elems); + + Status PushN(const PushNArgs& args); + + Status PopN(const PopNArgs& args); + + bool TryCleaningLock() { return cleaning_lock_.try_lock(); } + + void ReleaseCleaningLock() { cleaning_lock_.unlock(); } + + static CollectionIDType FetchID(DLRecord* record) { + assert(record != nullptr); + switch (record->GetRecordType()) { + case RecordType::ListElem: + return ExtractID(record->Key()); + case RecordType::ListHeader: + return DecodeID(record->Value()); + default: + GlobalLogger.Error("Wrong record type %u in ListID", + record->GetRecordType()); + kvdk_assert(false, "Wrong type in ListID"); + return 0; + } + } + + static bool MatchType(const DLRecord* record) { + RecordType type = record->GetRecordType(); + return type == RecordType::ListElem || type == RecordType::ListHeader; + } + + private: + // find the first live record of elem + std::deque::iterator findLiveRecord(StringView elem) { + auto iter = live_records_.begin(); + while (iter != live_records_.end()) { + if (equal_string_view((*iter)->Value(), elem)) { + return iter; + } + ++iter; + } + return live_records_.end(); + } + + friend ListIteratorImpl; + std::recursive_mutex list_lock_; + DLList dl_list_; + Allocator* kv_allocator_; + std::atomic size_; + // to avoid illegal access caused by cleaning skiplist by multi-thread + SpinMutex cleaning_lock_; + // we keep outdated records on list to support mvcc, so we track live records + // in a deque to support fast write operations + std::deque live_records_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/lock_table.hpp b/volatile/engine/lock_table.hpp new file mode 100644 index 00000000..453bac1a --- /dev/null +++ b/volatile/engine/lock_table.hpp @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#pragma once + +#include +#include + +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { + +// This class manages a table of locks, which indexed by hash value. It is able +// to lock multiple locks in the table by sequence to avoid dead lock +class LockTable { + public: + using HashValueType = std::uint64_t; + using MutexType = SpinMutex; + using ULockType = std::unique_lock; + using MultiGuardType = std::vector; + + LockTable(size_t n) : mutexes_{n} {} + + std::unique_lock AcquireLock(HashValueType hash) { + return std::unique_lock(*Mutex(hash)); + } + + void Lock(HashValueType hash) { Mutex(hash)->lock(); } + + void Unlock(HashValueType hash) { Mutex(hash)->unlock(); } + + void MultiLock(std::vector const& hashes) { + auto sorted = rearrange(hashes); + for (HashValueType hash : sorted) { + Lock(hash); + } + } + + void MultiLock(std::initializer_list hashes) { + MultiLock(std::vector{hashes}); + } + + void MultiUnlock(std::vector const& hashes) { + auto sorted = rearrange(hashes); + for (HashValueType hash : sorted) { + Unlock(hash); + } + } + + void MultiUnlock(std::initializer_list hashes) { + MultiUnlock(std::vector{hashes}); + } + + MultiGuardType MultiGuard(std::vector const& hashes) { + auto sorted = rearrange(hashes); + MultiGuardType guard; + for (HashValueType hash : sorted) { + guard.emplace_back(*Mutex(hash)); + } + return guard; + } + + MultiGuardType MultiGuard(std::initializer_list hashes) { + return MultiGuard(std::vector{hashes}); + } + + MutexType* Mutex(HashValueType hash) { + return &mutexes_[hash % mutexes_.size()]; + } + + private: + std::vector rearrange( + std::vector const& hashes) { + std::vector ret{hashes}; + size_t N = mutexes_.size(); + for (auto& hash : ret) { + hash %= N; + } + std::sort(ret.begin(), ret.end()); + size_t new_sz = std::unique(ret.begin(), ret.end()) - ret.begin(); + ret.resize(new_sz); + return ret; + } + + std::vector mutexes_; +}; + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/logger.cpp b/volatile/engine/logger.cpp new file mode 100644 index 00000000..b0f6c9f6 --- /dev/null +++ b/volatile/engine/logger.cpp @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ +#include "logger.hpp" + +namespace KVDK_NAMESPACE { + +void Logger::Debug(const char* format, ...) { + if (level_ <= LogLevel::Debug) { + va_list args; + va_start(args, format); + Log("[DEBUG]", format, args); + va_end(args); + } +} + +void Logger::Info(const char* format, ...) { + if (level_ <= LogLevel::Info) { + va_list args; + va_start(args, format); + Log("[INFO]", format, args); + va_end(args); + } +} + +void Logger::Error(const char* format, ...) { + if (level_ <= LogLevel::Error) { + va_list args; + va_start(args, format); + Log("[ERROR]", format, args); + va_end(args); + } +} + +void Logger::Log(const char* log_type, const char* format, va_list& args) { + if (log_file_ != nullptr) { + std::lock_guard lg(mut_); + auto now = std::chrono::system_clock::now(); + auto duration = + std::chrono::duration_cast(now - start_ts_); + fprintf(log_file_, "%s time %ld ms: ", log_type, duration.count()); + vfprintf(log_file_, format, args); + fflush(log_file_); + fsync(fileno(log_file_)); + } +} + +void Logger::Init(FILE* fp, LogLevel level) { + log_file_ = fp; + level_ = level; + start_ts_ = std::chrono::system_clock::now(); +} + +Logger GlobalLogger; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/engine/logger.hpp b/volatile/engine/logger.hpp similarity index 94% rename from engine/logger.hpp rename to volatile/engine/logger.hpp index 6bf978a1..315f7d8d 100644 --- a/engine/logger.hpp +++ b/volatile/engine/logger.hpp @@ -11,7 +11,7 @@ #include #include "alias.hpp" -#include "kvdk/configs.hpp" +#include "kvdk/volatile/configs.hpp" #define DO_LOG 1 diff --git a/volatile/engine/macros.hpp b/volatile/engine/macros.hpp new file mode 100644 index 00000000..e6e73520 --- /dev/null +++ b/volatile/engine/macros.hpp @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#pragma once + +#include +#include +#include + +#define to_hex(x) \ + std::hex << std::setfill('0') << std::setw(sizeof(decltype(x)) * 2) << x \ + << std::dec + +#ifndef KVDK_DEBUG_LEVEL +#pragma GCC warning "KVDK_DEBUG_LEVEL not defined, defaulted to 0" +#define KVDK_DEBUG_LEVEL 0 +#endif + +#define kvdk_assert(cond, msg) \ + { \ + if (KVDK_DEBUG_LEVEL > 0 && !(cond)) { \ + throw std::runtime_error{__FILE__ ":" + std::to_string(__LINE__) + \ + ":\t" + std::string{msg}}; \ + } \ + } + +#ifdef __GNUC__ +#define KVDK_LIKELY(x) __builtin_expect(!!(x), 1) +#define KVDK_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define KVDK_LIKELY(x) (x) +#define KVDK_UNLIKELY(x) (x) +#endif diff --git a/volatile/engine/snapshot.hpp b/volatile/engine/snapshot.hpp new file mode 100644 index 00000000..a16ce1c2 --- /dev/null +++ b/volatile/engine/snapshot.hpp @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { +struct Snapshot { + TimestampType timestamp; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/sorted_collection/iterator.hpp b/volatile/engine/sorted_collection/iterator.hpp new file mode 100644 index 00000000..477594f6 --- /dev/null +++ b/volatile/engine/sorted_collection/iterator.hpp @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include "../alias.hpp" +#include "skiplist.hpp" + +namespace KVDK_NAMESPACE { + +class KVEngine; + +class SortedIteratorImpl : public SortedIterator { + public: + SortedIteratorImpl(Skiplist* skiplist, const Allocator* kv_allocator, + const SnapshotImpl* snapshot, bool own_snapshot) + : skiplist_(skiplist), + snapshot_(snapshot), + own_snapshot_(own_snapshot), + dl_iter_(&skiplist->dl_list_, kv_allocator, snapshot) {} + + virtual ~SortedIteratorImpl() = default; + + virtual void Seek(const std::string& key) override { + assert(skiplist_); + Splice splice(skiplist_); + skiplist_->Seek(key, &splice); + dl_iter_.Locate(splice.next_data_record, true); + } + + virtual void SeekToFirst() override { dl_iter_.SeekToFirst(); } + + virtual void SeekToLast() override { dl_iter_.SeekToLast(); } + + virtual bool Valid() override { return dl_iter_.Valid(); } + + virtual void Next() override { dl_iter_.Next(); } + + virtual void Prev() override { dl_iter_.Prev(); } + + virtual std::string Key() override { + if (!Valid()) return ""; + return string_view_2_string(Skiplist::ExtractUserKey(dl_iter_.Key())); + } + + virtual std::string Value() override { + if (!Valid()) return ""; + return string_view_2_string(dl_iter_.Value()); + } + + private: + friend KVEngine; + + Skiplist* skiplist_; + const SnapshotImpl* snapshot_; + bool own_snapshot_; + DLListDataIterator dl_iter_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/sorted_collection/skiplist.cpp b/volatile/engine/sorted_collection/skiplist.cpp new file mode 100644 index 00000000..80035096 --- /dev/null +++ b/volatile/engine/sorted_collection/skiplist.cpp @@ -0,0 +1,950 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "skiplist.hpp" + +#include +#include + +#include "../kv_engine.hpp" +#include "../utils/codec.hpp" +#include "../utils/sync_point.hpp" +#include "../write_batch_impl.hpp" + +namespace KVDK_NAMESPACE { + +StringView SkiplistNode::UserKey() { return Skiplist::UserKey(this); } + +uint64_t SkiplistNode::SkiplistID() { return Skiplist::FetchID(this); } + +Skiplist::~Skiplist() { + destroyNodes(); + std::lock_guard lg_a(pending_delete_nodes_spin_); + for (SkiplistNode* node : pending_deletion_nodes_) { + SkiplistNode::DeleteNode(node, node_allocator_); + } + pending_deletion_nodes_.clear(); + std::lock_guard lg_b(obsolete_nodes_spin_); + for (SkiplistNode* node : obsolete_nodes_) { + SkiplistNode::DeleteNode(node, node_allocator_); + } + obsolete_nodes_.clear(); +} + +Skiplist::Skiplist(DLRecord* h, const std::string& name, CollectionIDType id, + Comparator comparator, Allocator* kv_allocator, + Allocator* node_allocator, HashTable* hash_table, + LockTable* lock_table, bool index_with_hashtable) + : Collection(name, id), + dl_list_(h, kv_allocator, lock_table), + size_(0), + comparator_(comparator), + kv_allocator_(kv_allocator), + node_allocator_(node_allocator), + hash_table_(hash_table), + record_locks_(lock_table), + index_with_hashtable_(index_with_hashtable) { + header_ = SkiplistNode::NewNode(name, h, kMaxHeight, node_allocator_); + + for (uint8_t i = 1; i <= kMaxHeight; i++) { + header_->RelaxedSetNext(i, nullptr); + } +}; + +Skiplist::WriteResult Skiplist::SetExpireTime(ExpireTimeType expired_time, + TimestampType timestamp) { + WriteResult ret; + DLRecord* header = HeaderRecord(); + auto request_size = + sizeof(DLRecord) + header->Key().size() + header->Value().size(); + SpaceEntry space_entry = kv_allocator_->Allocate(request_size); + if (space_entry.size == 0) { + ret.s = Status::MemoryOverflow; + return ret; + } + DLRecord* data_record = DLRecord::ConstructDLRecord( + kv_allocator_->offset2addr_checked(space_entry.offset), space_entry.size, + timestamp, RecordType::SortedHeader, RecordStatus::Normal, + kv_allocator_->addr2offset_checked(header), header->prev, header->next, + header->Key(), header->Value(), expired_time); + bool success = Skiplist::Replace(header, data_record, HeaderNode(), + kv_allocator_, record_locks_); + kvdk_assert(success, "existing header should be linked on its skiplist"); + ret.existing_record = header; + ret.dram_node = HeaderNode(); + ret.write_record = data_record; + return ret; +} + +void Skiplist::SeekNode(const StringView& key, SkiplistNode* start_node, + uint8_t start_height, uint8_t end_height, + Splice* result_splice) { + std::vector to_delete; + assert(start_node->height >= start_height && end_height >= 1); + SkiplistNode* prev = start_node; + PointerWithTag next; + for (uint8_t i = start_height; i >= end_height; i--) { + while (1) { + next = prev->Next(i); + // prev is logically deleted, roll back to prev height. + if (next.GetTag() == SkiplistNode::NodeStatus::Deleted) { + if (i < start_height) { + i++; + prev = result_splice->prevs[i]; + } else if (prev != start_node) { + // re-seek from this node for start height + i = start_height; + prev = start_node; + } else { + // this node has been deleted, so seek from header + kvdk_assert(result_splice->seeking_list != nullptr, + "skiplist must be set for seek operation!"); + return SeekNode(key, result_splice->seeking_list->HeaderNode(), + kMaxHeight, end_height, result_splice); + } + continue; + } + + if (next.Null()) { + result_splice->nexts[i] = nullptr; + result_splice->prevs[i] = prev; + break; + } + + // Physically remove deleted "next" nodes from skiplist + auto next_next = next->Next(i); + if (next_next.GetTag() == SkiplistNode::NodeStatus::Deleted) { + if (prev->CASNext(i, next, next_next.RawPointer())) { + if (--next->valid_links == 0) { + to_delete.push_back(next.RawPointer()); + } + } + // if prev is marked deleted before cas, cas will be failed, and prev + // will be roll back in next round + continue; + } + + DLRecord* next_data_record = next->record; + int cmp = Compare(key, next->UserKey()); + // pmem record maybe updated before comparing string, then the compare + // result will be invalid, so we need to do double check + if (next->record != next_data_record) { + continue; + } + + if (cmp > 0) { + prev = next.RawPointer(); + } else { + result_splice->nexts[i] = next.RawPointer(); + result_splice->prevs[i] = prev; + break; + } + } + } + if (to_delete.size() > 0) { + result_splice->seeking_list->obsoleteNodes(to_delete); + } +} + +void Skiplist::linkDLRecord(DLRecord* prev, DLRecord* next, DLRecord* linking, + Allocator* kv_allocator) { + uint64_t inserting_record_offset = kv_allocator->addr2offset(linking); + prev->next = inserting_record_offset; + + TEST_SYNC_POINT("KVEngine::DLList::LinkDLRecord::HalfLink"); + next->prev = inserting_record_offset; +} + +void Skiplist::Seek(const StringView& key, Splice* result_splice) { + result_splice->seeking_list = this; + SeekNode(key, header_, header_->Height(), 1, result_splice); + assert(result_splice->prevs[1] != nullptr); + DLRecord* prev_record = result_splice->prevs[1]->record; + DLRecord* next_record = nullptr; + while (1) { + next_record = kv_allocator_->offset2addr(prev_record->next); + if (next_record == HeaderRecord()) { + break; + } + + if (next_record == nullptr) { + return Seek(key, result_splice); + } + int cmp = Compare(key, UserKey(next_record)); + // pmem record maybe updated before comparing string, then the comparing + // result will be invalid, so we need to do double check + // + // Notice: In current implementation with the guard of snapshot mechanism, + // the record won't be freed during this operation, so this should not be + // happen + // if (!validateDLRecord(next_record)) { + // return Seek(key, result_splice); + // } + + if (cmp > 0) { + prev_record = next_record; + } else { + break; + } + } + result_splice->next_data_record = next_record; + result_splice->prev_data_record = prev_record; +} + +Status Skiplist::CheckIndex() { + DLListRecoveryUtils recovery_utils(kv_allocator_); + Splice splice(this); + splice.prev_data_record = HeaderRecord(); + for (uint8_t i = 1; i <= kMaxHeight; i++) { + splice.prevs[i] = header_; + } + + while (true) { + DLRecord* next_record = kv_allocator_->offset2addr_checked( + splice.prev_data_record->next); + if (next_record == HeaderRecord()) { + break; + } + SkiplistNode* next_node = splice.prevs[1]->RelaxedNext(1).RawPointer(); + if (IndexWithHashtable()) { + StringView key = next_record->Key(); + auto ret = hash_table_->Lookup(key, next_record->GetRecordType()); + if (ret.s != Status::Ok) { + GlobalLogger.Error( + "Check skiplist index error: record not exist in hash table\n"); + return Status::Abort; + } + + if (ret.entry.GetIndexType() == PointerType::SkiplistNode) { + if (ret.entry.GetIndex().skiplist_node != next_node) { + GlobalLogger.Error( + "Check skiplist index error: Dram node miss-match with hash " + "table\n"); + return Status::Abort; + } + } else { + if (ret.entry.GetIndex().dl_record != next_record) { + GlobalLogger.Error( + "Check skiplist index error: Dlrecord miss-match with hash " + "table\n"); + return Status::Abort; + } + } + } + + // Check dram linkage + if (next_node && next_node->record == next_record) { + for (uint8_t i = 1; i <= next_node->Height(); i++) { + if (splice.prevs[i]->RelaxedNext(i).RawPointer() != next_node) { + GlobalLogger.Error( + "Check skiplist index error: node linkage error\n"); + return Status::Abort; + } + splice.prevs[i] = next_node; + } + } + if (!recovery_utils.CheckLinkage(next_record)) { + return Status::Abort; + } + splice.prev_data_record = next_record; + } + + return Status::Ok; +} + +LockTable::MultiGuardType Skiplist::lockRecordPosition(const DLRecord* record, + Allocator* kv_allocator, + LockTable* lock_table) { + while (1) { + MemoryOffsetType prev_offset = record->prev; + MemoryOffsetType next_offset = record->next; + DLRecord* prev = kv_allocator->offset2addr_checked(prev_offset); + + auto guard = lock_table->MultiGuard({recordHash(prev), recordHash(record)}); + + // Check if the linkage has changed before we successfully acquire lock. + if (record->prev != prev_offset || record->next != next_offset) { + continue; + } + + return guard; + } +} + +bool Skiplist::lockInsertPosition(const StringView& inserting_key, + DLRecord* prev_record, DLRecord* next_record, + LockTable::ULockType* prev_record_lock) { + MemoryOffsetType prev_offset = + kv_allocator_->addr2offset_checked(prev_record); + MemoryOffsetType next_offset = + kv_allocator_->addr2offset_checked(next_record); + *prev_record_lock = record_locks_->AcquireLock(recordHash(prev_record)); + + // Check if the linkage has changed before we successfully acquire lock. + auto check_linkage = [&]() { + return prev_record->next == next_offset && next_record->prev == prev_offset; + }; + // Check id and order as prev and next may be both freed, then inserted + // to another position while keep linkage, before we lock them + // For example: + // Before lock: + // this skip list: record1 -> "prev" -> "next" -> record2 + // After lock: + // this skip list: "new record reuse prev" -> "new record reuse next" -> + // record1 -> record2 + // or: + // this skip list: record1 -> record2 + // another skip list:"new record reuse prev" -> "new record reuse next" -> + // In this case, inserting record will be mis-inserted between "new record + // reuse prev" and "new record reuse next" + // + // Notice: In current implementation with the guard of snapshot mechanism, the + // prev and next won't be freed during this operation, so id and order won't + // be changed anymore. We only check id and order in debug mode + auto check_id = [&]() { + return FetchID(next_record) == ID() && FetchID(prev_record) == ID(); + }; + + auto check_order = [&]() { + bool res = + /*check next*/ (next_record->GetRecordType() == + RecordType::SortedHeader || + Compare(inserting_key, UserKey(next_record)) <= 0) && + /*check prev*/ (prev_record->GetRecordType() == + RecordType::SortedHeader || + Compare(inserting_key, UserKey(prev_record)) > 0); + return res; + }; + if (!check_linkage()) { + *prev_record_lock = std::unique_lock(); + return false; + } + + kvdk_assert(check_id(), + "Check id of prev and next failed during skiplist insert\n"); + kvdk_assert(check_order(), + "Check key order of prev and next failed during skiplist " + "insert\n"); + + assert(prev_record->next == next_offset); + assert(next_record->prev == prev_offset); + + return true; +} + +Skiplist::WriteResult Skiplist::Write(SortedWriteArgs& args) { + WriteResult ret; + if (args.skiplist != this) { + ret.s = Status::InvalidArgument; + return ret; + } + if (args.op == WriteOp::Put) { + if (IndexWithHashtable()) { + ret = putPreparedWithHash(args.lookup_result, args.key, args.value, + args.ts, args.space); + } else { + kvdk_assert(args.seek_result != nullptr, ""); + ret = putPreparedNoHash(*args.seek_result, args.key, args.value, args.ts, + args.space); + } + if (ret.existing_record == nullptr || + ret.existing_record->GetRecordStatus() == RecordStatus::Outdated) { + UpdateSize(1); + } + } else { + if (IndexWithHashtable()) { + ret = deletePreparedWithHash(args.lookup_result, args.key, args.ts, + args.space); + } else { + kvdk_assert(args.seek_result != nullptr, ""); + DLRecord* existing_record = args.seek_result->next_data_record; + SkiplistNode* dram_node = nullptr; + if (args.seek_result->nexts[1] && + args.seek_result->nexts[1]->record == existing_record) { + dram_node = args.seek_result->nexts[1]; + } + ret = deletePreparedNoHash(existing_record, dram_node, args.key, args.ts, + args.space); + } + + if (ret.existing_record != nullptr && + ret.existing_record->GetRecordStatus() == RecordStatus::Normal) { + UpdateSize(-1); + } + } + return ret; +} + +SortedWriteArgs Skiplist::InitWriteArgs(const StringView& key, + const StringView& value, WriteOp op) { + SortedWriteArgs args; + args.collection = Name(); + args.skiplist = this; + args.key = key; + args.value = value; + args.op = op; + return args; +} + +Status Skiplist::PrepareWrite(SortedWriteArgs& args, TimestampType ts) { + kvdk_assert(args.op == WriteOp::Put || args.value.size() == 0, + "value of delete operation should be empty"); + if (args.skiplist != this) { + return Status::InvalidArgument; + } + bool op_delete = args.op == WriteOp::Delete; + std::string internal_key(InternalKey(args.key)); + bool allocate_space = true; + if (IndexWithHashtable()) { + if (op_delete) { + args.lookup_result = + hash_table_->Lookup(internal_key, RecordType::SortedElem); + } else { + args.lookup_result = + hash_table_->Lookup(internal_key, RecordType::SortedElem); + } + switch (args.lookup_result.s) { + case Status::Ok: { + if (op_delete && args.lookup_result.entry.GetRecordStatus() == + RecordStatus::Outdated) { + allocate_space = false; + } + break; + } + case Status::NotFound: { + if (op_delete) { + allocate_space = false; + } + break; + } + case Status::MemoryOverflow: { + return args.lookup_result.s; + } + default: + std::abort(); // never should reach + } + } else { + args.seek_result = std::unique_ptr(new Splice(args.skiplist)); + Seek(args.key, args.seek_result.get()); + auto key_exist = [&]() { + auto type = args.seek_result->next_data_record->GetRecordType(); + auto status = args.seek_result->next_data_record->GetRecordStatus(); + return type == RecordType::SortedElem && status == RecordStatus::Normal && + equal_string_view(args.seek_result->next_data_record->Key(), + internal_key); + }; + if (op_delete && !key_exist()) { + allocate_space = false; + } + } + + if (allocate_space) { + auto request_size = DLRecord::RecordSize(internal_key, args.value); + args.space = kv_allocator_->Allocate(request_size); + if (args.space.size == 0) { + return Status::MemoryOverflow; + } + } + + args.ts = ts; + return Status::Ok; +} + +Skiplist::WriteResult Skiplist::Delete(const StringView& key, + TimestampType timestamp) { + WriteResult ret; + SortedWriteArgs args = InitWriteArgs(key, "", WriteOp::Delete); + ret.s = PrepareWrite(args, timestamp); + if (ret.s == Status::Ok && args.space.size > 0) { + ret = Write(args); + } + return ret; +} + +Skiplist::WriteResult Skiplist::Put(const StringView& key, + const StringView& value, + TimestampType timestamp) { + WriteResult ret; + SortedWriteArgs args = InitWriteArgs(key, value, WriteOp::Put); + ret.s = PrepareWrite(args, timestamp); + if (ret.s == Status::Ok) { + ret = Write(args); + } + return ret; +} + +bool Skiplist::Replace(DLRecord* old_record, DLRecord* new_record, + SkiplistNode* dram_node, Allocator* kv_allocator, + LockTable* lock_table) { + bool ok = DLList::Replace(old_record, new_record, kv_allocator, lock_table); + if (ok && dram_node != nullptr) { + kvdk_assert(dram_node->record == old_record, + "Dram node not belong to old record in Skiplist::Replace"); + dram_node->record = new_record; + } + return ok; +} + +bool Skiplist::Remove(DLRecord* removing_record, SkiplistNode* dram_node, + Allocator* kv_allocator, LockTable* lock_table) { + bool ok = DLList::Remove(removing_record, kv_allocator, lock_table); + if (ok && dram_node) { + dram_node->MarkAsDeleted(); + } + return ok; +} + +SkiplistNode* Skiplist::NewNodeBuild(DLRecord* data_record, Allocator* alloc) { + SkiplistNode* dram_node = nullptr; + auto height = Skiplist::randomHeight(); + if (height > 0) { + StringView user_key = UserKey(data_record); + dram_node = SkiplistNode::NewNode(user_key, data_record, height, alloc); + if (dram_node == nullptr) { + GlobalLogger.Error("Memory overflow in Skiplist::NewNodeBuild\n"); + } + } + return dram_node; +} + +std::string Skiplist::EncodeSortedCollectionValue( + CollectionIDType id, const SortedCollectionConfigs& s_configs) { + std::string value_str; + + AppendUint64(&value_str, id); + AppendFixedString(&value_str, s_configs.comparator_name); + AppendUint32(&value_str, s_configs.index_with_hashtable); + + return value_str; +} + +Status Skiplist::DecodeSortedCollectionValue( + StringView value_str, CollectionIDType& id, + SortedCollectionConfigs& s_configs) { + if (!FetchUint64(&value_str, &id)) { + return Status::Abort; + } + if (!FetchFixedString(&value_str, &s_configs.comparator_name)) { + return Status::Abort; + } + if (!FetchUint32(&value_str, (uint32_t*)&s_configs.index_with_hashtable)) { + return Status::Abort; + } + + return Status::Ok; +} + +Status Skiplist::Get(const StringView& key, std::string* value) { + if (!IndexWithHashtable()) { + Splice splice(this); + Seek(key, &splice); + auto type = splice.next_data_record->GetRecordType(); + auto status = splice.next_data_record->GetRecordStatus(); + if (type == RecordType::SortedElem && status != RecordStatus::Outdated && + equal_string_view(key, UserKey(splice.next_data_record))) { + value->assign(splice.next_data_record->Value().data(), + splice.next_data_record->Value().size()); + return Status::Ok; + } else { + return Status::NotFound; + } + } else { + std::string internal_key = InternalKey(key); + auto ret = hash_table_->Lookup(internal_key, RecordType::SortedElem); + if (ret.s != Status::Ok || + ret.entry.GetRecordStatus() == RecordStatus::Outdated) { + return Status::NotFound; + } + + DLRecord* data_record; + switch (ret.entry.GetIndexType()) { + case PointerType::SkiplistNode: { + data_record = ret.entry.GetIndex().skiplist_node->record; + break; + } + case PointerType::DLRecord: { + data_record = ret.entry.GetIndex().dl_record; + break; + } + default: { + GlobalLogger.Error( + "Wrong hash index type while search sorted data in hash table\n"); + return Status::Abort; + } + } + kvdk_assert(data_record->GetRecordType() == RecordType::SortedElem, ""); + // As get is lockless, skiplist node may point to a new elem delete record + // after we get it from hashtable + if (data_record->GetRecordStatus() == RecordStatus::Outdated) { + return Status::NotFound; + } else { + value->assign(data_record->Value().data(), data_record->Value().size()); + return Status::Ok; + } + } +} + +Skiplist::WriteResult Skiplist::deletePreparedNoHash(DLRecord* existing_record, + SkiplistNode* dram_node, + const StringView& key, + TimestampType timestamp, + const SpaceEntry& space) { + kvdk_assert(existing_record != nullptr, ""); + WriteResult ret; + std::string internal_key(InternalKey(key)); + kvdk_assert(equal_string_view(existing_record->Key(), internal_key), ""); + ret.existing_record = existing_record; + ret.dram_node = dram_node; + DLList::WriteArgs args(internal_key, "", RecordType::SortedElem, + RecordStatus::Outdated, timestamp, space); + while (dl_list_.Update(args, existing_record) != Status::Ok) { + } + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + + if (dram_node) { + dram_node->record = ret.write_record; + } + return ret; +} + +Skiplist::WriteResult Skiplist::deletePreparedWithHash( + const HashTable::LookupResult& lookup_result, const StringView& key, + TimestampType timestamp, const SpaceEntry& space) { + std::string internal_key(InternalKey(key)); + assert(IndexWithHashtable()); + assert(lookup_result.s == Status::Ok); + assert(lookup_result.entry.GetRecordType() == RecordType::SortedElem && + lookup_result.entry.GetRecordStatus() == RecordStatus::Normal); + assert(space.size >= DLRecord::RecordSize(internal_key, "")); + DLRecord* existing_record; + SkiplistNode* dram_node; + + if (lookup_result.entry.GetIndexType() == PointerType::SkiplistNode) { + dram_node = lookup_result.entry.GetIndex().skiplist_node; + existing_record = dram_node->record; + } else { + dram_node = nullptr; + assert(lookup_result.entry.GetIndexType() == PointerType::DLRecord); + existing_record = lookup_result.entry.GetIndex().dl_record; + } + assert(timestamp > existing_record->GetTimestamp()); + + auto ret = + deletePreparedNoHash(existing_record, dram_node, key, timestamp, space); + + // until here, new record is already inserted to list + assert(ret.write_record != nullptr); + if (ret.dram_node == nullptr) { + hash_table_->Insert(lookup_result, RecordType::SortedElem, + RecordStatus::Outdated, ret.write_record, + PointerType::DLRecord); + } else { + ret.dram_node->record = ret.write_record; + hash_table_->Insert(lookup_result, RecordType::SortedElem, + RecordStatus::Outdated, ret.dram_node, + PointerType::SkiplistNode); + } + + return ret; +} + +Skiplist::WriteResult Skiplist::putPreparedWithHash( + const HashTable::LookupResult& lookup_result, const StringView& key, + const StringView& value, TimestampType timestamp, const SpaceEntry& space) { + WriteResult ret; + assert(IndexWithHashtable()); + std::string internal_key(InternalKey(key)); + DLList::WriteArgs args(internal_key, value, RecordType::SortedElem, + RecordStatus::Normal, timestamp, space); + + switch (lookup_result.s) { + case Status::Ok: { + if (lookup_result.entry.GetIndexType() == PointerType::SkiplistNode) { + ret.dram_node = lookup_result.entry.GetIndex().skiplist_node; + ret.existing_record = ret.dram_node->record; + } else { + ret.dram_node = nullptr; + assert(lookup_result.entry.GetIndexType() == PointerType::DLRecord); + ret.existing_record = lookup_result.entry.GetIndex().dl_record; + } + assert(timestamp > ret.existing_record->GetTimestamp()); + while (dl_list_.Update(args, ret.existing_record) != Status::Ok) { + } + + ret.write_record = + kv_allocator_->offset2addr_checked(space.offset); + ret.hash_entry_ptr = lookup_result.entry_ptr; + break; + } + case Status::NotFound: { + Splice splice(this); + Seek(key, &splice); + ret = putPreparedNoHash(splice, key, value, timestamp, space); + if (ret.s != Status::Ok) { + return ret; + } + + break; + } + case Status::MemoryOverflow: { + return ret; + } + default: + std::abort(); // never should reach + } + + // until here, new record is already inserted to list + assert(ret.write_record != nullptr); + if (ret.dram_node == nullptr) { + hash_table_->Insert(lookup_result, RecordType::SortedElem, + RecordStatus::Normal, ret.write_record, + PointerType::DLRecord); + } else { + ret.dram_node->record = ret.write_record; + hash_table_->Insert(lookup_result, RecordType::SortedElem, + RecordStatus::Normal, ret.dram_node, + PointerType::SkiplistNode); + } + + return ret; +} + +Skiplist::WriteResult Skiplist::putPreparedNoHash(Splice& seek_result, + const StringView& key, + const StringView& value, + TimestampType timestamp, + const SpaceEntry& space) { + WriteResult ret; + std::string internal_key(InternalKey(key)); + bool key_exist; + DLList::WriteArgs args(internal_key, value, RecordType::SortedElem, + RecordStatus::Normal, timestamp, space); + +seek_write_position: + key_exist = + !IndexWithHashtable() /* a hash indexed skiplist call this + function only if key not exist */ + && + seek_result.next_data_record->GetRecordType() == RecordType::SortedElem && + equal_string_view(seek_result.next_data_record->Key(), internal_key); + + if (key_exist) { + ret.existing_record = seek_result.next_data_record; + if (seek_result.nexts[1] && + seek_result.nexts[1]->record == ret.existing_record) { + ret.dram_node = seek_result.nexts[1]; + } + + while (dl_list_.Update(args, ret.existing_record) != Status::Ok) { + } + } else { + ret.existing_record = nullptr; + if (dl_list_.InsertBetween(args, seek_result.prev_data_record, + seek_result.next_data_record) != Status::Ok) { + seek_result = Splice(this); + Seek(key, &seek_result); + goto seek_write_position; + } + } + + ret.write_record = kv_allocator_->offset2addr_checked(space.offset); + + if (!key_exist) { + // create dram node for new record + ret.dram_node = Skiplist::NewNodeBuild(ret.write_record, node_allocator_); + if (ret.dram_node != nullptr) { + auto height = ret.dram_node->Height(); + for (int i = 1; i <= height; i++) { + while (1) { + auto now_next = seek_result.prevs[i]->Next(i); + // if next has been changed or been deleted, re-compute + if (now_next.RawPointer() == seek_result.nexts[i] && + now_next.GetTag() == SkiplistNode::NodeStatus::Normal) { + ret.dram_node->RelaxedSetNext(i, seek_result.nexts[i]); + if (seek_result.prevs[i]->CASNext(i, seek_result.nexts[i], + ret.dram_node)) { + break; + } + } else { + seek_result.Recompute(key, i); + } + } + } + } + } else if (ret.dram_node) { + ret.dram_node->record = ret.write_record; + } + return ret; +} + +void Skiplist::CleanObsoletedNodes() { + std::lock_guard lg_a(pending_delete_nodes_spin_); + if (pending_deletion_nodes_.size() > 0) { + for (SkiplistNode* node : pending_deletion_nodes_) { + // TODO: make sure the node is not referenced + SkiplistNode::DeleteNode(node, node_allocator_); + } + pending_deletion_nodes_.clear(); + } + + std::lock_guard lg_b(obsolete_nodes_spin_); + obsolete_nodes_.swap(pending_deletion_nodes_); +} + +void Skiplist::destroyAllRecords() { + std::vector to_free; + if (header_) { + DLRecord* header_record = HeaderRecord(); + DLRecord* to_destroy = nullptr; + do { + to_destroy = + kv_allocator_->offset2addr_checked(header_record->next); + StringView key = to_destroy->Key(); + auto ul = hash_table_->AcquireLock(key); + // We need to purge destroyed records one by one in case engine crashed + // during destroy + if (Skiplist::Remove(to_destroy, nullptr, kv_allocator_, record_locks_)) { + if (IndexWithHashtable()) { + auto lookup_result = + hash_table_->Lookup(key, to_destroy->GetRecordType()); + if (lookup_result.s == Status::Ok) { + DLRecord* hash_indexed_record = nullptr; + auto hash_index = lookup_result.entry.GetIndex(); + switch (lookup_result.entry.GetIndexType()) { + case PointerType::Skiplist: + hash_indexed_record = hash_index.skiplist->HeaderRecord(); + break; + case PointerType::SkiplistNode: + hash_indexed_record = hash_index.skiplist_node->record; + break; + case PointerType::DLRecord: + hash_indexed_record = hash_index.dl_record; + break; + default: + kvdk_assert(false, "Wrong hash index type of sorted record"); + } + + if (hash_indexed_record == to_destroy) { + hash_table_->Erase(lookup_result.entry_ptr); + } + } + } + + auto old_record = static_cast( + kv_allocator_->offset2addr(to_destroy->old_version)); + while (old_record) { + auto old_version = old_record->old_version; + to_free.emplace_back(kv_allocator_->addr2offset(old_record), + old_record->GetRecordSize()); + old_record->Destroy(); + old_record = kv_allocator_->offset2addr(old_version); + } + + to_free.emplace_back(kv_allocator_->addr2offset_checked(to_destroy), + to_destroy->GetRecordSize()); + to_destroy->Destroy(); + if (to_free.size() > kMaxCachedOldRecords) { + kv_allocator_->BatchFree(to_free); + to_free.clear(); + } + } + } while (to_destroy != + header_record /* header record should be the last detroyed one */); + } + kv_allocator_->BatchFree(to_free); +} + +void Skiplist::Destroy() { + destroyRecords(); + destroyNodes(); +} + +void Skiplist::DestroyAll() { + destroyAllRecords(); + destroyNodes(); +} + +void Skiplist::destroyNodes() { + if (header_) { + // To avoid memory leak (don't free created skiplist node), we should + // iterate the skiplist to find all deleted skiplist nodes. + // Notice: sometimes, a thread has just marked deleted skiplist node A, but + // another thread found this deleted skiplist node A and updated its linkage + // of lower height whiling seeking nodes. It is easy to ignore that the + // linkage of higher height of this deleted skiplist node A has not been + // changed. + for (int i = header_->Height(); i >= 1; --i) { + auto to_delete = header_->Next(i).RawPointer(); + while (to_delete) { + auto next = to_delete->Next(i).RawPointer(); + if (--to_delete->valid_links == 0) { + SkiplistNode::DeleteNode(to_delete, node_allocator_); + } + to_delete = next; + } + } + SkiplistNode::DeleteNode(header_, node_allocator_); + header_ = nullptr; + } +} + +void Skiplist::destroyRecords() { + std::vector to_free; + if (header_) { + DLRecord* header_record = HeaderRecord(); + DLRecord* to_destroy = nullptr; + do { + to_destroy = + kv_allocator_->offset2addr_checked(header_record->next); + StringView key = to_destroy->Key(); + auto ul = hash_table_->AcquireLock(key); + // We need to purge destroyed records one by one in case engine crashed + // during destroy + if (Skiplist::Remove(to_destroy, nullptr, kv_allocator_, record_locks_)) { + if (IndexWithHashtable()) { + auto lookup_result = + hash_table_->Lookup(key, to_destroy->GetRecordType()); + + if (lookup_result.s == Status::Ok) { + DLRecord* hash_indexed_record = nullptr; + auto hash_index = lookup_result.entry.GetIndex(); + switch (lookup_result.entry.GetIndexType()) { + case PointerType::Skiplist: + hash_indexed_record = hash_index.skiplist->HeaderRecord(); + break; + case PointerType::SkiplistNode: + hash_indexed_record = hash_index.skiplist_node->record; + break; + case PointerType::DLRecord: + hash_indexed_record = hash_index.dl_record; + break; + default: + kvdk_assert(false, "Wrong hash index type of sorted record"); + } + if (hash_indexed_record == to_destroy) { + hash_table_->Erase(lookup_result.entry_ptr); + } + } + } + to_destroy->Destroy(); + + to_free.emplace_back(kv_allocator_->addr2offset_checked(to_destroy), + to_destroy->GetRecordSize()); + } + + } while (to_destroy != + header_record /* header record should be the last detroyed one */); + } + + kv_allocator_->BatchFree(to_free); +} + +size_t Skiplist::Size() { return size_.load(std::memory_order_relaxed); } + +void Skiplist::UpdateSize(int64_t delta) { + kvdk_assert(delta >= 0 || size_.load() >= static_cast(-delta), + "Update skiplist size to negative"); + size_.fetch_add(delta, std::memory_order_relaxed); +} +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/sorted_collection/skiplist.hpp b/volatile/engine/sorted_collection/skiplist.hpp new file mode 100644 index 00000000..7cf39cd4 --- /dev/null +++ b/volatile/engine/sorted_collection/skiplist.hpp @@ -0,0 +1,582 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../alias.hpp" +#include "../collection.hpp" +#include "../dl_list.hpp" +#include "../hash_table.hpp" +#include "../lock_table.hpp" +#include "../structures.hpp" +#include "../utils/utils.hpp" +#include "../write_batch_impl.hpp" +#include "kvdk/volatile/engine.hpp" + +namespace KVDK_NAMESPACE { +static const uint8_t kMaxHeight = 32; +static const uint8_t kCacheHeight = 3; + +struct Splice; +class SortedIteratorImpl; + +struct SortedWriteArgs { + StringView collection; + StringView key; + StringView value; + WriteOp op; + Skiplist* skiplist; + SpaceEntry space; + TimestampType ts; + HashTable::LookupResult lookup_result; + std::unique_ptr seek_result; +}; + +/* Format: + * next pointers | DLRecord on kv memory | height | cached key size | + * cached key We only cache key if height > kCache height or there are enough + * space in the end of malloced space to cache the key (4B here). + * */ +struct SkiplistNode { + public: + enum class NodeStatus : uint8_t { + Normal = 0, + Deleted = 1, + }; + // Tagged pointers means this node has been logically removed from the list + std::atomic> next[0]; + // Doubly linked record on kv memory + DLRecord* record; + // TODO: save memory + uint16_t cached_key_size; + uint8_t height; + // How many height this node are linked on its skiplist. If this node is + // phisically remove from some height of a skiplist, then valid_links-=1. + // valid_links==0 means this node is removed from every height of the + // skiplist, free this node. + std::atomic valid_links{0}; + // 4 bytes for alignment, the actually allocated size may > 4 + char cached_key[4]; + + // Create a skiplist node, you should use same allocator for NewNode and + // DeleteNode + static SkiplistNode* NewNode(const StringView& key, DLRecord* data_record, + uint8_t height, Allocator* alloc) { + if (alloc == nullptr) { + alloc = global_memory_allocator(); + } + size_t size; + if (height >= kCacheHeight && key.size() > 4) { + size = sizeof(SkiplistNode) + 8 * height + key.size() - 4; + } else { + size = sizeof(SkiplistNode) + 8 * height; + } + SkiplistNode* node = nullptr; + + SpaceEntry entry = alloc->Allocate(size); + if (entry.size != 0) { + void* space = alloc->offset2addr(entry.offset); + node = (SkiplistNode*)((char*)space + 8 * height); + node->record = data_record; + node->height = height; + // make sure this will be linked to skiplist at all the height after + // creation + node->valid_links = height; + node->maybeCacheKey(key); + } + return node; + } + + // Destroy a skiplist node, you should use same allocator for NewNode and + // DeleteNode + static void DeleteNode(SkiplistNode* node, Allocator* alloc) { + if (alloc == nullptr) { + alloc = global_memory_allocator(); + } + uint64_t offset = alloc->addr2offset(node->heap_space_start()); + uint64_t size = node->allocated_size(); + alloc->Free(SpaceEntry(offset, size)); + } + + uint16_t Height() { return height; } + + StringView UserKey(); + + CollectionIDType SkiplistID(); + + PointerWithTag Next(int l) { + assert(l > 0 && l <= height && "should be less than node's height"); + return next[-l].load(std::memory_order_acquire); + } + + bool CASNext(int l, PointerWithTag expected, + PointerWithTag x) { + assert(l > 0 && l <= height && "should be less than node's height"); + return (next[-l].compare_exchange_strong(expected, x)); + } + + PointerWithTag RelaxedNext(int l) { + assert(l > 0 && l <= height && "should be less than node's height"); + return next[-l].load(std::memory_order_relaxed); + } + + void SetNext(int l, PointerWithTag x) { + assert(l > 0 && l <= height && "should be less than node's height"); + next[-l].store(x, std::memory_order_release); + } + + void RelaxedSetNext(int l, PointerWithTag x) { + assert(l > 0 && l <= height && "should be less than node's height"); + next[-l].store(x, std::memory_order_relaxed); + } + + // Logically delete node by tag next pointers from bottom to top + void MarkAsDeleted() { + for (int l = 1; l <= height; l++) { + while (1) { + auto next = RelaxedNext(l); + // This node alread tagged by another thread + if (next.GetTag() == NodeStatus::Deleted) { + break; + } + auto tagged = PointerWithTag( + next.RawPointer(), NodeStatus::Deleted); + if (CASNext(l, next, tagged)) { + break; + } + } + } + } + + bool IsDeleted() { return Next(1).GetTag() == NodeStatus::Deleted; } + + private: + SkiplistNode() {} + + void maybeCacheKey(const StringView& key) { + if (height >= kCacheHeight || key.size() <= 4) { + cached_key_size = key.size(); + memcpy(cached_key, key.data(), key.size()); + } else { + cached_key_size = 0; + } + } + + void* heap_space_start() { return (char*)this - height * 8; } + + uint64_t allocated_size() { + if (cached_key_size > 4) { + return sizeof(SkiplistNode) + 8 * height + cached_key_size - 4; + } else { + return sizeof(SkiplistNode) + 8 * height; + } + } +}; + +// A persistent sorted collection implemented as skiplist struct, data organized +// sorted by key +// +// The lowest level of the skiplist are stored on kv memory along with key and +// values, while higher level nodes are stored in DRAM, and re-construct at +// recovery. +// The insert and seek operations are indexed by the multi-level links and +// implemented in O(logn) time. Meanwhile, the skiplist nodes is also indexed by +// the global hash table, so the updates/delete and point read operations can be +// indexed by hash table and implemented in ~O(1) time +// Each skiplist has a header record stored on kv memory, the key of header +// record is the skiplist name, the value of header record is encoded by +// skiplist id and configs +class Skiplist : public Collection { + public: + // Result of a write operation + struct WriteResult { + Status s = Status::Ok; + DLRecord* existing_record = nullptr; + DLRecord* write_record = nullptr; + SkiplistNode* dram_node = nullptr; + HashEntry* hash_entry_ptr = nullptr; + }; + + Skiplist(DLRecord* h, const std::string& name, CollectionIDType id, + Comparator comparator, Allocator* kv_allocator, + Allocator* node_allocator, HashTable* hash_table, + LockTable* lock_table, bool index_with_hashtable); + + ~Skiplist() final; + + SkiplistNode* HeaderNode() { return header_; } + + DLRecord* HeaderRecord() { return header_->record; } + + const DLRecord* HeaderRecord() const { return header_->record; } + + bool IndexWithHashtable() { return index_with_hashtable_; } + + ExpireTimeType GetExpireTime() const final { + return HeaderRecord()->GetExpireTime(); + } + + bool HasExpired() const final { return HeaderRecord()->HasExpired(); } + + DLList* GetDLList() { return &dl_list_; } + + // Set this skiplist expire at expired_time + // + // Args: + // * expired_time: time to expire + // * timestamp: kvdk engine timestamp of calling this function + // + // Return Ok on success + WriteResult SetExpireTime(ExpireTimeType expired_time, + TimestampType timestamp); + + // Put "key, value" to the skiplist + // + // Args: + // * timestamp: kvdk engine timestamp of this operation + // + // Return Ok on success, with the writed data record, its dram node and + // updated data record if it exists + // + // Notice: the putting key should already been locked by engine + WriteResult Put(const StringView& key, const StringView& value, + TimestampType timestamp); + + // Get value of "key" from the skiplist + Status Get(const StringView& key, std::string* value); + + // Delete "key" from the skiplist by replace it with a delete record + // + // Args: + // * timestamp: kvdk engine timestamp of this operation + // + // Return Ok on success, with the writed delete record, its dram node and + // deleted record if it exists + // + // Notice: the deleting key should already been locked by engine + WriteResult Delete(const StringView& key, TimestampType timestamp); + + // Init args for put or delete operations + SortedWriteArgs InitWriteArgs(const StringView& key, const StringView& value, + WriteOp op); + + // Prepare neccessary resources for write, store lookup/seek result of key and + // required memory space to write new reocrd in args + // + // Args: + // * args: generated by InitWriteArgs + // + // Return: + // Ok on success + // MemoryOverflow if no enough kv memory space + // MemoryOverflow if no enough dram space + // + // Notice: args.key should already been locked by engine + Status PrepareWrite(SortedWriteArgs& args, TimestampType ts); + + // Do batch write according to args + // + // Args: + // * args: write args prepared by PrepareWrite() + // + // Return: + // Status Ok on success, with the writed delete record, its dram node and + // deleted record if existing + WriteResult Write(SortedWriteArgs& args); + + // Seek position of "key" on both dram and kv node in the skiplist, and + // store position in "result_splice". If "key" existing, the next pointers in + // splice point to node of "key" + void Seek(const StringView& key, Splice* result_splice); + + // Start seek from "start_node", find dram position of "key" in the skiplist + // between height "start_height" and "end"_height", and store position in + // "result_splice", if "key" existing, the next pointers in splice point to + // node of "key" + void SeekNode(const StringView& key, SkiplistNode* start_node, + uint8_t start_height, uint8_t end_height, + Splice* result_splice); + + // Destroy and free the whole skiplist, including skiplist nodes and kv + // records. + void Destroy(); + + // Destroy and free the whole skiplist with old version list. + void DestroyAll(); + + // check node linkage and hash index + Status CheckIndex(); + + void CleanObsoletedNodes(); + + // Return number of elements in skiplist + size_t Size(); + + void UpdateSize(int64_t delta); + + int Compare(const StringView& src_key, const StringView& target_key) { + return comparator_(src_key, target_key); + } + + static bool MatchType(DLRecord* record) { + RecordType type = record->GetRecordType(); + return type == RecordType::SortedElem || type == RecordType::SortedHeader; + } + + // Remove a dl record from its skiplist by unlinking + // + // Args: + // * purged_record:existing record to purge + // * dram_node:dram node of purging record, if it's a height 0 record, then + // pass nullptr + // + // Return: + // * true on success + // * false if purging_record not linked on a skiplist + // + // Notice: key of the purging record should already been locked by engine + static bool Remove(DLRecord* purging_record, SkiplistNode* dram_node, + Allocator* kv_allocator, LockTable* lock_table); + + // Replace "old_record" from its skiplist with "replacing_record", please make + // sure the key order is correct after replace + // + // Args: + // * old_record: existing record to be replaced + // * new_record: new reocrd to replace the older one + // * dram_node:dram node of old record, if it's a height 0 record, then + // pass nullptr + // + // Return: + // * true on success + // * false if old_record not linked on a skiplist + // + // Notice: key of the replacing record should already been locked by engine + static bool Replace(DLRecord* old_record, DLRecord* new_record, + SkiplistNode* dram_node, Allocator* kv_allocator, + LockTable* lock_table); + + // Build a skiplist node for "data_record" + static SkiplistNode* NewNodeBuild(DLRecord* data_record, Allocator* alloc); + + // Format: + // id (8 bytes) | configs + static std::string EncodeSortedCollectionValue( + CollectionIDType id, const SortedCollectionConfigs& s_configs); + + static Status DecodeSortedCollectionValue(StringView value_str, + CollectionIDType& id, + SortedCollectionConfigs& s_configs); + + inline static StringView UserKey(const SkiplistNode* node) { + assert(node != nullptr); + if (node->cached_key_size > 0) { + return StringView(node->cached_key, node->cached_key_size); + } + return ExtractUserKey(node->record->Key()); + } + + inline static StringView UserKey(const DLRecord* record) { + assert(record != nullptr); + return ExtractUserKey(record->Key()); + } + + inline static CollectionIDType FetchID(const SkiplistNode* node) { + assert(node != nullptr); + return FetchID(node->record); + } + + inline static CollectionIDType FetchID(const DLRecord* record) { + assert(record != nullptr); + switch (record->GetRecordType()) { + case RecordType::SortedElem: + return ExtractID(record->Key()); + break; + case RecordType::SortedHeader: + return DecodeID(record->Value()); + default: + GlobalLogger.Error("Wrong record type %u in SkiplistID", + record->GetRecordType()); + kvdk_assert(false, "Wrong type in SkiplistID"); + } + return 0; + } + + bool TryCleaningLock() { return cleaning_lock_.try_lock(); } + + void ReleaseCleaningLock() { cleaning_lock_.unlock(); } + + private: + friend SortedIteratorImpl; + + // put impl with prepared seek result and kv memory space + WriteResult putPreparedNoHash(Splice& seek_result, const StringView& key, + const StringView& value, + TimestampType timestamp, + const SpaceEntry& space); + + // put impl with prepared lookup result and kv memory space + WriteResult putPreparedWithHash(const HashTable::LookupResult& lookup_result, + const StringView& key, + const StringView& value, + TimestampType timestamp, + const SpaceEntry& space); + + // put impl with prepared existing record and kv memory space + WriteResult deletePreparedNoHash(DLRecord* existing_record, + SkiplistNode* dram_node, + const StringView& key, + TimestampType timestamp, + const SpaceEntry& space); + + // put impl with prepared lookup result of existing record and kv memory space + WriteResult deletePreparedWithHash( + const HashTable::LookupResult& lookup_result, const StringView& key, + TimestampType timestamp, const SpaceEntry& space); + + // Link DLRecord "linking" between "prev" and "next" + static void linkDLRecord(DLRecord* prev, DLRecord* next, DLRecord* linking, + Allocator* kv_allocator); + + inline void linkDLRecord(DLRecord* prev, DLRecord* next, DLRecord* linking) { + return linkDLRecord(prev, next, linking, kv_allocator_); + } + + // lock skiplist position to insert "key" by locking prev DLRecord and manage + // the lock with "prev_record_lock". + // + // Return true on success, return false if linkage of prev_record and + // next_record changed before succefully acquire lock + bool lockInsertPosition(const StringView& inserting_key, + DLRecord* prev_record, DLRecord* next_record, + LockTable::ULockType* prev_record_lock); + + // lock skiplist position of "record" by locking its prev DLRecord and the + // record itself + // Notice: we do not check if record is still correctly linked + static LockTable::MultiGuardType lockRecordPosition(const DLRecord* record, + Allocator* kv_allocator, + LockTable* lock_table); + + // lock skiplist position of "record" by locking its prev DLRecord and the + // record itself + // Notice: record must be a on list record, e.g. correctly linked by its + // predecessor + LockTable::MultiGuardType lockOnListRecord(const DLRecord* record) { + while (true) { + auto guard = lockRecordPosition(record, kv_allocator_, record_locks_); + DLRecord* prev = + kv_allocator_->offset2addr_checked(record->prev); + if (prev->next == kv_allocator_->addr2offset_checked(record)) { + return guard; + } + } + } + + void obsoleteNodes(const std::vector nodes) { + std::lock_guard lg(obsolete_nodes_spin_); + for (SkiplistNode* node : nodes) { + obsolete_nodes_.push_back(node); + } + } + + static uint8_t randomHeight() { + uint8_t height = 0; + while (height < kMaxHeight && fast_random_64() & 1) { + height++; + } + + return height; + } + + // Destroy sorted records, not including old version list. + void destroyRecords(); + + void destroyNodes(); + + // Destroy all sorted records including old version list. + void destroyAllRecords(); + + static LockTable::HashValueType recordHash(const DLRecord* record) { + kvdk_assert(record != nullptr, ""); + return XXH3_64bits(record, sizeof(const DLRecord*)); + } + + DLList dl_list_; + std::atomic size_; + Comparator comparator_ = compare_string_view; + // Allocate space for skiplist kv record + Allocator* kv_allocator_; + // Allocate space for skiplist high level nodes + Allocator* node_allocator_; + // TODO: use specified hash table for each skiplist + HashTable* hash_table_; + // locks to protect modification of records + LockTable* record_locks_; + bool index_with_hashtable_; + SkiplistNode* header_; + // nodes that unlinked on every height + std::vector obsolete_nodes_; + // to avoid concurrent access a just deleted node, a node can be safely + // deleted only if a certain interval is passes after being moved from + // obsolete_nodes_ to pending_deletion_nodes_, this is guaranteed by + // background thread of kvdk instance + std::vector pending_deletion_nodes_; + // protect obsolete_nodes_ + SpinMutex obsolete_nodes_spin_; + // protect pending_deletion_nodes_ + SpinMutex pending_delete_nodes_spin_; + // to avoid illegal access caused by cleaning skiplist by multi-thread + SpinMutex cleaning_lock_; +}; + +// A helper struct for locating a skiplist position +// +// nexts: next nodes on DRAM of a key position, or node of the key if it existed +// prevs: prev nodes on DRAM of a key position +// prev_data_record: previous record on kv memory of a key position +// next_data_record: next record on kv memory of a key position, or record of +// the key if it existed +// +// TODO: maybe we only need prev position +struct Splice { + // Seeking skiplist + Skiplist* seeking_list; + std::array nexts; + std::array prevs; + DLRecord* prev_data_record{nullptr}; + DLRecord* next_data_record{nullptr}; + + Splice(Skiplist* s) : seeking_list(s) {} + + void Recompute(const StringView& key, uint8_t l) { + SkiplistNode* start_node; + uint8_t start_height = l; + while (1) { + if (start_height > kMaxHeight || prevs[start_height] == nullptr) { + assert(seeking_list != nullptr); + start_height = kMaxHeight; + start_node = seeking_list->HeaderNode(); + } else if (prevs[start_height]->Next(start_height).GetTag() == + SkiplistNode::NodeStatus::Deleted) { + // If prev on this height has been deleted, roll back to higher height + start_height++; + continue; + } else { + start_node = prevs[start_height]; + } + seeking_list->SeekNode(key, start_node, start_height, l, this); + return; + } + } +}; + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/structures.hpp b/volatile/engine/structures.hpp new file mode 100644 index 00000000..8161f492 --- /dev/null +++ b/volatile/engine/structures.hpp @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include + +#include "alias.hpp" +#include "logger.hpp" +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { + +enum class PointerType : uint8_t { + // Value uninitialized considered as Invalid + Invalid = 0, + // Point to a string record + StringRecord = 1, + // Point to a doubly linked record + DLRecord = 2, + // Point to a dram skiplist node object + SkiplistNode = 3, + // Point to a dram Skiplist object + Skiplist = 4, + // Hash + HashList = 5, + // Element in Hash + HashElem = 6, + // List + List = 7, + // Point to a hash entry of hash table + HashEntry = 8, + // Allocated for later insertion + Allocated, + // Empty which point to nothing + Empty = 10, +}; + +// A pointer with additional information on high 16 bits +template +class PointerWithTag { + public: + static constexpr uint64_t kPointerMask = (((uint64_t)1 << 48) - 1); + + // TODO: Maybe explicit + PointerWithTag(PointerType* pointer) : tagged_pointer_((uint64_t)pointer) { + assert(sizeof(TagType) <= 2); + } + + explicit PointerWithTag(PointerType* pointer, TagType tag) + : tagged_pointer_((uint64_t)pointer | ((uint64_t)tag << 48)) { + assert(sizeof(TagType) <= 2); + } + + PointerWithTag() : tagged_pointer_(0) {} + + PointerType* RawPointer() { + return (PointerType*)(tagged_pointer_ & kPointerMask); + } + + const PointerType* RawPointer() const { + return (const PointerType*)(tagged_pointer_ & kPointerMask); + } + + bool Null() const { return RawPointer() == nullptr; } + + TagType GetTag() const { return static_cast(tagged_pointer_ >> 48); } + + void ClearTag() { tagged_pointer_ &= kPointerMask; } + + void SetTag(TagType tag) { tagged_pointer_ |= ((uint64_t)tag << 48); } + + const PointerType& operator*() const { return *RawPointer(); } + + PointerType& operator*() { return *(RawPointer()); } + + const PointerType* operator->() const { return RawPointer(); } + + PointerType* operator->() { return RawPointer(); } + + bool operator==(const PointerType* raw_pointer) { + return RawPointer() == raw_pointer; + } + + bool operator==(const PointerType* raw_pointer) const { + return RawPointer() == raw_pointer; + } + + private: + uint64_t tagged_pointer_; +}; +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/thread_manager.cpp b/volatile/engine/thread_manager.cpp new file mode 100644 index 00000000..a6510fbc --- /dev/null +++ b/volatile/engine/thread_manager.cpp @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "thread_manager.hpp" + +#include + +namespace KVDK_NAMESPACE { + +constexpr size_t kMaxRecycleID = 1024; + +std::shared_ptr ThreadManager::manager_(new ThreadManager); + +Thread::~Thread() { + if (manager != nullptr) { + manager->Release(*this); + } +} + +void ThreadManager::MaybeInitThread(Thread& t) { + if (t.id < 0) { + if (!recycle_id_.empty()) { + std::lock_guard lg(spin_); + if (!recycle_id_.empty()) { + auto it = recycle_id_.begin(); + t.manager = shared_from_this(); + t.id = *it; + recycle_id_.erase(it); + return; + } + } + int id = ids_.fetch_add(1, std::memory_order_relaxed); + t.manager = shared_from_this(); + t.id = id; + } +} + +void ThreadManager::Release(Thread& t) { + if (t.manager.get() == this && t.id >= 0 && + recycle_id_.size() < kMaxRecycleID) { + std::lock_guard lg(spin_); + recycle_id_.insert(t.id); + } + t.id = -1; + t.manager = nullptr; +} + +thread_local Thread this_thread; + +} // namespace KVDK_NAMESPACE diff --git a/engine/thread_manager.hpp b/volatile/engine/thread_manager.hpp similarity index 96% rename from engine/thread_manager.hpp rename to volatile/engine/thread_manager.hpp index 741f4122..cddae75e 100644 --- a/engine/thread_manager.hpp +++ b/volatile/engine/thread_manager.hpp @@ -8,7 +8,7 @@ #include #include "alias.hpp" -#include "kvdk/engine.hpp" +#include "kvdk/volatile/engine.hpp" #include "utils/utils.hpp" namespace KVDK_NAMESPACE { diff --git a/volatile/engine/utils/codec.hpp b/volatile/engine/utils/codec.hpp new file mode 100644 index 00000000..cfa89c35 --- /dev/null +++ b/volatile/engine/utils/codec.hpp @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include + +#include "../alias.hpp" +#include "../macros.hpp" + +namespace KVDK_NAMESPACE { + +inline std::string EncodeUint64(uint64_t value) { + return std::string((char*)&value, sizeof(uint64_t)); +} + +inline bool DecodeUint64(const StringView& src, uint64_t* value) { + if (src.size() < sizeof(uint64_t)) { + return false; + } + *value = *(uint64_t*)src.data(); + return true; +} + +inline void AppendUint64(std::string* dst, uint64_t value) { + dst->append((char*)&value, sizeof(uint64_t)); +} + +inline bool FetchUint64(StringView* src, uint64_t* value) { + if (src->size() < sizeof(uint64_t)) { + return false; + } + *value = *(uint64_t*)src->data(); + *src = StringView(src->data() + sizeof(uint64_t), + src->size() - sizeof(uint64_t)); + return true; +} + +inline std::string EncodeUint32(uint32_t value) { + return std::string((char*)&value, sizeof(uint32_t)); +} + +inline bool DecodeUint32(const StringView& src, uint32_t* value) { + if (src.size() < sizeof(uint32_t)) { + return false; + } + *value = *(uint32_t*)src.data(); + return true; +} + +inline void AppendUint32(std::string* dst, uint32_t value) { + dst->append((char*)&value, sizeof(uint32_t)); +} + +inline bool FetchUint32(StringView* src, uint32_t* value) { + if (src->size() < sizeof(uint32_t)) { + return false; + } + *value = *(uint32_t*)src->data(); + *src = StringView(src->data() + sizeof(uint32_t), + src->size() - sizeof(uint32_t)); + return true; +} + +// POD stands for Plain-old-data. +// We don't serialize object which may have pointers to other objects. +template +inline void AppendPOD(std::string* dst, T const& value) { + dst->append(reinterpret_cast(&value), sizeof(T)); +} + +template +inline bool FetchPOD(StringView* src, T* ret) { + if (src->size() < sizeof(T)) { + return false; + } + *ret = *reinterpret_cast(src->data()); + *src = StringView{src->data() + sizeof(T), src->size() - sizeof(T)}; + return true; +} + +template +inline T FetchPOD(StringView* src) { + T ret; + bool success = FetchPOD(src, &ret); + kvdk_assert(success, ""); + return ret; +} + +inline void AppendFixedString(std::string* dst, const StringView& str) { + AppendUint32(dst, str.size()); + dst->append(str.data(), str.size()); +} + +inline bool FetchFixedString(StringView* src, std::string* value) { + uint32_t size; + bool ret = FetchUint32(src, &size) && src->size() >= size; + if (ret) { + value->assign(src->data(), size); + *src = StringView(src->data() + size, src->size() - size); + } + return ret; +} + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/utils/sync_impl.hpp b/volatile/engine/utils/sync_impl.hpp new file mode 100644 index 00000000..28a79ede --- /dev/null +++ b/volatile/engine/utils/sync_impl.hpp @@ -0,0 +1,161 @@ +/* Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ + +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../alias.hpp" + +#if KVDK_DEBUG_LEVEL > 0 +namespace KVDK_NAMESPACE { + +struct SyncPointPair { + std::string producer; + std::string consumer; +}; + +struct SyncImpl { + class CrashPoint : public std::runtime_error { + public: + CrashPoint(std::string const& msg) : base{msg} {} + + private: + using base = std::runtime_error; + using base::what; + }; + + SyncImpl() : ready_(false) {} + void LoadDependency(const std::vector& dependencies) { + std::lock_guard lock(mutex_); + consumers_.clear(); + producers_.clear(); + cleared_points_.clear(); + for (const auto& dependency : dependencies) { + consumers_[dependency.producer].push_back(dependency.consumer); + producers_[dependency.consumer].push_back(dependency.producer); + point_table_.insert(dependency.consumer); + point_table_.insert(dependency.producer); + } + cv_.notify_all(); + } + + void EnableProcessing() { ready_ = true; } + + void DisableProcessing() { ready_ = false; } + + void Process(const std::string& point, void* func_arg) { + if (!ready_) { + return; + } + + if (point_table_.find(point) == point_table_.end()) { + return; + } + + std::unique_lock lock(mutex_); + + while (!IsClearedAllproducers(point)) { + cv_.wait(lock); + } + + auto callback_pair = callbacks_.find(point); + if (callback_pair != callbacks_.end()) { + num_callbacks_running_++; + mutex_.unlock(); + callback_pair->second(func_arg); + mutex_.lock(); + num_callbacks_running_--; + } + cleared_points_.insert(point); + + cv_.notify_all(); + } + + void SetCallBack(const std::string& point, + const std::function& callback) { + std::lock_guard lock(mutex_); + callbacks_[point] = callback; + point_table_.insert(point); + } + + void EnableCrashPoint(std::string const& name) { crash_points_.insert(name); } + + void Crash(std::string const& name, std::string const& msg) { + if (!ready_) { + return; + } + + if (crash_points_.find(name) == crash_points_.end()) { + return; + } + + throw CrashPoint{msg}; + } + + void ClearAllCallBacks() { + std::unique_lock lock(mutex_); + while (num_callbacks_running_ > 0) { + cv_.wait(lock); + } + callbacks_.clear(); + } + + bool IsClearedAllproducers(const std::string& point) { + for (const std::string& producer : producers_[point]) { + if (cleared_points_.find(producer) == cleared_points_.end()) return false; + } + return true; + } + + void ClearDependTrace() { + std::lock_guard lock(mutex_); + cleared_points_.clear(); + } + + void Reset() { + std::lock_guard lock(mutex_); + cleared_points_.clear(); + consumers_.clear(); + producers_.clear(); + callbacks_.clear(); + point_table_.clear(); + crash_points_.clear(); + num_callbacks_running_ = 0; + } + + virtual ~SyncImpl() {} + + private: + std::mutex mutex_; + std::condition_variable cv_; + std::atomic ready_; + std::unordered_set point_table_; + std::unordered_set crash_points_; + int num_callbacks_running_ = 0; + + std::unordered_map> consumers_; + std::unordered_map> producers_; + std::unordered_set cleared_points_; + std::unordered_map> callbacks_; +}; + +} // namespace KVDK_NAMESPACE + +#endif diff --git a/volatile/engine/utils/sync_point.cpp b/volatile/engine/utils/sync_point.cpp new file mode 100644 index 00000000..33daf0cf --- /dev/null +++ b/volatile/engine/utils/sync_point.cpp @@ -0,0 +1,57 @@ +/* Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ + +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "sync_point.hpp" + +#if KVDK_DEBUG_LEVEL > 0 + +namespace KVDK_NAMESPACE { +SyncPoint* SyncPoint::GetInstance() { + static SyncPoint sync_point; + return &sync_point; +} +SyncPoint::SyncPoint() : sync_impl_(new SyncImpl){}; + +SyncPoint::~SyncPoint() { delete sync_impl_; } + +void SyncPoint::Process(const std::string& point, void* func_arg) { + sync_impl_->Process(point, func_arg); +} + +void SyncPoint::LoadDependency(const std::vector& dependencies) { + sync_impl_->LoadDependency(dependencies); +} + +void SyncPoint::EnableCrashPoint(std::string const& name) { + sync_impl_->EnableCrashPoint(name); +} + +void SyncPoint::Crash(std::string const& name, std::string const& msg) { + sync_impl_->Crash(name, msg); +} + +void SyncPoint::EnableProcessing() { sync_impl_->EnableProcessing(); } + +void SyncPoint::DisableProcessing() { sync_impl_->DisableProcessing(); } + +void SyncPoint::SetCallBack(const std::string& point, + const std::function& callback) { + sync_impl_->SetCallBack(point, callback); +} + +void SyncPoint::ClearAllCallBacks() { sync_impl_->ClearAllCallBacks(); } + +void SyncPoint::ClearDependTrace() { sync_impl_->ClearDependTrace(); } + +void SyncPoint::Reset() { sync_impl_->Reset(); } + +} // namespace KVDK_NAMESPACE + +#endif diff --git a/volatile/engine/utils/sync_point.hpp b/volatile/engine/utils/sync_point.hpp new file mode 100644 index 00000000..c0968d85 --- /dev/null +++ b/volatile/engine/utils/sync_point.hpp @@ -0,0 +1,84 @@ +/* Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ + +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sync_impl.hpp" + +#if KVDK_DEBUG_LEVEL > 0 + +namespace KVDK_NAMESPACE { + +/* SyncPoint Guide: + * Developer could specify sync points in the codebase by TEST_SYNC_POINT. + * Each sync point represents a position in the execution stream of a thread. + * In the uint test, developer can set the relationship between the sync point + * by LoadDependency to reproduce a desired interleave of threads execution. + * Also can set the execution of the sync point by SetCallBack. Please see the + * example in the unit tests. + */ +class SyncPoint { + public: + using CrashPoint = SyncImpl::CrashPoint; + static SyncPoint* GetInstance(); + SyncPoint(const SyncPoint&) = delete; + SyncPoint& operator=(const SyncPoint&) = delete; + + ~SyncPoint(); + + void LoadDependency(const std::vector& dependencies); + void EnableProcessing(); + void DisableProcessing(); + + void SetCallBack(const std::string& point, + const std::function& callback); + + void EnableCrashPoint(std::string const& name); + + void Crash(std::string const& name, std::string const& msg); + + void ClearAllCallBacks(); + + void ClearDependTrace(); + + void Process(const std::string& point, void* func_arg = nullptr); + + void Reset(); + + private: + SyncPoint(); + SyncImpl* sync_impl_; +}; + +} // namespace KVDK_NAMESPACE + +#define TEST_SYNC_POINT(x) KVDK_NAMESPACE::SyncPoint::GetInstance()->Process(x) +#define TEST_SYNC_POINT_CALLBACK(x, y) \ + KVDK_NAMESPACE::SyncPoint::GetInstance()->Process(x, y) +#define TEST_CRASH_POINT(name, msg) \ + KVDK_NAMESPACE::SyncPoint::GetInstance()->Crash(name, msg) + +#else +#define TEST_SYNC_POINT(x) +#define TEST_SYNC_POINT_CALLBACK(x, y) +#define TEST_CRASH_POINT(name, msg) +#endif diff --git a/volatile/engine/utils/utils.cpp b/volatile/engine/utils/utils.cpp new file mode 100644 index 00000000..2b9a38e2 --- /dev/null +++ b/volatile/engine/utils/utils.cpp @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "utils.hpp" + +#include + +#include "../logger.hpp" + +namespace KVDK_NAMESPACE { + +int get_usable_pu(void) { + hwloc_topology_t topology; + hwloc_bitmap_t set; + int err; + + /* create a topology */ + err = hwloc_topology_init(&topology); + if (err < 0) { + GlobalLogger.Error("Failed to initialize the topology\n"); + return err; + } + err = hwloc_topology_load(topology); + if (err < 0) { + GlobalLogger.Error("Failed to load the topology\n"); + hwloc_topology_destroy(topology); + return err; + } + + /* retrieve the entire set of available PUs */ + [[gnu::unused]] hwloc_const_bitmap_t cset_available = + hwloc_topology_get_topology_cpuset(topology); + + /* retrieve the CPU binding of the current entire process */ + set = hwloc_bitmap_alloc(); + if (!set) { + GlobalLogger.Error("Failed to allocate a bitmap\n"); + hwloc_topology_destroy(topology); + return err; + } + err = hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS); + if (err < 0) { + GlobalLogger.Error("Failed to get cpu binding\n"); + hwloc_bitmap_free(set); + hwloc_topology_destroy(topology); + return err; + } + int pu = hwloc_bitmap_weight(set); + hwloc_topology_destroy(topology); + return pu; +} +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/utils/utils.hpp b/volatile/engine/utils/utils.hpp new file mode 100644 index 00000000..eb2c0d53 --- /dev/null +++ b/volatile/engine/utils/utils.hpp @@ -0,0 +1,416 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define XXH_INLINE_ALL +#include "xxhash.h" +#undef XXH_INLINE_ALL + +#include "../alias.hpp" +#include "../macros.hpp" +#include "codec.hpp" + +namespace KVDK_NAMESPACE { + +// A tool struct that call f on destroy +// use the macro defer(code) to do so +template +struct Defer { + F f; + Defer(F f) : f(f) {} + ~Defer() { f(); } +}; +template +Defer defer_func(F f) { + return Defer(f); +} +#define DEFER_1(x, y) x##y +#define DEFER_2(x, y) DEFER_1(x, y) +#define DEFER_3(x) DEFER_2(x, __COUNTER__) +#define defer(code) auto DEFER_3(_defer_) = defer_func([&]() { code; }) + +inline void atomic_memcpy_16(void* dst, void* src) { + ((std::atomic<__uint128_t>*)dst) + ->store(((std::atomic<__uint128_t>*)src)->load()); +} + +inline uint64_t hash_str(const char* str, uint64_t size) { + return XXH3_64bits(str, size); +} + +inline uint64_t xxh_hash(std::string const& val) { + return XXH3_64bits(val.data(), val.size()); +} + +inline uint64_t get_checksum(const void* data, uint64_t size) { + return XXH3_64bits(data, size); +} + +inline unsigned long long get_seed() { + unsigned long long seed = 0; + while (seed == 0 && _rdseed64_step(&seed) != 1) { + } + return seed; +} + +inline uint64_t fast_random_64() { + static std::mt19937_64 generator; + thread_local uint64_t seed = 0; + if (seed == 0) { + seed = generator(); + } + uint64_t x = seed; /* The state must be seeded with a nonzero value. */ + x ^= x >> 12; // a + x ^= x << 25; // b + x ^= x >> 27; // c + seed = x; + return x * 0x2545F4914F6CDD1D; +} + +inline uint64_t rdtsc() { + uint32_t lo, hi; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo) | (((uint64_t)hi) << 32); +} + +inline void atomic_load_16(void* dst, const void* src) { + (*(__uint128_t*)dst) = __atomic_load_16(src, std::memory_order_relaxed); +} + +inline void atomic_store_16(void* dst, const void* src) { + __atomic_store_16(dst, (*(__uint128_t*)src), std::memory_order_relaxed); +} + +inline void memcpy_16(void* dst, const void* src) { + __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); + _mm_storeu_si128(((__m128i*)dst) + 0, m0); +} + +inline void memcpy_8(void* dst, const void* src) { + *((uint64_t*)dst) = *((uint64_t*)src); +} + +inline void memcpy_4(void* dst, const void* src) { + *((uint32_t*)dst) = *((uint32_t*)src); +} + +inline void memcpy_2(void* dst, const void* src) { + *((uint16_t*)dst) = *((uint16_t*)src); +} + +inline void memcpy_1(void* dst, const void* src) { + *((uint8_t*)dst) = *((uint8_t*)src); +} + +inline std::string format_dir_path(const std::string& dir) { + return dir.back() == '/' ? dir : dir + "/"; +} + +inline bool file_exist(const std::string& name) { + bool exist = access(name.c_str(), 0) == 0; + return exist; +} + +inline int create_dir_if_missing(const std::string& name) { + int res = mkdir(name.c_str(), 0755) != 0; + if (res != 0) { + if (errno != EEXIST) { + return res; + } else { + struct stat s; + if (stat(name.c_str(), &s) == 0) { + return S_ISDIR(s.st_mode) ? 0 : res; + } + } + } + return res; +} + +inline std::string string_view_2_string(const StringView& src) { + return std::string(src.data(), src.size()); +} + +inline int compare_string_view(const StringView& src, + const StringView& target) { + auto size = std::min(src.size(), target.size()); + for (uint32_t i = 0; i < size; i++) { + if (src[i] != target[i]) { + return (unsigned char)src[i] - (unsigned char)target[i]; + } + } + return src.size() - target.size(); +} + +inline bool equal_string_view(const StringView& src, const StringView& target) { + if (src.size() == target.size()) { + return compare_string_view(src, target) == 0; + } + return false; +} + +class SpinMutex { + private: + std::atomic_flag locked_ = ATOMIC_FLAG_INIT; + + public: + SpinMutex() = default; + + void lock() { + while (!try_lock()) { + for (size_t i = 0; i != 64; ++i) { + _mm_pause(); + } + } + } + + void unlock() { locked_.clear(std::memory_order_release); } + + bool try_lock() { + if (locked_.test_and_set(std::memory_order_acquire)) { + return false; + } + return true; + } + + SpinMutex(const SpinMutex& s) = delete; + SpinMutex(SpinMutex&& s) = delete; + SpinMutex& operator=(const SpinMutex& s) = delete; +}; + +template +class SharedLock { + public: + explicit SharedLock(SharedMutex& mu) : device_{&mu} { + device_->lock_shared(); + } + + ~SharedLock() { + if (device_ != nullptr) { + device_->unlock_shared(); + } + } + + SharedLock(SharedLock const& other) : device_{other.device_} { + device_->lock_shared(); + } + + SharedLock& operator=(SharedLock const& other) { + if (device_ != nullptr) { + device_->unlock_shared(); + } + device_ = other.device_; + device_->lock_shared(); + return *this; + } + + SharedLock(SharedLock&& other) : device_{nullptr} { swap(other); } + + SharedLock& operator=(SharedLock&& other) { + if (device_ != nullptr) { + device_->unlock_shared(); + device_ = nullptr; + } + swap(other); + return *this; + } + + void swap(SharedLock& other) { + using std::swap; + swap(device_, other.device_); + } + + private: + SharedMutex* device_; +}; + +template +SharedLock LockShared(SharedMutex& mu) { + return SharedLock{mu}; +} + +class RWLock { + static constexpr std::int64_t reader_val{1}; + static constexpr std::int64_t writer_val{ + std::numeric_limits::min()}; + + // device.load() > 0 indicates only reader exists + // device.load() == 0 indicates no reader and no writer + // device.load() == writer_val indicates only writer exists, block readers + // Otherwise, writer has registered and is waiting for readers to leave + std::atomic_int64_t device{0}; + + public: + bool try_lock_shared() { + std::int64_t old = device.load(); + if (old < 0 || !device.compare_exchange_strong(old, old + reader_val)) { + // Other writer has acquired lock. + return false; + } + return true; + } + + void lock_shared() { + while (!try_lock_shared()) { + pause(); + } + return; + } + + void unlock_shared() { + device.fetch_sub(reader_val); + return; + } + + bool try_lock() { + std::int64_t old = device.load(); + if (old < 0 || !device.compare_exchange_strong(old, old + writer_val)) { + // Other writer has acquired lock. + return false; + } + while (device.load() != writer_val) { + // Block until all readers have left. + pause(); + } + return true; + } + + void lock() { + while (!try_lock()) { + pause(); + } + return; + } + + void unlock() { + device.fetch_sub(writer_val); + return; + } + + private: + void pause() { + for (size_t i = 0; i < 64; i++) { + _mm_pause(); + } + } +}; + +class Slice { + public: + Slice() : data_(nullptr), size_(0) {} + Slice(const char* data) : data_(data) { size_ = strlen(data_); } + Slice(const char* data, uint64_t size) : data_(data), size_(size) {} + + Slice(const std::string& str) : data_(str.data()), size_(str.size()) {} + Slice(const StringView& sv) : data_(sv.data()), size_(sv.size()) {} + + const char* data() const { return data_; } + + uint64_t& size() { return size_; } + + uint64_t size() const { return size_; } + + bool operator==(const Slice& b) { + if (b.size() == this->size_ && + memcmp(this->data_, b.data(), b.size()) == 0) { + return true; + } else { + return false; + } + } + + static int compare(const Slice& src, const Slice& target) { + auto size = std::min(src.size(), target.size()); + for (uint32_t i = 0; i < size; i++) { + if (src.data()[i] != target.data()[i]) { + return src.data()[i] - target.data()[i]; + } + } + return src.size() - target.size(); + } + + std::string to_string() { return std::string(data_, size_); } + + std::string to_string() const { return std::string(data_, size_); } + + private: + const char* data_; + uint64_t size_; +}; + +template +void compare_excange_if_larger(std::atomic& num, T target) { + while (true) { + T n = num.load(std::memory_order_relaxed); + if (n <= target) { + if (!num.compare_exchange_strong(n, target)) { + continue; + } + } + break; + } +} + +// Return the number of process unit (PU) that are bound to the kvdk instance +int get_usable_pu(void); + +namespace TimeUtils { +/* Return the UNIX time in microseconds */ +inline UnixTimeType unix_time(void) { + struct timeval tv; + UnixTimeType ust; + + gettimeofday(&tv, NULL); + ust = ((UnixTimeType)tv.tv_sec) * 1000000; + ust += tv.tv_usec; + return ust; +} + +inline int64_t millisecond_time() { return unix_time() / 1000; } + +inline bool CheckIsExpired(ExpireTimeType expired_time) { + if (expired_time >= 0 && expired_time <= millisecond_time()) { + return true; + } + return false; +} + +inline bool CheckTTL(TTLType ttl_time, UnixTimeType base_time) { + if (ttl_time != kPersistTime && + /* check overflow*/ ttl_time > INT64_MAX - base_time) { + return false; + } + return true; +} + +inline ExpireTimeType TTLToExpireTime( + TTLType ttl_time, UnixTimeType base_time = millisecond_time()) { + return ttl_time == kPersistTime ? kPersistTime : ttl_time + base_time; +} + +inline TTLType ExpireTimeToTTL(ExpireTimeType expire_time, + UnixTimeType base_time = millisecond_time()) { + return expire_time == kPersistTime ? kPersistTime : expire_time - base_time; +} + +} // namespace TimeUtils +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/version/old_records_cleaner.cpp b/volatile/engine/version/old_records_cleaner.cpp new file mode 100644 index 00000000..b3baa8db --- /dev/null +++ b/volatile/engine/version/old_records_cleaner.cpp @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "old_records_cleaner.hpp" + +#include "../kv_engine.hpp" +#include "../sorted_collection/skiplist.hpp" + +namespace KVDK_NAMESPACE { + +void OldRecordsCleaner::TryGlobalClean() { + std::vector space_to_free; + // Update recorded oldest snapshot up to state so we can know which records + // can be freed + kv_engine_->version_controller_.UpdateLocalOldestSnapshot(); + TimestampType oldest_snapshot_ts = + kv_engine_->version_controller_.LocalOldestSnapshotTS(); + + std::vector free_entries; + for (size_t i = 0; i < cleaner_thread_cache_.size(); i++) { + auto& cleaner_thread_cache = cleaner_thread_cache_[i]; + if (cleaner_thread_cache.pending_free_space_entries.size() > 0) { + std::lock_guard lg(cleaner_thread_cache.old_records_lock); + for (auto& space_entry : + cleaner_thread_cache.pending_free_space_entries) { + if (space_entry.release_time < oldest_snapshot_ts) { + free_entries.emplace_back(space_entry.entry); + } else { + global_pending_free_space_entries_.emplace_back(space_entry); + } + } + cleaner_thread_cache.pending_free_space_entries.clear(); + } + } + + auto iter = global_pending_free_space_entries_.begin(); + while (iter != global_pending_free_space_entries_.end()) { + if (iter->release_time < oldest_snapshot_ts) { + free_entries.emplace_back(iter->entry); + iter++; + } else { + break; + } + } + global_pending_free_space_entries_.erase( + global_pending_free_space_entries_.begin(), iter); + + if (free_entries.size() > 0) { + kv_engine_->kv_allocator_->BatchFree(space_to_free); + } +} + +void OldRecordsCleaner::maybeUpdateOldestSnapshot() { + // To avoid too many records pending free, we upadte global smallest + // snapshot regularly. We update it every kUpdateSnapshotRound to mitigate + // the overhead + constexpr size_t kUpdateSnapshotRound = 10000; + thread_local size_t round = 0; + if ((++round) % kUpdateSnapshotRound == 0) { + kv_engine_->version_controller_.UpdateLocalOldestSnapshot(); + } +} + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/version/old_records_cleaner.hpp b/volatile/engine/version/old_records_cleaner.hpp new file mode 100644 index 00000000..69db72a9 --- /dev/null +++ b/volatile/engine/version/old_records_cleaner.hpp @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include + +#include "../alias.hpp" +#include "../collection.hpp" +#include "../hash_table.hpp" +#include "../kv_engine_cleaner.hpp" +#include "../thread_manager.hpp" +#include "../utils/utils.hpp" +#include "kvdk/volatile/configs.hpp" +#include "version_controller.hpp" + +namespace KVDK_NAMESPACE { +class KVEngine; + +// OldRecordsCleaner is used to clean old version data records of kvdk +// +// To support multi-version machenism and consistent backup of kvdk, +// the updated/deleted records need to be ramained for a while until they are +// not refered by any snapshot +class OldRecordsCleaner { + public: + OldRecordsCleaner(KVEngine* kv_engine, uint32_t max_access_threads) + : kv_engine_(kv_engine), cleaner_thread_cache_(max_access_threads) { + assert(kv_engine_ != nullptr); + } + + // Try to clean global old records + void TryGlobalClean(); + + private: + struct CleanerThreadCache { + std::deque pending_free_space_entries{}; + SpinMutex old_records_lock; + }; + const uint64_t kLimitCachedDeleteRecords = 1000000; + + void maybeUpdateOldestSnapshot(); + + KVEngine* kv_engine_; + + Array cleaner_thread_cache_; + + std::deque global_pending_free_space_entries_; +}; + +} // namespace KVDK_NAMESPACE diff --git a/volatile/engine/version/version_controller.hpp b/volatile/engine/version/version_controller.hpp new file mode 100644 index 00000000..531e41d7 --- /dev/null +++ b/volatile/engine/version/version_controller.hpp @@ -0,0 +1,342 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include "../alias.hpp" +#include "../allocator.hpp" +#include "../thread_manager.hpp" +#include "kvdk/volatile/configs.hpp" + +namespace KVDK_NAMESPACE { +constexpr TimestampType kMaxTimestamp = UINT64_MAX; + +struct SnapshotImpl : public Snapshot { + explicit SnapshotImpl(const TimestampType& t) : timestamp(t) {} + + SnapshotImpl() = default; + + TimestampType GetTimestamp() const { return timestamp; } + + TimestampType timestamp = kMaxTimestamp; + SnapshotImpl* prev = nullptr; + SnapshotImpl* next = nullptr; +}; + +// A SnapshotList is a linked-list of global snapshots, new older snapshot is +// linked at the tail of list +class SnapshotList { + public: + SnapshotList() : head_() { + head_.prev = &head_; + head_.next = &head_; + } + + SnapshotImpl* New(TimestampType ts) { + SnapshotImpl* impl = new SnapshotImpl(ts); + impl->prev = &head_; + impl->next = head_.next; + head_.next->prev = impl; + head_.next = impl; + return impl; + } + + void Delete(const SnapshotImpl* impl) { + impl->prev->next = impl->next; + impl->next->prev = impl->prev; + delete impl; + } + + TimestampType OldestSnapshotTS() { + return empty() ? kMaxTimestamp : head_.prev->GetTimestamp(); + } + + ~SnapshotList() { + SnapshotImpl* curr = head_.next; + while (curr != &head_) { + SnapshotImpl* tmp = curr->next; + delete curr; + curr = tmp; + } + } + + private: + bool empty() { return head_.prev == &head_; } + + SnapshotImpl head_; +}; + +// VersionController manages snapshots and timestamp of a KVDK instance +// The snapshots include temporal snapshots that cached by each access thread of +// kvdk instance, and a global snapshot list that actively created by user +class VersionController { + public: + // LocalSnapshotHolder is used by kvdk functions such as Get(), SortedGet() + // and HashGet() internally to guarantee lockless reads will always read out + // valid data. LocalSnapshotHolder is thread_local, and one thread can hold + // atmost one at same time. + class LocalSnapshotHolder { + VersionController* owner_{nullptr}; + TimestampType ts_{}; + + public: + LocalSnapshotHolder(VersionController* o) : owner_{o} { + owner_->HoldLocalSnapshot(); + ts_ = owner_ + ->version_thread_cache_[ThreadManager::ThreadID() % + owner_->version_thread_cache_.size()] + .holding_snapshot.timestamp; + }; + LocalSnapshotHolder(LocalSnapshotHolder const&) = delete; + LocalSnapshotHolder& operator=(LocalSnapshotHolder const&) = delete; + LocalSnapshotHolder(LocalSnapshotHolder&& other) { + *this = std::move(other); + } + LocalSnapshotHolder& operator=(LocalSnapshotHolder&& other) { + kvdk_assert(owner_ == nullptr, ""); + kvdk_assert(other.owner_ != nullptr, ""); + std::swap(owner_, other.owner_); + std::swap(ts_, other.ts_); + return *this; + } + ~LocalSnapshotHolder() { + if (owner_ != nullptr) { + owner_->ReleaseLocalSnapshot(); + } + } + TimestampType Timestamp() { return ts_; } + }; + + // GlobalSnapshotHolder is hold internally by iterators. + // Create GlobalSnapshotHolder is more costly then InteralToken. + class GlobalSnapshotHolder { + VersionController* owner_{nullptr}; + SnapshotImpl* snap_{nullptr}; + + public: + GlobalSnapshotHolder(VersionController* o) : owner_{o} { + snap_ = owner_->NewGlobalSnapshot(); + }; + GlobalSnapshotHolder(GlobalSnapshotHolder const&) = delete; + GlobalSnapshotHolder& operator=(GlobalSnapshotHolder const&) = delete; + GlobalSnapshotHolder(GlobalSnapshotHolder&& other) { + *this = std::move(other); + } + GlobalSnapshotHolder& operator=(GlobalSnapshotHolder&& other) { + kvdk_assert(owner_ == nullptr, ""); + kvdk_assert(other.owner_ != nullptr, ""); + kvdk_assert(snap_ == nullptr, ""); + kvdk_assert(other.snap_ != nullptr, ""); + std::swap(owner_, other.owner_); + std::swap(snap_, other.snap_); + return *this; + } + ~GlobalSnapshotHolder() { + if (owner_ != nullptr) { + owner_->ReleaseSnapshot(snap_); + } + } + TimestampType Timestamp() { return snap_->GetTimestamp(); } + }; + + class BatchWriteToken { + VersionController* owner_{nullptr}; + TimestampType ts_{kMaxTimestamp}; + + public: + BatchWriteToken(VersionController* o) + : owner_{o}, ts_{owner_->GetCurrentTimestamp()} { + ts_ = owner_->GetCurrentTimestamp(); + auto& tc = + owner_->version_thread_cache_[ThreadManager::ThreadID() % + owner_->version_thread_cache_.size()]; + kvdk_assert(tc.batch_write_ts == kMaxTimestamp, ""); + tc.batch_write_ts = ts_; + } + BatchWriteToken(BatchWriteToken const&) = delete; + BatchWriteToken& operator=(BatchWriteToken const&) = delete; + BatchWriteToken(BatchWriteToken&& other) { swap(other); } + BatchWriteToken& operator=(BatchWriteToken&& other) { + swap(other); + return *this; + } + ~BatchWriteToken() { + if (owner_ != nullptr) { + auto& tc = + owner_->version_thread_cache_[ThreadManager::ThreadID() % + owner_->version_thread_cache_.size()]; + tc.batch_write_ts = kMaxTimestamp; + } + } + TimestampType Timestamp() { return ts_; } + + private: + void swap(BatchWriteToken& other) { + kvdk_assert(owner_ == nullptr, ""); + kvdk_assert(other.owner_ != nullptr, ""); + kvdk_assert(ts_ == kMaxTimestamp, ""); + kvdk_assert(other.ts_ != kMaxTimestamp, ""); + std::swap(owner_, other.owner_); + std::swap(ts_, other.ts_); + } + }; + + public: + VersionController(uint64_t max_access_threads) + : version_thread_cache_(max_access_threads) {} + + void Init(uint64_t base_timestamp) { + tsc_on_startup_ = rdtsc(); + base_timestamp_ = base_timestamp; + UpdateLocalOldestSnapshot(); + } + + LocalSnapshotHolder GetLocalSnapshotHolder() { + return LocalSnapshotHolder{this}; + } + + BatchWriteToken GetBatchWriteToken() { return BatchWriteToken{this}; } + + std::shared_ptr GetGlobalSnapshotToken() { + return std::make_shared(this); + } + + inline void HoldLocalSnapshot() { + kvdk_assert(ThreadManager::ThreadID() >= 0, + "Uninitialized thread in NewLocalSnapshot"); + kvdk_assert(version_thread_cache_[ThreadManager::ThreadID() % + version_thread_cache_.size()] + .holding_snapshot.timestamp == kMaxTimestamp, + "Previous LocalSnapshot not released yet!"); + version_thread_cache_[ThreadManager::ThreadID() % + version_thread_cache_.size()] + .holding_snapshot.timestamp = GetCurrentTimestamp(); + } + + inline void ReleaseLocalSnapshot() { + kvdk_assert(ThreadManager::ThreadID() >= 0, + "Uninitialized thread in ReleaseLocalSnapshot"); + version_thread_cache_[ThreadManager::ThreadID() % + version_thread_cache_.size()] + .holding_snapshot.timestamp = kMaxTimestamp; + } + + inline const SnapshotImpl& GetLocalSnapshot(size_t thread_num) { + kvdk_assert(thread_num < version_thread_cache_.size(), + "Wrong thread num in GetLocalSnapshot"); + return version_thread_cache_[thread_num].holding_snapshot; + } + + inline const SnapshotImpl& GetLocalSnapshot() { + kvdk_assert(ThreadManager::ThreadID() >= 0, + "Uninitialized thread in GetLocalSnapshot"); + return version_thread_cache_[ThreadManager::ThreadID() % + version_thread_cache_.size()] + .holding_snapshot; + } + + // Create a new global snapshot + SnapshotImpl* NewGlobalSnapshot(bool may_block = true) { + TimestampType ts = GetCurrentTimestamp(); + if (may_block) { + for (size_t i = 0; i < version_thread_cache_.size(); i++) { + while (ts >= version_thread_cache_[i].batch_write_ts) { + _mm_pause(); + } + } + } else { + for (size_t i = 0; i < version_thread_cache_.size(); i++) { + TimestampType batch_ts = version_thread_cache_[i].batch_write_ts; + ts = std::min(ts, batch_ts - 1); + } + } + + std::lock_guard lg(global_snapshots_lock_); + SnapshotImpl* new_global_snapshot = global_snapshots_.New(ts); + global_oldest_snapshot_ts_ = global_snapshots_.OldestSnapshotTS(); + return new_global_snapshot; + } + + // Release a global snapshot, it should be created by this instance + void ReleaseSnapshot(const SnapshotImpl* impl) { + std::lock_guard lg(global_snapshots_lock_); + global_snapshots_.Delete(impl); + global_oldest_snapshot_ts_ = global_snapshots_.OldestSnapshotTS(); + } + + inline TimestampType GetCurrentTimestamp() { + auto res = rdtsc() - tsc_on_startup_ + base_timestamp_; + return res; + } + + TimestampType LocalOldestSnapshotTS() { + return local_oldest_snapshot_.GetTimestamp(); + } + + TimestampType GlobalOldestSnapshotTs() { + return global_oldest_snapshot_ts_.load(); + } + + // Update recorded oldest snapshot up to state by iterating every thread + // holding snapshot + void UpdateLocalOldestSnapshot() { + // update local oldest snapshot + TimestampType ts = GetCurrentTimestamp(); + for (size_t i = 0; i < version_thread_cache_.size(); i++) { + auto& tc = version_thread_cache_[i]; + ts = std::min(tc.holding_snapshot.GetTimestamp(), ts); + } + local_oldest_snapshot_.timestamp = ts; + } + + private: + // Each access thread of the instance hold its own local snapshot in thread + // cache to avoid thread contention + struct alignas(64) VersionThreadCache { + VersionThreadCache() : holding_snapshot(kMaxTimestamp) {} + + SnapshotImpl holding_snapshot; + TimestampType batch_write_ts{kMaxTimestamp}; + char padding[64 - sizeof(holding_snapshot)]; + }; + + Array version_thread_cache_; + SnapshotList global_snapshots_; + SpinMutex global_snapshots_lock_; + // Known oldest snapshot of the instance, there is delay with the actual + // oldest snapshot until call UpdatedOldestSnapshot() + SnapshotImpl local_oldest_snapshot_{kMaxTimestamp}; + std::atomic global_oldest_snapshot_ts_{kMaxTimestamp}; + + // These two used to get current timestamp of the instance + // version_base_: The newest timestamp on instance closing last time + // tsc_on_startup_: The CPU tsc on instance start up + uint64_t base_timestamp_; + uint64_t tsc_on_startup_; +}; + +class CheckPoint { + public: + void MakeCheckpoint(const Snapshot* snapshot) { + checkpoint_ts_ = static_cast(snapshot)->GetTimestamp(); + } + + void MaybeRelease(const Snapshot* releasing_snapshot) { + if (static_cast(releasing_snapshot)->GetTimestamp() == + checkpoint_ts_) { + Release(); + } + } + + void Release() { checkpoint_ts_ = 0; } + + TimestampType CheckpointTS() { return checkpoint_ts_; } + + bool Valid() { return checkpoint_ts_ > 0; } + + private: + TimestampType checkpoint_ts_; +}; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/write_batch_impl.cpp b/volatile/engine/write_batch_impl.cpp new file mode 100644 index 00000000..53adcff2 --- /dev/null +++ b/volatile/engine/write_batch_impl.cpp @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include "write_batch_impl.hpp" + +#include "alias.hpp" + +namespace KVDK_NAMESPACE { + +void BatchWriteLog::EncodeTo(char* dst) { + // TODO + // This function should not be called without PMem support. + // Checking dst == nullptr is a temporary workaround. + if (dst == nullptr) { + return; + } + kvdk_assert(stage == Stage::Initializing, ""); + + size_t total_bytes; + total_bytes = + sizeof(total_bytes) + sizeof(timestamp_) + sizeof(stage) + + sizeof(string_logs_.size()) + + string_logs_.size() * sizeof(StringLogEntry) + + sizeof(sorted_logs_.size()) + + sorted_logs_.size() * sizeof(SortedLogEntry) + sizeof(hash_logs_.size()) + + hash_logs_.size() * sizeof(HashLogEntry) + sizeof(list_logs_.size()) + + list_logs_.size() * sizeof(ListLogEntry); + + std::string buffer; + buffer.reserve(total_bytes); + + AppendPOD(&buffer, total_bytes); + AppendPOD(&buffer, timestamp_); + AppendPOD(&buffer, stage); + + AppendPOD(&buffer, string_logs_.size()); + for (size_t i = 0; i < string_logs_.size(); i++) { + AppendPOD(&buffer, string_logs_[i]); + } + + AppendPOD(&buffer, sorted_logs_.size()); + for (size_t i = 0; i < sorted_logs_.size(); i++) { + AppendPOD(&buffer, sorted_logs_[i]); + } + + AppendPOD(&buffer, hash_logs_.size()); + for (size_t i = 0; i < hash_logs_.size(); i++) { + AppendPOD(&buffer, hash_logs_[i]); + } + + AppendPOD(&buffer, list_logs_.size()); + for (size_t i = 0; i < list_logs_.size(); i++) { + AppendPOD(&buffer, list_logs_[i]); + } + + kvdk_assert(buffer.size() == total_bytes, ""); + + memcpy(dst, buffer.data(), buffer.size()); + for (size_t i = 0; i < buffer.size(); i += 64) { + _mm_clflushopt(&dst[i]); + } + _mm_mfence(); +} + +void BatchWriteLog::DecodeFrom(char const* src) { + if (src == nullptr) { + return; + } + kvdk_assert( + string_logs_.empty() && sorted_logs_.empty() && hash_logs_.empty(), ""); + + size_t total_bytes = *reinterpret_cast(src); + if (total_bytes == 0) { + return; + } + + StringView sw{src, total_bytes}; + + total_bytes = FetchPOD(&sw); + timestamp_ = FetchPOD(&sw); + stage = FetchPOD(&sw); + + if (stage == Stage::Initializing || stage == Stage::Committed) { + // No need to deserialize furthermore. + return; + } + if (stage != Stage::Processing) { + kvdk_assert(false, "Invalid Stage, invalid Log!"); + return; + } + + string_logs_.resize(FetchPOD(&sw)); + for (size_t i = 0; i < string_logs_.size(); i++) { + string_logs_[i] = FetchPOD(&sw); + } + + sorted_logs_.resize(FetchPOD(&sw)); + for (size_t i = 0; i < sorted_logs_.size(); i++) { + sorted_logs_[i] = FetchPOD(&sw); + } + + hash_logs_.resize(FetchPOD(&sw)); + for (size_t i = 0; i < hash_logs_.size(); i++) { + hash_logs_[i] = FetchPOD(&sw); + } + + list_logs_.resize(FetchPOD(&sw)); + for (size_t i = 0; i < list_logs_.size(); i++) { + list_logs_[i] = FetchPOD(&sw); + } + + kvdk_assert(sw.size() == 0, ""); +} + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/engine/write_batch_impl.hpp b/volatile/engine/write_batch_impl.hpp new file mode 100644 index 00000000..848f3586 --- /dev/null +++ b/volatile/engine/write_batch_impl.hpp @@ -0,0 +1,311 @@ +#pragma once + +#include + +#include +#include +#include + +#include "alias.hpp" +#include "hash_table.hpp" +#include "kvdk/volatile/write_batch.hpp" +#include "utils/codec.hpp" +#include "utils/utils.hpp" + +namespace KVDK_NAMESPACE { + +class WriteBatchImpl final : public WriteBatch { + public: + struct StringOp { + StringOp(WriteOp o, const StringView& k, const StringView& v) + : op(o), key(string_view_2_string(k)), value(string_view_2_string(v)) {} + + WriteOp op; + std::string key; + std::string value; + }; + + struct SortedOp { + SortedOp(WriteOp o, const StringView& c, const StringView& k, + const StringView& v) + : op(o), + collection(string_view_2_string(c)), + key(string_view_2_string(k)), + value(string_view_2_string(v)) {} + + WriteOp op; + std::string collection; + std::string key; + std::string value; + }; + + struct HashOp { + HashOp(WriteOp o, const StringView& c, const StringView& k, + const StringView& v) + : op(o), + collection(string_view_2_string(c)), + key(string_view_2_string(k)), + value(string_view_2_string(v)) {} + + WriteOp op; + std::string collection; + std::string key; + std::string value; + }; + + struct HashEq { + size_t operator()(StringOp const& string_op) const { + return xxh_hash(string_op.key); + } + size_t operator()(SortedOp const& sorted_op) const { + return xxh_hash(sorted_op.collection) ^ xxh_hash(sorted_op.key); + } + size_t operator()(HashOp const& hash_op) const { + return xxh_hash(hash_op.collection) ^ xxh_hash(hash_op.key); + } + bool operator()(StringOp const& lhs, StringOp const& rhs) const { + return lhs.key == rhs.key; + } + bool operator()(SortedOp const& lhs, SortedOp const& rhs) const { + return lhs.collection == rhs.collection && lhs.key == rhs.key; + } + bool operator()(HashOp const& lhs, HashOp const& rhs) const { + return lhs.collection == rhs.collection && lhs.key == rhs.key; + } + }; + + void StringPut(const StringView key, const StringView value) final { + StringOp op{WriteOp::Put, key, value}; + string_ops_.erase(op); + string_ops_.insert(op); + } + + void StringDelete(const StringView key) final { + StringOp op{WriteOp::Delete, key, std::string{}}; + string_ops_.erase(op); + string_ops_.insert(op); + } + + void SortedPut(const StringView collection, const StringView key, + const StringView value) final { + SortedOp op{WriteOp::Put, collection, key, value}; + sorted_ops_.erase(op); + sorted_ops_.insert(op); + } + + void SortedDelete(const StringView collection, const StringView key) final { + SortedOp op{WriteOp::Delete, collection, key, std::string{}}; + sorted_ops_.erase(op); + sorted_ops_.insert(op); + } + + void HashPut(const StringView collection, const StringView key, + const StringView value) final { + HashOp op{WriteOp::Put, collection, key, value}; + hash_ops_.erase(op); + hash_ops_.insert(op); + } + + void HashDelete(const StringView collection, const StringView key) final { + HashOp op{WriteOp::Delete, collection, key, std::string{}}; + hash_ops_.erase(op); + hash_ops_.insert(op); + } + + void Clear() final { + string_ops_.clear(); + sorted_ops_.clear(); + hash_ops_.clear(); + } + + size_t Size() const final { + return string_ops_.size() + sorted_ops_.size() + hash_ops_.size(); + } + + using StringOpBatch = std::unordered_set; + using SortedOpBatch = std::unordered_set; + using HashOpBatch = std::unordered_set; + + StringOpBatch const& StringOps() const { return string_ops_; } + SortedOpBatch const& SortedOps() const { return sorted_ops_; } + HashOpBatch const& HashOps() const { return hash_ops_; } + + private: + StringOpBatch string_ops_; + SortedOpBatch sorted_ops_; + HashOpBatch hash_ops_; +}; + +struct StringWriteArgs { + StringView key; + StringView value; + WriteOp op; + SpaceEntry space; + TimestampType ts; + HashTable::LookupResult res; + StringRecord* new_rec; + + void Assign(WriteBatchImpl::StringOp const& string_op) { + key = string_op.key; + value = string_op.value; + op = string_op.op; + } +}; + +class BatchWriteLog { + public: + // The batch is first persisted to PMem with Stage::Initializing. + // After persisting is done, it enters Stage::Processing. + // When all batches are executed, it is marked as Stage::Committed + // and then purged from PMem. + // During recovery, a batch in + // Stage::Initializing is directly discarded and purged. + // Stage::Processing is rolled back. + // Stage::Committed is directly purged. + enum class Stage : size_t { + // Initializing must be 0 so that empty file can be skipped. + Initializing = 0, + Processing, + Committed, + }; + + enum class Op : size_t { Put, Delete }; + + struct StringLogEntry { + Op op; + MemoryOffsetType offset; + }; + + struct SortedLogEntry { + Op op; + MemoryOffsetType offset; + }; + + struct HashLogEntry { + Op op; + MemoryOffsetType offset; + }; + + struct ListLogEntry { + Op op; + MemoryOffsetType offset; + }; + + explicit BatchWriteLog() {} + + void SetTimestamp(TimestampType ts) { timestamp_ = ts; } + + void StringPut(MemoryOffsetType offset) { + string_logs_.emplace_back(StringLogEntry{Op::Put, offset}); + } + + void StringDelete(MemoryOffsetType offset) { + string_logs_.emplace_back(StringLogEntry{Op::Delete, offset}); + } + + void SortedPut(MemoryOffsetType offset) { + sorted_logs_.emplace_back(SortedLogEntry{Op::Put, offset}); + } + + void SortedDelete(MemoryOffsetType offset) { + sorted_logs_.emplace_back(SortedLogEntry{Op::Delete, offset}); + } + + void HashPut(MemoryOffsetType offset) { + hash_logs_.emplace_back(HashLogEntry{Op::Put, offset}); + } + + void HashDelete(MemoryOffsetType offset) { + hash_logs_.emplace_back(HashLogEntry{Op::Delete, offset}); + } + + void ListEmplace(MemoryOffsetType offset) { + list_logs_.emplace_back(ListLogEntry{Op::Put, offset}); + } + + void ListDelete(MemoryOffsetType offset) { + list_logs_.emplace_back(ListLogEntry{Op::Delete, offset}); + } + + void Clear() { + string_logs_.clear(); + sorted_logs_.clear(); + hash_logs_.clear(); + list_logs_.clear(); + } + + size_t Size() const { + return string_logs_.size() + sorted_logs_.size() + hash_logs_.size() + + list_logs_.size(); + } + + static size_t Capacity() { return (1UL << 20); } + + static size_t MaxBytes() { + static_assert(sizeof(HashLogEntry) >= sizeof(StringLogEntry), ""); + static_assert(sizeof(HashLogEntry) >= sizeof(SortedLogEntry), ""); + static_assert(sizeof(HashLogEntry) >= sizeof(ListLogEntry), ""); + return sizeof(size_t) + sizeof(TimestampType) + sizeof(Stage) + + sizeof(size_t) + Capacity() * sizeof(HashLogEntry); + } + + // Format of the BatchWriteLog + // total_bytes | timestamp | stage | + // N | StringLogEntry*N | + // M | SortedLogEntry*M + // K | HashLogEntry*K + // L | ListLogEntry*K + // dst is expected to have capacity of MaxBytes(). + void EncodeTo(char* dst); + + void DecodeFrom(char const* src); + + static void MarkProcessing(char* dst) { + if (dst == nullptr) { + return; + } + dst = &dst[sizeof(size_t) + sizeof(TimestampType)]; + *reinterpret_cast(dst) = Stage::Processing; + _mm_clflush(dst); + _mm_mfence(); + } + + static void MarkCommitted(char* dst) { + if (dst == nullptr) { + return; + } + dst = &dst[sizeof(size_t) + sizeof(TimestampType)]; + *reinterpret_cast(dst) = Stage::Committed; + _mm_clflush(dst); + _mm_mfence(); + } + + // For rollback + static void MarkInitializing(char* dst) { + dst = &dst[sizeof(size_t) + sizeof(TimestampType)]; + *reinterpret_cast(dst) = Stage::Initializing; + _mm_clflush(dst); + _mm_mfence(); + } + + using StringLog = std::vector; + using SortedLog = std::vector; + using HashLog = std::vector; + using ListLog = std::vector; + + StringLog const& StringLogs() const { return string_logs_; } + SortedLog const& SortedLogs() const { return sorted_logs_; } + HashLog const& HashLogs() const { return hash_logs_; } + ListLog const& ListLogs() const { return list_logs_; } + TimestampType Timestamp() const { return timestamp_; } + + private: + Stage stage{Stage::Initializing}; + TimestampType timestamp_; + StringLog string_logs_; + SortedLog sorted_logs_; + HashLog hash_logs_; + ListLog list_logs_; +}; + +} // namespace KVDK_NAMESPACE diff --git a/volatile/examples/tutorial/CMakeLists.txt b/volatile/examples/tutorial/CMakeLists.txt new file mode 100644 index 00000000..9613747a --- /dev/null +++ b/volatile/examples/tutorial/CMakeLists.txt @@ -0,0 +1,18 @@ +if (NOT BUILD_TUTORIAL) + return() +endif() + +enable_testing() + +file(GLOB sources + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.c) + +foreach(src ${sources}) + file(RELATIVE_PATH src_rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${src}) + string(REPLACE ".cpp" "" example_name ${src_rel_path}) + string(REPLACE ".c" "" example_name ${example_name}) + string(REGEX REPLACE "[/\\.]" "_" example_name ${example_name}) + add_executable(${example_name} ${src}) + target_link_libraries(${example_name} PUBLIC engine) +endforeach() diff --git a/volatile/examples/tutorial/c_api_tutorial.c b/volatile/examples/tutorial/c_api_tutorial.c new file mode 100644 index 00000000..af310b96 --- /dev/null +++ b/volatile/examples/tutorial/c_api_tutorial.c @@ -0,0 +1,766 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include +#include +#include +#include + +#include "kvdk/volatile/engine.h" + +// The KVDK instance is mounted as a directory +// /mnt/pmem0/tutorial_kvdk_example. +// Modify this path if necessary. +const char* pmem_path = "/mnt/pmem0/tutorial_kvdk_example"; + +static int StrCmp(const char* a, size_t alen, const char* b, size_t blen) { + int n = (alen < blen) ? alen : blen; + int r = memcmp(a, b, n); + if (r == 0) { + if (alen < blen) + r = -1; + else if (alen > blen) + r = +1; + } + return r; +} + +void AnonymousCollectionExample(KVDKEngine* kvdk_engine) { + const char* key1 = "key1"; + const char* key2 = "key2"; + const char* value1 = "value1"; + const char* value2 = "value2"; + char* read_v1; + char* read_v2; + size_t key1_len = strlen(key1); + size_t key2_len = strlen(key2); + size_t value1_len = strlen(value1); + size_t value2_len = strlen(value2); + size_t read_v1_len, read_v2_len; + int cmp; + KVDKWriteOptions* write_option = KVDKCreateWriteOptions(); + KVDKStatus s = + KVDKPut(kvdk_engine, key1, key1_len, value1, value1_len, write_option); + assert(s == Ok); + s = KVDKPut(kvdk_engine, key2, key1_len, value2, value2_len, write_option); + assert(s == Ok); + s = KVDKGet(kvdk_engine, key1, key1_len, &read_v1_len, &read_v1); + assert(s == Ok); + cmp = StrCmp(read_v1, read_v1_len, value1, value1_len); + assert(cmp == 0); + free(read_v1); + s = KVDKPut(kvdk_engine, key1, key1_len, value2, value2_len, write_option); + assert(s == Ok); + s = KVDKGet(kvdk_engine, key1, key1_len, &read_v1_len, &read_v1); + assert(s == Ok); + cmp = StrCmp(read_v1, read_v1_len, value2, value2_len); + assert(cmp == 0); + s = KVDKGet(kvdk_engine, key2, key2_len, &read_v2_len, &read_v2); + assert(s == Ok); + cmp = StrCmp(read_v2, read_v2_len, value2, value2_len); + assert(cmp == 0); + s = KVDKDelete(kvdk_engine, key1, key1_len); + assert(s == Ok); + s = KVDKDelete(kvdk_engine, key2, key2_len); + assert(s == Ok); + free(read_v1); + free(read_v2); + printf( + "Successfully performed Get, Put, Delete operations on anonymous " + "global collection.\n"); + + KVDKDestroyWriteOptions(write_option); +} + +// Reads and Writes on Named Sorted Collection +void SortedCollectionExample(KVDKEngine* kvdk_engine) { + const char* collection1 = "collection1"; + const char* collection2 = "collection2"; + const char* key1 = "key1"; + const char* key2 = "key2"; + const char* value1 = "value1"; + const char* value2 = "value2"; + char* read_v1; + char* read_v2; + size_t read_v1_len, read_v2_len; + int cmp; + + KVDKSortedCollectionConfigs* s_configs = KVDKCreateSortedCollectionConfigs(); + KVDKStatus s = KVDKSortedCreate(kvdk_engine, collection1, strlen(collection1), + s_configs); + assert(s == Ok); + s = KVDKSortedCreate(kvdk_engine, collection2, strlen(collection2), + s_configs); + assert(s == Ok); + s = KVDKSortedPut(kvdk_engine, collection1, strlen(collection1), key1, + strlen(key1), value1, strlen(value1)); + assert(s == Ok); + s = KVDKSortedPut(kvdk_engine, collection2, strlen(collection2), key2, + strlen(key2), value2, strlen(value2)); + assert(s == Ok); + s = KVDKSortedGet(kvdk_engine, collection1, strlen(collection1), key1, + strlen(key1), &read_v1_len, &read_v1); + assert(s == Ok); + cmp = StrCmp(read_v1, read_v1_len, value1, strlen(value1)); + assert(cmp == 0); + free(read_v1); + s = KVDKSortedPut(kvdk_engine, collection1, strlen(collection1), key1, + strlen(key1), value2, strlen(value2)); + assert(s == Ok); + s = KVDKSortedGet(kvdk_engine, collection1, strlen(collection1), key1, + strlen(key1), &read_v1_len, &read_v1); + assert(s == Ok); + cmp = StrCmp(read_v1, read_v1_len, value2, strlen(value2)); + assert(cmp == 0); + s = KVDKSortedGet(kvdk_engine, collection2, strlen(collection2), key2, + strlen(key2), &read_v2_len, &read_v2); + assert(s == Ok); + cmp = StrCmp(read_v2, read_v2_len, value2, strlen(value2)); + assert(cmp == 0); + s = KVDKSortedDelete(kvdk_engine, collection1, strlen(collection1), key1, + strlen(key1)); + assert(s == Ok); + s = KVDKSortedDelete(kvdk_engine, collection2, strlen(collection2), key2, + strlen(key2)); + assert(s == Ok); + s = KVDKSortedDestroy(kvdk_engine, collection1, strlen(collection1)); + assert(s == Ok); + s = KVDKSortedDestroy(kvdk_engine, collection2, strlen(collection2)); + assert(s == Ok); + s = KVDKSortedPut(kvdk_engine, collection1, strlen(collection1), key1, + strlen(key1), value1, strlen(value1)); + assert(s == NotFound); + free(read_v1); + free(read_v2); + s = KVDKSortedGet(kvdk_engine, collection1, strlen(collection1), key2, + strlen(key2), &read_v2_len, &read_v2); + assert(s == NotFound); + KVDKDestroySortedCollectionConfigs(s_configs); + printf( + "Successfully performed SortedGet, SortedPut, SortedDelete " + "operations on named " + "collections.\n"); +} + +void SortedIteratorExample(KVDKEngine* kvdk_engine) { + const char* nums[10] = {"4", "5", "0", "2", "9", "1", "3", "8", "6", "7"}; + const char* sorted_nums[10] = {"0", "1", "2", "3", "4", + "5", "6", "7", "8", "9"}; + const char* sorted_collection = "sorted_collection"; + KVDKSortedCollectionConfigs* s_configs = KVDKCreateSortedCollectionConfigs(); + KVDKStatus s = KVDKSortedCreate(kvdk_engine, sorted_collection, + strlen(sorted_collection), s_configs); + assert(s == Ok); + for (int i = 0; i < 10; ++i) { + char key[10] = "key", value[10] = "value"; + strcat(key, nums[i]); + strcat(value, nums[i]); + s = KVDKSortedPut(kvdk_engine, sorted_collection, strlen(sorted_collection), + key, strlen(key), value, strlen(value)); + assert(s == Ok); + } + // create sorted iterator + KVDKSortedIterator* kvdk_iter = KVDKSortedIteratorCreate( + kvdk_engine, sorted_collection, strlen(sorted_collection), NULL, NULL); + KVDKSortedIteratorSeekToFirst(kvdk_iter); + // Iterate through range ["key1", "key8"). + const char* beg = "key1"; + const char* end = "key8"; + + int i = 1; + for (KVDKSortedIteratorSeek(kvdk_iter, beg, strlen(beg)); + KVDKSortedIteratorValid(kvdk_iter); KVDKSortedIteratorNext(kvdk_iter)) { + char expected_key[10] = "key", expected_value[10] = "value"; + strcat(expected_key, sorted_nums[i]); + strcat(expected_value, sorted_nums[i]); + size_t key_len, val_len; + char *key_res, *val_res; + KVDKSortedIteratorKey(kvdk_iter, &key_res, &key_len); + KVDKSortedIteratorValue(kvdk_iter, &val_res, &val_len); + if (StrCmp(key_res, key_len, end, strlen(end)) > 0) { + free(key_res); + free(val_res); + break; + } + int cmp = StrCmp(key_res, key_len, expected_key, strlen(expected_key)); + assert(cmp == 0); + cmp = StrCmp(val_res, val_len, expected_value, strlen(expected_value)); + assert(cmp == 0); + free(key_res); + free(val_res); + ++i; + } + assert(i == 9); + + // Iterate through range ["key8", "key1"). + beg = "key8"; + end = "key1"; + + i = 8; + for (KVDKSortedIteratorSeek(kvdk_iter, beg, strlen(beg)); + KVDKSortedIteratorValid(kvdk_iter); KVDKSortedIteratorPrev(kvdk_iter)) { + char expected_key[10] = "key", expected_value[10] = "value"; + strcat(expected_key, sorted_nums[i]); + strcat(expected_value, sorted_nums[i]); + size_t key_len, val_len; + char *key_res, *val_res; + KVDKSortedIteratorKey(kvdk_iter, &key_res, &key_len); + KVDKSortedIteratorValue(kvdk_iter, &val_res, &val_len); + if (StrCmp(key_res, key_len, end, strlen(end)) < 0) { + free(key_res); + free(val_res); + break; + } + int cmp = StrCmp(key_res, key_len, expected_key, strlen(expected_key)); + assert(cmp == 0); + cmp = StrCmp(val_res, val_len, expected_value, strlen(expected_value)); + assert(cmp == 0); + free(key_res); + free(val_res); + --i; + } + assert(i == 0); + printf("Successfully iterated through a sorted named collections.\n"); + KVDKSortedIteratorDestroy(kvdk_engine, kvdk_iter); + KVDKDestroySortedCollectionConfigs(s_configs); +} + +int score_cmp(const char* a, size_t a_len, const char* b, size_t b_len) { + char a_buff[a_len + 1]; + char b_buff[b_len + 1]; + memcpy(a_buff, a, a_len); + memcpy(b_buff, b, b_len); + a_buff[a_len] = '\0'; + b_buff[b_len] = '\0'; + double scorea = atof(a_buff); + double scoreb = atof(b_buff); + if (scorea == scoreb) + return 0; + else if (scorea < scoreb) + return 1; + else + return -1; +} + +void CompFuncForSortedCollectionExample(KVDKEngine* kvdk_engine) { + const char* collection = "collection0"; + struct number_kv { + const char* number_key; + const char* value; + }; + + struct number_kv array[5] = { + {"100", "a"}, {"50", "c"}, {"40", "d"}, {"30", "b"}, {"90", "f"}}; + + struct number_kv expected_array[5] = { + {"100", "a"}, {"90", "f"}, {"50", "c"}, {"40", "d"}, {"30", "b"}}; + + // regitser compare function + const char* comp_name = "double_comp"; + KVDKRegisterCompFunc(kvdk_engine, comp_name, strlen(comp_name), score_cmp); + // create sorted collection + KVDKSortedCollectionConfigs* s_configs = KVDKCreateSortedCollectionConfigs(); + KVDKSetSortedCollectionConfigs(s_configs, comp_name, strlen(comp_name), 1); + KVDKStatus s = + KVDKSortedCreate(kvdk_engine, collection, strlen(collection), s_configs); + assert(s == Ok); + for (int i = 0; i < 5; ++i) { + s = KVDKSortedPut(kvdk_engine, collection, strlen(collection), + array[i].number_key, strlen(array[i].number_key), + array[i].value, strlen(array[i].value)); + assert(s == Ok); + } + KVDKSortedIterator* iter = KVDKSortedIteratorCreate( + kvdk_engine, collection, strlen(collection), NULL, NULL); + assert(iter != NULL); + + int i = 0; + for (KVDKSortedIteratorSeekToFirst(iter); KVDKSortedIteratorValid(iter); + KVDKSortedIteratorNext(iter)) { + size_t key_len, value_len; + char *key, *value; + KVDKSortedIteratorKey(iter, &key, &key_len); + KVDKSortedIteratorValue(iter, &value, &value_len); + if (StrCmp(key, key_len, expected_array[i].number_key, + strlen(expected_array[i].number_key)) != 0) { + printf("sort key error, current key: %s , but expected key: %s\n", key, + expected_array[i].number_key); + } + if (StrCmp(value, value_len, expected_array[i].value, + strlen(expected_array[i].value)) != 0) { + printf("sort value error, current value: %s , but expected value: %s\n", + value, expected_array[i].value); + } + free(key); + free(value); + ++i; + } + KVDKSortedIteratorDestroy(kvdk_engine, iter); + printf("Successfully collections sorted by number.\n"); + KVDKDestroySortedCollectionConfigs(s_configs); +} + +void BatchWriteAnonCollectionExample(KVDKEngine* kvdk_engine) { + const char* key1 = "key1"; + const char* key2 = "key2"; + const char* value1 = "value1"; + const char* value2 = "value2"; + char* read_v1; + char* read_v2; + size_t read_v1_len, read_v2_len; + KVDKWriteBatch* kvdk_wb = KVDKWriteBatchCreate(kvdk_engine); + KVDKWriteBatchStringPut(kvdk_wb, key1, strlen(key1), value1, strlen(value1)); + KVDKWriteBatchStringPut(kvdk_wb, key2, strlen(key2), value2, strlen(value2)); + KVDKWriteBatchStringDelete(kvdk_wb, key1, strlen(key1)); + KVDKWriteBatchStringPut(kvdk_wb, key1, strlen(key1), value2, strlen(value1)); + KVDKStatus s = KVDKBatchWrite(kvdk_engine, kvdk_wb); + assert(s == Ok); + s = KVDKGet(kvdk_engine, key1, strlen(key1), &read_v1_len, &read_v1); + assert(s == Ok); + int cmp = StrCmp(read_v1, read_v1_len, value2, strlen(value2)); + assert(cmp == 0); + s = KVDKGet(kvdk_engine, key2, strlen(key2), &read_v2_len, &read_v2); + assert(s == Ok); + cmp = StrCmp(read_v2, read_v2_len, value2, strlen(value2)); + assert(cmp == 0); + printf("Successfully performed BatchWrite on anonymous global collection.\n"); + KVDKWriteBatchDestory(kvdk_wb); + free(read_v1); + free(read_v2); +} + +void HashesCollectionExample(KVDKEngine* kvdk_engine) { + const char* nums[10] = {"9", "5", "2", "0", "7", "3", "1", "8", "6", "4"}; + const char* hash_collection = "hash_collection"; + KVDKStatus s = + KVDKHashCreate(kvdk_engine, hash_collection, strlen(hash_collection)); + assert(s == Ok); + for (int i = 0; i < 10; ++i) { + char key[10] = "key", value[10] = "value"; + strcat(key, nums[i]); + strcat(value, nums[i]); + s = KVDKHashPut(kvdk_engine, hash_collection, strlen(hash_collection), key, + strlen(key), value, strlen(value)); + assert(s == Ok); + size_t val_len; + char* val; + s = KVDKHashGet(kvdk_engine, hash_collection, strlen(hash_collection), key, + strlen(key), &val, &val_len); + assert(s == Ok); + int cmp = StrCmp(val, val_len, value, strlen(value)); + assert(cmp == 0); + free(val); + } + + s = KVDKHashDelete(kvdk_engine, hash_collection, strlen(hash_collection), + "key8", strlen("key8")); + assert(s == Ok); + // create hash iterator + KVDKHashIterator* kvdk_iter = KVDKHashIteratorCreate( + kvdk_engine, hash_collection, strlen(hash_collection), NULL, NULL); + assert(kvdk_iter != NULL); + int cnt = 0; + for (KVDKHashIteratorSeekToFirst(kvdk_iter); + KVDKHashIteratorIsValid(kvdk_iter); KVDKHashIteratorNext(kvdk_iter)) { + ++cnt; + } + assert(cnt == 9); + + cnt = 0; + for (KVDKHashIteratorSeekToLast(kvdk_iter); + KVDKHashIteratorIsValid(kvdk_iter); KVDKHashIteratorPrev(kvdk_iter)) { + ++cnt; + } + printf("Successfully performed Get Put Delete Iterate on HashList.\n"); + assert(cnt == 9); + KVDKHashIteratorDestroy(kvdk_engine, kvdk_iter); + + s = KVDKHashDestroy(kvdk_engine, hash_collection, strlen(hash_collection)); + assert(s == Ok); +} + +void ListsCollectionExample(KVDKEngine* kvdk_engine) { + const char* nums[10] = {"9", "5", "2", "0", "7", "3", "1", "8", "6", "4"}; + const char* list_collection = "list_collection"; + KVDKStatus s = + KVDKListCreate(kvdk_engine, list_collection, strlen(list_collection)); + assert(s == Ok); + for (int i = 0; i < 10; ++i) { + char key[10] = "key"; + strcat(key, nums[i]); + KVDKStatus s = KVDKListPushFront(kvdk_engine, list_collection, + strlen(list_collection), key, strlen(key)); + assert(s == Ok); + size_t key_len_res; + char* key_res; + s = KVDKListPopFront(kvdk_engine, list_collection, strlen(list_collection), + &key_res, &key_len_res); + assert(s == Ok); + int cmp = StrCmp(key_res, key_len_res, key, strlen(key)); + assert(cmp == 0); + free(key_res); + } + + for (int i = 0; i < 10; ++i) { + char key[10] = "value"; + strcat(key, nums[i]); + KVDKStatus s = KVDKListPushBack(kvdk_engine, list_collection, + strlen(list_collection), key, strlen(key)); + assert(s == Ok); + size_t key_len_res; + char* key_res; + s = KVDKListPopBack(kvdk_engine, list_collection, strlen(list_collection), + &key_res, &key_len_res); + assert(s == Ok); + int cmp = StrCmp(key_res, key_len_res, key, strlen(key)); + assert(cmp == 0); + free(key_res); + } + + s = KVDKListDestroy(kvdk_engine, list_collection, strlen(list_collection)); + assert(s == Ok); + + printf("Successfully performed RPush RPop LPush LPop on Lists.\n"); +} + +void ExpireExample(KVDKEngine* kvdk_engine) { + int64_t ttl_time; + char* got_val; + size_t val_len; + KVDKStatus s; + // For string + { + const char* key = "stringkey"; + const char* val = "stringval"; + // case: set expire time + KVDKWriteOptions* write_option = KVDKCreateWriteOptions(); + KVDKWriteOptionsSetTTLTime(write_option, 100); + s = KVDKPut(kvdk_engine, key, strlen(key), val, strlen(val), write_option); + assert(s == Ok); + s = KVDKGet(kvdk_engine, key, strlen(key), &val_len, &got_val); + assert(s == Ok); + int cmp = StrCmp(got_val, val_len, val, strlen(val)); + assert(cmp == 0); + free(got_val); + s = KVDKGetTTL(kvdk_engine, key, strlen(key), &ttl_time); + assert(s == Ok); + // case: reset expire time + s = KVDKExpire(kvdk_engine, key, strlen(key), INT32_MAX); + assert(s == Ok); + // case: change to persist key + s = KVDKExpire(kvdk_engine, key, strlen(key), INT64_MAX); + assert(s == Ok); + s = KVDKGetTTL(kvdk_engine, key, strlen(key), &ttl_time); + assert(s == Ok); + assert(ttl_time == INT64_MAX); + // case: keep expire time unchanged while updating a existing key + KVDKWriteOptionsSetUpdateTTL(write_option, 0); + KVDKWriteOptionsSetTTLTime(write_option, 0); + s = KVDKPut(kvdk_engine, key, strlen(key), val, strlen(val), write_option); + assert(s == Ok); + s = KVDKGetTTL(kvdk_engine, key, strlen(key), &ttl_time); + assert(s == Ok); + assert(ttl_time == INT64_MAX); + // case: key is expired. + s = KVDKExpire(kvdk_engine, key, strlen(key), 1); + assert(s == Ok); + sleep(1); + s = KVDKGet(kvdk_engine, key, strlen(key), &val_len, &got_val); + assert(s == NotFound); + // case: ttl time is negative or 0 + s = KVDKPut(kvdk_engine, key, strlen(key), val, strlen(val), write_option); + assert(s == Ok); + s = KVDKGet(kvdk_engine, key, strlen(key), &val_len, &got_val); + assert(s == NotFound); + // No need to free(got_val) + printf("Successfully expire string\n"); + + KVDKDestroyWriteOptions(write_option); + } + + { + const char* sorted_collection = "sorted_collection"; + const char* key = "sortedkey"; + const char* val = "sortedval"; + + // case: default persist key. + KVDKSortedCollectionConfigs* s_configs = + KVDKCreateSortedCollectionConfigs(); + s = KVDKSortedCreate(kvdk_engine, sorted_collection, + strlen(sorted_collection), s_configs); + assert(s == Ok || s == Existed); + s = KVDKGetTTL(kvdk_engine, sorted_collection, strlen(sorted_collection), + &ttl_time); + assert(s == Ok); + assert(ttl_time == INT64_MAX); + s = KVDKSortedPut(kvdk_engine, sorted_collection, strlen(sorted_collection), + key, strlen(key), val, strlen(val)); + assert(s == Ok); + // case: set expire_time + s = KVDKExpire(kvdk_engine, sorted_collection, strlen(sorted_collection), + INT32_MAX); + assert(s == Ok); + // case: change to persist key + s = KVDKExpire(kvdk_engine, sorted_collection, strlen(sorted_collection), + INT64_MAX); + s = KVDKGetTTL(kvdk_engine, sorted_collection, strlen(sorted_collection), + &ttl_time); + assert(s == Ok); + assert(ttl_time == INT64_MAX); + // case: key is expired. + s = KVDKExpire(kvdk_engine, sorted_collection, strlen(sorted_collection), + 1); + assert(s == Ok); + sleep(1); + s = KVDKSortedGet(kvdk_engine, sorted_collection, strlen(sorted_collection), + key, strlen(key), &val_len, &got_val); + assert(s == NotFound); + free(got_val); + printf("Successfully expire sorted\n"); + + KVDKDestroySortedCollectionConfigs(s_configs); + } + + { + const char* hash_collection = "hash"; + const char* key = "hashkey"; + const char* val = "hashval"; + + s = KVDKHashCreate(kvdk_engine, hash_collection, strlen(hash_collection)); + assert(s == Ok); + s = KVDKHashPut(kvdk_engine, hash_collection, strlen(hash_collection), key, + strlen(key), val, strlen(val)); + assert(s == Ok); + s = KVDKGetTTL(kvdk_engine, hash_collection, strlen(hash_collection), + &ttl_time); + assert(s == Ok); + assert(ttl_time == INT64_MAX); + + // case: set expire_time + s = KVDKExpire(kvdk_engine, hash_collection, strlen(hash_collection), 1); + assert(s == Ok); + // case: change to persist key + s = KVDKExpire(kvdk_engine, hash_collection, strlen(hash_collection), + INT64_MAX); + s = KVDKGetTTL(kvdk_engine, hash_collection, strlen(hash_collection), + &ttl_time); + assert(s == Ok); + assert(ttl_time == INT64_MAX); + // case: key is expired. + s = KVDKExpire(kvdk_engine, hash_collection, strlen(hash_collection), 1); + assert(s == Ok); + sleep(1); + s = KVDKHashGet(kvdk_engine, hash_collection, strlen(hash_collection), key, + strlen(key), &got_val, &val_len); + assert(s == NotFound); + + s = KVDKHashDestroy(kvdk_engine, hash_collection, strlen(hash_collection)); + assert(s == NotFound); + + printf("Successfully expire hash\n"); + } + return; +} + +typedef struct { + int incr_by; + int result; +} IncNArgs; + +int IncN(const char* old_val, size_t old_val_len, char** new_val, + size_t* new_val_len, void* args_pointer) { + assert(args_pointer); + + int old_num; + if (old_val == NULL) { + old_num = 0; + } else { + if (old_val_len != sizeof(int)) { + return KVDK_MODIFY_ABORT; + } + assert(old_val_len == sizeof(int)); + memcpy(&old_num, old_val, sizeof(int)); + } + + IncNArgs* args = (IncNArgs*)args_pointer; + *new_val = (char*)malloc(sizeof(int)); + if (*new_val == NULL) { + return KVDK_MODIFY_ABORT; + } + + *new_val_len = sizeof(int); + + args->result = old_num + args->incr_by; + memcpy(*new_val, &args->result, sizeof(int)); + return KVDK_MODIFY_WRITE; +} + +void ModifyExample(KVDKEngine* kvdk_engine) { + KVDKWriteOptions* write_option = KVDKCreateWriteOptions(); + char* incr_key = "incr"; + char* wrong_value_key = "wrong"; + IncNArgs args; + args.incr_by = 5; + + KVDKStatus s = KVDKPut(kvdk_engine, wrong_value_key, strlen(wrong_value_key), + "a", 1, write_option); + assert(s == Ok); + s = KVDKModify(kvdk_engine, wrong_value_key, strlen(wrong_value_key), IncN, + &args, free, write_option); + assert(s == Abort); + + int recycle = 100; + for (int i = 1; i <= recycle; i++) { + KVDKStatus s = KVDKModify(kvdk_engine, incr_key, strlen(incr_key), IncN, + &args, free, write_option); + assert(s == Ok); + assert(args.result == args.incr_by * (int)i); + + char* val; + size_t val_len; + s = KVDKGet(kvdk_engine, incr_key, strlen(incr_key), &val_len, &val); + assert(s == Ok); + assert(val_len == sizeof(int)); + int current_num; + memcpy(¤t_num, val, sizeof(int)); + assert(current_num == args.incr_by * i); + free(val); + } + KVDKDestroyWriteOptions(write_option); + printf("Successfully increase num by %d\n", args.incr_by); +} + +typedef struct { + char const* data; + size_t len; + size_t ret; +} HPutArgs; +// new_data is not touched as long as caller does not pass a free_func to +// KVDKHashModify This avoids a malloc() in HPutNXFunc and a free() in +// KVDKHashModify() +int HPutNXFunc(char const* old_data, size_t old_len, char** new_data, + size_t* new_len, void* args) { + HPutArgs* my_args = (HPutArgs*)args; + if (old_data == NULL) { + assert(old_len == 0); + *new_data = (char*)my_args->data; + *new_len = my_args->len; + my_args->ret = 1; + return KVDK_MODIFY_WRITE; + } else { + my_args->ret = 0; + return KVDK_MODIFY_NOOP; + } +} +int HPutFunc(char const* old_data, size_t old_len, char** new_data, + size_t* new_len, void* args) { + HPutArgs* my_args = (HPutArgs*)args; + *new_data = (char*)my_args->data; + *new_len = my_args->len; + if (old_data == NULL) { + assert(old_len == 0); + my_args->ret = 1; + } else { + my_args->ret = 0; + } + return KVDK_MODIFY_WRITE; +} + +void HashModifyExample(KVDKEngine* engine) { + char const* key = "my_hash"; + char const* field = "my_field"; + char const* value1 = "hello"; + char const* value2 = "my_field"; + char* resp_data; + size_t resp_len; + + HPutArgs args; + args.data = value1; + args.len = strlen(value1); + KVDKStatus s = KVDKHashCreate(engine, key, strlen(key)); + s = KVDKHashModify(engine, key, strlen(key), field, strlen(field), HPutNXFunc, + &args, NULL); + assert(s == Ok); + assert(args.ret == 1); + s = KVDKHashGet(engine, key, strlen(key), field, strlen(field), &resp_data, + &resp_len); + assert(s == Ok); + assert(resp_len == strlen(value1)); + assert(StrCmp(resp_data, resp_len, value1, strlen(value1)) == 0); + free(resp_data); + + args.data = value2; + args.len = strlen(value2); + s = KVDKHashModify(engine, key, strlen(key), field, strlen(field), HPutNXFunc, + &args, NULL); + assert(s == Ok); + assert(args.ret == 0); // Fail to set since the field already exists + s = KVDKHashGet(engine, key, strlen(key), field, strlen(field), &resp_data, + &resp_len); + assert(s == Ok); + assert(resp_len == strlen(value1)); + assert(StrCmp(resp_data, resp_len, value1, strlen(value1)) == + 0); // field is untouched + free(resp_data); + + s = KVDKHashModify(engine, key, strlen(key), field, strlen(field), HPutFunc, + &args, NULL); + assert(s == Ok); + assert(args.ret == 0); // Update, no field added + s = KVDKHashGet(engine, key, strlen(key), field, strlen(field), &resp_data, + &resp_len); + assert(s == Ok); + assert(resp_len == strlen(value2)); + assert(StrCmp(resp_data, resp_len, value2, strlen(value2)) == + 0); // field is updated + free(resp_data); + s = KVDKHashDestroy(engine, key, strlen(key)); + assert(s == Ok); + + printf("Successfully excecuted HSET and HSETNX.\n"); +} + +int main() { + // Initialize a KVDK instance. + KVDKConfigs* kvdk_configs = KVDKCreateConfigs(); + KVDKSetConfigs(kvdk_configs, 48, 1ull << 10, 1 << 4); + + const char* engine_path = "/mnt/pmem0/tutorial_kvdk_example"; + // open engine + KVDKEngine* kvdk_engine; + KVDKStatus s = KVDKOpen(engine_path, kvdk_configs, stdout, &kvdk_engine); + assert(s == Ok); + + // Modify Example + ModifyExample(kvdk_engine); + + // Anonymous Global Collection Example + AnonymousCollectionExample(kvdk_engine); + + // Named Sorted Collection Example + SortedCollectionExample(kvdk_engine); + + // Sorted Named Collection Example + SortedIteratorExample(kvdk_engine); + + CompFuncForSortedCollectionExample(kvdk_engine); + + // BatchWrite on Anonymous Global Collection Example + BatchWriteAnonCollectionExample(kvdk_engine); + + // Hashes Collection Example + HashesCollectionExample(kvdk_engine); + + // Listes Collection Example + ListsCollectionExample(kvdk_engine); + + // Expire Example + ExpireExample(kvdk_engine); + + HashModifyExample(kvdk_engine); + + KVDKDestroyConfigs(kvdk_configs); + KVDKCloseEngine(kvdk_engine); + return 0; +} diff --git a/volatile/examples/tutorial/cpp_api_tutorial.cpp b/volatile/examples/tutorial/cpp_api_tutorial.cpp new file mode 100644 index 00000000..4a8e2d96 --- /dev/null +++ b/volatile/examples/tutorial/cpp_api_tutorial.cpp @@ -0,0 +1,457 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "kvdk/volatile/engine.hpp" + +#define DEBUG // For assert + +using StringView = pmem::obj::string_view; +using kvdk::Snapshot; + +// The KVDK instance is mounted as a directory +// /mnt/pmem0/tutorial_kvdk_example. +// Modify this path if necessary. +const char* pmem_path = "/mnt/pmem0/tutorial_kvdk_example"; + +kvdk::Status status; +kvdk::Engine* engine = nullptr; + +static void test_anon_coll() { + std::string key1{"key1"}; + std::string key2{"key2"}; + std::string value1{"value1"}; + std::string value2{"value2"}; + std::string v; + kvdk::WriteOptions write_options; + + // Insert key1-value1 + status = engine->Put(key1, value1, write_options); + assert(status == kvdk::Status::Ok); + + // Get value1 by key1 + status = engine->Get(key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value1); + + // Update key1-value1 to key1-value2 + status = engine->Put(key1, value2, write_options); + assert(status == kvdk::Status::Ok); + + // Get value2 by key1 + status = engine->Get(key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value2); + + // Insert key2-value2 + status = engine->Put(key2, value2, write_options); + assert(status == kvdk::Status::Ok); + + // Delete key1-value2 + status = engine->Delete(key1); + assert(status == kvdk::Status::Ok); + + // Delete key2-value2 + status = engine->Delete(key2); + assert(status == kvdk::Status::Ok); + + printf( + "Successfully performed Get, Put, Delete operations on anonymous " + "global collection.\n"); + return; +} + +static void test_named_coll() { + std::string collection1{"my_collection_1"}; + std::string collection2{"my_collection_2"}; + std::string key1{"key1"}; + std::string key2{"key2"}; + std::string value1{"value1"}; + std::string value2{"value2"}; + std::string v; + status = engine->SortedCreate(collection1); + assert(status == kvdk::Status::Ok); + + status = engine->SortedCreate(collection2); + assert(status == kvdk::Status::Ok); + // Insert key1-value1 into "my_collection_1". + // Implicitly create a collection named "my_collection_1" in which + // key1-value1 is stored. + status = engine->SortedPut(collection1, key1, value1); + assert(status == kvdk::Status::Ok); + + // Get value1 by key1 in collection "my_collection_1" + status = engine->SortedGet(collection1, key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value1); + + // Insert key1-value2 into "my_collection_2". + // Implicitly create a collection named "my_collection_2" in which + // key1-value2 is stored. + status = engine->SortedPut(collection2, key1, value2); + assert(status == kvdk::Status::Ok); + + // Get value2 by key1 in collection "my_collection_2" + status = engine->SortedGet(collection2, key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value2); + + // Get value1 by key1 in collection "my_collection_1" + // key1-value2 is stored in "my_collection_2" + // Thus key1-value1 stored in "my_collection_1" is unaffected by operation + // engine->SortedPut(collection2, key1, value2). + status = engine->SortedGet(collection1, key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value1); + + // Insert key2-value2 into collection "my_collection_2" + // Collection "my_collection_2" already exists and no implicit collection + // creation occurs. + status = engine->SortedPut(collection2, key2, value2); + assert(status == kvdk::Status::Ok); + + // Delete key1-value1 in collection "my_collection_1" + // Although "my_collection_1" has no elements now, the collection itself is + // not deleted though. + status = engine->SortedDelete(collection1, key1); + assert(status == kvdk::Status::Ok); + + printf( + "Successfully performed SortedGet, SortedPut, SortedDelete operations on " + "named " + "collections.\n"); + return; +} + +static void test_iterator() { + std::string sorted_collection{"my_sorted_collection"}; + // Create Sorted Collection + status = engine->SortedCreate(sorted_collection); + assert(status == kvdk::Status::Ok); + // Create toy keys and values. + std::vector> kv_pairs; + for (int i = 0; i < 10; ++i) { + kv_pairs.emplace_back( + std::make_pair("key" + std::to_string(i), "value" + std::to_string(i))); + } + std::shuffle(kv_pairs.begin(), kv_pairs.end(), std::mt19937{42}); + // Print out kv_pairs to check if they are really shuffled. + printf("The shuffled kv-pairs are:\n"); + for (const auto& kv : kv_pairs) + printf("%s\t%s\n", kv.first.c_str(), kv.second.c_str()); + + // Populate collection "my_sorted_collection" with keys and values. + // kv_pairs are not necessarily sorted, but kv-pairs in collection + // "my_sorted_collection" are sorted. + for (int i = 0; i < 10; ++i) { + // Collection "my_sorted_collection" is implicitly created in first + // iteration + status = engine->SortedPut(sorted_collection, kv_pairs[i].first, + kv_pairs[i].second); + assert(status == kvdk::Status::Ok); + } + // Sort kv_pairs for checking the order of "my_sorted_collection". + std::sort(kv_pairs.begin(), kv_pairs.end()); + + // Iterate through collection "my_sorted_collection" + auto iter = engine->SortedIteratorCreate(sorted_collection); + if (!iter) { + fprintf(stderr, "Seek error\n"); + return; + } + iter->SeekToFirst(); + { + int i = 0; + while (iter->Valid()) { + assert(iter->Key() == kv_pairs[i].first); + assert(iter->Value() == kv_pairs[i].second); + iter->Next(); + ++i; + } + } + + // Iterate through range ["key1", "key8"). + std::string beg{"key1"}; + std::string end{"key8"}; + { + int i = 1; + iter->Seek(beg); + for (iter->Seek(beg); iter->Valid() && iter->Key() < end; iter->Next()) { + assert(iter->Key() == kv_pairs[i].first); + assert(iter->Value() == kv_pairs[i].second); + ++i; + } + } + + // Reversely iterate through range ["key8", "key1"). + beg = "key8"; + end = "key1"; + { + int i = 8; + for (iter->Seek(beg); iter->Valid() && iter->Key() > end; iter->Prev()) { + assert(iter->Key() == kv_pairs[i].first); + assert(iter->Value() == kv_pairs[i].second); + --i; + } + } + + printf("Successfully iterated through a sorted named collections.\n"); + engine->SortedIteratorRelease(iter); + return; +} + +static void test_batch_write() { + std::string key1{"key1"}; + std::string key2{"key2"}; + std::string value1{"value1"}; + std::string value2{"value2"}; + std::string v; + + auto batch = engine->WriteBatchCreate(); + batch->StringPut(key1, value1); + batch->StringPut(key1, value2); + batch->StringPut(key2, value2); + batch->StringDelete(key2); + + // If the batch is successfully written, there should be only key1-value2 in + // anonymous global collection. + status = engine->BatchWrite(batch); + assert(status == kvdk::Status::Ok); + + // Get value2 by key1 + status = engine->Get(key1, &v); + assert(status == kvdk::Status::Ok); + assert(v == value2); + + // Get value2 by key1 + status = engine->Get(key2, &v); + assert(status == kvdk::Status::NotFound); + // v is unchanged, but it is invalid. Always Check kvdk::Status before + // perform further operations! + assert(v == value2); + + printf("Successfully performed BatchWrite on anonymous global collection.\n"); + return; +} + +static void test_customer_sorted_func() { + std::string collection = "collection0"; + struct number_kv { + std::string number_key; + std::string value; + }; + + std::vector array = { + {"100", "a"}, {"50", "c"}, {"40", "d"}, {"30", "b"}, {"90", "f"}}; + + std::vector expected_array = { + {"100", "a"}, {"90", "f"}, {"50", "c"}, {"40", "d"}, {"30", "b"}}; + + // regitser compare function + std::string comp_name = "double_comp"; + auto score_cmp = [](const StringView& a, const StringView& b) -> int { + std::string str_a(a.data(), a.size()); + std::string str_b(b.data(), b.size()); + double scorea = std::stod(str_a); + double scoreb = std::stod(str_b); + if (scorea == scoreb) + return 0; + else if (scorea < scoreb) + return 1; + else + return -1; + }; + engine->registerComparator(comp_name, score_cmp); + // create sorted collection + kvdk::SortedCollectionConfigs s_configs; + s_configs.comparator_name = comp_name; + kvdk::Status s = engine->SortedCreate(collection, s_configs); + assert(s == Ok); + for (int i = 0; i < 5; ++i) { + s = engine->SortedPut(collection, array[i].number_key, array[i].value); + assert(s == Ok); + } + auto iter = engine->SortedIteratorCreate(collection); + + assert(iter != nullptr); + + int i = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string key = iter->Key(); + std::string value = iter->Value(); + if (key != expected_array[i].number_key) { + printf("sort key error, current key: %s , but expected key: %s\n", + key.c_str(), expected_array[i].number_key.c_str()); + } + if (value != expected_array[i].value) { + printf("sort value error, current value: %s , but expected value: %s\n", + value.c_str(), expected_array[i].value.c_str()); + } + ++i; + } + printf("Successfully collections sorted by number.\n"); + engine->SortedIteratorRelease(iter); +} + +static void test_expire() { + int64_t ttl_time; + std::string got_val; + kvdk::Status s; + // For string + { + std::string key = "stringkey"; + std::string val = "stringval"; + // case: set expire time + s = engine->Put(key, val, kvdk::WriteOptions{100}); + assert(s == kvdk::Status::Ok); + s = engine->Get(key, &got_val); + assert(s == kvdk::Status::Ok); + assert(got_val == val); + s = engine->GetTTL(key, &ttl_time); + assert(s == kvdk::Status::Ok); + // case: reset expire time + s = engine->Expire(key, INT32_MAX); + assert(s == kvdk::Status::Ok); + // case: change to persist key + s = engine->Expire(key, kvdk::kPersistTime); + assert(s == kvdk::Status::Ok); + s = engine->GetTTL(key, &ttl_time); + assert(s == kvdk::Status::Ok); + assert(ttl_time == kvdk::kPersistTime); + // case: key is expired. + s = engine->Expire(key, 1); + assert(s == kvdk::Status::Ok); + sleep(1); + s = engine->Get(key, &got_val); + assert(s == kvdk::Status::NotFound); + // case: ttl time is negative. + s = engine->Put(key, "Updatedval"); + assert(s == kvdk::Status::Ok); + s = engine->Expire(key, -1); + assert(s == kvdk::Status::Ok); + s = engine->GetTTL(key, &ttl_time); + assert(s == kvdk::Status::NotFound); + printf("Successfully expire string\n"); + } + + { + std::string sorted_collection = "sorted_collection"; + std::string key = "sortedkey"; + std::string val = "sortedval"; + + s = engine->SortedCreate(sorted_collection); + // case: default persist key. + s = engine->GetTTL(sorted_collection, &ttl_time); + assert(s == kvdk::Status::Ok); + assert(ttl_time == kvdk::kPersistTime); + s = engine->SortedPut(sorted_collection, key, val); + assert(s == kvdk::Status::Ok); + // case: set expire_time + s = engine->Expire(sorted_collection, INT32_MAX); + assert(s == kvdk::Status::Ok); + // case: change to persist key + s = engine->Expire(sorted_collection, kvdk::kPersistTime); + s = engine->GetTTL(sorted_collection, &ttl_time); + assert(s == kvdk::Status::Ok); + assert(ttl_time == kvdk::kPersistTime); + // case: key is expired. + s = engine->Expire(sorted_collection, 1); + assert(s == kvdk::Status::Ok); + sleep(1); + s = engine->SortedGet(sorted_collection, key, &got_val); + assert(s == kvdk::Status::NotFound); + printf("Successfully expire sorted\n"); + } + + { + std::string hash_collection = "hash_collection"; + std::string key = "hashkey"; + std::string val = "hashval"; + + // case: default persist key + s = engine->HashCreate(hash_collection); + assert(s == kvdk::Status::Ok); + s = engine->HashPut(hash_collection, key, val); + assert(s == kvdk::Status::Ok); + s = engine->GetTTL(hash_collection, &ttl_time); + assert(s == kvdk::Status::Ok); + assert(ttl_time == kvdk::kPersistTime); + + // case: set expire_time + s = engine->Expire(hash_collection, 1); + assert(s == kvdk::Status::Ok); + // case: change to persist key + s = engine->Expire(hash_collection, kvdk::kPersistTime); + s = engine->GetTTL(hash_collection, &ttl_time); + assert(s == kvdk::Status::Ok); + assert(ttl_time == kvdk::kPersistTime); + // case: key is expired. + s = engine->Expire(hash_collection, 1); + assert(s == kvdk::Status::Ok); + sleep(1); + s = engine->HashGet(hash_collection, key, &got_val); + assert(s == kvdk::Status::NotFound); + + s = engine->HashDestroy(hash_collection); + assert(s == kvdk::Status::NotFound); + printf("Successfully expire hash\n"); + } + + { + // TODO: add expire list, but now list api has changed. + } + return; +} + +int main() { + // Initialize a KVDK instance. + kvdk::Configs engine_configs; + { + // Configure for a tiny KVDK instance. + // Please refer to "Configuration" section in user documentation for + // details. + engine_configs.hash_bucket_num = (1ull << 10); + } + std::string engine_path{pmem_path}; + + // Purge old KVDK instance + [[gnu::unused]] int sink = + system(std::string{"rm -rf " + engine_path + "\n"}.c_str()); + + status = kvdk::Engine::Open(engine_path, &engine, engine_configs, stdout); + assert(status == kvdk::Status::Ok); + printf("Successfully opened a KVDK instance.\n"); + + // Reads and Writes on Anonymous Global Collection + test_anon_coll(); + + // Reads and Writes on Named Collection + test_named_coll(); + + // Iterating a Sorted Named Collection + test_iterator(); + + // Sorted Collection with Customer Sorted Function + test_customer_sorted_func(); + + // BatchWrite on Anonymous Global Collection + test_batch_write(); + + // Expire + test_expire(); + + // Close KVDK instance. + delete engine; + + // Remove persisted contents on PMem + return system(std::string{"rm -rf " + engine_path + "\n"}.c_str()); +} diff --git a/volatile/examples/tutorial/zset.c b/volatile/examples/tutorial/zset.c new file mode 100644 index 00000000..4e6af779 --- /dev/null +++ b/volatile/examples/tutorial/zset.c @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +// This is a example of single-thread redis zset implementation with kvdk c api + +#include "assert.h" +#include "kvdk/volatile/engine.h" +#include "malloc.h" +#include "string.h" + +#define NUM_MEMBERS 7 + +int StrCompare(const char* a, size_t a_len, const char* b, size_t b_len) { + int n = (a_len < b_len) ? a_len : b_len; + int cmp = memcmp(a, b, n); + if (cmp == 0) { + if (a_len < b_len) + cmp = -1; + else if (a_len > b_len) + cmp = 1; + } + return cmp; +} + +const char* comp_name = "score_comp"; +int ScoreCmp(const char* score_key_a, size_t a_len, const char* score_key_b, + size_t b_len) { + assert(a_len >= sizeof(int64_t)); + assert(b_len >= sizeof(int64_t)); + int cmp = (*(int64_t*)score_key_a) - (*(int64_t*)score_key_b); + return cmp == 0 ? StrCompare( + score_key_a + sizeof(int64_t), a_len - sizeof(int64_t), + score_key_b + sizeof(int64_t), b_len - sizeof(int64_t)) + : cmp; +} + +// Store score key in sorted collection to index score->member +void EncodeScoreKey(int64_t score, const char* member, size_t member_len, + char** score_key, size_t* score_key_len) { + *score_key_len = sizeof(int64_t) + member_len; + *score_key = (char*)malloc(*score_key_len); + memcpy(*score_key, &score, sizeof(int64_t)); + memcpy(*score_key + sizeof(int64_t), member, member_len); +} + +// Store member with string type to index member->score +void EncodeStringKey(const char* collection, size_t collection_len, + const char* member, size_t member_len, char** string_key, + size_t* string_key_len) { + *string_key_len = collection_len + member_len; + *string_key = (char*)malloc(*string_key_len); + memcpy(*string_key, collection, collection_len); + memcpy(*string_key + collection_len, member, member_len); +} + +// notice: we set member as a view of score key (which means no ownership) +void DecodeScoreKey(char* score_key, size_t score_key_len, char** member, + size_t* member_len, int64_t* score) { + assert(score_key_len > sizeof(int64_t)); + memcpy(score, score_key, sizeof(int64_t)); + *member = score_key + sizeof(int64_t); + *member_len = score_key_len - sizeof(int64_t); +} + +void PrintMemberScore(size_t index, char* member, size_t member_len, + int64_t score) { + char member_c_str[member_len + 1]; + memcpy(member_c_str, member, member_len); + member_c_str[member_len] = '\0'; + printf("(%lu)\"%s\"\n", 2 * index + 1, member_c_str); + printf("(%lu)\"%ld\"\n", 2 * index + 2, score); +} + +KVDKStatus KVDKZAdd(KVDKEngine* engine, const char* collection, + size_t collection_len, int64_t score, const char* member, + size_t member_len, KVDKWriteOptions* write_option) { + printf("ZADD %s %ld %s\n", collection, score, member); + + char* score_key; + char* string_key; + size_t score_key_len; + size_t string_key_len; + EncodeScoreKey(score, member, member_len, &score_key, &score_key_len); + EncodeStringKey(collection, collection_len, member, member_len, &string_key, + &string_key_len); + + KVDKStatus s = KVDKSortedPut(engine, collection, collection_len, score_key, + score_key_len, "", 0); + if (s == NotFound) { + KVDKSortedCollectionConfigs* s_config = KVDKCreateSortedCollectionConfigs(); + KVDKSetSortedCollectionConfigs( + s_config, comp_name, strlen(comp_name), + 0 /*we do not need hash index for score part*/); + s = KVDKSortedCreate(engine, collection, collection_len, s_config); + if (s == Ok) { + s = KVDKSortedPut(engine, collection, collection_len, score_key, + score_key_len, "", 0); + } + KVDKDestroySortedCollectionConfigs(s_config); + } + + if (s == Ok) { + s = KVDKPut(engine, string_key, string_key_len, (char*)&score, + sizeof(int64_t), write_option); + } + + free(score_key); + free(string_key); + + return s; +} + +KVDKStatus KVDKZPopMin(KVDKEngine* engine, const char* collection, + size_t collection_len, size_t n) { + printf("ZPOPMIN %s %lu\n", collection, n); + + KVDKStatus s; + KVDKSortedIterator* iter = + KVDKSortedIteratorCreate(engine, collection, collection_len, NULL, NULL); + if (iter == NULL) { + return Ok; + } + size_t cnt = 0; + + for (KVDKSortedIteratorSeekToFirst(iter); + KVDKSortedIteratorValid(iter) && n > 0; KVDKSortedIteratorNext(iter)) { + char* score_key; + char* string_key; + char* member; + size_t score_key_len; + size_t string_key_len; + size_t member_len; + int64_t score; + KVDKSortedIteratorKey(iter, &score_key, &score_key_len); + DecodeScoreKey(score_key, score_key_len, &member, &member_len, &score); + EncodeStringKey(collection, collection_len, member, member_len, &string_key, + &string_key_len); + s = KVDKSortedDelete(engine, collection, collection_len, score_key, + score_key_len); + if (s == Ok) { + s = KVDKDelete(engine, string_key, string_key_len); + } + + if (s == Ok) { + // do anything with poped key, like print + PrintMemberScore(cnt++, member, member_len, score); + (void)member; + } + + free(score_key); + free(string_key); + if (s != Ok || cnt == n) { + break; + } + } + KVDKSortedIteratorDestroy(engine, iter); + return s; +} + +KVDKStatus KVDKZPopMax(KVDKEngine* engine, const char* collection, + size_t collection_len, size_t n) { + printf("ZPOPMAX %s %lu\n", collection, n); + + KVDKStatus s; + KVDKSortedIterator* iter = + KVDKSortedIteratorCreate(engine, collection, collection_len, NULL, NULL); + if (iter == NULL) { + return Ok; + } + size_t cnt = 0; + + for (KVDKSortedIteratorSeekToLast(iter); + KVDKSortedIteratorValid(iter) && n > 0; KVDKSortedIteratorPrev(iter)) { + char* score_key; + char* string_key; + char* member; + size_t score_key_len; + size_t string_key_len; + size_t member_len; + int64_t score; + KVDKSortedIteratorKey(iter, &score_key, &score_key_len); + DecodeScoreKey(score_key, score_key_len, &member, &member_len, &score); + EncodeStringKey(collection, collection_len, member, member_len, &string_key, + &string_key_len); + s = KVDKSortedDelete(engine, collection, collection_len, score_key, + score_key_len); + if (s == Ok) { + s = KVDKDelete(engine, string_key, string_key_len); + } + + if (s == Ok) { + // do anything with poped key, like print + PrintMemberScore(cnt++, member, member_len, score); + (void)member; + } + + free(score_key); + free(string_key); + if (s != Ok || cnt == n) { + break; + } + } + KVDKSortedIteratorDestroy(engine, iter); + return s; +} + +KVDKStatus KVDKZRange(KVDKEngine* engine, const char* collection, + size_t collection_len, int64_t min_score, + int64_t max_score) { + printf("ZRANGE %s %ld %ld\n", collection, min_score, max_score); + + KVDKSortedIterator* iter = + KVDKSortedIteratorCreate(engine, collection, collection_len, NULL, NULL); + if (iter == NULL) { + return Ok; + } + size_t cnt = 0; + + for (KVDKSortedIteratorSeek(iter, (char*)&min_score, sizeof(int64_t)); + KVDKSortedIteratorValid(iter); KVDKSortedIteratorNext(iter)) { + char* score_key; + size_t score_key_len; + KVDKSortedIteratorKey(iter, &score_key, &score_key_len); + char* member; + size_t member_len; + int64_t score; + DecodeScoreKey(score_key, score_key_len, &member, &member_len, &score); + if (score <= max_score) { + // do any thing with in-range key, like print; + PrintMemberScore(cnt++, member, member_len, score); + free(score_key); + } else { + free(score_key); + break; + } + } + KVDKSortedIteratorDestroy(engine, iter); + return Ok; +} + +void ZSetTest(KVDKEngine* engine) { + const char* zset_name = "zset"; + const size_t zset_name_len = strlen(zset_name); + char* members[NUM_MEMBERS] = {"one", "two", "three", "four", + "five", "another_five", "six"}; + int64_t scores[NUM_MEMBERS] = {1, 2, 3, 4, 5, 5, 6}; + KVDKWriteOptions* write_option = KVDKCreateWriteOptions(); + for (size_t i = 0; i < NUM_MEMBERS; i++) { + KVDKStatus s = KVDKZAdd(engine, zset_name, zset_name_len, scores[i], + members[i], strlen(members[i]), write_option); + assert(s == Ok); + } + + assert(KVDKZRange(engine, zset_name, zset_name_len, 1, 6) == Ok); + assert(KVDKZPopMax(engine, zset_name, zset_name_len, 2) == Ok); + assert(KVDKZPopMin(engine, zset_name, zset_name_len, 2) == Ok); + assert(KVDKZRange(engine, zset_name, zset_name_len, 1, 6) == Ok); + KVDKDestroyWriteOptions(write_option); +} + +int main() { + KVDKConfigs* kvdk_configs = KVDKCreateConfigs(); + KVDKSetConfigs(kvdk_configs, 48, 1ull << 10, 1 << 4); + const char* engine_path = "/mnt/pmem0/kvdk_zset_example"; + KVDKEngine* kvdk_engine; + KVDKStatus s = KVDKOpen(engine_path, kvdk_configs, stdout, &kvdk_engine); + assert(s == Ok); + KVDKRegisterCompFunc(kvdk_engine, comp_name, strlen(comp_name), ScoreCmp); + ZSetTest(kvdk_engine); + + KVDKDestroyConfigs(kvdk_configs); + KVDKCloseEngine(kvdk_engine); + return 0; +} \ No newline at end of file diff --git a/volatile/extern/gtest b/volatile/extern/gtest new file mode 160000 index 00000000..e2239ee6 --- /dev/null +++ b/volatile/extern/gtest @@ -0,0 +1 @@ +Subproject commit e2239ee6043f73722e7aa812a459f54a28552929 diff --git a/volatile/extern/jemalloc b/volatile/extern/jemalloc new file mode 160000 index 00000000..36366f3c --- /dev/null +++ b/volatile/extern/jemalloc @@ -0,0 +1 @@ +Subproject commit 36366f3c4c741723369853c923e56999716398fc diff --git a/volatile/extern/jemalloc-cmake/CMakeLists.txt b/volatile/extern/jemalloc-cmake/CMakeLists.txt new file mode 100644 index 00000000..1310538b --- /dev/null +++ b/volatile/extern/jemalloc-cmake/CMakeLists.txt @@ -0,0 +1,108 @@ +if (NOT CMAKE_SYSTEM_NAME MATCHES "Linux") + message (FATAL_ERROR "Not supported for non-linux environment") +endif() + +message (STATUS "Configure to build jemalloc") + +set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000") + +set (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE "" CACHE STRING "Change default configuration string of jemalloc" ) +if (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE) + set (JEMALLOC_CONFIG_MALLOC_CONF "${JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE}") +endif() +message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}") + +set (JEMALLOC_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/jemalloc") + +set (SRCS + "${JEMALLOC_SOURCE_DIR}/src/arena.c" + "${JEMALLOC_SOURCE_DIR}/src/background_thread.c" + "${JEMALLOC_SOURCE_DIR}/src/base.c" + "${JEMALLOC_SOURCE_DIR}/src/bin.c" + "${JEMALLOC_SOURCE_DIR}/src/bin_info.c" + "${JEMALLOC_SOURCE_DIR}/src/bitmap.c" + "${JEMALLOC_SOURCE_DIR}/src/buf_writer.c" + "${JEMALLOC_SOURCE_DIR}/src/cache_bin.c" + "${JEMALLOC_SOURCE_DIR}/src/ckh.c" + "${JEMALLOC_SOURCE_DIR}/src/counter.c" + "${JEMALLOC_SOURCE_DIR}/src/ctl.c" + "${JEMALLOC_SOURCE_DIR}/src/decay.c" + "${JEMALLOC_SOURCE_DIR}/src/div.c" + "${JEMALLOC_SOURCE_DIR}/src/ecache.c" + "${JEMALLOC_SOURCE_DIR}/src/edata.c" + "${JEMALLOC_SOURCE_DIR}/src/edata_cache.c" + "${JEMALLOC_SOURCE_DIR}/src/ehooks.c" + "${JEMALLOC_SOURCE_DIR}/src/emap.c" + "${JEMALLOC_SOURCE_DIR}/src/eset.c" + "${JEMALLOC_SOURCE_DIR}/src/exp_grow.c" + "${JEMALLOC_SOURCE_DIR}/src/extent.c" + "${JEMALLOC_SOURCE_DIR}/src/extent_dss.c" + "${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c" + "${JEMALLOC_SOURCE_DIR}/src/fxp.c" + "${JEMALLOC_SOURCE_DIR}/src/hook.c" + "${JEMALLOC_SOURCE_DIR}/src/hpa.c" + "${JEMALLOC_SOURCE_DIR}/src/hpa_hooks.c" + "${JEMALLOC_SOURCE_DIR}/src/hpdata.c" + "${JEMALLOC_SOURCE_DIR}/src/inspect.c" + "${JEMALLOC_SOURCE_DIR}/src/jemalloc.c" + "${JEMALLOC_SOURCE_DIR}/src/large.c" + "${JEMALLOC_SOURCE_DIR}/src/log.c" + "${JEMALLOC_SOURCE_DIR}/src/malloc_io.c" + "${JEMALLOC_SOURCE_DIR}/src/mutex.c" + "${JEMALLOC_SOURCE_DIR}/src/nstime.c" + "${JEMALLOC_SOURCE_DIR}/src/pa.c" + "${JEMALLOC_SOURCE_DIR}/src/pac.c" + "${JEMALLOC_SOURCE_DIR}/src/pa_extra.c" + "${JEMALLOC_SOURCE_DIR}/src/pages.c" + "${JEMALLOC_SOURCE_DIR}/src/pai.c" + "${JEMALLOC_SOURCE_DIR}/src/peak_event.c" + "${JEMALLOC_SOURCE_DIR}/src/prof.c" + "${JEMALLOC_SOURCE_DIR}/src/prof_data.c" + "${JEMALLOC_SOURCE_DIR}/src/prof_log.c" + "${JEMALLOC_SOURCE_DIR}/src/prof_recent.c" + "${JEMALLOC_SOURCE_DIR}/src/prof_stats.c" + "${JEMALLOC_SOURCE_DIR}/src/prof_sys.c" + "${JEMALLOC_SOURCE_DIR}/src/psset.c" + "${JEMALLOC_SOURCE_DIR}/src/rtree.c" + "${JEMALLOC_SOURCE_DIR}/src/safety_check.c" + "${JEMALLOC_SOURCE_DIR}/src/san_bump.c" + "${JEMALLOC_SOURCE_DIR}/src/san.c" + "${JEMALLOC_SOURCE_DIR}/src/sc.c" + "${JEMALLOC_SOURCE_DIR}/src/sec.c" + "${JEMALLOC_SOURCE_DIR}/src/stats.c" + "${JEMALLOC_SOURCE_DIR}/src/sz.c" + "${JEMALLOC_SOURCE_DIR}/src/tcache.c" + "${JEMALLOC_SOURCE_DIR}/src/test_hooks.c" + "${JEMALLOC_SOURCE_DIR}/src/thread_event.c" + "${JEMALLOC_SOURCE_DIR}/src/ticker.c" + "${JEMALLOC_SOURCE_DIR}/src/tsd.c" + "${JEMALLOC_SOURCE_DIR}/src/witness.c" +) + +add_library(_jemalloc ${SRCS}) + +target_include_directories(_jemalloc SYSTEM PUBLIC include) +target_include_directories(_jemalloc SYSTEM PRIVATE "jemalloc/internal") +target_include_directories(_jemalloc SYSTEM PRIVATE "${JEMALLOC_SOURCE_DIR}/include") + +configure_file(include/jemalloc/internal/jemalloc_internal_defs.h.in + include/jemalloc/internal/jemalloc_internal_defs.h) +target_include_directories(_jemalloc SYSTEM PRIVATE + "${CMAKE_CURRENT_BINARY_DIR}/include/jemalloc/internal") + +target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) + +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_definitions(_jemalloc PRIVATE + # -DJEMALLOC_DEBUG=1 # Disable debug due to possible assertion failures + -DJEMALLOC_LOG=1) +endif () + +target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1) + +# for RTLD_NEXT +target_compile_options(_jemalloc PRIVATE -D_GNU_SOURCE) + +set_target_properties(_jemalloc PROPERTIES COMPILE_FLAGS "-w -fPIC") + +add_library(kvdk_extern_lib::jemalloc ALIAS _jemalloc) diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/internal/jemalloc_internal_defs.h.in b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/jemalloc_internal_defs.h.in new file mode 100644 index 00000000..0781740f --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -0,0 +1,435 @@ +/* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */ +#ifndef JEMALLOC_INTERNAL_DEFS_H_ +#define JEMALLOC_INTERNAL_DEFS_H_ +/* + * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all + * public APIs to be prefixed. This makes it possible, with some care, to use + * multiple allocators simultaneously. + */ +#define JEMALLOC_PREFIX "je_kvdk_" +#define JEMALLOC_CPREFIX "JE_KVDK_" + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +/* #undef JEMALLOC_OVERRIDE___LIBC_CALLOC */ +/* #undef JEMALLOC_OVERRIDE___LIBC_FREE */ +/* #undef JEMALLOC_OVERRIDE___LIBC_MALLOC */ +/* #undef JEMALLOC_OVERRIDE___LIBC_MEMALIGN */ +/* #undef JEMALLOC_OVERRIDE___LIBC_REALLOC */ +/* #undef JEMALLOC_OVERRIDE___LIBC_VALLOC */ +/* #undef JEMALLOC_OVERRIDE___LIBC_PVALLOC */ +/* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ + +/* + * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. + * For shared libraries, symbol visibility mechanisms prevent these symbols + * from being exported, but for static libraries, naming collisions are a real + * possibility. + */ +#define JEMALLOC_PRIVATE_NAMESPACE je_ + +/* + * Hyper-threaded CPUs may need a special instruction inside spin loops in + * order to yield to another virtual CPU. + */ +#define CPU_SPINWAIT __asm__ volatile("pause") +/* 1 if CPU_SPINWAIT is defined, 0 otherwise. */ +#define HAVE_CPU_SPINWAIT 1 + +/* + * Number of significant bits in virtual addresses. This may be less than the + * total number of bits in a pointer, e.g. on x64, for which the uppermost 16 + * bits are the same as bit 47. + */ +#define LG_VADDR 48 + +/* Defined if C11 atomics are available. */ +/* #undef JEMALLOC_C11_ATOMICS */ + +/* Defined if GCC __atomic atomics are available. */ +#define JEMALLOC_GCC_ATOMIC_ATOMICS +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS + +/* Defined if GCC __sync atomics are available. */ +#define JEMALLOC_GCC_SYNC_ATOMICS +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_SYNC_ATOMICS + +/* + * Defined if __builtin_clz() and __builtin_clzl() are available. + */ +#define JEMALLOC_HAVE_BUILTIN_CLZ + +/* + * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. + */ +/* #undef JEMALLOC_OS_UNFAIR_LOCK */ + +/* Defined if syscall(2) is usable. */ +#define JEMALLOC_USE_SYSCALL + +/* + * Defined if secure_getenv(3) is available. + */ +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ + +/* + * Defined if issetugid(2) is available. + */ +/* #undef JEMALLOC_HAVE_ISSETUGID */ + +/* Defined if pthread_atfork(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_ATFORK */ + +/* Defined if pthread_setname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_SETNAME_NP + +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC + +/* + * Defined if mach_absolute_time() is available. + */ +/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ + +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + +/* + * Defined if _malloc_thread_cleanup() exists. At least in the case of + * FreeBSD, pthread_key_create() allocates, which if used during malloc + * bootstrapping will cause recursion into the pthreads library. Therefore, if + * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in + * malloc_tsd. + */ +/* #undef JEMALLOC_MALLOC_THREAD_CLEANUP */ + +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#define JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +/* #undef JEMALLOC_MUTEX_INIT_CB */ + +/* Non-empty if the tls_model attribute is supported. */ +#define JEMALLOC_TLS_MODEL + +/* + * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables + * inline functions. + */ +/* #undef JEMALLOC_DEBUG */ + +/* JEMALLOC_STATS enables statistics calculation. */ +#define JEMALLOC_STATS + +/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */ +/* #undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API */ + +/* JEMALLOC_PROF enables allocation profiling. */ +/* #undef JEMALLOC_PROF */ + +/* Use libunwind for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBUNWIND */ + +/* Use libgcc for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBGCC */ + +/* Use gcc intrinsics for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_GCC */ + +/* JEMALLOC_PAGEID enabled page id */ +/* #undef JEMALLOC_PAGEID */ + +/* JEMALLOC_HAVE_PRCTL checks prctl */ +#define JEMALLOC_HAVE_PRCTL + +/* + * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage + * segment (DSS). + */ +#define JEMALLOC_DSS + +/* Support memory filling (junk/zero). */ +#define JEMALLOC_FILL + +/* Support utrace(2)-based tracing. */ +/* #undef JEMALLOC_UTRACE */ + +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + +/* Support optional abort() on OOM. */ +/* #undef JEMALLOC_XMALLOC */ + +/* Support lazy locking (avoid locking unless a second thread is launched). */ +/* #undef JEMALLOC_LAZY_LOCK */ + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +/* #undef LG_QUANTUM */ + +/* One page is 2^LG_PAGE bytes. */ +#define LG_PAGE 12 + +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + +/* + * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the + * system does not explicitly support huge pages; system calls that require + * explicit huge page support are separately configured. + */ +#define LG_HUGEPAGE 21 + +/* + * If defined, adjacent virtual memory mappings with identical attributes + * automatically coalesce, and they fragment when changes are made to subranges. + * This is the normal order of things for mmap()/munmap(), but on Windows + * VirtualAlloc()/VirtualFree() operations must be precisely matched, i.e. + * mappings do *not* coalesce/fragment. + */ +#define JEMALLOC_MAPS_COALESCE + +/* + * If defined, retain memory for later reuse by default rather than using e.g. + * munmap() to unmap freed extents. This is enabled on 64-bit Linux because + * common sequences of mmap()/munmap() calls will cause virtual memory map + * holes. + */ +#define JEMALLOC_RETAIN + +/* TLS is used to map arenas and magazine caches to threads. */ +#define JEMALLOC_TLS + +/* + * Used to mark unreachable code to quiet "end of non-void" compiler warnings. + * Don't use this directly; instead use unreachable() from util.h + */ +#define JEMALLOC_INTERNAL_UNREACHABLE __builtin_unreachable + +/* + * ffs*() functions to use for bitmapping. Don't use these directly; instead, + * use ffs_*() from util.h. + */ +#define JEMALLOC_INTERNAL_FFSLL __builtin_ffsll +#define JEMALLOC_INTERNAL_FFSL __builtin_ffsl +#define JEMALLOC_INTERNAL_FFS __builtin_ffs + +/* + * popcount*() functions to use for bitmapping. + */ +#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl +#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount + +/* + * If defined, explicitly attempt to more uniformly distribute large allocation + * pointer alignments across all cache indices. + */ +#define JEMALLOC_CACHE_OBLIVIOUS + +/* + * If defined, enable logging facilities. We make this a configure option to + * avoid taking extra branches everywhere. + */ +/* #undef JEMALLOC_LOG */ + +/* + * If defined, use readlinkat() (instead of readlink()) to follow + * /etc/malloc_conf. + */ +/* #undef JEMALLOC_READLINKAT */ + +/* + * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. + */ +/* #undef JEMALLOC_ZONE */ + +/* + * Methods for determining whether the OS overcommits. + * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's + * /proc/sys/vm.overcommit_memory file. + * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl. + */ +/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */ +#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY + +/* Defined if madvise(2) is available. */ +#define JEMALLOC_HAVE_MADVISE + +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#define JEMALLOC_HAVE_MADVISE_HUGE + +/* + * Methods for purging unused pages differ between operating systems. + * + * madvise(..., MADV_FREE) : This marks pages as being unused, such that they + * will be discarded rather than swapped out. + * madvise(..., MADV_DONTNEED) : If JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS is + * defined, this immediately discards pages, + * such that new pages will be demand-zeroed if + * the address region is later touched; + * otherwise this behaves similarly to + * MADV_FREE, though typically with higher + * system overhead. + */ +#define JEMALLOC_PURGE_MADVISE_FREE +#define JEMALLOC_PURGE_MADVISE_DONTNEED +#define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS + +/* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */ +#define JEMALLOC_DEFINE_MADVISE_FREE + +/* + * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise. + */ +#define JEMALLOC_MADVISE_DONTDUMP + +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + +/* + * Defined if transparent huge pages (THPs) are supported via the + * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. + */ +/* #undef JEMALLOC_THP */ + +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + +/* Define if operating system has alloca.h header. */ +#define JEMALLOC_HAS_ALLOCA_H + +/* C99 restrict keyword supported. */ +#define JEMALLOC_HAS_RESTRICT + +/* For use by hash code. */ +/* #undef JEMALLOC_BIG_ENDIAN */ + +/* sizeof(int) == 2^LG_SIZEOF_INT. */ +#define LG_SIZEOF_INT 2 + +/* sizeof(long) == 2^LG_SIZEOF_LONG. */ +#define LG_SIZEOF_LONG 3 + +/* sizeof(long long) == 2^LG_SIZEOF_LONG_LONG. */ +#define LG_SIZEOF_LONG_LONG 3 + +/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ +#define LG_SIZEOF_INTMAX_T 3 + +/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */ +/* #undef JEMALLOC_GLIBC_MALLOC_HOOK */ + +/* glibc memalign hook. */ +/* #undef JEMALLOC_GLIBC_MEMALIGN_HOOK */ + +/* pthread support */ +#define JEMALLOC_HAVE_PTHREAD + +/* dlsym() support */ +#define JEMALLOC_HAVE_DLSYM + +/* Adaptive mutex support in pthreads. */ +#define JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP + +/* GNU specific sched_getcpu support */ +#define JEMALLOC_HAVE_SCHED_GETCPU + +/* GNU specific sched_setaffinity support */ +#define JEMALLOC_HAVE_SCHED_SETAFFINITY + +/* + * If defined, all the features necessary for background threads are present. + */ +#define JEMALLOC_BACKGROUND_THREAD + +/* + * If defined, jemalloc symbols are not exported (doesn't work when + * JEMALLOC_PREFIX is not defined). + */ +/* #undef JEMALLOC_EXPORT */ + +/* config.malloc_conf options string. */ +#define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" + +/* If defined, jemalloc takes the malloc/free/etc. symbol names. */ +/* #undef JEMALLOC_IS_MALLOC */ + +/* + * Defined if strerror_r returns char * if _GNU_SOURCE is defined. + */ +#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE + +/* Performs additional safety checks when defined. */ +/* #undef JEMALLOC_OPT_SAFETY_CHECKS */ + +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + +/* If defined, realloc(ptr, 0) defaults to "free" instead of "alloc". */ +#define JEMALLOC_ZERO_REALLOC_DEFAULT_FREE + +#endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h new file mode 100644 index 00000000..fafc7889 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h @@ -0,0 +1,263 @@ +#ifndef JEMALLOC_PREAMBLE_H +#define JEMALLOC_PREAMBLE_H + +#include "jemalloc_internal_defs.h" +#include "jemalloc/internal/jemalloc_internal_decls.h" + +#if defined(JEMALLOC_UTRACE) || defined(JEMALLOC_UTRACE_LABEL) +#include +# if defined(JEMALLOC_UTRACE) +# define UTRACE_CALL(p, l) utrace(p, l) +# else +# define UTRACE_CALL(p, l) utrace("jemalloc_process", p, l) +# define JEMALLOC_UTRACE +# endif +#endif + +#define JEMALLOC_NO_DEMANGLE +#ifdef JEMALLOC_JET +# undef JEMALLOC_IS_MALLOC +# define JEMALLOC_N(n) jet_##n +# include "jemalloc/internal/public_namespace.h" +# define JEMALLOC_NO_RENAME +# include "../jemalloc.h" +# undef JEMALLOC_NO_RENAME +#else +# define JEMALLOC_N(n) je_##n +# include "../jemalloc.h" +#endif + +#if defined(JEMALLOC_OSATOMIC) +#include +#endif + +#ifdef JEMALLOC_ZONE +#include +#include +#include +#endif + +#include "jemalloc/internal/jemalloc_internal_macros.h" + +/* + * Note that the ordering matters here; the hook itself is name-mangled. We + * want the inclusion of hooks to happen early, so that we hook as much as + * possible. + */ +#ifndef JEMALLOC_NO_PRIVATE_NAMESPACE +# ifndef JEMALLOC_JET +# include "jemalloc/internal/private_namespace.h" +# else +# include "jemalloc/internal/private_namespace_jet.h" +# endif +#endif +#include "jemalloc/internal/test_hooks.h" + +#ifdef JEMALLOC_DEFINE_MADVISE_FREE +# define JEMALLOC_MADV_FREE 8 +#endif + +static const bool config_debug = +#ifdef JEMALLOC_DEBUG + true +#else + false +#endif + ; +static const bool have_dss = +#ifdef JEMALLOC_DSS + true +#else + false +#endif + ; +static const bool have_madvise_huge = +#ifdef JEMALLOC_HAVE_MADVISE_HUGE + true +#else + false +#endif + ; +static const bool config_fill = +#ifdef JEMALLOC_FILL + true +#else + false +#endif + ; +static const bool config_lazy_lock = +#ifdef JEMALLOC_LAZY_LOCK + true +#else + false +#endif + ; +static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; +static const bool config_prof = +#ifdef JEMALLOC_PROF + true +#else + false +#endif + ; +static const bool config_prof_libgcc = +#ifdef JEMALLOC_PROF_LIBGCC + true +#else + false +#endif + ; +static const bool config_prof_libunwind = +#ifdef JEMALLOC_PROF_LIBUNWIND + true +#else + false +#endif + ; +static const bool maps_coalesce = +#ifdef JEMALLOC_MAPS_COALESCE + true +#else + false +#endif + ; +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; +static const bool config_tls = +#ifdef JEMALLOC_TLS + true +#else + false +#endif + ; +static const bool config_utrace = +#ifdef JEMALLOC_UTRACE + true +#else + false +#endif + ; +static const bool config_xmalloc = +#ifdef JEMALLOC_XMALLOC + true +#else + false +#endif + ; +static const bool config_cache_oblivious = +#ifdef JEMALLOC_CACHE_OBLIVIOUS + true +#else + false +#endif + ; +/* + * Undocumented, for jemalloc development use only at the moment. See the note + * in jemalloc/internal/log.h. + */ +static const bool config_log = +#ifdef JEMALLOC_LOG + true +#else + false +#endif + ; +/* + * Are extra safety checks enabled; things like checking the size of sized + * deallocations, double-frees, etc. + */ +static const bool config_opt_safety_checks = +#ifdef JEMALLOC_OPT_SAFETY_CHECKS + true +#elif defined(JEMALLOC_DEBUG) + /* + * This lets us only guard safety checks by one flag instead of two; fast + * checks can guard solely by config_opt_safety_checks and run in debug mode + * too. + */ + true +#else + false +#endif + ; + +/* + * Extra debugging of sized deallocations too onerous to be included in the + * general safety checks. + */ +static const bool config_opt_size_checks = +#if defined(JEMALLOC_OPT_SIZE_CHECKS) || defined(JEMALLOC_DEBUG) + true +#else + false +#endif + ; + +static const bool config_uaf_detection = +#if defined(JEMALLOC_UAF_DETECTION) || defined(JEMALLOC_DEBUG) + true +#else + false +#endif + ; + +/* Whether or not the C++ extensions are enabled. */ +static const bool config_enable_cxx = +#ifdef JEMALLOC_ENABLE_CXX + true +#else + false +#endif +; + +#if defined(_WIN32) || defined(__APPLE__) || defined(JEMALLOC_HAVE_SCHED_GETCPU) +/* Currently percpu_arena depends on sched_getcpu. */ +#define JEMALLOC_PERCPU_ARENA +#endif +static const bool have_percpu_arena = +#ifdef JEMALLOC_PERCPU_ARENA + true +#else + false +#endif + ; +/* + * Undocumented, and not recommended; the application should take full + * responsibility for tracking provenance. + */ +static const bool force_ivsalloc = +#ifdef JEMALLOC_FORCE_IVSALLOC + true +#else + false +#endif + ; +static const bool have_background_thread = +#ifdef JEMALLOC_BACKGROUND_THREAD + true +#else + false +#endif + ; +static const bool config_high_res_timer = +#ifdef JEMALLOC_HAVE_CLOCK_REALTIME + true +#else + false +#endif + ; + +static const bool have_memcntl = +#ifdef JEMALLOC_HAVE_MEMCNTL + true +#else + false +#endif + ; + +#endif /* JEMALLOC_PREAMBLE_H */ diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/internal/public_namespace.h b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/public_namespace.h new file mode 100644 index 00000000..7e8b354a --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/public_namespace.h @@ -0,0 +1,25 @@ +#define je_aligned_alloc JEMALLOC_N(aligned_alloc) +#define je_calloc JEMALLOC_N(calloc) +#define je_dallocx JEMALLOC_N(dallocx) +#define je_free JEMALLOC_N(free) +#define je_mallctl JEMALLOC_N(mallctl) +#define je_mallctlbymib JEMALLOC_N(mallctlbymib) +#define je_mallctlnametomib JEMALLOC_N(mallctlnametomib) +#define je_malloc JEMALLOC_N(malloc) +#define je_malloc_conf JEMALLOC_N(malloc_conf) +#define je_malloc_conf_2_conf_harder JEMALLOC_N(malloc_conf_2_conf_harder) +#define je_malloc_message JEMALLOC_N(malloc_message) +#define je_malloc_stats_print JEMALLOC_N(malloc_stats_print) +#define je_malloc_usable_size JEMALLOC_N(malloc_usable_size) +#define je_mallocx JEMALLOC_N(mallocx) +#define je_smallocx_36366f3c4c741723369853c923e56999716398fc JEMALLOC_N(smallocx_36366f3c4c741723369853c923e56999716398fc) +#define je_nallocx JEMALLOC_N(nallocx) +#define je_posix_memalign JEMALLOC_N(posix_memalign) +#define je_rallocx JEMALLOC_N(rallocx) +#define je_realloc JEMALLOC_N(realloc) +#define je_sallocx JEMALLOC_N(sallocx) +#define je_sdallocx JEMALLOC_N(sdallocx) +#define je_xallocx JEMALLOC_N(xallocx) +#define je_memalign JEMALLOC_N(memalign) +#define je_valloc JEMALLOC_N(valloc) +#define je_pvalloc JEMALLOC_N(pvalloc) diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/internal/public_unnamespace.h b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/public_unnamespace.h new file mode 100644 index 00000000..f809b3f2 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/internal/public_unnamespace.h @@ -0,0 +1,25 @@ +#undef je_aligned_alloc +#undef je_calloc +#undef je_dallocx +#undef je_free +#undef je_mallctl +#undef je_mallctlbymib +#undef je_mallctlnametomib +#undef je_malloc +#undef je_malloc_conf +#undef je_malloc_conf_2_conf_harder +#undef je_malloc_message +#undef je_malloc_stats_print +#undef je_malloc_usable_size +#undef je_mallocx +#undef je_smallocx_36366f3c4c741723369853c923e56999716398fc +#undef je_nallocx +#undef je_posix_memalign +#undef je_rallocx +#undef je_realloc +#undef je_sallocx +#undef je_sdallocx +#undef je_xallocx +#undef je_memalign +#undef je_valloc +#undef je_pvalloc diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc.h new file mode 100644 index 00000000..f70f2f5c --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc.h @@ -0,0 +1,478 @@ +#ifndef JEMALLOC_H_ +#define JEMALLOC_H_ +#ifdef __cplusplus +extern "C" { +#endif + +/* Defined if __attribute__((...)) syntax is supported. */ +#define JEMALLOC_HAVE_ATTR + +/* Defined if alloc_size attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_ALLOC_SIZE + +/* Defined if format_arg(...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_ARG + +/* Defined if format(gnu_printf, ...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF + +/* Defined if format(printf, ...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF + +/* Defined if fallthrough attribute is supported. */ +/* #undef JEMALLOC_HAVE_ATTR_FALLTHROUGH */ + +/* Defined if cold attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_COLD + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +#define JEMALLOC_OVERRIDE_MEMALIGN +#define JEMALLOC_OVERRIDE_VALLOC +#define JEMALLOC_OVERRIDE_PVALLOC + +/* + * At least Linux omits the "const" in: + * + * size_t malloc_usable_size(const void *ptr); + * + * Match the operating system's prototype. + */ +#define JEMALLOC_USABLE_SIZE_CONST + +/* + * If defined, specify throw() for the public function prototypes when compiling + * with C++. The only justification for this is to match the prototypes that + * glibc defines. + */ +#define JEMALLOC_USE_CXX_THROW + +#ifdef _MSC_VER +# ifdef _WIN64 +# define LG_SIZEOF_PTR_WIN 3 +# else +# define LG_SIZEOF_PTR_WIN 2 +# endif +#endif + +/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ +#define LG_SIZEOF_PTR 3 + +/* + * Name mangling for public symbols is controlled by --with-mangling and + * --with-jemalloc-prefix. With default settings the je_ prefix is stripped by + * these macro definitions. + */ +#ifndef JEMALLOC_NO_RENAME +# define je_aligned_alloc je_kvdk_aligned_alloc +# define je_calloc je_kvdk_calloc +# define je_dallocx je_kvdk_dallocx +# define je_free je_kvdk_free +# define je_mallctl je_kvdk_mallctl +# define je_mallctlbymib je_kvdk_mallctlbymib +# define je_mallctlnametomib je_kvdk_mallctlnametomib +# define je_malloc je_kvdk_malloc +# define je_malloc_conf je_kvdk_malloc_conf +# define je_malloc_conf_2_conf_harder je_kvdk_malloc_conf_2_conf_harder +# define je_malloc_message je_kvdk_malloc_message +# define je_malloc_stats_print je_kvdk_malloc_stats_print +# define je_malloc_usable_size je_kvdk_malloc_usable_size +# define je_mallocx je_kvdk_mallocx +# define je_smallocx_36366f3c4c741723369853c923e56999716398fc je_kvdk_smallocx_36366f3c4c741723369853c923e56999716398fc +# define je_nallocx je_kvdk_nallocx +# define je_posix_memalign je_kvdk_posix_memalign +# define je_rallocx je_kvdk_rallocx +# define je_realloc je_kvdk_realloc +# define je_sallocx je_kvdk_sallocx +# define je_sdallocx je_kvdk_sdallocx +# define je_xallocx je_kvdk_xallocx +# define je_memalign je_kvdk_memalign +# define je_valloc je_kvdk_valloc +# define je_pvalloc je_kvdk_pvalloc +#endif + +#include +#include +#include +#include +#include + +#define JEMALLOC_VERSION "5.3.0-17-g36366f3c4c741723369853c923e56999716398fc" +#define JEMALLOC_VERSION_MAJOR 5 +#define JEMALLOC_VERSION_MINOR 3 +#define JEMALLOC_VERSION_BUGFIX 0 +#define JEMALLOC_VERSION_NREV 17 +#define JEMALLOC_VERSION_GID "36366f3c4c741723369853c923e56999716398fc" +#define JEMALLOC_VERSION_GID_IDENT 36366f3c4c741723369853c923e56999716398fc + +#define MALLOCX_LG_ALIGN(la) ((int)(la)) +#if LG_SIZEOF_PTR == 2 +# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1)) +#else +# define MALLOCX_ALIGN(a) \ + ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ + ffs((int)(((size_t)(a))>>32))+31)) +#endif +#define MALLOCX_ZERO ((int)0x40) +/* + * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 + * encodes MALLOCX_TCACHE_NONE. + */ +#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) +#define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) +/* + * Bias arena index bits so that 0 encodes "use an automatically chosen arena". + */ +#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) + +/* + * Use as arena index in "arena..{purge,decay,dss}" and + * "stats.arenas..*" mallctl interfaces to select all arenas. This + * definition is intentionally specified in raw decimal format to support + * cpp-based string concatenation, e.g. + * + * #define STRINGIFY_HELPER(x) #x + * #define STRINGIFY(x) STRINGIFY_HELPER(x) + * + * mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL, + * 0); + */ +#define MALLCTL_ARENAS_ALL 4096 +/* + * Use as arena index in "stats.arenas..*" mallctl interfaces to select + * destroyed arenas. + */ +#define MALLCTL_ARENAS_DESTROYED 4097 + +#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) +# define JEMALLOC_CXX_THROW throw() +#else +# define JEMALLOC_CXX_THROW +#endif + +#if defined(_MSC_VER) +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) __declspec(align(s)) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# ifndef JEMALLOC_EXPORT +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif +# endif +# define JEMALLOC_FORMAT_ARG(i) +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH +# define JEMALLOC_NOINLINE __declspec(noinline) +# ifdef __cplusplus +# define JEMALLOC_NOTHROW __declspec(nothrow) +# else +# define JEMALLOC_NOTHROW +# endif +# define JEMALLOC_SECTION(s) __declspec(allocate(s)) +# define JEMALLOC_RESTRICT_RETURN __declspec(restrict) +# if _MSC_VER >= 1900 && !defined(__EDG__) +# define JEMALLOC_ALLOCATOR __declspec(allocator) +# else +# define JEMALLOC_ALLOCATOR +# endif +# define JEMALLOC_COLD +#elif defined(JEMALLOC_HAVE_ATTR) +# define JEMALLOC_ATTR(s) __attribute__((s)) +# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) +# ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE +# define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s)) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2)) +# else +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# endif +# ifndef JEMALLOC_EXPORT +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG +# define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3)) +# else +# define JEMALLOC_FORMAT_ARG(i) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) +# elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i)) +# else +# define JEMALLOC_FORMAT_PRINTF(s, i) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FALLTHROUGH +# define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough) +# else +# define JEMALLOC_FALLTHROUGH +# endif +# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) +# define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) +# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +# ifdef JEMALLOC_HAVE_ATTR_COLD +# define JEMALLOC_COLD JEMALLOC_ATTR(__cold__) +# else +# define JEMALLOC_COLD +# endif +#else +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# define JEMALLOC_EXPORT +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH +# define JEMALLOC_NOINLINE +# define JEMALLOC_NOTHROW +# define JEMALLOC_SECTION(s) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +# define JEMALLOC_COLD +#endif + +#if (defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__)) && !defined(JEMALLOC_NO_RENAME) +# define JEMALLOC_SYS_NOTHROW +#else +# define JEMALLOC_SYS_NOTHROW JEMALLOC_NOTHROW +#endif + +/* + * The je_ prefix on the following public symbol declarations is an artifact + * of namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle.h). + */ +extern JEMALLOC_EXPORT const char *je_malloc_conf; +extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, + const char *s); + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_malloc(size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_calloc(size_t num, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); +JEMALLOC_EXPORT int JEMALLOC_SYS_NOTHROW je_posix_memalign( + void **memptr, size_t alignment, size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_aligned_alloc(size_t alignment, + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) + JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_realloc(void *ptr, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW je_free(void *ptr) + JEMALLOC_CXX_THROW; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_rallocx(void *ptr, size_t size, + int flags) JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_xallocx(void *ptr, size_t size, + size_t extra, int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_sallocx(const void *ptr, + int flags) JEMALLOC_ATTR(pure); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_sdallocx(void *ptr, size_t size, + int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); + +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctl(const char *name, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlnametomib(const char *name, + size_t *mibp, size_t *miblenp); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlbymib(const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print( + void (*write_cb)(void *, const char *), void *je_cbopaque, + const char *opts); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +#ifdef JEMALLOC_HAVE_MALLOC_SIZE +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_size( + const void *ptr); +#endif + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_memalign(size_t alignment, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_PVALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_pvalloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif + +typedef struct extent_hooks_s extent_hooks_t; + +/* + * void * + * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + * size_t alignment, bool *zero, bool *commit, unsigned arena_ind); + */ +typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *, + bool *, unsigned); + +/* + * bool + * extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * void + * extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef void (extent_destroy_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * bool + * extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t, + size_t, unsigned); + +/* + * bool + * extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t size_a, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + bool, unsigned); + +/* + * bool + * extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + * void *addr_b, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, + bool, unsigned); + +struct extent_hooks_s { + extent_alloc_t *alloc; + extent_dalloc_t *dalloc; + extent_destroy_t *destroy; + extent_commit_t *commit; + extent_decommit_t *decommit; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; + extent_split_t *split; + extent_merge_t *merge; +}; + +/* + * By default application code must explicitly refer to mangled symbol names, + * so that it is possible to use jemalloc in conjunction with another allocator + * in the same application. Define JEMALLOC_MANGLE in order to cause automatic + * name mangling that matches the API prefixing that happened as a result of + * --with-mangling and/or --with-jemalloc-prefix configuration settings. + */ +#ifdef JEMALLOC_MANGLE +# ifndef JEMALLOC_NO_DEMANGLE +# define JEMALLOC_NO_DEMANGLE +# endif +# define aligned_alloc je_aligned_alloc +# define calloc je_calloc +# define dallocx je_dallocx +# define free je_free +# define mallctl je_mallctl +# define mallctlbymib je_mallctlbymib +# define mallctlnametomib je_mallctlnametomib +# define malloc je_malloc +# define malloc_conf je_malloc_conf +# define malloc_conf_2_conf_harder je_malloc_conf_2_conf_harder +# define malloc_message je_malloc_message +# define malloc_stats_print je_malloc_stats_print +# define malloc_usable_size je_malloc_usable_size +# define mallocx je_mallocx +# define smallocx_36366f3c4c741723369853c923e56999716398fc je_smallocx_36366f3c4c741723369853c923e56999716398fc +# define nallocx je_nallocx +# define posix_memalign je_posix_memalign +# define rallocx je_rallocx +# define realloc je_realloc +# define sallocx je_sallocx +# define sdallocx je_sdallocx +# define xallocx je_xallocx +# define memalign je_memalign +# define valloc je_valloc +# define pvalloc je_pvalloc +#endif + +/* + * The je_* macros can be used as stable alternative names for the + * public jemalloc API if JEMALLOC_NO_DEMANGLE is defined. This is primarily + * meant for use in jemalloc itself, but it can be used by application code to + * provide isolation from the name mangling specified via --with-mangling + * and/or --with-jemalloc-prefix. + */ +#ifndef JEMALLOC_NO_DEMANGLE +# undef je_aligned_alloc +# undef je_calloc +# undef je_dallocx +# undef je_free +# undef je_mallctl +# undef je_mallctlbymib +# undef je_mallctlnametomib +# undef je_malloc +# undef je_malloc_conf +# undef je_malloc_conf_2_conf_harder +# undef je_malloc_message +# undef je_malloc_stats_print +# undef je_malloc_usable_size +# undef je_mallocx +# undef je_smallocx_36366f3c4c741723369853c923e56999716398fc +# undef je_nallocx +# undef je_posix_memalign +# undef je_rallocx +# undef je_realloc +# undef je_sallocx +# undef je_sdallocx +# undef je_xallocx +# undef je_memalign +# undef je_valloc +# undef je_pvalloc +#endif + +#ifdef __cplusplus +} +#endif +#endif /* JEMALLOC_H_ */ diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_defs.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_defs.h new file mode 100644 index 00000000..341c6e98 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_defs.h @@ -0,0 +1,56 @@ +/* include/jemalloc/jemalloc_defs.h. Generated from jemalloc_defs.h.in by configure. */ +/* Defined if __attribute__((...)) syntax is supported. */ +#define JEMALLOC_HAVE_ATTR + +/* Defined if alloc_size attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_ALLOC_SIZE + +/* Defined if format_arg(...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_ARG + +/* Defined if format(gnu_printf, ...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF + +/* Defined if format(printf, ...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF + +/* Defined if fallthrough attribute is supported. */ +/* #undef JEMALLOC_HAVE_ATTR_FALLTHROUGH */ + +/* Defined if cold attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_COLD + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +#define JEMALLOC_OVERRIDE_MEMALIGN +#define JEMALLOC_OVERRIDE_VALLOC +#define JEMALLOC_OVERRIDE_PVALLOC + +/* + * At least Linux omits the "const" in: + * + * size_t malloc_usable_size(const void *ptr); + * + * Match the operating system's prototype. + */ +#define JEMALLOC_USABLE_SIZE_CONST + +/* + * If defined, specify throw() for the public function prototypes when compiling + * with C++. The only justification for this is to match the prototypes that + * glibc defines. + */ +#define JEMALLOC_USE_CXX_THROW + +#ifdef _MSC_VER +# ifdef _WIN64 +# define LG_SIZEOF_PTR_WIN 3 +# else +# define LG_SIZEOF_PTR_WIN 2 +# endif +#endif + +/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ +#define LG_SIZEOF_PTR 3 diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_macros.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_macros.h new file mode 100644 index 00000000..c02e5608 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_macros.h @@ -0,0 +1,149 @@ +#include +#include +#include +#include +#include + +#define JEMALLOC_VERSION "5.3.0-17-g36366f3c4c741723369853c923e56999716398fc" +#define JEMALLOC_VERSION_MAJOR 5 +#define JEMALLOC_VERSION_MINOR 3 +#define JEMALLOC_VERSION_BUGFIX 0 +#define JEMALLOC_VERSION_NREV 17 +#define JEMALLOC_VERSION_GID "36366f3c4c741723369853c923e56999716398fc" +#define JEMALLOC_VERSION_GID_IDENT 36366f3c4c741723369853c923e56999716398fc + +#define MALLOCX_LG_ALIGN(la) ((int)(la)) +#if LG_SIZEOF_PTR == 2 +# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1)) +#else +# define MALLOCX_ALIGN(a) \ + ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ + ffs((int)(((size_t)(a))>>32))+31)) +#endif +#define MALLOCX_ZERO ((int)0x40) +/* + * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 + * encodes MALLOCX_TCACHE_NONE. + */ +#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) +#define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) +/* + * Bias arena index bits so that 0 encodes "use an automatically chosen arena". + */ +#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) + +/* + * Use as arena index in "arena..{purge,decay,dss}" and + * "stats.arenas..*" mallctl interfaces to select all arenas. This + * definition is intentionally specified in raw decimal format to support + * cpp-based string concatenation, e.g. + * + * #define STRINGIFY_HELPER(x) #x + * #define STRINGIFY(x) STRINGIFY_HELPER(x) + * + * mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL, + * 0); + */ +#define MALLCTL_ARENAS_ALL 4096 +/* + * Use as arena index in "stats.arenas..*" mallctl interfaces to select + * destroyed arenas. + */ +#define MALLCTL_ARENAS_DESTROYED 4097 + +#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) +# define JEMALLOC_CXX_THROW throw() +#else +# define JEMALLOC_CXX_THROW +#endif + +#if defined(_MSC_VER) +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) __declspec(align(s)) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# ifndef JEMALLOC_EXPORT +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif +# endif +# define JEMALLOC_FORMAT_ARG(i) +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH +# define JEMALLOC_NOINLINE __declspec(noinline) +# ifdef __cplusplus +# define JEMALLOC_NOTHROW __declspec(nothrow) +# else +# define JEMALLOC_NOTHROW +# endif +# define JEMALLOC_SECTION(s) __declspec(allocate(s)) +# define JEMALLOC_RESTRICT_RETURN __declspec(restrict) +# if _MSC_VER >= 1900 && !defined(__EDG__) +# define JEMALLOC_ALLOCATOR __declspec(allocator) +# else +# define JEMALLOC_ALLOCATOR +# endif +# define JEMALLOC_COLD +#elif defined(JEMALLOC_HAVE_ATTR) +# define JEMALLOC_ATTR(s) __attribute__((s)) +# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) +# ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE +# define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s)) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2)) +# else +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# endif +# ifndef JEMALLOC_EXPORT +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG +# define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3)) +# else +# define JEMALLOC_FORMAT_ARG(i) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) +# elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i)) +# else +# define JEMALLOC_FORMAT_PRINTF(s, i) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FALLTHROUGH +# define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough) +# else +# define JEMALLOC_FALLTHROUGH +# endif +# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) +# define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) +# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +# ifdef JEMALLOC_HAVE_ATTR_COLD +# define JEMALLOC_COLD JEMALLOC_ATTR(__cold__) +# else +# define JEMALLOC_COLD +# endif +#else +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# define JEMALLOC_EXPORT +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH +# define JEMALLOC_NOINLINE +# define JEMALLOC_NOTHROW +# define JEMALLOC_SECTION(s) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +# define JEMALLOC_COLD +#endif + +#if (defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__)) && !defined(JEMALLOC_NO_RENAME) +# define JEMALLOC_SYS_NOTHROW +#else +# define JEMALLOC_SYS_NOTHROW JEMALLOC_NOTHROW +#endif diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_mangle.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_mangle.h new file mode 100644 index 00000000..b87a1f33 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_mangle.h @@ -0,0 +1,72 @@ +/* + * By default application code must explicitly refer to mangled symbol names, + * so that it is possible to use jemalloc in conjunction with another allocator + * in the same application. Define JEMALLOC_MANGLE in order to cause automatic + * name mangling that matches the API prefixing that happened as a result of + * --with-mangling and/or --with-jemalloc-prefix configuration settings. + */ +#ifdef JEMALLOC_MANGLE +# ifndef JEMALLOC_NO_DEMANGLE +# define JEMALLOC_NO_DEMANGLE +# endif +# define aligned_alloc je_aligned_alloc +# define calloc je_calloc +# define dallocx je_dallocx +# define free je_free +# define mallctl je_mallctl +# define mallctlbymib je_mallctlbymib +# define mallctlnametomib je_mallctlnametomib +# define malloc je_malloc +# define malloc_conf je_malloc_conf +# define malloc_conf_2_conf_harder je_malloc_conf_2_conf_harder +# define malloc_message je_malloc_message +# define malloc_stats_print je_malloc_stats_print +# define malloc_usable_size je_malloc_usable_size +# define mallocx je_mallocx +# define smallocx_36366f3c4c741723369853c923e56999716398fc je_smallocx_36366f3c4c741723369853c923e56999716398fc +# define nallocx je_nallocx +# define posix_memalign je_posix_memalign +# define rallocx je_rallocx +# define realloc je_realloc +# define sallocx je_sallocx +# define sdallocx je_sdallocx +# define xallocx je_xallocx +# define memalign je_memalign +# define valloc je_valloc +# define pvalloc je_pvalloc +#endif + +/* + * The je_* macros can be used as stable alternative names for the + * public jemalloc API if JEMALLOC_NO_DEMANGLE is defined. This is primarily + * meant for use in jemalloc itself, but it can be used by application code to + * provide isolation from the name mangling specified via --with-mangling + * and/or --with-jemalloc-prefix. + */ +#ifndef JEMALLOC_NO_DEMANGLE +# undef je_aligned_alloc +# undef je_calloc +# undef je_dallocx +# undef je_free +# undef je_mallctl +# undef je_mallctlbymib +# undef je_mallctlnametomib +# undef je_malloc +# undef je_malloc_conf +# undef je_malloc_conf_2_conf_harder +# undef je_malloc_message +# undef je_malloc_stats_print +# undef je_malloc_usable_size +# undef je_mallocx +# undef je_smallocx_36366f3c4c741723369853c923e56999716398fc +# undef je_nallocx +# undef je_posix_memalign +# undef je_rallocx +# undef je_realloc +# undef je_sallocx +# undef je_sdallocx +# undef je_xallocx +# undef je_memalign +# undef je_valloc +# undef je_pvalloc +#endif diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_mangle_jet.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_mangle_jet.h new file mode 100644 index 00000000..d883fe69 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_mangle_jet.h @@ -0,0 +1,72 @@ +/* + * By default application code must explicitly refer to mangled symbol names, + * so that it is possible to use jemalloc in conjunction with another allocator + * in the same application. Define JEMALLOC_MANGLE in order to cause automatic + * name mangling that matches the API prefixing that happened as a result of + * --with-mangling and/or --with-jemalloc-prefix configuration settings. + */ +#ifdef JEMALLOC_MANGLE +# ifndef JEMALLOC_NO_DEMANGLE +# define JEMALLOC_NO_DEMANGLE +# endif +# define aligned_alloc jet_aligned_alloc +# define calloc jet_calloc +# define dallocx jet_dallocx +# define free jet_free +# define mallctl jet_mallctl +# define mallctlbymib jet_mallctlbymib +# define mallctlnametomib jet_mallctlnametomib +# define malloc jet_malloc +# define malloc_conf jet_malloc_conf +# define malloc_conf_2_conf_harder jet_malloc_conf_2_conf_harder +# define malloc_message jet_malloc_message +# define malloc_stats_print jet_malloc_stats_print +# define malloc_usable_size jet_malloc_usable_size +# define mallocx jet_mallocx +# define smallocx_36366f3c4c741723369853c923e56999716398fc jet_smallocx_36366f3c4c741723369853c923e56999716398fc +# define nallocx jet_nallocx +# define posix_memalign jet_posix_memalign +# define rallocx jet_rallocx +# define realloc jet_realloc +# define sallocx jet_sallocx +# define sdallocx jet_sdallocx +# define xallocx jet_xallocx +# define memalign jet_memalign +# define valloc jet_valloc +# define pvalloc jet_pvalloc +#endif + +/* + * The jet_* macros can be used as stable alternative names for the + * public jemalloc API if JEMALLOC_NO_DEMANGLE is defined. This is primarily + * meant for use in jemalloc itself, but it can be used by application code to + * provide isolation from the name mangling specified via --with-mangling + * and/or --with-jemalloc-prefix. + */ +#ifndef JEMALLOC_NO_DEMANGLE +# undef jet_aligned_alloc +# undef jet_calloc +# undef jet_dallocx +# undef jet_free +# undef jet_mallctl +# undef jet_mallctlbymib +# undef jet_mallctlnametomib +# undef jet_malloc +# undef jet_malloc_conf +# undef jet_malloc_conf_2_conf_harder +# undef jet_malloc_message +# undef jet_malloc_stats_print +# undef jet_malloc_usable_size +# undef jet_mallocx +# undef jet_smallocx_36366f3c4c741723369853c923e56999716398fc +# undef jet_nallocx +# undef jet_posix_memalign +# undef jet_rallocx +# undef jet_realloc +# undef jet_sallocx +# undef jet_sdallocx +# undef jet_xallocx +# undef jet_memalign +# undef jet_valloc +# undef jet_pvalloc +#endif diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_protos.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_protos.h new file mode 100644 index 00000000..082083d9 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_protos.h @@ -0,0 +1,77 @@ +/* + * The je_ prefix on the following public symbol declarations is an artifact + * of namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle.h). + */ +extern JEMALLOC_EXPORT const char *je_malloc_conf; +extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, + const char *s); + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_malloc(size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_calloc(size_t num, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); +JEMALLOC_EXPORT int JEMALLOC_SYS_NOTHROW je_posix_memalign( + void **memptr, size_t alignment, size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_aligned_alloc(size_t alignment, + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) + JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_realloc(void *ptr, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW je_free(void *ptr) + JEMALLOC_CXX_THROW; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_rallocx(void *ptr, size_t size, + int flags) JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_xallocx(void *ptr, size_t size, + size_t extra, int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_sallocx(const void *ptr, + int flags) JEMALLOC_ATTR(pure); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_sdallocx(void *ptr, size_t size, + int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); + +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctl(const char *name, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlnametomib(const char *name, + size_t *mibp, size_t *miblenp); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlbymib(const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print( + void (*write_cb)(void *, const char *), void *je_cbopaque, + const char *opts); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +#ifdef JEMALLOC_HAVE_MALLOC_SIZE +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_size( + const void *ptr); +#endif + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_memalign(size_t alignment, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_PVALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *je_pvalloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h new file mode 100644 index 00000000..80af5ea0 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h @@ -0,0 +1,77 @@ +/* + * The jet_ prefix on the following public symbol declarations is an artifact + * of namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle@install_suffix@.h). + */ +extern JEMALLOC_EXPORT const char *jet_malloc_conf; +extern JEMALLOC_EXPORT void (*jet_malloc_message)(void *cbopaque, + const char *s); + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_malloc(size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_calloc(size_t num, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); +JEMALLOC_EXPORT int JEMALLOC_SYS_NOTHROW jet_posix_memalign( + void **memptr, size_t alignment, size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_aligned_alloc(size_t alignment, + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) + JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_realloc(void *ptr, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW jet_free(void *ptr) + JEMALLOC_CXX_THROW; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *jet_mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *jet_rallocx(void *ptr, size_t size, + int flags) JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_xallocx(void *ptr, size_t size, + size_t extra, int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_sallocx(const void *ptr, + int flags) JEMALLOC_ATTR(pure); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_sdallocx(void *ptr, size_t size, + int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); + +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctl(const char *name, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctlnametomib(const char *name, + size_t *mibp, size_t *miblenp); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctlbymib(const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_malloc_stats_print( + void (*write_cb)(void *, const char *), void *jet_cbopaque, + const char *opts); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +#ifdef JEMALLOC_HAVE_MALLOC_SIZE +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_malloc_size( + const void *ptr); +#endif + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_memalign(size_t alignment, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_valloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_PVALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_pvalloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_rename.h new file mode 100644 index 00000000..dd61f3d7 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_rename.h @@ -0,0 +1,32 @@ +/* + * Name mangling for public symbols is controlled by --with-mangling and + * --with-jemalloc-prefix. With default settings the je_ prefix is stripped by + * these macro definitions. + */ +#ifndef JEMALLOC_NO_RENAME +# define je_aligned_alloc je_kvdk_aligned_alloc +# define je_calloc je_kvdk_calloc +# define je_dallocx je_kvdk_dallocx +# define je_free je_kvdk_free +# define je_mallctl je_kvdk_mallctl +# define je_mallctlbymib je_kvdk_mallctlbymib +# define je_mallctlnametomib je_kvdk_mallctlnametomib +# define je_malloc je_kvdk_malloc +# define je_malloc_conf je_kvdk_malloc_conf +# define je_malloc_conf_2_conf_harder je_kvdk_malloc_conf_2_conf_harder +# define je_malloc_message je_kvdk_malloc_message +# define je_malloc_stats_print je_kvdk_malloc_stats_print +# define je_malloc_usable_size je_kvdk_malloc_usable_size +# define je_mallocx je_kvdk_mallocx +# define je_smallocx_36366f3c4c741723369853c923e56999716398fc je_kvdk_smallocx_36366f3c4c741723369853c923e56999716398fc +# define je_nallocx je_kvdk_nallocx +# define je_posix_memalign je_kvdk_posix_memalign +# define je_rallocx je_kvdk_rallocx +# define je_realloc je_kvdk_realloc +# define je_sallocx je_kvdk_sallocx +# define je_sdallocx je_kvdk_sdallocx +# define je_xallocx je_kvdk_xallocx +# define je_memalign je_kvdk_memalign +# define je_valloc je_kvdk_valloc +# define je_pvalloc je_kvdk_pvalloc +#endif diff --git a/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h new file mode 100644 index 00000000..1a588743 --- /dev/null +++ b/volatile/extern/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h @@ -0,0 +1,77 @@ +typedef struct extent_hooks_s extent_hooks_t; + +/* + * void * + * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + * size_t alignment, bool *zero, bool *commit, unsigned arena_ind); + */ +typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *, + bool *, unsigned); + +/* + * bool + * extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * void + * extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef void (extent_destroy_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * bool + * extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t, + size_t, unsigned); + +/* + * bool + * extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t size_a, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + bool, unsigned); + +/* + * bool + * extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + * void *addr_b, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, + bool, unsigned); + +struct extent_hooks_s { + extent_alloc_t *alloc; + extent_dalloc_t *dalloc; + extent_destroy_t *destroy; + extent_commit_t *commit; + extent_decommit_t *decommit; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; + extent_split_t *split; + extent_merge_t *merge; +}; diff --git a/volatile/extern/libpmemobj++/string_view.hpp b/volatile/extern/libpmemobj++/string_view.hpp new file mode 100644 index 00000000..e3461d3a --- /dev/null +++ b/volatile/extern/libpmemobj++/string_view.hpp @@ -0,0 +1,1416 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2020-2021, Intel Corporation */ + +/** + * @file + * Our partial std::string_view implementation. + */ + +#ifndef LIBPMEMOBJ_CPP_STRING_VIEW +#define LIBPMEMOBJ_CPP_STRING_VIEW + +#include +#include +#include +#include +#include + +#if __cpp_lib_string_view +#include +#endif + +namespace pmem +{ + +namespace obj +{ + +#if __cpp_lib_string_view + +template > +using basic_string_view = std::basic_string_view; +using string_view = std::string_view; +using wstring_view = std::basic_string_view; +using u16string_view = std::basic_string_view; +using u32string_view = std::basic_string_view; + +#else + +/** + * Our partial std::string_view implementation. + * + * If C++17's std::string_view implementation is not available, this one + * is used to avoid unnecessary string copying. It's compatible with + * the std API, but it does not cover all functionalities. + * @ingroup data_view + */ +template > +class basic_string_view { +public: + /* Member types */ + using traits_type = Traits; + using value_type = CharT; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type &; + using const_reference = const value_type &; + using pointer = value_type *; + using const_pointer = const value_type *; + using const_iterator = const_pointer; + using iterator = const_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + static constexpr const size_type npos = + (std::numeric_limits::max)(); + + constexpr basic_string_view() noexcept; + constexpr basic_string_view(const CharT *data, size_type size); + constexpr basic_string_view(const std::basic_string &s); + constexpr basic_string_view(const CharT *data); + + /** Constructor initialized with the basic_string_view *rhs*. + * + * @param[in] rhs basic_string_view to initialize with + */ + constexpr basic_string_view(const basic_string_view &rhs) noexcept = + default; + + /** + * Replaces the view with that of *rhs*. + * + * @param[in] rhs basic_string_view to replace with + */ + basic_string_view & + operator=(const basic_string_view &rhs) noexcept = default; + + constexpr const_iterator begin() const noexcept; + constexpr const_iterator cbegin() const noexcept; + constexpr const_iterator end() const noexcept; + constexpr const_iterator cend() const noexcept; + constexpr const_reverse_iterator rbegin() const noexcept; + constexpr const_reverse_iterator crbegin() const noexcept; + constexpr const_reverse_iterator rend() const noexcept; + constexpr const_reverse_iterator crend() const noexcept; + + constexpr const CharT *data() const noexcept; + constexpr size_type size() const noexcept; + constexpr size_type length() const noexcept; + constexpr bool empty() const noexcept; + constexpr size_type max_size() const noexcept; + + const CharT &at(size_type pos) const; + constexpr const CharT &operator[](size_type pos) const noexcept; + constexpr const_reference front() const noexcept; + constexpr const_reference back() const noexcept; + + void remove_prefix(size_type n); + void remove_suffix(size_type n); + void swap(basic_string_view &v) noexcept; + + constexpr basic_string_view substr(size_type pos = 0, + size_type count = npos) const; + size_type copy(CharT *dest, size_type count, size_type pos = 0) const; + inline int compare(size_type pos1, size_type n1, + basic_string_view sv) const; + inline int compare(size_type pos1, size_type n1, basic_string_view sv, + size_type pos2, size_type n2) const; + inline int compare(const CharT *s) const noexcept; + inline int compare(size_type pos1, size_type n1, const CharT *s) const; + inline int compare(size_type pos1, size_type n1, const CharT *s, + size_type n2) const; + int compare(const basic_string_view &other) const noexcept; + + size_type find(basic_string_view str, size_type pos = 0) const noexcept; + size_type find(CharT ch, size_type pos = 0) const noexcept; + size_type find(const CharT *s, size_type pos = 0) const; + size_type find(const CharT *s, size_type pos, size_type count) const; + + size_type rfind(basic_string_view str, size_type pos = npos) const + noexcept; + size_type rfind(const CharT *s, size_type pos, size_type count) const; + size_type rfind(const CharT *s, size_type pos = npos) const; + size_type rfind(CharT ch, size_type pos = npos) const noexcept; + size_type find_first_of(basic_string_view str, size_type pos = 0) const + noexcept; + size_type find_first_of(const CharT *s, size_type pos, + size_type count) const; + size_type find_first_of(const CharT *s, size_type pos = 0) const; + size_type find_first_of(CharT ch, size_type pos = 0) const noexcept; + size_type find_first_not_of(basic_string_view str, + size_type pos = 0) const noexcept; + size_type find_first_not_of(const CharT *s, size_type pos, + size_type count) const; + size_type find_first_not_of(const CharT *s, size_type pos = 0) const; + size_type find_first_not_of(CharT ch, size_type pos = 0) const noexcept; + size_type find_last_of(basic_string_view str, + size_type pos = npos) const noexcept; + size_type find_last_of(const CharT *s, size_type pos, + size_type count) const; + size_type find_last_of(const CharT *s, size_type pos = npos) const; + size_type find_last_of(CharT ch, size_type pos = npos) const noexcept; + size_type find_last_not_of(basic_string_view str, + size_type pos = npos) const noexcept; + size_type find_last_not_of(const CharT *s, size_type pos, + size_type count) const; + size_type find_last_not_of(const CharT *s, size_type pos = npos) const; + size_type find_last_not_of(CharT ch, size_type pos = npos) const + noexcept; + +private: + const value_type *data_; + size_type size_; +}; + +/** + * The most typical string_view usage - the char specialization. + * @ingroup data_view + */ +using string_view = basic_string_view; + +/** + * The wide char specialization. + * @ingroup data_view + */ +using wstring_view = basic_string_view; + +/** + * The char16 specialization. + * @ingroup data_view + */ +using u16string_view = basic_string_view; + +/** + * The char32 specialization. + * @ingroup data_view + */ +using u32string_view = basic_string_view; + +/** + * Default constructor with empty data. + */ +template +constexpr inline basic_string_view::basic_string_view() noexcept + : data_(nullptr), size_(0) +{ +} + +/** + * Constructor initialized with *data* and its *size*. + * + * @param[in] data pointer to the C-like string to initialize with, + * it can contain null characters. + * @param[in] size length of the given data. + */ +template +constexpr inline basic_string_view::basic_string_view( + const CharT *data, size_type size) + : data_(data), size_(size) +{ +} + +/** + * Constructor initialized by the basic string *s*. + * + * @param[in] s reference to the string to initialize with. + */ +template +constexpr inline basic_string_view::basic_string_view( + const std::basic_string &s) + : data_(s.c_str()), size_(s.size()) +{ +} + +/** + * Constructor initialized by *data*. Size of the data will be set + * using Traits::length(). + * + * @param[in] data pointer to C-like string (char *) to initialize with, + * it has to end with the terminating null character. + */ +template +constexpr inline basic_string_view::basic_string_view( + const CharT *data) + : data_(data), size_(Traits::length(data)) +{ +} + +/** + * Returns an iterator to the first character of the view. + * + * @return const_iterator to the first character + */ +template +constexpr typename basic_string_view::const_iterator +basic_string_view::begin() const noexcept +{ + return cbegin(); +} + +/** + * Returns an iterator to the first character of the view. + * + * @return const_iterator to the first character + */ +template +constexpr typename basic_string_view::const_iterator +basic_string_view::cbegin() const noexcept +{ + return data_; +} + +/** + * Returns an iterator to the character following the last character of the + * view. This character acts as a placeholder, attempting to access it results + * in undefined behavior. + * + * @return const_iterator to the character following the last character. + */ +template +constexpr typename basic_string_view::const_iterator +basic_string_view::end() const noexcept +{ + return cend(); +} + +/** + * Returns an iterator to the character following the last character of the + * view. This character acts as a placeholder, attempting to access it results + * in undefined behavior. + * + * @return const_iterator to the character following the last character. + */ +template +constexpr typename basic_string_view::const_iterator +basic_string_view::cend() const noexcept +{ + return data_ + size_; +} + +/** + * Returns a reverse_iterator to the character following the last character of + * the view (reverse beginning). Reverse iterators iterate backwards: increasing + * them moves them towards the beginning of the string. + * + * @return const_reverse_iterator to the character following the last character. + */ +template +constexpr typename basic_string_view::const_reverse_iterator +basic_string_view::rbegin() const noexcept +{ + return reverse_iterator(cend()); +} + +/** + * Returns a reverse_iterator to the character following the last character of + * the view (reverse beginning). Reverse iterators iterate backwards: increasing + * them moves them towards the beginning of the string. + * + * @return const_reverse_iterator to the character following the last character. + */ +template +constexpr typename basic_string_view::const_reverse_iterator +basic_string_view::crbegin() const noexcept +{ + return reverse_iterator(cend()); +} + +/** + * Returns a reverse_iterator to the first character of the view. + * Reverse iterators iterate backwards: increasing + * them moves them towards the beginning of the string. + * This character acts as a placeholder, attempting to access it results + * in undefined behavior. + * + * @return const_reverse_iterator to the first character + */ +template +constexpr typename basic_string_view::const_reverse_iterator +basic_string_view::rend() const noexcept +{ + return reverse_iterator(cbegin()); +} + +/** + * Returns a reverse_iterator to the first character of the view. + * Reverse iterators iterate backwards: increasing + * them moves them towards the beginning of the string. + * This character acts as a placeholder, attempting to access it results + * in undefined behavior. + * + * @return const_reverse_iterator to the first character + */ +template +constexpr typename basic_string_view::const_reverse_iterator +basic_string_view::crend() const noexcept +{ + return reverse_iterator(cbegin()); +} + +/** + * Returns pointer to data stored in this pmem::obj::string_view. It may + * not contain the terminating null character. + * + * @return pointer to C-like string (char *), it may not end with null + * character. + */ +template +constexpr inline const CharT * +basic_string_view::data() const noexcept +{ + return data_; +} + +/** + * Returns that view is empty or not. + * + * @return true when size() == 0. + */ +template +constexpr inline bool +basic_string_view::empty() const noexcept +{ + return size() == 0; +} + +/** + * Returns the largest possible number of char-like objects that can be + * referred to by a basic_string_view. + * + * @return maximum number of characters. + */ +template +constexpr inline typename basic_string_view::size_type +basic_string_view::max_size() const noexcept +{ + return (std::numeric_limits::max)(); +} + +/** + * Returns count of characters stored in this pmem::obj::string_view + * data. + * + * @return the number of CharT elements in the view. + */ +template +constexpr inline typename basic_string_view::size_type +basic_string_view::size() const noexcept +{ + return size_; +} + +/** + * Returns count of characters stored in this pmem::obj::string_view data. + * + * @return the number of CharT elements in the view. + */ +template +constexpr inline typename basic_string_view::size_type +basic_string_view::length() const noexcept +{ + return size_; +} + +/** + * Returns reference to a character at position @param[in] pos . + * + * @return reference to the char. + */ +template +constexpr inline const CharT & + basic_string_view::operator[](size_t pos) const noexcept +{ + return data()[pos]; +} + +/** + * Returns reference to the character at position @param[in] pos and + * performs bound checking. + * + * @return reference to the char. + * + * @throw std::out_of_range when out of bounds occurs. + */ +template +inline const CharT & +basic_string_view::at(size_t pos) const +{ + if (pos >= size()) + throw std::out_of_range("Accessing a position out of bounds!"); + return data()[pos]; +} + +/** + * Returns reference to the last character in the view. + * The behavior is undefined if empty() == true. + * + * @return reference to the last character. + */ +template +constexpr inline const CharT & +basic_string_view::back() const noexcept +{ + return operator[](size() - 1); +} + +/** + * Returns reference to the first character in the view. + * The behavior is undefined if empty() == true. + * + * @return reference to the first character. + */ +template +constexpr inline const CharT & +basic_string_view::front() const noexcept +{ + return operator[](0); +} + +/** + * Moves the start of the view forward by n characters. + * The behavior is undefined if n > size(). + * + * @param[in] n number of characters to remove from the start of the view + */ +template +void +basic_string_view::remove_prefix(size_type n) +{ + data_ += n; + size_ -= n; +} + +/** + * Moves the end of the view back by n characters. + * The behavior is undefined if n > size(). + * + * @param[in] n number of characters to remove from the end of the view + */ +template +void +basic_string_view::remove_suffix(size_type n) +{ + size_ -= n; +} + +/** + * Exchanges the view with that of v. + * + * @param[in] v view to swap with + */ +template +void +basic_string_view::swap( + basic_string_view &v) noexcept +{ + std::swap(data_, v.data_); + std::swap(size_, v.size_); +} + +/** + * Finds the first substring equal to str. + * + * @param[in] str string to search for + * @param[in] pos position at which to start the search + * + * @return Position of the first character of the found substring or + * npos if no such substring is found. + */ +template +typename basic_string_view::size_type +basic_string_view::find(basic_string_view str, + size_type pos) const noexcept +{ + return find(str.data(), pos, str.size()); +} + +/** + * Finds the first character ch + * + * @param[in] ch character to search for + * @param[in] pos position at which to start the search + * + * @return Position of the first character equal to ch, or npos if no such + * character is found. + */ +template +typename basic_string_view::size_type +basic_string_view::find(CharT ch, size_type pos) const noexcept +{ + return find(&ch, pos, 1); +} + +/** + * Finds the first substring equal to the range [s, s+count). + * This range may contain null characters. + * + * @param[in] s pointer to the C-style string to search for + * @param[in] pos position at which to start the search + * @param[in] count length of the substring to search for + * + * @return Position of the first character of the found substring or + * npos if no such substring is found. + */ +template +typename basic_string_view::size_type +basic_string_view::find(const CharT *s, size_type pos, + size_type count) const +{ + auto sz = size(); + + if (pos > sz) + return npos; + + if (count == 0) + return pos; + + while (pos + count <= sz) { + auto found = traits_type::find(data() + pos, sz - pos, s[0]); + if (!found) + return npos; + pos = static_cast(std::distance(data(), found)); + if (traits_type::compare(found, s, count) == 0) { + return pos; + } + ++pos; + } + return npos; +} + +/** + * Finds the first substring equal to the C-style string pointed to by s. + * The length of the string is determined by the first null character. + * + * @param[in] s pointer to the C-style string to search for + * @param[in] pos position at which to start the search + * + * @return Position of the first character of the found substring or + * npos if no such substring is found. + */ +template +typename basic_string_view::size_type +basic_string_view::find(const CharT *s, size_type pos) const +{ + return find(s, pos, traits_type::length(s)); +} + +/** + * Finds the last substring equal to str. + * If npos or any value not smaller than size()-1 is passed as pos, whole string + * will be searched. + * @param[in] str string to search for + * @param[in] pos position at which to start the search + * + * @return Position (as an offset from the start of the string) of the first + * character of the found substring or npos if no such substring is found + */ +template +typename basic_string_view::size_type +basic_string_view::rfind(basic_string_view str, + size_type pos) const noexcept +{ + return rfind(str.data(), pos, str.size()); +} + +/** + * Finds the last substring equal to the range [s, s+count). + * This range can include null characters. When pos is specified, the search + * only includes sequences of characters that begin at or before position pos, + * ignoring any possible match beginning after pos. If npos or any value not + * smaller than size()-1 is passed as pos, whole string will be searched. + * + * @param[in] s pointer to the C-style string to search for + * @param[in] pos position at which to start the search + * @param[in] count length of the substring to search for + * + * @return Position (as an offset from the start of the string) of the first + * character of the found substring or npos if no such substring is found. If + * searching for an empty string returns pos unless pos > size(), in which + * case returns size(). + */ +template +typename basic_string_view::size_type +basic_string_view::rfind(const CharT *s, size_type pos, + size_type count) const +{ + if (count <= size()) { + pos = (std::min)(size() - count, pos); + do { + if (traits_type::compare(data() + pos, s, count) == 0) + return pos; + } while (pos-- > 0); + } + return npos; +} + +/** + * Finds the last substring equal to the C-style string pointed to by s. + * The length of the string is determined by the first null character. + * If npos or any value not smaller than size()-1 is passed as pos, whole string + * will be searched. + * + * @param[in] s pointer to the C-style string to search for + * @param[in] pos position at which to start the search + * + * @return Position (as an offset from the start of the string) of the first + * character of the found substring or npos if no such substring is found + */ +template +typename basic_string_view::size_type +basic_string_view::rfind(const CharT *s, size_type pos) const +{ + return rfind(s, pos, traits_type::length(s)); +} + +/** + * Finds the last character equal to ch. + * If npos or any value not smaller than size()-1 is passed as pos, whole string + * will be searched. + * + * @param[in] ch character to search for + * @param[in] pos position at which to start the search + * + * @return Position (as an offset from the start of the string) of the first + * character equal to ch or npos if no such character is found + */ +template +typename basic_string_view::size_type +basic_string_view::rfind(CharT ch, size_type pos) const noexcept +{ + return rfind(&ch, pos, 1); +} + +/** + * Finds the first character equal to any of the characters in str. + * + * @param[in] str string identifying characters to search for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_of(basic_string_view str, + size_type pos) const noexcept +{ + return find_first_of(str.data(), pos, str.size()); +} + +/** + * Finds the first character equal to any of the characters + * in the range [s, s+count). This range can include null characters. + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * @param[in] count length of the C-style string identifying characters to + * search for + * + * @return The position of the first character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_of(const CharT *s, size_type pos, + size_type count) const +{ + size_type first_of = npos; + for (const CharT *c = s; c != s + count; ++c) { + size_type found = find(*c, pos); + if (found != npos && found < first_of) + first_of = found; + } + return first_of; +} + +/** + * Finds the first character equal to any of the characters in the C-style + * string pointed to by s. The length of the string is determined by the first + * null character + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_of(const CharT *s, + size_type pos) const +{ + return find_first_of(s, pos, traits_type::length(s)); +} + +/** + * Finds the first character equal to ch + * + * @param[in] ch character to search for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_of(CharT ch, size_type pos) const + noexcept +{ + return find(ch, pos); +} + +/** + * Finds the first character equal to none of the characters in str. + * + * @param[in] str string identifying characters to search for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that does not match. + * If no such characters are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_not_of(basic_string_view str, + size_type pos) const + noexcept +{ + return find_first_not_of(str.data(), pos, str.size()); +} + +/** + * Finds the first character equal to none of the characters + * in the range [s, s+count). This range can include null characters. + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * @param[in] count length of the C-style string identifying characters to + * search for + * + * @return The position of the first character that does not match. + * If no such characters are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_not_of(const CharT *s, + size_type pos, + size_type count) const +{ + if (pos >= size()) + return npos; + + for (auto it = cbegin() + pos; it != cend(); ++it) + if (!traits_type::find(s, count, *it)) + return static_cast( + std::distance(cbegin(), it)); + return npos; +} + +/** + * Finds the first character equal to none of the characters in the C-style + * string pointed to by s. The length of the string is determined by the first + * null character + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that does not match. + * If no such characters are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_not_of(const CharT *s, + size_type pos) const +{ + return find_first_not_of(s, pos, traits_type::length(s)); +} + +/** + * Finds the first character not equal to ch + * + * @param[in] ch character to search for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that does not match. + * If no such characters are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_first_not_of(CharT ch, + size_type pos) const + noexcept +{ + return find_first_not_of(&ch, pos, 1); +} + +/** + * Finds the last character equal to any of the characters in str. + * + * @param[in] str string identifying characters to search for + * @param[in] pos position at which to start the search + * + * @return The position of the last character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_of(basic_string_view str, + size_type pos) const noexcept +{ + return find_last_of(str.data(), pos, str.size()); +} + +/** + * Finds the last character equal to any of the characters + * in the range [s, s+count). This range can include null characters. + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * @param[in] count length of the C-style string identifying characters to + * search for + * + * @return The position of the last character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_of(const CharT *s, size_type pos, + size_type count) const +{ + if (size() == 0 || count == 0) + return npos; + + bool found = false; + size_type last_of = 0; + for (const CharT *c = s; c != s + count; ++c) { + size_type position = rfind(*c, pos); + if (position != npos) { + found = true; + if (position > last_of) + last_of = position; + } + } + if (!found) + return npos; + return last_of; +} + +/** + * Finds the last character equal to any of the characters in the C-style string + * pointed to by s. The length of the string is determined by the + * first null character + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * + * @return The position of the last character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_of(const CharT *s, + size_type pos) const +{ + return find_last_of(s, pos, traits_type::length(s)); +} + +/** + * Finds the last character equal to ch + * + * @param[in] ch character to search for + * @param[in] pos position at which to start the search + * + * @return The position of the last character that matches. + * If no matches are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_of(CharT ch, size_type pos) const + noexcept +{ + return rfind(ch, pos); +} + +/** + * Finds the last character equal to none of the characters in str. + * + * @param[in] str string identifying characters to search for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that does not match. + * If no such characters are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_not_of(basic_string_view str, + size_type pos) const noexcept +{ + return find_last_not_of(str.data(), pos, str.size()); +} + +/** + * Finds the last character equal to none of the characters + * in the range [s, s+count). This range can include null characters. + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * @param[in] count length of the C-style string identifying characters to + * search for + * + * @return The position of the first character that does not match. + * If no such characters are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_not_of(const CharT *s, + size_type pos, + size_type count) const +{ + if (size() > 0) { + pos = (std::min)(pos, size() - 1); + do { + if (!traits_type::find(s, count, *(data() + pos))) + return pos; + + } while (pos-- > 0); + } + return npos; +} + +/** + * Finds the last character equal to none of the characters in the C-style + * string pointed to by s. The length of the string is determined by the first + * null character + * + * @param[in] s pointer to the C-style string identifying characters to search + * for + * @param[in] pos position at which to start the search + * + * @return Position of the first character not equal to any of the characters + * in the given string, or npos if no such character is found. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_not_of(const CharT *s, + size_type pos) const +{ + return find_last_not_of(s, pos, traits_type::length(s)); +} + +/** + * Finds the last character not equal to ch + * + * @param[in] ch character to search for + * @param[in] pos position at which to start the search + * + * @return The position of the first character that does not match. + * If no such characters are found, the function returns npos. + */ +template +typename basic_string_view::size_type +basic_string_view::find_last_not_of(CharT ch, + size_type pos) const noexcept +{ + return find_last_not_of(&ch, pos, 1); +} + +/** + * Returns a view of the substring [pos, pos + rcount), where rcount is the + * smaller of count and size() - pos. + * + * @param[in] pos position of the first character + * @param[in] count requested length + * + * @return view of the substring [pos, pos + rcount). + * + * @throw std::out_of_range if pos > size() + */ +template +constexpr basic_string_view +basic_string_view::substr(size_type pos, size_type count) const +{ + return pos > size() + ? throw std::out_of_range("string_view::substr") + : basic_string_view(data() + pos, + (std::min)(count, size() - pos)); +} + +/** + * Copies the substring [pos, pos + rcount) to the character array pointed to by + * dest, where rcount is the smaller of count and size() - pos. + * + * @param[in] dest pointer to the destination character string + * @param[in] pos position of the first character + * @param[in] count requested substring length + * + * @return number of characters copied. + * + * @throw std::out_of_range if pos > size() + */ +template +typename basic_string_view::size_type +basic_string_view::copy(CharT *dest, size_type count, + size_type pos) const +{ + if (pos > size()) + throw std::out_of_range("string_view::copy"); + size_type rlen = (std::min)(count, size() - pos); + Traits::copy(dest, data() + pos, rlen); + return rlen; +} + +/** + * Compares two character sequences. + * + * @param[in] pos1 position of the first character in this view to compare + * @param[in] n1 number of characters of this view to compare + * @param[in] sv view to compare + * + * @return negative value if this view is less than the other character + * sequence, zero if the both character sequences are equal, positive value if + * this view is greater than the other character sequence. + */ +template +inline int +basic_string_view::compare(size_type pos1, size_type n1, + basic_string_view sv) const +{ + return substr(pos1, n1).compare(sv); +} + +/** + * Compares two character sequences. + * + * @param[in] pos1 position of the first character in this view to compare + * @param[in] n1 number of characters of this view to compare + * @param[in] pos2 position of the first character of the given view to compare + * @param[in] n2 number of characters of the given view to compare + * @param[in] sv view to compare + * + * @return negative value if this view is less than the other character + * sequence, zero if the both character sequences are equal, positive value if + * this view is greater than the other character sequence. + */ +template +inline int +basic_string_view::compare(size_type pos1, size_type n1, + basic_string_view sv, size_type pos2, + size_type n2) const +{ + return substr(pos1, n1).compare(sv.substr(pos2, n2)); +} + +/** + * Compares two character sequences. + * + * @param[in] s pointer to the character string to compare to + * + * @return negative value if this view is less than the other character + * sequence, zero if the both character sequences are equal, positive value if + * this view is greater than the other character sequence. + */ +template +inline int +basic_string_view::compare(const CharT *s) const noexcept +{ + return compare(basic_string_view(s)); +} + +/** + * Compares two character sequences. + * + * @param[in] pos1 position of the first character in this view to compare + * @param[in] n1 number of characters of this view to compare + * @param[in] s pointer to the character string to compare to + * + * @return negative value if this view is less than the other character + * sequence, zero if the both character sequences are equal, positive value if + * this view is greater than the other character sequence. + */ +template +inline int +basic_string_view::compare(size_type pos1, size_type n1, + const CharT *s) const +{ + return substr(pos1, n1).compare(basic_string_view(s)); +} + +/** + * Compares two character sequences. + * + * @param[in] pos1 position of the first character in this view to compare + * @param[in] n1 number of characters of this view to compare + * @param[in] s pointer to the character string to compare to + * @param[in] n2 number of characters of the given string to compare + * + * @return negative value if this view is less than the other character + * sequence, zero if the both character sequences are equal, positive value if + * this view is greater than the other character sequence. + */ +template +inline int +basic_string_view::compare(size_type pos1, size_type n1, + const CharT *s, size_type n2) const +{ + return substr(pos1, n1).compare(basic_string_view(s, n2)); +} + +/** + * Compares this string_view with other. Works in the same way as + * std::basic_string::compare. + * + * @return 0 if both character sequences compare equal, + * positive value if this is lexicographically greater than other, + * negative value if this is lexicographically less than other. + */ +template +inline int +basic_string_view::compare(const basic_string_view &other) const + noexcept +{ + int ret = Traits::compare(data(), other.data(), + (std::min)(size(), other.size())); + if (ret != 0) + return ret; + if (size() < other.size()) + return -1; + if (size() > other.size()) + return 1; + return 0; +} + +/** + * Non-member equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator==(basic_string_view lhs, + basic_string_view rhs) +{ + return (lhs.size() != rhs.size() ? false : lhs.compare(rhs) == 0); +} + +/** + * Non-member equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator==( + basic_string_view lhs, + typename std::common_type>::type rhs) +{ + return (lhs.size() != rhs.size() ? false : lhs.compare(rhs) == 0); +} + +/** + * Non-member equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator==( + typename std::common_type>::type lhs, + basic_string_view rhs) +{ + return (lhs.size() != rhs.size() ? false : lhs.compare(rhs) == 0); +} + +/** + * Non-member not equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator!=(basic_string_view lhs, + basic_string_view rhs) +{ + return (lhs.size() != rhs.size() ? true : lhs.compare(rhs) != 0); +} + +/** + * Non-member not equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator!=( + typename std::common_type>::type lhs, + basic_string_view rhs) +{ + return (lhs.size() != rhs.size() ? true : lhs.compare(rhs) != 0); +} + +/** + * Non-member not equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator!=( + basic_string_view lhs, + typename std::common_type>::type rhs) +{ + return (lhs.size() != rhs.size() ? true : lhs.compare(rhs) != 0); +} + +/** + * Non-member less than operator. + * @relates basic_string_view + */ +template +constexpr bool +operator<(basic_string_view lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) < 0; +} + +/** + * Non-member less than operator. + * @relates basic_string_view + */ +template +constexpr bool +operator<(typename std::common_type>::type lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) < 0; +} + +/** + * Non-member less than operator. + * @relates basic_string_view + */ +template +constexpr bool +operator<(basic_string_view lhs, + typename std::common_type>::type rhs) +{ + return lhs.compare(rhs) < 0; +} + +/** + * Non-member less or equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator<=(basic_string_view lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) <= 0; +} + +/** + * Non-member less or equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator<=( + basic_string_view lhs, + typename std::common_type>::type rhs) +{ + return lhs.compare(rhs) <= 0; +} + +/** + * Non-member less or equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator<=( + typename std::common_type>::type lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) <= 0; +} + +/** + * Non-member greater than operator. + * @relates basic_string_view + */ +template +constexpr bool +operator>(basic_string_view lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) > 0; +} + +/** + * Non-member greater than operator. + * @relates basic_string_view + */ +template +constexpr bool +operator>(typename std::common_type>::type lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) > 0; +} + +/** + * Non-member greater than operator. + * @relates basic_string_view + */ +template +constexpr bool +operator>(basic_string_view lhs, + typename std::common_type>::type rhs) +{ + return lhs.compare(rhs) > 0; +} + +/** + * Non-member greater or equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator>=(basic_string_view lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) >= 0; +} + +/** + * Non-member greater or equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator>=( + typename std::common_type>::type lhs, + basic_string_view rhs) +{ + return lhs.compare(rhs) >= 0; +} + +/** + * Non-member greater or equal operator. + * @relates basic_string_view + */ +template +constexpr bool +operator>=( + basic_string_view lhs, + typename std::common_type>::type rhs) +{ + return lhs.compare(rhs) >= 0; +} +#endif + +} /* namespace obj */ +} /* namespace pmem */ + +#endif /* LIBPMEMOBJ_CPP_STRING_VIEW */ diff --git a/volatile/extern/numactl b/volatile/extern/numactl new file mode 160000 index 00000000..10c277c2 --- /dev/null +++ b/volatile/extern/numactl @@ -0,0 +1 @@ +Subproject commit 10c277c20768be9a563f75265bcd7e73954763ad diff --git a/volatile/extern/numactl-cmake/CMakeLists.txt b/volatile/extern/numactl-cmake/CMakeLists.txt new file mode 100644 index 00000000..69b4373a --- /dev/null +++ b/volatile/extern/numactl-cmake/CMakeLists.txt @@ -0,0 +1,38 @@ +if (NOT CMAKE_SYSTEM_NAME MATCHES "Linux") + message (FATAL_ERROR "Not supported for non-linux environment") +endif() + +message (STATUS "Configure to build libnuma") + +set (LIBNUMA_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/numactl") + +set (SRCS + "${LIBNUMA_SOURCE_DIR}/libnuma.c" + "${LIBNUMA_SOURCE_DIR}/syscall.c" + "${LIBNUMA_SOURCE_DIR}/distance.c" + "${LIBNUMA_SOURCE_DIR}/affinity.c" + "${LIBNUMA_SOURCE_DIR}/sysfs.c" + "${LIBNUMA_SOURCE_DIR}/rtnetlink.c" +) + +add_library(_numa ${SRCS}) + +target_include_directories(_numa PRIVATE include) +target_include_directories(_numa SYSTEM PUBLIC "${LIBNUMA_SOURCE_DIR}") + +set_target_properties(_numa PROPERTIES COMPILE_FLAGS "-w -fPIC") + +# disable SYMVER, it make shared libraries failed to link +add_custom_target( + _disable_symver + COMMAND sed -i "'25s/.*/#define SYMVER(a,b)/'" ${LIBNUMA_SOURCE_DIR}/util.h +) + +add_dependencies(_numa _disable_symver) + +add_custom_command( + TARGET _numa POST_BUILD + COMMAND sed -i "'25s/.*/#define SYMVER(a,b) __asm__ (\".symver \" a \",\" b);/'" ${LIBNUMA_SOURCE_DIR}/util.h +) + +add_library(kvdk_extern_lib::numa ALIAS _numa) diff --git a/volatile/extern/numactl-cmake/include/config.h b/volatile/extern/numactl-cmake/include/config.h new file mode 100644 index 00000000..1ea6cd79 --- /dev/null +++ b/volatile/extern/numactl-cmake/include/config.h @@ -0,0 +1,68 @@ +/* config.h. Generated from config.h.in by configure. */ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Checking for symver attribute */ +/* #undef HAVE_ATTRIBUTE_SYMVER */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Name of package */ +#define PACKAGE "numactl" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "numactl" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "numactl 2.0.14" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "numactl" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "2.0.14" + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* If the compiler supports a TLS storage class define it to that here */ +#define TLS __thread + +/* Version number of package */ +#define VERSION "2.0.14" diff --git a/volatile/extern/xxhash.h b/volatile/extern/xxhash.h new file mode 100644 index 00000000..f5b1bcaa --- /dev/null +++ b/volatile/extern/xxhash.h @@ -0,0 +1,5448 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher tests set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, such + * as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ +# ifdef XXH_NAMESPACE +# error "XXH_INLINE_ALL with XXH_NAMESPACE is not supported" + /* + * Note: Alternative: #undef all symbols (it's a pretty large list). + * Without #error: it compiles, but functions are actually not inlined. + */ +# endif +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, but they must + * still be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and is a more dispersed action. + * Meanwhile, renaming can be achieved in a single block + */ +# define XXH_IPREF(Id) XXH_INLINE_ ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 0 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is only useful when xxHash is compiled as a shared library, as it is + * independent of the version defined in the header. + * + * @return `XXH_VERSION_NUMBER` as of when the function was compiled. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Definitions +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is considered rather weak by today's standards. + * The @ref xxh3_family provides competitive speed for both 32-bit and 64-bit + * systems, and offers true 64/128 bit hash results. It provides a superior + * level of dispersion, and greatly reduces the risks of collisions. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incrememtal input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. It provides a superior level of + * dispersion, and greatly reduces the risks of collisions. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of produced hash values depends on secret's entropy. + * Technically, the secret must look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever unsure about the "randomness" of the blob of bytes, + * consider relabelling it as a "custom seed" instead, + * and employ "XXH3_generateSecret()" (see below) + * to generate a high entropy secret derived from the custom seed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v1; /*!< First accumulator lane */ + XXH32_hash_t v2; /*!< Second accumulator lane */ + XXH32_hash_t v3; /*!< Third accumulator lane */ + XXH32_hash_t v4; /*!< Fourth accumulator lane */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v1; /*!< First accumulator lane */ + XXH64_hash_t v2; /*!< Second accumulator lane */ + XXH64_hash_t v3; /*!< Third accumulator lane */ + XXH64_hash_t v4; /*!< Fourth accumulator lane */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH64_state_t */ + +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11+ */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * @note **This structure has a strict alignment requirement of 64 bytes.** Do + * not allocate this with `malloc()` or `new`, it will not be sufficiently + * aligned. Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack + * allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do not access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t reserved32; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length XXH3_SECRET_DEFAULT_SIZE + * into an already allocated buffer secretBuffer. + * The generated secret is _always_ XXH_SECRET_DEFAULT_SIZE bytes long. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be very long for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * this function can be used to generate a secret of proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even stupidly "low entropy" source such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all expected qualities. + * + * Supplying NULL as the customSeed copies the default secret into `secretBuffer`. + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API void XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize); + + +/* simple short-cut to pre-selected XXH3_128bits variant */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe if your compiler supports it, and *generally* as + * fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance (example: GCC + ARMv6). + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://stackoverflow.com/a/32095106/646947 for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 +/*! + * @def XXH_ACCEPT_NULL_INPUT_POINTER + * @brief Whether to add explicit `NULL` checks. + * + * If the input pointer is `NULL` and the length is non-zero, xxHash's default + * behavior is to dereference it, triggering a segfault. + * + * When this macro is enabled, xxHash actively checks the input for a null pointer. + * If it is, the result for null input pointers is the same as a zero-length input. + */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * adresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH_REROLL + * @brief Whether to reroll `XXH32_finalize` and `XXH64_finalize`. + * + * For performance, `XXH32_finalize` and `XXH64_finalize` use an unrolled loop + * in the form of a switch statement. + * + * This is not always desirable, as it generates larger code, and depending on + * the architecture, may even be slower + * + * This is automatically defined with `-Os`/`-Oz` on GCC and Clang. + */ +# define XXH_REROLL 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if !defined(__clang__) && defined(__GNUC__) && defined(__ARM_FEATURE_UNALIGNED) && defined(__ARM_ARCH) && (__ARM_ARCH == 6) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif !defined(__clang__) && ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + (defined(__GNUC__) && (defined(__ARM_ARCH) && __ARM_ARCH >= 7))) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH_REROLL +# if defined(__OPTIMIZE_SIZE__) +# define XXH_REROLL 1 +# else +# define XXH_REROLL 0 +# endif +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined(__GNUC__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#define XXH_STATIC_ASSERT(c) do { enum { XXH_sa = 1/(int)(!!(c)) }; } while (0) + + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://stackoverflow.com/a/32095106/646947 + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianess *** */ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, a runtime check (which is usually constant folded) + * is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ +static const xxh_u32 XXH_PRIME32_1 = 0x9E3779B1U; /*!< 0b10011110001101110111100110110001 */ +static const xxh_u32 XXH_PRIME32_2 = 0x85EBCA77U; /*!< 0b10000101111010111100101001110111 */ +static const xxh_u32 XXH_PRIME32_3 = 0xC2B2AE3DU; /*!< 0b11000010101100101010111000111101 */ +static const xxh_u32 XXH_PRIME32_4 = 0x27D4EB2FU; /*!< 0b00100111110101001110101100101111 */ +static const xxh_u32 XXH_PRIME32_5 = 0x165667B1U; /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if defined(__GNUC__) && defined(__SSE4_1__) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * This inline assembly hack forces acc into a normal register. This is the + * only thing that prevents GCC and Clang from autovectorizing the XXH32 + * loop (pragmas and attributes don't work for some resason) without globally + * disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * How this hack works: + * __asm__("" // Declare an assembly block but don't declare any instructions + * : // However, as an Input/Output Operand, + * "+r" // constrain a read/write operand (+) as a general purpose register (r). + * (acc) // and set acc as the operand + * ); + * + * Because of the 'r', the compiler has promised that seed will be in a + * general purpose register and the '+' says that it will be 'read/write', + * so it has to assume it has changed. It is like volatile without all the + * loads and stores. + * + * Since the argument has to be in a normal register (not an SSE register), + * each time XXH32_round is called, it is impossible to vectorize. + */ + __asm__("" : "+r" (acc)); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + /* Compact rerolled version */ + if (XXH_REROLL) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + /* fallthrough */ + case 8: XXH_PROCESS4; + /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + /* fallthrough */ + case 9: XXH_PROCESS4; + /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + /* fallthrough */ + case 10: XXH_PROCESS4; + /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + /* fallthrough */ + case 11: XXH_PROCESS4; + /* fallthrough */ + case 7: XXH_PROCESS4; + /* fallthrough */ + case 3: XXH_PROCESS1; + /* fallthrough */ + case 2: XXH_PROCESS1; + /* fallthrough */ + case 1: XXH_PROCESS1; + /* fallthrough */ + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input, len, seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + const xxh_u8* bEnd = input + len; + xxh_u32 h32; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (input==NULL) { + len=0; + bEnd=input=(const xxh_u8*)(size_t)16; + } +#endif + + if (len>=16) { + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + state.v2 = seed + XXH_PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - XXH_PRIME32_1; + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + xxh_u32 v1 = state->v1; + xxh_u32 v2 = state->v2; + xxh_u32 v3 = state->v3; + xxh_u32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + + XXH_rotl32(state->v2, 7) + + XXH_rotl32(state->v3, 12) + + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +/*! + * XXH_REROLL_XXH64: + * Whether to reroll the XXH64_finalize() loop. + * + * Just like XXH32, we can unroll the XXH64_finalize() loop. This can be a + * performance gain on 64-bit hosts, as only one jump is required. + * + * However, on 32-bit hosts, because arithmetic needs to be done with two 32-bit + * registers, and 64-bit arithmetic needs to be simulated, it isn't beneficial + * to unroll. The code becomes ridiculously large (the largest function in the + * binary on i386!), and rerolling it saves anywhere from 3kB to 20kB. It is + * also slightly faster because it fits into cache better and is more likely + * to be inlined by the compiler. + * + * If XXH_REROLL is defined, this is ignored and the loop is always rerolled. + */ +#ifndef XXH_REROLL_XXH64 +# if (defined(__ILP32__) || defined(_ILP32)) /* ILP32 is often defined on 32-bit GCC family */ \ + || !(defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) /* x86-64 */ \ + || defined(_M_ARM64) || defined(__aarch64__) || defined(__arm64__) /* aarch64 */ \ + || defined(__PPC64__) || defined(__PPC64LE__) || defined(__ppc64__) || defined(__powerpc64__) /* ppc64 */ \ + || defined(__mips64__) || defined(__mips64)) /* mips64 */ \ + || (!defined(SIZE_MAX) || SIZE_MAX < ULLONG_MAX) /* check limits */ +# define XXH_REROLL_XXH64 1 +# else +# define XXH_REROLL_XXH64 0 +# endif +#endif /* !defined(XXH_REROLL_XXH64) */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://stackoverflow.com/a/32095106/646947 + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +static const xxh_u64 XXH_PRIME64_1 = 0x9E3779B185EBCA87ULL; /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +static const xxh_u64 XXH_PRIME64_2 = 0xC2B2AE3D27D4EB4FULL; /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +static const xxh_u64 XXH_PRIME64_3 = 0x165667B19E3779F9ULL; /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +static const xxh_u64 XXH_PRIME64_4 = 0x85EBCA77C2B2AE63ULL; /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +static const xxh_u64 XXH_PRIME64_5 = 0x27D4EB2F165667C5ULL; /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1_64 do { \ + h64 ^= (*ptr++) * XXH_PRIME64_5; \ + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; \ +} while (0) + +#define XXH_PROCESS4_64 do { \ + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; \ + ptr += 4; \ + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; \ +} while (0) + +#define XXH_PROCESS8_64 do { \ + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); \ + ptr += 8; \ + h64 ^= k1; \ + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; \ +} while (0) + + /* Rerolled version for 32-bit targets is faster and much smaller. */ + if (XXH_REROLL || XXH_REROLL_XXH64) { + len &= 31; + while (len >= 8) { + XXH_PROCESS8_64; + len -= 8; + } + if (len >= 4) { + XXH_PROCESS4_64; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1_64; + --len; + } + return XXH64_avalanche(h64); + } else { + switch(len & 31) { + case 24: XXH_PROCESS8_64; + /* fallthrough */ + case 16: XXH_PROCESS8_64; + /* fallthrough */ + case 8: XXH_PROCESS8_64; + return XXH64_avalanche(h64); + + case 28: XXH_PROCESS8_64; + /* fallthrough */ + case 20: XXH_PROCESS8_64; + /* fallthrough */ + case 12: XXH_PROCESS8_64; + /* fallthrough */ + case 4: XXH_PROCESS4_64; + return XXH64_avalanche(h64); + + case 25: XXH_PROCESS8_64; + /* fallthrough */ + case 17: XXH_PROCESS8_64; + /* fallthrough */ + case 9: XXH_PROCESS8_64; + XXH_PROCESS1_64; + return XXH64_avalanche(h64); + + case 29: XXH_PROCESS8_64; + /* fallthrough */ + case 21: XXH_PROCESS8_64; + /* fallthrough */ + case 13: XXH_PROCESS8_64; + /* fallthrough */ + case 5: XXH_PROCESS4_64; + XXH_PROCESS1_64; + return XXH64_avalanche(h64); + + case 26: XXH_PROCESS8_64; + /* fallthrough */ + case 18: XXH_PROCESS8_64; + /* fallthrough */ + case 10: XXH_PROCESS8_64; + XXH_PROCESS1_64; + XXH_PROCESS1_64; + return XXH64_avalanche(h64); + + case 30: XXH_PROCESS8_64; + /* fallthrough */ + case 22: XXH_PROCESS8_64; + /* fallthrough */ + case 14: XXH_PROCESS8_64; + /* fallthrough */ + case 6: XXH_PROCESS4_64; + XXH_PROCESS1_64; + XXH_PROCESS1_64; + return XXH64_avalanche(h64); + + case 27: XXH_PROCESS8_64; + /* fallthrough */ + case 19: XXH_PROCESS8_64; + /* fallthrough */ + case 11: XXH_PROCESS8_64; + XXH_PROCESS1_64; + XXH_PROCESS1_64; + XXH_PROCESS1_64; + return XXH64_avalanche(h64); + + case 31: XXH_PROCESS8_64; + /* fallthrough */ + case 23: XXH_PROCESS8_64; + /* fallthrough */ + case 15: XXH_PROCESS8_64; + /* fallthrough */ + case 7: XXH_PROCESS4_64; + /* fallthrough */ + case 3: XXH_PROCESS1_64; + /* fallthrough */ + case 2: XXH_PROCESS1_64; + /* fallthrough */ + case 1: XXH_PROCESS1_64; + /* fallthrough */ + case 0: return XXH64_avalanche(h64); + } + } + /* impossible to reach */ + XXH_ASSERT(0); + return 0; /* unreachable, but some compilers complain without it */ +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + const xxh_u8* bEnd = input + len; + xxh_u64 h64; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (input==NULL) { + len=0; + bEnd=input=(const xxh_u8*)(size_t)32; + } +#endif + + if (len>=32) { + const xxh_u8* const limit = bEnd - 32; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (xxh_u64) len; + + return XXH64_finalize(h64, input, len, align); +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ + +/*! @ingroup xxh64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH64_state_t state; /* use a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + state.v2 = seed + XXH_PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - XXH_PRIME64_1; + /* do not write into reserved64, might be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64)); + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH64_state_t* state, const void* input, size_t len) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + xxh_u64 v1 = state->v1; + xxh_u64 v2 = state->v2; + xxh_u64 v3 = state->v3; + xxh_u64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + xxh_u64 const v1 = state->v1; + xxh_u64 const v2 = state->v2; + xxh_u64 const v3 = state->v3; + xxh_u64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + + + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) +# if defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# elif defined(__ARM_NEON__) || defined(__ARM_NEON) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# endif +#elif defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minumum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment reqired for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif defined(__GNUC__) /* msvc support maybe later */ \ + && (defined(__ARM_NEON__) || defined(__ARM_NEON)) \ + && (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) +# define XXH_VECTOR XXH_NEON +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && defined(__GNUC__) \ + && !defined(__aarch64__) && !defined(__arm64__) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# include +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs, rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if defined(__GNUC__) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif defined(_M_X64) || defined(_M_IA64) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs, rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len < 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(8 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + __asm__ ("" : "+r" (seed64)); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ALIGN(64) __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { XXH_ALIGN(64) __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, -(xxh_i64)seed64); + + XXH_ALIGN(64) const __m512i* const src = (const __m512i*) XXH3_kSecret; + XXH_ALIGN(64) __m512i* const dest = ( __m512i*) customSecret; + int i; + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards ‘const’ qualifier". */ + union { + XXH_ALIGN(64) const __m512i* cp; + XXH_ALIGN(64) void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { XXH_ALIGN(32) __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { XXH_ALIGN(32) __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64, -(xxh_i64)seed64, (xxh_i64)seed64); + + XXH_ALIGN(64) const __m256i* const src = (const __m256i*) XXH3_kSecret; + XXH_ALIGN(64) __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + * The asm hack causes Clang to assume that XXH3_kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + __asm__("" : "+r" (dest)); +# endif + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { XXH_ALIGN(16) __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { XXH_ALIGN(16) __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + // MSVC 32bit mode does not support _mm_set_epi64x before 2015 + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, -(xxh_i64)seed64 }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64); +# endif + int i; + + XXH_ALIGN(64) const float* const src = (float const*) XXH3_kSecret; + XXH_ALIGN(XXH_SEC_ALIGN) __m128i* dest = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + __asm__("" : "+r" (dest)); +# endif + + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm_add_epi64(_mm_castps_si128(_mm_load_ps(src+i*4)), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + XXH_ALIGN(16) uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(uint64x2_t); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + xxh_u64x2* const xacc = (xxh_u64x2*) acc; /* presumed aligned */ + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + xacc[i] += product; + + /* swap high and low halves */ +#ifdef __s390x__ + xacc[i] += vec_permi(data_vec, data_vec, 2); +#else + xacc[i] += vec_xxpermdi(data_vec, data_vec, 2); +#endif + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const data_val = XXH_readLE64(xinput + 8*i); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8); + xacc[i ^ 1] += data_val; /* swap adjacent lanes */ + xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i); + xxh_u64 acc64 = xacc[i]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[i] = acc64; + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), long MOVK chains stall the + * integer pipelines: + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + __asm__("" : "+r" (kSecretPtr)); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + __asm__("" : "+r" (result64)); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + * Since the function is not inlined, the compiler may not be able to understand that, + * in some scenarios, its `secret` argument is actually a compile time constant. + * This variant enforces that the compiler can detect that, + * and uses this opportunity to streamline the generated code for better performance. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_64bits_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_64bits_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_64bits_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_64bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* state, + const xxh_u8* input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + + state->totalLen += len; + + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(state->acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* Consume input by a multiple of internal buffer size */ + if (input+XXH3_INTERNALBUFFER_SIZE < bEnd) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(state->acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + XXH_ASSERT(input < bEnd); + + /* Some remaining input (always) : buffer it */ + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->seed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize) +{ + XXH_ASSERT(secretBuffer != NULL); + if (customSeedSize == 0) { + memcpy(secretBuffer, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return; + } + XXH_ASSERT(customSeed != NULL); + + { size_t const segmentSize = sizeof(XXH128_hash_t); + size_t const nbSegments = XXH_SECRET_DEFAULT_SIZE / segmentSize; + XXH128_canonical_t scrambler; + XXH64_hash_t seeds[12]; + size_t segnb; + XXH_ASSERT(nbSegments == 12); + XXH_ASSERT(segmentSize * nbSegments == XXH_SECRET_DEFAULT_SIZE); /* exact multiple */ + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + + /* + * Copy customSeed to seeds[], truncating or repeating as necessary. + */ + { size_t toFill = XXH_MIN(customSeedSize, sizeof(seeds)); + size_t filled = toFill; + memcpy(seeds, customSeed, toFill); + while (filled < sizeof(seeds)) { + toFill = XXH_MIN(filled, sizeof(seeds) - filled); + memcpy((char*)seeds + filled, seeds, toFill); + filled += toFill; + } } + + /* generate secret */ + memcpy(secretBuffer, &scrambler, sizeof(scrambler)); + for (segnb=1; segnb < nbSegments; segnb++) { + size_t const segmentStart = segnb * segmentSize; + XXH128_canonical_t segment; + XXH128_canonicalFromHash(&segment, + XXH128(&scrambler, sizeof(scrambler), XXH_readLE64(seeds + segnb) + segnb) ); + memcpy((char*)secretBuffer + segmentStart, &segment, sizeof(segment)); + } } +} + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All the functions are actually the same as for 64-bit streaming variant. + * The only difference is the finalizatiom routine. + */ + +static void +XXH3_128bits_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + XXH3_64bits_reset_internal(statePtr, seed, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_128bits_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_128bits_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_128bits_reset(statePtr); + if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_128bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + memcpy(dst, &hash.high64, sizeof(hash.high64)); + memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif + +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXH_IMPLEMENTATION */ + + +#if defined (__cplusplus) +} +#endif diff --git a/volatile/include/kvdk/volatile/comparator.hpp b/volatile/include/kvdk/volatile/comparator.hpp new file mode 100644 index 00000000..8eb7594a --- /dev/null +++ b/volatile/include/kvdk/volatile/comparator.hpp @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include +#include + +#include "types.hpp" + +namespace KVDK_NAMESPACE { +using StringView = pmem::obj::string_view; +using Comparator = + std::function; + +class ComparatorTable { + public: + // Register a string compare function to the table + // + // Return true on success, return false if comparator_name already existed + bool RegisterComparator(const StringView& comparator_name, + Comparator comp_func) { + std::string name(comparator_name.data(), comparator_name.size()); + if (comparator_table_.find(name) == comparator_table_.end()) { + comparator_table_.emplace(name, comp_func); + return true; + } else { + return false; + } + } + + // Return a registered comparator "comparator_name" on success, return nullptr + // if it's not existing + Comparator GetComparator(const StringView& comparator_name) { + std::string name(comparator_name.data(), comparator_name.size()); + auto iter = comparator_table_.find(name); + if (iter != comparator_table_.end()) { + return iter->second; + } + return nullptr; + }; + + private: + std::unordered_map comparator_table_; +}; +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/include/kvdk/volatile/configs.hpp b/volatile/include/kvdk/volatile/configs.hpp new file mode 100644 index 00000000..9b881105 --- /dev/null +++ b/volatile/include/kvdk/volatile/configs.hpp @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include + +#include "comparator.hpp" +#include "types.hpp" + +namespace KVDK_NAMESPACE { + +enum class LogLevel : uint8_t { + All = 0, + Debug, + Info, + Error, + None, +}; + +// Configs of created sorted collection +// For correctness of encoding, please add new config field in the end of the +// existing fields +struct SortedCollectionConfigs { + std::string comparator_name = "default"; + int index_with_hashtable = 1; +}; + +struct Configs { + // TODO: rename to concurrent internal threads + // + // Max number of concurrent threads read/write the kvdk instance internally. + // Set it to the number of the hyper-threads to get best performance + // + // Notice: you can call KVDK API with any number of threads, but if your + // parallel threads more than max_access_threads, the performance will be + // degraded due to synchronization cost + uint64_t max_access_threads = 64; + + // The number of bucket groups in the hash table. + // + // It should be 2^n and should smaller than 2^32. + // The original hash table size would be (kHashBucketSize(128 by default) * + // hash_bucket_num) + // More buckets means less write contention, faster hash + // search, but more internal memory fragmentation. + uint64_t hash_bucket_num = (1 << 27); + + // The number of buckets per hash slot. + // + // The hash slot is the minimum unit of write lock and hot-spot cache, less + // buckets in a slot means better hot spot read performance, less write + // contentions and more memory consumption + uint32_t num_buckets_per_slot = 1; + + // Time interval to do background work in seconds + // + // In KVDK, a background thread will regularly organize free space, + // frequent execution will lead to better space utilization, but more + // influence to foreground performance + double background_work_interval = 5.0; + + // Time interval that the background thread report memory usage by + // GlobalLogger. report_memory_usage_interval less than + // background_work_interval will be ignored. + double report_memory_usage_interval = 1000000.0; + + // Log information to show + LogLevel log_level = LogLevel::Info; + + // Optional optimization strategy for few large skiplists by multi-thread + // recovery a skiplist. The optimization can get better performance when + // having few large skiplists. Default is to close optimization. + bool opt_large_sorted_collection_recovery = false; + + // If a checkpoint is made in last open, recover the instance to the + // checkpoint version if this true + // + // Notice: If opening a backup instance, this will always be set to true + // during recovery. + bool recover_to_checkpoint = false; + + // If customer compare functions is used in a kvdk engine, these functions + // should be registered to the comparator before open engine + ComparatorTable comparator; + + // Background clean thread numbers. + uint64_t clean_threads = 8; + + // Set the memory nodes where volatile KV memory allocator binds to. + // There is no effect when the configured value is empty or invalid. + // Note: Currently, only the jemalloc allocator support memory binding. + // jemalloc is enabled by cmake option -DWITH_JEMALLOC=ON. + std::string dest_memory_nodes = ""; +}; + +struct WriteOptions { + WriteOptions(TTLType _ttl_time = kPersistTTL, bool _update_ttl = true) + : ttl_time(_ttl_time), update_ttl(_update_ttl) {} + + // expired time in milliseconod, should be kPersistTime for no expiration + // data, or a certain interge which be in the range [INT64_MIN , INT64_MAX]. + TTLType ttl_time; + + // determine whether to update expired time if key already existed + bool update_ttl; +}; + +} // namespace KVDK_NAMESPACE diff --git a/volatile/include/kvdk/volatile/engine.h b/volatile/include/kvdk/volatile/engine.h new file mode 100644 index 00000000..fd831a41 --- /dev/null +++ b/volatile/include/kvdk/volatile/engine.h @@ -0,0 +1,308 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +#include "types.h" + +typedef struct KVDKEngine KVDKEngine; +typedef struct KVDKConfigs KVDKConfigs; +typedef struct KVDKWriteOptions KVDKWriteOptions; +typedef struct KVDKWriteBatch KVDKWriteBatch; +typedef struct KVDKSortedIterator KVDKSortedIterator; +typedef struct KVDKListIterator KVDKListIterator; +typedef struct KVDKHashIterator KVDKHashIterator; +typedef struct KVDKSnapshot KVDKSnapshot; +typedef struct KVDKSortedCollectionConfigs KVDKSortedCollectionConfigs; +typedef struct KVDKRegex KVDKRegex; + +extern KVDKRegex* KVDKRegexCreate(char const* data, size_t len); +extern void KVDKRegexDestroy(KVDKRegex* re); + +extern KVDKConfigs* KVDKCreateConfigs(void); +extern void KVDKSetConfigs(KVDKConfigs* kv_config, uint64_t max_access_threads, + uint64_t hash_bucket_num, + uint32_t num_buckets_per_slot); +extern void KVDKConfigRegisterCompFunc( + KVDKConfigs* kv_config, const char* compara_name, size_t compara_len, + int (*compare)(const char* src, size_t src_len, const char* target, + size_t target_len)); +extern void KVDKDestroyConfigs(KVDKConfigs* kv_config); + +extern KVDKWriteOptions* KVDKCreateWriteOptions(void); +extern void KVDKDestroyWriteOptions(KVDKWriteOptions*); +extern void KVDKWriteOptionsSetTTLTime(KVDKWriteOptions*, int64_t); +extern void KVDKWriteOptionsSetUpdateTTL(KVDKWriteOptions* kv_options, + int update_ttl); + +extern KVDKSortedCollectionConfigs* KVDKCreateSortedCollectionConfigs(); +extern void KVDKSetSortedCollectionConfigs(KVDKSortedCollectionConfigs* configs, + const char* comp_func_name, + size_t comp_func_len, + int index_with_hashtable); +extern void KVDKDestroySortedCollectionConfigs( + KVDKSortedCollectionConfigs* configs); + +extern KVDKStatus KVDKOpen(const char* name, const KVDKConfigs* config, + FILE* log_file, KVDKEngine** engine); +extern KVDKStatus KVDKBackup(KVDKEngine* engine, const char* backup_path, + size_t backup_path_len, KVDKSnapshot* snapshot); +extern KVDKStatus KVDKRestore(const char* name, const char* backup_log, + const KVDKConfigs* config, FILE* log_file, + KVDKEngine** engine); +extern void KVDKCloseEngine(KVDKEngine* engine); +extern KVDKSnapshot* KVDKGetSnapshot(KVDKEngine* engine, int make_checkpoint); +extern void KVDKReleaseSnapshot(KVDKEngine* engine, KVDKSnapshot* snapshot); + +extern int KVDKRegisterCompFunc(KVDKEngine* engine, const char* compara_name, + size_t compara_len, + int (*compare)(const char* src, size_t src_len, + const char* target, + size_t target_len)); + +// For BatchWrite +extern KVDKWriteBatch* KVDKWriteBatchCreate(KVDKEngine* engine); +extern void KVDKWriteBatchDestory(KVDKWriteBatch* batch); +extern void KVDKWRiteBatchClear(KVDKWriteBatch* batch); +extern void KVDKWriteBatchStringPut(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* val_data, + size_t val_len); +extern void KVDKWriteBatchStringDelete(KVDKWriteBatch* batch, + char const* key_data, size_t key_len); +extern void KVDKWriteBatchSortedPut(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* field_data, + size_t field_len, char const* val_data, + size_t val_len); +extern void KVDKWriteBatchSortedDelete(KVDKWriteBatch* batch, + char const* key_data, size_t key_len, + char const* field_data, + size_t field_len); +extern void KVDKWriteBatchHashPut(KVDKWriteBatch* batch, char const* key_data, + size_t key_len, char const* field_data, + size_t field_len, char const* val_data, + size_t val_len); +extern void KVDKWriteBatchHashDelete(KVDKWriteBatch* batch, + char const* key_data, size_t key_len, + char const* field_data, size_t field_len); +extern KVDKStatus KVDKBatchWrite(KVDKEngine* engine, + KVDKWriteBatch const* batch); + +// For Anonymous Global Collection +extern KVDKStatus KVDKGet(KVDKEngine* engine, const char* key, size_t key_len, + size_t* val_len, char** val); +extern KVDKStatus KVDKPut(KVDKEngine* engine, const char* key, size_t key_len, + const char* val, size_t val_len, + const KVDKWriteOptions* write_option); +extern KVDKStatus KVDKDelete(KVDKEngine* engine, const char* key, + size_t key_len); + +// Modify value of existing key in the engine +// +// * modify_func: customized function to modify existing value of key. See +// definition of KVDKModifyFunc (types.h) for more details. +// * modify_args: customized arguments of modify_func. +// * free_func: function to free allocated space for new value in +// modify_func, pall NULL if not need to free +// +// Return Ok if modify success. +// Return Abort if modify function abort modifying. +// Return other non-Ok status on any error. +extern KVDKStatus KVDKModify(KVDKEngine* engine, const char* key, + size_t key_len, KVDKModifyFunc modify_func, + void* modify_args, KVDKFreeFunc free_func, + const KVDKWriteOptions* write_option); + +/// Sorted +/// ////////////////////////////////////////////////////////////////////// +extern KVDKStatus KVDKSortedCreate(KVDKEngine* engine, + const char* collection_name, + size_t collection_len, + KVDKSortedCollectionConfigs* configs); +extern KVDKStatus KVDKSortedDestroy(KVDKEngine* engine, + const char* collection_name, + size_t collection_len); +extern KVDKStatus KVDKSortedSize(KVDKEngine* engine, const char* collection, + size_t collection_len, size_t* size); +extern KVDKStatus KVDKSortedPut(KVDKEngine* engine, const char* collection, + size_t collection_len, const char* key, + size_t key_len, const char* val, + size_t val_len); +extern KVDKStatus KVDKSortedDelete(KVDKEngine* engine, const char* collection, + size_t collection_len, const char* key, + size_t key_len); +extern KVDKStatus KVDKSortedGet(KVDKEngine* engine, const char* collection, + size_t collection_len, const char* key, + size_t key_len, size_t* val_len, char** val); +extern KVDKSortedIterator* KVDKSortedIteratorCreate(KVDKEngine* engine, + const char* collection, + size_t collection_len, + KVDKSnapshot* snapshot, + KVDKStatus* s); +extern void KVDKSortedIteratorDestroy(KVDKEngine* engine, + KVDKSortedIterator* iterator); +extern void KVDKSortedIteratorSeekToFirst(KVDKSortedIterator* iter); +extern void KVDKSortedIteratorSeekToLast(KVDKSortedIterator* iter); +extern void KVDKSortedIteratorSeek(KVDKSortedIterator* iter, const char* str, + size_t str_len); +extern void KVDKSortedIteratorNext(KVDKSortedIterator* iter); +extern void KVDKSortedIteratorPrev(KVDKSortedIterator* iter); +extern unsigned char KVDKSortedIteratorValid(KVDKSortedIterator* iter); +extern void KVDKSortedIteratorKey(KVDKSortedIterator* iter, char** key, + size_t* key_len); +extern void KVDKSortedIteratorValue(KVDKSortedIterator* iter, char** value, + size_t* val_len); + +/// Hash ////////////////////////////////////////////////////////////////////// +extern KVDKStatus KVDKHashCreate(KVDKEngine* engine, char const* key_data, + size_t key_len); +extern KVDKStatus KVDKHashDestroy(KVDKEngine* engine, char const* key_data, + size_t key_len); +extern KVDKStatus KVDKHashLength(KVDKEngine* engine, char const* key_data, + size_t key_len, size_t* len); +extern KVDKStatus KVDKHashGet(KVDKEngine* engine, const char* key_data, + size_t key_len, const char* field_data, + size_t field_len, char** val_data, + size_t* val_len); +extern KVDKStatus KVDKHashPut(KVDKEngine* engine, const char* key_data, + size_t key_len, const char* field_data, + size_t field_len, const char* val_data, + size_t val_len); +extern KVDKStatus KVDKHashDelete(KVDKEngine* engine, const char* key_data, + size_t key_len, const char* field_data, + size_t field_len); +extern KVDKStatus KVDKHashModify(KVDKEngine* engine, const char* key_data, + size_t key_len, const char* field_data, + size_t field_len, KVDKModifyFunc modify_func, + void* args, KVDKFreeFunc free_func); + +/// HashIterator ////////////////////////////////////////////////////////////// +extern KVDKHashIterator* KVDKHashIteratorCreate(KVDKEngine* engine, + char const* key_data, + size_t key_len, + KVDKSnapshot* snapshot, + KVDKStatus* s); +extern void KVDKHashIteratorDestroy(KVDKEngine* engine, KVDKHashIterator* iter); +extern void KVDKHashIteratorPrev(KVDKHashIterator* iter); +extern void KVDKHashIteratorNext(KVDKHashIterator* iter); +extern void KVDKHashIteratorSeekToFirst(KVDKHashIterator* iter); +extern void KVDKHashIteratorSeekToLast(KVDKHashIterator* iter); +extern int KVDKHashIteratorIsValid(KVDKHashIterator* iter); +extern void KVDKHashIteratorGetKey(KVDKHashIterator* iter, char** field_data, + size_t* field_len); +extern void KVDKHashIteratorGetValue(KVDKHashIterator* iter, char** value_data, + size_t* value_len); +extern int KVDKHashIteratorMatchKey(KVDKHashIterator* iter, + KVDKRegex const* re); + +/// List ////////////////////////////////////////////////////////////////////// +extern KVDKStatus KVDKListCreate(KVDKEngine* engine, char const* key_data, + size_t key_len); +extern KVDKStatus KVDKListDestroy(KVDKEngine* engine, char const* key_data, + size_t key_len); +extern KVDKStatus KVDKListSize(KVDKEngine* engine, char const* key_data, + size_t key_len, size_t* len); +extern KVDKStatus KVDKListPushFront(KVDKEngine* engine, char const* key_data, + size_t key_len, char const* elem_data, + size_t elem_len); +extern KVDKStatus KVDKListPushBack(KVDKEngine* engine, char const* key_data, + size_t key_len, char const* elem_data, + size_t elem_len); +extern KVDKStatus KVDKListPopFront(KVDKEngine* engine, char const* key_data, + size_t key_len, char** elem_data, + size_t* elem_len); +extern KVDKStatus KVDKListPopBack(KVDKEngine* engine, char const* key_data, + size_t key_len, char** elem_data, + size_t* elem_len); +extern KVDKStatus KVDKListBatchPushFront(KVDKEngine* engine, + char const* key_data, size_t key_len, + char const* const* elems_data, + size_t const* elems_len, + size_t elems_cnt); +extern KVDKStatus KVDKListBatchPushBack(KVDKEngine* engine, + char const* key_data, size_t key_len, + char const* const* elems_data, + size_t const* elems_len, + size_t elems_cnt); +extern KVDKStatus KVDKListBatchPopFront( + KVDKEngine* engine, char const* key_data, size_t key_len, size_t n, + void (*cb)(char const* elem_data, size_t elem_len, void* args), void* args); +extern KVDKStatus KVDKListBatchPopBack( + KVDKEngine* engine, char const* key_data, size_t key_len, size_t n, + void (*cb)(char const* elem_data, size_t elem_len, void* args), void* args); +extern KVDKStatus KVDKListMove(KVDKEngine* engine, char const* src_data, + size_t src_len, int src_pos, + char const* dst_data, size_t dst_len, + int dst_pos, char** elem_data, size_t* elem_len); +extern KVDKStatus KVDKListInsertAt(KVDKEngine* engine, char const* list_name, + size_t list_name_len, char const* elem_data, + size_t elem_len, long index); +extern KVDKStatus KVDKListInsertBefore(KVDKEngine* engine, + char const* list_name, + size_t list_name_len, + char const* elem_data, size_t elem_len, + char const* pos_elem, + size_t pos_elem_len); +extern KVDKStatus KVDKListInsertAfter(KVDKEngine* engine, char const* list_name, + size_t list_name_len, + char const* elem_data, size_t elem_len, + char const* pos_elem, + size_t pos_elem_len); +extern KVDKStatus KVDKListErase(KVDKEngine* engine, char const* list_name, + size_t list_len, long index, char** elem_data, + size_t* elem_len); +extern KVDKStatus KVDKListReplace(KVDKEngine* engine, char const* list_name, + size_t list_name_len, long index, + char const* elem, size_t elem_len); +/// ListIterator ////////////////////////////////////////////////////////////// +extern KVDKListIterator* KVDKListIteratorCreate(KVDKEngine* engine, + char const* key_data, + size_t key_len, KVDKStatus* s); +extern void KVDKListIteratorDestroy(KVDKEngine* engine, KVDKListIterator* iter); +extern void KVDKListIteratorPrev(KVDKListIterator* iter); +extern void KVDKListIteratorNext(KVDKListIterator* iter); +extern void KVDKListIteratorPrevElem(KVDKListIterator* iter, + char const* elem_data, size_t elem_len); +extern void KVDKListIteratorNextElem(KVDKListIterator* iter, + char const* elem_data, size_t elem_len); +extern void KVDKListIteratorSeekToFirst(KVDKListIterator* iter); +extern void KVDKListIteratorSeekToLast(KVDKListIterator* iter); +extern void KVDKListIteratorSeekToFirstElem(KVDKListIterator* iter, + char const* elem_data, + size_t elem_len); +extern void KVDKListIteratorSeekToLastElem(KVDKListIterator* iter, + char const* elem_data, + size_t elem_len); +extern void KVDKListIteratorSeekPos(KVDKListIterator* iter, long pos); +extern int KVDKListIteratorIsValid(KVDKListIterator* iter); +extern void KVDKListIteratorGetValue(KVDKListIterator* iter, char** elem_data, + size_t* elem_len); + +/* ttl_time is negetive or positive number, If ttl_time == INT64_MAX, + * the key is persistent; If ttl_time <=0, the key is expired immediately. + */ +extern KVDKStatus KVDKExpire(KVDKEngine* engine, const char* str, + size_t str_len, int64_t ttl_time); +/* ttl_time is INT64_MAX and return Status::Ok if the key is persist + * ttl_time is 0 and return Status::NotFound if the key is expired or doesnot + * exist. + * ttl_time is certain positive number and return Status::Ok if the key hasn't + * expired and exist. + */ +extern KVDKStatus KVDKGetTTL(KVDKEngine* engine, const char* str, + size_t str_len, int64_t* ttl_time); + +extern KVDKStatus KVDKTypeOf(KVDKEngine* engine, char const* key_data, + size_t key_len, KVDKValueType* type); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif // extern "C" diff --git a/volatile/include/kvdk/volatile/engine.hpp b/volatile/include/kvdk/volatile/engine.hpp new file mode 100644 index 00000000..3fb01b08 --- /dev/null +++ b/volatile/include/kvdk/volatile/engine.hpp @@ -0,0 +1,514 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include +#include + +#include "comparator.hpp" +#include "configs.hpp" +#include "iterator.hpp" +#include "snapshot.hpp" +#include "types.hpp" +#include "write_batch.hpp" + +namespace kvdk = KVDK_NAMESPACE; + +namespace KVDK_NAMESPACE { +// This is the abstraction of a persistent KVDK instance +class Engine { + public: + // Open a new KVDK instance or restore a existing KVDK instance + // + // Args: + // * engine_path: indicates the dir path that persist the instance + // * engine_ptr: store the pointer to restored instance + // * configs: engine configs4 + // * log_file: file to print out runtime logs + // + // Return: + // Return Status::Ok on sucess, return other status for any error + // + // To close the instance, just delete *engine_ptr. + static Status Open(const StringView engine_path, Engine** engine_ptr, + const Configs& configs, FILE* log_file = stdout); + + // Restore a KVDK instance from a backup log file. + // + // Args: + // * engine_path: indicates the dir path that persist the instance + // * backup_log: the backup log file restored from + // * engine_ptr: store the pointer to restored instance + // * configs: engine configs + // * log_file: file to print out runtime logs + // + // Return: + // Return Status::Ok on sucess, return other status for any error + // + // Notice: + // "engine_path" should be an empty dir + static Status Restore(const StringView engine_path, + const StringView backup_log, Engine** engine_ptr, + const Configs& configs, FILE* log_file = stdout); + + // Get type of key, it can be String, SortedCollection, HashCollection or List + // + // Return: + // Status::Ok and store type to "*type" on success + // Status::NotFound if key does not exist + virtual Status TypeOf(StringView key, ValueType* type) = 0; + + // Insert a STRING-type KV to set "key" to hold "value". + // + // Args: + // *options: customized write options + // + // Return: + // Status Ok on success . + // Status::WrongType if key exists but is a collection. + // Status::MemoryOverflow if DRAM exhausted. + virtual Status Put(const StringView key, const StringView value, + const WriteOptions& options = WriteOptions()) = 0; + + // Search the STRING-type KV of "key" in the kvdk instance. + // + // Return: + // Return Status::Ok and store the corresponding value to *value on success. + // Return Status::NotFound if the "key" does not exist. + virtual Status Get(const StringView key, std::string* value) = 0; + + // Remove STRING-type KV of "key". + // + // Return: + // Status::Ok on success or the "key" did not exist + // Status::WrongType if key exists but is a collection type + // Status::MemoryOverflow if DRAM exhausted. + virtual Status Delete(const StringView key) = 0; + + // Modify value of existing key in the engine + // + // Args: + // * modify_func: customized function to modify existing value of key. See + // definition of ModifyFunc (types.hpp) for more details. + // * modify_args: customized arguments of modify_func. + // + // Return: + // Return Status::Ok if modify success. + // Return Status::Abort if modify function abort modifying. + // Return other non-Ok status on any error. + virtual Status Modify(const StringView key, ModifyFunc modify_func, + void* modify_args, + const WriteOptions& options = WriteOptions()) = 0; + + // Atomically do a batch of operations (Put or Delete) to the instance, these + // operations either all succeed, or all fail. The data will be rollbacked if + // the instance crash during a batch write + // + // Return: + // Status::Ok on success + // Status::NotFound if a collection operated by the batch does not exist + // Status::MemoryOverflow if DRAM exhausted. + // + // Notice: + // BatchWrite has no isolation guaranteed, if you need it, you should use + // Transaction API + virtual Status BatchWrite(std::unique_ptr const& batch) = 0; + + // Create a write batch for BatchWrite operation + virtual std::unique_ptr WriteBatchCreate() = 0; + + // time to *expired_time on success. + // + // Args: + // * key: STRING-type key or collection name to search. + // * ttl_time: store TTL result. + // + // Return: + // Status::Ok and store ttl time to *ttl_time on success + // Status::NotFound if key is expired or does not exist + virtual Status GetTTL(const StringView key, int64_t* ttl_time) = 0; + + // Set ttl_time for STRING-type or Collection type data + // + // Args: + // * key: STRING-type key or collection name to set ttl_time. + // * ttl_time: ttl time to set. if ttl_time == kPersistTTL, the name will not + // be expired. If ttl_time <=0, the name is expired immediately. + // + // Return: + // Status::Ok on success. + // Status::NotFound if key does not exist. + virtual Status Expire(const StringView key, int64_t ttl_time) = 0; + + // Create a empty sorted collection with configs. You should always create + // collection before you do any operations on it + // + // Args: + // * configs: customized config of creating collection + // + // Return: + // Status::Ok on success + // Status::Existed if sorted collection already existed + // Status::WrongType if collection existed but not a sorted collection + // Status::MemoryOverflow if DRAM exhausted. + virtual Status SortedCreate( + const StringView collection, + const SortedCollectionConfigs& configs = SortedCollectionConfigs()) = 0; + + // Destroy a sorted collection + // Return: + // Status::Ok on success + // Status::WrongType if collection existed but not a sorted collection + // Status::MemoryOverflow if runtime/KV memory exhausted + virtual Status SortedDestroy(const StringView collection) = 0; + + // Get number of elements in a sorted collection + // + // Return: + // Status::Ok on success + // Status::NotFound if collection not exist + virtual Status SortedSize(const StringView collection, size_t* size) = 0; + + // Insert a KV to set "key" in sorted collection "collection" + // to hold "value" + // Return: + // Status::Ok on success. + // Status::NotFound if collection not exist. + // Status::WrongType if collection exists but is not a sorted collection. + // Status::MemoryOverflow if DRAM exhausted. + virtual Status SortedPut(const StringView collection, const StringView key, + const StringView value) = 0; + + // Search the KV of "key" in sorted collection "collection" + // + // Return: + // Status::Ok and store the corresponding value to *value on success. + // Status::NotFound If the "collection" or "key" does not exist. + virtual Status SortedGet(const StringView collection, const StringView key, + std::string* value) = 0; + + // Remove KV of "key" in the sorted collection "collection". + // + // Return: + // Status::Ok on success or key not existed in collection + // Status::NotFound if collection not exist + // Status::WrongType if collection exists but is not a sorted collection. + // Status::MemoryOverflow if DRAM exhausted. + virtual Status SortedDelete(const StringView collection, + const StringView key) = 0; + + // Create a KV iterator on sorted collection "collection", which is able to + // sequentially iterate all KVs in the "collection". + // + // Args: + // * snapshot: iterator will iterate all elems a t "snapshot" version, if + // snapshot is nullptr, then a internal snapshot will be created at current + // version and the iterator will be created on it + // * status: store operation status if not null + // + // Return: + // Return A pointer to iterator on success. + // Return nullptr if collection not exist or any other errors, and store error + // status to "status" + // + // Notice: + // 1. Iterator will be invalid after the passed snapshot is released + // 2. Please release the iterator as soon as it is not needed, as the holding + // snapshot will forbid newer data being freed + virtual SortedIterator* SortedIteratorCreate(const StringView collection, + Snapshot* snapshot = nullptr, + Status* s = nullptr) = 0; + + // Release a sorted iterator and its holding resouces + virtual void SortedIteratorRelease(SortedIterator*) = 0; + + /// List APIs /////////////////////////////////////////////////////////////// + + // Create an empty List. + // Return: + // Status::WrongType if list name existed but is not a List. + // Status::Existed if a List named list already exists. + // Status::MemoryOverflow if DRAM exhausted. + // Status::Ok if successfully created the List. + virtual Status ListCreate(StringView list) = 0; + + // Destroy a List associated with key + // Return: + // Status::WrongType if list name is not a List. + // Status::Ok if successfully destroyed the List or the List not existed + // Status::MemoryOverflow if DRAM exhausted + virtual Status ListDestroy(StringView list) = 0; + + // Total elements in List. + // Return: + // Status::InvalidDataSize if list name is too long + // Status::WrongType if list name is not a List. + // Status::NotFound if list does not exist or has expired. + // Status::Ok and length of List if List exists. + virtual Status ListSize(StringView list, size_t* sz) = 0; + + // Push element as first element of List + // Return: + // Status::InvalidDataSize if list name or elem is too long + // Status::WrongType if list name is not a List. + // Status::MemoryOverflow if DRAM exhausted. + // Status::Ok if operation succeeded. + virtual Status ListPushFront(StringView list, StringView elem) = 0; + + // Push element as last element of List + // Return: + // Status::InvalidDataSize if list name or elem is too long + // Status::WrongType if list name in the instance is not a List. + // Status::MemoryOverflow if DRAM exhausted. + // Status::Ok if operation succeeded. + virtual Status ListPushBack(StringView list, StringView elem) = 0; + + // Pop first element of list + // Return: + // Status::InvalidDataSize if list name is too long + // Status::WrongType if list is not a List. + // Status::NotFound if list does not exist or has expired. + // Status::Ok and element if operation succeeded. + virtual Status ListPopFront(StringView list, std::string* elem) = 0; + + // Pop last element of List + // Return: + // Status::InvalidDataSize if list name is too long + // Status::WrongType if list is not a List. + // Status::NotFound if list does not exist or has expired. + // Status::Ok and element if operation succeeded. + virtual Status ListPopBack(StringView list, std::string* elem) = 0; + + // Push multiple elements to the front of List + // Return: + // Status::InvalidDataSize if list name or elem is too long + // Status::WrongType if list is not a List. + // Status::MemoryOverflow if DRAM exhausted. + // Status::Ok if operation succeeded. + virtual Status ListBatchPushFront(StringView list, + std::vector const& elems) = 0; + virtual Status ListBatchPushFront(StringView list, + std::vector const& elems) = 0; + + // Push multiple elements to the back of List + // Return: + // Status::InvalidDataSize if list or elem is too long + // Status::WrongType if list name is not a List. + // Status::MemoryOverflow if DRAM exhausted. + // Status::Ok if operation succeeded. + virtual Status ListBatchPushBack(StringView list, + std::vector const& elems) = 0; + virtual Status ListBatchPushBack(StringView list, + std::vector const& elems) = 0; + + // Pop first N element of List + // Return: + // Status::InvalidDataSize if list is too long + // Status::WrongType if list is not a List. + // Status::NotFound if list does not exist or has expired. + // Status::Ok and element if operation succeeded. + virtual Status ListBatchPopFront(StringView list, size_t n, + std::vector* elems) = 0; + + // Pop last N element of List + // Return: + // Status::InvalidDataSize if list is too long + // Status::WrongType if list is not a List. + // Status::NotFound if list does not exist or has expired. + // Status::Ok and element if operation succeeded. + virtual Status ListBatchPopBack(StringView list, size_t n, + std::vector* elems) = 0; + + // Move element in src List at src_pos to dst List at dst_pos + // src_pos and dst_pos can only be 0, indicating List front, + // or -1, indicating List back. + // Return: + // Status::WrongType if src or dst is not a List. + // Status::NotFound if list or element not exist + // Status::Ok and moved element if operation succeeded. + virtual Status ListMove(StringView src_list, ListPos src_pos, + StringView dst_list, ListPos dst_pos, + std::string* elem) = 0; + + // Insert a element to a list at index, the index can be positive or + // negative + // Return: + // Status::InvalidDataSize if list name is too large + // Status::WrongType if collection is not a list + // Status::NotFound if collection not found or index is beyond list size + // Status::Ok if operation succeeded + virtual Status ListInsertAt(StringView list, StringView elem, long index) = 0; + + // Insert an element before element "pos" in list "collection" + // Return: + // Status::InvalidDataSize if elem is too long. + // Status::MemoryOverflow if DRAM exhausted. + // Status::NotFound if List of the ListIterator has expired or been + // deleted, or "pos" not exist in the list + // Status::Ok if operation succeeded. + virtual Status ListInsertBefore(StringView list, StringView elem, + StringView pos) = 0; + + // Insert an element after element "pos" in list "collection" + // Return: + // Status::InvalidDataSize if elem is too long. + // Status::MemoryOverflow if DRAM exhausted. + // Status::NotFound if List of the ListIterator has expired or been + // deleted, or "pos" not exist in the list + // Status::Ok if operation succeeded. + virtual Status ListInsertAfter(StringView list, StringView elem, + StringView pos) = 0; + + // Remove the element at index + // Return: + // Status::NotFound if the index beyond list size. + // Status::Ok if operation succeeded, and store removed elem in "elem" + virtual Status ListErase(StringView list, long index, std::string* elem) = 0; + + // Replace the element at index + // Return: + // Status::InvalidDataSize if elem is too long + // Status::NotFound if if the index beyond list size. + // Status::Ok if operation succeeded. + virtual Status ListReplace(StringView list, long index, StringView elem) = 0; + + // Create a KV iterator on list "list", which is able to iterate all elems in + // the list + // + // Args: + // * snapshot: iterator will iterate all elems a t "snapshot" version, if + // snapshot is nullptr, then a internal snapshot will be created at current + // version and the iterator will be created on it + // * status: store operation status if not null + // + // Return: + // Return A pointer to iterator on success. + // Return nullptr if list not exist or any other errors, and store error + // status to "status" + // + // Notice: + // 1. Iterator will be invalid after the passed snapshot is released + // 2. Please release the iterator as soon as it is not needed, as the holding + // snapshot will forbid newer data being freed + virtual ListIterator* ListIteratorCreate(StringView list, + Snapshot* snapshot = nullptr, + Status* status = nullptr) = 0; + // Release a ListIterator and its holding resources + virtual void ListIteratorRelease(ListIterator*) = 0; + + /// Hash APIs /////////////////////////////////////////////////////////////// + + // Create a empty hash collection. You should always create collection before + // you do any operations on it + // + // Return: + // Status::Ok on success + // Status::Existed if hash collection already existed + // Status::WrongType if collection existed but not a hash collection + // Status::MemoryOverflow if DRAM exhausted + virtual Status HashCreate(StringView collection) = 0; + + // Destroy a hash collection + // Return: + // Status::Ok on success + // Status::WrongType if collection existed but not a hash collection + // Status::MemoryOverflow if DRAM exhausted + virtual Status HashDestroy(StringView collection) = 0; + + // Get number of elements in a hash collection + // + // Return: + // Status::Ok on success + // Status::NotFound if collection not exist + virtual Status HashSize(StringView collection, size_t* len) = 0; + + // Search the KV of "key" in hash collection "collection" + // + // Return: + // Status::Ok and store the corresponding value to *value on success. + // Status::NotFound If the "collection" or "key" does not exist. + virtual Status HashGet(StringView collection, StringView key, + std::string* value) = 0; + + // Insert a KV to set "key" in hash collection "collection" + // to hold "value" + // Return: + // Status::Ok on success. + // Status::NotFound if collection not exist. + // Status::WrongType if collection exists but is not a hash collection. + // Status::MemoryOverflow if DRAM exhausted + virtual Status HashPut(StringView collection, StringView key, + StringView value) = 0; + + // Remove KV of "key" in the hash collection "collection". + // + // Return: + // Status::Ok on success or key not existed in collection + // Status::NotFound if collection not exist + // Status::WrongType if collection exists but is not a hash collection. + // Status::MemoryOverflow if DRAM exhausted. + virtual Status HashDelete(StringView collection, StringView key) = 0; + + // Modify value of a existing key in a hash collection + // + // Args: + // * modify_func: customized function to modify existing value of key. See + // definition of ModifyFunc (types.hpp) for more details. + // * modify_args: customized arguments of modify_func. + // + // Return: + // Return Status::Ok if modify success. + // Return Status::Abort if modify function abort modifying. + // Return other non-Ok status on any error. + virtual Status HashModify(StringView collection, StringView key, + ModifyFunc modify_func, void* cb_args) = 0; + + // Create a KV iterator on hash collection "collection", which is able to + // iterate all elems in the collection at "snapshot" version, if snapshot is + // nullptr, then a internal snapshot will be created at current version and + // the iterator will be created on it + // + // Notice: + // 1. Iterator will be invalid after the passed snapshot is released + // 2. Please release the iterator as soon as it is not needed, as the holding + // snapshot will forbid newer data being freed + virtual HashIterator* HashIteratorCreate(StringView collection, + Snapshot* snapshot = nullptr, + Status* s = nullptr) = 0; + virtual void HashIteratorRelease(HashIterator*) = 0; + + /// Other /////////////////////////////////////////////////////////////////// + + // Get a snapshot of the instance at this moment. + // If set make_checkpoint to true, a persistent checkpoint will be made until + // this snapshot is released. You can recover KVDK instance to the checkpoint + // version during recovery, then the checkpoint will be removed. + // + // Notice: + // 1. You can maintain multiple snapshot but only the last checkpoint. + // 2. Please release the snapshot as soon as it is not needed, as it will + // forbid newer data being freed + virtual Snapshot* GetSnapshot(bool make_checkpoint) = 0; + + // Make a backup on "snapshot" to "backup_log" + virtual Status Backup(const pmem::obj::string_view backup_log, + const Snapshot* snapshot) = 0; + + // Release a snapshot of the instance + virtual void ReleaseSnapshot(const Snapshot*) = 0; + + // Register a customized comparator to the engine on runtime + // + // Return: + // Return true on success + // Return false if a comparator of comparator_name already existed + virtual bool registerComparator(const StringView& comparator_name, + Comparator) = 0; + + // Close the instance on exit. + virtual ~Engine() = 0; +}; + +} // namespace KVDK_NAMESPACE diff --git a/volatile/include/kvdk/volatile/iterator.hpp b/volatile/include/kvdk/volatile/iterator.hpp new file mode 100644 index 00000000..10e7e981 --- /dev/null +++ b/volatile/include/kvdk/volatile/iterator.hpp @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +#include + +#include "types.hpp" + +namespace KVDK_NAMESPACE { + +class SortedIterator { + public: + virtual void Seek(const std::string& key) = 0; + + virtual void SeekToFirst() = 0; + + virtual void SeekToLast() = 0; + + virtual bool Valid() = 0; + + virtual void Next() = 0; + + virtual void Prev() = 0; + + virtual std::string Key() = 0; + + virtual std::string Value() = 0; + + virtual ~SortedIterator() = default; +}; + +class ListIterator { + public: + virtual void Seek(long index) = 0; + + virtual void SeekToFirst() = 0; + + virtual void SeekToFirst(StringView elem) = 0; + + virtual void SeekToLast() = 0; + + virtual void SeekToLast(StringView elem) = 0; + + virtual void Next() = 0; + + virtual void Next(StringView elem) = 0; + + virtual void Prev() = 0; + + virtual void Prev(StringView elem) = 0; + + virtual bool Valid() const = 0; + + virtual std::string Value() const = 0; + + virtual ~ListIterator() = default; +}; + +class HashIterator { + public: + virtual void SeekToFirst() = 0; + + virtual void SeekToLast() = 0; + + virtual bool Valid() const = 0; + + virtual void Next() = 0; + + virtual void Prev() = 0; + + virtual std::string Key() const = 0; + + virtual std::string Value() const = 0; + + virtual bool MatchKey(std::regex const& re) = 0; + + virtual ~HashIterator() = default; +}; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/include/kvdk/volatile/snapshot.hpp b/volatile/include/kvdk/volatile/snapshot.hpp new file mode 100644 index 00000000..91b29622 --- /dev/null +++ b/volatile/include/kvdk/volatile/snapshot.hpp @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#pragma once + +namespace KVDK_NAMESPACE { + +// A snapshot indicates a immutable view of a KVDK engine at a certain time +struct Snapshot {}; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/include/kvdk/volatile/types.h b/volatile/include/kvdk/volatile/types.h new file mode 100644 index 00000000..ab0e1f9e --- /dev/null +++ b/volatile/include/kvdk/volatile/types.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ +#pragma once + +#include + +#define KVDK_MODIFY_WRITE 0 +#define KVDK_MODIFY_DELETE 1 +#define KVDK_MODIFY_ABORT 2 +#define KVDK_MODIFY_NOOP 3 + +#define KVDK_LIST_FRONT 0 +#define KVDK_LIST_BACK 1 + +// Customized modify function used in KVDKModify, indicate how to modify +// existing value +// +// Below is args of the function, "input" is passed by KVDK engine, and the +// function is responsible to fill outputs in "output" args. +// +// *(input) old_val: existing value of key, or nullptr if key not exist +// *(input) old_val_len: length of "old_val" +// *(output) new_val: store new value after modifying. It's responsible of +// KVDKModifyFunc to allocate space for *new_val and fill modify result here if +// the function returns KVDK_MODIFY_WRITE. +// *(output) new_val_len: length of "new_val", It's responsible of +// KVDKModifyFunc to fill it +// * args: customer args +// +// return KVDK_MODIFY_WRITE indicates to update existing value to +// "new_value" +// return KVDK_MODIFY_DELETE indicates to delete the kv from engine +// return KVDK_MODIFY_ABORT indicates the existing kv should not be +// modified and abort the operation +typedef int (*KVDKModifyFunc)(const char* old_val, size_t old_val_len, + char** new_val, size_t* new_val_len, void* args); +// Used in KVDKModify, indicate how to free allocated space in KVDKModifyFunc +typedef void (*KVDKFreeFunc)(void*); + +#define GENERATE_ENUM(ENUM) ENUM, +#define GENERATE_STRING(STRING) #STRING, + +#define KVDK_TYPES(GEN) \ + GEN(String) \ + GEN(SortedCollection) \ + GEN(HashCollection) \ + GEN(List) + +typedef enum { KVDK_TYPES(GENERATE_ENUM) } KVDKValueType; + +__attribute__((unused)) static char const* KVDKValueTypeString[] = { + KVDK_TYPES(GENERATE_STRING)}; + +#define KVDK_STATUS(GEN) \ + GEN(Ok) \ + GEN(NotFound) \ + GEN(Outdated) \ + GEN(WrongType) \ + GEN(Existed) \ + GEN(OperationFail) \ + GEN(OutOfRange) \ + GEN(MemoryOverflow) \ + GEN(NotSupported) \ + GEN(InvalidBatchSize) \ + GEN(InvalidDataSize) \ + GEN(InvalidArgument) \ + GEN(IOError) \ + GEN(InvalidConfiguration) \ + GEN(Fail) \ + GEN(Abort) + +typedef enum { KVDK_STATUS(GENERATE_ENUM) } KVDKStatus; + +__attribute__((unused)) static char const* KVDKStatusStrings[] = { + KVDK_STATUS(GENERATE_STRING)}; diff --git a/volatile/include/kvdk/volatile/types.hpp b/volatile/include/kvdk/volatile/types.hpp new file mode 100644 index 00000000..1d99c922 --- /dev/null +++ b/volatile/include/kvdk/volatile/types.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include +#include +#include + +#include "libpmemobj++/string_view.hpp" +#include "types.h" + +namespace KVDK_NAMESPACE { +using StringView = pmem::obj::string_view; +using CollectionIDType = std::uint64_t; +using IndexType = std::int64_t; + +using UnixTimeType = std::int64_t; +using ExpireTimeType = UnixTimeType; +using TTLType = std::int64_t; + +using Status = KVDKStatus; +using ValueType = KVDKValueType; + +enum class ListPos : int { + Front = KVDK_LIST_FRONT, + Back = KVDK_LIST_BACK, +}; + +enum class ModifyOperation : int { + Write = KVDK_MODIFY_WRITE, + Delete = KVDK_MODIFY_DELETE, + Abort = KVDK_MODIFY_ABORT, + Noop = KVDK_MODIFY_NOOP +}; + +// Customized modify function used in Engine::Modify, indicate how to modify +// existing value +// +// Below is args of the function, "input" is passed by KVDK engine, and the +// function is responsible to fill outputs in "output" args. +// +// *(input) key: associated key of modify operation +// *(input) old_value: existing value of key, or nullptr if key not exist +// *(output) new_value: store new value after modifying "existing_value". Caller +// is responsible to fill the modify result here if the function returns +// ModifyOperation::Write. +// * args: customer args +// +// return ModifyOperation::Write indicates to update existing value to +// "new_value" +// return ModifyOperation::Delete indicates to delete the kv from engine +// return ModifyOperation::Abort indicates the existing kv should not be +// modified and abort the operation +using ModifyFunc = std::function; + +constexpr ExpireTimeType kPersistTime = INT64_MAX; +constexpr TTLType kPersistTTL = INT64_MAX; +constexpr TTLType kInvalidTTL = 0; +constexpr uint32_t kMinMemoryBlockSize = 32; +} // namespace KVDK_NAMESPACE + +#if !__cpp_lib_string_view +#include + +#include + +template +std::basic_ostream& operator<<( + std::basic_ostream& out, + pmem::obj::basic_string_view sv) { + return std::__ostream_insert(out, sv.data(), sv.size()); +} + +namespace std { +template <> +struct hash { + size_t operator()(pmem::obj::string_view const& sv) const noexcept { + return _Hash_impl::hash(sv.data(), sv.size()); + } +}; +} // namespace std + +#endif // !__cpp_lib_string_view diff --git a/volatile/include/kvdk/volatile/write_batch.hpp b/volatile/include/kvdk/volatile/write_batch.hpp new file mode 100644 index 00000000..b156dff1 --- /dev/null +++ b/volatile/include/kvdk/volatile/write_batch.hpp @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ +#pragma once + +#include +#include + +#include "types.hpp" + +namespace KVDK_NAMESPACE { + +class WriteBatch { + public: + virtual void StringPut(const StringView key, const StringView value) = 0; + virtual void StringDelete(const StringView key) = 0; + + virtual void SortedPut(const StringView collection, const StringView key, + const StringView value) = 0; + virtual void SortedDelete(const StringView collection, + const StringView key) = 0; + + virtual void HashPut(const StringView collection, const StringView key, + const StringView value) = 0; + virtual void HashDelete(const StringView collection, + const StringView key) = 0; + + virtual void Clear() = 0; + + virtual size_t Size() const = 0; + + virtual ~WriteBatch() = default; +}; + +} // namespace KVDK_NAMESPACE \ No newline at end of file diff --git a/volatile/java/CMakeLists.txt b/volatile/java/CMakeLists.txt new file mode 100644 index 00000000..045864dd --- /dev/null +++ b/volatile/java/CMakeLists.txt @@ -0,0 +1,194 @@ +cmake_minimum_required(VERSION 3.4) + +if(${CMAKE_VERSION} VERSION_LESS "3.11.4") + message("Please consider switching to CMake 3.11.4 or newer") +endif() + +set(CMAKE_JAVA_COMPILE_FLAGS -source 7) + +set(KVDKJNI_VERSION 1.0.0-SNAPSHOT) +set(KVDKJNI_JAR_FILE kvdkjni-${KVDKJNI_VERSION}.jar) + +set(JNI_NATIVE_SOURCES + kvdkjni/configs.cc + kvdkjni/engine.cc + kvdkjni/iterator.cc + kvdkjni/native_bytes_handle.cc + kvdkjni/write_batch.cc +) + +set(JAVA_MAIN_SOURCES + src/main/java/io/pmem/kvdk/AbstractNativeReference.java + src/main/java/io/pmem/kvdk/Configs.java + src/main/java/io/pmem/kvdk/Engine.java + src/main/java/io/pmem/kvdk/Iterator.java + src/main/java/io/pmem/kvdk/Status.java + src/main/java/io/pmem/kvdk/KVDKException.java + src/main/java/io/pmem/kvdk/KVDKObject.java + src/main/java/io/pmem/kvdk/NativeBytesHandle.java + src/main/java/io/pmem/kvdk/NativeLibraryLoader.java + src/main/java/io/pmem/kvdk/WriteBatch.java + src/main/java/io/pmem/kvdk/WriteOptions.java +) + +set(JAVA_TEST_SOURCES + src/test/java/io/pmem/kvdk/ConfigsTest.java + src/test/java/io/pmem/kvdk/EngineTest.java + src/test/java/io/pmem/kvdk/EngineTestBase.java + src/test/java/io/pmem/kvdk/IteratorTest.java + src/test/java/io/pmem/kvdk/WriteBatchTest.java +) + +include(FindJava) +include(UseJava) +find_package(JNI) +include(ExternalProject) + +include_directories(${JNI_INCLUDE_DIRS}) +include_directories(${PROJECT_SOURCE_DIR}/java) + +set(MVN_LOCAL ~/.m2/repository) +set(MVN_CENTRAL https://repo1.maven.org/maven2) + +set(JAVA_TEST_LIBDIR ${PROJECT_SOURCE_DIR}/java/test-libs) +set(JAVA_TMP_JAR ${JAVA_TEST_LIBDIR}/tmp.jar) + +set(JAVA_JUNIT_JAR_SOURCE /junit/junit/4.12/junit-4.12.jar) +set(JAVA_JUNIT_JAR ${JAVA_TEST_LIBDIR}/junit-4.12.jar) +set(JAVA_JUNIT_JAR_CHECKSUM_256 59721f0805e223d84b90677887d9ff567dc534d7c502ca903c0c2b17f05c116a) + +set(JAVA_TESTCLASSPATH ${JAVA_JUNIT_JAR}) + +set(JNI_OUTPUT_DIR ${PROJECT_SOURCE_DIR}/java/include) +file(MAKE_DIRECTORY ${JNI_OUTPUT_DIR}) + +if(${Java_VERSION_MINOR} VERSION_LESS_EQUAL "7" AND ${Java_VERSION_MAJOR} STREQUAL "1") + message(FATAL_ERROR "Detected Java 7 or older (${Java_VERSION_STRING}), minimum required version is Java 8") +endif() + +# Download dependencies for Java unit tests +if(NOT EXISTS ${JAVA_TEST_LIBDIR}) + file(MAKE_DIRECTORY ${JAVA_TEST_LIBDIR}) +endif() + +if(NOT EXISTS ${JAVA_JUNIT_JAR}) + if(EXISTS ${MVN_LOCAL}${JAVA_JUNIT_JAR_SOURCE}) + message("Copying from ${MVN_LOCAL}${JAVA_JUNIT_JAR_SOURCE} to ${JAVA_JUNIT_JAR}") + file(COPY ${MVN_LOCAL}${JAVA_JUNIT_JAR_SOURCE} DESTINATION ${JAVA_TEST_LIBDIR}) + else() + message("Downloading ${MVN_CENTRAL}${JAVA_JUNIT_JAR_SOURCE} to ${JAVA_TMP_JAR}") + execute_process(COMMAND curl --fail --insecure --location ${MVN_CENTRAL}${JAVA_JUNIT_JAR_SOURCE} + RESULT_VARIABLE error_code + OUTPUT_VARIABLE error_message + OUTPUT_FILE ${JAVA_TMP_JAR}) + + if(NOT error_code EQUAL 0) + message(FATAL_ERROR "Failed downloading ${MVN_CENTRAL}${JAVA_JUNIT_JAR_SOURCE} to ${JAVA_JUNIT_JAR}: ${error_message}") + endif() + file(RENAME ${JAVA_TMP_JAR} ${JAVA_JUNIT_JAR}) + endif() +endif() + +# Verify checksum of downloaded files +file(SHA256 ${JAVA_JUNIT_JAR} TEMP_FILE_CHECKSUM) +if (NOT ${JAVA_JUNIT_JAR_CHECKSUM_256} STREQUAL ${TEMP_FILE_CHECKSUM}) + message(FATAL_ERROR "Checksum mismatch, file: ${JAVA_JUNIT_JAR}, expected: ${JAVA_JUNIT_JAR_CHECKSUM_256}, is: ${TEMP_FILE_CHECKSUM}") +endif() + +if(${Java_VERSION_MAJOR} VERSION_GREATER_EQUAL "10" AND ${CMAKE_VERSION} VERSION_LESS "3.11.4") + # Java 10 and newer don't have javah, but the alternative GENERATE_NATIVE_HEADERS requires CMake 3.11.4 or newer + message(FATAL_ERROR "Detected Java 10 or newer (${Java_VERSION_STRING}), to build with CMake please upgrade CMake to 3.11.4 or newer") + +elseif(${CMAKE_VERSION} VERSION_LESS "3.11.4") + # Old CMake + message("Using an old CMAKE (${CMAKE_VERSION}) - JNI headers generated in separate step") + add_jar( + kvdkjni_classes + SOURCES + ${JAVA_MAIN_SOURCES} + ${JAVA_TEST_SOURCES} + INCLUDE_JARS ${JAVA_TESTCLASSPATH} + ) + +else () + # Java 1.8 or newer prepare the JAR... + message("Preparing Jar for JDK ${Java_VERSION_STRING}") + add_jar( + kvdkjni_classes + SOURCES + ${JAVA_MAIN_SOURCES} + ${JAVA_TEST_SOURCES} + INCLUDE_JARS ${JAVA_TESTCLASSPATH} + GENERATE_NATIVE_HEADERS kvdkjni_headers DESTINATION ${JNI_OUTPUT_DIR} + ) + +endif() + +if(${CMAKE_VERSION} VERSION_LESS "3.11.4") + # Old CMake ONLY generate JNI headers, otherwise JNI is handled in add_jar step above + message("Preparing JNI headers for old CMake (${CMAKE_VERSION})") + set(NATIVE_JAVA_CLASSES + io.pmem.kvdk.Configs + ) + + create_javah( + TARGET kvdkjni_headers + CLASSES ${NATIVE_JAVA_CLASSES} + CLASSPATH kvdkjni_classes ${JAVA_TESTCLASSPATH} + OUTPUT_DIR ${JNI_OUTPUT_DIR} + ) +endif() + +set(KVDKJNI_SHARED_LIB kvdkjni${ARTIFACT_SUFFIX}) +add_library(${KVDKJNI_SHARED_LIB} SHARED ${JNI_NATIVE_SOURCES}) +add_dependencies(${KVDKJNI_SHARED_LIB} kvdkjni_headers) +target_link_libraries(${KVDKJNI_SHARED_LIB} ${KVDK_STATIC_LIB}) + +# find needed shared libraries. +# libatomic.so.1 +find_library(ATOMIC_SHARED_LIB NAMES "${CMAKE_SHARED_LIBRARY_PREFIX}atomic${CMAKE_SHARED_LIBRARY_SUFFIX}.1") +find_package_handle_standard_args(atomic REQUIRED_VARS ATOMIC_SHARED_LIB) +if(NOT atomic_FOUND) + message(FATAL_ERROR "libatomic.so.1 not found") +endif() + +# libstdc++.so.6 +find_library(STD_CPP_SHARED_LIB NAMES "${CMAKE_SHARED_LIBRARY_PREFIX}stdc++${CMAKE_SHARED_LIBRARY_SUFFIX}.6") +find_package_handle_standard_args(std_cpp REQUIRED_VARS STD_CPP_SHARED_LIB) +if(NOT std_cpp_FOUND) + message(FATAL_ERROR "libstdc++.so.6 not found") +endif() + +# copy shared libraries to java dir for unit tests +SET(JAVA_RESOURCES_DIR ${PROJECT_SOURCE_DIR}/java/src/main/resources/) +if(NOT EXISTS ${JAVA_RESOURCES_DIR}) + file(MAKE_DIRECTORY ${JAVA_RESOURCES_DIR}) +endif() + +set(ATOMIC_SHARED_LIB_FILE ${CMAKE_SHARED_LIBRARY_PREFIX}atomic${CMAKE_SHARED_LIBRARY_SUFFIX}.1) +set(STD_CPP_SHARED_LIB_FILE ${CMAKE_SHARED_LIBRARY_PREFIX}stdc++${CMAKE_SHARED_LIBRARY_SUFFIX}.6) +set(KVDKJNI_SHARED_LIB_FILE ${CMAKE_SHARED_LIBRARY_PREFIX}${KVDKJNI_SHARED_LIB}${CMAKE_SHARED_LIBRARY_SUFFIX}) + +add_custom_command(TARGET ${KVDKJNI_SHARED_LIB} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${ATOMIC_SHARED_LIB} + ${JAVA_RESOURCES_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${STD_CPP_SHARED_LIB} + ${JAVA_RESOURCES_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + $ + ${JAVA_RESOURCES_DIR} + COMMENT "Copying JNI library file to: ${JAVA_RESOURCES_DIR}" +) + +# build jar +add_custom_command(TARGET ${KVDKJNI_SHARED_LIB} POST_BUILD + COMMAND cp kvdkjni_classes.jar ${KVDKJNI_JAR_FILE} + COMMAND cp ${ATOMIC_SHARED_LIB} ${ATOMIC_SHARED_LIB_FILE} + COMMAND jar -uf ${KVDKJNI_JAR_FILE} ${ATOMIC_SHARED_LIB_FILE} + COMMAND cp ${STD_CPP_SHARED_LIB} ${STD_CPP_SHARED_LIB_FILE} + COMMAND jar -uf ${KVDKJNI_JAR_FILE} ${STD_CPP_SHARED_LIB_FILE} + COMMAND jar -uf ${KVDKJNI_JAR_FILE} ${KVDKJNI_SHARED_LIB_FILE} + COMMENT "Building jar file: ${CMAKE_CURRENT_BINARY_DIR}/${KVDKJNI_JAR_FILE}" +) diff --git a/volatile/java/README.md b/volatile/java/README.md new file mode 100644 index 00000000..7fcd9f15 --- /dev/null +++ b/volatile/java/README.md @@ -0,0 +1,89 @@ +## Build + +### System requirements +* JDK >= 1.8 +* Apache Maven >= 3.1.0 + +### Build KVDK JNI library +Add `-DWITH_JNI=ON` to cmake command options. For example: + +```bash +cd kvdk +mkdir -p build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Release -DCHECK_CPP_STYLE=ON -DWITH_JNI=ON && make -j +``` + +### Unit Test with Apache Maven +```bash +cd kvdk/java +mvn clean test +``` + +## Release +### Install to local Maven repository +```bash +cd kvdk/java +mvn clean install +``` + +### Release to Maven Central +1. Sign and deploy jar [to OSSRH](https://central.sonatype.org/publish/publish-manual/) +2. Release deployment [from OSSRH to Maven Central](https://central.sonatype.org/publish/release/) + +## Run examples +```bash +cd kvdk/java/examples +mvn clean package + +export PMEM_IS_PMEM_FORCE=1 +mkdir -p /tmp/kvdk-test-dir + +java -cp target/kvdkjni-examples-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.examples.KVDKExamples +``` + +## Run benchmark +```bash +cd kvdk/java/benchmark +mvn clean package + +mkdir -p /mnt/pmem0/kvdk/bench-dir + +export type=string +# or export type=sorted + +# fill example +numactl --cpunodebind=0 --membind=0 java -cp target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark -fill=true -latency=true -path=/mnt/pmem0/kvdk/bench-dir -space=412316860416 -value_size=120 -num_kv=536870912 -num_operations=536870912 -type=$type -threads=32 + +# random batch insert example +numactl --cpunodebind=0 --membind=0 java -cp target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark -fill=false -latency=true -path=/mnt/pmem0/kvdk/bench-dir -space=412316860416 -value_size=120 -num_kv=536870912 -num_operations=536870912 -type=$type -threads=32 -timeout=30 -read_ratio=0 -existing_keys_ratio=0 -batch_size=100 + +# random insert example +numactl --cpunodebind=0 --membind=0 java -cp target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark -fill=false -latency=true -path=/mnt/pmem0/kvdk/bench-dir -space=412316860416 -value_size=120 -num_kv=536870912 -num_operations=536870912 -type=$type -threads=32 -timeout=30 -read_ratio=0 -existing_keys_ratio=0 + +# range scan example +numactl --cpunodebind=0 --membind=0 java -cp target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark -fill=false -latency=true -path=/mnt/pmem0/kvdk/bench-dir -space=412316860416 -value_size=120 -num_kv=536870912 -num_operations=10737418240 -type=$type -threads=32 -timeout=30 -read_ratio=1 -scan=1 + +# random read example +numactl --cpunodebind=0 --membind=0 java -cp target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark -fill=false -latency=true -path=/mnt/pmem0/kvdk/bench-dir -space=412316860416 -value_size=120 -num_kv=536870912 -num_operations=10737418240 -type=$type -threads=32 -timeout=30 -read_ratio=1 + +# random read write example (9R:1W) +numactl --cpunodebind=0 --membind=0 java -cp target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark -fill=false -latency=true -path=/mnt/pmem0/kvdk/bench-dir -space=412316860416 -value_size=120 -num_kv=536870912 -num_operations=536870912 -type=$type -threads=32 -timeout=30 -read_ratio=0.9 + +# random update example +numactl --cpunodebind=0 --membind=0 java -cp target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark -fill=false -latency=true -path=/mnt/pmem0/kvdk/bench-dir -space=412316860416 -value_size=120 -num_kv=536870912 -num_operations=10737418240 -type=$type -threads=32 -timeout=30 -read_ratio=0 +``` + +## Cross Platform + +The KVDK Java library contains the needed shared libaries (`.so` files), which will be loaded when they are not present in system library paths. + +A Java application relying on KVDK Java library can be run on **64-bit** servers without building the KVDK C++ code or installing its dependencies (e.g. ndctl, pmdk). + +The only requirement is the `libc` version `GLIBC_2.18 or higher` can be found. + +We tested without installing KVDK C++ library or its dependencies on below platforms, in which the `libc` is new enough: +* Fedora >= 20 +* Ubuntu >= 14.04 +* Centos >= 8 + +For older platforms, you may need to upgrade `libc` in system path or where `LD_LIBRARY_PATH` points to. diff --git a/volatile/java/benchmark/pom.xml b/volatile/java/benchmark/pom.xml new file mode 100644 index 00000000..a2b66e69 --- /dev/null +++ b/volatile/java/benchmark/pom.xml @@ -0,0 +1,102 @@ + + + 4.0.0 + + io.pmem + kvdkjni-benchmark + 1.0.0-SNAPSHOT + + KVDK JNI Benchmark + KVDK JNI Benchmark. + https://github.com/pmem/kvdk + 2021 + + + + BSD-3-Clause + hhttps://github.com/pmem/kvdk/blob/main/LICENSE + repo + + + + + scm:git:https://github.com/pmem/kvdk.git + scm:git:https://github.com/pmem/kvdk.git + scm:git:https://github.com/pmem/kvdk.git + + + + Intel + https://www.intel.com + + + + 1.0.0-SNAPSHOT + 1.8 + 1.8 + UTF-8 + 2.7.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${project.build.source} + ${project.build.target} + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + target + -Djava.library.path=${project.build.directory} + false + false + + ${project.build.directory}/* + + + 1 + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + 1.7 + + + + + + + spotless-check + compile + + check + + + + + + + + + + io.pmem + kvdkjni + ${kvdkjni.version} + + + diff --git a/volatile/java/benchmark/scripts/benchmark_all.sh b/volatile/java/benchmark/scripts/benchmark_all.sh new file mode 100755 index 00000000..70de46c6 --- /dev/null +++ b/volatile/java/benchmark/scripts/benchmark_all.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +BASEDIR=$(dirname "$0") + +# Note: The directory $pmem_path will be removed from file system automatically before benchmarking +pmem_path=/mnt/pmem0/kvdk/bench-dir +space=412316860416 +num_kv=536870912 + +# languages +for lang in java native; do + echo "#################################### $lang ####################################" + + bin="java -cp $BASEDIR/../target/kvdkjni-benchmark-1.0.0-SNAPSHOT.jar:$BASEDIR/../../target/kvdkjni-1.0.0-SNAPSHOT.jar io.pmem.kvdk.benchmark.KVDKBenchmark" + if [ "$lang" == "native" ]; then + bin=$BASEDIR/../../../build/bench + fi + + # value_sizes + for value_size in 120 4096; do + echo "#################################### value_size: $value_size ####################################" + if [[ "$value_size" == "4096" ]]; then + # calculation logic for num_kv: + # header_size = type == string ? 24 : 48 + # key_size = 8 + # num_kv = space // 4 // (header_size + key_size + average_value_size) + + num_kv=24778657 + fi + + # benchmarks + for type in string sorted blackhole; do + echo "#################################### $type ####################################" + rm -rf $pmem_path + + num_ops1=$num_kv + num_ops2=10737418240 + + if [ "$type" == "blackhole" ]; then + num_ops1=$num_ops2 + fi + + # fill + echo "#################################### fill" + numactl -N 0 -m 0 $bin -populate=true -latency=true -path=$pmem_path -space=$space -value_size=$value_size -num_kv=$num_kv -num_operations=$num_ops1 -max_access_threads=64 -threads=64 -type=$type -num_collection=16 -fill=true + + # batch insert random + echo "#################################### batch insert random" + numactl -N 0 -m 0 $bin -populate=true -latency=true -path=$pmem_path -space=$space -value_size=$value_size -num_kv=$num_kv -num_operations=$num_ops1 -max_access_threads=64 -threads=64 -type=$type -num_collection=16 -fill=false -timeout=30 -read_ratio=0 -existing_keys_ratio=0 -batch_size=100 + + # insert random + echo "#################################### insert random" + numactl -N 0 -m 0 $bin -populate=true -latency=true -path=$pmem_path -space=$space -value_size=$value_size -num_kv=$num_kv -num_operations=$num_ops1 -max_access_threads=64 -threads=64 -type=$type -num_collection=16 -fill=false -timeout=30 -read_ratio=0 -existing_keys_ratio=0 + + # range scan + echo "#################################### range scan" + numactl -N 0 -m 0 $bin -populate=true -latency=true -path=$pmem_path -space=$space -value_size=$value_size -num_kv=$num_kv -num_operations=$num_ops2 -max_access_threads=64 -threads=64 -type=$type -num_collection=16 -fill=false -timeout=30 -read_ratio=1 -scan=1 + + # read random + echo "#################################### read random" + numactl -N 0 -m 0 $bin -populate=true -latency=true -path=$pmem_path -space=$space -value_size=$value_size -num_kv=$num_kv -num_operations=$num_ops2 -max_access_threads=64 -threads=64 -type=$type -num_collection=16 -fill=false -timeout=30 -read_ratio=1 + + # read write random (9R:1W) + echo "#################################### read write random 9R:1W" + numactl -N 0 -m 0 $bin -populate=true -latency=true -path=$pmem_path -space=$space -value_size=$value_size -num_kv=$num_kv -num_operations=$num_ops1 -max_access_threads=64 -threads=64 -type=$type -num_collection=16 -fill=false -timeout=30 -read_ratio=0.9 + + # update random + echo "#################################### update random" + numactl -N 0 -m 0 $bin -populate=true -latency=true -path=$pmem_path -space=$space -value_size=$value_size -num_kv=$num_kv -num_operations=$num_ops2 -max_access_threads=64 -threads=64 -type=$type -num_collection=16 -fill=false -timeout=30 -read_ratio=0 + done + done +done diff --git a/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/KVDKBenchmark.java b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/KVDKBenchmark.java new file mode 100644 index 00000000..afdee6ad --- /dev/null +++ b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/KVDKBenchmark.java @@ -0,0 +1,1033 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk.benchmark; + +import io.pmem.kvdk.Configs; +import io.pmem.kvdk.Engine; +import io.pmem.kvdk.Iterator; +import io.pmem.kvdk.KVDKException; +import io.pmem.kvdk.NativeBytesHandle; +import io.pmem.kvdk.WriteBatch; +import io.pmem.kvdk.benchmark.util.ConstantLongGenerator; +import io.pmem.kvdk.benchmark.util.LongGenerator; +import io.pmem.kvdk.benchmark.util.RandomLongGenerator; +import io.pmem.kvdk.benchmark.util.RangeLongGenerator; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class KVDKBenchmark { + private static final long MAX_VALUE_SIZE = 102400; + private static final long VALUE_RANDOM_SEED = 42; + private static final int WARMUP_SECONDS = 2; + private static final int MAX_LATENCY = 10000000; // Unit: 100 ns + + String path; + long numKv; + long numOperations; + boolean fill; + long timeout; + long valueSize; + String valueSizeDistributionStr; + int threads; + double readRatio; + double existingKeysRatio; + boolean latency; + String type; + boolean scan; + int numCollection; + long batchSize; + String keyDistributionStr; + boolean populate; + int maxAccessThreads; + long space; + boolean optLargeSortedCollectionRestore; + boolean useDevdaxMode; + long hashBucketNum; + + Map flags; + + List collectionNames; + List closeableObjects; + long maxRandomKey; + int writeThreads; + int readThreads; + ExecutorService executor; + + List readOpsLog; + List writeOpsLog; + List readNotFoundCountLog; + int lastValidLogIndex; + + DataType dataType; + KeyDistribution keyDistribution; + ValueSizeDistribution valueSizeDistribution; + + AtomicLong finishedThreads; + AtomicBoolean isTimeout; + AtomicLong readOps; + AtomicLong writeOps; + AtomicLong readNotFoundCount; + + Engine kvdkEngine; + byte[] valuePool; + long numOperationsPerThread; + NativeBytesHandle[] collectionNameHandles; + LongGenerator[] randomKeyGenerators; + LongGenerator[] rangeKeyGenerators; + LongGenerator[] randomValueSizeGenerators; + LongGenerator[] constantValueSizeGenerators; + long[][] latencies; + + static { + Engine.loadLibrary(); + } + + public KVDKBenchmark(Map flags) { + path = (String) flags.get(Flag.path); + numKv = (Long) flags.get(Flag.num_kv); + numOperations = (Long) flags.get(Flag.num_operations); + fill = (Boolean) flags.get(Flag.fill); + timeout = (Long) flags.get(Flag.timeout); + valueSize = (Long) flags.get(Flag.value_size); + valueSizeDistributionStr = (String) flags.get(Flag.value_size_distribution); + threads = ((Long) flags.get(Flag.threads)).intValue(); + readRatio = (Double) flags.get(Flag.read_ratio); + existingKeysRatio = (Double) flags.get(Flag.existing_keys_ratio); + latency = (Boolean) flags.get(Flag.latency); + type = (String) flags.get(Flag.type); + scan = (Boolean) flags.get(Flag.scan); + numCollection = ((Long) flags.get(Flag.num_collection)).intValue(); + batchSize = (Long) flags.get(Flag.batch_size); + keyDistributionStr = (String) flags.get(Flag.key_distribution); + populate = (Boolean) flags.get(Flag.populate); + maxAccessThreads = ((Long) flags.get(Flag.max_access_threads)).intValue(); + space = (Long) flags.get(Flag.space); + optLargeSortedCollectionRestore = + (Boolean) flags.get(Flag.opt_large_sorted_collection_restore); + useDevdaxMode = (Boolean) flags.get(Flag.use_devdax_mode); + hashBucketNum = (Long) flags.get(Flag.hash_bucket_num); + + this.flags = flags; + closeableObjects = new ArrayList<>(); + finishedThreads = new AtomicLong(); + isTimeout = new AtomicBoolean(false); + readOps = new AtomicLong(); + writeOps = new AtomicLong(); + readNotFoundCount = new AtomicLong(); + readOpsLog = new ArrayList<>(); + writeOpsLog = new ArrayList<>(); + readNotFoundCountLog = new ArrayList<>(); + + printFlags(); + processBenchmarkConfigs(); + generateValuePool(); + } + + private void printFlags() { + System.out.println("flags:"); + for (Flag flag : Flag.values()) { + System.out.format(" -%s=%s%n", flag.name(), flags.get(flag).toString()); + } + } + + private void processBenchmarkConfigs() { + if (type.equals("sorted")) { + dataType = DataType.Sorted; + } else if (type.equals("string")) { + dataType = DataType.String; + } else if (type.equals("blackhole")) { + dataType = DataType.Blackhole; + } else { + throw new IllegalArgumentException("Unsupported -type: " + type); + } + + if (dataType.equals(DataType.Sorted)) { + collectionNames = new ArrayList<>(); + collectionNameHandles = new NativeBytesHandle[numCollection]; + for (int i = 0; i < numCollection; i++) { + collectionNames.add("Collection_" + i); + } + } + + if (scan && dataType.equals(DataType.String)) { + throw new IllegalArgumentException("Unsupported to scan for -type: " + type); + } + + if (valueSize > MAX_VALUE_SIZE) { + throw new IllegalArgumentException( + "-value_size is too large: " + + valueSize + + ", should be less than: " + + MAX_VALUE_SIZE); + } + + maxRandomKey = existingKeysRatio == 0 ? Long.MAX_VALUE : (long) (numKv / existingKeysRatio); + + if (fill) { + assert (readRatio == 0); + System.out.println("Fill mode, -num_operations is ignored, will use -num_kv"); + + if (threads > maxAccessThreads) { + threads = maxAccessThreads; // overwrite + System.out.println( + "Fill mode, -threads is reset to -max_access_threads: " + maxAccessThreads); + } + System.out.println("Fill mode, -threads: " + threads); + System.out.println("Fill mode, -key_distribution is reset to: range"); + + keyDistribution = KeyDistribution.Range; + numOperationsPerThread = numKv / threads + 1; + rangeKeyGenerators = new LongGenerator[threads]; + for (int i = 0; i < threads; i++) { + rangeKeyGenerators[i] = + new RangeLongGenerator( + i * numOperationsPerThread, (i + 1) * numOperationsPerThread, 1); + } + } else { + numOperationsPerThread = numOperations / threads + 1; + + if (keyDistributionStr.equals("random")) { + keyDistribution = KeyDistribution.Uniform; + } else { + throw new IllegalArgumentException( + "Invalid -key_distribution: " + keyDistributionStr); + } + + randomKeyGenerators = new LongGenerator[threads]; + for (int i = 0; i < threads; i++) { + randomKeyGenerators[i] = new RandomLongGenerator(0, maxRandomKey); + } + } + System.out.println("Num of operations per thread: " + numOperationsPerThread); + + if (valueSizeDistributionStr.equals("constant")) { + valueSizeDistribution = ValueSizeDistribution.Constant; + constantValueSizeGenerators = new LongGenerator[threads]; + for (int i = 0; i < threads; i++) { + constantValueSizeGenerators[i] = new ConstantLongGenerator(valueSize); + } + } else if (valueSizeDistributionStr.equals("random")) { + valueSizeDistribution = ValueSizeDistribution.Uniform; + randomValueSizeGenerators = new LongGenerator[threads]; + for (int i = 0; i < threads; i++) { + randomValueSizeGenerators[i] = new RandomLongGenerator(1, valueSize + 1); + } + } else { + throw new IllegalArgumentException( + "Invalid -value_size_distribution: " + valueSizeDistributionStr); + } + + writeThreads = fill ? threads : (int) (threads - (readRatio * 100 * threads / 100)); + readThreads = threads - writeThreads; + + if (latency) { + System.out.println("Latency stat is enabled."); + latencies = new long[threads][MAX_LATENCY]; + } + } + + private void initKVDKEngine() throws KVDKException { + Configs configs = new Configs(); + + configs.setMaxAccessThreads(maxAccessThreads); + configs.setOptLargeSortedCollectionRecovery(optLargeSortedCollectionRestore); + configs.setUseDevDaxMode(useDevdaxMode); + configs.setHashBucketNum(hashBucketNum); + + kvdkEngine = Engine.open(path, configs); + + configs.close(); + closeableObjects.add(kvdkEngine); + } + + private void generateValuePool() { + StringBuilder sb = new StringBuilder(); + Random random = new Random(VALUE_RANDOM_SEED); + int low = 'a'; + for (int i = 0; i < valueSize; i++) { + sb.append((char) (low + random.nextInt(26))); + } + + valuePool = sb.toString().getBytes(); + } + + private void createSortedCollections() throws KVDKException { + System.out.format("Creating %d Sorted Collections", numCollection); + for (int i = 0; i < collectionNames.size(); i++) { + String collectionName = collectionNames.get(i); + NativeBytesHandle nameHandle = new NativeBytesHandle(collectionName.getBytes()); + collectionNameHandles[i] = nameHandle; + closeableObjects.add(nameHandle); + kvdkEngine.sortedCreate(nameHandle); + } + } + + private void startTasks() { + System.out.println("Init " + readThreads + " readers and " + writeThreads + " writers."); + executor = Executors.newCachedThreadPool(); + for (int i = 0; i < writeThreads; i++) { + executor.submit(new WriteTask(i)); + } + for (int i = writeThreads; i < threads; i++) { + Callable task = scan ? new ScanTask(i) : new ReadTask(i); + executor.submit(task); + } + } + + private void shutdownTasks() throws InterruptedException { + System.out.println("Shutdown task executor."); + executor.shutdown(); + boolean finished = executor.awaitTermination(10, TimeUnit.SECONDS); + if (!finished) { + System.out.println("Executor was not finished after 10 seconds. Force shutdown."); + executor.shutdownNow(); + } + System.out.println("Executor was shut down."); + } + + private void printProgress() throws InterruptedException { + System.out.println("----------------------------------------------------------"); + System.out.format( + "%-15s%-15s%-15s%-15s%-15s%-15s\n", + "Time(ms)", "Read Ops", "Write Ops", "Not Found", "Total Read", "Total Write"); + long start = System.currentTimeMillis(); + + readOpsLog.add(Long.valueOf(0)); + writeOpsLog.add(Long.valueOf(0)); + readNotFoundCountLog.add(Long.valueOf(0)); + lastValidLogIndex = 0; + while (true) { + Thread.sleep(1000); + long duration = System.currentTimeMillis() - start; + + readOpsLog.add(readOps.get()); + writeOpsLog.add(writeOps.get()); + readNotFoundCountLog.add(readNotFoundCount.get()); + + int idx = readOpsLog.size() - 1; + System.out.format( + "%-15d%-15d%-15d%-15d%-15d%-15d\n", + duration, + readOpsLog.get(idx) - readOpsLog.get(idx - 1), + writeOpsLog.get(idx) - writeOpsLog.get(idx - 1), + readNotFoundCountLog.get(idx) - readNotFoundCountLog.get(idx - 1), + readOpsLog.get(idx), + writeOpsLog.get(idx)); + + if (finishedThreads.get() == 0 || idx <= WARMUP_SECONDS) { + lastValidLogIndex = idx; + } + + if (finishedThreads.get() == threads) { + break; + } + + if (!fill && duration >= timeout * 1000) { + // Signal time out + System.out.println("Benchmark timeout (second): " + timeout); + isTimeout.set(true); + break; + } + } + + long duration = System.currentTimeMillis() - start; + System.out.println("Benchmark finished, duration (ms): " + duration); + } + + private void printLatency() { + long ro = readOps.get(); + if (ro > 0 && readThreads > 0 && !scan) { + double avg = 0; + double l50 = 0; + double l99 = 0; + double l995 = 0; + double l999 = 0; + double l9999 = 0; + + long accumulatedOps = 0; + long accumulatedLatency = 0; + for (int i = 1; i < MAX_LATENCY; i++) { + for (int j = 0; j < readThreads; j++) { + accumulatedOps += latencies[writeThreads + j][i]; + accumulatedLatency += latencies[writeThreads + j][i] * i; + + if (l50 == 0 && (double) accumulatedOps / ro > 0.5) { + l50 = (double) i / 10; + } else if (l99 == 0 && (double) accumulatedOps / ro > 0.99) { + l99 = (double) i / 10; + } else if (l995 == 0 && (double) accumulatedOps / ro > 0.995) { + l995 = (double) i / 10; + } else if (l999 == 0 && (double) accumulatedOps / ro > 0.999) { + l999 = (double) i / 10; + } else if (l9999 == 0 && (double) accumulatedOps / ro > 0.9999) { + l9999 = (double) i / 10; + } + } + } + avg = accumulatedLatency / ro / 10; + + System.out.format( + "read lantencies (us): Avg: %.2f, P50: %.2f, P99: %.2f, " + + "P99.5: %.2f, P99.9: %.2f, P99.99: %.2f\n", + avg, l50, l99, l995, l999, l9999); + } + + long wo = writeOps.get(); + if (wo > 0 && writeThreads > 0) { + double avg = 0; + double l50 = 0; + double l99 = 0; + double l995 = 0; + double l999 = 0; + double l9999 = 0; + + long accumulatedOps = 0; + long accumulatedLatency = 0; + for (int i = 1; i < MAX_LATENCY; i++) { + for (int j = 0; j < writeThreads; j++) { + accumulatedOps += latencies[j][i]; + accumulatedLatency += latencies[j][i] * i; + + if (l50 == 0 && (double) accumulatedOps / wo > 0.5) { + l50 = (double) i / 10; + } else if (l99 == 0 && (double) accumulatedOps / wo > 0.99) { + l99 = (double) i / 10; + } else if (l995 == 0 && (double) accumulatedOps / wo > 0.995) { + l995 = (double) i / 10; + } else if (l999 == 0 && (double) accumulatedOps / wo > 0.999) { + l999 = (double) i / 10; + } else if (l9999 == 0 && (double) accumulatedOps / wo > 0.9999) { + l9999 = (double) i / 10; + } + } + } + avg = accumulatedLatency / wo / 10; + + System.out.format( + "write lantencies (us): Avg: %.2f, P50: %.2f, P99: %.2f, " + + "P99.5: %.2f, P99.9: %.2f, P99.99: %.2f\n", + avg, l50, l99, l995, l999, l9999); + } + } + + private void printFinalStats() { + int timeElapsedInSeconds = 0; + long totalValidRead = 0; + long totalValidWrite = 0; + + if (lastValidLogIndex <= WARMUP_SECONDS) { + timeElapsedInSeconds = lastValidLogIndex; + totalValidRead = readOpsLog.get(lastValidLogIndex); + totalValidWrite = writeOpsLog.get(lastValidLogIndex); + } else { + timeElapsedInSeconds = lastValidLogIndex - WARMUP_SECONDS; + totalValidRead = readOpsLog.get(lastValidLogIndex) - readOpsLog.get(WARMUP_SECONDS); + totalValidWrite = writeOpsLog.get(lastValidLogIndex) - writeOpsLog.get(WARMUP_SECONDS); + } + + if (timeElapsedInSeconds == 0) { + return; + } + + System.out.println("----------------------------------------------------------"); + System.out.println( + "Average Read Ops:\t" + + totalValidRead / timeElapsedInSeconds + + ". Average Write Ops:\t" + + totalValidWrite / timeElapsedInSeconds); + + if (latency) { + printLatency(); + } + } + + private void closeObjects() throws Exception { + Collections.reverse(closeableObjects); + for (AutoCloseable c : closeableObjects) { + c.close(); + } + } + + public void run() throws Exception { + if (!dataType.equals(DataType.Blackhole)) { + initKVDKEngine(); + } + + if (dataType.equals(DataType.Sorted)) { + createSortedCollections(); + } + + startTasks(); + + try { + printProgress(); + } finally { + shutdownTasks(); + + printFinalStats(); + + closeObjects(); + } + } + + private enum DataType { + String, + Sorted, + Blackhole; + } + + private enum KeyDistribution { + Range, + Uniform + } + + private enum ValueSizeDistribution { + Constant, + Uniform + } + + private enum Flag { + path("/mnt/pmem0/kvdk", "Instance path") { + @Override + protected Object parseValue(String value) { + return value; + } + }, + num_kv((1L << 30), "Number of KVs to place") { + @Override + protected Object parseValue(String value) { + return parseLongPositive("num_kv", value); + } + }, + num_operations( + (1L << 30), + "Number of total operations. Asserted to be equal to num_kv if \n" + + "\t(fill == true).") { + @Override + protected Object parseValue(String value) { + return parseLongPositive("num_operations", value); + } + }, + fill(false, "Fill num_kv uniform kv pairs to a new instance") { + @Override + protected Object parseValue(String value) { + return parseBoolean(value); + } + }, + timeout(30L, "Time (seconds) to benchmark, this is valid only if fill=false") { + @Override + protected Object parseValue(String value) { + return parseLongPositive("timeout", value); + } + }, + value_size(120L, "Value size of KV") { + @Override + protected Object parseValue(String value) { + return parseLongPositive("value_size", value); + } + }, + value_size_distribution( + "constant", + "Distribution of value size to write, can be constant/random,\n" + + "\tdefault is constant. If set to random, the max value size\n" + + "\twill be -value_size.") { + @Override + protected Object parseValue(String value) { + return value; + } + }, + threads(10L, "Number of concurrent threads to run benchmark") { + @Override + protected Object parseValue(String value) { + return parseLongPositive("threads", value); + } + }, + read_ratio(0.0, "Read threads = threads * read_ratio") { + @Override + protected Object parseValue(String value) { + return parseDoubleRatio("read_ratio", value); + } + }, + existing_keys_ratio( + 1.0, + "Ratio of keys to read / write that existed in the filled instance, for\n" + + "\texample, if set to\n" + + "\t1, all writes will be updates, and all read keys will be existed") { + @Override + protected Object parseValue(String value) { + return parseDoubleRatio("existing_keys_ratio", value); + } + }, + latency(false, "Stat operation latencies") { + @Override + protected Object parseValue(String value) { + return parseBoolean(value); + } + }, + type("string", "Storage engine to benchmark, can be string, sorted or blackhole") { + @Override + protected Object parseValue(String value) { + return value; + } + }, + scan( + false, + "If set true, read threads will do scan operations, this is valid\n" + + "\tonly if we benchmark sorted engine") { + @Override + protected Object parseValue(String value) { + return parseBoolean(value); + } + }, + num_collection(1L, "Number of collections in the instance to benchmark") { + @Override + protected Object parseValue(String value) { + return parseLongNotNegtive("num_collection", value); + } + }, + batch_size( + 0L, + "Size of write batch. If batch>0, write string type kv with atomic batch\n" + + "\twrite, this is valid only if we benchmark string engine") { + @Override + protected Object parseValue(String value) { + return parseLongNotNegtive("batch_size", value); + } + }, + key_distribution( + "random", + "Distribution of benchmark keys, if fill is true, this param will\n" + + "\tbe ignored and only uniform distribution will be used") { + @Override + protected Object parseValue(String value) { + return value; + } + }, + populate( + false, + "Populate pmem space while creating a new instance. This can improve write\n" + + "\tperformance in runtime, but will take long time to init the instance") { + @Override + protected Object parseValue(String value) { + return parseBoolean(value); + } + }, + max_access_threads(32L, "Max access threads of the instance") { + @Override + protected Object parseValue(String value) { + return parseLongPositive("max_access_threads", value); + } + }, + space((256L << 30), "Max usable PMem space of the instance") { + @Override + protected Object parseValue(String value) { + return parseLongPositive("space", value); + } + }, + opt_large_sorted_collection_restore( + true, + "Optional optimization strategy which Multi-thread recovery a\n" + + "\tskiplist. When having few large skiplists, the optimization can\n" + + "\tget better performance") { + @Override + protected Object parseValue(String value) { + return parseBoolean(value); + } + }, + use_devdax_mode(false, "Use devdax device for kvdk") { + @Override + protected Object parseValue(String value) { + return parseBoolean(value); + } + }, + hash_bucket_num( + (1L << 27), + "The number of bucket groups in the hash table.\n" + + "\tIt should be 2^n and should smaller than 2^32.") { + @Override + protected Object parseValue(String value) { + Long ret = parseLongPositive("hash_bucket_num", value); + if (!isPowerOfTwo(ret) || (int) (Math.ceil((Math.log(ret) / Math.log(2)))) >= 32) { + throw new IllegalArgumentException( + "Invalid value for -hash_bucket_num: " + ret); + } + return ret; + } + }; + + private Flag(Object defaultValue, String description) { + this.defaultValue = defaultValue; + this.description = description; + } + + public Object getDefaultValue() { + return defaultValue; + } + + public String getDescription() { + return description; + } + + public boolean parseBoolean(String value) { + if (value.equals("1")) { + return true; + } else if (value.equals("0")) { + return false; + } + return Boolean.parseBoolean(value); + } + + public Long parseLongNotNegtive(String name, String value) { + Long ret = Long.parseLong(value); + if (ret < 0) { + throw new IllegalArgumentException("Invalid value for -" + name + ": " + ret); + } + return ret; + } + + public Long parseLongPositive(String name, String value) { + Long ret = Long.parseLong(value); + if (ret <= 0) { + throw new IllegalArgumentException("Invalid value for -" + name + ": " + ret); + } + return ret; + } + + public Double parseDoubleRatio(String name, String value) { + Double ret = Double.parseDouble(value); + if (ret < 0 || ret > 1) { + throw new IllegalArgumentException("Invalid value for -" + name + ": " + ret); + } + return ret; + } + + static boolean isPowerOfTwo(long n) { + return (int) (Math.ceil((Math.log(n) / Math.log(2)))) + == (int) (Math.floor(((Math.log(n) / Math.log(2))))); + } + + protected abstract Object parseValue(String value); + + private final Object defaultValue; + private final String description; + } + + abstract class BenchmarkTask implements Callable { + protected int tid; + + public BenchmarkTask(int tid) { + this.tid = tid; + } + + @Override + public Void call() throws Exception { + run(); + return null; + } + + protected abstract void run() throws KVDKException; + + protected long generateKey() { + switch (keyDistribution) { + case Range: + return rangeKeyGenerators[tid].gen(); + case Uniform: + return randomKeyGenerators[tid].gen(); + default: + throw new UnsupportedOperationException(); + } + } + + protected long generateValueSize() { + switch (valueSizeDistribution) { + case Constant: + return constantValueSizeGenerators[tid].gen(); + case Uniform: + return randomValueSizeGenerators[tid].gen(); + default: + throw new UnsupportedOperationException(); + } + } + + public byte[] longToBytes(long x) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(x); + return buffer.array(); + } + } + + private class WriteTask extends BenchmarkTask { + public WriteTask(int tid) { + super(tid); + } + + @Override + public void run() throws KVDKException { + WriteBatch batch = null; + if (kvdkEngine != null) { + batch = kvdkEngine.writeBatchCreate(); + } + + for (long operations = 0; operations < numOperationsPerThread; operations++) { + if (isTimeout.get()) { + break; + } + + long num = generateKey(); + byte[] key = longToBytes(num); + int generatedValueSize = (int) generateValueSize(); + + long start = 0; + if (latency) { + start = System.nanoTime(); + } + + switch (dataType) { + case String: + if (batchSize == 0) { + kvdkEngine.put(key, 0, key.length, valuePool, 0, generatedValueSize); + } else { + batch.stringPut(key, 0, key.length, valuePool, 0, generatedValueSize); + if ((operations + 1) % batchSize == 0) { + kvdkEngine.batchWrite(batch); + batch.clear(); + } + } + break; + case Sorted: + int cid = (int) (num % numCollection); + if (batchSize == 0) { + kvdkEngine.sortedPut( + collectionNameHandles[cid], + key, + 0, + key.length, + valuePool, + 0, + generatedValueSize); + } else { + batch.sortedPut( + collectionNameHandles[cid], + key, + 0, + key.length, + valuePool, + 0, + generatedValueSize); + if ((operations + 1) % batchSize == 0) { + kvdkEngine.batchWrite(batch); + batch.clear(); + } + } + break; + case Blackhole: + break; + default: + throw new UnsupportedOperationException(); + } + + if (latency) { + long lat = System.nanoTime() - start; + if (lat / 100 >= MAX_LATENCY) { + throw new RuntimeException("Write latency overflow"); + } + latencies[tid][(int) (lat / 100)]++; + } + + if ((operations + 1) % 1000 == 0) { + writeOps.addAndGet(1000); + } + } + + if (batch != null) { + batch.close(); + } + + finishedThreads.incrementAndGet(); + } + } + + private class ReadTask extends BenchmarkTask { + public ReadTask(int tid) { + super(tid); + } + + @Override + public void run() throws KVDKException { + long readNotFound = 0; + for (long operations = 0; operations < numOperationsPerThread; operations++) { + if (isTimeout.get()) { + break; + } + + long num = generateKey(); + byte[] key = longToBytes(num); + byte[] value = null; + + long start = 0; + if (latency) { + start = System.nanoTime(); + } + + switch (dataType) { + case String: + value = kvdkEngine.get(key); + break; + case Sorted: + int cid = (int) (num % numCollection); + value = kvdkEngine.sortedGet(collectionNameHandles[cid], key); + break; + case Blackhole: + break; + default: + throw new UnsupportedOperationException(); + } + + if (latency) { + long lat = System.nanoTime() - start; + if (lat / 100 >= MAX_LATENCY) { + throw new RuntimeException("Read latency overflow"); + } + latencies[tid][(int) (lat / 100)]++; + } + + if (value == null && !dataType.equals(DataType.Blackhole)) { + // Not found + if (++readNotFound % 1000 == 0) { + readNotFoundCount.addAndGet(1000); + } + } + + if ((operations + 1) % 1000 == 0) { + readOps.addAndGet(1000); + } + } + + finishedThreads.incrementAndGet(); + } + } + + private class ScanTask extends BenchmarkTask { + public ScanTask(int tid) { + super(tid); + } + + @Override + public void run() throws KVDKException { + long readNotFound = 0; + for (long operations = 0, operations_counted = 0; + operations < numOperationsPerThread; ) { + if (isTimeout.get()) { + break; + } + + long num = generateKey(); + byte[] key = longToBytes(num); + byte[] value = null; + + final int scanLength = 100; + switch (dataType) { + case Sorted: + int cid = (int) (num % numCollection); + Iterator iter = kvdkEngine.sortedIteratorCreate(collectionNameHandles[cid]); + iter.seek(key); + for (int i = 0; i < scanLength && iter.isValid(); i++, iter.next()) { + key = iter.key(); + value = iter.value(); + + operations++; + if (operations >= operations_counted + 1000) { + readOps.addAndGet(operations - operations_counted); + operations_counted = operations; + } + } + kvdkEngine.sortedIteratorRelease(iter); + + break; + case Blackhole: + operations += 1024; + readOps.addAndGet(1024); + break; + case String: + default: + throw new UnsupportedOperationException(); + } + + if ((operations + 1) % 1000 == 0) { + writeOps.addAndGet(1000); + } + } + + finishedThreads.incrementAndGet(); + } + } + + public static void printHelp() { + System.out.println("usage:"); + for (Flag flag : Flag.values()) { + System.out.format(" -%s%n\t%s%n", flag.name(), flag.getDescription()); + if (flag.getDefaultValue() != null) { + System.out.format("\tDefault: %s%n", flag.getDefaultValue().toString()); + } + } + } + + private static Map parseArgs(String[] args) { + Map flags = new EnumMap(Flag.class); + for (Flag flag : Flag.values()) { + if (flag.getDefaultValue() != null) { + flags.put(flag, flag.getDefaultValue()); + } + } + + for (String arg : args) { + boolean valid = false; + if (arg.equals("-h") || arg.equals("--help") || arg.equals("-?") || arg.equals("?")) { + printHelp(); + System.exit(0); + } + if (arg.startsWith("-")) { + try { + String[] parts = arg.substring(1).split("="); + if (parts.length >= 1) { + Flag key = Flag.valueOf(parts[0]); + if (key != null) { + Object value = null; + if (parts.length >= 2) { + value = key.parseValue(parts[1]); + } + flags.put(key, value); + valid = true; + } + } + } catch (Exception e) { + } + } + if (!valid) { + System.err.println("Invalid argument: " + arg); + System.err.println("Run with -h to show help information"); + System.exit(1); + } + } + + return flags; + } + + public static void main(String[] args) throws Exception { + Map flags = parseArgs(args); + new KVDKBenchmark(flags).run(); + } +} diff --git a/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/ConstantLongGenerator.java b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/ConstantLongGenerator.java new file mode 100644 index 00000000..17669d51 --- /dev/null +++ b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/ConstantLongGenerator.java @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk.benchmark.util; + +public class ConstantLongGenerator implements LongGenerator { + private long value; + + public ConstantLongGenerator(long value) { + this.value = value; + } + + @Override + public long gen() { + return value; + } +} diff --git a/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/LongGenerator.java b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/LongGenerator.java new file mode 100644 index 00000000..fd3f9165 --- /dev/null +++ b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/LongGenerator.java @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk.benchmark.util; + +public interface LongGenerator { + long gen(); +} diff --git a/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RandomLongGenerator.java b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RandomLongGenerator.java new file mode 100644 index 00000000..76b6260f --- /dev/null +++ b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RandomLongGenerator.java @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk.benchmark.util; + +import java.util.SplittableRandom; + +public class RandomLongGenerator implements LongGenerator { + private SplittableRandom random; + private long low; + private long high; + + public RandomLongGenerator(long low, long high) { + this.low = low; + this.high = high; + this.random = new SplittableRandom(); + } + + @Override + public long gen() { + return random.nextLong(low, high); + } +} diff --git a/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RangeLongGenerator.java b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RangeLongGenerator.java new file mode 100644 index 00000000..ccf54180 --- /dev/null +++ b/volatile/java/benchmark/src/main/java/io/pmem/kvdk/benchmark/util/RangeLongGenerator.java @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk.benchmark.util; + +public class RangeLongGenerator implements LongGenerator { + private long low; + private long high; + private long step; + private long curr; + + public RangeLongGenerator(long low, long high, long step) { + this.low = low; + this.high = high; + this.step = step; + this.curr = this.low; + } + + @Override + public long gen() { + long old = curr; + curr += step; + + assert (old < high); + return old; + } +} diff --git a/volatile/java/examples/pom.xml b/volatile/java/examples/pom.xml new file mode 100644 index 00000000..18b3b831 --- /dev/null +++ b/volatile/java/examples/pom.xml @@ -0,0 +1,102 @@ + + + 4.0.0 + + io.pmem + kvdkjni-examples + 1.0.0-SNAPSHOT + + KVDK JNI Examples + KVDK JNI Examples. + https://github.com/pmem/kvdk + 2021 + + + + BSD-3-Clause + hhttps://github.com/pmem/kvdk/blob/main/LICENSE + repo + + + + + scm:git:https://github.com/pmem/kvdk.git + scm:git:https://github.com/pmem/kvdk.git + scm:git:https://github.com/pmem/kvdk.git + + + + Intel + https://www.intel.com + + + + 1.0.0-SNAPSHOT + 1.8 + 1.8 + UTF-8 + 2.7.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${project.build.source} + ${project.build.target} + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + target + -Djava.library.path=${project.build.directory} + false + false + + ${project.build.directory}/* + + + 1 + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + 1.7 + + + + + + + spotless-check + compile + + check + + + + + + + + + + io.pmem + kvdkjni + ${kvdkjni.version} + + + diff --git a/volatile/java/examples/src/main/java/io/pmem/kvdk/exmaples/KVDKExamples.java b/volatile/java/examples/src/main/java/io/pmem/kvdk/exmaples/KVDKExamples.java new file mode 100644 index 00000000..e075d00d --- /dev/null +++ b/volatile/java/examples/src/main/java/io/pmem/kvdk/exmaples/KVDKExamples.java @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk.examples; + +import io.pmem.kvdk.Configs; +import io.pmem.kvdk.Engine; +import io.pmem.kvdk.Iterator; +import io.pmem.kvdk.KVDKException; +import io.pmem.kvdk.NativeBytesHandle; +import io.pmem.kvdk.WriteBatch; + +public class KVDKExamples { + protected Configs engineConfigs; + protected Engine kvdkEngine; + + public void prepare() throws KVDKException { + engineConfigs = new Configs(); + engineConfigs.setHashBucketNum(1L << 10); + engineConfigs.setMaxAccessThreads(4); + + kvdkEngine = Engine.open("/tmp/kvdk-test-dir", engineConfigs); + engineConfigs.close(); + } + + public void runAnonymousCollection() throws KVDKException { + System.out.println(); + System.out.println("Output of runAnonymousCollection: "); + + // put + String key = "sssss"; + String value = "22222"; + kvdkEngine.put(key.getBytes(), value.getBytes()); + + // expire + kvdkEngine.expire(key.getBytes(), 1000); + + // get + System.out.println("value: " + new String(kvdkEngine.get(key.getBytes()))); + + // delete + kvdkEngine.delete(key.getBytes()); + } + + public void runSortedCollection() throws KVDKException { + System.out.println(); + System.out.println("Output of runSortedCollection: "); + + String name = "collection\u0000\nname"; + NativeBytesHandle nameHandle = new NativeBytesHandle(name.getBytes()); + + // create + kvdkEngine.sortedCreate(nameHandle); + + String key1 = "key\u00001"; + String key2 = "key\u00002"; + String key3 = "key\u00003"; + + String value1 = "value\u00003"; + String value2 = "value\u00002"; + String value3 = "value\u00001"; + + // disordered put + kvdkEngine.sortedPut(nameHandle, key3.getBytes(), value3.getBytes()); + kvdkEngine.sortedPut(nameHandle, key1.getBytes(), value1.getBytes()); + kvdkEngine.sortedPut(nameHandle, key2.getBytes(), value2.getBytes()); + + // print sorted result + System.out.println("Sorted by key:"); + Iterator iter = kvdkEngine.sortedIteratorCreate(nameHandle); + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + System.out.println( + new String(iter.key()).replace("\0", "[\\0]") + + ": " + + new String(iter.value()).replace("\0", "[\\0]")); + } + + // close iterator + iter.close(); + + // destroy sorted collection + kvdkEngine.sortedDestroy(nameHandle); + + // close name handle + nameHandle.close(); + } + + public void runWriteBatch() throws KVDKException { + System.out.println(); + System.out.println("Output of runWriteBatch: "); + + String key = "key1"; + String value1 = "value1"; + String value2 = "value2"; + + // batch + WriteBatch batch = kvdkEngine.writeBatchCreate(); + batch.stringPut(key.getBytes(), value1.getBytes()); + batch.stringPut(key.getBytes(), value2.getBytes()); + + // write + kvdkEngine.batchWrite(batch); + + // get + kvdkEngine.get(key.getBytes()); + + // print + System.out.println("value: " + new String(kvdkEngine.get(key.getBytes()))); + + // delete + kvdkEngine.delete(key.getBytes()); + } + + public void close() { + kvdkEngine.close(); + } + + public static void main(String[] args) { + try { + KVDKExamples examples = new KVDKExamples(); + examples.prepare(); + examples.runAnonymousCollection(); + examples.runSortedCollection(); + examples.runWriteBatch(); + examples.close(); + } catch (KVDKException ex) { + ex.printStackTrace(); + } + } +} diff --git a/volatile/java/kvdkjni/configs.cc b/volatile/java/kvdkjni/configs.cc new file mode 100644 index 00000000..d8e3c1ff --- /dev/null +++ b/volatile/java/kvdkjni/configs.cc @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include + +#include "include/io_pmem_kvdk_Configs.h" +#include "kvdkjni/kvdkjni.h" + +/* + * Class: io_pmem_kvdk_Configs + * Method: newConfigs + * Signature: ()J + */ +jlong Java_io_pmem_kvdk_Configs_newConfigs(JNIEnv*, jclass) { + auto* cfg = new KVDK_NAMESPACE::Configs(); + return GET_CPLUSPLUS_POINTER(cfg); +} + +/* + * Class: io_pmem_kvdk_Configs + * Method: setMaxAccessThreads + * Signature: (JJ)V + */ +void Java_io_pmem_kvdk_Configs_setMaxAccessThreads(JNIEnv*, jobject, + jlong handle, jlong num) { + reinterpret_cast(handle)->max_access_threads = num; +} + +/* + * Class: io_pmem_kvdk_Configs + * Method: setHashBucketNum + * Signature: (JJ)V + */ +void Java_io_pmem_kvdk_Configs_setHashBucketNum(JNIEnv*, jobject, jlong handle, + jlong num) { + reinterpret_cast(handle)->hash_bucket_num = num; +} + +/* + * Class: io_pmem_kvdk_Configs + * Method: setDestMemoryNodes + * Signature: (JLjava/lang/String;)V + */ +void Java_io_pmem_kvdk_Configs_setDestMemoryNodes(JNIEnv* env, jobject, + jlong handle, + jstring jnodes) { + const char* nodes_chars = env->GetStringUTFChars(jnodes, nullptr); + if (nodes_chars == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + std::string nodes_str = nodes_chars; + reinterpret_cast(handle)->dest_memory_nodes = + nodes_str; + env->ReleaseStringUTFChars(jnodes, nodes_chars); +} + +/* + * Class: io_pmem_kvdk_Configs + * Method: setOptLargeSortedCollectionRecovery + * Signature: (JZ)V + */ +void Java_io_pmem_kvdk_Configs_setOptLargeSortedCollectionRecovery( + JNIEnv*, jobject, jlong handle, jboolean opt) { + reinterpret_cast(handle) + ->opt_large_sorted_collection_recovery = opt; +} + +/* + * Class: io_pmem_kvdk_Configs + * Method: setCleanThreads + * Signature: (JJ)V + */ +void Java_io_pmem_kvdk_Configs_setCleanThreads(JNIEnv*, jobject, jlong handle, + jlong num_threds) { + reinterpret_cast(handle)->clean_threads = + num_threds; +} + +/* + * Class: io_pmem_kvdk_Configs + * Method: closeInternal + * Signature: (J)V + */ +void Java_io_pmem_kvdk_Configs_closeInternal(JNIEnv*, jobject, jlong handle) { + auto* cfg = reinterpret_cast(handle); + assert(cfg != nullptr); + delete cfg; +} diff --git a/volatile/java/kvdkjni/cplusplus_to_java_convert.h b/volatile/java/kvdkjni/cplusplus_to_java_convert.h new file mode 100644 index 00000000..705e797b --- /dev/null +++ b/volatile/java/kvdkjni/cplusplus_to_java_convert.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#pragma once + +#define GET_CPLUSPLUS_POINTER(_pointer) \ + static_cast(reinterpret_cast(_pointer)) diff --git a/volatile/java/kvdkjni/engine.cc b/volatile/java/kvdkjni/engine.cc new file mode 100644 index 00000000..324319bc --- /dev/null +++ b/volatile/java/kvdkjni/engine.cc @@ -0,0 +1,438 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include + +#include "include/io_pmem_kvdk_Engine.h" +#include "kvdkjni/kvdkjni.h" + +/* + * Class: io_pmem_kvdk_Engine + * Method: open + * Signature: (Ljava/lang/String;J)V + */ +jlong Java_io_pmem_kvdk_Engine_open(JNIEnv* env, jclass, jstring jengine_path, + jlong jcfg_handle) { + const char* engine_path_chars = env->GetStringUTFChars(jengine_path, nullptr); + if (engine_path_chars == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + std::string engine_path_str = engine_path_chars; + + auto* cfg = reinterpret_cast(jcfg_handle); + KVDK_NAMESPACE::Engine* engine = nullptr; + KVDK_NAMESPACE::Status s = + KVDK_NAMESPACE::Engine::Open(engine_path_str, &engine, *cfg, stdout); + + env->ReleaseStringUTFChars(jengine_path, engine_path_chars); + + if (s == KVDK_NAMESPACE::Status::Ok) { + return GET_CPLUSPLUS_POINTER(engine); + } else { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + return 0; + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: put + * Signature: (J[BII[BIIJZ)V + */ +void Java_io_pmem_kvdk_Engine_put(JNIEnv* env, jobject, jlong handle, + jbyteArray key, jint key_off, jint key_len, + jbyteArray value, jint value_off, + jint value_len, jlong ttl_in_millis, + jboolean update_ttl_if_existed) { + auto* engine = reinterpret_cast(handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + jbyte* value_bytes = new jbyte[value_len]; + env->GetByteArrayRegion(value, value_off, value_len, value_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + delete[] value_bytes; + return; + } + + KVDK_NAMESPACE::WriteOptions write_options; + write_options.ttl_time = ttl_in_millis; + write_options.update_ttl = update_ttl_if_existed; + + auto s = + engine->Put(std::string(reinterpret_cast(key_bytes), key_len), + std::string(reinterpret_cast(value_bytes), value_len), + write_options); + + delete[] key_bytes; + delete[] value_bytes; + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: expire + * Signature: (J[BIIJ)V + */ +void Java_io_pmem_kvdk_Engine_expire__J_3BIIJ(JNIEnv* env, jobject, + jlong handle, jbyteArray key, + jint key_off, jint key_len, + jlong ttl_in_millis) { + auto* engine = reinterpret_cast(handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + auto s = engine->Expire( + std::string(reinterpret_cast(key_bytes), key_len), ttl_in_millis); + + delete[] key_bytes; + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: expire + * Signature: (JJIJ)V + */ +void Java_io_pmem_kvdk_Engine_expire__JJIJ(JNIEnv* env, jobject, + jlong engine_handle, + jlong name_handle, jint name_len, + jlong ttl_in_millis) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + auto s = engine->Expire(std::string(name_chars, name_len), ttl_in_millis); + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: get + * Signature: (J[BII)[B + */ +jbyteArray Java_io_pmem_kvdk_Engine_get(JNIEnv* env, jobject, jlong handle, + jbyteArray key, jint key_off, + jint key_len) { + auto* engine = reinterpret_cast(handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return nullptr; + } + + std::string value; + auto s = engine->Get(std::string(reinterpret_cast(key_bytes), key_len), + &value); + + delete[] key_bytes; + + if (s == KVDK_NAMESPACE::Status::NotFound) { + return nullptr; + } + + if (s == KVDK_NAMESPACE::Status::Ok) { + jbyteArray ret = KVDK_NAMESPACE::JniUtil::createJavaByteArray( + env, value.c_str(), value.size()); + return ret; + } + + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + return nullptr; +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: delete + * Signature: (J[BII)V + */ +void Java_io_pmem_kvdk_Engine_delete(JNIEnv* env, jobject, jlong handle, + jbyteArray key, jint key_off, + jint key_len) { + auto* engine = reinterpret_cast(handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + auto s = + engine->Delete(std::string(reinterpret_cast(key_bytes), key_len)); + + delete[] key_bytes; + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: sortedCreate + * Signature: (JJI)V + */ +void Java_io_pmem_kvdk_Engine_sortedCreate(JNIEnv* env, jobject, + jlong engine_handle, + jlong name_handle, jint name_len) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + auto s = engine->SortedCreate(std::string(name_chars, name_len)); + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: sortedDestroy + * Signature: (JJI)V + */ +void Java_io_pmem_kvdk_Engine_sortedDestroy(JNIEnv* env, jobject, + jlong engine_handle, + jlong name_handle, jint name_len) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + auto s = engine->SortedDestroy(std::string(name_chars, name_len)); + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: sortedSize + * Signature: (JJI)J + */ +jlong Java_io_pmem_kvdk_Engine_sortedSize(JNIEnv* env, jobject, + jlong engine_handle, + jlong name_handle, jint name_len) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + size_t size = 0; + auto s = engine->SortedSize(std::string(name_chars, name_len), &size); + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } + + return size; +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: sortedPut + * Signature: (JJI[BII[BII)V + */ +void Java_io_pmem_kvdk_Engine_sortedPut(JNIEnv* env, jobject, + jlong engine_handle, jlong name_handle, + jint name_len, jbyteArray key, + jint key_off, jint key_len, + jbyteArray value, jint value_off, + jint value_len) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + jbyte* value_bytes = new jbyte[value_len]; + env->GetByteArrayRegion(value, value_off, value_len, value_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + delete[] value_bytes; + return; + } + + auto s = engine->SortedPut( + std::string(name_chars, name_len), + std::string(reinterpret_cast(key_bytes), key_len), + std::string(reinterpret_cast(value_bytes), value_len)); + + delete[] key_bytes; + delete[] value_bytes; + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: sortedGet + * Signature: (JJI[BII)[B + */ +jbyteArray Java_io_pmem_kvdk_Engine_sortedGet(JNIEnv* env, jobject, + jlong engine_handle, + jlong name_handle, jint name_len, + jbyteArray key, jint key_off, + jint key_len) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return nullptr; + } + + std::string value; + auto s = engine->SortedGet( + std::string(name_chars, name_len), + std::string(reinterpret_cast(key_bytes), key_len), &value); + + delete[] key_bytes; + + if (s == KVDK_NAMESPACE::Status::NotFound) { + return nullptr; + } + + if (s == KVDK_NAMESPACE::Status::Ok) { + jbyteArray ret = KVDK_NAMESPACE::JniUtil::createJavaByteArray( + env, value.c_str(), value.size()); + return ret; + } + + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + return nullptr; +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: sortedDelete + * Signature: (JJI[BII)V + */ +void Java_io_pmem_kvdk_Engine_sortedDelete(JNIEnv* env, jobject, + jlong engine_handle, + jlong name_handle, jint name_len, + jbyteArray key, jint key_off, + jint key_len) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + auto s = engine->SortedDelete( + std::string(name_chars, name_len), + std::string(reinterpret_cast(key_bytes), key_len)); + + delete[] key_bytes; + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: sortedIteratorCreate + * Signature: (JJI)J + */ +jlong Java_io_pmem_kvdk_Engine_sortedIteratorCreate(JNIEnv* env, jobject, + jlong engine_handle, + jlong name_handle, + jint name_len) { + auto* engine = reinterpret_cast(engine_handle); + auto* name_chars = reinterpret_cast(name_handle); + + KVDK_NAMESPACE::Status s; + KVDK_NAMESPACE::SortedIterator* iter = engine->SortedIteratorCreate( + std::string(name_chars, name_len), nullptr, &s); + + if (s == KVDK_NAMESPACE::Status::Ok) { + return GET_CPLUSPLUS_POINTER(iter); + } else { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + return 0; + } +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: closeInternal + * Signature: (J)V + */ +void Java_io_pmem_kvdk_Engine_closeInternal(JNIEnv*, jobject, jlong handle) { + auto* engine = reinterpret_cast(handle); + assert(engine != nullptr); + delete engine; +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: writeBatchCreate + * Signature: (J)J + */ +jlong Java_io_pmem_kvdk_Engine_writeBatchCreate(JNIEnv*, jobject, + jlong handle) { + auto* engine = reinterpret_cast(handle); + std::unique_ptr ptr = engine->WriteBatchCreate(); + return GET_CPLUSPLUS_POINTER(ptr.release()); +} + +/* + * Class: io_pmem_kvdk_Engine + * Method: batchWrite + * Signature: (JJ)V + */ +void Java_io_pmem_kvdk_Engine_batchWrite(JNIEnv* env, jobject, + jlong engine_handle, + jlong batch_handle) { + auto* engine = reinterpret_cast(engine_handle); + auto* batch = reinterpret_cast(batch_handle); + std::unique_ptr ptr(batch); + auto s = engine->BatchWrite(ptr); + ptr.release(); + + if (s != KVDK_NAMESPACE::Status::Ok) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew(env, s); + } +} diff --git a/volatile/java/kvdkjni/iterator.cc b/volatile/java/kvdkjni/iterator.cc new file mode 100644 index 00000000..e7631ec7 --- /dev/null +++ b/volatile/java/kvdkjni/iterator.cc @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include + +#include "include/io_pmem_kvdk_Iterator.h" +#include "kvdkjni/kvdkjni.h" + +/* + * Class: io_pmem_kvdk_Iterator + * Method: closeInternal + * Signature: (J)V + */ +void Java_io_pmem_kvdk_Iterator_closeInternal(JNIEnv*, jobject, + jlong iterator_handle, + jlong engine_handle) { + auto* iterator = + reinterpret_cast(iterator_handle); + auto* engine = reinterpret_cast(engine_handle); + assert(iterator != nullptr); + assert(engine != nullptr); + + engine->SortedIteratorRelease(iterator); +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: seek + * Signature: (J[BII)V + */ +void Java_io_pmem_kvdk_Iterator_seek(JNIEnv* env, jobject, jlong handle, + jbyteArray key, jint key_off, + jint key_len) { + auto* iterator = reinterpret_cast(handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + iterator->Seek(std::string(reinterpret_cast(key_bytes), key_len)); + + delete[] key_bytes; +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: seekToFirst + * Signature: (J)V + */ +void Java_io_pmem_kvdk_Iterator_seekToFirst(JNIEnv*, jobject, jlong handle) { + auto* iterator = reinterpret_cast(handle); + iterator->SeekToFirst(); +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: seekToLast + * Signature: (J)V + */ +void Java_io_pmem_kvdk_Iterator_seekToLast(JNIEnv*, jobject, jlong handle) { + auto* iterator = reinterpret_cast(handle); + iterator->SeekToLast(); +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: isValid + * Signature: (J)Z + */ +jboolean Java_io_pmem_kvdk_Iterator_isValid(JNIEnv*, jobject, jlong handle) { + auto* iterator = reinterpret_cast(handle); + return iterator->Valid(); +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: next + * Signature: (J)V + */ +void Java_io_pmem_kvdk_Iterator_next(JNIEnv*, jobject, jlong handle) { + auto* iterator = reinterpret_cast(handle); + iterator->Next(); +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: prev + * Signature: (J)V + */ +void Java_io_pmem_kvdk_Iterator_prev(JNIEnv*, jobject, jlong handle) { + auto* iterator = reinterpret_cast(handle); + iterator->Prev(); +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: key + * Signature: (J)[B + */ +jbyteArray Java_io_pmem_kvdk_Iterator_key(JNIEnv* env, jobject, jlong handle) { + auto* iterator = reinterpret_cast(handle); + + std::string key = iterator->Key(); + jbyteArray ret = KVDK_NAMESPACE::JniUtil::createJavaByteArray( + env, key.c_str(), key.size()); + return ret; +} + +/* + * Class: io_pmem_kvdk_Iterator + * Method: value + * Signature: (J)[B + */ +jbyteArray Java_io_pmem_kvdk_Iterator_value(JNIEnv* env, jobject, + jlong handle) { + auto* iterator = reinterpret_cast(handle); + + std::string value = iterator->Value(); + jbyteArray ret = KVDK_NAMESPACE::JniUtil::createJavaByteArray( + env, value.c_str(), value.size()); + return ret; +} diff --git a/volatile/java/kvdkjni/kvdkjni.h b/volatile/java/kvdkjni/kvdkjni.h new file mode 100644 index 00000000..f87696c6 --- /dev/null +++ b/volatile/java/kvdkjni/kvdkjni.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#ifndef JAVA_KVDKJNI_KVDKJNI_H_ +#define JAVA_KVDKJNI_KVDKJNI_H_ + +#include +#include + +#include + +#include "kvdk/volatile/engine.hpp" +#include "kvdkjni/cplusplus_to_java_convert.h" + +namespace KVDK_NAMESPACE { + +// Helper class to get Java class by name from C++. +class JavaClass { + public: + /* + * Gets and initializes a Java Class + * + * @param env A pointer to the Java environment + * @param jclazz_name The fully qualified JNI name of the Java Class + * e.g. "java/lang/String" + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env, const char* jclazz_name) { + jclass jclazz = env->FindClass(jclazz_name); + assert(jclazz != nullptr); + return jclazz; + } +}; + +// The portal class for io.pmem.kvdk.Status +class StatusJni : public JavaClass { + public: + /* + * Get the Java Class io.pmem.kvdk.Status + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "io/pmem/kvdk/Status"); + } + + /* + * Create a new Java io.pmem.kvdk.Status object with the same properties as + * the provided C++ KVDK_NAMESPACE::Status object + * + * @param env A pointer to the Java environment + * @param status The KVDK_NAMESPACE::Status object + * + * @return A reference to a Java io.pmem.kvdk.Status object, or nullptr + * if an an exception occurs + */ + static jobject construct(JNIEnv* env, const Status& status) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", "(B)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jobject jstatus = env->NewObject(jclazz, mid, toJavaStatusCode(status)); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + return jstatus; + } + + static jobject construct(JNIEnv* env, const Status* status) { + return construct(env, *status); + } + + // Returns the equivalent io.pmem.kvdk.Status.Code for the provided + // C++ KVDK_NAMESPACE::Status enum + static jbyte toJavaStatusCode(const KVDK_NAMESPACE::Status& status) { + switch (status) { + case KVDK_NAMESPACE::Status::Ok: + return 0x0; + case KVDK_NAMESPACE::Status::NotFound: + return 0x1; + case KVDK_NAMESPACE::Status::Outdated: + return 0x2; + case KVDK_NAMESPACE::Status::WrongType: + return 0x3; + case KVDK_NAMESPACE::Status::Existed: + return 0x4; + case KVDK_NAMESPACE::Status::OperationFail: + return 0x5; + case KVDK_NAMESPACE::Status::OutOfRange: + return 0x6; + case KVDK_NAMESPACE::Status::MemoryOverflow: + return 0x7; + case KVDK_NAMESPACE::Status::NotSupported: + return 0x8; + case KVDK_NAMESPACE::Status::InvalidBatchSize: + return 0x9; + case KVDK_NAMESPACE::Status::InvalidDataSize: + return 0xA; + case KVDK_NAMESPACE::Status::InvalidArgument: + return 0xB; + case KVDK_NAMESPACE::Status::IOError: + return 0xC; + case KVDK_NAMESPACE::Status::InvalidConfiguration: + return 0xD; + case KVDK_NAMESPACE::Status::Fail: + return 0xE; + case KVDK_NAMESPACE::Status::Abort: + return 0xF; + default: + return 0x7F; // undefined + } + } +}; + +// Java Exception template +template +class JavaException : public JavaClass { + public: + /* + * Create and throw a java exception with the provided message + * + * @param env A pointer to the Java environment + * @param msg The message for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const std::string& msg) { + jclass jclazz = DERIVED::getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + std::cerr << "JavaException::ThrowNew - Error: unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + const jint rs = env->ThrowNew(jclazz, msg.c_str()); + if (rs != JNI_OK) { + // exception could not be thrown + std::cerr << "JavaException::ThrowNew - Fatal: could not throw exception!" + << std::endl; + return env->ExceptionCheck(); + } + + return true; + } +}; + +// The portal class for io.pmem.kvdk.KVDKException +class KVDKExceptionJni : public JavaException { + public: + /* + * Get the Java Class io.pmem.kvdk.KVDKException + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaException::getJClass(env, "io/pmem/kvdk/KVDKException"); + } + + /* + * Create and throw a Java KVDKException with the provided message + * + * @param env A pointer to the Java environment + * @param msg The message for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const std::string& msg) { + return JavaException::ThrowNew(env, msg); + } + + /* + * Create and throw a Java KVDKException with the provided status + * + * If s == Status::Ok, then this function will not throw any exception. + * + * @param env A pointer to the Java environment + * @param s The status for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const Status& s) { + if (s == Status::Ok) { + return false; + } + + // get the KVDKException class + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + std::cerr + << "KVDKExceptionJni::ThrowNew/class - Error: unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // get the constructor of io.pmem.kvdk.KVDKException + jmethodID mid = + env->GetMethodID(jclazz, "", "(Lio/pmem/kvdk/Status;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + std::cerr + << "KVDKExceptionJni::ThrowNew/cstr - Error: unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // get the Java status object + jobject jstatus = StatusJni::construct(env, s); + if (jstatus == nullptr) { + // exception occcurred + std::cerr << "KVDKExceptionJni::ThrowNew/StatusJni - Error: unexpected " + "exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // construct the KVDKException + jthrowable kvdk_exception = + reinterpret_cast(env->NewObject(jclazz, mid, jstatus)); + if (env->ExceptionCheck()) { + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (kvdk_exception != nullptr) { + env->DeleteLocalRef(kvdk_exception); + } + std::cerr << "KVDKExceptionJni::ThrowNew/NewObject - Error: unexpected " + "exception!" + << std::endl; + return true; + } + + // throw the KVDKException + const jint rs = env->Throw(kvdk_exception); + if (rs != JNI_OK) { + // exception could not be thrown + std::cerr + << "KVDKExceptionJni::ThrowNew - Fatal: could not throw exception!" + << std::endl; + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (kvdk_exception != nullptr) { + env->DeleteLocalRef(kvdk_exception); + } + return env->ExceptionCheck(); + } + + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (kvdk_exception != nullptr) { + env->DeleteLocalRef(kvdk_exception); + } + + return true; + } +}; + +class JniUtil { + public: + static jbyteArray createJavaByteArray(JNIEnv* env, const char* bytes, + const size_t size) { + // Limitation for java array size is vm specific + // In general it cannot exceed Integer.MAX_VALUE (2^31 - 1) + // Current HotSpot VM limitation for array size is Integer.MAX_VALUE - 5 + // (2^31 - 1 - 5) It means that the next call to env->NewByteArray can still + // end with OutOfMemoryError("Requested array size exceeds VM limit") coming + // from VM + static const size_t MAX_JARRAY_SIZE = (static_cast(1)) << 31; + if (size > MAX_JARRAY_SIZE) { + KVDK_NAMESPACE::KVDKExceptionJni::ThrowNew( + env, "Requested array size exceeds VM limit"); + return nullptr; + } + + const jsize jlen = static_cast(size); + jbyteArray jbytes = env->NewByteArray(jlen); + if (jbytes == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetByteArrayRegion( + jbytes, 0, jlen, + const_cast(reinterpret_cast(bytes))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jbytes); + return nullptr; + } + + return jbytes; + } +}; + +} // namespace KVDK_NAMESPACE + +#endif // JAVA_KVDKJNI_KVDKJNI_H_ diff --git a/volatile/java/kvdkjni/native_bytes_handle.cc b/volatile/java/kvdkjni/native_bytes_handle.cc new file mode 100644 index 00000000..ae877c7b --- /dev/null +++ b/volatile/java/kvdkjni/native_bytes_handle.cc @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include + +#include "include/io_pmem_kvdk_NativeBytesHandle.h" +#include "kvdkjni/kvdkjni.h" + +/* + * Class: io_pmem_kvdk_NativeBytesHandle + * Method: newNativeBytes + * Signature: ([B)J + */ +jlong Java_io_pmem_kvdk_NativeBytesHandle_newNativeBytes(JNIEnv* env, jclass, + jbyteArray bytes) { + int len = env->GetArrayLength(bytes); + jbyte* b = new jbyte[len]; + env->GetByteArrayRegion(bytes, 0, len, b); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] b; + return 0; + } + + return GET_CPLUSPLUS_POINTER(b); +} + +/* + * Class: io_pmem_kvdk_NativeBytesHandle + * Method: closeInternal + * Signature: (J)V + */ +void Java_io_pmem_kvdk_NativeBytesHandle_closeInternal(JNIEnv*, jobject, + jlong handle) { + auto* b = reinterpret_cast(handle); + assert(b != nullptr); + delete[] b; +} diff --git a/volatile/java/kvdkjni/write_batch.cc b/volatile/java/kvdkjni/write_batch.cc new file mode 100644 index 00000000..6e0c9008 --- /dev/null +++ b/volatile/java/kvdkjni/write_batch.cc @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include + +#include "include/io_pmem_kvdk_WriteBatch.h" +#include "kvdkjni/kvdkjni.h" + +/* + * Class: io_pmem_kvdk_WriteBatch + * Method: closeInternal + * Signature: (J)V + */ +void Java_io_pmem_kvdk_WriteBatch_closeInternal(JNIEnv*, jobject, + jlong batch_handle) { + auto* batch = reinterpret_cast(batch_handle); + assert(batch != nullptr); + delete batch; +} + +/* + * Class: io_pmem_kvdk_WriteBatch + * Method: stringPut + * Signature: (J[BII[BII)V + */ +void Java_io_pmem_kvdk_WriteBatch_stringPut(JNIEnv* env, jobject, + jlong batch_handle, jbyteArray key, + jint key_off, jint key_len, + jbyteArray value, jint value_off, + jint value_len) { + auto* batch = reinterpret_cast(batch_handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + jbyte* value_bytes = new jbyte[value_len]; + env->GetByteArrayRegion(value, value_off, value_len, value_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + delete[] value_bytes; + return; + } + + batch->StringPut( + std::string(reinterpret_cast(key_bytes), key_len), + std::string(reinterpret_cast(value_bytes), value_len)); + + delete[] key_bytes; + delete[] value_bytes; +} + +/* + * Class: io_pmem_kvdk_WriteBatch + * Method: stringDelete + * Signature: (J[BII)V + */ +void Java_io_pmem_kvdk_WriteBatch_stringDelete(JNIEnv* env, jobject, + jlong batch_handle, + jbyteArray key, jint key_off, + jint key_len) { + auto* batch = reinterpret_cast(batch_handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + batch->StringDelete(std::string(reinterpret_cast(key_bytes), key_len)); + + delete[] key_bytes; +} + +/* + * Class: io_pmem_kvdk_WriteBatch + * Method: sortedPut + * Signature: (JJI[BII[BII)V + */ +void Java_io_pmem_kvdk_WriteBatch_sortedPut(JNIEnv* env, jobject, + jlong batch_handle, + jlong name_handle, jint name_len, + jbyteArray key, jint key_off, + jint key_len, jbyteArray value, + jint value_off, jint value_len) { + auto* batch = reinterpret_cast(batch_handle); + auto* name_chars = reinterpret_cast(name_handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + jbyte* value_bytes = new jbyte[value_len]; + env->GetByteArrayRegion(value, value_off, value_len, value_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + delete[] value_bytes; + return; + } + + batch->SortedPut( + std::string(name_chars, name_len), + std::string(reinterpret_cast(key_bytes), key_len), + std::string(reinterpret_cast(value_bytes), value_len)); + + delete[] key_bytes; + delete[] value_bytes; +} + +/* + * Class: io_pmem_kvdk_WriteBatch + * Method: sortedDelete + * Signature: (JJI[BII)V + */ +void Java_io_pmem_kvdk_WriteBatch_sortedDelete(JNIEnv* env, jobject, + jlong batch_handle, + jlong name_handle, jint name_len, + jbyteArray key, jint key_off, + jint key_len) { + auto* batch = reinterpret_cast(batch_handle); + auto* name_chars = reinterpret_cast(name_handle); + + jbyte* key_bytes = new jbyte[key_len]; + env->GetByteArrayRegion(key, key_off, key_len, key_bytes); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key_bytes; + return; + } + + batch->SortedDelete(std::string(name_chars, name_len), + std::string(reinterpret_cast(key_bytes), key_len)); + + delete[] key_bytes; +} + +/* + * Class: io_pmem_kvdk_WriteBatch + * Method: clear + * Signature: (J)V + */ +void Java_io_pmem_kvdk_WriteBatch_clear(JNIEnv*, jobject, jlong batch_handle) { + auto* batch = reinterpret_cast(batch_handle); + batch->Clear(); +} + +/* + * Class: io_pmem_kvdk_WriteBatch + * Method: size + * Signature: (J)J + */ +jlong Java_io_pmem_kvdk_WriteBatch_size(JNIEnv*, jobject, jlong batch_handle) { + auto* batch = reinterpret_cast(batch_handle); + return batch->Size(); +} \ No newline at end of file diff --git a/volatile/java/pom.xml b/volatile/java/pom.xml new file mode 100644 index 00000000..807360d2 --- /dev/null +++ b/volatile/java/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + io.pmem + kvdkjni + 1.0.0-SNAPSHOT + + KVDK JNI + KVDK fat jar that contains .so file. + + https://github.com/pmem/kvdk + 2021 + + + + BSD-3-Clause + hhttps://github.com/pmem/kvdk/blob/main/LICENSE + repo + + + + + scm:git:https://github.com/pmem/kvdk.git + scm:git:https://github.com/pmem/kvdk.git + scm:git:https://github.com/pmem/kvdk.git + + + + Intel + https://www.intel.com + + + + 1.8 + 1.8 + UTF-8 + 2.7.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${project.build.source} + ${project.build.target} + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + target + -Djava.library.path=${project.build.directory} + false + false + + ${project.build.directory}/* + + + 1 + libasan.so.4 + detect_leaks=0:handle_segv=0 + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + 1.7 + + + + + + + spotless-check + compile + + check + + + + + + + + + + junit + junit + 4.13.1 + test + + + diff --git a/volatile/java/src/main/java/io/pmem/kvdk/AbstractNativeReference.java b/volatile/java/src/main/java/io/pmem/kvdk/AbstractNativeReference.java new file mode 100644 index 00000000..2ec4cfe3 --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/AbstractNativeReference.java @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** Immutable reference to native C++ object. */ +public abstract class AbstractNativeReference implements AutoCloseable { + /** Indicating whether this reference is responsible to free the native C++ object. */ + protected final AtomicBoolean owningHandle_; + + /** + * Constrctor. + * + * @param owningHandle + */ + protected AbstractNativeReference(final boolean owningHandle) { + this.owningHandle_ = new AtomicBoolean(owningHandle); + } + + public boolean isOwningHandle() { + return owningHandle_.get(); + } + + /** Release the responsibility of freeing native C++ object. */ + protected final void disOwnNativeHandle() { + owningHandle_.set(false); + } + + @Override + public void close() { + if (owningHandle_.compareAndSet(true, false)) { + closeInternal(); + } + } + + protected abstract void closeInternal(); +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/Configs.java b/volatile/java/src/main/java/io/pmem/kvdk/Configs.java new file mode 100644 index 00000000..aee992de --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/Configs.java @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +/** KVDK engine configs. */ +public class Configs extends KVDKObject { + static { + Engine.loadLibrary(); + } + + public Configs() { + super(newConfigs()); + } + + public Configs setMaxAccessThreads(final long num_threads) { + setMaxAccessThreads(nativeHandle_, num_threads); + return this; + } + + public Configs setHashBucketNum(final long num) { + setHashBucketNum(nativeHandle_, num); + return this; + } + + public Configs setDestMemoryNodes(final String nodes) { + setDestMemoryNodes(nativeHandle_, nodes); + return this; + } + + public Configs setOptLargeSortedCollectionRecovery(final boolean opt) { + setOptLargeSortedCollectionRecovery(nativeHandle_, opt); + return this; + } + + public Configs setUseDevDaxMode(final boolean use) { + setUseDevDaxMode(nativeHandle_, use); + return this; + } + + public Configs setCleanThreads(final long num_threads) { + setCleanThreads(nativeHandle_, num_threads); + return this; + } + + // Native methods + private static native long newConfigs(); + + private native void setMaxAccessThreads(long handle, long num); + + private native void setHashBucketNum(long handle, long num); + + private native void setDestMemoryNodes(long handle, String nodes); + + private native void setOptLargeSortedCollectionRecovery(long handle, boolean opt); + + private native void setUseDevDaxMode(long handle, boolean use); + + private native void setCleanThreads(long handle, long num); + + @Override + protected final native void closeInternal(long handle); +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/Engine.java b/volatile/java/src/main/java/io/pmem/kvdk/Engine.java new file mode 100644 index 00000000..ec952aa9 --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/Engine.java @@ -0,0 +1,426 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** The KVDK engine, providing the APIs for Key-Value operations. */ +public class Engine extends KVDKObject { + private static final String atomicLibraryFileName = "libatomic.so.1"; + private static final String stdLibraryFileName = "libstdc++.so.6"; + private static final String jniLibraryFileName = System.mapLibraryName("kvdkjni"); + + /** + * Current state of loading native library. This must be defined before calling loading + * loadLibrary(). + */ + private static final AtomicReference libraryLoaded = + new AtomicReference<>(LibraryState.NOT_LOADED); + + static { + Engine.loadLibrary(); + } + + protected Engine(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Open a KVDK engine instance. + * + * @param path A directory to PMem + * @param configs KVDK engine configs + * @return + * @throws KVDKException + */ + public static Engine open(final String path, final Configs configs) throws KVDKException { + final Engine engine = new Engine(open(path, configs.getNativeHandle())); + return engine; + } + + public void put(final byte[] key, final byte[] value) throws KVDKException { + WriteOptions defaulWriteOptions = new WriteOptions(); + put( + nativeHandle_, + key, + 0, + key.length, + value, + 0, + value.length, + defaulWriteOptions.getTtlInMillis(), + defaulWriteOptions.isUpdateTtlIfExisted()); + } + + public void put( + final byte[] key, + int keyOffset, + int keyLength, + final byte[] value, + int valueOffset, + int valueLength) + throws KVDKException { + WriteOptions defaulWriteOptions = new WriteOptions(); + put( + nativeHandle_, + key, + keyOffset, + keyLength, + value, + valueOffset, + valueLength, + defaulWriteOptions.getTtlInMillis(), + defaulWriteOptions.isUpdateTtlIfExisted()); + } + + public void put(final byte[] key, final byte[] value, final WriteOptions wOptions) + throws KVDKException { + put( + nativeHandle_, + key, + 0, + key.length, + value, + 0, + value.length, + wOptions.getTtlInMillis(), + wOptions.isUpdateTtlIfExisted()); + } + + public void put( + final byte[] key, + int keyOffset, + int keyLength, + final byte[] value, + int valueOffset, + int valueLength, + final WriteOptions wOptions) + throws KVDKException { + put( + nativeHandle_, + key, + keyOffset, + keyLength, + value, + valueOffset, + valueLength, + wOptions.getTtlInMillis(), + wOptions.isUpdateTtlIfExisted()); + } + + public void expire(final byte[] key, final long ttlInMillis) throws KVDKException { + expire(nativeHandle_, key, 0, key.length, ttlInMillis); + } + + public void expire(final byte[] key, int keyOffset, int keyLength, final long ttlInMillis) + throws KVDKException { + expire(nativeHandle_, key, keyOffset, keyLength, ttlInMillis); + } + + public void expire(final NativeBytesHandle nameHandle, final long ttlInMillis) + throws KVDKException { + expire(nativeHandle_, nameHandle.getNativeHandle(), nameHandle.getLength(), ttlInMillis); + } + + /** + * @param key + * @return Value as byte array, null if the specified key doesn't exist. + * @throws KVDKException + */ + public byte[] get(final byte[] key) throws KVDKException { + return get(nativeHandle_, key, 0, key.length); + } + + /** + * @param key + * @return Value as byte array, null if the specified key doesn't exist. + * @throws KVDKException + */ + public byte[] get(final byte[] key, int keyOffset, int keyLength) throws KVDKException { + return get(nativeHandle_, key, keyOffset, keyLength); + } + + public void delete(final byte[] key) throws KVDKException { + delete(nativeHandle_, key, 0, key.length); + } + + public void delete(final byte[] key, int keyOffset, int keyLength) throws KVDKException { + delete(nativeHandle_, key, keyOffset, keyLength); + } + + public void sortedCreate(final NativeBytesHandle nameHandle) throws KVDKException { + sortedCreate(nativeHandle_, nameHandle.getNativeHandle(), nameHandle.getLength()); + } + + public void sortedDestroy(final NativeBytesHandle nameHandle) throws KVDKException { + sortedDestroy(nativeHandle_, nameHandle.getNativeHandle(), nameHandle.getLength()); + } + + public long sortedSize(final NativeBytesHandle nameHandle) throws KVDKException { + return sortedSize(nativeHandle_, nameHandle.getNativeHandle(), nameHandle.getLength()); + } + + public void sortedPut(final NativeBytesHandle nameHandle, final byte[] key, final byte[] value) + throws KVDKException { + sortedPut( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + 0, + key.length, + value, + 0, + value.length); + } + + public void sortedPut( + final NativeBytesHandle nameHandle, + final byte[] key, + int keyOffset, + int keyLength, + final byte[] value, + int valueOffset, + int valueLength) + throws KVDKException { + sortedPut( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + keyOffset, + keyLength, + value, + valueOffset, + valueLength); + } + + /** + * @param nameHandle + * @param key + * @return Value as byte array, null if the specified key doesn't exist. + * @throws KVDKException + */ + public byte[] sortedGet(final NativeBytesHandle nameHandle, final byte[] key) + throws KVDKException { + return sortedGet( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + 0, + key.length); + } + + /** + * @param nameHandle + * @param key + * @return Value as byte array, null if the specified key doesn't exist. + * @throws KVDKException + */ + public byte[] sortedGet( + final NativeBytesHandle nameHandle, final byte[] key, int keyOffset, int keyLength) + throws KVDKException { + return sortedGet( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + keyOffset, + keyLength); + } + + public void sortedDelete(final NativeBytesHandle nameHandle, final byte[] key) + throws KVDKException { + sortedDelete( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + 0, + key.length); + } + + public void sortedDelete( + final NativeBytesHandle nameHandle, final byte[] key, int keyOffset, int keyLength) + throws KVDKException { + sortedDelete( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + keyOffset, + keyLength); + } + + public Iterator sortedIteratorCreate(final NativeBytesHandle nameHandle) throws KVDKException { + long iteratorHandle = + sortedIteratorCreate( + nativeHandle_, nameHandle.getNativeHandle(), nameHandle.getLength()); + + return new Iterator(iteratorHandle, nativeHandle_); + } + + public void sortedIteratorRelease(final Iterator iterator) throws KVDKException { + iterator.close(); + } + + public WriteBatch writeBatchCreate() { + long batchHandle = writeBatchCreate(nativeHandle_); + return new WriteBatch(batchHandle); + } + + public void batchWrite(WriteBatch batch) throws KVDKException { + batchWrite(nativeHandle_, batch.getNativeHandle()); + } + + // Native methods + @Override + protected final native void closeInternal(long handle); + + private static native long open(final String path, final long cfg_handle) throws KVDKException; + + private native void put( + long handle, + byte[] key, + int keyOffset, + int keyLength, + byte[] value, + int valueOffset, + int valueLength, + long ttl, + boolean updateTtlIfExisted); + + private native void expire( + long handle, byte[] key, int keyOffset, int keyLength, long ttlInMillis); + + private native void expire(long handle, long nameHandle, int nameLenth, long ttlInMillis); + + private native byte[] get(long handle, byte[] key, int keyOffset, int keyLength); + + private native void delete(long handle, byte[] key, int keyOffset, int keyLength); + + private native void sortedCreate(long engineHandle, long nameHandle, int nameLenth); + + private native void sortedDestroy(long engineHandle, long nameHandle, int nameLenth); + + private native long sortedSize(long engineHandle, long nameHandle, int nameLenth); + + private native void sortedPut( + long engineHandle, + long nameHandle, + int nameLenth, + byte[] key, + int keyOffset, + int keyLength, + byte[] value, + int valueOffset, + int valueLength); + + private native byte[] sortedGet( + long engineHandle, + long nameHandle, + int nameLenth, + byte[] key, + int keyOffset, + int keyLength); + + private native void sortedDelete( + long engineHandle, + long nameHandle, + int nameLenth, + byte[] key, + int keyOffset, + int keyLength); + + private native long sortedIteratorCreate(long engineHandle, long nameHandle, int nameLenth); + + private native long writeBatchCreate(long handle); + + private native void batchWrite(long engineHandle, long batchHandle); + + private enum LibraryState { + NOT_LOADED, + LOADING, + LOADED + } + + /** + * Loads the necessary library files. Calling this method twice will have no effect. By default + * the method extracts the shared library for loading at java.io.tmpdir, however, you can + * override this temporary location by setting the environment variable KVDK_SHARED_LIB_DIR. + */ + public static void loadLibrary() { + if (libraryLoaded.get() == LibraryState.LOADED) { + return; + } + + if (libraryLoaded.compareAndSet(LibraryState.NOT_LOADED, LibraryState.LOADING)) { + final String tmpDir = System.getenv("KVDK_SHARED_LIB_DIR"); + + try { + NativeLibraryLoader.getInstance().loadLibrary(tmpDir); + } catch (final IOException e) { + libraryLoaded.set(LibraryState.NOT_LOADED); + throw new RuntimeException("Unable to load the KVDK shared library", e); + } + + libraryLoaded.set(LibraryState.LOADED); + return; + } + + while (libraryLoaded.get() == LibraryState.LOADING) { + try { + Thread.sleep(10); + } catch (final InterruptedException e) { + // ignore + } + } + } + + /** + * Tries to load the necessary library files from the given list of directories. + * + * @param paths a list of strings where each describes a directory of a library. + */ + public static void loadLibrary(final List paths) { + if (libraryLoaded.get() == LibraryState.LOADED) { + return; + } + + if (libraryLoaded.compareAndSet(LibraryState.NOT_LOADED, LibraryState.LOADING)) { + boolean success = false; + UnsatisfiedLinkError err = null; + for (final String path : paths) { + try { + System.load(path + "/" + atomicLibraryFileName); + System.load(path + "/" + stdLibraryFileName); + System.load(path + "/" + jniLibraryFileName); + success = true; + break; + } catch (final UnsatisfiedLinkError e) { + err = e; + } + } + if (!success) { + libraryLoaded.set(LibraryState.NOT_LOADED); + throw err; + } + + libraryLoaded.set(LibraryState.LOADED); + return; + } + + while (libraryLoaded.get() == LibraryState.LOADING) { + try { + Thread.sleep(10); + } catch (final InterruptedException e) { + // ignore + } + } + } +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/Iterator.java b/volatile/java/src/main/java/io/pmem/kvdk/Iterator.java new file mode 100644 index 00000000..b0896cf3 --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/Iterator.java @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +/** Iterator to Key-Values in KVDK. */ +public class Iterator extends KVDKObject { + static { + Engine.loadLibrary(); + } + + private final long engineHandle; + + protected Iterator(final long iteratorHandle, final long engineHandle) { + super(iteratorHandle); + this.engineHandle = engineHandle; + } + + @Override + protected void closeInternal(long handle) { + closeInternal(handle, engineHandle); + } + + public void seek(final byte[] key) { + seek(nativeHandle_, key, 0, key.length); + } + + public void seek(final byte[] key, final int keyOffset, int keyLength) { + seek(nativeHandle_, key, keyOffset, keyLength); + } + + public void seekToFirst() { + seekToFirst(nativeHandle_); + } + + public void seekToLast() { + seekToLast(nativeHandle_); + } + + public boolean isValid() { + return isValid(nativeHandle_); + } + + public void next() { + next(nativeHandle_); + } + + public void prev() { + prev(nativeHandle_); + } + + public byte[] key() { + return key(nativeHandle_); + } + + public byte[] value() { + return value(nativeHandle_); + } + + // Native methods + protected native void closeInternal(long iteratorHandle, long engineHandle); + + private native void seek(long handle, byte[] key, int keyOffset, int keyLength); + + private native void seekToFirst(long handle); + + private native void seekToLast(long handle); + + private native boolean isValid(long handle); + + private native void next(long handle); + + private native void prev(long handle); + + private native byte[] key(long handle); + + private native byte[] value(long handle); +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/KVDKException.java b/volatile/java/src/main/java/io/pmem/kvdk/KVDKException.java new file mode 100644 index 00000000..a9b9e03e --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/KVDKException.java @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +/** Exception class defined for KVDK. */ +public class KVDKException extends Exception { + private final Status status; + + public KVDKException(final String msg) { + this(msg, null); + } + + public KVDKException(final String msg, final Status status) { + super(msg); + this.status = status; + } + + public KVDKException(final Status status) { + super(status.getMessage() != null ? status.getMessage() : status.getCode().name()); + this.status = status; + } + + public Status getStatus() { + return status; + } +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/KVDKObject.java b/volatile/java/src/main/java/io/pmem/kvdk/KVDKObject.java new file mode 100644 index 00000000..032536fd --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/KVDKObject.java @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +/** Base calss for KVDK object that hold a reference to native C++ object. */ +public abstract class KVDKObject extends AbstractNativeReference { + /** An immutable reference to the value of the C++ pointer. */ + protected final long nativeHandle_; + + protected KVDKObject(final long nativeHandle) { + super(true); + this.nativeHandle_ = nativeHandle; + } + + @Override + protected void closeInternal() { + closeInternal(nativeHandle_); + } + + protected abstract void closeInternal(final long handle); + + public long getNativeHandle() { + return nativeHandle_; + } +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/NativeBytesHandle.java b/volatile/java/src/main/java/io/pmem/kvdk/NativeBytesHandle.java new file mode 100644 index 00000000..7f1d0200 --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/NativeBytesHandle.java @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +/** + * This class is used to handle reference to native string view. Holding native string view avoids + * repetitive coversion of bytes from Java to C++. + */ +public class NativeBytesHandle extends KVDKObject { + static { + Engine.loadLibrary(); + } + + private final int length; + private final byte[] bytes; + + public NativeBytesHandle(final byte[] bytes) { + super(newNativeBytes(bytes)); + this.length = bytes.length; + this.bytes = bytes; + } + + public int getLength() { + return length; + } + + public byte[] getBytes() { + return bytes; + } + + // Native methods + private static native long newNativeBytes(byte[] bytes); + + @Override + protected native void closeInternal(long handle); +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/NativeLibraryLoader.java b/volatile/java/src/main/java/io/pmem/kvdk/NativeLibraryLoader.java new file mode 100644 index 00000000..a9318523 --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/NativeLibraryLoader.java @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +/** + * This class is used to load the KVDK shared library from system or jar. When loading from jar, the + * shared library is extracted to a temp folder and loaded from there. + */ +public class NativeLibraryLoader { + private static final NativeLibraryLoader instance = new NativeLibraryLoader(); + private static boolean initialized = false; + + private static final String jniLibraryName = "kvdkjni"; + + private static final String[][] requiredLibraries = { + {"libatomic.so.1", "libatomic", ".so.1"}, + {"libstdc++.so.6", "libstdc++", ".so.6"}, + {"libkvdkjni.so", "libkvdkjni", ".so"} + }; + + public static NativeLibraryLoader getInstance() { + return instance; + } + + public synchronized void loadLibrary(final String tmpDir) throws IOException { + try { + // try system dynamic library + System.loadLibrary(jniLibraryName); + return; + } catch (final UnsatisfiedLinkError ule) { + // ignore - then try from jar + } + + // try jar + loadLibraryFromJar(tmpDir); + } + + void loadLibraryFromJar(final String tmpDir) throws IOException { + if (!initialized) { + for (int i = 0; i < requiredLibraries.length; i++) { + String[] libraryInfo = requiredLibraries[i]; + System.load( + loadLibraryFromJarToTemp( + tmpDir, libraryInfo[0], libraryInfo[1], libraryInfo[2]) + .getAbsolutePath()); + } + + initialized = true; + } + } + + File loadLibraryFromJarToTemp( + final String tmpDir, + String libraryFileName, + String tempFilePrefix, + String tempFileSuffix) + throws IOException { + InputStream is = null; + try { + // attempt to look up the static library in the jar file + is = getClass().getClassLoader().getResourceAsStream(libraryFileName); + + if (is == null) { + throw new RuntimeException(libraryFileName + " was not found inside JAR."); + } + + // create a temporary file to copy the library to + final File temp; + if (tmpDir == null || tmpDir.isEmpty()) { + temp = File.createTempFile(tempFilePrefix, tempFileSuffix); + } else { + final File parentDir = new File(tmpDir); + if (!parentDir.exists()) { + throw new RuntimeException( + "Directory: " + parentDir.getAbsolutePath() + " does not exist!"); + } + temp = new File(parentDir, libraryFileName); + if (temp.exists() && !temp.delete()) { + throw new RuntimeException( + "File: " + + temp.getAbsolutePath() + + " already exists and cannot be removed."); + } + if (!temp.createNewFile()) { + throw new RuntimeException( + "File: " + temp.getAbsolutePath() + " could not be created."); + } + } + if (!temp.exists()) { + throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); + } else { + temp.deleteOnExit(); + } + + // copy the library from the Jar file to the temp destination + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // return the temporary library file + return temp; + + } finally { + if (is != null) { + is.close(); + } + } + } + + /** Private constructor to disallow instantiation */ + private NativeLibraryLoader() {} +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/Status.java b/volatile/java/src/main/java/io/pmem/kvdk/Status.java new file mode 100644 index 00000000..2836c1ee --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/Status.java @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import java.util.Objects; + +/** Java class to represent the Status enum in C++ KVDK. */ +public class Status { + private final Code code; + private final String message; + + public Status(final Code code, final String message) { + this.code = code; + this.message = message; + } + + protected Status(final byte code) { + this(code, null); + } + + protected Status(final byte code, final String message) { + this.code = Code.getCode(code); + this.message = message; + } + + public Code getCode() { + return code; + } + + public String getMessage() { + return message; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return code == status.code && Objects.equals(message, status.message); + } + + @Override + public int hashCode() { + return Objects.hash(code, message); + } + + // should stay in sync with include/kvdk/types.h:KVDKStatus and + // java/kvdkjni/kvdkjni.h:toJavaStatusCode + public enum Code { + Ok((byte) 0x0), + NotFound((byte) 0x1), + Outdated((byte) 0x2), + WrongType((byte) 0x3), + Existed((byte) 0x4), + OperationFail((byte) 0x5), + OutOfRange((byte) 0x6), + MemoryOverflow((byte) 0x7), + NotSupported((byte) 0x8), + InvalidBatchSize((byte) 0x9), + InvalidDataSize((byte) 0xA), + InvalidArgument((byte) 0xB), + IOError((byte) 0xC), + InvalidConfiguration((byte) 0xD), + Fail((byte) 0xE), + Abort((byte) 0xF), + Undefined((byte) 0x7F); + + private final byte value; + + Code(final byte value) { + this.value = value; + } + + public static Code getCode(final byte value) { + for (final Code code : Code.values()) { + if (code.value == value) { + return code; + } + } + throw new IllegalArgumentException("Illegal value provided for Code (" + value + ")."); + } + + /** + * Returns the byte value of the enumerations value. + * + * @return byte representation + */ + public byte getValue() { + return value; + } + } +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/WriteBatch.java b/volatile/java/src/main/java/io/pmem/kvdk/WriteBatch.java new file mode 100644 index 00000000..58d7a903 --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/WriteBatch.java @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +/** A WriteBatch can be used to prepare multiple operations before a commit. */ +public class WriteBatch extends KVDKObject { + static { + Engine.loadLibrary(); + } + + protected WriteBatch(final long handle) { + super(handle); + } + + public void stringPut(final byte[] key, final byte[] value) { + stringPut(nativeHandle_, key, 0, key.length, value, 0, value.length); + } + + public void stringPut( + final byte[] key, + int keyOffset, + int keyLength, + final byte[] value, + int valueOffset, + int valueLength) { + stringPut(nativeHandle_, key, keyOffset, keyLength, value, valueOffset, valueLength); + } + + public void stringDelete(final byte[] key) { + stringDelete(nativeHandle_, key, 0, key.length); + } + + public void stringDelete(final byte[] key, int keyOffset, int keyLength) { + stringDelete(nativeHandle_, key, keyOffset, keyLength); + } + + public void sortedPut( + final NativeBytesHandle nameHandle, final byte[] key, final byte[] value) { + sortedPut( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + 0, + key.length, + value, + 0, + value.length); + } + + public void sortedPut( + final NativeBytesHandle nameHandle, + final byte[] key, + int keyOffset, + int keyLength, + final byte[] value, + int valueOffset, + int valueLength) { + sortedPut( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + keyOffset, + keyLength, + value, + valueOffset, + valueLength); + } + + public void sortedDelete(final NativeBytesHandle nameHandle, final byte[] key) { + sortedDelete( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + 0, + key.length); + } + + public void sortedDelete( + final NativeBytesHandle nameHandle, final byte[] key, int keyOffset, int keyLength) { + sortedDelete( + nativeHandle_, + nameHandle.getNativeHandle(), + nameHandle.getLength(), + key, + keyOffset, + keyLength); + } + + public void clear() { + clear(nativeHandle_); + } + + public long size() { + return size(nativeHandle_); + } + + // Native methods + @Override + protected final native void closeInternal(long handle); + + private native void stringPut( + long handle, + byte[] key, + int keyOffset, + int keyLength, + byte[] value, + int valueOffset, + int valueLength); + + private native void stringDelete(long handle, byte[] key, int keyOffset, int keyLength); + + private native void sortedPut( + long engineHandle, + long nameHandle, + int nameLenth, + byte[] key, + int keyOffset, + int keyLength, + byte[] value, + int valueOffset, + int valueLength); + + private native void sortedDelete( + long engineHandle, + long nameHandle, + int nameLenth, + byte[] key, + int keyOffset, + int keyLength); + + private native void clear(long handle); + + private native long size(long handle); +} diff --git a/volatile/java/src/main/java/io/pmem/kvdk/WriteOptions.java b/volatile/java/src/main/java/io/pmem/kvdk/WriteOptions.java new file mode 100644 index 00000000..b93f20bd --- /dev/null +++ b/volatile/java/src/main/java/io/pmem/kvdk/WriteOptions.java @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +/** Options of expiration */ +public class WriteOptions { + private long ttlInMillis; + private boolean updateTtlIfExisted; + + public WriteOptions() { + this(Long.MAX_VALUE, true); + } + + public WriteOptions(long ttlInMillis, boolean updateTtlIfExisted) { + this.ttlInMillis = ttlInMillis; + this.updateTtlIfExisted = updateTtlIfExisted; + } + + public long getTtlInMillis() { + return ttlInMillis; + } + + public boolean isUpdateTtlIfExisted() { + return updateTtlIfExisted; + } +} diff --git a/volatile/java/src/test/java/io/pmem/kvdk/ConfigsTest.java b/volatile/java/src/test/java/io/pmem/kvdk/ConfigsTest.java new file mode 100644 index 00000000..9fbf157a --- /dev/null +++ b/volatile/java/src/test/java/io/pmem/kvdk/ConfigsTest.java @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import org.junit.Test; + +public class ConfigsTest { + @Test + public void testCreateConfigs() { + Configs configs = new Configs(); + + configs.setMaxAccessThreads(8); + + configs.close(); + } +} diff --git a/volatile/java/src/test/java/io/pmem/kvdk/EngineTest.java b/volatile/java/src/test/java/io/pmem/kvdk/EngineTest.java new file mode 100644 index 00000000..ea6cab42 --- /dev/null +++ b/volatile/java/src/test/java/io/pmem/kvdk/EngineTest.java @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import static org.junit.Assert.assertEquals; + +import io.pmem.kvdk.Status.Code; +import org.junit.Test; + +public class EngineTest extends EngineTestBase { + @Test + public void testOpenAndCloseKVDKEngine() {} + + @Test + public void testAnonymousCollection() throws KVDKException { + String key = "key1"; + String value1 = "value1"; + + // put + kvdkEngine.put(key.getBytes(), value1.getBytes()); + assertEquals(value1, new String(kvdkEngine.get(key.getBytes()))); + + // rewrite + String value2 = "value2"; + kvdkEngine.put(key.getBytes(), value2.getBytes()); + assertEquals(value2, new String(kvdkEngine.get(key.getBytes()))); + + // delete + kvdkEngine.delete(key.getBytes()); + assertEquals(null, kvdkEngine.get(key.getBytes())); + + // delete nonexistent key: OK + kvdkEngine.delete(key.getBytes()); + + // put special characters + String value3 = "value\0with_zero"; + kvdkEngine.put(key.getBytes(), value3.getBytes()); + assertEquals(value3, new String(kvdkEngine.get(key.getBytes()))); + } + + @Test + public void testSortedCollection() throws KVDKException { + String name = "collection\0\nname"; + NativeBytesHandle nameHandle = new NativeBytesHandle(name.getBytes()); + + // create + kvdkEngine.sortedCreate(nameHandle); + + // duplicate creation: Not OK + try { + kvdkEngine.sortedCreate(nameHandle); + } catch (KVDKException ex) { + // should be Existed + assertEquals(ex.getStatus().getCode(), Code.Existed); + } + + // destroy + kvdkEngine.sortedDestroy(nameHandle); + + // create again: OK + kvdkEngine.sortedCreate(nameHandle); + + String key = "key\01"; + String value = "value\01"; + + // put + kvdkEngine.sortedPut(nameHandle, key.getBytes(), value.getBytes()); + + // size + assertEquals(1, kvdkEngine.sortedSize(nameHandle)); + + // get + assertEquals(value, new String(kvdkEngine.sortedGet(nameHandle, key.getBytes()))); + + // delete + kvdkEngine.sortedDelete(nameHandle, key.getBytes()); + assertEquals(null, kvdkEngine.sortedGet(nameHandle, key.getBytes())); + + // size + assertEquals(0, kvdkEngine.sortedSize(nameHandle)); + + // delete nonexistent key: OK + kvdkEngine.sortedDelete(nameHandle, key.getBytes()); + + // destroy + kvdkEngine.sortedDestroy(nameHandle); + + // destroy destroyed sorted collection: OK + kvdkEngine.sortedDestroy(nameHandle); + + // delete on destroyed sorted collection: OK + kvdkEngine.sortedDelete(nameHandle, key.getBytes()); + + // put on destroyed sorted collection: Not OK + try { + kvdkEngine.sortedPut(nameHandle, key.getBytes(), value.getBytes()); + } catch (KVDKException ex) { + // should be NotFound + assertEquals(ex.getStatus().getCode(), Code.NotFound); + } + + // size on destroyed sorted collection: Not OK + try { + kvdkEngine.sortedSize(nameHandle); + } catch (KVDKException ex) { + // should be NotFound + assertEquals(ex.getStatus().getCode(), Code.NotFound); + } + + // close name handle + nameHandle.close(); + } + + @Test + public void testExpiration() throws KVDKException, InterruptedException { + String key1 = "key1"; + String value1 = "value1"; + + // put + kvdkEngine.put(key1.getBytes(), value1.getBytes()); + assertEquals(value1, new String(kvdkEngine.get(key1.getBytes()))); + + // expire + kvdkEngine.expire(key1.getBytes(), 199); + Thread.sleep(200); + + // check + assertEquals(null, kvdkEngine.get(key1.getBytes())); + } + + @Test + public void testExpirationWithWriteOptions() throws KVDKException, InterruptedException { + String key1 = "key1"; + String value1 = "value1"; + + WriteOptions writeOptions = new WriteOptions(199, true); + + // put + kvdkEngine.put(key1.getBytes(), value1.getBytes(), writeOptions); + assertEquals(value1, new String(kvdkEngine.get(key1.getBytes()))); + + Thread.sleep(200); + + // check + assertEquals(null, kvdkEngine.get(key1.getBytes())); + } +} diff --git a/volatile/java/src/test/java/io/pmem/kvdk/EngineTestBase.java b/volatile/java/src/test/java/io/pmem/kvdk/EngineTestBase.java new file mode 100644 index 00000000..3790d85c --- /dev/null +++ b/volatile/java/src/test/java/io/pmem/kvdk/EngineTestBase.java @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import java.io.File; +import java.io.IOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +public class EngineTestBase { + protected Engine kvdkEngine; + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void init() throws KVDKException, IOException { + File tempDir = folder.newFolder(); + String enginePath = tempDir.getAbsolutePath(); + + Configs engineConfigs = new Configs(); + engineConfigs.setHashBucketNum(1L << 10); + engineConfigs.setMaxAccessThreads(4); + + kvdkEngine = Engine.open(enginePath, engineConfigs); + engineConfigs.close(); + } + + @After + public void teardown() { + kvdkEngine.close(); + } +} diff --git a/volatile/java/src/test/java/io/pmem/kvdk/IteratorTest.java b/volatile/java/src/test/java/io/pmem/kvdk/IteratorTest.java new file mode 100644 index 00000000..a5e2a924 --- /dev/null +++ b/volatile/java/src/test/java/io/pmem/kvdk/IteratorTest.java @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class IteratorTest extends EngineTestBase { + + @Test + public void testSortedCollectionIterator() throws KVDKException { + String name = "collection\0\nname"; + NativeBytesHandle nameHandle = new NativeBytesHandle(name.getBytes()); + + // create + kvdkEngine.sortedCreate(nameHandle); + + String key1 = "key\01"; + String key2 = "key\02"; + String key3 = "key\03"; + + String value1 = "value\03"; + String value2 = "value\02"; + String value3 = "value\01"; + + // disordered put + kvdkEngine.sortedPut(nameHandle, key3.getBytes(), value3.getBytes()); + kvdkEngine.sortedPut(nameHandle, key1.getBytes(), value1.getBytes()); + kvdkEngine.sortedPut(nameHandle, key2.getBytes(), value2.getBytes()); + + // create iterator + Iterator iter = kvdkEngine.sortedIteratorCreate(nameHandle); + assertFalse(iter.isValid()); + + // seekToFirst + iter.seekToFirst(); + assertTrue(iter.isValid()); + assertEquals(key1, new String(iter.key())); + assertEquals(value1, new String(iter.value())); + + // next + iter.next(); + assertTrue(iter.isValid()); + assertEquals(key2, new String(iter.key())); + assertEquals(value2, new String(iter.value())); + + iter.next(); + assertTrue(iter.isValid()); + assertEquals(key3, new String(iter.key())); + assertEquals(value3, new String(iter.value())); + + iter.next(); + assertFalse(iter.isValid()); + + // seekToLast + iter.seekToLast(); + assertTrue(iter.isValid()); + assertEquals(key3, new String(iter.key())); + assertEquals(value3, new String(iter.value())); + + // prev + iter.prev(); + assertTrue(iter.isValid()); + assertEquals(key2, new String(iter.key())); + assertEquals(value2, new String(iter.value())); + + iter.prev(); + assertTrue(iter.isValid()); + assertEquals(key1, new String(iter.key())); + assertEquals(value1, new String(iter.value())); + + iter.prev(); + assertFalse(iter.isValid()); + + // close iterator + iter.close(); + + // destroy sorted collection + kvdkEngine.sortedDestroy(nameHandle); + + // close name handle + nameHandle.close(); + } +} diff --git a/volatile/java/src/test/java/io/pmem/kvdk/WriteBatchTest.java b/volatile/java/src/test/java/io/pmem/kvdk/WriteBatchTest.java new file mode 100644 index 00000000..26c41e4b --- /dev/null +++ b/volatile/java/src/test/java/io/pmem/kvdk/WriteBatchTest.java @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +package io.pmem.kvdk; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +public class WriteBatchTest extends EngineTestBase { + @Test + public void testWriteBatchString() throws KVDKException { + String key1 = "key1"; + String key2 = "key2"; + String value1 = "value1"; + String value2 = "value2"; + + WriteBatch batch = kvdkEngine.writeBatchCreate(); + batch.stringPut(key1.getBytes(), value1.getBytes()); + batch.stringPut(key1.getBytes(), value2.getBytes()); + batch.stringPut(key2.getBytes(), value2.getBytes()); + batch.stringDelete(key2.getBytes()); + + // If the batch is successfully written, there should be only key1-value2 in + // anonymous global collection. + kvdkEngine.batchWrite(batch); + assertEquals(value2, new String(kvdkEngine.get(key1.getBytes()))); + assertNull(kvdkEngine.get(key2.getBytes())); + } + + @Test + public void testWriteBatchSorted() throws KVDKException { + String name = "collection_name"; + String key1 = "key1"; + String key2 = "key2"; + String value1 = "value1"; + String value2 = "value2"; + + NativeBytesHandle nameHandle = new NativeBytesHandle(name.getBytes()); + + kvdkEngine.sortedCreate(nameHandle); + + WriteBatch batch = kvdkEngine.writeBatchCreate(); + batch.sortedPut(nameHandle, key1.getBytes(), value1.getBytes()); + batch.sortedPut(nameHandle, key1.getBytes(), value2.getBytes()); + batch.sortedPut(nameHandle, key2.getBytes(), value2.getBytes()); + batch.sortedDelete(nameHandle, key2.getBytes()); + + // If the batch is successfully written, there should be only key1-value2 in + // the sorted collection. + kvdkEngine.batchWrite(batch); + assertEquals(1, kvdkEngine.sortedSize(nameHandle)); + assertEquals(value2, new String(kvdkEngine.sortedGet(nameHandle, key1.getBytes()))); + assertNull(kvdkEngine.sortedGet(nameHandle, key2.getBytes())); + + batch.close(); + nameHandle.close(); + } +} diff --git a/volatile/kvdk.pc.in b/volatile/kvdk.pc.in new file mode 100644 index 00000000..53c6c512 --- /dev/null +++ b/volatile/kvdk.pc.in @@ -0,0 +1,12 @@ +version=@VERSION@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ +includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: kvdk +Description: kvdk - key-value store for Persistent Memory +Version: ${version} +URL: https://github.com/pmem/kvdk +Requires.private: @PKG_CONFIG_REQUIRES@ +Libs: -L${libdir} -lengine +Cflags: -I${includedir} diff --git a/volatile/scripts/benchmark_impl.py b/volatile/scripts/benchmark_impl.py new file mode 100644 index 00000000..30a009ee --- /dev/null +++ b/volatile/scripts/benchmark_impl.py @@ -0,0 +1,184 @@ +import os +import datetime +import sys +import datetime +import git +from select import select + +from git import repo + + +def __fill(exec, shared_para, data_type, report_path): + new_para = shared_para + " -fill=1 -type={}".format(data_type) + report = report_path + "_fill" + print("Fill {}".format(data_type)) + cmd = "{0} {1} > {2}".format(exec, new_para, report) + print(cmd) + os.system(cmd) + + +def read_random(exec, shared_para, data_type, report_path, num_operations): + new_para = shared_para + \ + " -fill=0 -type={} -read_ratio=1 -num_operations={}".format(data_type, num_operations) + report = report_path + ("read_random" if (data_type != "list") else "pop") + print("Read random {}".format(data_type)) + cmd = "{0} {1} > {2}".format(exec, new_para, report) + print(cmd) + os.system(cmd) + + +def insert_random(exec, shared_para, data_type, report_path, num_operations): + new_para = shared_para + \ + " -fill=0 -type={} -read_ratio=0 -existing_keys_ratio=0 -num_operations={}".format(data_type, num_operations) + report = report_path + ("insert_random" if (data_type != "list") else "push") + print("Insert random {}".format(data_type)) + cmd = "{0} {1} > {2}".format(exec, new_para, report) + print(cmd) + os.system(cmd) + + +def batch_insert_random(exec, shared_para, data_type, report_path, num_operations): + if data_type != "string" and data_type != "sorted" and data_type != "blackhole" and data_type != "hash": + return + new_para = shared_para + \ + " -fill=0 -type={} -read_ratio=0 -batch_size=100"\ + " -existing_keys_ratio=0 -num_operations={}".format(data_type, num_operations) + report = report_path + "batch_insert_random" + print("Batch insert random {}".format(data_type)) + cmd = "{0} {1} > {2}".format(exec, new_para, report) + print(cmd) + os.system(cmd) + + +def update_random(exec, shared_para, data_type, report_path, num_operations): + if data_type == "list": + return + new_para = shared_para + \ + " -fill=0 -type={} -read_ratio=0 -num_operations={}".format(data_type, num_operations) + report = report_path + "update_random" + print("Update random {}".format(data_type)) + cmd = "{0} {1} > {2}".format(exec, new_para, report) + print(cmd) + os.system(cmd) + + +def read_write_random(exec, shared_para, data_type, report_path, num_operations): + ratio = 0.9 + if data_type == "list": + ratio = 0.5 + new_para = shared_para + \ + " -fill=0 -type={} -read_ratio={} -num_operations={}".format(data_type, ratio, num_operations) + report = report_path + ("read_write_random" if (data_type != "list") else "pushpop") + print("Read write random {}".format(data_type)) + cmd = "{0} {1} > {2}".format(exec, new_para, report) + print(cmd) + os.system(cmd) + + +def range_scan(exec, shared_para, data_type, report_path, num_operations): + if data_type == "string" or data_type == "list": + return + new_para = shared_para + \ + " -fill=0 -type={} -read_ratio=1 -scan=1 -num_operations={}".format(data_type, num_operations) + report = report_path + "range_scan" + print("Range scan {}".format(data_type)) + cmd = "{0} {1} > {2}".format(exec, new_para, report) + print(cmd) + os.system(cmd) + + +def confirm(dir): + timeout = 60 + print("Instance path : {}, it will be removed and recreated, confirm? (y/n) (Automatically confirm in {} seconds)".format(dir, timeout)) + rlist, _, _ = select([sys.stdin], [], [], timeout) + y = 'n' + if rlist: + y = sys.stdin.readline() + else: + print("Automatically confirmed after {} seconds!".format(timeout)) + y = 'y' + if y != 'y' and y != 'y\n': + exit(1) + + +def run_benchmark( + data_type, + exec, + pmem_path, + pmem_size, + populate_on_fill, + n_thread, + num_collection, + timeout, + key_distribution, + value_size, + value_size_distribution, + benchmarks, +): + confirm(pmem_path) + os.system("rm -rf {0}".format(pmem_path)) + + # calculate num kv to fill + header_sz = 24 if (key_distribution == 'string') else 40 + k_sz = 8 + avg_v_sz = value_size if (value_size_distribution == 'constant') else (value_size // 2) + avg_kv_size = (header_sz + k_sz + avg_v_sz + 63) // 64 * 64 + + # 1/4 for fill, 1/4 for insert, 1/4 for batch_write + num_kv = pmem_size // 4 // avg_kv_size + + # create report dir + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M") + git_hash = git.Repo(search_parent_directories=True).head.object.hexsha + report_path = "./results/commit-{0}/data_type-{1}-key_dist-{2}/vsize-{3}-vsize_dist-{4}-threads-{5}-collections-{6}-time-{7}/".format( + git_hash[0:8], + data_type, + key_distribution, + value_size, + value_size_distribution, + n_thread, + num_collection, + timestamp) + os.system("mkdir -p {}".format(report_path)) + + # run benchmarks + print("Run benchmarks for data type :{}, value size distribution: {}".format( + data_type, value_size_distribution)) + shared_para = \ + "-path={0} "\ + "-space={1} "\ + "-populate={2} "\ + "-num_kv={3} "\ + "-threads={4} "\ + "-max_access_threads={5} "\ + "-num_collection={6} "\ + "-timeout={7} "\ + "-value_size={8} "\ + "-value_size_distribution={9} "\ + "-key_distribution={10} ".format( + pmem_path, + pmem_size, + populate_on_fill, + num_kv, + n_thread, + n_thread, + num_collection, + timeout, + value_size, + value_size_distribution, + key_distribution) + # we always fill data before run benchmarks + __fill(exec, shared_para, data_type, report_path) + for benchmark in benchmarks: + if (benchmark == read_random) \ + or (benchmark == update_random) \ + or (benchmark == range_scan) \ + or (data_type == "blackhole") : + num_operations = 1024 * 1024 * 1024 * 10 + else: + # __fill, insert_random, batch_insert_random, read_write_random + num_operations = num_kv + + benchmark(exec, shared_para, data_type, report_path, num_operations) + + os.system("rm -rf {0}".format(pmem_path)) diff --git a/volatile/scripts/clang_format.sh b/volatile/scripts/clang_format.sh new file mode 100755 index 00000000..d4664613 --- /dev/null +++ b/volatile/scripts/clang_format.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +src_dirs=( benchmark engine examples include tests java ) +file_types=( h hpp c cpp cc ) + +for dir in "${src_dirs[@]}" +do + for type in "${file_types[@]}" + do + find "../$dir/" -iname "*.$type" | xargs -I '{}' clang-format-9 -i '{}' + done +done diff --git a/volatile/scripts/clang_tidy.sh b/volatile/scripts/clang_tidy.sh new file mode 100755 index 00000000..8646a08b --- /dev/null +++ b/volatile/scripts/clang_tidy.sh @@ -0,0 +1,19 @@ +#!/bin/ + +includes='-I../include -I../extern' +checks='abseil-*,bugprone-*,cert-*,clang-analyzer-*,cppcoreguidelines-*,-cppcoreguidelines-pro-type-reinterpret-cast,misc-*' + +# Just too much +# find ../ -iname *.cpp -o -iname *.hpp | xargs -I '{}' clang-tidy-13 '{}' -checks=$checks -- $includes + +i=1; +j=$#; +if [ $j -eq 0 ]; then + echo "usage: ./clang_tidy.sh [... ]" +fi +while [ $i -le $j ] +do + clang-tidy-13 $1 -checks=$checks -- $includes + i=$((i + 1)); + shift 1; +done diff --git a/volatile/scripts/cppstyle b/volatile/scripts/cppstyle new file mode 100755 index 00000000..e21a345f --- /dev/null +++ b/volatile/scripts/cppstyle @@ -0,0 +1,44 @@ +#!/usr/bin/perl -w +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017, Intel Corporation + +use strict; +use Text::Diff; + +my $clangfmt = shift or die; +my $mode = shift or die; + +sub check { + my ($file) = @_; + my $original; + my $formatted; + + $formatted = `$clangfmt -style=file "$file"`; + + if ($mode eq 'check') { + local $/=undef; + open FILE, "$file" or die "Couldn't open file: $file"; + binmode FILE; + $original = ; + close FILE; + + my $diff = diff \$original, \$formatted; + + if ($diff ne "") { + print "Style check using $clangfmt for file $file failed\n"; + print $diff; + die "Style check using $clangfmt for file $file failed\n"; + } + } elsif ($mode eq 'format') { + local $/=undef; + open FILE, '>', "$file" or die "Couldn't open file: $file"; + print FILE "$formatted"; + close FILE; + } else { + die 'unknown mode'; + } +} + +foreach(@ARGV) { + check($_) +} diff --git a/volatile/scripts/run_benchmark.py b/volatile/scripts/run_benchmark.py new file mode 100644 index 00000000..ce8990f2 --- /dev/null +++ b/volatile/scripts/run_benchmark.py @@ -0,0 +1,64 @@ +import benchmark_impl +import sys +import itertools + +numanode = 0 +pmem_path = "/mnt/pmem{}/kvdk_benchmark".format(numanode) +bin = "../build/bench" +exec = "numactl --cpunodebind={0} --membind={0} {1}".format(numanode, bin) + +num_thread = 64 +value_sizes = [120] +# constant: value size always be "value_size", +# random: value size uniformly distributed in [1, value_size] +value_size_distributions = ['constant'] +timeout = 30 # For operations other than fill +populate_on_fill = 1 # For fill only +pmem_size = 384 * 1024 * 1024 * 1024 # we need enough space to test insert +num_collection = 16 + +benchmarks = [ + benchmark_impl.batch_insert_random, + benchmark_impl.insert_random, + benchmark_impl.range_scan, + benchmark_impl.read_random, + benchmark_impl.read_write_random, + benchmark_impl.update_random] + +data_types = [] + +if __name__ == "__main__": + usage = 'usage: run_benchmark.py [data type] [key distribution]\n\ + data type can be "string", "sorted", "hash", "list", or "all"\n\ + key distribution can be "random" or "zipf" or "all".' + if len(sys.argv) != 3: + print(usage) + exit(1) + if sys.argv[1] == 'string': + data_types = ['string'] + elif sys.argv[1] == 'sorted': + data_types = ['sorted'] + elif sys.argv[1] == 'hash': + data_types = ['hash'] + elif sys.argv[1] == 'list': + data_types = ['list'] + elif sys.argv[1] == 'blackhole': + data_types = ['blackhole'] + elif sys.argv[1] == 'all': + data_types = ['blackhole', 'string', 'sorted', 'hash', 'list'] + else: + print(usage) + exit(1) + if sys.argv[2] == 'random': + key_distributions = ['random'] + elif sys.argv[2] == 'zipf': + key_distributions = ['zipf'] + elif sys.argv[2] == 'all': + key_distributions = ['random', 'zipf'] + else: + print(usage) + exit(1) + + for [data_type, value_size, vsz_dist, k_dist] in itertools.product(data_types, value_sizes, value_size_distributions, key_distributions): + benchmark_impl.run_benchmark(data_type, exec, pmem_path, pmem_size, populate_on_fill, + num_thread, num_collection, timeout, k_dist, value_size, vsz_dist, benchmarks) diff --git a/volatile/scripts/test_coverage.sh b/volatile/scripts/test_coverage.sh new file mode 100755 index 00000000..85c96699 --- /dev/null +++ b/volatile/scripts/test_coverage.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# The shell is for view test coverage. + +COVERAGE_FILE=test_coverage.info +REPORT_FOLDER=test_coverage_report +./../build/dbtest +lcov --rc lcov_branch_coverage=1 -c -d $(pwd)/../build/ -o ${COVERAGE_FILE}_tmp +lcov --rc lcov_branch_coverage=1 -e ${COVERAGE_FILE}_tmp "*engine*" -o ${COVERAGE_FILE} +genhtml --rc genhtml_branch_coverage=1 ${COVERAGE_FILE} -o ${REPORT_FOLDER} diff --git a/volatile/tests/CMakeLists.txt b/volatile/tests/CMakeLists.txt new file mode 100644 index 00000000..eab028a2 --- /dev/null +++ b/volatile/tests/CMakeLists.txt @@ -0,0 +1,16 @@ +# For kvdk api general test +set(TEST_SOURCES tests.cpp) +add_executable(dbtest ${TEST_SOURCES}) +target_link_libraries(dbtest PUBLIC engine gtest gtest_main) + +# C API test +add_executable(c_api_test + c_api_test_list.cpp + c_api_test_hash.cpp + ) +target_link_libraries(c_api_test PUBLIC engine gtest gtest_main) + +# For stress tests +set(TEST_SOURCES2 stress_test.cpp) +add_executable(dbstress_test ${TEST_SOURCES2}) +target_link_libraries(dbstress_test PUBLIC engine gtest gtest_main) diff --git a/volatile/tests/c_api_test.hpp b/volatile/tests/c_api_test.hpp new file mode 100644 index 00000000..94850eed --- /dev/null +++ b/volatile/tests/c_api_test.hpp @@ -0,0 +1,51 @@ +#pragma once +#ifndef KVDK_C_API_TEST_HPP +#define KVDK_C_API_TEST_HPP + +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ +#include + +#include + +#include "gtest/gtest.h" +#include "kvdk/volatile/engine.h" +#include "test_util.h" + +class EngineCAPITestBase : public testing::Test { + protected: + KVDKEngine* engine{nullptr}; + KVDKConfigs* configs; + const std::string db_path{"/mnt/pmem0/kvdk_c_api_test"}; + + virtual void SetUp() override { + purgeDB(); + configs = KVDKCreateConfigs(); + KVDKSetConfigs(configs, 32, 1024, 1); + ASSERT_EQ(KVDKOpen(db_path.c_str(), configs, stdout, &engine), + KVDKStatus::Ok) + << "Fail to open the KVDK instance"; + } + + virtual void TearDown() { + KVDKCloseEngine(engine); + KVDKDestroyConfigs(configs); + purgeDB(); + } + + void RebootDB() { + KVDKCloseEngine(engine); + ASSERT_EQ(KVDKOpen(db_path.c_str(), configs, stdout, &engine), + KVDKStatus::Ok) + << "Fail to open the KVDK instance"; + } + + private: + void purgeDB() { + std::string cmd = "rm -rf " + db_path + "\n"; + [[gnu::unused]] int _sink = system(cmd.c_str()); + } +}; + +#endif // KVDK_C_API_TEST_HPP diff --git a/volatile/tests/c_api_test_hash.cpp b/volatile/tests/c_api_test_hash.cpp new file mode 100644 index 00000000..c042a081 --- /dev/null +++ b/volatile/tests/c_api_test_hash.cpp @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include "c_api_test.hpp" + +struct FetchAddArgs { + size_t old_val; + size_t new_val; + size_t n; +}; + +int FetchAdd(char const* old_val_data, size_t old_val_len, char** new_val_data, + size_t* new_val_len, void* args) { + FetchAddArgs* fa_args = (FetchAddArgs*)args; + if (old_val_data != nullptr) { + assert(sizeof(size_t) == old_val_len); + fa_args->old_val = *(size_t*)old_val_data; + } else { + fa_args->old_val = 0; + } + fa_args->new_val = fa_args->old_val + fa_args->n; + *new_val_len = sizeof(size_t); + *new_val_data = (char*)&fa_args->new_val; + return KVDK_MODIFY_WRITE; +} + +TEST_F(EngineCAPITestBase, Hash) { + size_t num_threads = 16; + size_t count = 1000; + + std::string key{"Hash"}; + ASSERT_EQ(KVDKHashCreate(engine, key.data(), key.size()), KVDKStatus::Ok); + using umap = std::unordered_map; + std::vector local_copies(num_threads); + std::mutex mu; + + auto HPut = [&](size_t tid) { + umap& local_copy = local_copies[tid]; + for (size_t j = 0; j < count; j++) { + std::string field{std::to_string(tid) + "_" + GetRandomString(10)}; + std::string value{GetRandomString(120)}; + ASSERT_EQ(KVDKHashPut(engine, key.data(), key.size(), field.data(), + field.size(), value.data(), value.size()), + KVDKStatus::Ok); + local_copy[field] = value; + } + }; + + auto HGet = [&](size_t tid) { + umap const& local_copy = local_copies[tid]; + for (auto const& kv : local_copy) { + char* resp_data; + size_t resp_len; + ASSERT_EQ(KVDKHashGet(engine, key.data(), key.size(), kv.first.data(), + kv.first.size(), &resp_data, &resp_len), + KVDKStatus::Ok); + ASSERT_EQ(std::string(resp_data, resp_len), kv.second); + free(resp_data); + } + }; + + auto HDelete = [&](size_t tid) { + umap& local_copy = local_copies[tid]; + std::string sink; + for (size_t i = 0; i < count / 2; i++) { + auto iter = local_copy.begin(); + char* resp_data; + size_t resp_len; + ASSERT_EQ(KVDKHashDelete(engine, key.data(), key.size(), + iter->first.data(), iter->first.size()), + KVDKStatus::Ok); + ASSERT_EQ(KVDKHashGet(engine, key.data(), key.size(), iter->first.data(), + iter->first.size(), &resp_data, &resp_len), + KVDKStatus::NotFound); + local_copy.erase(iter); + } + }; + + auto HashSize = [&](size_t) { + size_t len = 0; + ASSERT_EQ(KVDKHashLength(engine, key.data(), key.size(), &len), + KVDKStatus::Ok); + size_t cnt = 0; + for (size_t tid = 0; tid < num_threads; tid++) { + cnt += local_copies[tid].size(); + } + ASSERT_EQ(len, cnt); + }; + + auto HashIterate = [&](size_t tid) { + umap combined; + for (size_t tid = 0; tid < num_threads; tid++) { + umap const& local_copy = local_copies[tid]; + for (auto const& kv : local_copy) { + combined[kv.first] = kv.second; + } + } + + KVDKHashIterator* iter = + KVDKHashIteratorCreate(engine, key.data(), key.size(), NULL, NULL); + + ASSERT_NE(iter, nullptr); + size_t cnt = 0; + for (KVDKHashIteratorSeekToFirst(iter); KVDKHashIteratorIsValid(iter); + KVDKHashIteratorNext(iter)) { + ++cnt; + char* field_data; + size_t field_len; + char* value_data; + size_t value_len; + KVDKHashIteratorGetKey(iter, &field_data, &field_len); + KVDKHashIteratorGetValue(iter, &value_data, &value_len); + ASSERT_EQ(combined[std::string(field_data, field_len)], + std::string(value_data, value_len)); + free(field_data); + free(value_data); + } + ASSERT_EQ(cnt, combined.size()); + + cnt = 0; + for (KVDKHashIteratorSeekToLast(iter); KVDKHashIteratorIsValid(iter); + KVDKHashIteratorPrev(iter)) { + ++cnt; + char* field_data; + size_t field_len; + char* value_data; + size_t value_len; + KVDKHashIteratorGetKey(iter, &field_data, &field_len); + KVDKHashIteratorGetValue(iter, &value_data, &value_len); + ASSERT_EQ(combined[std::string(field_data, field_len)], + std::string(value_data, value_len)); + free(field_data); + free(value_data); + } + ASSERT_EQ(cnt, combined.size()); + + std::string re_str1{".*"}; + std::string re_str2{std::to_string(tid) + "_.*"}; + KVDKRegex* re1 = KVDKRegexCreate(re_str1.data(), re_str1.size()); + KVDKRegex* re2 = KVDKRegexCreate(re_str2.data(), re_str2.size()); + size_t match_cnt1 = 0; + size_t match_cnt2 = 0; + for (KVDKHashIteratorSeekToFirst(iter); KVDKHashIteratorIsValid(iter); + KVDKHashIteratorNext(iter)) { + match_cnt1 += KVDKHashIteratorMatchKey(iter, re1); + match_cnt2 += KVDKHashIteratorMatchKey(iter, re2); + } + ASSERT_EQ(match_cnt1, combined.size()); + ASSERT_EQ(match_cnt2, local_copies[tid].size()); + KVDKRegexDestroy(re2); + KVDKRegexDestroy(re1); + + KVDKHashIteratorDestroy(engine, iter); + }; + + std::string counter{"counter"}; + auto HashModify = [&](size_t) { + FetchAddArgs args; + args.n = 1; + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(KVDKHashModify(engine, key.data(), key.size(), counter.data(), + counter.size(), FetchAdd, &args, NULL), + KVDKStatus::Ok); + } + }; + + for (size_t i = 0; i < 3; i++) { + RebootDB(); + LaunchNThreads(num_threads, HPut); + LaunchNThreads(num_threads, HGet); + LaunchNThreads(num_threads, HDelete); + LaunchNThreads(num_threads, HashIterate); + LaunchNThreads(num_threads, HashSize); + LaunchNThreads(num_threads, HPut); + LaunchNThreads(num_threads, HGet); + LaunchNThreads(num_threads, HDelete); + LaunchNThreads(num_threads, HashIterate); + LaunchNThreads(num_threads, HashSize); + } + LaunchNThreads(num_threads, HashModify); + char* resp_data; + size_t resp_len; + ASSERT_EQ(KVDKHashGet(engine, key.data(), key.size(), counter.data(), + counter.size(), &resp_data, &resp_len), + KVDKStatus::Ok); + ASSERT_EQ(resp_len, sizeof(size_t)); + ASSERT_EQ(*(size_t*)resp_data, num_threads * count); + free(resp_data); +} diff --git a/volatile/tests/c_api_test_list.cpp b/volatile/tests/c_api_test_list.cpp new file mode 100644 index 00000000..9c04f3e7 --- /dev/null +++ b/volatile/tests/c_api_test_list.cpp @@ -0,0 +1,367 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021-2022 Intel Corporation + */ + +#include +#include +#include + +#include "c_api_test.hpp" + +void ConcatStrings(char const* elem_data, size_t elem_len, void* arg) { + std::string* buffer = static_cast(arg); + buffer->append(elem_data, elem_len); + buffer->append(1, '\n'); +} + +TEST_F(EngineCAPITestBase, List) { + size_t num_threads = 16; + size_t count = 1000; + + std::vector> elems_vec(num_threads); + std::vector list_vec(num_threads); + for (size_t i = 0; i < num_threads; i++) { + list_vec[i] = "List_" + std::to_string(i); + ASSERT_EQ(KVDKListCreate(engine, list_vec[i].data(), list_vec[i].size()), + KVDKStatus::Ok); + for (size_t j = 0; j < count; j++) { + elems_vec[i].push_back(std::to_string(i) + "_" + std::to_string(j)); + } + } + + std::vector> list_copy_vec(num_threads); + + auto ListIteratorGetValue = [&](KVDKListIterator* iter) { + char* value; + size_t sz; + // Read the value at the ListIterator + KVDKListIteratorGetValue(iter, &value, &sz); + std::string ret{value, sz}; + free(value); + return ret; + }; + + auto ListPopFront = [&](std::string const& key) { + char* value; + size_t sz; + KVDKStatus s = + KVDKListPopFront(engine, key.data(), key.size(), &value, &sz); + std::string ret{value, sz}; + free(value); + return std::make_pair(s, ret); + }; + + auto ListPopBack = [&](std::string const& key) { + char* value; + size_t sz; + KVDKStatus s = KVDKListPopBack(engine, key.data(), key.size(), &value, &sz); + std::string ret{value, sz}; + free(value); + return std::make_pair(s, ret); + }; + + auto LPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t sz; + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(KVDKListPushFront(engine, key.data(), key.size(), + elems[j].data(), elems[j].size()), + KVDKStatus::Ok); + list_copy.push_front(elems[j]); + ASSERT_EQ(KVDKListSize(engine, key.data(), key.size(), &sz), + KVDKStatus::Ok); + ASSERT_EQ(sz, list_copy.size()); + } + }; + + auto RPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t sz; + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(KVDKListPushBack(engine, key.data(), key.size(), + elems[j].data(), elems[j].size()), + KVDKStatus::Ok); + list_copy.push_back(elems[j]); + ASSERT_EQ(KVDKListSize(engine, key.data(), key.size(), &sz), + KVDKStatus::Ok); + ASSERT_EQ(sz, list_copy.size()); + } + }; + + auto LPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t len; + for (size_t j = 0; j < count; j++) { + if (list_copy.empty()) { + break; + } + ASSERT_EQ(std::make_pair(KVDKStatus::Ok, list_copy.front()), + ListPopFront(key)); + list_copy.pop_front(); + ASSERT_TRUE((KVDKListSize(engine, key.data(), key.size(), &len) == + KVDKStatus::NotFound && + list_copy.empty()) || + len == list_copy.size()); + } + }; + + auto RPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t len; + for (size_t j = 0; j < count; j++) { + if (list_copy.empty()) { + break; + } + ASSERT_EQ(std::make_pair(KVDKStatus::Ok, list_copy.back()), + ListPopBack(key)); + list_copy.pop_back(); + ASSERT_TRUE((KVDKListSize(engine, key.data(), key.size(), &len) == + KVDKStatus::NotFound && + list_copy.empty()) || + len == list_copy.size()); + } + }; + + auto ListIterate = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + + KVDKListIterator* iter = + KVDKListIteratorCreate(engine, key.data(), key.size(), NULL); + if (iter != nullptr) { + KVDKListIteratorSeekPos(iter, 0); + for (auto iter2 = list_copy.begin(); iter2 != list_copy.end(); iter2++) { + ASSERT_TRUE(KVDKListIteratorIsValid(iter)); + ASSERT_EQ(ListIteratorGetValue(iter), *iter2); + KVDKListIteratorNext(iter); + } + + KVDKListIteratorSeekPos(iter, -1); + for (auto iter2 = list_copy.rbegin(); iter2 != list_copy.rend(); + iter2++) { + ASSERT_TRUE(KVDKListIteratorIsValid(iter)); + ASSERT_EQ(ListIteratorGetValue(iter), *iter2); + KVDKListIteratorPrev(iter); + } + } + KVDKListIteratorDestroy(engine, iter); + }; + + auto ConvertParams = [](std::vector const& elems) { + std::vector elems_data; + std::vector elems_len; + for (auto const& elem : elems) { + elems_data.push_back(elem.data()); + elems_len.push_back(elem.size()); + } + return std::make_pair(elems_data, elems_len); + }; + + auto ListInsertPutRemove = [&](size_t tid) { + auto const& list_name = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t len; + std::string elem; + size_t const insert_pos = 5; + + ASSERT_EQ(KVDKListSize(engine, list_name.data(), list_name.size(), &len), + KVDKStatus::Ok); + ASSERT_GT(len, insert_pos); + + KVDKListIterator* iter = KVDKListIteratorCreate(engine, list_name.data(), + list_name.size(), NULL); + ASSERT_NE(iter, nullptr); + + KVDKListIteratorSeekPos(iter, insert_pos); + auto iter2 = std::next(list_copy.begin(), insert_pos); + auto iter_elem = ListIteratorGetValue(iter); + ASSERT_EQ(iter_elem, *iter2); + + elem = *iter2 + "_before"; + ASSERT_EQ(KVDKListInsertBefore(engine, list_name.data(), list_name.size(), + elem.data(), elem.size(), iter_elem.data(), + iter_elem.size()), + KVDKStatus::Ok); + iter2 = list_copy.insert(iter2, elem); + KVDKListIteratorDestroy(engine, iter); + iter = KVDKListIteratorCreate(engine, list_name.data(), list_name.size(), + NULL); + KVDKListIteratorSeekPos(iter, insert_pos); + ASSERT_EQ(ListIteratorGetValue(iter), *iter2); + + auto replace_pos = insert_pos - 2; + KVDKListIteratorPrev(iter); + KVDKListIteratorPrev(iter); + --iter2; + --iter2; + ASSERT_EQ(ListIteratorGetValue(iter), *iter2); + elem = *iter2 + "_new"; + ASSERT_EQ(KVDKListReplace(engine, list_name.data(), list_name.size(), + replace_pos, elem.data(), elem.size()), + KVDKStatus::Ok); + *iter2 = elem; + KVDKListIteratorDestroy(engine, iter); + iter = KVDKListIteratorCreate(engine, list_name.data(), list_name.size(), + NULL); + KVDKListIteratorSeekPos(iter, replace_pos); + ASSERT_EQ(ListIteratorGetValue(iter), *iter2); + + auto erase_pos = replace_pos - 2; + KVDKListIteratorPrev(iter); + KVDKListIteratorPrev(iter); + --iter2; + --iter2; + ASSERT_EQ(ListIteratorGetValue(iter), *iter2); + char* value; + size_t sz; + ASSERT_EQ(KVDKListErase(engine, list_name.data(), list_name.size(), + erase_pos, &value, &sz), + KVDKStatus::Ok); + iter2 = list_copy.erase(iter2); + KVDKListIteratorDestroy(engine, iter); + iter = KVDKListIteratorCreate(engine, list_name.data(), list_name.size(), + NULL); + KVDKListIteratorSeekPos(iter, erase_pos); + ASSERT_EQ(ListIteratorGetValue(iter), *iter2); + KVDKListIteratorDestroy(engine, iter); + }; + + auto LBatchPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + for (size_t j = 0; j < count; j++) { + list_copy.push_front(elems[j]); + } + auto param = ConvertParams(elems); + ASSERT_EQ(KVDKListBatchPushFront(engine, key.data(), key.size(), + param.first.data(), param.second.data(), + elems.size()), + KVDKStatus::Ok); + size_t sz; + ASSERT_EQ(KVDKListSize(engine, key.data(), key.size(), &sz), + KVDKStatus::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto RBatchPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + for (size_t j = 0; j < count; j++) { + list_copy.push_back(elems[j]); + } + auto param = ConvertParams(elems); + ASSERT_EQ(KVDKListBatchPushBack(engine, key.data(), key.size(), + param.first.data(), param.second.data(), + elems.size()), + KVDKStatus::Ok); + size_t sz; + ASSERT_EQ(KVDKListSize(engine, key.data(), key.size(), &sz), + KVDKStatus::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto LBatchPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + std::string buffer1; + std::string buffer2; + ASSERT_EQ(KVDKListBatchPopFront(engine, key.data(), key.size(), count, + ConcatStrings, &buffer1), + KVDKStatus::Ok); + for (size_t j = 0; j < count && !list_copy.empty(); j++) { + ConcatStrings(list_copy.front().data(), list_copy.front().size(), + &buffer2); + list_copy.pop_front(); + } + ASSERT_EQ(buffer1, buffer2); + size_t sz; + ASSERT_EQ(KVDKListSize(engine, key.data(), key.size(), &sz), + KVDKStatus::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto RBatchPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + std::string buffer1; + std::string buffer2; + ASSERT_EQ(KVDKListBatchPopBack(engine, key.data(), key.size(), count, + ConcatStrings, &buffer1), + KVDKStatus::Ok); + for (size_t j = 0; j < count && !list_copy.empty(); j++) { + ConcatStrings(list_copy.back().data(), list_copy.back().size(), &buffer2); + list_copy.pop_back(); + } + ASSERT_EQ(buffer1, buffer2); + size_t sz; + ASSERT_EQ(KVDKListSize(engine, key.data(), key.size(), &sz), + KVDKStatus::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto RPushLPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + + ASSERT_FALSE(list_copy.empty()); + + auto elem_copy = list_copy.front(); + list_copy.push_back(elem_copy); + list_copy.pop_front(); + + char* elem_data; + size_t elem_len; + ASSERT_EQ(KVDKListMove(engine, key.data(), key.size(), KVDK_LIST_FRONT, + key.data(), key.size(), KVDK_LIST_BACK, &elem_data, + &elem_len), + KVDKStatus::Ok); + ASSERT_EQ(std::string(elem_data, elem_len), elem_copy); + free(elem_data); + + size_t sz; + ASSERT_EQ(KVDKListSize(engine, key.data(), key.size(), &sz), + KVDKStatus::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + for (size_t i = 0; i < 3; i++) { + LaunchNThreads(num_threads, LPop); + LaunchNThreads(num_threads, RPop); + LaunchNThreads(num_threads, LPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LBatchPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RBatchPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LBatchPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RBatchPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LPush); + LaunchNThreads(num_threads, ListIterate); + for (size_t j = 0; j < 100; j++) { + LaunchNThreads(num_threads, ListInsertPutRemove); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPushLPop); + LaunchNThreads(num_threads, ListIterate); + } + RebootDB(); + } +} diff --git a/volatile/tests/stress_test.cpp b/volatile/tests/stress_test.cpp new file mode 100644 index 00000000..933d5b31 --- /dev/null +++ b/volatile/tests/stress_test.cpp @@ -0,0 +1,983 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kvdk/volatile/engine.hpp" +#include "kvdk/volatile/types.hpp" +#include "test_util.h" + +DEFINE_bool( + verbose, false, + "If set true, stress test will print a progress bar for operations."); + +DEFINE_string(path, "/mnt/pmem0/kvdk_stress_test", + "Path of KVDK instance on PMem."); + +using kvdk::StringView; + +namespace kvdk_testing { + +using KeyType = StringView; +using ValueType = StringView; +using CollectionNameType = StringView; + +// Operators are just wrappers of the engine and collection name +// They offer an universal interface for calling KVEngine APIs +class HashesOperator { + kvdk::Engine*& engine; + CollectionNameType collection_name; + + public: + HashesOperator() = delete; + HashesOperator(kvdk::Engine*& e, CollectionNameType cn) + : engine{e}, collection_name{cn} {} + kvdk::Status operator()(KeyType key, std::string* value_got) { + return engine->HashGet(collection_name, key, value_got); + } + kvdk::Status operator()(KeyType key, ValueType value) { + return engine->HashPut(collection_name, key, value); + } + kvdk::Status operator()(KeyType key) { + return engine->HashDelete(collection_name, key); + } +}; + +class SortedOperator { + kvdk::Engine*& engine; + CollectionNameType collection_name; + + public: + SortedOperator() = delete; + SortedOperator(kvdk::Engine*& e, CollectionNameType cn) + : engine{e}, collection_name{cn} {} + kvdk::Status operator()(KeyType key, std::string* value_got) { + return engine->SortedGet(collection_name, key, value_got); + } + kvdk::Status operator()(KeyType key, ValueType value) { + return engine->SortedPut(collection_name, key, value); + } + kvdk::Status operator()(KeyType key) { + return engine->SortedDelete(collection_name, key); + } +}; + +class StringOperator { + kvdk::Engine*& engine; + CollectionNameType collection_name; + + public: + StringOperator() = delete; + // For convenince, introducing empty collection_name for global anonymous + // collection + StringOperator(kvdk::Engine*& e, CollectionNameType cn) + : engine{e}, collection_name{} { + if (cn != collection_name) throw; + } + kvdk::Status operator()(KeyType key, std::string* value_got) { + return engine->Get(key, value_got); + } + kvdk::Status operator()(KeyType key, ValueType value) { + return engine->Put(key, value); + } + kvdk::Status operator()(KeyType key) { return engine->Delete(key); } +}; + +enum class IteratingDirection { Forward, Backward }; + +// A ShadowKVEngine operates on one KVEngine collection, +// including the global anonymous collection(string). +// User should call EvenXSetOddXSet() first to modify KVEngine, +// then call UpdatePossibleStates() to update possible_state to +// keep track of the state of the KVEngine. +template +class ShadowKVEngine { + public: + struct StateAndValue { + enum class State { Existing, Deleted } state; + ValueType value; + + bool operator==(StateAndValue other) { + bool match_state = (state == other.state); + bool match_value = + ((state == State::Existing) && (value == other.value)) || + (state == State::Deleted); + return match_state && match_value; + } + + friend std::ostream& operator<<(std::ostream& out, + StateAndValue const& vstate) { + out << "State: " + << (vstate.state == StateAndValue::State::Deleted ? "Deleted" + : "Existing") + << "\t" + << "Value: " << vstate.value << "\n"; + return out; + } + }; + + struct SingleOp { + enum class OpType { Get, Put, Delete } op; + KeyType key; + ValueType value; // Empty for Delete, expected for Get + + // For printing error message + friend std::ostream& operator<<(std::ostream& out, SingleOp const& sop) { + out << "Op: " + << (sop.op == OpType::Get + ? "Get" + : (sop.op == OpType::Put ? "Put" : "Delete")) + << "\n" + << "Key: " << sop.key << "\n" + << "Value: " << sop.value << "\n"; + return out; + } + }; + + using OperationQueue = std::deque; + using PossibleStates = std::unordered_multimap; + using StagedChanges = std::unordered_map; + + private: + kvdk::Engine*& engine; + CollectionNameType collection_name; + EngineOperator engine_operator; + size_t const n_thread; + // A Key may have multiple possible StateAndValue. + // possible_state keep track of these StateAndValues + PossibleStates possible_state; + std::vector task_queues; + + public: + ShadowKVEngine() = delete; + ShadowKVEngine(kvdk::Engine*& e, CollectionNameType cn, size_t nt) + : engine{e}, + collection_name{cn}, + engine_operator{engine, collection_name}, + n_thread{nt}, + possible_state{}, + task_queues(n_thread) {} + + // Execute task_queues in ShadowKVEngine + // Update possible_state + /// TODO: make this private, put it in operateKVEngine(), which should + /// run multiple threads. + void UpdatePossibleStates() { + std::cout << "[Testing] Updating Engine State" << std::endl; + + // Some keys are overwritten by operations in operateKVEngine(), + // states and values before calling operateKVEngine() are + // no longer possible. + { + ProgressBar pbar{std::cout, "", n_thread, 1, FLAGS_verbose}; + for (size_t tid = 0; tid < n_thread; tid++) { + for (auto const& sop : task_queues[tid]) { + possible_state.erase(sop.key); + } + pbar.Update(tid + 1); + } + } + + // Squash every task queue and merge into possible_state + { + ProgressBar pbar{std::cout, "", n_thread, 1, FLAGS_verbose}; + for (size_t tid = 0; tid < n_thread; tid++) { + StagedChanges squashed_changes{task_queues[tid].size() * 2}; + for (auto const& sop : task_queues[tid]) { + switch (sop.op) { + case SingleOp::OpType::Get: { + // Get will not change the state of any KV + continue; + } + case SingleOp::OpType::Put: { + squashed_changes[sop.key] = + StateAndValue{StateAndValue::State::Existing, sop.value}; + continue; + } + case SingleOp::OpType::Delete: { + squashed_changes[sop.key] = + StateAndValue{StateAndValue::State::Deleted, ValueType{}}; + continue; + } + } + } + for (auto const& kvs : squashed_changes) { + possible_state.emplace(kvs); + } + pbar.Update(tid + 1); + } + } + task_queues.clear(); + task_queues.resize(n_thread); + } + + // Modify KVEngine by Put + void EvenXSetOddXSet(size_t tid, std::vector const& keys, + std::vector const& values) { + task_queues[tid] = generateOperations(keys, values, false); + operateKVEngine(tid, (tid == 0) ? FLAGS_verbose : false); + } + + // Modify KVEngine by Put and Delete + void EvenXSetOddXDelete(size_t tid, std::vector const& keys, + std::vector const& values) { + task_queues[tid] = generateOperations(keys, values, true); + operateKVEngine(tid, (tid == 0) ? FLAGS_verbose : false); + } + + // Check KVEngine by iterating through it. + // Iterated KVs are looked up in possible_state. + template + void CheckIterator(Iterator* iterator, IteratingDirection direction) { + PossibleStates possible_state_copy{possible_state}; + + // Iterating forward or backward. + { + ASSERT_TRUE(iterator != nullptr) << "Invalid Iterator"; + switch (direction) { + case IteratingDirection::Forward: { + std::cout << "[Testing] Iterating forward." << std::endl; + iterator->SeekToFirst(); + break; + } + case IteratingDirection::Backward: { + std::cout << "[Testing] Iterating backward." << std::endl; + iterator->SeekToLast(); + break; + } + } + + ProgressBar pbar{std::cout, "", possible_state.size(), 1000, + FLAGS_verbose}; + while (iterator->Valid()) { + auto key = iterator->Key(); + auto value = iterator->Value(); + + checkState(key, {StateAndValue::State::Existing, value}); + + possible_state_copy.erase(key); + pbar.Update(possible_state.size() - possible_state_copy.size()); + + switch (direction) { + case IteratingDirection::Forward: + iterator->Next(); + break; + case IteratingDirection::Backward: + iterator->Prev(); + break; + } + } + // Remaining kv-pairs in possible_kv_pairs are deleted kv-pairs + { + while (!possible_state_copy.empty()) { + auto key = possible_state_copy.begin()->first; + + checkState(key, {StateAndValue::State::Deleted, ValueType{}}); + + possible_state_copy.erase(key); + pbar.Update(possible_state.size() - possible_state_copy.size()); + } + } + } + } + + // Check KVEngine by get every key in possible_state + // and check its value and state. + void CheckGetter() { + kvdk::Status status; + std::string value_got; + PossibleStates possible_state_copy{possible_state}; + { + std::cout << "[Testing] Checking by Get" << std::endl; + ProgressBar pbar{std::cout, "", possible_state.size(), 1000, + FLAGS_verbose}; + while (!possible_state_copy.empty()) { + auto key = possible_state_copy.begin()->first; + + status = engine_operator(key, &value_got); + switch (status) { + case kvdk::Status::Ok: { + checkState(key, {StateAndValue::State::Existing, value_got}); + break; + } + case kvdk::Status::NotFound: { + checkState(key, {StateAndValue::State::Deleted, ValueType{}}); + break; + } + default: { + ASSERT_TRUE(false) << "Invalid kvdk status in CheckGetter."; + break; + } + } + + possible_state_copy.erase(key); + pbar.Update(possible_state.size() - possible_state_copy.size()); + } + } + } + + private: + // Excecute task_queues in KVEngine by calling EngineOperator + // ShadowKVEngine remains unchanged + void operateKVEngine(size_t tid, bool enable_progress_bar) { + OperationQueue const& tasks = task_queues[tid]; + + kvdk::Status status; + std::string value_got; + size_t progress = 0; + ProgressBar progress_bar{std::cout, "", tasks.size(), 100, + enable_progress_bar}; + /// TODO: Catch kill point and clean up tasks + for (auto const& task : tasks) { + switch (task.op) { + case SingleOp::OpType::Get: { + status = engine_operator(task.key, &value_got); + ASSERT_EQ(status, kvdk::Status::Ok) + << "Key cannot be queried with Get\n" + << "Key: " << task.key << "\n"; + ASSERT_EQ(task.value, value_got) + << "Value got does not match expected\n" + << "Value got:\n" + << value_got << "\n" + << "Expected:\n" + << task.value << "\n"; + break; + } + case SingleOp::OpType::Put: { + status = engine_operator(task.key, task.value); + ASSERT_EQ(status, kvdk::Status::Ok) << "Fail to set key\n" + << "Key: " << task.key << "\n"; + break; + } + case SingleOp::OpType::Delete: { + status = engine_operator(task.key); + ASSERT_EQ(status, kvdk::Status::Ok) << "Fail to delete key\n" + << "Key: " << task.key << "\n"; + break; + } + } + ++progress; + progress_bar.Update(progress); + } + } + + // Check whether a key and corresponding state is in possible_state + void checkState(KeyType key, StateAndValue vstate) { + auto ranges = possible_state.equal_range(key); + + bool match = false; + for (auto iter = ranges.first; iter != ranges.second; ++iter) { + match = (match || (vstate == iter->second)); + } + std::stringstream ss; + if (!match) { + for (auto iter = ranges.first; iter != ranges.second; ++iter) { + ss << iter->second << "\n"; + } + } + + ASSERT_TRUE(match) << "Key and State supplied is not possible:\n" + << "Key: " << key << "\n" + << "Supplied:\n" + << vstate << "\n" + << "Possible:\n" + << ss.str() << "\n"; + } + + static OperationQueue generateOperations(std::vector const& keys, + std::vector const& values, + bool interleaved_set_delete) { + OperationQueue queue(keys.size()); + for (size_t i = 0; i < queue.size(); i++) { + if (i % 2 == 0 || !interleaved_set_delete) { + queue[i] = {SingleOp::OpType::Put, keys[i], values[i]}; + } else { + queue[i] = {SingleOp::OpType::Delete, keys[i], ValueType{}}; + } + } + return queue; + } +}; +} // namespace kvdk_testing + +class EngineTestBase : public testing::Test { + protected: + kvdk::Engine* engine = nullptr; + kvdk::Configs configs; + kvdk::Status status; + + /// The following parameters are used to configure the test. + /// Override SetUpParameters to provide different parameters + /// Default configure parameters + size_t n_hash_bucket; + size_t t_background_work_interval; + + /// Test specific parameters + size_t n_thread; + size_t n_kv_per_thread; + // These parameters set the range of sizes of keys and values + size_t sz_key_min; + size_t sz_key_max; + size_t sz_value_min; + size_t sz_value_max; + + // Actual keys an values used by thread for insertion + std::vector> grouped_keys; + std::vector> grouped_values; + + using ShadowHashes = + kvdk_testing::ShadowKVEngine; + using ShadowSorted = + kvdk_testing::ShadowKVEngine; + using ShadowString = + kvdk_testing::ShadowKVEngine; + + std::unordered_map> + shadow_hashes_engines; + std::unordered_map> + shadow_sorted_engines; + std::unique_ptr shadow_string_engine; + + private: + std::vector key_pool; + std::vector value_pool; + std::default_random_engine rand{42}; + + protected: + /// Other tests should overload this function to setup parameters + virtual void SetUpParameters() = 0; + + virtual void SetUp() override { + purgeDB(); + + SetUpParameters(); + + configs.hash_bucket_num = n_hash_bucket; + configs.background_work_interval = t_background_work_interval; + configs.max_access_threads = n_thread + 1; + configs.log_level = kvdk::LogLevel::Debug; + + prepareKVPairs(); + + status = kvdk::Engine::Open(FLAGS_path, &engine, configs, stderr); + ASSERT_EQ(status, kvdk::Status::Ok) << "Fail to open the KVDK instance"; + } + + virtual void TearDown() { + delete engine; + purgeDB(); + } + + void RebootDB() { + delete engine; + + status = kvdk::Engine::Open(FLAGS_path, &engine, configs, stderr); + ASSERT_EQ(status, kvdk::Status::Ok) << "Fail to open the KVDK instance"; + } + + void ShuffleAllKeysValuesWithinThread() { + for (size_t tid = 0; tid < n_thread; tid++) { + shuffleKeys(tid); + shuffleValues(tid); + } + } + + void HashesAllHSet(std::string const& collection_name) { + ShuffleAllKeysValuesWithinThread(); + auto ModifyEngine = [&](int tid) { + shadow_hashes_engines[collection_name]->EvenXSetOddXSet( + tid, grouped_keys[tid], grouped_values[tid]); + }; + + std::cout << "[Testing] Execute HashesAllHSet in " << collection_name << "." + << std::endl; + LaunchNThreads(n_thread, ModifyEngine); + shadow_hashes_engines[collection_name]->UpdatePossibleStates(); + } + + void HashesEvenHSetOddHDelete(std::string const& collection_name) { + ShuffleAllKeysValuesWithinThread(); + auto ModifyEngine = [&](int tid) { + shadow_hashes_engines[collection_name]->EvenXSetOddXDelete( + tid, grouped_keys[tid], grouped_values[tid]); + }; + + std::cout << "[Testing] Execute HashesEvenHSetOddHDelete in " + << collection_name << "." << std::endl; + LaunchNThreads(n_thread, ModifyEngine); + shadow_hashes_engines[collection_name]->UpdatePossibleStates(); + } + + void SortedAllPut(std::string const& collection_name) { + ShuffleAllKeysValuesWithinThread(); + auto ModifyEngine = [&](int tid) { + shadow_sorted_engines[collection_name]->EvenXSetOddXSet( + tid, grouped_keys[tid], grouped_values[tid]); + }; + + std::cout << "[Testing] Execute SortedAllPut in " << collection_name << "." + << std::endl; + LaunchNThreads(n_thread, ModifyEngine); + shadow_sorted_engines[collection_name]->UpdatePossibleStates(); + } + + void SortedEvenPutOddDelete(std::string const& collection_name) { + ShuffleAllKeysValuesWithinThread(); + auto ModifyEngine = [&](int tid) { + shadow_sorted_engines[collection_name]->EvenXSetOddXDelete( + tid, grouped_keys[tid], grouped_values[tid]); + }; + + std::cout << "[Testing] Execute SortedEvenPutOddDelete in " + << collection_name << "." << std::endl; + LaunchNThreads(n_thread, ModifyEngine); + shadow_sorted_engines[collection_name]->UpdatePossibleStates(); + } + + void StringAllSet() { + ShuffleAllKeysValuesWithinThread(); + auto ModifyEngine = [&](int tid) { + shadow_string_engine->EvenXSetOddXSet(tid, grouped_keys[tid], + grouped_values[tid]); + }; + + std::cout << "[Testing] Execute StringAllSet " << std::endl; + LaunchNThreads(n_thread, ModifyEngine); + shadow_string_engine->UpdatePossibleStates(); + } + + void StringEvenSetOddDelete() { + ShuffleAllKeysValuesWithinThread(); + auto ModifyEngine = [&](int tid) { + shadow_string_engine->EvenXSetOddXDelete(tid, grouped_keys[tid], + grouped_values[tid]); + }; + + std::cout << "[Testing] Execute StringEvenSetOddDelete " << std::endl; + LaunchNThreads(n_thread, ModifyEngine); + shadow_string_engine->UpdatePossibleStates(); + } + + void CheckHashesCollection(std::string collection_name) { + std::cout << "[Testing] Checking Hashes Collection: " << collection_name + << std::endl; + shadow_hashes_engines[collection_name]->CheckGetter(); + auto iter = engine->HashIteratorCreate(collection_name); + shadow_hashes_engines[collection_name]->CheckIterator( + iter, kvdk_testing::IteratingDirection::Forward); + shadow_hashes_engines[collection_name]->CheckIterator( + iter, kvdk_testing::IteratingDirection::Backward); + engine->HashIteratorRelease(iter); + } + + void CheckSortedCollection(std::string collection_name) { + std::cout << "[Testing] Checking Sorted Collection: " << collection_name + << std::endl; + shadow_sorted_engines[collection_name]->CheckGetter(); + auto iter = engine->SortedIteratorCreate(collection_name); + shadow_sorted_engines[collection_name]->CheckIterator( + iter, kvdk_testing::IteratingDirection::Forward); + shadow_sorted_engines[collection_name]->CheckIterator( + iter, kvdk_testing::IteratingDirection::Backward); + engine->SortedIteratorRelease(iter); + } + + void CheckStrings() { + std::cout << "[Testing] Checking strings." << std::endl; + shadow_string_engine->CheckGetter(); + } + + void InitializeStrings() { + shadow_string_engine.reset( + new ShadowString{engine, kvdk_testing::CollectionNameType{}, n_thread}); + } + + void InitializeHashes(std::string const& collection_name) { + shadow_hashes_engines[collection_name].reset( + new ShadowHashes{engine, collection_name, n_thread}); + } + + void InitializeSorted(std::string const& collection_name) { + shadow_sorted_engines[collection_name].reset( + new ShadowSorted{engine, collection_name, n_thread}); + } + + void RemoveOutDatedSorted(std::string const& collection_name) { + if (shadow_sorted_engines.find(collection_name) != + shadow_sorted_engines.end()) { + shadow_sorted_engines.erase(collection_name); + } + } + + void RemoveOutDatedHashes(std::string const& collection_name) { + if (shadow_hashes_engines.find(collection_name) != + shadow_hashes_engines.end()) { + shadow_hashes_engines.erase(collection_name); + } + } + + private: + void purgeDB() { + std::string cmd = "rm -rf " + FLAGS_path + "\n"; + [[gnu::unused]] int _sink = system(cmd.data()); + } + + void shuffleKeys(size_t tid) { + std::shuffle(grouped_keys[tid].begin(), grouped_keys[tid].end(), rand); + } + + void shuffleValues(size_t tid) { + std::shuffle(grouped_values[tid].begin(), grouped_values[tid].end(), rand); + } + + void prepareKVPairs() { + key_pool.reserve(n_thread * n_kv_per_thread); + value_pool.reserve(n_kv_per_thread); + grouped_keys.resize(n_thread); + grouped_values.resize(n_thread); + + for (size_t tid = 0; tid < n_thread; tid++) { + grouped_keys[tid].reserve(n_kv_per_thread); + grouped_values[tid].reserve(n_kv_per_thread); + } + + std::cout << "[Testing] Generating string for keys and values" << std::endl; + { + ProgressBar progress_gen_kv{std::cout, "", n_kv_per_thread, 1, + FLAGS_verbose}; + for (size_t i = 0; i < n_kv_per_thread; i++) { + value_pool.push_back(GetRandomString(sz_value_min, sz_value_max)); + for (size_t tid = 0; tid < n_thread; tid++) { + key_pool.push_back(GetRandomString(sz_key_min, sz_key_max)); + } + + if ((i + 1) % 1000 == 0 || (i + 1) == n_kv_per_thread) { + progress_gen_kv.Update(i + 1); + } + } + } + std::cout << "[Testing] Generating string_view for keys and values" + << std::endl; + { + ProgressBar progress_gen_kv_view{std::cout, "", n_thread, 1, + FLAGS_verbose}; + for (size_t tid = 0; tid < n_thread; tid++) { + for (size_t i = 0; i < n_kv_per_thread; i++) { + grouped_keys[tid].emplace_back(key_pool[i * n_thread + tid]); + grouped_values[tid].emplace_back(value_pool[i]); + } + progress_gen_kv_view.Update(tid + 1); + } + } + } +}; + +class EngineStressTest : public EngineTestBase { + protected: + virtual void SetUpParameters() override final { + /// Default configure parameters + // Less buckets to increase hash collisions + n_hash_bucket = (1ULL << 20); + t_background_work_interval = 1; + + /// Test specific parameters + n_thread = 32; + // 1M keys per thread, totaling about 32M(actually less) records + n_kv_per_thread = (1ULL << 20); + // These parameters set the range of sizes of keys and values + sz_key_min = 2; + sz_key_max = 16; + sz_value_min = 0; + sz_value_max = 1024; + } + // Shared among EngineStressTest + const size_t n_reboot = 3; +}; + +TEST_F(EngineStressTest, HashesHSetOnly) { + std::string global_collection_name{"HashesCollection"}; + InitializeHashes(global_collection_name); + ASSERT_EQ(engine->HashCreate(global_collection_name), kvdk::Status::Ok); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + HashesAllHSet(global_collection_name); + CheckHashesCollection(global_collection_name); + + RebootDB(); + CheckHashesCollection(global_collection_name); + } +} + +TEST_F(EngineStressTest, HashesHSetAndHDelete) { + std::string global_collection_name{"HashesCollection"}; + InitializeHashes(global_collection_name); + ASSERT_EQ(engine->HashCreate(global_collection_name), kvdk::Status::Ok); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + HashesEvenHSetOddHDelete(global_collection_name); + CheckHashesCollection(global_collection_name); + + RebootDB(); + CheckHashesCollection(global_collection_name); + } +} + +TEST_F(EngineStressTest, SortedPutOnly) { + for (int index_with_hashtable : {0, 1}) { + std::string global_collection_name{"SortedCollection" + + index_with_hashtable}; + InitializeSorted(global_collection_name); + + kvdk::SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = index_with_hashtable; + ASSERT_EQ(engine->SortedCreate(global_collection_name, s_configs), + kvdk::Status::Ok); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + std::cout << "Sorted collection index with hashtable: " + << index_with_hashtable << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + SortedAllPut(global_collection_name); + CheckSortedCollection(global_collection_name); + + RebootDB(); + CheckSortedCollection(global_collection_name); + } + } +} + +TEST_F(EngineStressTest, SortedPutAndDelete) { + for (int index_with_hashtable : {0, 1}) { + std::string global_collection_name{"SortedCollection" + + index_with_hashtable}; + InitializeSorted(global_collection_name); + + kvdk::SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = index_with_hashtable; + ASSERT_EQ(engine->SortedCreate(global_collection_name, s_configs), + kvdk::Status::Ok); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + std::cout << "Sorted collection index with hashtable: " + << index_with_hashtable << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + SortedEvenPutOddDelete(global_collection_name); + CheckSortedCollection(global_collection_name); + + RebootDB(); + CheckSortedCollection(global_collection_name); + } + } +} + +TEST_F(EngineStressTest, StringSetOnly) { + InitializeStrings(); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + StringAllSet(); + CheckStrings(); + + RebootDB(); + CheckStrings(); + } +} + +TEST_F(EngineStressTest, StringSetAndDelete) { + InitializeStrings(); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + StringEvenSetOddDelete(); + CheckStrings(); + + RebootDB(); + CheckStrings(); + } +} + +class EngineHotspotTest : public EngineTestBase { + protected: + virtual void SetUpParameters() override final { + /// Default configure parameters + // Less buckets to increase hash collisions + n_hash_bucket = (1ULL << 20); + t_background_work_interval = 1; + + /// Test specific parameters + // Too many threads will make this test too slow + n_thread = 4; + // 1M keys per thread + n_kv_per_thread = (1ULL << 20); + // 0-sized key "" is a hotspot, which may reveal many defects + // These parameters set the range of sizes of keys and values + sz_key_min = 0; + sz_key_max = 8; + sz_value_min = 0; + sz_value_max = 128; + } + + size_t n_repeat = 10; + size_t n_reboot = 3; +}; + +TEST_F(EngineHotspotTest, HashesMultipleHotspot) { + std::string global_collection_name{"HashesCollection"}; + InitializeHashes(global_collection_name); + ASSERT_EQ(engine->HashCreate(global_collection_name), kvdk::Status::Ok); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + for (size_t i = 0; i < n_repeat; i++) { + HashesAllHSet(global_collection_name); + CheckHashesCollection(global_collection_name); + HashesEvenHSetOddHDelete(global_collection_name); + CheckHashesCollection(global_collection_name); + } + RebootDB(); + CheckHashesCollection(global_collection_name); + } +} + +TEST_F(EngineHotspotTest, SortedMultipleHotspot) { + for (int index_with_hashtable : {0, 1}) { + std::string global_collection_name{"SortedCollection" + + index_with_hashtable}; + InitializeSorted(global_collection_name); + + kvdk::SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = index_with_hashtable; + ASSERT_EQ(engine->SortedCreate(global_collection_name, s_configs), + kvdk::Status::Ok); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + std::cout << "Sorted collection index with hashtable: " + << index_with_hashtable << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + for (size_t i = 0; i < n_repeat; i++) { + SortedAllPut(global_collection_name); + CheckSortedCollection(global_collection_name); + SortedEvenPutOddDelete(global_collection_name); + CheckSortedCollection(global_collection_name); + } + RebootDB(); + CheckSortedCollection(global_collection_name); + } + } +} + +TEST_F(EngineHotspotTest, StringMultipleHotspot) { + InitializeStrings(); + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + + for (size_t i = 0; i < n_repeat; i++) { + StringAllSet(); + CheckStrings(); + StringEvenSetOddDelete(); + CheckStrings(); + } + RebootDB(); + CheckStrings(); + } +} + +TEST_F(EngineStressTest, BackgroundCleanerTest) { + std::string global_sorted_collection_name{"SortedCollection_withHashTable"}; + std::string global_hash_collection_name{"HashCollection"}; + + kvdk::SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = 1; + + std::cout << "[Testing] Modify, check, reboot and check engine for " + << n_reboot << " times." << std::endl; + for (size_t i = 0; i < n_reboot; i++) { + InitializeSorted(global_sorted_collection_name); + InitializeHashes(global_hash_collection_name); + InitializeStrings(); + std::cout << "[Testing] Repeat: " << i + 1 << std::endl; + std::cout << " Create Sorted collection index with hashtable: " << 1 + << std::endl; + ASSERT_EQ(engine->SortedCreate(global_sorted_collection_name, s_configs), + kvdk::Status::Ok); + std::cout << "Create Hash collection" << std::endl; + ASSERT_EQ(engine->HashCreate(global_hash_collection_name), + kvdk::Status::Ok); + for (size_t j = 0; j < 2; j++) { + // Sorted + SortedAllPut(global_sorted_collection_name); + CheckSortedCollection(global_sorted_collection_name); + SortedEvenPutOddDelete(global_sorted_collection_name); + CheckSortedCollection(global_sorted_collection_name); + // String + StringAllSet(); + CheckStrings(); + StringEvenSetOddDelete(); + CheckStrings(); + // Hash + HashesAllHSet(global_hash_collection_name); + CheckHashesCollection(global_hash_collection_name); + HashesEvenHSetOddHDelete(global_hash_collection_name); + CheckHashesCollection(global_hash_collection_name); + } + + RebootDB(); + CheckSortedCollection(global_sorted_collection_name); + CheckHashesCollection(global_hash_collection_name); + ASSERT_EQ(engine->SortedDestroy(global_sorted_collection_name), + kvdk::Status::Ok); + RemoveOutDatedSorted(global_sorted_collection_name); + ASSERT_EQ(engine->Expire(global_hash_collection_name, -1), + kvdk::Status::Ok); + RemoveOutDatedHashes(global_hash_collection_name); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/volatile/tests/test_util.h b/volatile/tests/test_util.h new file mode 100644 index 00000000..163e93d3 --- /dev/null +++ b/volatile/tests/test_util.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Create a string that contains 8 bytes from uint64_t. */ +inline std::string uint64_to_string(uint64_t& key) { + return std::string(reinterpret_cast(&key), 8); +} + +inline void random_str(char* str, unsigned int size) { + for (unsigned int i = 0; i < size; i++) { + switch (rand() % 3) { + case 0: + str[i] = rand() % 10 + '0'; + break; + case 1: + str[i] = rand() % 26 + 'A'; + break; + case 2: + str[i] = rand() % 26 + 'a'; + break; + default: + break; + } + } + str[size] = 0; +} + +// Return a string of length len with random characters in ['a', 'z'] +inline std::string GetRandomString(size_t len) { + static std::default_random_engine re; + std::string str; + str.reserve(len); + for (size_t i = 0; i < len; i++) str.push_back('a' + re() % 26); + return str; +} + +// Return a string of length in [min_len, max_len] with random characters in +// ['a', 'z'] +inline std::string GetRandomString(size_t min_len, size_t max_len) { + static std::default_random_engine re; + size_t len = min_len + re() % (max_len - min_len + 1); + return GetRandomString(len); +} + +inline void LaunchNThreads(int n_thread, std::function func, + int id_start = 0) { + std::vector ts; + for (int i = id_start; i < id_start + n_thread; i++) { + ts.emplace_back(std::thread(func, i)); + } + for (auto& t : ts) t.join(); +} + +class ProgressBar { + private: + std::ostream& out_stream_; + std::string tag_; + size_t total_progress_; + size_t current_progress_; + size_t last_report_; + size_t report_interval_; + size_t bar_length_; + size_t step_; + bool enabled_; + + bool flush_newline_{false}; + + static constexpr char symbol_done_{'#'}; + static constexpr char symbol_fill_{'-'}; + + public: + explicit ProgressBar(std::ostream& out, std::string tag, + size_t total_progress, size_t report_interval, + bool enabled, size_t bar_length = 50) + : out_stream_{out}, + tag_{tag}, + total_progress_{total_progress}, + current_progress_{0}, + last_report_{0}, + report_interval_{report_interval}, + bar_length_{bar_length}, + step_{total_progress / bar_length}, + enabled_{enabled} { + assert(total_progress_ > 0); + assert(bar_length_ > 0); + if (step_ == 0) { + step_ = 1; + bar_length_ = total_progress; + } + // Actual bar length may be 1 char longer than given bar length + // 51 = 2048 / (2048 / 50). This prevents overflowing the bar + bar_length_ = total_progress_ / step_; + + showProgress(); + } + + void Update(size_t current_progress) { + assert(!flush_newline_ && "Trying to update a completed progress!"); + assert(current_progress_ < current_progress && + current_progress <= total_progress_); + + current_progress_ = current_progress; + if (current_progress_ == total_progress_) { + flush_newline_ = true; + } + + if (last_report_ + report_interval_ <= current_progress_ || + current_progress_ == total_progress_) { + showProgress(); + last_report_ = current_progress_; + } + } + + ~ProgressBar() { + if (!flush_newline_) { + flush_newline_ = true; + showProgress(); + } + } + + private: + void showProgress() { + if (!enabled_) return; + + assert(current_progress_ <= current_progress_); + + out_stream_ << "\r" << tag_ << std::setw(10) << std::right + << current_progress_ << "/" << std::setw(10) << std::left + << total_progress_ << "\t" + << "["; + + { + size_t n_step_done = current_progress_ / step_; + for (size_t i = 0; i < n_step_done; i++) out_stream_ << symbol_done_; + for (size_t i = 0; i < bar_length_ - n_step_done; i++) + out_stream_ << symbol_fill_; + } + + out_stream_ << "]" << std::flush; + + if (flush_newline_) out_stream_ << std::endl; + } +}; diff --git a/volatile/tests/tests.cpp b/volatile/tests/tests.cpp new file mode 100644 index 00000000..34ed875c --- /dev/null +++ b/volatile/tests/tests.cpp @@ -0,0 +1,2573 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include "../engine/kv_engine.hpp" +#include "../engine/utils/sync_point.hpp" +#include "kvdk/volatile/engine.hpp" +#include "test_util.h" + +DEFINE_string(path, "/mnt/pmem0/kvdk_unit_test", + "Path of KVDK instance on PMem."); + +using namespace KVDK_NAMESPACE; +static const uint64_t str_pool_length = 1024000; + +using PutOpsFunc = + std::function; +using DeleteOpsFunc = std::function; + +using DestroyFunc = std::function; + +using GetOpsFunc = std::function; + +enum class Types { String, Sorted, Hash }; + +class EngineBasicTest : public testing::Test { + protected: + Engine* engine = nullptr; + Configs configs; + std::string db_path; + std::string backup_path; + std::string backup_log; + std::string str_pool; + + virtual void SetUp() override { + str_pool.resize(str_pool_length); + random_str(&str_pool[0], str_pool_length); + // No logs by default, for debug, set it to All + configs.log_level = LogLevel::Debug; + configs.hash_bucket_num = (1 << 10); + // For faster test, no interval so it would not block engine closing + configs.background_work_interval = 0.1; + configs.max_access_threads = 8; + db_path = FLAGS_path; + backup_path = FLAGS_path + "_backup"; + backup_log = FLAGS_path + ".backup"; + char cmd[1024]; + sprintf(cmd, "rm -rf %s && rm -rf %s && rm -rf %s\n", db_path.c_str(), + backup_path.c_str(), backup_log.c_str()); + int res __attribute__((unused)) = system(cmd); + config_option_ = OptionConfig::Default; + cnt_ = 500; + } + + virtual void TearDown() { +#if KVDK_DEBUG_LEVEL > 0 + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->Reset(); +#endif + Destroy(); + } + + std::string FastRandomString(int len) { + return std::string( + str_pool.data() + fast_random_64() % (str_pool_length - len), len); + } + + void Destroy() { + // delete db_path + char cmd[1024]; + sprintf(cmd, "rm -rf %s && rm -rf %s && rm -rf %s\n", db_path.c_str(), + backup_path.c_str(), backup_log.c_str()); + int res __attribute__((unused)) = system(cmd); + } + + bool ChangeConfig() { + config_option_++; + if (config_option_ >= End) { + return false; + } else { + ReCreateEngine(); + return engine != nullptr; + } + } + + void ReCreateEngine() { + delete engine; + engine = nullptr; + Destroy(); + configs = CurrentConfigs(); + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + } + + void Reboot() {} + + // Return the current configuration. + Configs CurrentConfigs() { + switch (config_option_) { + case OptRestore: + configs.opt_large_sorted_collection_recovery = true; + break; + default: + break; + } + return configs; + } + + // Put/Get/Delete + void TestString(uint64_t n_threads) { + auto StringPutFunc = [&](const std::string&, const std::string& key, + const std::string& value) -> Status { + return engine->Put(key, value); + }; + + auto StringGetFunc = [&](const std::string&, const std::string& key, + std::string* value) -> Status { + return engine->Get(key, value); + }; + + auto StringDeleteFunc = [&](const std::string&, + const std::string& key) -> Status { + return engine->Delete(key); + }; + + testEmptyKey("", StringPutFunc, StringGetFunc, StringDeleteFunc); + auto global_func = [=](uint64_t id) { + this->createBasicOperationTest("", StringPutFunc, StringGetFunc, + StringDeleteFunc, id); + }; + LaunchNThreads(n_threads, global_func); + } + + void TestGlobalSortedCollection(const std::string& collection, + const SortedCollectionConfigs& s_configs) { + auto SortedPutFunc = [&](const std::string& collection, + const std::string& key, + const std::string& value) -> Status { + return engine->SortedPut(collection, key, value); + }; + + auto SortedGetFunc = [&](const std::string& collection, + const std::string& key, + std::string* value) -> Status { + return engine->SortedGet(collection, key, value); + }; + + auto SortedDeleteFunc = [&](const std::string& collection, + const std::string& key) -> Status { + return engine->SortedDelete(collection, key); + }; + + auto SortedDestroyFunc = [&](const std::string& collection) { + return engine->SortedDestroy(collection); + }; + + ASSERT_EQ(engine->SortedCreate(collection, s_configs), Status::Ok); + + testDestroy(collection, SortedDestroyFunc, SortedPutFunc, SortedGetFunc, + SortedDeleteFunc); + + ASSERT_EQ(engine->SortedCreate(collection, s_configs), Status::Ok); + + auto global_func = [=](uint64_t id) { + this->createBasicOperationTest(collection, SortedPutFunc, SortedGetFunc, + SortedDeleteFunc, id); + }; + LaunchNThreads(configs.max_access_threads, global_func); + } + + void TestLocalSortedCollection(Engine* engine, const std::string& collection, + const SortedCollectionConfigs& s_configs) { + auto SortedPutFunc = [&](const std::string& collection, + const std::string& key, + const std::string& value) -> Status { + return engine->SortedPut(collection, key, value); + }; + + auto SortedGetFunc = [&](const std::string& collection, + const std::string& key, + std::string* value) -> Status { + return engine->SortedGet(collection, key, value); + }; + + auto SortedDeleteFunc = [&](const std::string& collection, + const std::string& key) -> Status { + return engine->SortedDelete(collection, key); + }; + + auto SortedDestroyFunc = [&](const std::string& collection) { + return engine->SortedDestroy(collection); + }; + + auto AccessTest = [&](uint64_t id) { + std::string thread_local_collection = collection + std::to_string(id); + ASSERT_EQ(engine->SortedCreate(thread_local_collection, s_configs), + Status::Ok); + + testEmptyKey(thread_local_collection, SortedPutFunc, SortedGetFunc, + SortedDeleteFunc); + testDestroy(thread_local_collection, SortedDestroyFunc, SortedPutFunc, + SortedGetFunc, SortedDeleteFunc); + + ASSERT_EQ(engine->SortedCreate(thread_local_collection, s_configs), + Status::Ok); + createBasicOperationTest(thread_local_collection, SortedPutFunc, + SortedGetFunc, SortedDeleteFunc, id); + }; + LaunchNThreads(configs.max_access_threads, AccessTest); + } + + void TestSortedIterator(const std::string& collection, + bool is_local = false) { + auto IteratingThrough = [&](uint32_t id) { + int entries = 0; + std::string new_collection = collection; + if (is_local) { + new_collection += std::to_string(id); + } + + size_t collection_size; + ASSERT_EQ(engine->SortedSize(new_collection, &collection_size), + Status::Ok); + + auto iter = engine->SortedIteratorCreate(new_collection); + ASSERT_TRUE(iter != nullptr); + // forward iterator + iter->SeekToFirst(); + if (iter->Valid()) { + ++entries; + std::string prev = iter->Key(); + iter->Next(); + while (iter->Valid()) { + ++entries; + std::string k = iter->Key(); + iter->Next(); + ASSERT_EQ(true, k.compare(prev) > 0); + prev = k; + } + } + ASSERT_EQ(collection_size, entries); + if (is_local) { + ASSERT_EQ(cnt_, entries); + } else { + ASSERT_EQ(cnt_ * configs.max_access_threads, entries); + } + + // backward iterator + iter->SeekToLast(); + if (iter->Valid()) { + --entries; + std::string next = iter->Key(); + iter->Prev(); + while (iter->Valid()) { + --entries; + std::string k = iter->Key(); + iter->Prev(); + ASSERT_EQ(true, k.compare(next) < 0); + next = k; + } + } + ASSERT_EQ(entries, 0); + engine->SortedIteratorRelease(iter); + }; + LaunchNThreads(configs.max_access_threads, IteratingThrough); + } + + private: + void testEmptyKey(const std::string& collection, PutOpsFunc PutFunc, + GetOpsFunc GetFunc, DeleteOpsFunc DeleteFunc) { + std::string key, val, got_val; + key = "", val = "val"; + ASSERT_EQ(PutFunc(collection, key, val), Status::Ok); + ASSERT_EQ(GetFunc(collection, key, &got_val), Status::Ok); + ASSERT_EQ(val, got_val); + ASSERT_EQ(DeleteFunc(collection, key), Status::Ok); + ASSERT_EQ(GetFunc(collection, key, &got_val), Status::NotFound); + } + + void testDestroy(const std::string& collection, DestroyFunc DestroyFunc, + PutOpsFunc PutFunc, GetOpsFunc GetFunc, + DeleteOpsFunc DeleteFunc) { + std::string key{"test_key"}; + std::string val{"test_val"}; + std::string got_val; + ASSERT_EQ(PutFunc(collection, key, val), Status::Ok); + ASSERT_EQ(GetFunc(collection, key, &got_val), Status::Ok); + ASSERT_EQ(val, got_val); + ASSERT_EQ(DestroyFunc(collection), Status::Ok); + ASSERT_EQ(PutFunc(collection, key, val), Status::NotFound); + ASSERT_EQ(GetFunc(collection, key, &got_val), Status::NotFound); + ASSERT_EQ(DeleteFunc(collection, key), Status::Ok); + } + + void createBasicOperationTest(const std::string& collection, + PutOpsFunc PutFunc, GetOpsFunc GetFunc, + DeleteOpsFunc DeleteFunc, uint32_t id) { + std::string val1, val2, got_val1, got_val2; + int t_cnt = cnt_; + while (t_cnt--) { + std::string key1(std::to_string(id) + "_" + std::to_string(t_cnt)); + std::string key2(std::to_string(id) + "@" + std::to_string(t_cnt)); + val1 = FastRandomString(fast_random_64() % 1024); + val2 = FastRandomString(fast_random_64() % 1024); + + // Put + ASSERT_EQ(PutFunc(collection, key1, val1), Status::Ok); + ASSERT_EQ(PutFunc(collection, key2, val2), Status::Ok); + + // Get + ASSERT_EQ(GetFunc(collection, key1, &got_val1), Status::Ok); + ASSERT_EQ(val1, got_val1); + ASSERT_EQ(GetFunc(collection, key2, &got_val2), Status::Ok); + ASSERT_EQ(val2, got_val2); + + // Delete + ASSERT_EQ(DeleteFunc(collection, key1), Status::Ok); + ASSERT_EQ(GetFunc(collection, key1, &got_val1), Status::NotFound); + + // Update + val2 = FastRandomString(fast_random_64() % 1024); + ASSERT_EQ(PutFunc(collection, key2, val2), Status::Ok); + ASSERT_EQ(GetFunc(collection, key2, &got_val2), Status::Ok); + ASSERT_EQ(got_val2, val2); + } + } + + // Sequence of option configurations to try + enum OptionConfig { Default, MultiThread, OptRestore, End }; + int config_option_; + int cnt_; +}; + +class BatchWriteTest : public EngineBasicTest {}; + +TEST_F(EngineBasicTest, TestUniqueKey) { + std::string sorted_collection("sorted_collection"); + std::string hash_collection("unordered_collection"); + std::string list("list"); + std::string str("str"); + std::string elem_key("elem"); + std::string val("val"); + + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + ASSERT_EQ(engine->Put(str, val), Status::Ok); + + ASSERT_EQ(engine->SortedCreate(sorted_collection), Status::Ok); + + ASSERT_EQ(engine->HashCreate(hash_collection), Status::Ok); + ASSERT_EQ(engine->HashPut(hash_collection, elem_key, val), Status::Ok); + + ASSERT_EQ(engine->ListCreate(list), Status::Ok); + ASSERT_EQ(engine->ListPushBack(list, elem_key), Status::Ok); + + std::string got_val; + // Test string + for (const std::string& string_key : + {sorted_collection, hash_collection, list, str}) { + Status op_ret = string_key == str ? Status::Ok : Status::WrongType; + std::string new_val("new_str_val"); + // Put + ASSERT_EQ(engine->Put(string_key, new_val), op_ret); + // Get + ASSERT_EQ(engine->Get(string_key, &got_val), op_ret); + if (op_ret == Status::Ok) { + ASSERT_EQ(got_val, new_val); + } + } + + // Test sorted + for (const std::string& collection_name : + {sorted_collection, hash_collection, list, str}) { + Status create_ret = collection_name == sorted_collection + ? Status::Existed + : Status::WrongType; + Status op_ret = + collection_name == sorted_collection ? Status::Ok : Status::WrongType; + std::string new_val("new_sorted_val"); + // Create + ASSERT_EQ(engine->SortedCreate(collection_name), create_ret); + // Put + ASSERT_EQ(engine->SortedPut(collection_name, elem_key, new_val), op_ret); + + // Get + ASSERT_EQ(engine->SortedGet(collection_name, elem_key, &got_val), op_ret); + + if (op_ret == Status::Ok) { + ASSERT_EQ(got_val, new_val); + } + // Delete elem + ASSERT_EQ(engine->SortedDelete(collection_name, elem_key), op_ret); + } + + // Test unordered + for (const std::string& collection_name : + {sorted_collection, hash_collection, list, str}) { + Status create_ret = collection_name == hash_collection ? Status::Existed + : Status::WrongType; + Status op_ret = + collection_name == hash_collection ? Status::Ok : Status::WrongType; + std::string new_val("new_unordered_val"); + // Create + ASSERT_EQ(engine->HashCreate(collection_name), create_ret); + // Put + ASSERT_EQ(engine->HashPut(collection_name, elem_key, new_val), op_ret); + // Get + ASSERT_EQ(engine->HashGet(collection_name, elem_key, &got_val), op_ret); + if (op_ret == Status::Ok) { + ASSERT_EQ(got_val, new_val); + } + // Delete + ASSERT_EQ(engine->HashDelete(collection_name, elem_key), op_ret); + } + + // Test list + for (const std::string& collection_name : + {sorted_collection, hash_collection, list, str}) { + Status create_ret = + collection_name == list ? Status::Existed : Status::WrongType; + Status op_ret = collection_name == list ? Status::Ok : Status ::WrongType; + std::string new_val_back("new_back_val"); + std::string new_val_front("new_front_val"); + size_t length; + std::string got_val_back; + std::string got_val_front; + + // Create + ASSERT_EQ(engine->ListCreate(collection_name), create_ret); + // Push + ASSERT_EQ(engine->ListPushBack(collection_name, new_val_back), op_ret); + ASSERT_EQ(engine->ListPushFront(collection_name, new_val_front), op_ret); + // Pop + ASSERT_EQ(engine->ListPopBack(collection_name, &got_val_back), op_ret); + ASSERT_EQ(engine->ListPopFront(collection_name, &got_val_front), op_ret); + // Length + ASSERT_EQ(engine->ListSize(collection_name, &length), op_ret); + + if (op_ret == Status::Ok) { + ASSERT_EQ(got_val_back, new_val_back); + ASSERT_EQ(got_val_front, new_val_front); + ASSERT_EQ(length, 1); + } + } + delete engine; +} + +TEST_F(EngineBasicTest, TypeOfKey) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + std::unordered_map key_types; + for (auto type : {ValueType::String, ValueType::HashCollection, + ValueType::List, ValueType::SortedCollection}) { + std::string key = KVDKValueTypeString[type]; + key_types[key] = type; + ValueType type_resp; + switch (type) { + case ValueType::String: { + ASSERT_EQ(engine->Put(key, ""), Status::Ok); + break; + } + case ValueType::HashCollection: { + ASSERT_EQ(engine->HashCreate(key), Status::Ok); + break; + } + case ValueType::List: { + ASSERT_EQ(engine->ListCreate(key), Status::Ok); + break; + } + case ValueType::SortedCollection: { + ASSERT_EQ(engine->SortedCreate(key), Status::Ok); + break; + } + } + ASSERT_EQ(engine->TypeOf(key, &type_resp), Status::Ok); + ASSERT_EQ(type_resp, type); + ASSERT_EQ(engine->TypeOf("non-exist", &type_resp), Status::NotFound); + } + delete engine; +} + +// Test iterator/backup/checkpoint on a snapshot +TEST_F(EngineBasicTest, TestBasicSnapshot) { + uint32_t num_threads = 16; + int count = 100; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + std::string sorted_collection("sorted_collection"); + std::string hash_collection("hash_collection"); + std::string list("list"); + std::string sorted_collection_after_snapshot( + "sorted_collection_after_snapshot"); + std::string hash_collection_after_snapshot("hash_collection_after_snapshot"); + std::string list_after_snapshot("list_after_snapshot"); + ASSERT_EQ(engine->SortedCreate(sorted_collection), Status::Ok); + ASSERT_EQ(engine->HashCreate(hash_collection), Status::Ok); + ASSERT_EQ(engine->ListCreate(list), Status::Ok); + + bool snapshot_done(false); + std::atomic_uint64_t set_finished_threads(0); + SpinMutex spin; + std::condition_variable_any cv; + + // Insert kv, then update/delete them and insert new kv after snapshot + auto WriteThread = [&](uint32_t id) { + int cnt = count; + // Insert + while (cnt--) { + // Insert + std::string key1(std::string(id + 1, 'a') + std::to_string(cnt)); + std::string key2(std::string(id + 1, 'b') + std::to_string(cnt)); + WriteOptions write_options; + ASSERT_EQ(engine->Put(key1, key1, write_options), Status::Ok); + ASSERT_EQ(engine->Put(key2, key2, write_options), Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection, key1, key1), Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection, key2, key2), Status::Ok); + ASSERT_EQ(engine->HashPut(hash_collection, key1, key1), Status::Ok); + ASSERT_EQ(engine->HashPut(hash_collection, key2, key2), Status::Ok); + ASSERT_EQ(engine->ListPushBack(list, key1), Status::Ok); + ASSERT_EQ(engine->ListPushBack(list, key2), Status::Ok); + } + // Wait snapshot done + set_finished_threads.fetch_add(1); + { + std::unique_lock ul(spin); + while (!snapshot_done) { + cv.wait(ul); + } + } + + cnt = count; + while (cnt--) { + // Update, Delete, and Insert new + std::string key1(std::string(id + 1, 'a') + std::to_string(cnt)); + std::string key2(std::string(id + 1, 'b') + std::to_string(cnt)); + std::string key3(std::string(id + 1, 'c') + std::to_string(cnt)); + ASSERT_EQ(engine->Put(key1, "updated " + key1), Status::Ok); + ASSERT_EQ(engine->Delete(key1), Status::Ok); + ASSERT_EQ(engine->Put(key3, key3), Status::Ok); + + ASSERT_EQ(engine->SortedPut(sorted_collection, key1, "updated " + key1), + Status::Ok); + ASSERT_EQ(engine->SortedDelete(sorted_collection, key2), Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection, key3, key3), Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection_after_snapshot, key1, key1), + Ok); + + ASSERT_EQ(engine->HashPut(hash_collection, key1, "updated " + key1), + Status::Ok); + ASSERT_EQ(engine->HashDelete(hash_collection, key2), Status::Ok); + ASSERT_EQ(engine->HashPut(hash_collection, key3, key3), Status::Ok); + ASSERT_EQ(engine->HashPut(hash_collection_after_snapshot, key1, key1), + Ok); + + std::string elem; + ASSERT_EQ(engine->ListPopBack(list, &elem), Status::Ok); + ASSERT_EQ(engine->ListPopBack(list, &elem), Status::Ok); + ASSERT_EQ(engine->ListPushBack(list, key3), Status::Ok); + ASSERT_EQ(engine->ListPushBack(list_after_snapshot, key1), Ok); + } + }; + + std::vector ths; + for (size_t i = 0; i < num_threads; i++) { + ths.emplace_back(std::thread(WriteThread, i)); + } + // wait until all threads insert done + while (set_finished_threads.load() != num_threads) { + asm volatile("pause"); + } + Snapshot* snapshot = engine->GetSnapshot(true); + // Insert a new collection after snapshot + ASSERT_EQ(engine->SortedCreate(sorted_collection_after_snapshot), Status::Ok); + ASSERT_EQ(engine->HashCreate(hash_collection_after_snapshot), Status::Ok); + ASSERT_EQ(engine->ListCreate(list_after_snapshot), Status::Ok); + { + std::lock_guard ul(spin); + snapshot_done = true; + cv.notify_all(); + } + engine->Backup(backup_log, snapshot); + for (auto& t : ths) { + t.join(); + } + + { // sorted snapshot iterator + SortedIterator* sorted_snapshot_iter = + engine->SortedIteratorCreate(sorted_collection, snapshot); + // Destroyed collection still should be accessable by snapshot_iter + engine->SortedDestroy(sorted_collection); + + uint64_t snapshot_iter_cnt = 0; + sorted_snapshot_iter->SeekToFirst(); + while (sorted_snapshot_iter->Valid()) { + ASSERT_TRUE(sorted_snapshot_iter->Valid()); + snapshot_iter_cnt++; + ASSERT_EQ(sorted_snapshot_iter->Key(), sorted_snapshot_iter->Value()); + sorted_snapshot_iter->Next(); + } + ASSERT_EQ(snapshot_iter_cnt, num_threads * count * 2); + engine->SortedIteratorRelease(sorted_snapshot_iter); + + sorted_snapshot_iter = engine->SortedIteratorCreate( + sorted_collection_after_snapshot, snapshot); + sorted_snapshot_iter->SeekToFirst(); + ASSERT_FALSE(sorted_snapshot_iter->Valid()); + engine->SortedIteratorRelease(sorted_snapshot_iter); + } + + { // Hash snapshot iterator + auto hash_snapshot_iter = + engine->HashIteratorCreate(hash_collection, snapshot); + engine->HashDestroy(hash_collection); + + uint64_t snapshot_iter_cnt = 0; + hash_snapshot_iter->SeekToFirst(); + while (hash_snapshot_iter->Valid()) { + ASSERT_TRUE(hash_snapshot_iter->Valid()); + snapshot_iter_cnt++; + ASSERT_EQ(hash_snapshot_iter->Key(), hash_snapshot_iter->Value()); + hash_snapshot_iter->Next(); + } + ASSERT_EQ(snapshot_iter_cnt, num_threads * count * 2); + engine->HashIteratorRelease(hash_snapshot_iter); + + hash_snapshot_iter = + engine->HashIteratorCreate(hash_collection_after_snapshot, snapshot); + hash_snapshot_iter->SeekToFirst(); + ASSERT_FALSE(hash_snapshot_iter->Valid()); + engine->HashIteratorRelease(hash_snapshot_iter); + } + + { // List snapshot iterator + auto list_snapshot_iter = engine->ListIteratorCreate(list, snapshot); + engine->ListDestroy(list); + + uint64_t snapshot_iter_cnt = 0; + list_snapshot_iter->SeekToFirst(); + while (list_snapshot_iter->Valid()) { + ASSERT_TRUE(list_snapshot_iter->Valid()); + snapshot_iter_cnt++; + ASSERT_TRUE(list_snapshot_iter->Value()[0] == 'a' || + list_snapshot_iter->Value()[0] == 'b'); + list_snapshot_iter->Next(); + } + ASSERT_EQ(snapshot_iter_cnt, num_threads * count * 2); + engine->ListIteratorRelease(list_snapshot_iter); + + list_snapshot_iter = + engine->ListIteratorCreate(list_after_snapshot, snapshot); + list_snapshot_iter->SeekToFirst(); + ASSERT_FALSE(list_snapshot_iter->Valid()); + engine->ListIteratorRelease(list_snapshot_iter); + } + + delete engine; + + auto Validation = [&]() { + // Test backup and checkpoint instance + // All changes after snapshot should not be seen in backup and checkpoint + // Writes on backup should work well + size_t size; + ASSERT_EQ(engine->SortedSize(sorted_collection, &size), Status::Ok); + ASSERT_EQ(size, num_threads * count * 2); + ASSERT_EQ(engine->HashSize(hash_collection, &size), Status::Ok); + ASSERT_EQ(size, num_threads * count * 2); + ASSERT_EQ(engine->ListSize(list, &size), Status::Ok); + ASSERT_EQ(size, num_threads * count * 2); + for (uint32_t id = 0; id < num_threads; id++) { + int cnt = count; + std::string got_v1, got_v2, got_v3; + while (cnt--) { + std::string key1(std::string(id + 1, 'a') + std::to_string(cnt)); + std::string key2(std::string(id + 1, 'b') + std::to_string(cnt)); + std::string key3(std::string(id + 1, 'c') + std::to_string(cnt)); + // string + ASSERT_EQ(engine->Get(key1, &got_v1), Status::Ok); + ASSERT_EQ(engine->Get(key2, &got_v2), Status::Ok); + ASSERT_EQ(engine->Get(key3, &got_v3), Status::NotFound); + ASSERT_EQ(got_v1, key1); + ASSERT_EQ(got_v2, key2); + // sorted + ASSERT_EQ(engine->SortedGet(sorted_collection, key1, &got_v1), + Status::Ok); + ASSERT_EQ(engine->SortedGet(sorted_collection, key2, &got_v2), + Status::Ok); + ASSERT_EQ(engine->SortedGet(sorted_collection, key3, &got_v3), + Status::NotFound); + ASSERT_EQ(got_v1, key1); + ASSERT_EQ(got_v2, key2); + ASSERT_EQ( + engine->SortedGet(sorted_collection_after_snapshot, key1, &got_v1), + Status::NotFound); + // hash + ASSERT_EQ(engine->HashGet(hash_collection, key1, &got_v1), Status::Ok); + ASSERT_EQ(engine->HashGet(hash_collection, key2, &got_v2), Status::Ok); + ASSERT_EQ(engine->HashGet(hash_collection, key3, &got_v3), + Status::NotFound); + ASSERT_EQ(got_v1, key1); + ASSERT_EQ(got_v2, key2); + ASSERT_EQ( + engine->HashGet(hash_collection_after_snapshot, key1, &got_v1), + Status::NotFound); + } + } + + { // sorted iterator + uint64_t sorted_iter_cnt = 0; + auto sorted_iter = engine->SortedIteratorCreate(sorted_collection); + ASSERT_TRUE(sorted_iter != nullptr); + sorted_iter->SeekToFirst(); + while (sorted_iter->Valid()) { + ASSERT_TRUE(sorted_iter->Valid()); + sorted_iter_cnt++; + ASSERT_EQ(sorted_iter->Key(), sorted_iter->Value()); + sorted_iter->Next(); + } + ASSERT_EQ(sorted_iter_cnt, num_threads * count * 2); + engine->SortedIteratorRelease(sorted_iter); + ASSERT_EQ(engine->SortedIteratorCreate(sorted_collection_after_snapshot), + nullptr); + } + + { // hash iterator + uint64_t hash_iter_cnt = 0; + auto hash_iter = engine->HashIteratorCreate(hash_collection); + ASSERT_TRUE(hash_iter != nullptr); + hash_iter->SeekToFirst(); + while (hash_iter->Valid()) { + ASSERT_TRUE(hash_iter->Valid()); + hash_iter_cnt++; + ASSERT_EQ(hash_iter->Key(), hash_iter->Value()); + hash_iter->Next(); + } + engine->HashIteratorRelease(hash_iter); + ASSERT_EQ(hash_iter_cnt, num_threads * count * 2); + ASSERT_EQ(engine->HashIteratorCreate(hash_collection_after_snapshot), + nullptr); + } + + { // list iterator + uint64_t list_iter_cnt = 0; + auto list_iter = engine->ListIteratorCreate(list); + ASSERT_TRUE(list_iter != nullptr); + list_iter->SeekToFirst(); + while (list_iter->Valid()) { + ASSERT_TRUE(list_iter->Valid()); + list_iter_cnt++; + ASSERT_TRUE(list_iter->Value()[0] == 'a' || + list_iter->Value()[0] == 'b'); + list_iter->Next(); + } + engine->ListIteratorRelease(list_iter); + ASSERT_EQ(list_iter_cnt, num_threads * count * 2); + ASSERT_EQ(engine->ListIteratorCreate(hash_collection_after_snapshot), + nullptr); + } + }; + + std::vector opt_restore_skiplists{0, 1}; + for (auto is_opt : opt_restore_skiplists) { + configs.opt_large_sorted_collection_recovery = is_opt; + + ASSERT_EQ( + Engine::Restore(backup_path, backup_log, &engine, configs, stdout), + Status::Ok); + Validation(); + delete engine; + } +} + +TEST_F(EngineBasicTest, TestBasicStringOperations) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + do { + TestString(configs.max_access_threads); + } while (ChangeConfig()); + delete engine; +} + +TEST_F(EngineBasicTest, TestStringModify) { + struct IncNArgs { + size_t incr_by; + size_t result; + }; + auto IncN = [](const std::string* old_val, std::string* new_value, + void* modify_args) { + assert(modify_args); + IncNArgs* args = static_cast(modify_args); + size_t old_num; + if (old_val == nullptr) { + // if key not exist, start from 0 + old_num = 0; + } else { + if (old_val->size() != sizeof(size_t)) { + return ModifyOperation::Abort; + } else { + memcpy(&old_num, old_val->data(), sizeof(size_t)); + } + } + args->result = old_num + args->incr_by; + + new_value->assign((char*)&args->result, sizeof(size_t)); + return ModifyOperation::Write; + }; + + int num_threads = 16; + int ops_per_thread = 1000; + uint64_t incr_by = 5; + + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + std::string incr_key = "plus"; + + std::string wrong_value_key = "wrong_value"; + ASSERT_EQ(engine->Put(wrong_value_key, std::string(10, 'a')), Status::Ok); + + auto TestModify = [&](int) { + IncNArgs args{5, 0}; + ASSERT_EQ(engine->Modify(wrong_value_key, IncN, &args), Status::Abort); + for (int i = 0; i < ops_per_thread; i++) { + size_t prev_num = args.result; + ASSERT_EQ(engine->Modify(incr_key, IncN, &args), Status::Ok); + ASSERT_TRUE(args.result > prev_num); + } + }; + + LaunchNThreads(num_threads, TestModify); + std::string val; + size_t val_num; + ASSERT_EQ(engine->Get(incr_key, &val), Status::Ok); + ASSERT_EQ(val.size(), sizeof(size_t)); + memcpy(&val_num, val.data(), sizeof(size_t)); + ASSERT_EQ(val_num, ops_per_thread * num_threads * incr_by); + delete engine; +} + +TEST_F(BatchWriteTest, Sorted) { + size_t num_threads = 1; + for (int index_with_hashtable : {0, 1}) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + size_t batch_size = 100; + size_t count = 100 * batch_size; + + std::string collection_name{"sorted" + + std::to_string(index_with_hashtable)}; + SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = index_with_hashtable; + ASSERT_EQ(engine->SortedCreate(collection_name), Status::Ok); + + std::vector> elems(num_threads); + std::vector> values(num_threads); + for (size_t tid = 0; tid < num_threads; tid++) { + for (size_t i = 0; i < count; i++) { + elems[tid].push_back(std::to_string(tid) + "_" + std::to_string(i)); + values[tid].emplace_back(); + } + } + + auto Put = [&](size_t tid) { + for (size_t i = 0; i < count; i++) { + values[tid][i] = GetRandomString(120); + ASSERT_EQ( + engine->SortedPut(collection_name, elems[tid][i], values[tid][i]), + Status::Ok); + } + }; + + auto BatchWrite = [&](size_t tid) { + auto batch = engine->WriteBatchCreate(); + for (size_t i = 0; i < count; i++) { + if (i % 2 == 0) { + values[tid][i] = GetRandomString(120); + batch->SortedPut(collection_name, elems[tid][i], values[tid][i]); + } else { + values[tid][i].clear(); + batch->SortedDelete(collection_name, elems[tid][i]); + } + if ((i + 1) % batch_size == 0) { + // Delete a non-existing elem + batch->SortedDelete(collection_name, "non-existing"); + ASSERT_EQ(engine->BatchWrite(batch), Status::Ok); + batch->Clear(); + } + } + }; + + auto Check = [&](size_t tid) { + for (size_t i = 0; i < count; i++) { + std::string val_resp; + if (values[tid][i].empty()) { + ASSERT_EQ( + engine->SortedGet(collection_name, elems[tid][i], &val_resp), + Status::NotFound); + } else { + ASSERT_EQ( + engine->SortedGet(collection_name, elems[tid][i], &val_resp), + Status::Ok); + ASSERT_EQ(values[tid][i], val_resp); + } + } + }; + + LaunchNThreads(num_threads, BatchWrite); + LaunchNThreads(num_threads, Check); + + Reboot(); + LaunchNThreads(num_threads, Check); + LaunchNThreads(num_threads, Put); + LaunchNThreads(num_threads, Check); + LaunchNThreads(num_threads, BatchWrite); + LaunchNThreads(num_threads, Check); + + delete engine; + } +} + +TEST_F(BatchWriteTest, String) { + size_t num_threads = 16; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + size_t batch_size = 100; + size_t count = 100 * batch_size; + std::vector> keys(num_threads); + std::vector> values(num_threads); + for (size_t tid = 0; tid < num_threads; tid++) { + for (size_t i = 0; i < count; i++) { + keys[tid].push_back(std::to_string(tid) + "_" + std::to_string(i)); + values[tid].emplace_back(); + } + } + + auto Put = [&](size_t tid) { + for (size_t i = 0; i < count; i++) { + values[tid][i] = GetRandomString(120); + ASSERT_EQ(engine->Put(keys[tid][i], values[tid][i]), Status::Ok); + } + }; + + auto BatchWrite = [&](size_t tid) { + auto batch = engine->WriteBatchCreate(); + for (size_t i = 0; i < count; i++) { + if (i % 2 == 0) { + values[tid][i] = GetRandomString(120); + // The first Put is overwritten by the second Put. + batch->StringPut(keys[tid][i], GetRandomString(120)); + batch->StringPut(keys[tid][i], values[tid][i]); + } else { + values[tid][i].clear(); + batch->StringDelete(keys[tid][i]); + batch->StringDelete(keys[tid][i]); + } + if ((i + 1) % batch_size == 0) { + // Delete a non-existing key + batch->StringDelete("non-existing"); + ASSERT_EQ(batch->Size(), batch_size + 1); + ASSERT_EQ(engine->BatchWrite(batch), Status::Ok); + batch->Clear(); + } + } + }; + + auto Check = [&](size_t tid) { + for (size_t i = 0; i < count; i++) { + std::string val_resp; + if (values[tid][i].empty()) { + ASSERT_EQ(engine->Get(keys[tid][i], &val_resp), Status::NotFound); + } else { + ASSERT_EQ(engine->Get(keys[tid][i], &val_resp), Status::Ok); + ASSERT_EQ(values[tid][i], val_resp); + } + } + }; + + LaunchNThreads(num_threads, BatchWrite); + LaunchNThreads(num_threads, Check); + + Reboot(); + LaunchNThreads(num_threads, Check); + LaunchNThreads(num_threads, Put); + LaunchNThreads(num_threads, Check); + LaunchNThreads(num_threads, BatchWrite); + LaunchNThreads(num_threads, Check); + + delete engine; +} + +TEST_F(BatchWriteTest, Hash) { + size_t num_threads = 16; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + size_t batch_size = 100; + size_t count = 100 * batch_size; + + std::string key{"hash"}; + ASSERT_EQ(engine->HashCreate(key), Status::Ok); + + std::vector> fields(num_threads); + std::vector> values(num_threads); + for (size_t tid = 0; tid < num_threads; tid++) { + for (size_t i = 0; i < count; i++) { + fields[tid].push_back(std::to_string(tid) + "_" + std::to_string(i)); + values[tid].emplace_back(); + } + } + + auto Put = [&](size_t tid) { + for (size_t i = 0; i < count; i++) { + values[tid][i] = GetRandomString(120); + ASSERT_EQ(engine->HashPut(key, fields[tid][i], values[tid][i]), + Status::Ok); + } + }; + + auto BatchWrite = [&](size_t tid) { + auto batch = engine->WriteBatchCreate(); + for (size_t i = 0; i < count; i++) { + if (i % 2 == 0) { + values[tid][i] = GetRandomString(120); + batch->HashPut(key, fields[tid][i], values[tid][i]); + } else { + values[tid][i].clear(); + batch->HashDelete(key, fields[tid][i]); + } + if ((i + 1) % batch_size == 0) { + // Delete a non-existing key + batch->HashDelete(key, "non-existing"); + ASSERT_EQ(engine->BatchWrite(batch), Status::Ok); + batch->Clear(); + } + } + }; + + auto Check = [&](size_t tid) { + for (size_t i = 0; i < count; i++) { + std::string val_resp; + if (values[tid][i].empty()) { + ASSERT_EQ(engine->HashGet(key, fields[tid][i], &val_resp), + Status::NotFound); + } else { + ASSERT_EQ(engine->HashGet(key, fields[tid][i], &val_resp), Status::Ok); + ASSERT_EQ(values[tid][i], val_resp); + } + } + }; + + LaunchNThreads(num_threads, BatchWrite); + LaunchNThreads(num_threads, Check); + + Reboot(); + + LaunchNThreads(num_threads, Check); + LaunchNThreads(num_threads, Put); + LaunchNThreads(num_threads, Check); + LaunchNThreads(num_threads, BatchWrite); + LaunchNThreads(num_threads, Check); + + delete engine; +} + +TEST_F(EngineBasicTest, TestLocalSortedCollection) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + do { + for (int index_with_hashtable : {0, 1}) { + SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = index_with_hashtable; + TestLocalSortedCollection(engine, + "hash_index" + + std::to_string(index_with_hashtable) + + "thread_skiplist", + s_configs); + TestSortedIterator("hash_index" + std::to_string(index_with_hashtable) + + "thread_skiplist", + true); + } + } while (ChangeConfig()); + + delete engine; +} + +TEST_F(EngineBasicTest, TestGlobalSortedCollection) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + do { + for (int index_with_hashtable : {0, 1}) { + SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = index_with_hashtable; + std::string collection = + std::to_string(index_with_hashtable) + "global_skiplist"; + TestGlobalSortedCollection(collection, s_configs); + TestSortedIterator(collection, false); + } + } while (ChangeConfig()); + delete engine; +} + +TEST_F(EngineBasicTest, TestSeek) { + std::string val; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + // Test Seek + std::string collection = "col1"; + ASSERT_EQ(engine->SortedCreate(collection), Status::Ok); + uint64_t z = 0; + auto zero_filled_str = uint64_to_string(z); + ASSERT_EQ(engine->SortedPut(collection, zero_filled_str, zero_filled_str), + Status::Ok); + ASSERT_EQ(engine->SortedGet(collection, zero_filled_str, &val), Status::Ok); + auto iter = engine->SortedIteratorCreate(collection); + ASSERT_NE(iter, nullptr); + iter->Seek(zero_filled_str); + ASSERT_TRUE(iter->Valid()); + + // Test SeekToFirst + collection.assign("col2"); + ASSERT_EQ(engine->SortedCreate(collection), Status::Ok); + ASSERT_EQ(engine->SortedPut(collection, "foo", "bar"), Status::Ok); + ASSERT_EQ(engine->SortedGet(collection, "foo", &val), Status::Ok); + ASSERT_EQ(engine->SortedDelete(collection, "foo"), Status::Ok); + ASSERT_EQ(engine->SortedGet(collection, "foo", &val), Status::NotFound); + ASSERT_EQ(engine->SortedPut(collection, "foo2", "bar2"), Status::Ok); + engine->SortedIteratorRelease(iter); + iter = engine->SortedIteratorCreate(collection); + ASSERT_NE(iter, nullptr); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->Value(), "bar2"); + engine->SortedIteratorRelease(iter); + delete engine; +} + +TEST_F(EngineBasicTest, TestStringLargeValue) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + for (size_t sz = 1024; sz < (1UL << 30); sz *= 2) { + std::string key{"large"}; + std::string value(sz, 'a'); + std::string sink; + + ASSERT_EQ(engine->Put(key, value), Status::Ok); + ASSERT_EQ(engine->Get(key, &sink), Status::Ok); + ASSERT_EQ(value, sink); + } + delete engine; +} + +TEST_F(EngineBasicTest, TestList) { + size_t num_threads = 1; + size_t count = 1000; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + std::vector> elems_vec(num_threads); + std::vector list_vec(num_threads); + for (size_t i = 0; i < num_threads; i++) { + list_vec[i] = "List_" + std::to_string(i); + ASSERT_EQ(engine->ListCreate(list_vec[i]), Status::Ok); + ASSERT_EQ(engine->ListDestroy(list_vec[i]), Status::Ok); + ASSERT_EQ(engine->ListCreate(list_vec[i]), Status::Ok); + for (size_t j = 0; j < count; j++) { + elems_vec[i].push_back(std::to_string(i) + "_" + std::to_string(j)); + } + } + std::vector> list_copy_vec(num_threads); + + auto LPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t sz; + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(engine->ListPushFront(key, elems[j]), Status::Ok); + list_copy.push_front(elems[j]); + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + } + }; + + auto RPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t sz; + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(engine->ListPushBack(key, elems[j]), Status::Ok); + list_copy_vec[tid].push_back(elems[j]); + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + } + }; + + auto LPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + std::string value_got; + size_t sz; + for (size_t j = 0; j < count; j++) { + if (list_copy.empty()) { + ASSERT_EQ(engine->ListPopFront(key, &value_got), Status::NotFound); + break; + } + ASSERT_EQ(engine->ListPopFront(key, &value_got), Status::Ok); + ASSERT_EQ(list_copy.front(), value_got); + list_copy.pop_front(); + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + } + }; + + auto RPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + std::string value_got; + size_t sz; + for (size_t j = 0; j < count; j++) { + if (list_copy.empty()) { + ASSERT_EQ(engine->ListPopFront(key, &value_got), Status::NotFound); + break; + } + ASSERT_EQ(engine->ListPopBack(key, &value_got), Status::Ok); + ASSERT_EQ(list_copy.back(), value_got); + list_copy.pop_back(); + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + } + }; + + auto LBatchPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + for (size_t j = 0; j < count; j++) { + list_copy.push_front(elems[j]); + } + ASSERT_EQ(engine->ListBatchPushFront(key, elems), Status::Ok); + size_t sz; + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto RBatchPush = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto const& elems = elems_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + for (size_t j = 0; j < count; j++) { + list_copy.push_back(elems[j]); + } + ASSERT_EQ(engine->ListBatchPushBack(key, elems), Status::Ok); + size_t sz; + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto LBatchPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + std::vector elems_resp; + ASSERT_EQ(engine->ListBatchPopFront(key, count, &elems_resp), Status::Ok); + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(list_copy.front(), elems_resp[j]); + list_copy.pop_front(); + } + size_t sz; + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto RBatchPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + std::vector elems_resp; + ASSERT_EQ(engine->ListBatchPopBack(key, count, &elems_resp), Status::Ok); + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(list_copy.back(), elems_resp[j]); + list_copy.pop_back(); + } + size_t sz; + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto RPushLPop = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + + auto elem_copy = list_copy.front(); + list_copy.push_back(elem_copy); + list_copy.pop_front(); + + std::string elem; + ASSERT_EQ(engine->ListMove(key, ListPos::Front, key, ListPos::Back, &elem), + Status::Ok); + ASSERT_EQ(elem, elem_copy); + + size_t sz; + ASSERT_EQ(engine->ListSize(key, &sz), Status::Ok); + ASSERT_EQ(sz, list_copy.size()); + }; + + auto ListIterate = [&](size_t tid) { + auto const& key = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + + auto iter = engine->ListIteratorCreate(key); + ASSERT_TRUE((list_copy.empty() && iter == nullptr) || (iter != nullptr)); + if (iter != nullptr) { + iter->Seek(0); + for (auto iter2 = list_copy.begin(); iter2 != list_copy.end(); iter2++) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->Value(), *iter2); + iter->Next(); + } + + iter->Seek(-1); + for (auto iter2 = list_copy.rbegin(); iter2 != list_copy.rend(); + iter2++) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->Value(), *iter2); + iter->Prev(); + } + engine->ListIteratorRelease(iter); + } + }; + + auto ListInsertPutRemove = [&](size_t tid) { + auto const& list_name = list_vec[tid]; + auto& list_copy = list_copy_vec[tid]; + size_t len; + size_t const insert_pos = 5; + std::string elem; + + ASSERT_EQ(engine->ListSize(list_name, &len), Status::Ok); + ASSERT_GT(len, insert_pos); + + auto iter = engine->ListIteratorCreate(list_name); + ASSERT_NE(iter, nullptr); + + iter->Seek(insert_pos); + auto iter2 = std::next(list_copy.begin(), insert_pos); + ASSERT_EQ(iter->Value(), *iter2); + + elem = *iter2 + "_before"; + ASSERT_EQ(engine->ListInsertBefore(list_name, elem, iter->Value()), + Status::Ok); + iter2 = list_copy.insert(iter2, elem); + engine->ListIteratorRelease(iter); + iter = engine->ListIteratorCreate(list_name); + iter->Seek(insert_pos); + ASSERT_EQ(iter->Value(), *iter2); + + auto replace_pos = insert_pos - 2; + iter->Prev(); + iter->Prev(); + --iter2; + --iter2; + ASSERT_EQ(iter->Value(), *iter2); + elem = *iter2 + "_new"; + ASSERT_EQ(engine->ListReplace(list_name, replace_pos, elem), Status::Ok); + *iter2 = elem; + engine->ListIteratorRelease(iter); + iter = engine->ListIteratorCreate(list_name); + iter->Seek(replace_pos); + ASSERT_EQ(iter->Value(), *iter2); + + auto erase_pos = replace_pos - 2; + iter->Prev(); + iter->Prev(); + --iter2; + --iter2; + ASSERT_EQ(iter->Value(), *iter2); + std::string value; + ASSERT_EQ(engine->ListErase(list_name, erase_pos, &value), Status::Ok); + ASSERT_EQ(value, *iter2); + iter2 = list_copy.erase(iter2); + engine->ListIteratorRelease(iter); + iter = engine->ListIteratorCreate(list_name); + iter->Seek(erase_pos); + ASSERT_EQ(iter->Value(), *iter2); + engine->ListIteratorRelease(iter); + }; + + for (size_t i = 0; i < 3; i++) { + LaunchNThreads(num_threads, LPop); + LaunchNThreads(num_threads, RPop); + LaunchNThreads(num_threads, LPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LBatchPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RBatchPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LBatchPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RBatchPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPop); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPush); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, LPush); + LaunchNThreads(num_threads, ListIterate); + for (size_t j = 0; j < 100; j++) { + LaunchNThreads(num_threads, ListInsertPutRemove); + LaunchNThreads(num_threads, ListIterate); + LaunchNThreads(num_threads, RPushLPop); + LaunchNThreads(num_threads, ListIterate); + } + Reboot(); + } + + delete engine; +} + +TEST_F(EngineBasicTest, TestHash) { + size_t num_threads = 1; + size_t count = 1000; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + std::string key{"Hash"}; + ASSERT_EQ(engine->HashCreate(key), Status::Ok); + ASSERT_EQ(engine->HashDestroy(key), Status::Ok); + ASSERT_EQ(engine->HashCreate(key), Status::Ok); + using umap = std::unordered_map; + std::vector local_copies(num_threads); + std::mutex mu; + + auto HPut = [&](size_t tid) { + umap& local_copy = local_copies[tid]; + for (size_t j = 0; j < count; j++) { + std::string field{std::to_string(tid) + "_" + GetRandomString(10)}; + std::string value{GetRandomString(120)}; + ASSERT_EQ(engine->HashPut(key, field, value), Status::Ok); + local_copy[field] = value; + } + }; + + auto HGet = [&](size_t tid) { + umap const& local_copy = local_copies[tid]; + for (auto const& kv : local_copy) { + std::string resp; + ASSERT_EQ(engine->HashGet(key, kv.first, &resp), Status::Ok); + ASSERT_EQ(resp, kv.second) << "Field:\t" << kv.first << "\n"; + } + }; + + auto HDelete = [&](size_t tid) { + umap& local_copy = local_copies[tid]; + std::string sink; + for (size_t i = 0; i < count / 2; i++) { + auto iter = local_copy.begin(); + ASSERT_EQ(engine->HashDelete(key, iter->first), Status::Ok); + ASSERT_EQ(engine->HashGet(key, iter->first, &sink), Status::NotFound); + local_copy.erase(iter); + } + }; + + auto HashSize = [&](size_t) { + size_t len = 0; + ASSERT_EQ(engine->HashSize(key, &len), Status::Ok); + size_t cnt = 0; + for (size_t tid = 0; tid < num_threads; tid++) { + cnt += local_copies[tid].size(); + } + ASSERT_EQ(len, cnt); + }; + + auto HashIterate = [&](size_t tid) { + umap combined; + for (size_t tid = 0; tid < num_threads; tid++) { + umap const& local_copy = local_copies[tid]; + for (auto const& kv : local_copy) { + combined[kv.first] = kv.second; + } + } + + auto iter = engine->HashIteratorCreate(key); + + ASSERT_NE(iter, nullptr); + size_t cnt = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++cnt; + ASSERT_EQ(combined[iter->Key()], iter->Value()); + } + ASSERT_EQ(cnt, combined.size()); + + cnt = 0; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + ++cnt; + ASSERT_EQ(combined[iter->Key()], iter->Value()); + } + ASSERT_EQ(cnt, combined.size()); + + std::regex re1{".*"}; + std::regex re2{std::to_string(tid) + "_.*"}; + size_t match_cnt1 = 0; + size_t match_cnt2 = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + match_cnt1 += iter->MatchKey(re1) ? 1 : 0; + match_cnt2 += iter->MatchKey(re2) ? 1 : 0; + } + ASSERT_EQ(match_cnt1, combined.size()); + ASSERT_EQ(match_cnt2, local_copies[tid].size()); + engine->HashIteratorRelease(iter); + }; + + std::string counter{"counter"}; + auto HashModify = [&](size_t) { + struct FetchAddArgs { + size_t old; + size_t n; + }; + auto FetchAdd = [](std::string const* old_val, std::string* new_value, + void* args) { + FetchAddArgs* fa_args = static_cast(args); + if (old_val != nullptr) { + try { + fa_args->old = std::stoul(*old_val); + } catch (std::invalid_argument const&) { + return ModifyOperation::Abort; + } catch (std::out_of_range const&) { + return ModifyOperation::Abort; + } + } else { + fa_args->old = 0; + } + new_value->assign(std::to_string(fa_args->old + fa_args->n)); + return ModifyOperation::Write; + }; + + FetchAddArgs args; + args.n = 1; + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(engine->HashModify(key, counter, FetchAdd, &args), Status::Ok); + } + }; + + for (size_t i = 0; i < 3; i++) { + GlobalLogger.Debug("### round %lu ####\n", i); + Reboot(); + LaunchNThreads(num_threads, HPut); + LaunchNThreads(num_threads, HGet); + LaunchNThreads(num_threads, HDelete); + LaunchNThreads(num_threads, HashIterate); + LaunchNThreads(num_threads, HashSize); + LaunchNThreads(num_threads, HPut); + LaunchNThreads(num_threads, HGet); + LaunchNThreads(num_threads, HDelete); + LaunchNThreads(num_threads, HashIterate); + LaunchNThreads(num_threads, HashSize); + } + LaunchNThreads(num_threads, HashModify); + std::string resp; + ASSERT_EQ(engine->HashGet(key, counter, &resp), Status::Ok); + ASSERT_EQ(resp, std::to_string(num_threads * count)); + + delete engine; +} + +TEST_F(EngineBasicTest, TestStringHotspot) { + size_t n_thread_reading = 16; + size_t n_thread_writing = 16; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + size_t count = 100000; + std::string key{"SuperHotspot"}; + std::string val1(1024, 'a'); + std::string val2(1023, 'b'); + + ASSERT_EQ(engine->Put(key, val1), Status::Ok); + + auto EvenWriteOddRead = [&](uint32_t id) { + for (size_t i = 0; i < count; i++) { + if (id % 2 == 0) { + // Even Write + if (id % 4 == 0) { + ASSERT_EQ(engine->Put(key, val1), Status::Ok); + } else { + ASSERT_EQ(engine->Put(key, val2), Status::Ok); + } + } else { + // Odd Read + std::string got_val; + ASSERT_EQ(engine->Get(key, &got_val), Status::Ok); + bool match = false; + match = match || (got_val == val1); + match = match || (got_val == val2); + if (!match) { + std::string msg; + msg.append("Wrong value!\n"); + msg.append("The value should be 1024 of a's or 1023 of b's.\n"); + msg.append("Actual result is:\n"); + msg.append(got_val); + msg.append("\n"); + msg.append("Length: "); + msg.append(std::to_string(got_val.size())); + msg.append("\n"); + GlobalLogger.Error(msg.data()); + } + ASSERT_TRUE(match); + } + } + }; + + LaunchNThreads(n_thread_reading + n_thread_writing, EvenWriteOddRead); + delete engine; +} + +TEST_F(EngineBasicTest, TestSortedHotspot) { + size_t n_thread_reading = 16; + size_t n_thread_writing = 16; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + size_t count = 100000; + std::string collection_name{"collection"}; + std::vector keys{"SuperHotSpot0", "SuperHotSpot2", + "SuperHotSpot1"}; + std::string val1(1024, 'a'); + std::string val2(1024, 'b'); + ASSERT_EQ(engine->SortedCreate(collection_name), Status::Ok); + + for (const std::string& key : keys) { + ASSERT_EQ(engine->SortedPut(collection_name, key, val1), Status::Ok); + + auto EvenWriteOddRead = [&](uint32_t id) { + for (size_t i = 0; i < count; i++) { + if (id % 2 == 0) { + // Even Write + if (id % 4 == 0) { + ASSERT_EQ(engine->SortedPut(collection_name, key, val1), + Status::Ok); + } else { + ASSERT_EQ(engine->SortedPut(collection_name, key, val2), + Status::Ok); + } + } else { + // Odd Read + std::string got_val; + ASSERT_EQ(engine->SortedGet(collection_name, key, &got_val), + Status::Ok); + bool match = false; + match = match || (got_val == val1); + match = match || (got_val == val2); + if (!match) { + std::string msg; + msg.append("Wrong value!\n"); + msg.append("The value should be 1024 of a's or 1023 of b's.\n"); + msg.append("Actual result is:\n"); + msg.append(got_val); + msg.append("\n"); + msg.append("Length: "); + msg.append(std::to_string(got_val.size())); + msg.append("\n"); + GlobalLogger.Error(msg.data()); + } + ASSERT_TRUE(match); + } + } + }; + + LaunchNThreads(n_thread_reading + n_thread_writing, EvenWriteOddRead); + } + delete engine; +} + +TEST_F(EngineBasicTest, TestSortedCustomCompareFunction) { + using kvpair = std::pair; + size_t num_threads = 16; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + std::vector collections{"collection0", "collection1", + "collection2"}; + + auto cmp0 = [](const StringView& a, const StringView& b) -> int { + double scorea = std::stod(string_view_2_string(a)); + double scoreb = std::stod(string_view_2_string(b)); + if (scorea == scoreb) + return 0; + else if (scorea < scoreb) + return 1; + else + return -1; + }; + + auto cmp1 = [](const StringView& a, const StringView& b) -> int { + double scorea = std::stod(string_view_2_string(a)); + double scoreb = std::stod(string_view_2_string(b)); + if (scorea == scoreb) + return 0; + else if (scorea > scoreb) + return 1; + else + return -1; + }; + + size_t count = 10; + std::vector key_values(count); + std::map dedup_kvs; + std::generate(key_values.begin(), key_values.end(), [&]() { + const char v = rand() % (90 - 65 + 1) + 65; + std::string k = std::to_string(rand() % 100); + dedup_kvs[k] = v; + return std::make_pair(k, std::string(1, v)); + }); + + // register compare function + engine->registerComparator("collection0_cmp", cmp0); + engine->registerComparator("collection1_cmp", cmp1); + for (size_t i = 0; i < collections.size(); ++i) { + Status s; + if (i < 2) { + std::string comp_name = "collection" + std::to_string(i) + "_cmp"; + SortedCollectionConfigs s_configs; + s_configs.comparator_name = comp_name; + s = engine->SortedCreate(collections[i], s_configs); + } else { + s = engine->SortedCreate(collections[i]); + } + ASSERT_EQ(s, Status::Ok); + } + for (size_t i = 0; i < collections.size(); ++i) { + auto Write = [&](size_t) { + for (size_t j = 0; j < count; j++) { + ASSERT_EQ(engine->SortedPut(collections[i], key_values[j].first, + key_values[j].second), + Status::Ok); + } + }; + LaunchNThreads(num_threads, Write); + } + + for (size_t i = 0; i < collections.size(); ++i) { + std::vector expected_res(dedup_kvs.begin(), dedup_kvs.end()); + if (i == 0) { + std::sort(expected_res.begin(), expected_res.end(), + [&](const kvpair& a, const kvpair& b) -> bool { + return cmp0(a.first, b.first) <= 0; + }); + + } else if (i == 1) { + std::sort(expected_res.begin(), expected_res.end(), + [&](const kvpair& a, const kvpair& b) -> bool { + return cmp1(a.first, b.first) <= 0; + }); + } + auto iter = engine->SortedIteratorCreate(collections[i]); + ASSERT_TRUE(iter != nullptr); + iter->SeekToFirst(); + size_t cnt = 0; + while (iter->Valid()) { + std::string key = iter->Key(); + std::string val = iter->Value(); + ASSERT_EQ(key, expected_res[cnt].first); + ASSERT_EQ(val, expected_res[cnt].second); + iter->Next(); + cnt++; + } + engine->SortedIteratorRelease(iter); + } + delete engine; +} + +TEST_F(EngineBasicTest, TestHashTableIterator) { + size_t num_threads = 32; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + std::string collection_name = "sortedcollection"; + engine->SortedCreate(collection_name); + auto MixedPut = [&](size_t id) { + if (id % 2 == 0) { + ASSERT_EQ(engine->Put("stringkey" + std::to_string(id), "stringval"), + Status::Ok); + } else { + ASSERT_EQ( + engine->SortedPut(collection_name, "sortedkey" + std::to_string(id), + "sortedval"), + Status::Ok); + } + }; + LaunchNThreads(num_threads, MixedPut); + + auto test_kvengine = static_cast(engine); + auto hash_table = test_kvengine->GetHashTable(); + size_t total_entry_num = 0; + // Hash Table Iterator + // scan hash table with locked slot. + { + auto hashtable_iter = hash_table->GetIterator(0, hash_table->GetSlotsNum()); + while (hashtable_iter.Valid()) { + auto slot_iter = hashtable_iter.Slot(); + while (slot_iter.Valid()) { + switch (slot_iter->GetIndexType()) { + case PointerType::StringRecord: { + total_entry_num++; + ASSERT_EQ(string_view_2_string( + slot_iter->GetIndex().string_record->Value()), + "stringval"); + break; + } + case PointerType::Skiplist: { + total_entry_num++; + ASSERT_EQ( + string_view_2_string(slot_iter->GetIndex().skiplist->Name()), + collection_name); + break; + } + case PointerType::SkiplistNode: { + total_entry_num++; + ASSERT_EQ(string_view_2_string( + slot_iter->GetIndex().skiplist_node->record->Value()), + "sortedval"); + break; + } + case PointerType::DLRecord: { + total_entry_num++; + ASSERT_EQ( + string_view_2_string(slot_iter->GetIndex().dl_record->Value()), + "sortedval"); + break; + } + default: + ASSERT_EQ((slot_iter->GetIndexType() == PointerType::Invalid) || + (slot_iter->GetIndexType() == PointerType::Empty), + true); + break; + } + slot_iter++; + } + hashtable_iter.Next(); + } + ASSERT_EQ(total_entry_num, num_threads + 1); + } + delete engine; +} + +TEST_F(EngineBasicTest, TestExpireAPI) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + std::string got_val; + int64_t ttl_time; + std::string key = "expired_key"; + std::string val(10, 'a'); + std::string val2(10, 'b'); + std::string list_collection = "ListCollection"; + std::string sorted_collection = "SortedCollection"; + std::string hashes_collection = "HashesCollection"; + int64_t normal_ttl_time = 10000; /* 10s */ + int64_t max_ttl_time = INT64_MAX - 1; + + // For string + { + // key is expired. Check expired time when reading. + WriteOptions write_options1{1, true}; + ASSERT_EQ(engine->Put(key, val, write_options1), Status::Ok); + sleep(1); + ASSERT_EQ(engine->Get(key, &got_val), Status::NotFound); + + // update kv pair with new expired time. + WriteOptions write_options2{INT64_MAX / 1000, true}; + ASSERT_EQ(engine->Put(key, val2, write_options2), Status::Ok); + ASSERT_EQ(engine->Get(key, &got_val), Status::Ok); + ASSERT_EQ(got_val, val2); + + // test update_ttl option + WriteOptions write_options3{1, false}; + ASSERT_EQ(engine->Put(key, val, write_options3), Status::Ok); + sleep(1); + ASSERT_EQ(engine->Get(key, &got_val), Status::Ok); + ASSERT_EQ(got_val, val); + ASSERT_EQ( + engine->Modify( + key, + [=](const std::string* old_value, std::string* new_value, void*) { + if (old_value != nullptr) { + new_value->assign(val2); + return ModifyOperation::Write; + } + return ModifyOperation::Abort; + }, + nullptr, write_options3), + Status::Ok); + sleep(1); + ASSERT_EQ(engine->Get(key, &got_val), Status::Ok); + ASSERT_EQ(got_val, val2); + + // Get expired time. + ASSERT_EQ(engine->GetTTL(key, &ttl_time), Status::Ok); + + // reset expired time for string record. + ASSERT_EQ(engine->Expire(key, normal_ttl_time), Status::Ok); + + // set negative ttl time. + std::string expire_key = "expired_key1"; + std::string expire_val = "expired_val1"; + ASSERT_EQ(engine->Put(expire_key, expire_val, WriteOptions{}), Status::Ok); + ASSERT_EQ(engine->GetTTL(expire_key, &ttl_time), Status::Ok); + ASSERT_EQ(ttl_time, kPersistTime); + ASSERT_EQ(engine->Expire(expire_key, -30), Status::Ok); + ASSERT_EQ(engine->GetTTL(expire_key, &ttl_time), Status::NotFound); + } + + // For sorte collection + { + Status s = engine->SortedCreate(sorted_collection); + ASSERT_EQ(s, Status::Ok); + ASSERT_EQ( + engine->SortedPut(sorted_collection, "sorted" + key, "sorted" + val), + Status::Ok); + // Set expired time for collection + ASSERT_EQ(engine->Expire(sorted_collection, max_ttl_time), + Status::InvalidArgument); + ASSERT_EQ( + engine->SortedPut(sorted_collection, "sorted2" + key, "sorted2" + val), + Status::Ok); + ASSERT_EQ(engine->GetTTL(sorted_collection, &ttl_time), Status::Ok); + // check sorted_collection is persist; + ASSERT_EQ(ttl_time, kPersistTTL); + // reset expired time for collection + ASSERT_EQ(engine->Expire(sorted_collection, 2), Status::Ok); + sleep(2); + ASSERT_EQ(engine->SortedGet(sorted_collection, "sorted" + key, &got_val), + Status::NotFound); + ASSERT_EQ(engine->GetTTL(sorted_collection, &ttl_time), Status::NotFound); + ASSERT_EQ(ttl_time, kInvalidTTL); + + // set negative or 0 ttl time. + ASSERT_EQ(engine->SortedCreate(sorted_collection), Status::Ok); + ASSERT_EQ(engine->GetTTL(sorted_collection, &ttl_time), Status::Ok); + ASSERT_EQ(ttl_time, kPersistTime); + ASSERT_EQ(engine->Expire(sorted_collection, 0), Status::Ok); + ASSERT_EQ(engine->GetTTL(sorted_collection, &ttl_time), Status::NotFound); + ASSERT_EQ(ttl_time, kInvalidTTL); + } + + // For hashes collection + { + ASSERT_EQ(engine->HashCreate(hashes_collection), Status::Ok); + ASSERT_EQ( + engine->HashPut(hashes_collection, "hashes" + key, "hashes" + val), + Status::Ok); + // Set expired time for collection, max_ttl_time is overflow. + ASSERT_EQ(engine->Expire(hashes_collection, max_ttl_time), + Status::InvalidArgument); + ASSERT_EQ( + engine->HashPut(hashes_collection, "hashes2" + key, "hashes2" + val), + Status::Ok); + + // reset expired time for collection + ASSERT_EQ(engine->Expire(hashes_collection, normal_ttl_time), Status::Ok); + ASSERT_EQ(engine->HashGet(hashes_collection, "hashes" + key, &got_val), + Status::Ok); + ASSERT_EQ(got_val, "hashes" + val); + // get collection ttl time + sleep(2); + ASSERT_EQ(engine->GetTTL(hashes_collection, &ttl_time), Status::Ok); + } + + // For list + { + ASSERT_EQ(engine->ListCreate(list_collection), Status::Ok); + ASSERT_EQ(engine->ListPushFront(list_collection, "list" + val), Status::Ok); + // Set expired time for collection + ASSERT_EQ(engine->Expire(list_collection, max_ttl_time), + Status::InvalidArgument); + ASSERT_EQ(engine->GetTTL(list_collection, &ttl_time), Status::Ok); + // check list is persist + ASSERT_EQ(ttl_time, kPersistTime); + // reset expired time for collection + ASSERT_EQ(engine->Expire(list_collection, normal_ttl_time), Status::Ok); + ASSERT_EQ(engine->GetTTL(list_collection, &ttl_time), Status::Ok); + } + + // Close engine and Recovery + Reboot(); + + // Get string record expired time + ASSERT_EQ(engine->GetTTL(key, &ttl_time), Status::Ok); + + // Get sorted record expired time + ASSERT_EQ(engine->GetTTL(sorted_collection, &ttl_time), Status::NotFound); + + // Get hashes record expired time + ASSERT_EQ(engine->GetTTL(hashes_collection, &ttl_time), Status::Ok); + + // Get list record expired time + ASSERT_EQ(engine->GetTTL(list_collection, &ttl_time), Status::Ok); + delete engine; +} + +TEST_F(EngineBasicTest, TestbackgroundDestroyCollections) { + size_t n_thread_writing = 16; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + TTLType ttl = 1000; // 1s + int cnt = 100; + size_t num_thread = n_thread_writing; + + auto list0_push = [&](size_t id) { + std::string list_key0 = "listkey0" + std::to_string(id); + ASSERT_EQ(engine->ListCreate(list_key0), Status::Ok); + for (int i = 0; i < cnt; ++i) { + ASSERT_EQ( + engine->ListPushFront(list_key0, "list_elem" + std::to_string(i)), + Status::Ok); + } + ASSERT_EQ(engine->Expire(list_key0, ttl), Status::Ok); + }; + auto list1_push = [&](size_t id) { + std::string list_key1 = "listkey1" + std::to_string(id); + ASSERT_EQ(engine->ListCreate(list_key1), Status::Ok); + for (int i = 0; i < cnt; ++i) { + ASSERT_EQ( + engine->ListPushFront(list_key1, "list_elem" + std::to_string(i)), + Status::Ok); + } + }; + auto hash0_push = [&](size_t id) { + std::string hash_key0 = "hashkey0" + std::to_string(id); + ASSERT_EQ(engine->HashCreate(hash_key0), Status::Ok); + for (int i = 0; i < cnt; ++i) { + std::string str = std::to_string(i); + ASSERT_EQ( + engine->HashPut(hash_key0, "hash_elem" + str, "hash_value" + str), + Status::Ok); + } + ASSERT_EQ(engine->Expire(hash_key0, ttl), Status::Ok); + }; + + auto sorted0_push = [&](size_t id) { + std::string sorted_key0 = "sortedkey0" + std::to_string(id); + ASSERT_EQ(engine->SortedCreate(sorted_key0), Status::Ok); + for (int i = 0; i < cnt; ++i) { + std::string str = std::to_string(i); + ASSERT_EQ(engine->SortedPut(sorted_key0, "sorted_elem" + str, + "sorted_value" + str), + Status::Ok); + } + ASSERT_EQ(engine->Expire(sorted_key0, ttl), Status::Ok); + }; + + LaunchNThreads(num_thread, list0_push); + LaunchNThreads(num_thread, list1_push); + LaunchNThreads(num_thread, hash0_push); + LaunchNThreads(num_thread, sorted0_push); + + sleep(2); + for (size_t i = 0; i < num_thread; ++i) { + std::string str = std::to_string(i); + TTLType got_ttl; + ASSERT_EQ(engine->GetTTL("hashkey0" + str, &got_ttl), Status::NotFound); + ASSERT_EQ(engine->GetTTL("listkey0" + str, &got_ttl), Status::NotFound); + ASSERT_EQ(engine->GetTTL("sortedkey0" + str, &got_ttl), Status::NotFound); + ASSERT_EQ(engine->GetTTL("listkey1" + str, &got_ttl), Status::Ok); + } + + delete engine; +} + +// ========================= Sync Point ====================================== + +#if KVDK_DEBUG_LEVEL > 0 +// Example: +// {key0, val0} <-> {key2, val2} +// thread1 insert : {key0, val0} <-> {key1, val1} <-> {key2, val2} +// thread2: iter +TEST_F(EngineBasicTest, TestSortedSyncPoint) { + Configs test_config = configs; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, test_config, stdout), + Status::Ok); + std::vector ths; + std::string collection_name = "skiplist"; + ASSERT_EQ(engine->SortedCreate(collection_name), Status::Ok); + + engine->SortedPut(collection_name, "key0", "val0"); + engine->SortedPut(collection_name, "key2", "val2"); + + std::atomic first_record(false); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->Reset(); + SyncPoint::GetInstance()->LoadDependency( + {{"KVEngine::DLList::LinkDLRecord::HalfLink", "Test::Iter::key0"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + // insert + ths.emplace_back(std::thread([&]() { + engine->SortedPut(collection_name, "key1", "val1"); + std::string got_val; + ASSERT_EQ(engine->SortedGet(collection_name, "key1", &got_val), Status::Ok); + })); + + // Iter + ths.emplace_back(std::thread([&]() { + sleep(1); + auto sorted_iter = engine->SortedIteratorCreate(collection_name); + sorted_iter->SeekToLast(); + if (sorted_iter->Valid()) { + std::string next = sorted_iter->Key(); + ASSERT_EQ(next, "key2"); + sorted_iter->Prev(); + while (sorted_iter->Valid()) { + std::string k = sorted_iter->Key(); + TEST_SYNC_POINT("Test::Iter::" + k); + if (k == "key0") { + sorted_iter->Next(); + ASSERT_EQ(sorted_iter->Key(), "key1"); + sorted_iter->Prev(); + } + sorted_iter->Prev(); + ASSERT_EQ(true, k.compare(next) < 0); + next = k; + } + } + engine->SortedIteratorRelease(sorted_iter); + })); + for (auto& thread : ths) { + thread.join(); + } + delete engine; +} + +TEST_F(EngineBasicTest, TestHashTableRangeIter) { + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + std::string key = "stringkey"; + std::string val = "stringval"; + std::string updated_val = "stringupdatedval"; + + ASSERT_EQ(engine->Put(key, val), Status::Ok); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->Reset(); + SyncPoint::GetInstance()->LoadDependency( + {{"ScanHashTable", "KVEngine::stringPutImpl::BeforeLock"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + auto StringUpdate = [&]() { + ASSERT_EQ(engine->Put(key, updated_val), Status::Ok); + }; + + auto HashTableScan = [&]() { + auto test_kvengine = static_cast(engine); + auto hash_table = test_kvengine->GetHashTable(); + auto hashtable_iter = hash_table->GetIterator(0, hash_table->GetSlotsNum()); + while (hashtable_iter.Valid()) { + auto slot_lock = hashtable_iter.AcquireSlotLock(); + auto slot_iter = hashtable_iter.Slot(); + while (slot_iter.Valid()) { + if (slot_iter->GetIndexType() == PointerType::StringRecord) { + TEST_SYNC_POINT("ScanHashTable"); + sleep(2); + ASSERT_EQ(slot_iter->GetIndex().string_record->Key(), key); + ASSERT_EQ(slot_iter->GetIndex().string_record->Value(), val); + } + slot_iter++; + } + hashtable_iter.Next(); + } + }; + + std::vector ts; + ts.emplace_back(std::thread(StringUpdate)); + ts.emplace_back(std::thread(HashTableScan)); + for (auto& t : ts) t.join(); + delete engine; +} + +TEST_F(EngineBasicTest, TestBackGroundCleaner) { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->Reset(); + // abandon background cleaner thread + SyncPoint::GetInstance()->SetCallBack( + "KVEngine::backgroundCleaner::NothingToDo", [&](void* close_reclaimer) { + *((std::atomic_bool*)close_reclaimer) = true; + return; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + int cnt = 100; + + auto PutString = [&]() { + for (int i = 0; i < cnt; ++i) { + std::string key = std::to_string(i) + "stringk"; + std::string val = std::to_string(i) + "stringval"; + ASSERT_EQ(engine->Put(key, val, WriteOptions{INT32_MAX}), Status::Ok); + } + }; + + auto ExpireString = [&]() { + for (int i = 0; i < cnt; ++i) { + // string + std::string key = std::to_string(i) + "stringk"; + std::string got_val; + if (engine->Get(key, &got_val) == Status::Ok) { + ASSERT_EQ(engine->Expire(key, 1), Status::Ok); + } + } + }; + + auto GetString = [&]() { + for (int i = 0; i < cnt; ++i) { + // string + std::string key = std::to_string(i) + "stringk"; + std::string got_val; + int64_t ttl_time; + Status s = engine->GetTTL(key, &ttl_time); + if (s == Status::Ok) { + if (ttl_time > 1) { + ASSERT_EQ(ttl_time <= INT32_MAX, true); + } + } else { + ASSERT_EQ(s, Status::NotFound); + ASSERT_EQ(ttl_time, kInvalidTTL); + } + } + }; + + auto PutSorted = [&]() { + for (int i = 0; i < cnt; ++i) { + for (int index_with_hashtable : {0, 1}) { + std::string sorted_collection = + std::to_string(i) + "sorted" + std::to_string(index_with_hashtable); + std::string key = std::to_string(i) + "sortedk"; + std::string val = std::to_string(i) + "sortedval"; + SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = index_with_hashtable; + ASSERT_EQ(engine->SortedCreate(sorted_collection, s_configs), + Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection, key, val), Status::Ok); + bool set_expire = fast_random_64() % 2 == 0; + if (set_expire) { + ASSERT_EQ(engine->Expire(sorted_collection, 1), Status::Ok); + } + } + } + }; + + auto GetSorted = [&]() { + for (int i = 0; i < cnt; ++i) { + for (int index_with_hashtable : {0, 1}) { + std::string sorted_collection = + std::to_string(i) + "sorted" + std::to_string(index_with_hashtable); + std::string key = std::to_string(i) + "sortedk"; + std::string val = std::to_string(i) + "sortedval"; + std::string got_val; + int64_t ttl_time; + Status s = engine->GetTTL(sorted_collection, &ttl_time); + if (s == Status::Ok) { + if (ttl_time == kPersistTime) { + ASSERT_EQ(engine->SortedGet(sorted_collection, key, &got_val), + Status::Ok); + ASSERT_EQ(got_val, val); + } else { + ASSERT_TRUE(ttl_time <= 1); + } + } else { + ASSERT_EQ(s, Status::NotFound); + ASSERT_EQ(ttl_time, kInvalidTTL); + ASSERT_EQ(engine->SortedGet(sorted_collection, key, &got_val), + Status::NotFound); + } + } + } + }; + + std::string sorted_collection = "sorted_collection"; + std::string list_collection = "list_collection"; + std::string hashlist_collection = "hashlist_collection"; + + auto CreateAndDestroySorted = [&]() { + std::string key = "sorted_key"; + ASSERT_EQ(engine->SortedCreate(sorted_collection), Status::Ok); + for (int i = 0; i < cnt; ++i) { + auto new_key = key + std::to_string(i); + ASSERT_EQ(engine->SortedPut(sorted_collection, new_key, "sorted_value"), + Status::Ok); + ASSERT_EQ( + engine->SortedPut(sorted_collection, new_key, "sorted_update_value"), + Status::Ok); + if (i % 2 != 0) { + ASSERT_EQ(engine->SortedDelete(sorted_collection, new_key), Status::Ok); + } + } + ASSERT_EQ(engine->SortedDestroy(sorted_collection), Status::Ok); + ASSERT_EQ(engine->SortedCreate(sorted_collection), Status::Ok); + }; + + auto CreateAndDestroyList = [&]() { + std::string key = "list_key"; + std::string got_key; + ASSERT_EQ(engine->ListCreate(list_collection), Status::Ok); + for (int i = 0; i < cnt; ++i) { + auto new_key = key + std::to_string(i); + ASSERT_EQ(engine->ListPushFront(list_collection, new_key), Status::Ok); + ASSERT_EQ(engine->ListPopBack(list_collection, &got_key), Status::Ok); + ASSERT_EQ(got_key, new_key); + } + ASSERT_EQ(engine->ListDestroy(list_collection), Status::Ok); + ASSERT_EQ(engine->ListCreate(list_collection), Status::Ok); + }; + + auto CreateAndDestroyHashList = [&]() { + std::string key = "hashlist_key"; + ASSERT_EQ(engine->HashCreate(hashlist_collection), Status::Ok); + for (int i = 0; i < cnt; ++i) { + auto new_key = key + std::to_string(i); + ASSERT_EQ(engine->HashPut(hashlist_collection, new_key, "hashlist_value"), + Status::Ok); + ASSERT_EQ(engine->HashPut(hashlist_collection, new_key, + "hashlist_update_value"), + Status::Ok); + if (i % 2 != 0) { + ASSERT_EQ(engine->HashDelete(hashlist_collection, new_key), Status::Ok); + } + } + ASSERT_EQ(engine->HashDestroy(hashlist_collection), Status::Ok); + ASSERT_EQ(engine->HashCreate(hashlist_collection), Status::Ok); + }; + + auto ExpiredClean = [&]() { + auto test_kvengine = static_cast(engine); + auto cleaner = test_kvengine->EngineCleaner(); + cleaner->Start(); + sleep(2); + cleaner->Close(); + }; + + { + std::vector ts; + ts.emplace_back(std::thread(PutString)); + ts.emplace_back(std::thread(ExpireString)); + ts.emplace_back(std::thread(ExpiredClean)); + + for (auto& t : ts) t.join(); + + // check + GetString(); + } + + { + std::vector ts; + ts.emplace_back(std::thread(PutString)); + ts.emplace_back(std::thread(ExpireString)); + ts.emplace_back(std::thread(ExpiredClean)); + ts.emplace_back(std::thread(GetString)); + for (auto& t : ts) t.join(); + } + + { + PutSorted(); + auto t = std::thread(ExpiredClean); + GetSorted(); + t.join(); + } + + { + CreateAndDestroySorted(); + CreateAndDestroyHashList(); + CreateAndDestroyList(); + ExpiredClean(); + { + size_t size; + ASSERT_EQ(engine->SortedSize(sorted_collection, &size), Status::Ok); + ASSERT_EQ(size, 0); + ASSERT_EQ(engine->HashSize(hashlist_collection, &size), Status::Ok); + ASSERT_EQ(size, 0); + ASSERT_EQ(engine->ListSize(list_collection, &size), Status::Ok); + ASSERT_EQ(size, 0); + } + } + delete engine; +} + +TEST_F(EngineBasicTest, TestBackGroundIterNoHashIndexSkiplist) { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->Reset(); + // abandon background cleaner thread + SyncPoint::GetInstance()->SetCallBack( + "KVEngine::backgroundCleaner::NothingToDo", [&](void* close_reclaimer) { + *((std::atomic_bool*)close_reclaimer) = true; + return; + }); + SyncPoint::GetInstance()->LoadDependency( + {{"KVEngine::BackgroundCleaner::IterSkiplist::UnlinkDeleteRecord", + "KVEngine::SkiplistNoHashIndex::Put"}}); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + std::string collection_name = "Skiplist_with_hash_index"; + SortedCollectionConfigs s_configs; + s_configs.index_with_hashtable = false; + ASSERT_EQ(engine->SortedCreate(collection_name, s_configs), Status::Ok); + int cnt = 100; + + // Two case: (1) record->old_record->old_record; + // (2)record->delete_record->old_record + auto PutAndDeleteSorted = [&]() { + for (int i = 0; i < cnt; ++i) { + std::string key = "sorted_key" + std::to_string(i); + std::string value = "sorted_value" + std::to_string(i); + ASSERT_EQ(engine->SortedPut(collection_name, key, value), Status::Ok); + if ((i % 2) == 0) { + ASSERT_EQ(engine->SortedDelete(collection_name, key), Status::Ok); + } else { + ASSERT_EQ(engine->SortedPut(collection_name, key, + "update_value" + std::to_string(i)), + Status::Ok); + } + + TEST_SYNC_POINT("KVEngine::SkiplistNoHashIndex::Put"); + ASSERT_EQ(engine->SortedPut(collection_name, key, + "update_value_again" + std::to_string(i)), + Status::Ok); + } + }; + + auto backgroundCleaner = [&]() { + auto test_kvengine = static_cast(engine); + auto cleaner = test_kvengine->EngineCleaner(); + cleaner->Start(); + sleep(2); + cleaner->Close(); + }; + std::vector ts; + ts.emplace_back(PutAndDeleteSorted); + ts.emplace_back(backgroundCleaner); + for (auto& t : ts) t.join(); + + int entries = 0; + // iterating sorted collection + auto iter = engine->SortedIteratorCreate(collection_name); + ASSERT_TRUE(iter != nullptr); + // forward iterator + iter->SeekToFirst(); + if (iter->Valid()) { + ++entries; + std::string prev = iter->Key(); + iter->Next(); + while (iter->Valid()) { + ++entries; + std::string k = iter->Key(); + iter->Next(); + ASSERT_EQ(true, k.compare(prev) > 0); + prev = k; + } + } + engine->SortedIteratorRelease(iter); + ASSERT_EQ(entries, cnt); + delete engine; +} + +TEST_F(EngineBasicTest, TestDynamicCleaner) { + enum class OpType { insert, update, outdated }; + OpType op; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->Reset(); + // abandon background cleaner thread + SyncPoint::GetInstance()->SetCallBack( + "KVEngine::backgroundCleaner::NothingToDo", [&](void* close_reclaimer) { + *((std::atomic_bool*)close_reclaimer) = true; + return; + }); + SyncPoint::GetInstance()->SetCallBack("KVEngine::Cleaner::AdjustCleanWorkers", + [&](void* advice_thread_num) { + if (op == OpType::update) { + *((size_t*)advice_thread_num) = 6; + } else if (op == OpType::outdated) { + *((size_t*)advice_thread_num) = 8; + } else { + *((size_t*)advice_thread_num) = 1; + } + return; + }); + SyncPoint::GetInstance()->EnableProcessing(); + configs.hash_bucket_num = 256; + configs.clean_threads = 8; + ASSERT_EQ(Engine::Open(db_path.c_str(), &engine, configs, stdout), + Status::Ok); + + std::string sorted_collection = "sorted_collection"; + std::string common_str_key = "string_key"; + std::string common_sorted_key = "sorted_key"; + std::string common_value = "val"; + ASSERT_EQ(engine->SortedCreate(sorted_collection), Status::Ok); + auto test_kvengine = static_cast(engine); + auto space_cleaner = test_kvengine->EngineCleaner(); + space_cleaner->Start(); + + size_t cnt = 16; + // only insert + op = OpType::insert; + for (size_t id = 0; id < cnt; ++id) { + std::string value = std::to_string(id) + common_value; + std::string str_key = std::to_string(id) + common_str_key; + std::string sorted_key = std::to_string(id) + common_sorted_key; + ASSERT_EQ(engine->Put(str_key, value), Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection, sorted_key, value), + Status::Ok); + } + ASSERT_EQ(space_cleaner->ActiveThreadNum(), 1); + + // update + op = OpType::update; + sleep(1); + for (size_t id = 0; id < cnt; ++id) { + std::string str_key = std::to_string(id) + common_str_key; + std::string sorted_key = std::to_string(id) + common_sorted_key; + std::string value = std::to_string(id) + common_value; + for (int i = 0; i < 100; ++i) { + auto update_val = value + std::to_string(i); + ASSERT_EQ(engine->Put(str_key, update_val), Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection, sorted_key, update_val), + Status::Ok); + } + } + ASSERT_EQ(space_cleaner->ActiveThreadNum(), 6); + op = OpType::outdated; + sleep(1); + for (size_t id = 0; id < cnt; ++id) { + std::string str_key = std::to_string(id) + common_str_key; + std::string sorted_key = std::to_string(id) + common_sorted_key; + ASSERT_EQ(engine->Expire(str_key, -1), Status::Ok); + ASSERT_EQ(engine->SortedDelete(sorted_collection, sorted_key), Status::Ok); + } + ASSERT_EQ(space_cleaner->ActiveThreadNum(), 8); + + op = OpType::insert; + sleep(1); + for (size_t id = 0; id < cnt; ++id) { + std::string value = std::to_string(id) + common_value; + std::string str_key = std::to_string(id) + common_str_key; + std::string sorted_key = std::to_string(id) + common_sorted_key; + ASSERT_EQ(engine->Put(str_key, value), Status::Ok); + ASSERT_EQ(engine->SortedPut(sorted_collection, sorted_key, value), + Status::Ok); + } + ASSERT_EQ(space_cleaner->ActiveThreadNum(), 1); + + delete engine; +} +#endif + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +}