add more docs

storage/blockchain/src/constants.rs

@@ -14,21 +14,6 @@
 /// <https://github.com/monero-project/monero/blob/c8214782fb2a769c57382a999eaf099691c836e7/src/blockchain_db/lmdb/db_lmdb.cpp#L57>
 pub const DATABASE_VERSION: u64 = 0;
 
-//---------------------------------------------------------------------------------------------------- Error Messages
-/// Corrupt database error message.
-///
-/// The error message shown to end-users in panic
-/// messages if we think the database is corrupted.
-///
-/// This is meant to be user-friendly.
-pub const DATABASE_CORRUPT_MSG: &str = r"Cuprate has encountered a fatal error. The database may be corrupted.
-
-TODO: instructions on:
-1. What to do
-2. How to fix (re-sync, recover, etc)
-3. General advice for preventing corruption
-4. etc";
-
 //---------------------------------------------------------------------------------------------------- Tests
 #[cfg(test)]
 mod test {}

storage/blockchain/src/lib.rs

@@ -117,7 +117,7 @@ compile_error!("Cuprate is only compatible with 64-bit CPUs");
 mod constants;
 mod free;
 
-pub use constants::{DATABASE_CORRUPT_MSG, DATABASE_VERSION};
+pub use constants::DATABASE_VERSION;
 pub use cuprate_database;
 pub use free::open;
 

storage/txpool/Cargo.toml

@@ -1,35 +1,42 @@
 [package]
-name        = "cuprate-txpool"
-version     = "0.0.0"
-edition     = "2021"
+name = "cuprate-txpool"
+version = "0.0.0"
+edition = "2021"
 description = "Cuprate's transaction pool database"
-license     = "MIT"
-authors     = ["Boog900"]
-repository  = "https://github.com/Cuprate/cuprate/tree/main/storage/txpool"
-keywords    = ["cuprate", "txpool", "transaction", "pool", "database"]
+license = "MIT"
+authors = ["Boog900"]
+repository = "https://github.com/Cuprate/cuprate/tree/main/storage/txpool"
+keywords = ["cuprate", "txpool", "transaction", "pool", "database"]
 
 [features]
+default = ["heed", "service"]
+# default     = ["redb", "service"]
+# default     = ["redb-memory", "service"]
+heed = ["cuprate-database/heed"]
+redb = ["cuprate-database/redb"]
+redb-memory = ["cuprate-database/redb-memory"]
+service = ["dep:tower", "dep:rayon", "dep:cuprate-database-service"]
 
 [dependencies]
 cuprate-database = { path = "../database", features = ["heed"] }
-cuprate-database-service = { path = "../service" }
-cuprate-types    = { path = "../../types" }
-cuprate-helper   = { path = "../../helper" , default-features = false, features = ["constants"] }
+cuprate-database-service = { path = "../service", optional = true }
+cuprate-types = { path = "../../types" }
+cuprate-helper = { path = "../../helper", default-features = false, features = ["constants"] }
 
-monero-serai     = { workspace = true, features = ["std"] }
-bytemuck         = { workspace = true, features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
-bitflags         = { workspace = true, features = ["std", "serde", "bytemuck"] }
-thiserror        = { workspace = true }
-hex              = { workspace = true }
-tower            = { workspace = true }
+monero-serai = { workspace = true, features = ["std"] }
+bytemuck = { workspace = true, features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
+bitflags = { workspace = true, features = ["std", "serde", "bytemuck"] }
+thiserror = { workspace = true }
+hex = { workspace = true }
 
-rayon            = { workspace = true }
+tower = { workspace = true, optional = true }
+rayon = { workspace = true, optional = true }
 
-serde            = { workspace = true, optional = true }
+serde = { workspace = true, optional = true }
 
 [dev-dependencies]
 cuprate-test-utils = { path = "../../test-utils" }
 
-tokio              = { workspace = true }
-tempfile           = { workspace = true }
-hex-literal        = { workspace = true }
+tokio = { workspace = true }
+tempfile = { workspace = true }
+hex-literal = { workspace = true }

storage/txpool/README.md

@@ -0,0 +1,124 @@
+Cuprate's tx-pool database.
+
+This documentation is mostly for practical usage of `cuprate_txpool`.
+
+For a high-level overview, see the database section in
+[Cuprate's architecture book](https://architecture.cuprate.org).
+
+If you're looking for a database crate, consider using the lower-level
+[`cuprate-database`](https://doc.cuprate.org/cuprate_database)
+crate that this crate is built on-top of.
+
+# Purpose
+
+This crate does 3 things:
+
+1. Uses [`cuprate_database`] as a base database layer
+1. Implements various transaction pool related [operations](ops), [tables], and [types]
+1. Exposes a [`tower::Service`] backed by a thread-pool
+
+Each layer builds on-top of the previous.
+
+As a user of `cuprate_txpool`, consider using the higher-level [`service`] module,
+or at the very least the [`ops`] module instead of interacting with the `cuprate_database` traits directly.
+
+# `cuprate_database`
+
+Consider reading `cuprate_database`'s crate documentation before this crate, as it is the first layer.
+
+If/when this crate needs is used, be sure to use the version that this crate re-exports, e.g.:
+
+```rust
+use cuprate_txpool::{
+    cuprate_database::RuntimeError,
+};
+```
+
+This ensures the types/traits used from `cuprate_database` are the same ones used by `cuprate_txpool` internally.
+
+# Feature flags
+
+The `service` module requires the `service` feature to be enabled.
+See the module for more documentation.
+
+Different database backends are enabled by the feature flags:
+
+- `heed` (LMDB)
+- `redb`
+
+The default is `heed`.
+
+`tracing` is always enabled and cannot be disabled via feature-flag.
+<!-- FIXME: tracing should be behind a feature flag -->
+
+# Invariants when not using `service`
+
+`cuprate_txpool` can be used without the `service` feature enabled but
+there are some things that must be kept in mind when doing so.
+
+Failing to uphold these invariants may cause panics.
+
+1. `LMDB` requires the user to resize the memory map resizing (see [`cuprate_database::RuntimeError::ResizeNeeded`]
+1. `LMDB` has a maximum reader transaction count,
+   currently, [it is set to `126`](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799)
+1. `LMDB`
+   has [maximum key/value byte size](http://www.lmdb.tech/doc/group__internal.html#gac929399f5d93cef85f874b9e9b1d09e0)
+   which must not be exceeded
+
+# Examples
+
+The below is an example of using `cuprate_txpool`'s
+lowest API, i.e. using a mix of this crate and `cuprate_database`'s traits directly -
+**this is NOT recommended.**
+
+For examples of the higher-level APIs, see:
+
+- [`ops`]
+- [`service`]
+
+```rust
+use cuprate_txpool::{
+    cuprate_database::{
+        ConcreteEnv,
+        Env, EnvInner,
+        DatabaseRo, DatabaseRw, TxRo, TxRw,
+    },
+    config::ConfigBuilder,
+    tables::{Tables, TablesMut, OpenTables},
+};
+
+# fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Create a configuration for the database environment.
+    let tmp_dir = tempfile::tempdir()?;
+    let db_dir = tmp_dir.path().to_owned();
+    let config = ConfigBuilder::new()
+        .db_directory(db_dir.into())
+        .build();
+
+    // Initialize the database environment.
+    let env = cuprate_txpool::open(config)?;
+
+    // Open up a transaction + tables for writing.
+    let env_inner = env.env_inner();
+    let tx_rw = env_inner.tx_rw()?;
+    let mut tables = env_inner.open_tables_mut(&tx_rw)?;
+
+    // ⚠️ Write data to the tables directly.
+    // (not recommended, use `ops` or `service`).
+    const KEY_IMAGE: [u8; 32] = [88; 32];
+    const TX_HASH: [u8; 32] = [88; 32];
+    tables.spent_key_images_mut().put(&KEY_IMAGE, &TX_HASH)?;
+
+    // Commit the data written.
+    drop(tables);
+    TxRw::commit(tx_rw)?;
+
+    // Read the data, assert it is correct.
+    let tx_ro = env_inner.tx_ro()?;
+    let tables = env_inner.open_tables(&tx_ro)?;
+    let (key_image, tx_hash) = tables.spent_key_images().first()?;
+    assert_eq!(key_image, KEY_IMAGE);
+    assert_eq!(tx_hash, TX_HASH);
+    # Ok(())
+}
+```

storage/txpool/src/config.rs

@@ -1,9 +1,12 @@
-use cuprate_database::config::{Config as DbConfig, SyncMode};
-use cuprate_database::resize::ResizeAlgorithm;
+//! The transaction pool [`Config`].
+use std::{borrow::Cow, path::Path};
+
+use cuprate_database::{
+    config::{Config as DbConfig, SyncMode},
+    resize::ResizeAlgorithm,
+};
 use cuprate_database_service::ReaderThreads;
-use cuprate_helper::fs::{cuprate_blockchain_dir, cuprate_txpool_dir};
-use std::borrow::Cow;
-use std::path::Path;
+use cuprate_helper::fs::cuprate_txpool_dir;
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
@@ -40,7 +43,7 @@ impl ConfigBuilder {
         Self {
             db_directory: None,
             db_config: cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(
-                cuprate_blockchain_dir(),
+                cuprate_txpool_dir(),
             )),
             reader_threads: None,
             max_txpool_weight: None,
@@ -51,15 +54,15 @@ impl ConfigBuilder {
     ///
     /// # Default values
     /// If [`ConfigBuilder::db_directory`] was not called,
-    /// the default [`cuprate_blockchain_dir`] will be used.
+    /// the default [`cuprate_txpool_dir`] will be used.
     ///
     /// For all other values, [`Default::default`] is used.
     pub fn build(self) -> Config {
         // INVARIANT: all PATH safety checks are done
         // in `helper::fs`. No need to do them here.
         let db_directory = self
             .db_directory
-            .unwrap_or_else(|| Cow::Borrowed(cuprate_blockchain_dir()));
+            .unwrap_or_else(|| Cow::Borrowed(cuprate_txpool_dir()));
 
         let reader_threads = self.reader_threads.unwrap_or_default();
 
@@ -121,7 +124,7 @@ impl ConfigBuilder {
     #[must_use]
     pub fn fast(mut self) -> Self {
         self.db_config =
-            cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_blockchain_dir()))
+            cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_txpool_dir()))
                 .fast();
 
         self.reader_threads = Some(ReaderThreads::OnePerThread);
@@ -135,7 +138,7 @@ impl ConfigBuilder {
     #[must_use]
     pub fn low_power(mut self) -> Self {
         self.db_config =
-            cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_blockchain_dir()))
+            cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_txpool_dir()))
                 .low_power();
 
         self.reader_threads = Some(ReaderThreads::One);
@@ -145,7 +148,7 @@ impl ConfigBuilder {
 
 impl Default for ConfigBuilder {
     fn default() -> Self {
-        let db_directory = Cow::Borrowed(cuprate_blockchain_dir());
+        let db_directory = Cow::Borrowed(cuprate_txpool_dir());
         Self {
             db_directory: Some(db_directory.clone()),
             db_config: cuprate_database::config::ConfigBuilder::new(db_directory),
@@ -180,7 +183,7 @@ impl Config {
     /// Create a new [`Config`] with sane default settings.
     ///
     /// The [`DbConfig::db_directory`]
-    /// will be set to [`cuprate_blockchain_dir`].
+    /// will be set to [`cuprate_txpool_dir`].
     ///
     /// All other values will be [`Default::default`].
     ///

storage/txpool/src/free.rs

@@ -1,4 +1,4 @@
-//! General free functions (related to the database).
+//! General free functions (related to the tx-pool database).
 
 //---------------------------------------------------------------------------------------------------- Import
 use cuprate_database::{ConcreteEnv, Env, EnvInner, InitError, RuntimeError, TxRw};

storage/txpool/src/lib.rs

@@ -1,9 +1,12 @@
+#![doc = include_str!("../README.md")]
+
 pub mod config;
 mod free;
-mod ops;
+pub mod ops;
+#[cfg(feature = "service")]
 pub mod service;
 pub mod tables;
-mod types;
+pub mod types;
 
 pub use config::Config;
 pub use free::open;