diff --git a/Cargo.lock b/Cargo.lock index ea3ff4a3..c718e3c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1049,7 +1049,7 @@ dependencies = [ [[package]] name = "dalek-ff-group" version = "0.4.1" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "crypto-bigint", "curve25519-dalek", @@ -1193,7 +1193,7 @@ dependencies = [ [[package]] name = "flexible-transcript" version = "0.3.2" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "blake2", "digest", @@ -1729,7 +1729,7 @@ dependencies = [ [[package]] name = "monero-address" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "monero-io", @@ -1742,7 +1742,7 @@ dependencies = [ [[package]] name = "monero-borromean" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "monero-generators", @@ -1755,7 +1755,7 @@ dependencies = [ [[package]] name = "monero-bulletproofs" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "monero-generators", @@ -1770,7 +1770,7 @@ dependencies = [ [[package]] name = "monero-clsag" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "dalek-ff-group", @@ -1790,7 +1790,7 @@ dependencies = [ [[package]] name = "monero-generators" version = "0.4.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "dalek-ff-group", @@ -1804,7 +1804,7 @@ dependencies = [ [[package]] name = "monero-io" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "std-shims", @@ -1813,7 +1813,7 @@ dependencies = [ [[package]] name = "monero-mlsag" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "monero-generators", @@ -1827,7 +1827,7 @@ dependencies = [ [[package]] name = "monero-primitives" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "monero-generators", @@ -1840,9 +1840,8 @@ dependencies = [ [[package]] name = "monero-rpc" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ - "async-trait", "curve25519-dalek", "hex", "monero-address", @@ -1857,7 +1856,7 @@ dependencies = [ [[package]] name = "monero-serai" version = "0.1.4-alpha" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "curve25519-dalek", "hex-literal", @@ -1875,9 +1874,8 @@ dependencies = [ [[package]] name = "monero-simple-request-rpc" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ - "async-trait", "digest_auth", "hex", "monero-rpc", @@ -1885,6 +1883,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1922,6 +1930,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "page_size" version = "0.6.0" @@ -2537,6 +2551,15 @@ dependencies = [ "keccak", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -2555,7 +2578,7 @@ dependencies = [ [[package]] name = "simple-request" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "http-body-util", "hyper", @@ -2605,7 +2628,7 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "std-shims" version = "0.1.1" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=d7ecab6#d7ecab605ec20d959d3e1439bf93251f84a6c0ed" dependencies = [ "hashbrown", "spin", @@ -2925,6 +2948,9 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "nu-ansi-term", + "sharded-slab", + "thread_local", "tracing-core", ] diff --git a/Cargo.toml b/Cargo.toml index fa348ccd..b590208a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ futures = { version = "0.3.30", default-features = false } hex = { version = "0.4.3", default-features = false } hex-literal = { version = "0.4", default-features = false } indexmap = { version = "2.5.0", default-features = false } -monero-serai = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce", default-features = false } +monero-serai = { git = "https://github.com/Cuprate/serai.git", rev = "d7ecab6", default-features = false } paste = { version = "1.0.15", default-features = false } pin-project = { version = "1.1.5", default-features = false } randomx-rs = { git = "https://github.com/Cuprate/randomx-rs.git", rev = "0028464", default-features = false } @@ -88,8 +88,8 @@ tracing-subscriber = { version = "0.3.18", default-features = false } tracing = { version = "0.1.40", default-features = false } ## workspace.dev-dependencies -monero-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" } -monero-simple-request-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" } +monero-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d7ecab6" } +monero-simple-request-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d7ecab6" } tempfile = { version = "3" } pretty_assertions = { version = "1.4.1" } proptest = { version = "1" } diff --git a/binaries/cuprated/Cargo.toml b/binaries/cuprated/Cargo.toml index 32d1b284..f86a5ea8 100644 --- a/binaries/cuprated/Cargo.toml +++ b/binaries/cuprated/Cargo.toml @@ -70,7 +70,7 @@ tokio-util = { workspace = true } tokio-stream = { workspace = true } tokio = { workspace = true } tower = { workspace = true } -tracing-subscriber = { workspace = true } +tracing-subscriber = { workspace = true, features = ["ansi", "std", "fmt"] } tracing = { workspace = true } [lints] diff --git a/binaries/cuprated/src/blockchain.rs b/binaries/cuprated/src/blockchain.rs index a06f3fa7..59f1de2b 100644 --- a/binaries/cuprated/src/blockchain.rs +++ b/binaries/cuprated/src/blockchain.rs @@ -25,6 +25,7 @@ mod manager; mod syncer; mod types; +pub use manager::init_blockchain_manager; use types::{ ConcreteBlockVerifierService, ConcreteTxVerifierService, ConsensusBlockchainReadHandle, }; diff --git a/binaries/cuprated/src/config.rs b/binaries/cuprated/src/config.rs index 6556b3e7..2a55bebb 100644 --- a/binaries/cuprated/src/config.rs +++ b/binaries/cuprated/src/config.rs @@ -1,8 +1,11 @@ //! cuprated config -use serde::{Deserialize, Serialize}; + use cuprate_consensus::ContextConfig; use cuprate_helper::network::Network; +use cuprate_p2p::block_downloader::BlockDownloaderConfig; use cuprate_p2p_core::ClearNet; +use serde::{Deserialize, Serialize}; +use std::time::Duration; mod p2p; mod storage; @@ -18,23 +21,23 @@ pub fn config() -> Config { #[serde(deny_unknown_fields, default)] pub struct Config { network: Network, - + p2p: P2PConfig, - - storage: StorageConfig + + storage: StorageConfig, } impl Config { pub fn network(&self) -> Network { self.network } - + pub fn clearnet_p2p_config(&self) -> cuprate_p2p::P2PConfig { cuprate_p2p::P2PConfig { network: self.network, outbound_connections: self.p2p.clear_net.general.outbound_connections, extra_outbound_connections: self.p2p.clear_net.general.extra_outbound_connections, - max_inbound_connections:self.p2p.clear_net.general.max_inbound_connections, + max_inbound_connections: self.p2p.clear_net.general.max_inbound_connections, gray_peers_percent: self.p2p.clear_net.general.gray_peers_percent, server_config: Some(self.p2p.clear_net.server.clone()), p2p_port: self.p2p.clear_net.general.p2p_port, @@ -42,16 +45,26 @@ impl Config { address_book_config: self.p2p.clear_net.general.address_book_config.clone(), } } - + pub fn context_config(&self) -> ContextConfig { match self.network { Network::Mainnet => ContextConfig::main_net(), Network::Stagenet => ContextConfig::stage_net(), - Network::Testnet => ContextConfig::test_net() + Network::Testnet => ContextConfig::test_net(), } } pub fn blockchain_config(&self) -> cuprate_blockchain::config::Config { self.storage.blockchain.clone() } + + pub fn block_downloader_config(&self) -> BlockDownloaderConfig { + BlockDownloaderConfig { + buffer_size: 50_000_000, + in_progress_queue_size: 50_000_000, + check_client_pool_interval: Duration::from_secs(30), + target_batch_size: 5_000_000, + initial_batch_size: 1, + } + } } diff --git a/binaries/cuprated/src/config/p2p.rs b/binaries/cuprated/src/config/p2p.rs index f085696d..f89b2460 100644 --- a/binaries/cuprated/src/config/p2p.rs +++ b/binaries/cuprated/src/config/p2p.rs @@ -35,7 +35,7 @@ pub struct SharedNetConfig { impl Default for SharedNetConfig { fn default() -> Self { Self { - outbound_connections: 32, + outbound_connections: 64, extra_outbound_connections: 8, max_inbound_connections: 128, gray_peers_percent: 0.7, diff --git a/binaries/cuprated/src/main.rs b/binaries/cuprated/src/main.rs index f7dd97de..68d0e64f 100644 --- a/binaries/cuprated/src/main.rs +++ b/binaries/cuprated/src/main.rs @@ -39,7 +39,8 @@ fn main() { let async_rt = init_tokio_rt(&config); async_rt.block_on(async move { - blockchain:: check_add_genesis(&mut bc_read_handle, &mut bc_write_handle, config.network()).await; + blockchain::check_add_genesis(&mut bc_read_handle, &mut bc_write_handle, config.network()) + .await; let (block_verifier, _tx_verifier, context_svc) = blockchain::init_consensus(bc_read_handle.clone(), config.context_config()) @@ -47,14 +48,24 @@ fn main() { .unwrap(); let net = cuprate_p2p::initialize_network( - p2p::request_handler:: { + p2p::request_handler::P2pProtocolRequestHandlerMaker { blockchain_read_handle: bc_read_handle.clone(), }, p2p::core_sync_service::CoreSyncService(context_svc.clone()), config.clearnet_p2p_config(), ) - .await - .unwrap(); + .await + .unwrap(); + + blockchain::init_blockchain_manager( + net, + bc_write_handle, + bc_read_handle, + context_svc, + block_verifier, + config.block_downloader_config(), + ) + .await; // TODO: this can be removed as long as the main thread does not exit, so when command handling // is added diff --git a/binaries/cuprated/src/p2p.rs b/binaries/cuprated/src/p2p.rs index 7d3c6ea6..a91a6435 100644 --- a/binaries/cuprated/src/p2p.rs +++ b/binaries/cuprated/src/p2p.rs @@ -2,5 +2,5 @@ //! //! Will handle initiating the P2P and contains a protocol request handler. -pub mod request_handler; pub mod core_sync_service; +pub mod request_handler; diff --git a/binaries/cuprated/src/p2p/core_sync_service.rs b/binaries/cuprated/src/p2p/core_sync_service.rs index 20f510d4..34c91e47 100644 --- a/binaries/cuprated/src/p2p/core_sync_service.rs +++ b/binaries/cuprated/src/p2p/core_sync_service.rs @@ -48,4 +48,4 @@ impl Service for CoreSyncService { )) }) } -} \ No newline at end of file +} diff --git a/binaries/cuprated/src/p2p/request_handler.rs b/binaries/cuprated/src/p2p/request_handler.rs index 0bf6ee23..1d84570a 100644 --- a/binaries/cuprated/src/p2p/request_handler.rs +++ b/binaries/cuprated/src/p2p/request_handler.rs @@ -5,7 +5,7 @@ use std::{ }; use bytes::Bytes; -use futures::{future::BoxFuture, FutureExt}; +use futures::{future::BoxFuture, FutureExt, TryFutureExt}; use monero_serai::{block::Block, transaction::Transaction}; use tower::{Service, ServiceExt}; @@ -18,7 +18,9 @@ use cuprate_helper::{ map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}, }; use cuprate_p2p::constants::MAX_BLOCK_BATCH_LEN; -use cuprate_p2p_core::{client::PeerInformation, NetworkZone, ProtocolRequest, ProtocolResponse}; +use cuprate_p2p_core::{ + client::PeerInformation, NetZoneAddress, NetworkZone, ProtocolRequest, ProtocolResponse, +}; use cuprate_types::{ blockchain::{BlockchainReadRequest, BlockchainResponse}, BlockCompleteEntry, MissingTxsInBlock, TransactionBlobs, @@ -37,8 +39,8 @@ pub struct P2pProtocolRequestHandlerMaker { pub blockchain_read_handle: BlockchainReadHandle, } -impl Service> for P2pProtocolRequestHandlerMaker { - type Response = P2pProtocolRequestHandler; +impl Service> for P2pProtocolRequestHandlerMaker { + type Response = P2pProtocolRequestHandler; type Error = tower::BoxError; type Future = Ready>; @@ -46,7 +48,7 @@ impl Service> for P2pProtocolRequestHandlerMa Poll::Ready(Ok(())) } - fn call(&mut self, peer_information: PeerInformation) -> Self::Future { + fn call(&mut self, peer_information: PeerInformation) -> Self::Future { // TODO: check sync info? let blockchain_read_handle = self.blockchain_read_handle.clone(); @@ -60,16 +62,16 @@ impl Service> for P2pProtocolRequestHandlerMa /// The P2P protocol request handler. #[derive(Clone)] -pub struct P2pProtocolRequestHandler { +pub struct P2pProtocolRequestHandler { /// The [`PeerInformation`] for this peer. - peer_information: PeerInformation, + peer_information: PeerInformation, /// The [`BlockchainReadHandle`] blockchain_read_handle: BlockchainReadHandle, } -impl Service for P2pProtocolRequestHandler { +impl Service for P2pProtocolRequestHandler { type Response = ProtocolResponse; - type Error = anyhow::Error; + type Error = tower::BoxError; type Future = BoxFuture<'static, Result>; fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { @@ -78,21 +80,26 @@ impl Service for P2pProtocolRequestHandler { fn call(&mut self, request: ProtocolRequest) -> Self::Future { match request { - ProtocolRequest::GetObjects(r) => { - get_objects(r, self.blockchain_read_handle.clone()).boxed() - } - ProtocolRequest::GetChain(r) => { - get_chain(r, self.blockchain_read_handle.clone()).boxed() - } + ProtocolRequest::GetObjects(r) => get_objects(r, self.blockchain_read_handle.clone()) + .map_err(Into::into) + .boxed(), + ProtocolRequest::GetChain(r) => get_chain(r, self.blockchain_read_handle.clone()) + .map_err(Into::into) + .boxed(), ProtocolRequest::FluffyMissingTxs(r) => { - fluffy_missing_txs(r, self.blockchain_read_handle.clone()).boxed() + fluffy_missing_txs(r, self.blockchain_read_handle.clone()) + .map_err(Into::into) + .boxed() } ProtocolRequest::NewBlock(_) => ready(Err(anyhow::anyhow!( "Peer sent a full block when we support fluffy blocks" - ))) + ) + .into())) .boxed(), ProtocolRequest::NewFluffyBlock(r) => { - new_fluffy_block(r, self.blockchain_read_handle.clone()).boxed() + new_fluffy_block(r, self.blockchain_read_handle.clone()) + .map_err(Into::into) + .boxed() } ProtocolRequest::GetTxPoolCompliment(_) | ProtocolRequest::NewTransactions(_) => { ready(Ok(ProtocolResponse::NA)).boxed()