Make the Raft library compatible with no_std environments.

The std target implementation remains unchanged. The majority of changes are switching to using core and alloc
where std used to be. The no_std flavor uses no_std compatible collections and synchronization primitives.
The thiserror crate doesn't have no_std support hence the error types where reimplemented manually. The code
generation for the protobuf is also doesn't have no_std support so to minimize the changes throghout the
codebase instead the change contains modified autogenerated wrappers for the Prost generated code.

.github/workflows/ci.yml

@@ -3,7 +3,7 @@ name: CI
 on:
   pull_request:
   push:
-    branches: 
+    branches:
     - master
 
 env:
@@ -45,12 +45,18 @@ jobs:
       run: cargo fmt --all -- --check
     - name: check clippy
       if: ${{ matrix.rust == 'stable' && matrix.os == 'ubuntu-latest' }}
-      run: cargo clippy --all --all-targets -- -D clippy::all && cargo clippy --no-default-features --features prost-codec -- -D clippy::all
-    - run: cargo test --all -- --nocapture
+      run: cargo clippy --all --all-targets -- -D clippy::all && cargo clippy --no-default-features --features prost-codec --features std -- -D clippy::all
+    - name: check no_std compatibility
+      run: cargo build --no-default-features --features prost-codec --target x86_64-unknown-none
+    - name: run tests with no-std enabled
+      # No-std implementation uses feature that can only be compiled by nightly version
+      if: ${{ matrix.rust == 'nightly' }}
+      run: cargo test --all --no-default-features --features=prost-codec --features=default-logger --no-fail-fast -- --nocapture
+    - run: cargo test --all  --no-fail-fast -- --nocapture
     # Validate benches still work.
     - run: cargo bench --all -- --test
     # Because failpoints inject failure in code path, which will affect all concurrently running tests, Hence they need to be synchronized, which make tests slow.
     # Only package harness has failpoints tests.
     - run: cargo test --tests --features failpoints --package harness -- --nocapture
     # TODO: There is a bug in protobuf-build that cached size is not supported yet, so do not test harness.
-    - run: cargo test --no-default-features --features prost-codec -- --nocapture
+    - run: cargo test --no-default-features --features prost-codec --features std -- --nocapture

Cargo.toml

@@ -9,34 +9,40 @@ readme = "README.md"
 homepage = "https://github.com/tikv/raft-rs"
 documentation = "https://docs.rs/raft"
 description = "The rust language implementation of Raft algorithm."
-categories = ["algorithms", "database-implementations"]
+categories = ["algorithms", "database-implementations", "no-std"]
 edition = "2021"
 
 [workspace]
 members = ["proto", "harness", "datadriven"]
 
 [features]
-default = ["protobuf-codec", "default-logger"]
+default = ["protobuf-codec", "default-logger", "std"]
+std = ["thiserror", "fxhash", "raft-proto/std"]
 # Enable failpoints
 failpoints = ["fail/failpoints"]
-protobuf-codec = ["raft-proto/protobuf-codec", "bytes"]
-prost-codec = ["raft-proto/prost-codec"]
+protobuf-codec = ["raft-proto/protobuf-codec", "raft-proto/std", "bytes", "protobuf"]
+prost-codec = ["raft-proto/prost-codec", "prost"]
 default-logger = ["slog-stdlog", "slog-envlogger", "slog-term"]
 
 # Make sure to synchronize updates with Harness.
 [dependencies]
-bytes = { version = "1", optional = true }
-fxhash = "0.2.1"
+bytes = { version = "1", optional = true, default-features = false }
+fxhash = { version = "0.2.1", optional = true }
+ahash = { version = "0.8.3", default-features = false }
 fail = { version = "0.4", optional = true }
 getset = "0.1.1"
-protobuf = "2"
-thiserror = "1.0"
+protobuf = { version = "2", optional = true }
+prost = { version = "0.11", optional = true, default-features = false, features = ["prost-derive"] }
+thiserror = { version = "1.0", optional = true }
 raft-proto = { path = "proto", version = "0.7.0", default-features = false }
-rand = "0.8"
-slog = "2.2"
+rand = { version = "0.8", default-features = false, features = ["alloc", "small_rng", "getrandom"] }
+getrandom = { version = "0.2.10", default-features = false, features = ["rdrand"] }
+slog = { version = "2.2", default-features = false }
 slog-envlogger = { version = "2.1.0", optional = true }
 slog-stdlog = { version = "4", optional = true }
 slog-term = { version = "2.4.0", optional = true }
+spin = { version = "0.9.8"}
+hashbrown = { version = "0.14.0"}
 
 [dev-dependencies]
 criterion = "0.3"

examples/five_mem_node/main.rs

@@ -11,7 +11,8 @@ use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 use std::{str, thread};
 
-use protobuf::Message as PbMessage;
+#[cfg(feature = "protobuf-codec")]
+use protobuf::Message as _;
 use raft::storage::MemStorage;
 use raft::{prelude::*, StateRole};
 use regex::Regex;

harness/Cargo.toml

@@ -12,7 +12,7 @@ categories = []
 edition = "2018"
 
 [features]
-default = ["protobuf-codec", "raft/default-logger"]
+default = ["protobuf-codec", "raft/default-logger", "raft/std"]
 # Enable failpoints
 failpoints = ["fail/failpoints"]
 protobuf-codec = ["raft/protobuf-codec"]

harness/tests/integration_cases/test_raft.rs

@@ -19,6 +19,7 @@ use std::collections::HashMap;
 use std::panic::{self, AssertUnwindSafe};
 
 use harness::*;
+#[cfg(feature = "protobuf-codec")]
 use protobuf::Message as PbMessage;
 use raft::eraftpb::*;
 use raft::storage::{GetEntriesContext, MemStorage};
@@ -397,7 +398,7 @@ fn test_progress_flow_control() {
     // election, and the first proposal (only one proposal gets sent
     // because we're in probe state).
     assert_eq!(ms.len(), 1);
-    assert_eq!(ms[0].msg_type, MessageType::MsgAppend);
+    assert_eq!(ms[0].get_msg_type(), MessageType::MsgAppend);
     assert_eq!(ms[0].entries.len(), 2);
     assert_eq!(ms[0].entries[0].data.len(), 0);
     assert_eq!(ms[0].entries[1].data.len(), 1000);
@@ -410,7 +411,7 @@ fn test_progress_flow_control() {
     ms = r.read_messages();
     assert_eq!(ms.len(), 3);
     for (i, m) in ms.iter().enumerate() {
-        if m.msg_type != MessageType::MsgAppend {
+        if m.get_msg_type() != MessageType::MsgAppend {
             panic!("{}: expected MsgAppend, got {:?}", i, m.msg_type);
         }
         if m.entries.len() != 2 {
@@ -426,7 +427,7 @@ fn test_progress_flow_control() {
     ms = r.read_messages();
     assert_eq!(ms.len(), 2);
     for (i, m) in ms.iter().enumerate() {
-        if m.msg_type != MessageType::MsgAppend {
+        if m.get_msg_type() != MessageType::MsgAppend {
             panic!("{}: expected MsgAppend, got {:?}", i, m.msg_type);
         }
     }
@@ -3336,7 +3337,7 @@ fn test_commit_after_remove_node() -> Result<()> {
     let ents = next_ents(&mut r, &s);
     assert_eq!(ents.len(), 1);
     assert_eq!(ents[0].get_entry_type(), EntryType::EntryNormal);
-    assert_eq!(ents[0].data.as_ref(), b"hello");
+    assert_eq!(ents[0].get_data(), b"hello");
 
     Ok(())
 }
@@ -4452,7 +4453,7 @@ fn test_conf_change_check_before_campaign() {
     let mut cc = ConfChange::default();
     cc.set_change_type(ConfChangeType::RemoveNode);
     cc.node_id = 3;
-    e.data = protobuf::Message::write_to_bytes(&cc).unwrap().into();
+    e.data = cc.write_to_bytes().unwrap().into();
     m.mut_entries().push(e);
     nt.send(vec![m]);
 

harness/tests/integration_cases/test_raw_node.rs

@@ -15,6 +15,7 @@
 // limitations under the License.
 
 use harness::Network;
+#[cfg(feature = "protobuf-codec")]
 use protobuf::{Message as PbMessage, ProtobufEnum as _};
 use raft::eraftpb::*;
 use raft::storage::{GetEntriesContext, MemStorage};
@@ -887,7 +888,7 @@ fn prepare_async_entries(raw_node: &mut RawNode<MemStorage>, s: &MemStorage) {
     // election, and the first proposal (only one proposal gets sent
     // because we're in probe state).
     assert_eq!(msgs.len(), 1);
-    assert_eq!(msgs[0].msg_type, MessageType::MsgAppend);
+    assert_eq!(msgs[0].get_msg_type(), MessageType::MsgAppend);
     assert_eq!(msgs[0].entries.len(), 2);
     let _ = raw_node.advance_append(rd);
 
@@ -928,7 +929,7 @@ fn test_raw_node_with_async_entries() {
     s.wl().append(&entries).unwrap();
     let msgs = rd.messages();
     assert_eq!(msgs.len(), 5);
-    assert_eq!(msgs[0].msg_type, MessageType::MsgAppend);
+    assert_eq!(msgs[0].get_msg_type(), MessageType::MsgAppend);
     assert_eq!(msgs[0].entries.len(), 2);
     let _ = raw_node.advance_append(rd);
 }
@@ -1014,7 +1015,7 @@ fn test_raw_node_async_entries_with_leader_change() {
     // election, and the first proposal (only one proposal gets sent
     // because we're in probe state).
     assert_eq!(msgs.len(), 1);
-    assert_eq!(msgs[0].msg_type, MessageType::MsgAppend);
+    assert_eq!(msgs[0].get_msg_type(), MessageType::MsgAppend);
     assert_eq!(msgs[0].entries.len(), 2);
     let _ = raw_node.advance_append(rd);
 
@@ -1846,7 +1847,7 @@ fn test_committed_entries_pagination_after_restart() {
     let (mut entries, mut size) = (vec![], 0);
     for i in 2..=10 {
         let e = new_entry(1, i, Some("test data"));
-        size += e.compute_size() as u64;
+        size += raft::util::compute_size(&e) as u64;
         entries.push(e);
     }
     s.inner.wl().append(&entries).unwrap();

harness/tests/test_util/mod.rs

@@ -14,6 +14,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// This is necessary to support prost and rust-protobuf at the same time.
+#![allow(clippy::useless_conversion)]
+
 use harness::*;
 use raft::eraftpb::*;
 use raft::storage::MemStorage;
@@ -130,7 +133,7 @@ pub const SOME_DATA: Option<&'static str> = Some("somedata");
 
 pub fn new_message_with_entries(from: u64, to: u64, ty: MessageType, ents: Vec<Entry>) -> Message {
     let mut m = Message {
-        msg_type: ty,
+        msg_type: ty.into(),
         to,
         from,
         ..Default::default()
@@ -179,7 +182,7 @@ pub fn new_snapshot(index: u64, term: u64, voters: Vec<u64>) -> Snapshot {
 
 pub fn conf_change(ty: ConfChangeType, node_id: u64) -> ConfChange {
     ConfChange {
-        change_type: ty,
+        change_type: ty.into(),
         node_id,
         ..Default::default()
     }

proto/Cargo.toml

@@ -13,15 +13,16 @@ categories = ["algorithms", "database-implementations"]
 build = "build.rs"
 
 [features]
-default = ["protobuf-codec"]
-protobuf-codec = ["protobuf-build/protobuf-codec", "bytes", "protobuf/bytes"]
+default = ["protobuf-codec", "std"]
+std = []
+protobuf-codec = ["protobuf-build/protobuf-codec", "bytes", "protobuf/bytes", "protobuf"]
 prost-codec = ["protobuf-build/prost-codec", "prost", "lazy_static"]
 
 [build-dependencies]
 protobuf-build = { version = "0.15.1", default-features = false }
 
 [dependencies]
-bytes = { version = "1", optional = true }
-lazy_static = { version = "1", optional = true }
-prost = { version = "0.11", optional = true }
-protobuf = "2"
+bytes = { version = "1", optional = true, default-features = false }
+lazy_static = { version = "1", optional = true, default-features = false, features = ["spin_no_std"] }
+prost = { version = "0.11", optional = true, default-features = false, features = ["prost-derive"] }
+protobuf = { version = "2", optional = true }

proto/build.rs

@@ -4,6 +4,7 @@ use protobuf_build::Builder;
 
 fn main() {
     let base = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string());
+
     Builder::new()
         .search_dir_for_protos(&format!("{}/proto", base))
         .includes(&[format!("{}/include", base), format!("{}/proto", base)])

proto/src/confchange.rs

@@ -1,10 +1,22 @@
 // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
 
+extern crate alloc;
+
 use crate::eraftpb::{
     ConfChange, ConfChangeSingle, ConfChangeTransition, ConfChangeType, ConfChangeV2,
 };
-use std::borrow::Cow;
-use std::fmt::Write;
+use alloc::borrow::Cow;
+use alloc::format;
+use alloc::string::String;
+use alloc::vec;
+use alloc::vec::Vec;
+use core::fmt::Write;
+use core::iter::Iterator;
+use core::option::Option;
+use core::option::Option::{None, Some};
+use core::result::Result;
+use core::result::Result::{Err, Ok};
+use core::write;
 
 /// Creates a `ConfChangeSingle`.
 pub fn new_conf_change_single(node_id: u64, ty: ConfChangeType) -> ConfChangeSingle {

proto/src/lib.rs

@@ -3,7 +3,9 @@
 // We use `default` method a lot to be support prost and rust-protobuf at the
 // same time. And reassignment can be optimized by compiler.
 #![allow(clippy::field_reassign_with_default)]
+#![cfg_attr(not(feature = "std"), no_std)]
 
+extern crate alloc;
 mod confchange;
 mod confstate;
 
@@ -19,6 +21,19 @@ pub use crate::protos::eraftpb;
 #[allow(renamed_and_removed_lints)]
 #[allow(bare_trait_objects)]
 mod protos {
+
+    #[cfg(feature = "prost-codec")]
+    pub mod eraftpb {
+
+        use core::concat;
+        use core::env;
+        use core::include;
+
+        include!(concat!(env!("OUT_DIR"), "/protos/eraftpb.rs"));
+        include!("wrappers.rs");
+    }
+
+    #[cfg(feature = "protobuf-codec")]
     include!(concat!(env!("OUT_DIR"), "/protos/mod.rs"));
 
     use self::eraftpb::Snapshot;
@@ -40,6 +55,8 @@ pub mod prelude {
 
 pub mod util {
     use crate::eraftpb::ConfState;
+    use core::convert::From;
+    use core::iter::IntoIterator;
 
     impl<Iter1, Iter2> From<(Iter1, Iter2)> for ConfState
     where