equihash: Fix some Rust API issues

components/equihash/Cargo.toml

@@ -9,12 +9,21 @@ license = "MIT OR Apache-2.0"
 edition = "2021"
 rust-version = "1.56.1"
 
+[features]
+default = []
+
+## Builds the C++ tromp solver and Rust FFI layer.
+solver = ["dep:cc"]
+
 [dependencies]
 blake2b_simd = "1"
 byteorder = "1"
 
 [build-dependencies]
-cc = "1"
+cc = { version = "1", optional = true }
+
+[dev-dependencies]
+hex = "0.4"
 
 [lib]
 bench = false

components/equihash/build.rs

@@ -1,6 +1,13 @@
+//! Build script for the equihash tromp solver in C.
+
 fn main() {
+    #[cfg(feature = "solver")]
     cc::Build::new()
         .include("tromp/")
         .file("tromp/equi_miner.c")
         .compile("equitromp");
+
+    // Tell Cargo to only rerun this build script if the tromp C files or headers change.
+    #[cfg(feature = "solver")]
+    println!("cargo:rerun-if-changed=tromp");
 }

components/equihash/src/blake2b.rs

@@ -2,6 +2,9 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://www.opensource.org/licenses/mit-license.php .
 
+// This module uses unsafe code for FFI into blake2b.
+#![allow(unsafe_code)]
+
 use blake2b_simd::{State, PERSONALBYTES};
 
 use std::ptr;

components/equihash/src/lib.rs

@@ -27,5 +27,7 @@ mod test_vectors;
 
 pub use verify::{is_valid_solution, Error};
 
+#[cfg(feature = "solver")]
 mod blake2b;
+#[cfg(feature = "solver")]
 pub mod tromp;

components/equihash/src/tromp.rs

@@ -1,3 +1,5 @@
+//! Rust interface to the tromp equihash solver.
+
 use std::marker::{PhantomData, PhantomPinned};
 use std::slice;
 
@@ -15,7 +17,6 @@
 extern "C" {
     #[allow(improper_ctypes)]
     fn equi_new(
-        n_threads: u32,
         blake2b_clone: extern "C" fn(state: *const State) -> *mut State,
         blake2b_free: extern "C" fn(state: *mut State),
         blake2b_update: extern "C" fn(state: *mut State, input: *const u8, input_len: usize),
@@ -30,23 +31,30 @@
     fn equi_digiteven(eq: *mut CEqui, r: u32, id: u32);
     fn equi_digitK(eq: *mut CEqui, id: u32);
     fn equi_nsols(eq: *const CEqui) -> usize;
-    fn equi_sols(eq: *const CEqui) -> *const *const u32;
+    /// Returns `equi_nsols()` solutions of length `2^K`, in a single memory allocation.
+    fn equi_sols(eq: *const CEqui) -> *const u32;
 }
 
-unsafe fn worker(p: verify::Params, curr_state: &State) -> Vec<Vec<u32>> {
-    // Create solver and initialize it.
-    let eq = equi_new(
-        1,
-        blake2b::blake2b_clone,
-        blake2b::blake2b_free,
-        blake2b::blake2b_update,
-        blake2b::blake2b_finalize,
-    );
+/// Performs a single equihash solver run with equihash parameters `p` and hash state `curr_state`.
+/// Returns zero or more unique solutions.
+///
+/// # SAFETY
+///
+/// The parameters to this function must match the hard-coded parameters in the C++ code.
+///
+/// This function uses unsafe code for FFI into the tromp solver.
+#[allow(unsafe_code)]
+#[allow(clippy::print_stdout)]
+unsafe fn worker(eq: *mut CEqui, p: verify::Params, curr_state: &State) -> Vec<Vec<u32>> {
+    // SAFETY: caller must supply a valid `eq` instance.
+    //
+    // Review Note: nsols is set to zero in C++ here
     equi_setstate(eq, curr_state);
 
     // Initialization done, start algo driver.
     equi_digit0(eq, 0);
     equi_clearslots(eq);
+    // SAFETY: caller must supply a `p` instance that matches the hard-coded values in the C code.
     for r in 1..p.k {
         if (r & 1) != 0 {
             equi_digitodd(eq, r, 0)
@@ -55,25 +63,69 @@
         };
         equi_clearslots(eq);
     }
+    // Review Note: nsols is increased here, but only if the solution passes the strictly ordered check.
+    // With 256 nonces, we get to around 6/9 digits strictly ordered.
     equi_digitK(eq, 0);
 
     let solutions = {
         let nsols = equi_nsols(eq);
         let sols = equi_sols(eq);
-        let solutions = slice::from_raw_parts(sols, nsols);
         let solution_len = 1 << p.k;
+        //println!("{nsols} solutions of length {solution_len} at {sols:?}");
+
+        // SAFETY:
+        // - caller must supply a `p` instance that matches the hard-coded values in the C code.
+        // - `sols` is a single allocation containing at least `nsols` solutions.
+        // - this slice is a shared ref to the memory in a valid `eq` instance supplied by the caller.
+        let solutions: &[u32] = slice::from_raw_parts(sols, nsols * solution_len);
+
+        /*
+        println!(
+            "{nsols} solutions of length {solution_len} as a slice of length {:?}",
+            solutions.len()
+        );
+        */
+
+        let mut chunks = solutions.chunks_exact(solution_len);
+
+        // SAFETY:
+        // - caller must supply a `p` instance that matches the hard-coded values in the C code.
+        // - each solution contains `solution_len` u32 values.
+        // - the temporary slices are shared refs to a valid `eq` instance supplied by the caller.
+        // - the bytes in the shared ref are copied before they are returned.
+        // - dropping `solutions: &[u32]` does not drop the underlying memory owned by `eq`.
+        let mut solutions = (&mut chunks)
+            .map(|solution| solution.to_vec())
+            .collect::<Vec<_>>();
+
+        assert_eq!(chunks.remainder().len(), 0);
+
+        // Sometimes the solver returns identical solutions.
+        solutions.sort();
+        solutions.dedup();
 
         solutions
-            .iter()
-            .map(|solution| slice::from_raw_parts(*solution, solution_len).to_vec())
-            .collect::<Vec<_>>()
     };
 
-    equi_free(eq);
+    /*
+    println!(
+        "{} solutions as cloned vectors of length {:?}",
+        solutions.len(),
+        solutions
+            .iter()
+            .map(|solution| solution.len())
+            .collect::<Vec<_>>()
+    );
+    */
 
     solutions
 }
 
+/// Performs multiple equihash solver runs with equihash parameters `200, 9`, initialising the hash with
+/// the supplied partial `input`. Between each run, generates a new nonce of length `N` using the
+/// `next_nonce` function.
+///
+/// Returns zero or more unique solutions.
 pub fn solve_200_9<const N: usize>(
     input: &[u8],
     mut next_nonce: impl FnMut() -> Option<[u8; N]>,
@@ -82,49 +134,190 @@
     let mut state = verify::initialise_state(p.n, p.k, p.hash_output());
     state.update(input);
 
-    loop {
+    // Create solver and initialize it.
+    //
+    // # SAFETY
+    // - the parameters 200,9 match the hard-coded parameters in the C++ code.
+    // - tromp is compiled without multi-threading support, so each instance can only support 1 thread.
+    // - the blake2b functions are in the correct order in Rust and C++ initializers.
+    #[allow(unsafe_code)]
+    let eq = unsafe {
+        equi_new(
+            blake2b::blake2b_clone,
+            blake2b::blake2b_free,
+            blake2b::blake2b_update,
+            blake2b::blake2b_finalize,
+        )
+    };
+
+    let solutions = loop {
         let nonce = match next_nonce() {
             Some(nonce) => nonce,
             None => break vec![],
         };
 
         let mut curr_state = state.clone();
+        // Review Note: these hashes are changing when the nonce changes
         curr_state.update(&nonce);
 
-        let solutions = unsafe { worker(p, &curr_state) };
+        // SAFETY:
+        // - the parameters 200,9 match the hard-coded parameters in the C++ code.
+        // - the eq instance is initilized above.
+        #[allow(unsafe_code)]
+        let solutions = unsafe { worker(eq, p, &curr_state) };
         if !solutions.is_empty() {
             break solutions;
         }
+    };
+
+    // SAFETY:
+    // - the eq instance is initilized above, and not used after this point.
+    #[allow(unsafe_code)]
+    unsafe {
+        equi_free(eq)
+    };
+
+    solutions
+}
+
+/// Performs multiple equihash solver runs with equihash parameters `200, 9`, initialising the hash with
+/// the supplied partial `input`. Between each run, generates a new nonce of length `N` using the
+/// `next_nonce` function.
+///
+/// Returns zero or more unique compressed solutions.
+pub fn solve_200_9_compressed<const N: usize>(
+    input: &[u8],
+    next_nonce: impl FnMut() -> Option<[u8; N]>,
+) -> Vec<Vec<u8>> {
+    // https://github.com/zcash/zcash/blob/6fdd9f1b81d3b228326c9826fa10696fc516444b/src/pow/tromp/equi.h#L34
+    const DIGIT_BITS: usize = 200 / (9 + 1);
+    let solutions = solve_200_9(input, next_nonce);
+
+    let mut solutions: Vec<Vec<u8>> = solutions
+        .iter()
+        .map(|solution| get_minimal_from_indices(solution, DIGIT_BITS))
+        .collect();
+
+    // Just in case the solver returns solutions that become the same when compressed.
+    solutions.sort();
+    solutions.dedup();
+
+    solutions
+}
+
+// Rough translation of GetMinimalFromIndices() from:
+// https://github.com/zcash/zcash/blob/6fdd9f1b81d3b228326c9826fa10696fc516444b/src/crypto/equihash.cpp#L130-L145
+fn get_minimal_from_indices(indices: &[u32], digit_bits: usize) -> Vec<u8> {
+    let index_bytes = (u32::BITS / 8) as usize;
+    let digit_bytes = ((digit_bits + 1) + 7) / 8;
+    assert!(digit_bytes <= index_bytes);
+
+    let len_indices = indices.len() * index_bytes;
+    let min_len = (digit_bits + 1) * len_indices / (8 * index_bytes);
+    let byte_pad = index_bytes - digit_bytes;
+
+    // Rough translation of EhIndexToArray(index, array_pointer) from:
+    // https://github.com/zcash/zcash/blob/6fdd9f1b81d3b228326c9826fa10696fc516444b/src/crypto/equihash.cpp#L123-L128
+    //
+    // Big-endian so that lexicographic array comparison is equivalent to integer comparison.
+    let array: Vec<u8> = indices
+        .iter()
+        .flat_map(|index| index.to_be_bytes())
+        .collect();
+    assert_eq!(array.len(), len_indices);
+
+    compress_array(array, min_len, digit_bits + 1, byte_pad)
+}
+
+// Rough translation of CompressArray() from:
+// https://github.com/zcash/zcash/blob/6fdd9f1b81d3b228326c9826fa10696fc516444b/src/crypto/equihash.cpp#L39-L76
+fn compress_array(array: Vec<u8>, out_len: usize, bit_len: usize, byte_pad: usize) -> Vec<u8> {
+    let mut out = Vec::with_capacity(out_len);
+
+    let index_bytes = (u32::BITS / 8) as usize;
+    assert!(bit_len >= 8);
+    assert!(8 * index_bytes >= 7 + bit_len);
+
+    let in_width: usize = (bit_len + 7) / 8 + byte_pad;
+    assert!(out_len == bit_len * array.len() / (8 * in_width));
+
+    let bit_len_mask: u32 = (1 << (bit_len as u32)) - 1;
+
+    // The acc_bits least-significant bits of acc_value represent a bit sequence
+    // in big-endian order.
+    let mut acc_bits: usize = 0;
+    let mut acc_value: u32 = 0;
+
+    let mut j: usize = 0;
+    for _i in 0..out_len {
+        // When we have fewer than 8 bits left in the accumulator, read the next
+        // input element.
+        if acc_bits < 8 {
+            acc_value <<= bit_len;
+            for x in byte_pad..in_width {
+                acc_value |= (
+                    // Apply bit_len_mask across byte boundaries
+                    (array[j + x] & ((bit_len_mask >> (8 * (in_width - x - 1))) as u8)) as u32
+                )
+                    .wrapping_shl(8 * (in_width - x - 1) as u32); // Big-endian
+            }
+            j += in_width;
+            acc_bits += bit_len;
+        }
+
+        acc_bits -= 8;
+        out.push((acc_value >> acc_bits) as u8);
     }
+
+    out
 }
 
 #[cfg(test)]
 mod tests {
-    use super::solve_200_9;
+    use super::solve_200_9_compressed;
 
     #[test]
+    #[allow(clippy::print_stdout)]
     fn run_solver() {
         let input = b"Equihash is an asymmetric PoW based on the Generalised Birthday problem.";
-        let mut nonce = [
+        let mut nonce: [u8; 32] = [
             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
             0, 0, 0,
         ];
+        let mut nonces = 0..=32_u32;
+        let nonce_count = nonces.clone().count();
 
-        let solutions = solve_200_9(input, || {
-            nonce[0] += 1;
-            if nonce[0] == 0 {
-                None
-            } else {
-                Some(nonce)
-            }
+        let solutions = solve_200_9_compressed(input, || {
+            let variable_nonce = nonces.next()?;
+            println!("Using variable nonce [0..4] of {}", variable_nonce);
+
+            let variable_nonce = variable_nonce.to_le_bytes();
+            nonce[0] = variable_nonce[0];
+            nonce[1] = variable_nonce[1];
+            nonce[2] = variable_nonce[2];
+            nonce[3] = variable_nonce[3];
+
+            Some(nonce)
         });
 
         if solutions.is_empty() {
-            println!("Found no solutions");
+            // Expected solution rate is documented at:
+            // https://github.com/tromp/equihash/blob/master/README.md
+            panic!("Found no solutions after {nonce_count} runs, expected 1.88 solutions per run",);
         } else {
             println!("Found {} solutions:", solutions.len());
-            for solution in solutions {
-                println!("- {:?}", solution);
+            for (sol_num, solution) in solutions.iter().enumerate() {
+                println!("Validating solution {sol_num}:-\n{}", hex::encode(solution));
+                crate::is_valid_solution(200, 9, input, &nonce, solution).unwrap_or_else(|error| {
+                    panic!(
+                        "unexpected invalid equihash 200, 9 solution:\n\
+                             error: {error:?}\n\
+                             input: {input:?}\n\
+                             nonce: {nonce:?}\n\
+                             solution: {solution:?}"
+                    )
+                });
+                println!("Solution {sol_num} is valid!\n");
             }
         }
     }