Add soroban-bench-utils, add benchmark tests to measure metering accuracy

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "soroban-native-sdk-macros",
     "soroban-test-wasms",
     "soroban-synth-wasm",
+    "soroban-bench-utils",
 ]
 
 exclude = ["soroban-test-wasms/wasm-workspace"]

soroban-bench-utils/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "soroban-bench-utils"
+description = "Utilities for benchmarking soroban."
+homepage = "https://github.com/stellar/rs-soroban-env"
+repository = "https://github.com/stellar/rs-soroban-env"
+authors = ["Stellar Development Foundation <[email protected]>"]
+license = "Apache-2.0"
+version.workspace = true
+edition = "2021"
+rust-version = "1.71"
+publish = false
+
+[dependencies]
+soroban-env-common = { workspace = true }
+tracking-allocator = "0.4.0"
+
+[target.'cfg(target_os = "linux")'.dependencies]
+perf-event = "0.4.7"

soroban-bench-utils/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod tracker;
+pub use tracker::HostTracker;
+pub use tracking_allocator;

soroban-bench-utils/src/rusagev4.rs

@@ -0,0 +1,62 @@
+// Generated with
+//
+//    bindgen --no-layout-tests \
+//            --allowlist-var RUSAGE_INFO_V4 \
+//            --allowlist-function proc_pid_rusage \
+//            --allowlist-type rusage_info_v4 \
+//             -o rusagev4.rs \
+//              /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libproc.h
+
+#![allow(non_camel_case_types)]
+
+/* automatically generated by rust-bindgen 0.60.1 */
+
+pub const RUSAGE_INFO_V4: u32 = 4;
+pub type rusage_info_t = *mut ::std::os::raw::c_void;
+#[repr(C)]
+#[derive(Debug, Copy, Clone, Default)]
+pub struct rusage_info_v4 {
+    pub ri_uuid: [u8; 16usize],
+    pub ri_user_time: u64,
+    pub ri_system_time: u64,
+    pub ri_pkg_idle_wkups: u64,
+    pub ri_interrupt_wkups: u64,
+    pub ri_pageins: u64,
+    pub ri_wired_size: u64,
+    pub ri_resident_size: u64,
+    pub ri_phys_footprint: u64,
+    pub ri_proc_start_abstime: u64,
+    pub ri_proc_exit_abstime: u64,
+    pub ri_child_user_time: u64,
+    pub ri_child_system_time: u64,
+    pub ri_child_pkg_idle_wkups: u64,
+    pub ri_child_interrupt_wkups: u64,
+    pub ri_child_pageins: u64,
+    pub ri_child_elapsed_abstime: u64,
+    pub ri_diskio_bytesread: u64,
+    pub ri_diskio_byteswritten: u64,
+    pub ri_cpu_time_qos_default: u64,
+    pub ri_cpu_time_qos_maintenance: u64,
+    pub ri_cpu_time_qos_background: u64,
+    pub ri_cpu_time_qos_utility: u64,
+    pub ri_cpu_time_qos_legacy: u64,
+    pub ri_cpu_time_qos_user_initiated: u64,
+    pub ri_cpu_time_qos_user_interactive: u64,
+    pub ri_billed_system_time: u64,
+    pub ri_serviced_system_time: u64,
+    pub ri_logical_writes: u64,
+    pub ri_lifetime_max_phys_footprint: u64,
+    pub ri_instructions: u64,
+    pub ri_cycles: u64,
+    pub ri_billed_energy: u64,
+    pub ri_serviced_energy: u64,
+    pub ri_interval_max_phys_footprint: u64,
+    pub ri_runnable_time: u64,
+}
+extern "C" {
+    pub fn proc_pid_rusage(
+        pid: ::std::os::raw::c_int,
+        flavor: ::std::os::raw::c_int,
+        buffer: *mut rusage_info_t,
+    ) -> ::std::os::raw::c_int;
+}

soroban-bench-utils/src/tracker.rs

@@ -0,0 +1,171 @@
+use std::{
+    alloc::System,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
+    time::Instant,
+};
+use tracking_allocator::{
+    AllocationGroupToken, AllocationGuard, AllocationRegistry, AllocationTracker, Allocator,
+};
+
+use self::cpu::InstructionCounter;
+
+#[global_allocator]
+static GLOBAL: Allocator<System> = Allocator::system();
+
+#[derive(Clone)]
+struct MemTracker(Arc<AtomicU64>);
+impl AllocationTracker for MemTracker {
+    fn allocated(
+        &self,
+        _addr: usize,
+        _object_input: usize,
+        wrapped_input: usize,
+        _group_id: tracking_allocator::AllocationGroupId,
+    ) {
+        self.0.fetch_add(wrapped_input as u64, Ordering::SeqCst);
+    }
+
+    // We do not count memory deallocation because we are trying to estimate the worst-case
+    // memory cost under a variety of possible memory allocation models. This mimics the arena
+    // model, where memory deallocations do not happen until the very end. This is not perfect
+    // as any transient objects are counted as actual memory cost, but it is the worst-case
+    // estimate.
+    fn deallocated(
+        &self,
+        _addr: usize,
+        _object_input: usize,
+        _wrapped_input: usize,
+        _source_group_id: tracking_allocator::AllocationGroupId,
+        _current_group_id: tracking_allocator::AllocationGroupId,
+    ) {
+        // No-Op, see comment above.
+        ()
+    }
+}
+
+#[cfg(target_os = "linux")]
+mod cpu {
+    pub struct InstructionCounter(perf_event::Counter);
+    impl InstructionCounter {
+        pub fn new() -> Self {
+            InstructionCounter(
+                perf_event::Builder::new()
+                    .build()
+                    .expect("perf_event::Builder::new().build()"),
+            )
+        }
+        pub fn begin(&mut self) {
+            self.0.reset().expect("perf_event::Counter::reset");
+            self.0.enable().expect("perf_event::Counter::enable");
+        }
+        pub fn end_and_count(&mut self) -> u64 {
+            self.0.disable().expect("perf_event::Counter::disable");
+            self.0.read().expect("perf_event::Counter::read")
+        }
+    }
+}
+
+#[cfg(target_os = "macos")]
+#[path = "."]
+mod cpu {
+
+    #[path = "rusagev4.rs"]
+    mod rusagev4;
+
+    pub struct InstructionCounter(u64);
+    impl InstructionCounter {
+        fn get() -> u64 {
+            use rusagev4::*;
+            use std::os::raw::c_int;
+            let mut ri = rusage_info_v4::default();
+            let pid: c_int = std::process::id() as c_int;
+            let ptr: *mut rusage_info_v4 = &mut ri;
+            let ret: c_int = unsafe { proc_pid_rusage(pid, RUSAGE_INFO_V4 as c_int, ptr.cast()) };
+            assert!(ret == 0, "proc_pid_rusage failed");
+            ri.ri_instructions
+        }
+        pub fn new() -> Self {
+            InstructionCounter(Self::get())
+        }
+        pub fn begin(&mut self) {
+            self.0 = Self::get();
+        }
+
+        pub fn end_and_count(&mut self) -> u64 {
+            let curr = Self::get();
+            curr - self.0
+        }
+    }
+}
+
+#[cfg(not(any(target_os = "linux", target_os = "macos")))]
+mod cpu {
+    pub struct InstructionCounter(u64);
+    impl InstructionCounter {
+        pub fn new() -> Self {
+            InstructionCounter(0)
+        }
+        pub fn begin(&mut self) {}
+        pub fn end_and_count(&mut self) -> u64 {
+            0
+        }
+    }
+}
+
+pub struct HostTracker<'a> {
+    cpu_insn_counter: InstructionCounter,
+    mem_tracker: MemTracker,
+    start_time: Instant,
+    alloc_guard: Option<AllocationGuard<'a>>,
+}
+
+impl<'a> HostTracker<'a> {
+    pub fn start(token: Option<&'a mut AllocationGroupToken>) -> Self {
+        // Setup the instrumentation
+        let mut cpu_insn_counter = cpu::InstructionCounter::new();
+        let mem_tracker = MemTracker(Arc::new(AtomicU64::new(0)));
+        AllocationRegistry::set_global_tracker(mem_tracker.clone())
+            .expect("no other global tracker should be set yet");
+        AllocationRegistry::enable_tracking();
+
+        // start the cpu and mem measurement
+        mem_tracker.0.store(0, Ordering::SeqCst);
+        let alloc_guard: Option<AllocationGuard> = if let Some(t) = token {
+            Some(t.enter())
+        } else {
+            None
+        };
+
+        let start_time = Instant::now();
+        cpu_insn_counter.begin();
+
+        HostTracker {
+            cpu_insn_counter,
+            mem_tracker,
+            start_time,
+            alloc_guard,
+        }
+    }
+
+    pub fn stop(mut self) -> (u64, u64, u64) {
+        // collect the metrics
+        let cpu_insns = self.cpu_insn_counter.end_and_count();
+        let stop_time = Instant::now();
+        if let Some(g) = self.alloc_guard {
+            drop(g)
+        }
+
+        let mem_bytes = self.mem_tracker.0.load(Ordering::SeqCst);
+        let time_nsecs = stop_time.duration_since(self.start_time).as_nanos() as u64;
+
+        AllocationRegistry::disable_tracking();
+        unsafe {
+            AllocationRegistry::clear_global_tracker();
+        }
+
+        (cpu_insns, mem_bytes, time_nsecs)
+    }
+}

soroban-env-host/Cargo.toml

@@ -44,11 +44,11 @@ tracy-client = { version = "=0.15.2", features = ["enable", "timer-fallback"], d
 env_logger = "0.9.0"
 itertools = "0.10.3"
 log = "0.4.17"
-tracking-allocator = "0.4.0"
 tabwriter = "1.2.1"
 thousands = "0.2.0"
 soroban-test-wasms = { package = "soroban-test-wasms", path = "../soroban-test-wasms" }
 soroban-synth-wasm = { package = "soroban-synth-wasm", path = "../soroban-synth-wasm" }
+soroban-bench-utils = { package = "soroban-bench-utils", path = "../soroban-bench-utils" }
 bytes-lit = "0.0.5"
 textplots = "0.8.0"
 wasmprinter = "0.2.41"
@@ -60,9 +60,6 @@ linregress = "0.5.1"
 testutils = ["soroban-env-common/testutils"]
 tracy = ["dep:tracy-client"]
 
-[target.'cfg(target_os = "linux")'.dev-dependencies]
-perf-event = "0.4.7"
-
 [[bench]]
 required-features = ["testutils"]
 harness = false