From 241e7ebf076e020441e9ed4f28bebd4c3f51180b Mon Sep 17 00:00:00 2001 From: Henry Boisdequin <65845077+henryboisdequin@users.noreply.github.com> Date: Mon, 1 Mar 2021 19:37:05 +0530 Subject: [PATCH] use `criterion` for benches --- .gitignore | 4 + CHANGELOG.md | 1 + Cargo.toml | 7 +- benches/bench.rs | 352 ++++++++---------------------------------- benches/bench_util.rs | 16 ++ rust-toolchain | 1 + 6 files changed, 89 insertions(+), 292 deletions(-) create mode 100644 benches/bench_util.rs create mode 100644 rust-toolchain diff --git a/.gitignore b/.gitignore index 693699042b..95cc49265c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Rust /target **/*.rs.bk Cargo.lock + +# Editors +.vscode diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d127a5608..8bb6eb7a7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Changed - The minimum Rust version has been bumped to 1.49.0. (#230) +- `hashbrown`'s benchmarks now use `criterion` ## [v0.10.0] - 2021-01-16 diff --git a/Cargo.toml b/Cargo.toml index 91e2e1e90c..bac786a315 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,11 +29,16 @@ bumpalo = { version = "3.5.0", optional = true } [dev-dependencies] lazy_static = "1.4" -rand = { version = "0.7.3", features = ["small_rng"] } +rand = { version = "0.8.3", features = ["small_rng"] } rayon = "1.0" fnv = "1.0.7" serde_test = "1.0" doc-comment = "0.3.1" +criterion = "0.3.4" + +[[bench]] +name = "bench" +harness = false [features] default = ["ahash", "inline-more"] diff --git a/benches/bench.rs b/benches/bench.rs index 568c513e10..cde5bddc32 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -1,331 +1,101 @@ -// This benchmark suite contains some benchmarks along a set of dimensions: -// Hasher: std default (SipHash) and crate default (AHash). -// Int key distribution: low bit heavy, top bit heavy, and random. -// Task: basic functionality: insert, insert_erase, lookup, lookup_fail, iter -#![feature(test)] +// This file contains all the benchmarks from `hashbrown`. +// To bench, `hashbrown` use a crates called `criterion`. -extern crate test; +mod bench_util; -use test::{black_box, Bencher}; - -use hashbrown::hash_map::DefaultHashBuilder; +use crate::bench_util::DropType; +use criterion::{black_box, criterion_group, criterion_main, Criterion}; use hashbrown::{HashMap, HashSet}; -use std::{ - collections::hash_map::RandomState, - sync::atomic::{self, AtomicUsize}, -}; - -const SIZE: usize = 1000; - -// The default hashmap when using this crate directly. -type AHashMap = HashMap; -// This uses the hashmap from this crate with the default hasher of the stdlib. -type StdHashMap = HashMap; - -// A random key iterator. -#[derive(Clone, Copy)] -struct RandomKeys { - state: usize, -} - -impl RandomKeys { - fn new() -> Self { - RandomKeys { state: 0 } - } -} - -impl Iterator for RandomKeys { - type Item = usize; - fn next(&mut self) -> Option { - // Add 1 then multiply by some 32 bit prime. - self.state = self.state.wrapping_add(1).wrapping_mul(3787392781); - Some(self.state) - } -} - -// Just an arbitrary side effect to make the maps not shortcircuit to the non-dropping path -// when dropping maps/entries (most real world usages likely have drop in the key or value) -lazy_static::lazy_static! { - static ref SIDE_EFFECT: AtomicUsize = AtomicUsize::new(0); -} - -#[derive(Clone)] -struct DropType(usize); -impl Drop for DropType { - fn drop(&mut self) { - SIDE_EFFECT.fetch_add(self.0, atomic::Ordering::SeqCst); - } -} - -macro_rules! bench_suite { - ($bench_macro:ident, $bench_ahash_serial:ident, $bench_std_serial:ident, - $bench_ahash_highbits:ident, $bench_std_highbits:ident, - $bench_ahash_random:ident, $bench_std_random:ident) => { - $bench_macro!($bench_ahash_serial, AHashMap, 0..); - $bench_macro!($bench_std_serial, StdHashMap, 0..); - $bench_macro!( - $bench_ahash_highbits, - AHashMap, - (0..).map(usize::swap_bytes) - ); - $bench_macro!( - $bench_std_highbits, - StdHashMap, - (0..).map(usize::swap_bytes) - ); - $bench_macro!($bench_ahash_random, AHashMap, RandomKeys::new()); - $bench_macro!($bench_std_random, StdHashMap, RandomKeys::new()); - }; -} - -macro_rules! bench_insert { - ($name:ident, $maptype:ident, $keydist:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut m = $maptype::with_capacity_and_hasher(SIZE, Default::default()); - b.iter(|| { - m.clear(); - for i in ($keydist).take(SIZE) { - m.insert(i, (DropType(i), [i; 20])); - } - black_box(&mut m); - }); - eprintln!("{}", SIDE_EFFECT.load(atomic::Ordering::SeqCst)); - } - }; -} - -bench_suite!( - bench_insert, - insert_ahash_serial, - insert_std_serial, - insert_ahash_highbits, - insert_std_highbits, - insert_ahash_random, - insert_std_random -); - -macro_rules! bench_grow_insert { - ($name:ident, $maptype:ident, $keydist:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - b.iter(|| { - let mut m = $maptype::default(); - for i in ($keydist).take(SIZE) { - m.insert(i, DropType(i)); - } - black_box(&mut m); - }) - } - }; -} - -bench_suite!( - bench_grow_insert, - grow_insert_ahash_serial, - grow_insert_std_serial, - grow_insert_ahash_highbits, - grow_insert_std_highbits, - grow_insert_ahash_random, - grow_insert_std_random -); - -macro_rules! bench_insert_erase { - ($name:ident, $maptype:ident, $keydist:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut base = $maptype::default(); - for i in ($keydist).take(SIZE) { - base.insert(i, DropType(i)); - } - let skip = $keydist.skip(SIZE); - b.iter(|| { - let mut m = base.clone(); - let mut add_iter = skip.clone(); - let mut remove_iter = $keydist; - // While keeping the size constant, - // replace the first keydist with the second. - for (add, remove) in (&mut add_iter).zip(&mut remove_iter).take(SIZE) { - m.insert(add, DropType(add)); - black_box(m.remove(&remove)); - } - black_box(m); - }); - eprintln!("{}", SIDE_EFFECT.load(atomic::Ordering::SeqCst)); - } - }; -} -bench_suite!( - bench_insert_erase, - insert_erase_ahash_serial, - insert_erase_std_serial, - insert_erase_ahash_highbits, - insert_erase_std_highbits, - insert_erase_ahash_random, - insert_erase_std_random -); - -macro_rules! bench_lookup { - ($name:ident, $maptype:ident, $keydist:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut m = $maptype::default(); - for i in $keydist.take(SIZE) { - m.insert(i, DropType(i)); - } - - b.iter(|| { - for i in $keydist.take(SIZE) { - black_box(m.get(&i)); - } - }); - eprintln!("{}", SIDE_EFFECT.load(atomic::Ordering::SeqCst)); - } - }; -} - -bench_suite!( - bench_lookup, - lookup_ahash_serial, - lookup_std_serial, - lookup_ahash_highbits, - lookup_std_highbits, - lookup_ahash_random, - lookup_std_random -); - -macro_rules! bench_lookup_fail { - ($name:ident, $maptype:ident, $keydist:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut m = $maptype::default(); - let mut iter = $keydist; - for i in (&mut iter).take(SIZE) { - m.insert(i, DropType(i)); - } - - b.iter(|| { - for i in (&mut iter).take(SIZE) { - black_box(m.get(&i)); - } - }) - } - }; -} - -bench_suite!( - bench_lookup_fail, - lookup_fail_ahash_serial, - lookup_fail_std_serial, - lookup_fail_ahash_highbits, - lookup_fail_std_highbits, - lookup_fail_ahash_random, - lookup_fail_std_random -); - -macro_rules! bench_iter { - ($name:ident, $maptype:ident, $keydist:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut m = $maptype::default(); - for i in ($keydist).take(SIZE) { - m.insert(i, DropType(i)); - } - - b.iter(|| { - for i in &m { - black_box(i); - } - }) - } - }; -} - -bench_suite!( - bench_iter, - iter_ahash_serial, - iter_std_serial, - iter_ahash_highbits, - iter_std_highbits, - iter_ahash_random, - iter_std_random -); - -#[bench] -fn clone_small(b: &mut Bencher) { +fn clone_small(c: &mut Criterion) { let mut m = HashMap::new(); for i in 0..10 { m.insert(i, DropType(i)); } - b.iter(|| { - black_box(m.clone()); - }) + c.bench_function("clone_small", |b| { + b.iter(|| { + black_box(m.clone()); + }) + }); } -#[bench] -fn clone_from_small(b: &mut Bencher) { +fn clone_from_small(c: &mut Criterion) { let mut m = HashMap::new(); let mut m2 = HashMap::new(); for i in 0..10 { m.insert(i, DropType(i)); } - b.iter(|| { - m2.clone_from(&m); - black_box(&mut m2); - }) + c.bench_function("clone_from_small", |b| { + b.iter(|| { + m2.clone_from(&m); + black_box(&mut m2); + }) + }); } -#[bench] -fn clone_large(b: &mut Bencher) { +fn clone_large(c: &mut Criterion) { let mut m = HashMap::new(); for i in 0..1000 { m.insert(i, DropType(i)); } - b.iter(|| { - black_box(m.clone()); - }) + c.bench_function("clone_large", |b| { + b.iter(|| { + black_box(m.clone()); + }) + }); } -#[bench] -fn clone_from_large(b: &mut Bencher) { +fn clone_from_large(c: &mut Criterion) { let mut m = HashMap::new(); let mut m2 = HashMap::new(); for i in 0..1000 { m.insert(i, DropType(i)); } - b.iter(|| { - m2.clone_from(&m); - black_box(&mut m2); - }) + c.bench_function("clone_from_large", |b| { + b.iter(|| { + m2.clone_from(&m); + black_box(&mut m2); + }) + }); } -#[bench] -fn rehash_in_place(b: &mut Bencher) { - b.iter(|| { - let mut set = HashSet::new(); +fn rehash_in_place(c: &mut Criterion) { + c.bench_function("rehash_in_place", |b| { + b.iter(|| { + let mut set = HashSet::new(); - // Each loop triggers one rehash - for _ in 0..10 { - for i in 0..224 { - set.insert(i); - } + // Each loop triggers one rehash + for _ in 0..10 { + for i in 0..224 { + set.insert(i); + } - assert_eq!( - set.capacity(), - 224, - "The set must be at or close to capacity to trigger a re hashing" - ); + assert_eq!( + set.capacity(), + 224, + "The set must be at or close to capacity to trigger a re hashing" + ); - for i in 100..1400 { - set.remove(&(i - 100)); - set.insert(i); + for i in 100..1400 { + set.remove(&(i - 100)); + set.insert(i); + } + set.clear(); } - set.clear(); - } + }) }); } + +criterion_group!( + benches, + clone_small, + clone_from_small, + clone_large, + clone_from_large, + rehash_in_place +); +criterion_main!(benches); diff --git a/benches/bench_util.rs b/benches/bench_util.rs new file mode 100644 index 0000000000..a2ab143341 --- /dev/null +++ b/benches/bench_util.rs @@ -0,0 +1,16 @@ +use std::sync::atomic::{self, AtomicUsize}; + +// Just an arbitrary side effect to make the maps not short circuit to the non-dropping path +// when dropping maps/entries (most real world usages likely have drop in the key or value) +lazy_static::lazy_static! { + static ref SIDE_EFFECT: AtomicUsize = AtomicUsize::new(0); +} + +#[derive(Clone)] +pub(crate) struct DropType(pub usize); + +impl Drop for DropType { + fn drop(&mut self) { + SIDE_EFFECT.fetch_add(self.0, atomic::Ordering::SeqCst); + } +} diff --git a/rust-toolchain b/rust-toolchain new file mode 100644 index 0000000000..7f3a46a841 --- /dev/null +++ b/rust-toolchain @@ -0,0 +1 @@ +1.49.0