From 1678ce748e2e68ac7de7da68c925a6d385faa220 Mon Sep 17 00:00:00 2001 From: Pass Automated Testing Suite Date: Thu, 18 Apr 2024 00:32:46 +0200 Subject: [PATCH] Add support to run generators through TestU01 test suites --- .github/workflows/build-and-test.yml | 4 +- README.md | 19 ++++++ bin/crush.ml | 97 ++++++++++++++++++++++++++++ bin/dune | 5 ++ lib/dune | 1 + 5 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 bin/crush.ml create mode 100644 bin/dune diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f993f67..c0d1346 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -44,8 +44,8 @@ jobs: run: | opam install . --deps-only --with-test --with-doc --yes - - name: build - run: opam exec -- dune build + - name: build library + run: opam exec -- dune build lib - name: test run: opam exec -- dune runtest --instrument-with bisect_ppx --force diff --git a/README.md b/README.md index 65e308c..06e0988 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,28 @@ get_floats 10 rng [] |> fst ``` Supported bitgenerators include: `PCG64`, `Philox64`, `Xoshiro256` and `SFC64`. +## Empirical Randomness Testing +Running the test suite provided by [TestU01][6] on the supported generators is supported. +To build the test executable one needs to run `dune build bin`. To see the available +command options run `dune exec -- crush -help`. Below is a sample output from running +`dune exec -- crush pcg64` to test `PCG64` on the Small Crush test suite: +```shell +$ dune exec -- crush pcg64 + +========= Summary results of SmallCrush ========= + + Version: TestU01 1.2.3 + Generator: pcg64 + Number of statistics: 15 + Total CPU time: 00:01:40.64 + + All tests were passed +``` + [1]: https://codecov.io/gh/zoj613/bitgenerators/graph/badge.svg?token=KOOG2Y1SH5 [2]: https://img.shields.io/github/actions/workflow/status/zoj613/bitgenerators/build-and-test.yml?branch=main [3]: https://img.shields.io/github/license/zoj613/bitgenerators [4]: https://zoj613.github.io/bitgenerators/bitgenerators/Bitgen/index.html [5]: https://www.pcg-random.org/posts/developing-a-seed_seq-alternative.html +[6]: https://www.semanticscholar.org/paper/TestU01%3A-A-C-library-for-empirical-testing-of-L'Ecuyer-Simard/ba61b9f0b400b6a375eca7f7ecdb18ad871fa9e8 diff --git a/bin/crush.ml b/bin/crush.ml new file mode 100644 index 0000000..eb8190c --- /dev/null +++ b/bin/crush.ml @@ -0,0 +1,97 @@ +open Stdint +open Bitgen + + +module type S = sig + type t + val next_uint32 : t -> uint32 * t + val initialize : SeedSequence.t -> t +end + + +let mask1, mask2, mask3, mask4 = + Uint32.(of_int 0x55555555, of_int 0x33333333, of_int 0x0F0F0F0F, of_int 0x00FF00FF) + + +(* https://graphics.stanford.edu/~seander/bithacks.html + https://www.pcg-random.org/posts/how-to-test-with-testu01.html *) +let rev_bits v = + let open Uint32 in + (* swap odd and even bits *) + let v = logor (logand (shift_right v 1) mask1) (shift_left (logand v mask1) 1) in + (* swap consecutive pairs *) + let v = logor (logand (shift_right v 2) mask2) (shift_left (logand v mask2) 2) in + (* swap nibbles *) + let v = logor (logand (shift_right v 4) mask3) (shift_left (logand v mask3) 4) in + (* swap bytes *) + let v = logor (logand (shift_right v 8) mask4) (shift_left (logand v mask4) 8) in + (* swap 2-byte-long pairs *) + logor (shift_right v 16) (shift_left v 16) + + +(* TestU01 can only be used for 32bit input and thus causes it to be more sensitive + to flaws in the most-significant bits than the least significant bits. + It is important to test general-purpose generators in bit-reversed form, + to verify their suitability for applications which use the low-order bits. + Thankfully, the bitgenerator interface provide a next_uint32 function for + all the 64-bit PRNG's which outputs the the high and then low 32 bits of the + unsigned random 64 bits. *) +let make_int32 (module M : S) ~rev = + let ss = SeedSequence.initialize [] in + let t = ref (M.initialize ss) in + let bits () = match M.next_uint32 !t, rev with + | (u, t'), false -> t := t'; Uint32.to_int32 u + | (u, t'), true -> t := t'; rev_bits u |> Uint32.to_int32 + in + bits + + +let to_module = function + | "xoshiro256" -> (module Xoshiro256 : S) + | "pcg64" -> (module PCG64 : S) + | "philox64" -> (module Philox64 : S) + | "sfc64" -> (module SFC64 : S) + | _ -> failwith "Unknown PRNG" + + +let testsuite = function + | "smallcrush" -> TestU01.Bbattery.small_crush + | "crush" -> TestU01.Bbattery.crush + | "bigcrush" -> TestU01.Bbattery.big_crush + | _ -> failwith "Unknown test name" + + +let is_rev name = function + | false -> name + | true -> name ^ "-rev" + + +let usage = " +Run TestU01's test suite on the supported bitgenerators. +It offers several batteries of tests including 'Small Crush' +(which consists of 10 tests), 'Crush' (96 tests), and 'Big Crush' (106 tests). +Bitgenerators to select from include: sfc64, philox64, pcg64 and xoshiro256. + +crush [-name] [-pvalue] [-rev] [-verbose] + +E.g `crush -name bigcrush -rev pcg64` (which runs Big Crush on the reversed bits of PCG64 generator). +" + + +let () = + let pvalue = ref 0.01 + and test = ref "smallcrush" + and prng = ref "" + and verbose = ref false + and rev = ref false in + let spec = [ + ("-pvalue", Arg.Set_float pvalue, "Threshold for suspect p-values. Defaults to 0.01"); + ("-name", Arg.Set_string test, "Name of the test to run. Should be one of {smallcrush, crush, bigcrush}.\n Defaults to 'smallcrush'"); + ("-rev", Arg.Set rev, "Run tests on reversed bits of the input."); + ("-verbose", Arg.Set verbose, "In addition to the summary report, write the result of each test to stdout."); + ] in + Arg.parse spec (fun x -> prng := x) usage; + TestU01.Swrite.set_basic !verbose; + Probdist.Gofw.set_suspectp !pvalue; + TestU01.Unif01.create_extern_gen_int32 (is_rev !prng !rev) (to_module !prng |> make_int32 ~rev:!rev) + |> testsuite !test diff --git a/bin/dune b/bin/dune new file mode 100644 index 0000000..20db808 --- /dev/null +++ b/bin/dune @@ -0,0 +1,5 @@ +(executable + (public_name crush) + (name crush) + (ocamlopt_flags (:standard -O3)) + (libraries bitgenerators testu01)) diff --git a/lib/dune b/lib/dune index 0a1dd6f..ca2b67a 100644 --- a/lib/dune +++ b/lib/dune @@ -2,5 +2,6 @@ (name bitgen) (public_name bitgenerators) (libraries stdint ctypes) + (ocamlopt_flags (:standard -O3)) (instrumentation (backend bisect_ppx)))