From 2d7b3e93fd89b87ea1a15eed2874840011c5ae71 Mon Sep 17 00:00:00 2001 From: Luiz Irber Date: Sat, 25 May 2024 14:22:12 -0700 Subject: [PATCH] Replace serde_json with simd_json where supported --- Cargo.lock | 184 +++++++++++++++++++++++++++++++++- flake.lock | 18 ++-- flake.nix | 5 - src/core/Cargo.toml | 3 +- src/core/benches/minhash.rs | 4 +- src/core/src/collection.rs | 14 +-- src/core/src/errors.rs | 2 +- src/core/src/ffi/signature.rs | 4 +- src/core/src/signature.rs | 26 ++--- src/core/tests/minhash.rs | 52 +++++----- src/core/tests/storage.rs | 6 +- 11 files changed, 247 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 559f38fc84..8fbdb177e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,12 +19,30 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aliasable" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -596,6 +614,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "funty" version = "2.0.0" @@ -639,13 +666,33 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "halfbrown" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8588661a8607108a5ca69cab034063441a0413a0b041c13618a7dd348021ef6f" +dependencies = [ + "hashbrown 0.14.5", + "serde", +] + [[package]] name = "hashbrown" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" dependencies = [ - "ahash", + "ahash 0.7.6", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.11", + "allocator-api2", ] [[package]] @@ -788,6 +835,70 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.154" @@ -1373,6 +1484,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "ref-cast" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.46", +] + [[package]] name = "regex" version = "1.5.6" @@ -1412,7 +1543,7 @@ dependencies = [ "bitvec", "bytecheck", "bytes", - "hashbrown", + "hashbrown 0.12.1", "ptr_meta", "rend", "rkyv_derive", @@ -1595,6 +1726,22 @@ dependencies = [ "wide", ] +[[package]] +name = "simd-json" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "570c430b3d902ea083097e853263ae782dfe40857d93db019a12356c8e8143fa" +dependencies = [ + "getrandom", + "halfbrown", + "lexical-core", + "ref-cast", + "serde", + "serde_json", + "simdutf8", + "value-trait", +] + [[package]] name = "simdutf8" version = "0.1.4" @@ -1654,6 +1801,7 @@ dependencies = [ "roots", "serde", "serde_json", + "simd-json", "statrs", "streaming-stats", "tempfile", @@ -1849,6 +1997,18 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +[[package]] +name = "value-trait" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad8db98c1e677797df21ba03fca7d3bf9bec3ca38db930954e4fe6e1ea27eb4" +dependencies = [ + "float-cmp", + "halfbrown", + "itoa", + "ryu", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2193,6 +2353,26 @@ version = "1.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.46", +] + [[package]] name = "zstd-sys" version = "2.0.7+zstd.1.5.4" diff --git a/flake.lock b/flake.lock index ad45360618..2afc8bf0d8 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1706925685, - "narHash": "sha256-hVInjWMmgH4yZgA4ZtbgJM1qEAel72SYhP5nOWX4UIM=", + "lastModified": 1716619601, + "narHash": "sha256-9dUxZf8MOqJH3vjbhrz7LH4qTcnRsPSBU1Q50T7q/X8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "79a13f1437e149dc7be2d1290c74d378dad60814", + "rev": "47e03a624662ce399e55c45a5f6da698fc72c797", "type": "github" }, "original": { @@ -33,11 +33,11 @@ ] }, "locked": { - "lastModified": 1707099356, - "narHash": "sha256-ph483MDKLi9I/gndYOieVP41es633DOOmPjEI50x5KU=", + "lastModified": 1716603336, + "narHash": "sha256-81u/zd7V+XRTq88zwRLxw5GnwZyEiAvGA2BvAXUe864=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "61dfa5a8129f7edbe9150253c68f673f87b16fb1", + "rev": "4d0f1e4d5d65c23cdbb77e4b0d91940be7309bd4", "type": "github" }, "original": { @@ -66,11 +66,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 5c3795f1d4..2456c8ed1f 100644 --- a/flake.nix +++ b/flake.nix @@ -43,11 +43,6 @@ export MACOSX_DEPLOYMENT_TARGET=10.14 ''; - # Work around https://github.com/NixOS/nixpkgs/issues/166205. - env = lib.optionalAttrs stdenv.cc.isClang { - NIX_LDFLAGS = "-l${stdenv.cc.libcxx.cxxabi.libName}"; - }; - buildInputs = lib.optionals stdenv.isDarwin [ pkgs.libiconv pkgs.darwin.apple_sdk.frameworks.Security ]; nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook bindgenHook ]; diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index f4d6d652d4..ca3db9267c 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -55,8 +55,8 @@ rayon = { version = "1.10.0", optional = true } rkyv = { version = "0.7.44", optional = true } roaring = "0.10.4" roots = "0.0.8" +simd-json = "0.13.10" serde = { version = "1.0.202", features = ["derive"] } -serde_json = "1.0.117" statrs = "0.16.0" streaming-stats = "0.2.3" thiserror = "1.0" @@ -98,6 +98,7 @@ skip_feature_sets = [ js-sys = "0.3.68" web-sys = { version = "0.3.69", features = ["console", "File", "FileReaderSync"] } wasm-bindgen = { version = "0.2.89", features = ["serde-serialize"] } +serde_json = { version = "1.0.117" } [target.'cfg(all(target_arch = "wasm32"))'.dependencies] chrono = { version = "0.4.32", features = ["wasmbind"] } diff --git a/src/core/benches/minhash.rs b/src/core/benches/minhash.rs index c2e27b0249..e65c58d743 100644 --- a/src/core/benches/minhash.rs +++ b/src/core/benches/minhash.rs @@ -16,7 +16,7 @@ fn intersection(c: &mut Criterion) { filename.push("../../tests/test-data/gather-abund/genome-s10.fa.gz.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let mut sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let mut sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); let mh = if let Sketch::MinHash(mh) = &sigs.swap_remove(0).sketches()[0] { mh.clone() } else { @@ -27,7 +27,7 @@ fn intersection(c: &mut Criterion) { filename.push("../../tests/test-data/gather-abund/genome-s11.fa.gz.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let mut sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let mut sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); let mh2 = if let Sketch::MinHash(mh) = &sigs.swap_remove(0).sketches()[0] { mh.clone() } else { diff --git a/src/core/src/collection.rs b/src/core/src/collection.rs index 9f708381ef..356fefe320 100644 --- a/src/core/src/collection.rs +++ b/src/core/src/collection.rs @@ -216,7 +216,7 @@ mod test { filename.push("../../tests/test-data/47+63-multisig.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); selection.set_scaled(2000); @@ -242,7 +242,7 @@ mod test { filename.push("../../tests/test-data/47+63-multisig.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); selection.set_scaled(500); @@ -263,7 +263,7 @@ mod test { filename.push("../../tests/test-data/genome-s11.fa.gz.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); assert_eq!(sigs.len(), 4); // create Selection object let mut selection = Selection::default(); @@ -285,7 +285,7 @@ mod test { filename.push("../../tests/test-data/genome-s11.fa.gz.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); let sigs_copy = sigs.clone(); assert_eq!(sigs.len(), 4); // create Selection object @@ -315,7 +315,7 @@ mod test { filename.push("../../tests/test-data/47+63-multisig.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); assert_eq!(sigs.len(), 6); // create Selection object let mut selection = Selection::default(); @@ -336,7 +336,7 @@ mod test { filename.push("../../tests/test-data/47+63-multisig.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); selection.set_scaled(2000); @@ -387,7 +387,7 @@ mod test { .push("../../tests/test-data/prot/hp/GCA_001593925.1_ASM159392v1_protein.faa.gz.sig"); let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); selection.set_moltype(HashFunctions::Murmur64Hp); diff --git a/src/core/src/errors.rs b/src/core/src/errors.rs index 90c028eb38..3706f5bf80 100644 --- a/src/core/src/errors.rs +++ b/src/core/src/errors.rs @@ -56,7 +56,7 @@ pub enum SourmashError { StorageError(#[from] crate::storage::StorageError), #[error(transparent)] - SerdeError(#[from] serde_json::error::Error), + SerdeError(#[from] simd_json::Error), #[error(transparent)] NifflerError(#[from] niffler::Error), diff --git a/src/core/src/ffi/signature.rs b/src/core/src/ffi/signature.rs index 06a0bd9fe5..a525f1f912 100644 --- a/src/core/src/ffi/signature.rs +++ b/src/core/src/ffi/signature.rs @@ -193,7 +193,7 @@ unsafe fn signature_eq(ptr: *const SourmashSignature, other: *const SourmashSign ffi_fn! { unsafe fn signature_save_json(ptr: *const SourmashSignature) -> Result { let sig = SourmashSignature::as_rust(ptr); - let st = serde_json::to_string(sig)?; + let st = simd_json::to_string(sig)?; Ok(SourmashStr::from_string(st)) } } @@ -248,7 +248,7 @@ unsafe fn signatures_save_buffer(ptr: *const *const SourmashSignature, size: usi } else { Box::new(&mut buffer) }; - serde_json::to_writer(&mut writer, &rsigs)?; + simd_json::to_writer(&mut writer, &rsigs)?; } let b = buffer.into_boxed_slice(); diff --git a/src/core/src/signature.rs b/src/core/src/signature.rs index 0ab8190f98..2bbb98cb22 100644 --- a/src/core/src/signature.rs +++ b/src/core/src/signature.rs @@ -577,7 +577,7 @@ impl Signature { { let (rdr, _format) = niffler::get_reader(Box::new(rdr))?; - let sigs: Vec = serde_json::from_reader(rdr)?; + let sigs: Vec = simd_json::from_reader(rdr)?; Ok(sigs) } @@ -788,7 +788,7 @@ impl ToWriter for Signature { where W: io::Write, { - serde_json::to_writer(writer, &vec![&self])?; + simd_json::to_writer(writer, &vec![&self])?; Ok(()) } } @@ -928,7 +928,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); assert_eq!(sigs.len(), 4); @@ -1049,7 +1049,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); assert_eq!(sigs.len(), 1); @@ -1065,7 +1065,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); assert_eq!(sigs.len(), 1); @@ -1089,7 +1089,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); assert_eq!(sigs.len(), 1); @@ -1114,7 +1114,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); assert_eq!(sigs.len(), 1); @@ -1138,7 +1138,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); @@ -1164,7 +1164,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); @@ -1184,7 +1184,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); @@ -1204,7 +1204,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); @@ -1225,7 +1225,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); @@ -1243,7 +1243,7 @@ mod test { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); // create Selection object let mut selection = Selection::default(); diff --git a/src/core/tests/minhash.rs b/src/core/tests/minhash.rs index 195091056f..a32c6f63a7 100644 --- a/src/core/tests/minhash.rs +++ b/src/core/tests/minhash.rs @@ -383,7 +383,7 @@ fn load_save_minhash_sketches() { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); let sig = sigs.get(0).unwrap(); let sketches = sig.sketches(); @@ -392,11 +392,11 @@ fn load_save_minhash_sketches() { if let Sketch::MinHash(mh) = &sketches[0] { let bmh: KmerMinHashBTree = mh.clone().into(); { - serde_json::to_writer(&mut buffer, &bmh).unwrap(); + simd_json::to_writer(&mut buffer, &bmh).unwrap(); } - let new_mh: KmerMinHash = serde_json::from_reader(&buffer[..]).unwrap(); - let new_bmh: KmerMinHashBTree = serde_json::from_reader(&buffer[..]).unwrap(); + let new_mh: KmerMinHash = simd_json::from_reader(&buffer[..]).unwrap(); + let new_bmh: KmerMinHashBTree = simd_json::from_reader(&buffer[..]).unwrap(); assert_eq!(mh.md5sum(), new_mh.md5sum()); assert_eq!(bmh.md5sum(), new_bmh.md5sum()); @@ -430,11 +430,11 @@ fn load_save_minhash_sketches() { buffer.clear(); let imh: KmerMinHash = bmh.clone().into(); { - serde_json::to_writer(&mut buffer, &imh).unwrap(); + simd_json::to_writer(&mut buffer, &imh).unwrap(); } - let new_mh: KmerMinHash = serde_json::from_reader(&buffer[..]).unwrap(); - let new_bmh: KmerMinHashBTree = serde_json::from_reader(&buffer[..]).unwrap(); + let new_mh: KmerMinHash = simd_json::from_reader(&buffer[..]).unwrap(); + let new_bmh: KmerMinHashBTree = simd_json::from_reader(&buffer[..]).unwrap(); assert_eq!(mh.md5sum(), new_mh.md5sum()); assert_eq!(bmh.md5sum(), new_bmh.md5sum()); @@ -484,7 +484,7 @@ fn load_save_minhash_sketches_abund() { let file = File::open(filename).unwrap(); let reader = BufReader::new(file); - let sigs: Vec = serde_json::from_reader(reader).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(reader).expect("Loading error"); let sig = sigs.get(0).unwrap(); let sketches = sig.sketches(); @@ -493,11 +493,11 @@ fn load_save_minhash_sketches_abund() { if let Sketch::MinHash(mh) = &sketches[0] { let bmh: KmerMinHashBTree = mh.clone().into(); { - serde_json::to_writer(&mut buffer, &bmh).unwrap(); + simd_json::to_writer(&mut buffer, &bmh).unwrap(); } - let new_mh: KmerMinHash = serde_json::from_reader(&buffer[..]).unwrap(); - let new_bmh: KmerMinHashBTree = serde_json::from_reader(&buffer[..]).unwrap(); + let new_mh: KmerMinHash = simd_json::from_reader(&buffer[..]).unwrap(); + let new_bmh: KmerMinHashBTree = simd_json::from_reader(&buffer[..]).unwrap(); assert_eq!(mh.md5sum(), new_mh.md5sum()); assert_eq!(bmh.md5sum(), new_bmh.md5sum()); @@ -541,11 +541,11 @@ fn load_save_minhash_sketches_abund() { buffer.clear(); let imh: KmerMinHash = bmh.clone().into(); { - serde_json::to_writer(&mut buffer, &imh).unwrap(); + simd_json::to_writer(&mut buffer, &imh).unwrap(); } - let new_mh: KmerMinHash = serde_json::from_reader(&buffer[..]).unwrap(); - let new_bmh: KmerMinHashBTree = serde_json::from_reader(&buffer[..]).unwrap(); + let new_mh: KmerMinHash = simd_json::from_reader(&buffer[..]).unwrap(); + let new_bmh: KmerMinHashBTree = simd_json::from_reader(&buffer[..]).unwrap(); assert_eq!(mh.md5sum(), new_mh.md5sum()); assert_eq!(bmh.md5sum(), new_bmh.md5sum()); @@ -671,14 +671,14 @@ fn load_save_minhash_dayhoff(seq in "FLYS*CWLPGQRMTHINKVADER{0,1000}") { let mut buffer_b = vec![]; { - serde_json::to_writer(&mut buffer_a, &a).unwrap(); - serde_json::to_writer(&mut buffer_b, &b).unwrap(); + simd_json::to_writer(&mut buffer_a, &a).unwrap(); + simd_json::to_writer(&mut buffer_b, &b).unwrap(); } assert_eq!(buffer_a, buffer_b); - let c: KmerMinHash = serde_json::from_reader(&buffer_b[..]).unwrap(); - let d: KmerMinHashBTree = serde_json::from_reader(&buffer_a[..]).unwrap(); + let c: KmerMinHash = simd_json::from_reader(&buffer_b[..]).unwrap(); + let d: KmerMinHashBTree = simd_json::from_reader(&buffer_a[..]).unwrap(); assert!((a.similarity(&c, false, false).unwrap() - b.similarity(&d, false, false).unwrap()).abs() < EPSILON); assert!((a.similarity(&c, true, false).unwrap() - b.similarity(&d, true, false).unwrap()).abs() < EPSILON); @@ -699,14 +699,14 @@ fn load_save_minhash_hp(seq in "FLYS*CWLPGQRMTHINKVADER{0,1000}") { let mut buffer_b = vec![]; { - serde_json::to_writer(&mut buffer_a, &a).unwrap(); - serde_json::to_writer(&mut buffer_b, &b).unwrap(); + simd_json::to_writer(&mut buffer_a, &a).unwrap(); + simd_json::to_writer(&mut buffer_b, &b).unwrap(); } assert_eq!(buffer_a, buffer_b); - let c: KmerMinHash = serde_json::from_reader(&buffer_b[..]).unwrap(); - let d: KmerMinHashBTree = serde_json::from_reader(&buffer_a[..]).unwrap(); + let c: KmerMinHash = simd_json::from_reader(&buffer_b[..]).unwrap(); + let d: KmerMinHashBTree = simd_json::from_reader(&buffer_a[..]).unwrap(); assert!((a.similarity(&c, false, false).unwrap() - b.similarity(&d, false, false).unwrap()).abs() < EPSILON); assert!((a.similarity(&c, true, false).unwrap() - b.similarity(&d, true, false).unwrap()).abs() < EPSILON); @@ -727,14 +727,14 @@ fn load_save_minhash_dna(seq in "ACGTN{0,1000}") { let mut buffer_b = vec![]; { - serde_json::to_writer(&mut buffer_a, &a).unwrap(); - serde_json::to_writer(&mut buffer_b, &b).unwrap(); + simd_json::to_writer(&mut buffer_a, &a).unwrap(); + simd_json::to_writer(&mut buffer_b, &b).unwrap(); } assert_eq!(buffer_a, buffer_b); - let c: KmerMinHash = serde_json::from_reader(&buffer_b[..]).unwrap(); - let d: KmerMinHashBTree = serde_json::from_reader(&buffer_a[..]).unwrap(); + let c: KmerMinHash = simd_json::from_reader(&buffer_b[..]).unwrap(); + let d: KmerMinHashBTree = simd_json::from_reader(&buffer_a[..]).unwrap(); assert!((a.similarity(&c, false, false).unwrap() - b.similarity(&d, false, false).unwrap()).abs() < EPSILON); assert!((a.similarity(&c, true, false).unwrap() - b.similarity(&d, true, false).unwrap()).abs() < EPSILON); diff --git a/src/core/tests/storage.rs b/src/core/tests/storage.rs index e0d355d6b0..c5b1ed3371 100644 --- a/src/core/tests/storage.rs +++ b/src/core/tests/storage.rs @@ -12,9 +12,9 @@ fn zipstorage_load_file() -> Result<(), Box> { let zs = ZipStorage::from_file(filename.to_str().unwrap())?; - let data = zs.load("v6.sbt.json")?; + let mut data = zs.load("v6.sbt.json")?; - let description: serde_json::Value = serde_json::from_slice(&data[..])?; + let description: simd_json::borrowed::Value = simd_json::from_slice(&mut data[..])?; assert_eq!(description["version"], 6); Ok(()) @@ -69,7 +69,7 @@ fn zipstorage_parallel_access() -> Result<(), Box> { .par_iter() .map(|path| { let data = zs.load(path).unwrap(); - let sigs: Vec = serde_json::from_reader(&data[..]).expect("Loading error"); + let sigs: Vec = simd_json::from_reader(&data[..]).expect("Loading error"); sigs.iter() .map(|v| v.sketches().iter().map(|mh| mh.size()).sum::()) .sum::()