From 4670c02d78767c4c9923b675be51ef2a3da0541f Mon Sep 17 00:00:00 2001 From: "liuqiang.06" Date: Mon, 11 Nov 2024 13:03:11 +0800 Subject: [PATCH] chore: spilit crates --- Cargo.toml | 20 ++-- sonic_number/Cargo.toml | 7 ++ sonic_number/README.md | 3 + sonic_number/src/arch/aarch64.rs | 11 ++ sonic_number/src/arch/fallback.rs | 11 ++ sonic_number/src/arch/mod.rs | 12 ++ sonic_number/src/arch/x86_64.rs | 108 ++++++++++++++++++ {src/util/num => sonic_number/src}/common.rs | 0 {src/util/num => sonic_number/src}/decimal.rs | 0 {src/util/num => sonic_number/src}/float.rs | 0 {src/util/num => sonic_number/src}/lemire.rs | 0 .../num/mod.rs => sonic_number/src/lib.rs | 34 +++--- {src/util/num => sonic_number/src}/slow.rs | 0 {src/util/num => sonic_number/src}/table.rs | 0 sonic_simd/Cargo.toml | 7 ++ sonic_simd/README.md | 8 ++ {src/util/simd => sonic_simd/src}/avx2.rs | 1 - {src/util/simd => sonic_simd/src}/bits.rs | 24 ---- src/util/simd/mod.rs => sonic_simd/src/lib.rs | 10 +- {src/util/simd => sonic_simd/src}/neon.rs | 12 +- {src/util/simd => sonic_simd/src}/sse2.rs | 0 {src/util/simd => sonic_simd/src}/traits.rs | 4 + {src/util/simd => sonic_simd/src}/v128.rs | 0 {src/util/simd => sonic_simd/src}/v256.rs | 13 ++- {src/util/simd => sonic_simd/src}/v512.rs | 13 ++- src/error.rs | 10 ++ src/parser.rs | 16 +-- src/serde/de.rs | 38 +++--- src/serde/number.rs | 7 +- src/util/arch/aarch64.rs | 14 +-- src/util/arch/fallback.rs | 12 -- src/util/arch/x86_64.rs | 107 ----------------- src/util/mod.rs | 5 - src/util/string.rs | 11 +- 34 files changed, 275 insertions(+), 243 deletions(-) create mode 100644 sonic_number/Cargo.toml create mode 100644 sonic_number/README.md create mode 100644 sonic_number/src/arch/aarch64.rs create mode 100644 sonic_number/src/arch/fallback.rs create mode 100644 sonic_number/src/arch/mod.rs create mode 100644 sonic_number/src/arch/x86_64.rs rename {src/util/num => sonic_number/src}/common.rs (100%) rename {src/util/num => sonic_number/src}/decimal.rs (100%) rename {src/util/num => sonic_number/src}/float.rs (100%) rename {src/util/num => sonic_number/src}/lemire.rs (100%) rename src/util/num/mod.rs => sonic_number/src/lib.rs (96%) rename {src/util/num => sonic_number/src}/slow.rs (100%) rename {src/util/num => sonic_number/src}/table.rs (100%) create mode 100644 sonic_simd/Cargo.toml create mode 100644 sonic_simd/README.md rename {src/util/simd => sonic_simd/src}/avx2.rs (99%) rename {src/util/simd => sonic_simd/src}/bits.rs (84%) rename src/util/simd/mod.rs => sonic_simd/src/lib.rs (85%) rename {src/util/simd => sonic_simd/src}/neon.rs (95%) rename {src/util/simd => sonic_simd/src}/sse2.rs (100%) rename {src/util/simd => sonic_simd/src}/traits.rs (95%) rename {src/util/simd => sonic_simd/src}/v128.rs (100%) rename {src/util/simd => sonic_simd/src}/v256.rs (89%) rename {src/util/simd => sonic_simd/src}/v512.rs (89%) diff --git a/Cargo.toml b/Cargo.toml index 5c3dcfd..e3a3bb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,15 +15,17 @@ version = "0.3.15" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bumpalo = "3.13" -bytes = "1.8" -cfg-if = "1.0" -faststr = { version = "0.2", features = ["serde"] } -itoa = "1.0" -ryu = "1.0" -serde = { version = "1.0", features = ["rc", "derive"] } -simdutf8 = "0.1" -thiserror = "2.0" +bumpalo = "3.13" +bytes = "1.8" +cfg-if = "1.0" +faststr = { version = "0.2", features = ["serde"] } +itoa = "1.0" +ryu = "1.0" +serde = { version = "1.0", features = ["rc", "derive"] } +simdutf8 = "0.1" +sonic_number = { path = "./sonic_number" } +sonic_simd = { path = "./sonic_simd" } +thiserror = "2.0" [dev-dependencies] bytes = { version = "1.4", features = ["serde"] } diff --git a/sonic_number/Cargo.toml b/sonic_number/Cargo.toml new file mode 100644 index 0000000..af2845a --- /dev/null +++ b/sonic_number/Cargo.toml @@ -0,0 +1,7 @@ +[package] +edition = "2021" +name = "sonic_number" +version = "0.1.0" + +[dependencies] +cfg-if = "1.0" diff --git a/sonic_number/README.md b/sonic_number/README.md new file mode 100644 index 0000000..5f89fa4 --- /dev/null +++ b/sonic_number/README.md @@ -0,0 +1,3 @@ +# sonic_number + +A fast number parsing library based on SIMD. \ No newline at end of file diff --git a/sonic_number/src/arch/aarch64.rs b/sonic_number/src/arch/aarch64.rs new file mode 100644 index 0000000..ea1d5c1 --- /dev/null +++ b/sonic_number/src/arch/aarch64.rs @@ -0,0 +1,11 @@ +#[inline(always)] +pub unsafe fn simd_str2int(c: &[u8], need: usize) -> (u64, usize) { + debug_assert!(need < 17); + let mut sum = 0u64; + let mut i = 0; + while i < need && c.get_unchecked(i).is_ascii_digit() { + sum = (c.get_unchecked(i) - b'0') as u64 + sum * 10; + i += 1; + } + (sum, i) +} diff --git a/sonic_number/src/arch/fallback.rs b/sonic_number/src/arch/fallback.rs new file mode 100644 index 0000000..ea1d5c1 --- /dev/null +++ b/sonic_number/src/arch/fallback.rs @@ -0,0 +1,11 @@ +#[inline(always)] +pub unsafe fn simd_str2int(c: &[u8], need: usize) -> (u64, usize) { + debug_assert!(need < 17); + let mut sum = 0u64; + let mut i = 0; + while i < need && c.get_unchecked(i).is_ascii_digit() { + sum = (c.get_unchecked(i) - b'0') as u64 + sum * 10; + i += 1; + } + (sum, i) +} diff --git a/sonic_number/src/arch/mod.rs b/sonic_number/src/arch/mod.rs new file mode 100644 index 0000000..dd85ba6 --- /dev/null +++ b/sonic_number/src/arch/mod.rs @@ -0,0 +1,12 @@ +cfg_if::cfg_if! { + if #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq", target_feature = "avx2", target_feature = "sse2"))] { + mod x86_64; + pub use x86_64::*; + } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { + mod aarch64; + pub use aarch64::*; + } else { + mod fallback; + pub use fallback::*; + } +} diff --git a/sonic_number/src/arch/x86_64.rs b/sonic_number/src/arch/x86_64.rs new file mode 100644 index 0000000..8679256 --- /dev/null +++ b/sonic_number/src/arch/x86_64.rs @@ -0,0 +1,108 @@ +use std::arch::x86_64::*; + +macro_rules! packadd_1 { + ($v:ident) => { + let delta = _mm_set1_epi64x(0x010A010A010A010A); + $v = _mm_maddubs_epi16($v, delta); + }; +} + +macro_rules! packadd_2 { + ($v:ident) => { + let delta = _mm_set1_epi64x(0x0001006400010064); + $v = _mm_madd_epi16($v, delta); + }; +} + +macro_rules! packadd_4 { + ($v:ident) => { + $v = _mm_packus_epi32($v, $v); + let delta = _mm_set_epi16(0, 0, 0, 0, 1, 10000, 1, 10000); + $v = _mm_madd_epi16($v, delta); + }; +} + +// simd add for 5 ~ 8 digits +macro_rules! simd_add_5_8 { + ($v:ident, $and:literal) => {{ + $v = _mm_slli_si128($v, 16 - $and); + packadd_1!($v); + packadd_2!($v); + (_mm_extract_epi32($v, 2) as u64) * 10000 + (_mm_extract_epi32($v, 3) as u64) + }}; +} + +// simd add for 9 ~ 15 digits +macro_rules! simd_add_9_15 { + ($v:ident, $and:literal) => {{ + $v = _mm_slli_si128($v, 16 - $and); + packadd_1!($v); + packadd_2!($v); + packadd_4!($v); + (_mm_extract_epi32($v, 0) as u64) * 100000000 + (_mm_extract_epi32($v, 1) as u64) + }}; +} + +macro_rules! simd_add_16 { + ($v:ident) => {{ + packadd_1!($v); + packadd_2!($v); + packadd_4!($v); + (_mm_extract_epi32($v, 0) as u64) * 100000000 + (_mm_extract_epi32($v, 1) as u64) + }}; +} +#[inline(always)] +pub unsafe fn simd_str2int(c: &[u8], need: usize) -> (u64, usize) { + debug_assert!(need <= 16); + let data = _mm_loadu_si128(c.as_ptr() as *const __m128i); + let zero = _mm_setzero_si128(); + let nine = _mm_set1_epi8(9); + let zero_c = _mm_set1_epi8(b'0' as i8); + + let mut data = _mm_sub_epi8(data, zero_c); + let lt_zero = _mm_cmpgt_epi8(zero, data); + let gt_nine = _mm_cmpgt_epi8(data, nine); + + let is_num_end = _mm_or_si128(lt_zero, gt_nine); + let is_num_end_int = _mm_movemask_epi8(is_num_end); + + // get the real parsed count + let mut count = need; + if is_num_end_int != 0 { + let digits = is_num_end_int.trailing_zeros() as usize; + if digits < need { + count = digits; + } + } + + let sum = match count { + 1 => _mm_extract_epi8(data, 0) as u64, + 2 => (_mm_extract_epi8(data, 0) * 10 + _mm_extract_epi8(data, 1)) as u64, + 3 => { + // shift to clear the non-digit ascii in vector + data = _mm_slli_si128(data, 16 - 3); + packadd_1!(data); + // add the highest two lanes + (_mm_extract_epi16(data, 6) * 100 + _mm_extract_epi16(data, 7)) as u64 + } + 4 => { + data = _mm_slli_si128(data, 16 - 4); + packadd_1!(data); + (_mm_extract_epi16(data, 6) * 100 + _mm_extract_epi16(data, 7)) as u64 + } + 5 => simd_add_5_8!(data, 5), + 6 => simd_add_5_8!(data, 6), + 7 => simd_add_5_8!(data, 7), + 8 => simd_add_5_8!(data, 8), + 9 => simd_add_9_15!(data, 9), + 10 => simd_add_9_15!(data, 10), + 11 => simd_add_9_15!(data, 11), + 12 => simd_add_9_15!(data, 12), + 13 => simd_add_9_15!(data, 13), + 14 => simd_add_9_15!(data, 14), + 15 => simd_add_9_15!(data, 15), + 16 => simd_add_16!(data), + _ => unreachable!(), + }; + (sum, count) +} diff --git a/src/util/num/common.rs b/sonic_number/src/common.rs similarity index 100% rename from src/util/num/common.rs rename to sonic_number/src/common.rs diff --git a/src/util/num/decimal.rs b/sonic_number/src/decimal.rs similarity index 100% rename from src/util/num/decimal.rs rename to sonic_number/src/decimal.rs diff --git a/src/util/num/float.rs b/sonic_number/src/float.rs similarity index 100% rename from src/util/num/float.rs rename to sonic_number/src/float.rs diff --git a/src/util/num/lemire.rs b/sonic_number/src/lemire.rs similarity index 100% rename from src/util/num/lemire.rs rename to sonic_number/src/lemire.rs diff --git a/src/util/num/mod.rs b/sonic_number/src/lib.rs similarity index 96% rename from src/util/num/mod.rs rename to sonic_number/src/lib.rs index 277401e..869eb74 100644 --- a/src/util/num/mod.rs +++ b/sonic_number/src/lib.rs @@ -1,3 +1,4 @@ +mod arch; mod common; mod decimal; mod float; @@ -6,7 +7,7 @@ mod slow; mod table; use self::{common::BiasedFp, float::RawFloat, table::POWER_OF_FIVE_128}; -use crate::{error::ErrorCode, util::arch::simd_str2int}; +use crate::arch::simd_str2int; const FLOATING_LONGEST_DIGITS: usize = 17; const F64_BITS: u32 = 64; @@ -16,7 +17,7 @@ const F64_EXP_BIAS: i32 = 1023; const F64_SIG_MASK: u64 = 0x000F_FFFF_FFFF_FFFF; #[derive(Debug)] -pub(crate) enum ParserNumber { +pub enum ParserNumber { Unsigned(u64), /// Always less than zero. Signed(i64), @@ -24,6 +25,12 @@ pub(crate) enum ParserNumber { Float(f64), } +#[derive(Debug)] +pub enum Error { + InvalidNumber, + FloatMustBeFinite, +} + macro_rules! match_digit { ($data:expr, $i:expr, $pattern:pat) => { $i < $data.len() && matches!($data[$i], $pattern) @@ -45,18 +52,18 @@ macro_rules! digit { macro_rules! check_digit { ($data:expr, $i:expr) => { if !($i < $data.len() && $data[$i].is_ascii_digit()) { - return Err(ErrorCode::InvalidNumber); + return Err(Error::InvalidNumber); } }; } #[inline(always)] -fn parse_exponent(data: &[u8], index: &mut usize) -> Result { +fn parse_exponent(data: &[u8], index: &mut usize) -> Result { let mut exponent: i32 = 0; let mut negative = false; if *index >= data.len() { - return Err(ErrorCode::InvalidNumber); + return Err(Error::InvalidNumber); } match data[*index] { @@ -114,7 +121,7 @@ fn parse_number_fraction( exponent: &mut i32, mut need: isize, dot_pos: usize, -) -> Result { +) -> Result { debug_assert!(need < FLOATING_LONGEST_DIGITS as isize); // native implement: @@ -152,11 +159,7 @@ fn parse_number_fraction( } #[inline(always)] -pub(crate) fn parse_number( - data: &[u8], - index: &mut usize, - negative: bool, -) -> Result { +pub fn parse_number(data: &[u8], index: &mut usize, negative: bool) -> Result { let mut significant: u64 = 0; let mut exponent: i32 = 0; let mut trunc = false; @@ -250,7 +253,7 @@ pub(crate) fn parse_number( } let mut digits_cnt = *index - digit_start; if digits_cnt == 0 { - return Err(ErrorCode::InvalidNumber); + return Err(Error::InvalidNumber); } // slow path for too long integer @@ -331,7 +334,7 @@ fn parse_float( negative: bool, trunc: bool, raw_num: &[u8], -) -> Result { +) -> Result { // parse double fast if significant >> 52 == 0 && (-22..=(22 + 15)).contains(&exponent) { if let Some(mut float) = parse_float_fast(exponent, significant) { @@ -375,7 +378,7 @@ fn parse_float( // check inf for float if float.is_infinite() { - return Err(ErrorCode::FloatMustBeFinite); + return Err(Error::FloatMustBeFinite); } Ok(ParserNumber::Float(float)) } @@ -469,8 +472,7 @@ const POW10_FLOAT: [f64; 23] = [ #[cfg(test)] mod test { - use super::parse_number; - use crate::util::num::ParserNumber; + use crate::{parse_number, ParserNumber}; fn test_parse_ok(input: &str, expect: f64) { assert_eq!(input.parse::().unwrap(), expect); diff --git a/src/util/num/slow.rs b/sonic_number/src/slow.rs similarity index 100% rename from src/util/num/slow.rs rename to sonic_number/src/slow.rs diff --git a/src/util/num/table.rs b/sonic_number/src/table.rs similarity index 100% rename from src/util/num/table.rs rename to sonic_number/src/table.rs diff --git a/sonic_simd/Cargo.toml b/sonic_simd/Cargo.toml new file mode 100644 index 0000000..f17e131 --- /dev/null +++ b/sonic_simd/Cargo.toml @@ -0,0 +1,7 @@ +[package] +edition = "2021" +name = "sonic_simd" +version = "0.1.0" + +[dependencies] +cfg-if = "1.0" diff --git a/sonic_simd/README.md b/sonic_simd/README.md new file mode 100644 index 0000000..891d13d --- /dev/null +++ b/sonic_simd/README.md @@ -0,0 +1,8 @@ + +# sonic_simd + +A portable SIMD library that provides low-level APIs for x86, ARM. Other platforms will use the fallback scalar implementation. + +TODO: + +1. support RISC-V. \ No newline at end of file diff --git a/src/util/simd/avx2.rs b/sonic_simd/src/avx2.rs similarity index 99% rename from src/util/simd/avx2.rs rename to sonic_simd/src/avx2.rs index a1c1481..c185748 100644 --- a/src/util/simd/avx2.rs +++ b/sonic_simd/src/avx2.rs @@ -1,6 +1,5 @@ use std::{ arch::x86_64::*, - mem::transmute, ops::{BitAnd, BitOr, BitOrAssign}, }; diff --git a/src/util/simd/bits.rs b/sonic_simd/src/bits.rs similarity index 84% rename from src/util/simd/bits.rs rename to sonic_simd/src/bits.rs index ac39549..64e23f8 100644 --- a/src/util/simd/bits.rs +++ b/sonic_simd/src/bits.rs @@ -1,5 +1,3 @@ -use serde::de; - use super::traits::BitMask; macro_rules! impl_bits { @@ -102,25 +100,3 @@ impl BitMask for NeonBits { Self(self.0 & u64::MAX >> (n * 4)) } } - -pub fn combine_u16(lo: u16, hi: u16) -> u32 { - #[cfg(target_endian = "little")] - { - (lo as u32) | ((hi as u32) << 16) - } - #[cfg(target_endian = "big")] - { - (hi as u32) | ((lo as u32) << 16) - } -} - -pub fn combine_u32(lo: u32, hi: u32) -> u64 { - #[cfg(target_endian = "little")] - { - (lo as u64) | ((hi as u64) << 32) - } - #[cfg(target_endian = "big")] - { - (hi as u64) | ((lo as u64) << 32) - } -} diff --git a/src/util/simd/mod.rs b/sonic_simd/src/lib.rs similarity index 85% rename from src/util/simd/mod.rs rename to sonic_simd/src/lib.rs index 49245f4..e902a67 100644 --- a/src/util/simd/mod.rs +++ b/sonic_simd/src/lib.rs @@ -1,5 +1,5 @@ -#[allow(unused_imports)] -#[allow(unused)] +#![allow(non_camel_case_types)] + pub mod bits; mod traits; @@ -9,7 +9,7 @@ cfg_if::cfg_if! { mod sse2; use self::sse2::*; } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { - pub(crate) mod neon; + pub mod neon; use self::neon::*; } else { // TODO: support wasm @@ -35,12 +35,12 @@ pub use self::traits::{BitMask, Mask, Simd}; mod v512; use self::v512::*; -#[cfg(all(target_feature = "neon", target_arch = "aarch64"))] pub type u8x16 = Simd128u; - pub type u8x32 = Simd256u; pub type u8x64 = Simd512u; +pub type i8x16 = Simd128i; pub type i8x32 = Simd256i; +pub type i8x64 = Simd512i; pub type m8x32 = Mask256; diff --git a/src/util/simd/neon.rs b/sonic_simd/src/neon.rs similarity index 95% rename from src/util/simd/neon.rs rename to sonic_simd/src/neon.rs index c6e5d63..4d8156a 100644 --- a/src/util/simd/neon.rs +++ b/sonic_simd/src/neon.rs @@ -1,7 +1,4 @@ -use std::{ - arch::aarch64::*, - ops::{BitAnd, BitOr, BitOrAssign}, -}; +use std::arch::aarch64::*; use super::{bits::NeonBits, Mask, Simd}; @@ -151,12 +148,7 @@ impl std::ops::BitOrAssign for Mask128 { } #[inline(always)] -pub(crate) unsafe fn to_bitmask64( - v0: uint8x16_t, - v1: uint8x16_t, - v2: uint8x16_t, - v3: uint8x16_t, -) -> u64 { +pub unsafe fn to_bitmask64(v0: uint8x16_t, v1: uint8x16_t, v2: uint8x16_t, v3: uint8x16_t) -> u64 { let bit_mask = std::mem::transmute::<[u8; 16], uint8x16_t>(BIT_MASK_TAB); let t0 = vandq_u8(v0, bit_mask); diff --git a/src/util/simd/sse2.rs b/sonic_simd/src/sse2.rs similarity index 100% rename from src/util/simd/sse2.rs rename to sonic_simd/src/sse2.rs diff --git a/src/util/simd/traits.rs b/sonic_simd/src/traits.rs similarity index 95% rename from src/util/simd/traits.rs rename to sonic_simd/src/traits.rs index 793993f..da702a7 100644 --- a/src/util/simd/traits.rs +++ b/sonic_simd/src/traits.rs @@ -7,18 +7,22 @@ pub trait Simd: Sized { type Element; type Mask: Mask; + /// # Safety unsafe fn from_slice_unaligned_unchecked(slice: &[u8]) -> Self { debug_assert!(slice.len() >= Self::LANES); Self::loadu(slice.as_ptr()) } + /// # Safety unsafe fn write_to_slice_unaligned_unchecked(&self, slice: &mut [u8]) { debug_assert!(slice.len() >= Self::LANES); self.storeu(slice.as_mut_ptr()); } + /// # Safety unsafe fn loadu(ptr: *const u8) -> Self; + /// # Safety unsafe fn storeu(&self, ptr: *mut u8); fn eq(&self, rhs: &Self) -> Self::Mask; diff --git a/src/util/simd/v128.rs b/sonic_simd/src/v128.rs similarity index 100% rename from src/util/simd/v128.rs rename to sonic_simd/src/v128.rs diff --git a/src/util/simd/v256.rs b/sonic_simd/src/v256.rs similarity index 89% rename from src/util/simd/v256.rs rename to sonic_simd/src/v256.rs index 9fb7de7..08f435b 100644 --- a/src/util/simd/v256.rs +++ b/sonic_simd/src/v256.rs @@ -1,6 +1,6 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; -use super::{bits::combine_u16, Mask, Mask128, Simd, Simd128i, Simd128u}; +use super::{Mask, Mask128, Simd, Simd128i, Simd128u}; #[derive(Debug)] #[repr(transparent)] @@ -22,10 +22,19 @@ impl Mask for Mask256 { fn bitmask(self) -> Self::BitMask { cfg_if::cfg_if! { if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { - use std::arch::aarch64::uint8x16_t; let(v0, v1) = self.0; unsafe { super::neon::to_bitmask32(v0.0, v1.0) } } else { + fn combine_u16(lo: u16, hi: u16) -> u32 { + #[cfg(target_endian = "little")] + { + (lo as u32) | ((hi as u32) << 16) + } + #[cfg(target_endian = "big")] + { + (hi as u32) | ((lo as u32) << 16) + } + } combine_u16(self.0 .0.bitmask(), self.0 .1.bitmask()) } } diff --git a/src/util/simd/v512.rs b/sonic_simd/src/v512.rs similarity index 89% rename from src/util/simd/v512.rs rename to sonic_simd/src/v512.rs index 6d06104..0533290 100644 --- a/src/util/simd/v512.rs +++ b/sonic_simd/src/v512.rs @@ -1,6 +1,6 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; -use super::{bits::combine_u32, Mask, Mask256, Simd, Simd256i, Simd256u}; +use super::{Mask, Mask256, Simd, Simd256i, Simd256u}; #[derive(Debug)] #[repr(transparent)] @@ -22,12 +22,21 @@ impl Mask for Mask512 { fn bitmask(self) -> Self::BitMask { cfg_if::cfg_if! { if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { - use std::arch::aarch64::uint8x16_t; let (v0, v1) = self.0; let (m0, m1) = v0.0; let (m2, m3) = v1.0; unsafe { super::neon::to_bitmask64(m0.0, m1.0, m2.0, m3.0) } } else { + fn combine_u32(lo: u32, hi: u32) -> u64 { + #[cfg(target_endian = "little")] + { + (lo as u64) | ((hi as u64) << 32) + } + #[cfg(target_endian = "big")] + { + (hi as u64) | ((lo as u64) << 32) + } + } combine_u32(self.0 .0.bitmask(), self.0 .1.bitmask()) } } diff --git a/src/error.rs b/src/error.rs index d610562..76b0611 100644 --- a/src/error.rs +++ b/src/error.rs @@ -12,6 +12,7 @@ use serde::{ de::{self, Unexpected}, ser, }; +use sonic_number::Error as NumberError; use thiserror::Error as ErrorTrait; use crate::reader::Position; @@ -291,6 +292,15 @@ pub(crate) enum ErrorCode { SerExpectKeyIsStrOrNum(Unexpected<'static>), } +impl From for ErrorCode { + fn from(err: NumberError) -> Self { + match err { + NumberError::InvalidNumber => ErrorCode::InvalidNumber, + NumberError::FloatMustBeFinite => ErrorCode::FloatMustBeFinite, + } + } +} + impl Error { #[cold] pub(crate) fn syntax(code: ErrorCode, json: &[u8], index: usize) -> Self { diff --git a/src/parser.rs b/src/parser.rs index 172f79f..b53da35 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -7,10 +7,11 @@ use std::{ use faststr::FastStr; use serde::de::{self, Expected, Unexpected}; - -use super::reader::Reader; +use sonic_number::{parse_number, ParserNumber}; #[cfg(all(target_feature = "neon", target_arch = "aarch64"))] -use crate::util::simd::bits::NeonBits; +use sonic_simd::bits::NeonBits; +use sonic_simd::{i8x32, m8x32, u8x32, u8x64, Mask, Simd}; + use crate::{ config::DeserializeCfg, error::{ @@ -23,10 +24,10 @@ use crate::{ tree::{MultiIndex, MultiKey, PointerTreeInner, PointerTreeNode}, PointerTree, }, + reader::Reader, + serde::de::invalid_type_number, util::{ arch::{get_nonspace_bits, prefix_xor}, - num::{parse_number, ParserNumber}, - simd::{i8x32, m8x32, u8x32, u8x64, Mask, Simd}, string::*, unicode::{codepoint_to_utf8, hex_to_u32_nocheck}, }, @@ -262,7 +263,7 @@ where let data = reader.as_u8_slice(); let ret = parse_number(data, &mut now, neg); reader.set_index(now); - ret.map_err(|code| self.error(code)) + ret.map_err(|err| self.error(err.into())) } // TODO: optimize me, avoid clone twice. @@ -330,6 +331,7 @@ where .map_err(|e| self.error(e))?; self.read.set_ptr(src); let s = str_from_raw_parts(start, cnt); + dbg!(&s); check_visit!(self, vis.visit_raw_str(s, raw)) } ParseStatus::None => { @@ -2023,7 +2025,7 @@ where de::Error::invalid_type(Unexpected::Bool(false), exp) } c @ b'-' | c @ b'0'..=b'9' => match self.parse_number(c) { - Ok(n) => n.invalid_type(exp), + Ok(n) => invalid_type_number(&n, exp), Err(err) => return err, }, b'"' => { diff --git a/src/serde/de.rs b/src/serde/de.rs index b35a37f..571142c 100644 --- a/src/serde/de.rs +++ b/src/serde/de.rs @@ -7,6 +7,7 @@ use serde::{ de::{self, Expected, Unexpected}, forward_to_deserialize_any, }; +use sonic_number::ParserNumber; use crate::{ error::{ @@ -16,11 +17,9 @@ use crate::{ }, parser::{as_str, ParseStatus, ParsedSlice, Parser, Reference}, reader::{Read, Reader}, - util::num::ParserNumber, value::{node::Value, shared::Shared}, JsonInput, }; - const MAX_ALLOWED_DEPTH: u8 = u8::MAX; ////////////////////////////////////////////////////////////////////////////// @@ -262,24 +261,22 @@ impl<'a, R> Drop for DepthGuard<'a, R> { } } -impl ParserNumber { - fn visit<'de, V>(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - match self { - ParserNumber::Float(x) => visitor.visit_f64(x), - ParserNumber::Unsigned(x) => visitor.visit_u64(x), - ParserNumber::Signed(x) => visitor.visit_i64(x), - } +fn visit_number<'de, V>(num: &ParserNumber, visitor: V) -> Result +where + V: de::Visitor<'de>, +{ + match *num { + ParserNumber::Float(x) => visitor.visit_f64(x), + ParserNumber::Unsigned(x) => visitor.visit_u64(x), + ParserNumber::Signed(x) => visitor.visit_i64(x), } +} - pub(crate) fn invalid_type(self, exp: &dyn Expected) -> Error { - match self { - ParserNumber::Float(x) => de::Error::invalid_type(Unexpected::Float(x), exp), - ParserNumber::Unsigned(x) => de::Error::invalid_type(Unexpected::Unsigned(x), exp), - ParserNumber::Signed(x) => de::Error::invalid_type(Unexpected::Signed(x), exp), - } +pub(crate) fn invalid_type_number(num: &ParserNumber, exp: &dyn Expected) -> Error { + match *num { + ParserNumber::Float(x) => de::Error::invalid_type(Unexpected::Float(x), exp), + ParserNumber::Unsigned(x) => de::Error::invalid_type(Unexpected::Unsigned(x), exp), + ParserNumber::Signed(x) => de::Error::invalid_type(Unexpected::Signed(x), exp), } } @@ -305,7 +302,7 @@ impl<'de, R: Reader<'de>> Deserializer { }; let value = match peek { - c @ b'-' | c @ b'0'..=b'9' => tri!(self.parser.parse_number(c)).visit(visitor), + c @ b'-' | c @ b'0'..=b'9' => visit_number(&tri!(self.parser.parse_number(c)), visitor), _ => Err(self.peek_invalid_type(peek, &visitor)), }; @@ -380,6 +377,7 @@ impl<'de, R: Reader<'de>> Deserializer { if self.parser.read.index() == 0 { // will parse the JSON inplace let cfg = self.parser.cfg; + dbg!(cfg); let json = self.parser.read.as_u8_slice(); // get n to check trailing characters in later @@ -476,7 +474,7 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer tri!(self.parser.parse_literal("alse")); visitor.visit_bool(false) } - c @ b'-' | c @ b'0'..=b'9' => tri!(self.parser.parse_number(c)).visit(visitor), + c @ b'-' | c @ b'0'..=b'9' => visit_number(&tri!(self.parser.parse_number(c)), visitor), b'"' => match tri!(self.parser.parse_str_impl(&mut self.scratch)) { Reference::Borrowed(s) => visitor.visit_borrowed_str(s), Reference::Copied(s) => visitor.visit_str(s), diff --git a/src/serde/number.rs b/src/serde/number.rs index b32bf91..25d386a 100644 --- a/src/serde/number.rs +++ b/src/serde/number.rs @@ -1,9 +1,8 @@ // The code is cloned from [serde_json](https://github.com/serde-rs/json) and modified necessary parts. -use crate::{ - error::make_error, - util::{num::ParserNumber, private::Sealed}, -}; +use sonic_number::ParserNumber; + +use crate::{error::make_error, util::private::Sealed}; /// Represents a JSON number, whether integer or floating point. #[derive(Clone, PartialEq, Eq, Hash)] diff --git a/src/util/arch/aarch64.rs b/src/util/arch/aarch64.rs index 564d068..7ea63f5 100644 --- a/src/util/arch/aarch64.rs +++ b/src/util/arch/aarch64.rs @@ -68,22 +68,10 @@ pub unsafe fn get_nonspace_bits(data: &[u8; 64]) -> u64 { vtstq_u8(v, white_mask) } - !crate::util::simd::neon::to_bitmask64( + !sonic_simd::neon::to_bitmask64( chunk_nonspace_bits(vld1q_u8(data.as_ptr())), chunk_nonspace_bits(vld1q_u8(data.as_ptr().offset(16))), chunk_nonspace_bits(vld1q_u8(data.as_ptr().offset(32))), chunk_nonspace_bits(vld1q_u8(data.as_ptr().offset(48))), ) } - -#[inline(always)] -pub unsafe fn simd_str2int(c: &[u8], need: usize) -> (u64, usize) { - debug_assert!(need < 17); - let mut sum = 0u64; - let mut i = 0; - while i < need && c.get_unchecked(i).is_ascii_digit() { - sum = (c.get_unchecked(i) - b'0') as u64 + sum * 10; - i += 1; - } - (sum, i) -} diff --git a/src/util/arch/fallback.rs b/src/util/arch/fallback.rs index 960a395..6373431 100644 --- a/src/util/arch/fallback.rs +++ b/src/util/arch/fallback.rs @@ -21,15 +21,3 @@ pub unsafe fn get_nonspace_bits(data: &[u8; 64]) -> u64 { } mask } - -#[inline(always)] -pub unsafe fn simd_str2int(c: &[u8], need: usize) -> (u64, usize) { - debug_assert!(need < 17); - let mut sum = 0u64; - let mut i = 0; - while i < need && c.get_unchecked(i).is_ascii_digit() { - sum = (c.get_unchecked(i) - b'0') as u64 + sum * 10; - i += 1; - } - (sum, i) -} diff --git a/src/util/arch/x86_64.rs b/src/util/arch/x86_64.rs index c70c499..7bcda4b 100644 --- a/src/util/arch/x86_64.rs +++ b/src/util/arch/x86_64.rs @@ -58,110 +58,3 @@ pub unsafe fn get_nonspace_bits(data: &[u8; 64]) -> u64 { !space } } - -macro_rules! packadd_1 { - ($v:ident) => { - let delta = _mm_set1_epi64x(0x010A010A010A010A); - $v = _mm_maddubs_epi16($v, delta); - }; -} - -macro_rules! packadd_2 { - ($v:ident) => { - let delta = _mm_set1_epi64x(0x0001006400010064); - $v = _mm_madd_epi16($v, delta); - }; -} - -macro_rules! packadd_4 { - ($v:ident) => { - $v = _mm_packus_epi32($v, $v); - let delta = _mm_set_epi16(0, 0, 0, 0, 1, 10000, 1, 10000); - $v = _mm_madd_epi16($v, delta); - }; -} - -// simd add for 5 ~ 8 digits -macro_rules! simd_add_5_8 { - ($v:ident, $and:literal) => {{ - $v = _mm_slli_si128($v, 16 - $and); - packadd_1!($v); - packadd_2!($v); - (_mm_extract_epi32($v, 2) as u64) * 10000 + (_mm_extract_epi32($v, 3) as u64) - }}; -} - -// simd add for 9 ~ 15 digits -macro_rules! simd_add_9_15 { - ($v:ident, $and:literal) => {{ - $v = _mm_slli_si128($v, 16 - $and); - packadd_1!($v); - packadd_2!($v); - packadd_4!($v); - (_mm_extract_epi32($v, 0) as u64) * 100000000 + (_mm_extract_epi32($v, 1) as u64) - }}; -} - -macro_rules! simd_add_16 { - ($v:ident) => {{ - packadd_1!($v); - packadd_2!($v); - packadd_4!($v); - (_mm_extract_epi32($v, 0) as u64) * 100000000 + (_mm_extract_epi32($v, 1) as u64) - }}; -} -#[inline(always)] -pub unsafe fn simd_str2int(c: &[u8], need: usize) -> (u64, usize) { - debug_assert!(need <= 16); - let data = _mm_loadu_si128(c.as_ptr() as *const __m128i); - let zero = _mm_setzero_si128(); - let nine = _mm_set1_epi8(9); - let zero_c = _mm_set1_epi8(b'0' as i8); - - let mut data = _mm_sub_epi8(data, zero_c); - let lt_zero = _mm_cmpgt_epi8(zero, data); - let gt_nine = _mm_cmpgt_epi8(data, nine); - - let is_num_end = _mm_or_si128(lt_zero, gt_nine); - let is_num_end_int = _mm_movemask_epi8(is_num_end); - - // get the real parsed count - let mut count = need; - if is_num_end_int != 0 { - let digits = is_num_end_int.trailing_zeros() as usize; - if digits < need { - count = digits; - } - } - - let sum = match count { - 1 => _mm_extract_epi8(data, 0) as u64, - 2 => (_mm_extract_epi8(data, 0) * 10 + _mm_extract_epi8(data, 1)) as u64, - 3 => { - // shift to clear the non-digit ascii in vector - data = _mm_slli_si128(data, 16 - 3); - packadd_1!(data); - // add the highest two lanes - (_mm_extract_epi16(data, 6) * 100 + _mm_extract_epi16(data, 7)) as u64 - } - 4 => { - data = _mm_slli_si128(data, 16 - 4); - packadd_1!(data); - (_mm_extract_epi16(data, 6) * 100 + _mm_extract_epi16(data, 7)) as u64 - } - 5 => simd_add_5_8!(data, 5), - 6 => simd_add_5_8!(data, 6), - 7 => simd_add_5_8!(data, 7), - 8 => simd_add_5_8!(data, 8), - 9 => simd_add_9_15!(data, 9), - 10 => simd_add_9_15!(data, 10), - 11 => simd_add_9_15!(data, 11), - 12 => simd_add_9_15!(data, 12), - 13 => simd_add_9_15!(data, 13), - 14 => simd_add_9_15!(data, 14), - 15 => simd_add_9_15!(data, 15), - 16 => simd_add_16!(data), - _ => unreachable!(), - }; - (sum, count) -} diff --git a/src/util/mod.rs b/src/util/mod.rs index 315dda4..c06a6d2 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,15 +1,10 @@ pub(crate) mod arch; -pub(crate) mod num; pub(crate) mod private; pub(crate) mod reborrow; pub(crate) mod string; pub(crate) mod unicode; pub(crate) mod utf8; -#[allow(non_camel_case_types)] -#[allow(unused_imports)] -pub(crate) mod simd; - #[cfg(test)] pub(crate) mod mock; diff --git a/src/util/string.rs b/src/util/string.rs index b85d0b0..5bb8d61 100644 --- a/src/util/string.rs +++ b/src/util/string.rs @@ -5,17 +5,16 @@ use std::{ }; #[cfg(not(all(target_feature = "neon", target_arch = "aarch64")))] -use crate::util::simd::u8x32; +use sonic_simd::u8x32; #[cfg(all(target_feature = "neon", target_arch = "aarch64"))] -use crate::util::simd::{bits::NeonBits, u8x16}; +use sonic_simd::{bits::NeonBits, u8x16}; +use sonic_simd::{BitMask, Mask, Simd}; + use crate::{ error::ErrorCode::{ self, ControlCharacterWhileParsingString, InvalidEscape, InvalidUnicodeCodePoint, }, - util::{ - simd::{BitMask, Mask, Simd}, - unicode::handle_unicode_codepoint_mut, - }, + util::unicode::handle_unicode_codepoint_mut, }; #[inline(always)]