From f25621ffaaa1d3e56bef11a3770b0fbf7b7e7407 Mon Sep 17 00:00:00 2001 From: Laiho Date: Wed, 21 Aug 2024 13:38:56 +0300 Subject: [PATCH] add threading enum --- src/node/src/lib.rs | 19 +++++---- src/parser/src/e2e_test.rs | 11 +++--- src/parser/src/parse_demo.rs | 45 +++++++++++++--------- src/parser/src/second_pass/collect_data.rs | 1 - src/python/src/lib.rs | 20 +++++----- src/wasm/src/lib.rs | 13 ++++--- 6 files changed, 59 insertions(+), 50 deletions(-) diff --git a/src/node/src/lib.rs b/src/node/src/lib.rs index 2be61de8..735646b2 100644 --- a/src/node/src/lib.rs +++ b/src/node/src/lib.rs @@ -20,7 +20,6 @@ use std::collections::HashMap; use std::fs::File; use std::hash::RandomState; use std::result::Result; -use std::time::Instant; fn parse_demo(bytes: BytesVariant, parser: &mut Parser) -> Result { match bytes { @@ -52,7 +51,7 @@ pub fn parse_voice(path_or_buf: Either) -> napi::Result out, @@ -85,7 +84,7 @@ pub fn list_game_events(path_or_buf: Either) -> napi::Result) -> napi::Result) -> napi::Result huffman_lookup_table: &huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = parse_demo(bytes, &mut parser)?; let mut hm: HashMap = HashMap::default(); @@ -209,7 +208,7 @@ pub fn parse_event( huffman_lookup_table: &huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = parse_demo(bytes, &mut parser)?; let s = match serde_json::to_value(&output.game_events) { Ok(s) => s, @@ -271,7 +270,7 @@ pub fn parse_events( huffman_lookup_table: &huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = parse_demo(bytes, &mut parser)?; let s = match serde_json::to_value(&output.game_events) { Ok(s) => s, @@ -331,7 +330,7 @@ pub fn parse_ticks( order_by_steamid: order_by_steamid, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = parse_demo(bytes, &mut parser)?; real_names.push("tick".to_owned()); real_names.push("steamid".to_owned()); @@ -402,7 +401,7 @@ pub fn parse_player_info(path_or_buf: Either) -> napi::Result s, @@ -431,7 +430,7 @@ pub fn parse_player_skins(path_or_buf: Either) -> napi::Result s, diff --git a/src/parser/src/e2e_test.rs b/src/parser/src/e2e_test.rs index 1eb66d39..f69d5075 100644 --- a/src/parser/src/e2e_test.rs +++ b/src/parser/src/e2e_test.rs @@ -315,7 +315,7 @@ pub fn _create_ge_tests() { order_by_steamid: false, }; - let mut ds = Parser::new(settings, true); + let mut ds = Parser::new(settings, crate::parse_demo::ParsingMode::ForceMultiThreaded); // ds.is_multithreadable = false; let file = File::open("test_demo.dem".to_string()).unwrap(); let mmap = unsafe { MmapOptions::new().map(&file).unwrap() }; @@ -688,7 +688,7 @@ pub fn _create_tests() { order_by_steamid: false, }; - let mut ds = Parser::new(settings, true); + let mut ds = Parser::new(settings, crate::parse_demo::ParsingMode::ForceMultiThreaded); let file = File::open("test_demo.dem".to_string()).unwrap(); let mmap = unsafe { MmapOptions::new().map(&file).unwrap() }; let d = ds.parse_demo(&mmap).unwrap(); @@ -1060,7 +1060,7 @@ fn create_data() -> (DemoOutput, PropController, BTreeMap order_by_steamid: false, }; - let mut ds = Parser::new(settings, false); + let mut ds = Parser::new(settings, crate::parse_demo::ParsingMode::ForceMultiThreaded); let file = File::open("test_demo.dem".to_string()).unwrap(); let mmap = unsafe { MmapOptions::new().map(&file).unwrap() }; let out1 = ds.parse_demo(&mmap).unwrap(); @@ -1083,7 +1083,7 @@ fn create_data() -> (DemoOutput, PropController, BTreeMap huffman_lookup_table: &huf, order_by_steamid: false, }; - let mut ds = Parser::new(settings, false); + let mut ds = Parser::new(settings, crate::parse_demo::ParsingMode::ForceMultiThreaded); let file = File::open("test_demo.dem".to_string()).unwrap(); let mmap = unsafe { MmapOptions::new().map(&file).unwrap() }; let out2 = ds.parse_demo(&mmap).unwrap(); @@ -1198,7 +1198,8 @@ mod tests { huffman_lookup_table: &huf, order_by_steamid: false, }; - let mut ds = Parser::new(settings, true); + + let mut ds = Parser::new(settings, crate::parse_demo::ParsingMode::ForceMultiThreaded); let file = File::open("test_demo.dem").unwrap(); let mmap = unsafe { MmapOptions::new().map(&file).unwrap() }; let output = ds.parse_demo(&mmap).unwrap(); diff --git a/src/parser/src/parse_demo.rs b/src/parser/src/parse_demo.rs index 9ade207c..d1e75474 100644 --- a/src/parser/src/parse_demo.rs +++ b/src/parser/src/parse_demo.rs @@ -41,34 +41,43 @@ pub struct DemoOutput { pub struct Parser<'a> { input: ParserInputs<'a>, - pub force_singlethread: bool, + pub parsing_mode: ParsingMode, +} +#[derive(PartialEq)] +pub enum ParsingMode { + ForceSingleThreaded, + ForceMultiThreaded, + Normal, } impl<'a> Parser<'a> { - pub fn new(input: ParserInputs<'a>, force_singlethread: bool) -> Self { + pub fn new(input: ParserInputs<'a>, parsing_mode: ParsingMode) -> Self { Parser { input: input, - force_singlethread: force_singlethread, + parsing_mode: parsing_mode, } } pub fn parse_demo(&mut self, demo_bytes: &[u8]) -> Result { - // Single threaded second pass - if !check_multithreadability(&self.input.wanted_player_props) || self.force_singlethread { - let mut first_pass_parser = FirstPassParser::new(&self.input); - let first_pass_output = first_pass_parser.parse_demo(&demo_bytes, false)?; - return self.second_pass_single_threaded(demo_bytes, first_pass_output); - } // Multi threaded second pass - let (sender, receiver) = channel(); - let mut fp = FrameParser::new(); - return thread::scope(|s| { - let _handle = s.spawn(|| fp.par_start(demo_bytes, sender)); - let mut first_pass_parser = FirstPassParser::new(&self.input); - let first_pass_output = first_pass_parser.parse_demo(&demo_bytes, true).unwrap(); - let out = self.second_pass_threaded_with_channels(demo_bytes, first_pass_output, receiver); - out - }); + if self.parsing_mode == ParsingMode::ForceMultiThreaded + || check_multithreadability(&self.input.wanted_player_props) + && !(self.parsing_mode == ParsingMode::ForceSingleThreaded) + { + let (sender, receiver) = channel(); + let mut fp = FrameParser::new(); + return thread::scope(|s| { + let _handle = s.spawn(|| fp.par_start(demo_bytes, sender)); + let mut first_pass_parser = FirstPassParser::new(&self.input); + let first_pass_output = first_pass_parser.parse_demo(&demo_bytes, true).unwrap(); + let out = self.second_pass_threaded_with_channels(demo_bytes, first_pass_output, receiver); + out + }); + } + // Single threaded second pass + let mut first_pass_parser = FirstPassParser::new(&self.input); + let first_pass_output = first_pass_parser.parse_demo(&demo_bytes, false)?; + return self.second_pass_single_threaded(demo_bytes, first_pass_output); } fn second_pass_single_threaded( diff --git a/src/parser/src/second_pass/collect_data.rs b/src/parser/src/second_pass/collect_data.rs index 5693d35c..d9e33e5b 100644 --- a/src/parser/src/second_pass/collect_data.rs +++ b/src/parser/src/second_pass/collect_data.rs @@ -10,7 +10,6 @@ use crate::maps::PAINTKITS; use crate::maps::PLAYER_COLOR; use crate::maps::STICKER_ID_TO_NAME; use crate::maps::WEAPINDICIES; -use crate::second_pass::entities::Entity; use crate::second_pass::entities::EntityType; use crate::second_pass::parser_settings::SecondPassParser; use crate::second_pass::variants::PropColumn; diff --git a/src/python/src/lib.rs b/src/python/src/lib.rs index 28c017ea..a28d6de0 100644 --- a/src/python/src/lib.rs +++ b/src/python/src/lib.rs @@ -70,7 +70,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -97,7 +97,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -131,7 +131,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -201,7 +201,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -245,7 +245,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -325,7 +325,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -423,7 +423,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -479,7 +479,7 @@ impl DemoParser { huffman_lookup_table: &self.huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), @@ -507,7 +507,7 @@ impl DemoParser { huffman_lookup_table: &vec![], order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(PyValueError::new_err(format!("{e}"))), @@ -558,7 +558,7 @@ impl DemoParser { huffman_lookup_table: &arc_huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, parser::parse_demo::ParsingMode::Normal); let output = match parser.parse_demo(&self.mmap) { Ok(output) => output, Err(e) => return Err(Exception::new_err(format!("{e}"))), diff --git a/src/wasm/src/lib.rs b/src/wasm/src/lib.rs index 57d312a2..fe468eb8 100644 --- a/src/wasm/src/lib.rs +++ b/src/wasm/src/lib.rs @@ -1,6 +1,7 @@ use parser::first_pass::parser_settings::rm_user_friendly_names; use parser::first_pass::parser_settings::ParserInputs; use parser::parse_demo::Parser; +use parser::parse_demo::ParsingMode::ForceSingleThreaded; use parser::second_pass::parser_settings::create_huffman_lookup_table; use parser::second_pass::variants::soa_to_aos; use parser::second_pass::variants::OutputSerdeHelperStruct; @@ -57,7 +58,7 @@ pub fn parseEvent( huffman_lookup_table: &arc_huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, ForceSingleThreaded); let output = match parser.parse_demo(&file) { Ok(output) => output, @@ -119,7 +120,7 @@ pub fn parseEvents( huffman_lookup_table: &arc_huf, order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, ForceSingleThreaded); let output = match parser.parse_demo(&file) { Ok(output) => output, @@ -149,7 +150,7 @@ pub fn listGameEvents(fileBytes: Vec) -> Result { huffman_lookup_table: &arc_huf.clone(), order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, ForceSingleThreaded); let output = match parser.parse_demo(&fileBytes) { Ok(output) => output, @@ -209,7 +210,7 @@ pub fn parseTicks( huffman_lookup_table: &arc_huf.clone(), order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, ForceSingleThreaded); let output = match parser.parse_demo(&file) { Ok(output) => output, @@ -263,7 +264,7 @@ pub fn parseGrenades(file: Vec) -> Result { huffman_lookup_table: &arc_huf.clone(), order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, ForceSingleThreaded); let output = match parser.parse_demo(&file) { Ok(output) => output, @@ -295,7 +296,7 @@ pub fn parseHeader(file: Vec) -> Result { huffman_lookup_table: &arc_huf.clone(), order_by_steamid: false, }; - let mut parser = Parser::new(settings, false); + let mut parser = Parser::new(settings, ForceSingleThreaded); let output = match parser.parse_demo(&file) { Ok(output) => output, Err(e) => return Err(JsError::new(&format!("{}", e))),