-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
📦 NEW: New value net. Add data truncation script
1 parent
3b80994
commit f672b76
Showing
5 changed files
with
107,241 additions
and
107,177 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
use princhess::train::TrainingPosition; | ||
|
||
use std::env; | ||
use std::fs::File; | ||
use std::io::{self, BufRead, BufReader, BufWriter, Write}; | ||
use std::time::Instant; | ||
|
||
const BUFFER_COUNT: usize = 1 << 16; | ||
|
||
fn main() { | ||
let mut args = env::args(); | ||
args.next(); | ||
|
||
let truncate_to = args.next().unwrap().parse::<usize>().unwrap(); | ||
let files = args.collect::<Vec<String>>(); | ||
|
||
for input in files { | ||
let file = File::open(input.clone()).unwrap(); | ||
let positions = file.metadata().unwrap().len() as usize / TrainingPosition::SIZE; | ||
|
||
let buffer_size = BUFFER_COUNT * TrainingPosition::SIZE; | ||
let mut buffer = BufReader::with_capacity(buffer_size, file); | ||
|
||
let out_file = format!("{}.{}m.truncated", input, truncate_to); | ||
let mut writer = | ||
BufWriter::new(File::create(out_file).unwrap()); | ||
|
||
let start = Instant::now(); | ||
println!( | ||
"Truncating {} positions from {} to {}m positions...", | ||
positions, input, truncate_to | ||
); | ||
|
||
let mut processed = 0; | ||
|
||
while let Ok(bytes) = buffer.fill_buf() { | ||
if bytes.is_empty() { | ||
break; | ||
} | ||
|
||
let data = TrainingPosition::read_batch(bytes); | ||
TrainingPosition::write_batch(&mut writer, data).unwrap(); | ||
|
||
processed += data.len(); | ||
|
||
print!( | ||
"{:>8} / {} ({:2}%)\r", | ||
processed, | ||
truncate_to * 1000000, | ||
processed * 100 / (truncate_to * 1000000), | ||
); | ||
io::stdout().flush().unwrap(); | ||
|
||
let consumed = bytes.len(); | ||
buffer.consume(consumed); | ||
|
||
if processed >= truncate_to * 1000000 { | ||
break; | ||
} | ||
} | ||
|
||
println!("Wrote {}m positions in {:?}", truncate_to, start.elapsed()); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,56 +1,56 @@ | ||
[ | ||
-88, -59, 114, -124, -90, 13, -116, -21, | ||
-38, 45, -47, -82, 152, -34, -58, -14, | ||
60, -57, 7, -105, -20, 25, 121, -147, | ||
80, -29, 14, 46, -42, -6, 3, 95, | ||
-63, -43, -1, -39, 10, -161, -175, -95, | ||
43, -101, -80, 157, -83, 62, -46, -77, | ||
-89, 81, 27, -46, -62, -37, -53, 0, | ||
-19, -34, -35, -66, -89, -121, -33, -28, | ||
-97, -97, -242, -22, -36, -16, -142, 1, | ||
-5, -130, -143, -3, -35, -50, 33, -23, | ||
132, -73, 7, -22, -139, -174, -49, 136, | ||
-102, -136, -127, 74, -75, -5, -17, -113, | ||
81, -80, -3, -23, -66, -84, -132, -120, | ||
-51, -112, 126, -6, -68, -14, -196, -89, | ||
-123, 58, -164, -88, -58, 43, -123, -50, | ||
-8, -85, 25, -44, -67, -44, -101, -20, | ||
|
||
-42, -16, -167, 16, -142, -137, -32, -52, | ||
94, -69, -17, -101, -121, 27, -43, -25, | ||
-6, 23, -87, -108, 23, -13, -46, -92, | ||
-117, -81, -51, -54, 142, -136, -126, -11, | ||
-140, -200, -119, -3, -102, -7, -117, -18, | ||
-69, -55, -104, -21, -85, -84, 9, -32, | ||
-1, -127, -52, -194, -34, 61, -92, -170, | ||
-87, -59, 23, -19, 35, -30, -27, -90, | ||
-30, -19, -191, -38, -48, -81, -87, -146, | ||
-112, -134, -13, -182, -74, -125, -141, 93, | ||
-150, -107, -32, -17, -161, -32, 6, -175, | ||
-117, -3, -161, -4, -77, -133, -46, -108, | ||
2, -3, -74, -43, -28, -64, -20, -112, | ||
-47, -27, -45, -86, -32, 87, 45, 76, | ||
-56, 73, -5, -7, -35, -1, -67, -62, | ||
-34, 1, -99, -34, -74, -114, -15, -25, | ||
|
||
0, -109, -8, -13, -64, -166, 71, -27, | ||
-33, -125, -58, -104, -87, 4, -116, -16, | ||
100, 5, -27, 5, -228, -19, -36, 133, | ||
-100, -68, -4, -58, -81, -43, -157, -108, | ||
38, -40, -52, -58, -49, -77, -33, -11, | ||
-13, -4, -53, -167, -48, -77, -87, 0, | ||
50, -34, 8, -112, -90, -33, -50, -26, | ||
-74, -11, -207, -70, -1, 73, -4, -10, | ||
-23, -19, -12, -30, -27, -93, -51, -54, | ||
-27, -4, 108, -63, -197, 180, -4, -103, | ||
-42, -102, 0, -28, 26, -78, -29, -101, | ||
78, -124, -45, -37, -115, -127, 15, 47, | ||
-6, -25, -129, -134, -152, -158, -89, -5, | ||
-89, -14, -163, -121, 40, -168, -19, -153, | ||
-110, -1, -62, -186, 17, -13, -25, 8, | ||
-134, -64, -83, 7, -5, -155, -67, -85, | ||
|
||
8, 76, -33, -49, -121, -13, -77, 21, | ||
-104, 69, 0, -1, -18, -59, 167, -82, | ||
25, 43, -8, 9, -8, -281, -55, 14, | ||
-87, 3, -24, 30, 59, -159, -18, 23, | ||
-157, -20, -70, -76, -63, 23, -136, -31, | ||
5, -101, -2, -33, -107, -45, -41, -63, | ||
-70, -51, -179, -141, -11, 11, -10, -184, | ||
4, -6, -140, -8, -164, -87, -8, 12, | ||
140, 2, -29, 0, -45, -48, -106, -110, | ||
114, 150, -165, -69, 27, -14, -79, -29, | ||
-51, -65, -66, -192, -18, -104, 51, -157, | ||
-112, -16, -24, -120, -62, 116, 0, -42, | ||
-167, -237, -87, -95, -39, -123, -4, -47, | ||
0, -100, -34, -69, -100, -74, -34, -61, | ||
-300, 23, 15, -82, 104, 0, 25, -100, | ||
79, -46, 6, -40, -115, -63, 52, 24, | ||
|
||
-94, -164, -18, -1, -103, 25, -48, -72, | ||
-188, -155, -46, -19, -197, -86, -133, -159, | ||
-90, 88, -46, 52, 33, -65, -50, 1, | ||
-88, -43, -6, -88, 126, -121, -149, -21, | ||
-13, -171, -47, -21, -30, -117, -84, 31, | ||
-114, -1, -80, -53, -21, -135, 74, 74, | ||
-201, -10, -10, -29, -86, -179, -88, -73, | ||
-78, -184, -130, -68, -21, -165, -151, -56, | ||
32, -11, -89, 81, -208, -82, -71, -111, | ||
41, -28, -82, -33, -35, -43, 120, -80, | ||
78, -21, -97, -113, -88, -42, -79, -159, | ||
-22, -41, -226, -18, -105, -25, -30, -108, | ||
-97, -84, -76, -145, 56, -120, -34, -99, | ||
-92, -77, -37, -14, -32, -70, -28, 79, | ||
17, -39, -67, -38, -35, 20, 26, -30, | ||
104, 55, 13, 35, -121, -8, -9, -10, | ||
|
||
-133, -109, -9, -57, 87, -169, -59, -1, | ||
-19, -14, -30, 72, -14, -107, -28, -66, | ||
-49, -41, -85, -36, -8, -51, -93, -49, | ||
22, 87, -52, -52, -71, 19, 170, -18, | ||
-112, -13, 2, -76, 31, 0, -47, -72, | ||
-13, -38, 38, 45, -160, -83, -81, -32, | ||
26, 120, -128, 43, -60, -164, 8, -142, | ||
-48, -85, -111, 70, -16, -5, -166, -90, | ||
-58, 20, -108, 84, 46, -8, -89, -22, | ||
1, -99, 18, -91, -52, -71, -198, 0, | ||
2, -26, 6, -39, -51, -141, -27, -33, | ||
-43, -21, 73, -39, -180, -49, -103, 8, | ||
-118, -91, -156, -99, 9, 87, -22, -61, | ||
69, -35, 21, -104, -7, -172, -112, 19, | ||
67, -3, 10, -72, -15, -6, -134, -69, | ||
-78, -79, -92, 70, -54, -56, -40, -107, | ||
|
||
] |
Oops, something went wrong.