Skip to content

Commit

Permalink
📦 NEW: New value net. Add data truncation script
Browse files Browse the repository at this point in the history
  • Loading branch information
@ianagbip1oti
ianagbip1oti authored Mar 11, 2024
1 parent 3b80994 commit f672b76
Showing 5 changed files with 107,241 additions and 107,177 deletions.
64 changes: 64 additions & 0 deletions src/bin/data-truncate.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
use princhess::train::TrainingPosition;

use std::env;
use std::fs::File;
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::time::Instant;

const BUFFER_COUNT: usize = 1 << 16;

fn main() {
let mut args = env::args();
args.next();

let truncate_to = args.next().unwrap().parse::<usize>().unwrap();
let files = args.collect::<Vec<String>>();

for input in files {
let file = File::open(input.clone()).unwrap();
let positions = file.metadata().unwrap().len() as usize / TrainingPosition::SIZE;

let buffer_size = BUFFER_COUNT * TrainingPosition::SIZE;
let mut buffer = BufReader::with_capacity(buffer_size, file);

let out_file = format!("{}.{}m.truncated", input, truncate_to);
let mut writer =
BufWriter::new(File::create(out_file).unwrap());

let start = Instant::now();
println!(
"Truncating {} positions from {} to {}m positions...",
positions, input, truncate_to
);

let mut processed = 0;

while let Ok(bytes) = buffer.fill_buf() {
if bytes.is_empty() {
break;
}

let data = TrainingPosition::read_batch(bytes);
TrainingPosition::write_batch(&mut writer, data).unwrap();

processed += data.len();

print!(
"{:>8} / {} ({:2}%)\r",
processed,
truncate_to * 1000000,
processed * 100 / (truncate_to * 1000000),
);
io::stdout().flush().unwrap();

let consumed = bytes.len();
buffer.consume(consumed);

if processed >= truncate_to * 1000000 {
break;
}
}

println!("Wrote {}m positions in {:?}", truncate_to, start.elapsed());
}
}
96 changes: 48 additions & 48 deletions src/value/hidden_bias
View file
Original file line number Diff line number Diff line change
@@ -1,56 +1,56 @@
[
-88, -59, 114, -124, -90, 13, -116, -21,
-38, 45, -47, -82, 152, -34, -58, -14,
60, -57, 7, -105, -20, 25, 121, -147,
80, -29, 14, 46, -42, -6, 3, 95,
-63, -43, -1, -39, 10, -161, -175, -95,
43, -101, -80, 157, -83, 62, -46, -77,
-89, 81, 27, -46, -62, -37, -53, 0,
-19, -34, -35, -66, -89, -121, -33, -28,
-97, -97, -242, -22, -36, -16, -142, 1,
-5, -130, -143, -3, -35, -50, 33, -23,
132, -73, 7, -22, -139, -174, -49, 136,
-102, -136, -127, 74, -75, -5, -17, -113,
81, -80, -3, -23, -66, -84, -132, -120,
-51, -112, 126, -6, -68, -14, -196, -89,
-123, 58, -164, -88, -58, 43, -123, -50,
-8, -85, 25, -44, -67, -44, -101, -20,

-42, -16, -167, 16, -142, -137, -32, -52,
94, -69, -17, -101, -121, 27, -43, -25,
-6, 23, -87, -108, 23, -13, -46, -92,
-117, -81, -51, -54, 142, -136, -126, -11,
-140, -200, -119, -3, -102, -7, -117, -18,
-69, -55, -104, -21, -85, -84, 9, -32,
-1, -127, -52, -194, -34, 61, -92, -170,
-87, -59, 23, -19, 35, -30, -27, -90,
-30, -19, -191, -38, -48, -81, -87, -146,
-112, -134, -13, -182, -74, -125, -141, 93,
-150, -107, -32, -17, -161, -32, 6, -175,
-117, -3, -161, -4, -77, -133, -46, -108,
2, -3, -74, -43, -28, -64, -20, -112,
-47, -27, -45, -86, -32, 87, 45, 76,
-56, 73, -5, -7, -35, -1, -67, -62,
-34, 1, -99, -34, -74, -114, -15, -25,

0, -109, -8, -13, -64, -166, 71, -27,
-33, -125, -58, -104, -87, 4, -116, -16,
100, 5, -27, 5, -228, -19, -36, 133,
-100, -68, -4, -58, -81, -43, -157, -108,
38, -40, -52, -58, -49, -77, -33, -11,
-13, -4, -53, -167, -48, -77, -87, 0,
50, -34, 8, -112, -90, -33, -50, -26,
-74, -11, -207, -70, -1, 73, -4, -10,
-23, -19, -12, -30, -27, -93, -51, -54,
-27, -4, 108, -63, -197, 180, -4, -103,
-42, -102, 0, -28, 26, -78, -29, -101,
78, -124, -45, -37, -115, -127, 15, 47,
-6, -25, -129, -134, -152, -158, -89, -5,
-89, -14, -163, -121, 40, -168, -19, -153,
-110, -1, -62, -186, 17, -13, -25, 8,
-134, -64, -83, 7, -5, -155, -67, -85,

8, 76, -33, -49, -121, -13, -77, 21,
-104, 69, 0, -1, -18, -59, 167, -82,
25, 43, -8, 9, -8, -281, -55, 14,
-87, 3, -24, 30, 59, -159, -18, 23,
-157, -20, -70, -76, -63, 23, -136, -31,
5, -101, -2, -33, -107, -45, -41, -63,
-70, -51, -179, -141, -11, 11, -10, -184,
4, -6, -140, -8, -164, -87, -8, 12,
140, 2, -29, 0, -45, -48, -106, -110,
114, 150, -165, -69, 27, -14, -79, -29,
-51, -65, -66, -192, -18, -104, 51, -157,
-112, -16, -24, -120, -62, 116, 0, -42,
-167, -237, -87, -95, -39, -123, -4, -47,
0, -100, -34, -69, -100, -74, -34, -61,
-300, 23, 15, -82, 104, 0, 25, -100,
79, -46, 6, -40, -115, -63, 52, 24,

-94, -164, -18, -1, -103, 25, -48, -72,
-188, -155, -46, -19, -197, -86, -133, -159,
-90, 88, -46, 52, 33, -65, -50, 1,
-88, -43, -6, -88, 126, -121, -149, -21,
-13, -171, -47, -21, -30, -117, -84, 31,
-114, -1, -80, -53, -21, -135, 74, 74,
-201, -10, -10, -29, -86, -179, -88, -73,
-78, -184, -130, -68, -21, -165, -151, -56,
32, -11, -89, 81, -208, -82, -71, -111,
41, -28, -82, -33, -35, -43, 120, -80,
78, -21, -97, -113, -88, -42, -79, -159,
-22, -41, -226, -18, -105, -25, -30, -108,
-97, -84, -76, -145, 56, -120, -34, -99,
-92, -77, -37, -14, -32, -70, -28, 79,
17, -39, -67, -38, -35, 20, 26, -30,
104, 55, 13, 35, -121, -8, -9, -10,

-133, -109, -9, -57, 87, -169, -59, -1,
-19, -14, -30, 72, -14, -107, -28, -66,
-49, -41, -85, -36, -8, -51, -93, -49,
22, 87, -52, -52, -71, 19, 170, -18,
-112, -13, 2, -76, 31, 0, -47, -72,
-13, -38, 38, 45, -160, -83, -81, -32,
26, 120, -128, 43, -60, -164, 8, -142,
-48, -85, -111, 70, -16, -5, -166, -90,
-58, 20, -108, 84, 46, -8, -89, -22,
1, -99, 18, -91, -52, -71, -198, 0,
2, -26, 6, -39, -51, -141, -27, -33,
-43, -21, 73, -39, -180, -49, -103, 8,
-118, -91, -156, -99, 9, 87, -22, -61,
69, -35, 21, -104, -7, -172, -112, 19,
67, -3, 10, -72, -15, -6, -134, -69,
-78, -79, -92, 70, -54, -56, -40, -107,

]
Loading

0 comments on commit f672b76

Please sign in to comment.