bin support (#6)

* chore: minor cleanup

* chore: tweak nix flake for intellij

* feat: optional serde dependency for wads

* fix: ignore wad source buffer in serde ser

* fix: serde ignore -> skip typo

* feat: optional wasm_bindgen dep

* fix: make wad chunk fields public

* feat: wasm support for WadChunkCompression

* fix: attempt to get wasm working

* fix: remove unnecessary expect

* feat: add ruzstd support (for wasm)

* fix: remove wasm stuff

* chore: remove commented debug junk

* chore: remove more debug logs

* feat: ReaderExt specific Error type

* feat: impl len prefixed string, bool + mat4 read helpers

* feat: serde derives for Color

* feat: bin read support

* fix: force PropertyValueEnum to be exhaustive

* fix: make PropertyValueEnum variants public

* chore: lint cleanup

* feat: store BinTreeObject class_hash

* fix: propagate invalid bin prop kind error

* fix: remove debug logs

* feat: u8/f32 color distinction (+ fix bin colors)

* feat: do bin object legacy read fallback hack

* chore: add cargo-hack to nix flake

* feat: impl BinProperty::size()

* feat: size check for struct bin value

* feat: size check for container value

* fix: rename InvalidPropertyType err -> InvalidPropertyTypePrimitive

* feat: check for nested containers and non-prim map keys

* chore: remove todo

* feat: basic bin read test

* feat: proper size checks + add io::Seek constraint everywhere

* chore (nix): add cargo-expand + remove duplicated input?

* feat: add write helpers + make vec helpers more generic

* feat: impl AsRef for Color<T>

* feat: add WriteProperty trait

* feat: impl to_writer for BinProperty

* feat: impl WriteProperty for primitives

* feat: impl WriteProperty for OptionalValue

* feat: impl to_writer and kind for BinPropertyEnum

* fix: update bin read test

* fix: make all read/write trait bounds the same (+ Seek + ?Sized)

* feat: impl WriteProperty for ContainerValue

* feat: impl WriteProperty for Struct/EmbeddedValue

fix: unimplemented comment in struct writer

* chore: better doc comment for measure()

* feat: impl WriteProperty for MapValue

* chore: warning cleanup

* feat: impl WriteProperty for StringValue

* feat: impl WriteProperty for UnorderedContainerValue

* fix: explicitly don't allow legacy optional write (for now)

* fix: rebase woes

* fix: update bin read test snapshot

* feat: impl to_writer for BinTree/BinTreeObject

* feat: `window` seek helper function

* fix: size seeking issue

* chore (nix): add gdb to flake

* feat: add bin roundtrip test + reorg tests

* fix: remove real data tests

crates/league-toolkit/Cargo.toml

@@ -9,7 +9,7 @@ default = ["zstd"]
 zstd = ["dep:zstd"]
 ruzstd = ["dep:ruzstd"]
 
-serde = ["dep:serde"]
+serde = ["dep:serde", "glam/serde"]
 rust_backends = [
   "flate2/rust_backend",
   "ruzstd",
@@ -20,7 +20,6 @@ bitflags = "2.5.0"
 byteorder = "1.5.0"
 flate2 = "1.0.30"
 glam = { version = "0.27.0", features = ["glam-assert"] }
-insta = "1.39.0"
 lazy_static = "1.4.0"
 log = "0.4.21"
 memchr = "2.7.2"
@@ -31,6 +30,13 @@ zstd = { version = "0.13", default-features = false, optional = true }
 ruzstd = { version = "0.7", optional = true }
 
 serde = { version = "1.0.204", features = ["derive"], optional = true }
+paste = "1.0.15"
+miette = "7.2.0"
+enum_dispatch = "0.3.13"
 
 [dev-dependencies]
+league-toolkit = {path = ".", features = ["serde"]}
 approx = "0.5.1"
+insta = { version = "1.39.0", features = ["ron"] }
+serde = { version = "*", features = ["derive"] }
+glam = { version = "*", features = ["glam-assert", "serde"] }

crates/league-toolkit/src/core/mesh/error.rs

@@ -7,7 +7,9 @@ pub enum ParseError {
     #[error("Invalid '{0}' - got '{1}'")]
     InvalidField(&'static str, String),
     #[error("IO Error - {0}")]
-    ReaderError(#[from] std::io::Error),
+    IOError(#[from] std::io::Error),
     #[error("UTF-8 Error - {0}")]
     Utf8Error(#[from] std::str::Utf8Error),
+    #[error(transparent)]
+    ReaderError(#[from] crate::util::ReaderError),
 }

crates/league-toolkit/src/core/mesh/skinned/mod.rs

@@ -1,6 +1,3 @@
-use std::io::Read;
-
-use byteorder::ReadBytesExt;
 use glam::Vec3;
 use num_enum::{IntoPrimitive, TryFromPrimitive};
 

crates/league-toolkit/src/core/mesh/static/face.rs

@@ -37,7 +37,7 @@ impl StaticMeshFace {
             material,
             vertex_ids,
             uvs: (vec2(uvs.0, uvs.3), vec2(uvs.1, uvs.4), vec2(uvs.2, uvs.5)),
-            colors: (Color::ONE, Color::ONE, Color::ONE),
+            colors: (Color::<f32>::ONE, Color::<f32>::ONE, Color::<f32>::ONE),
         })
     }
 }

crates/league-toolkit/src/core/mesh/static/mod.rs

@@ -1,6 +1,3 @@
-use std::io::Read;
-
-use byteorder::ReadBytesExt;
 use glam::Vec3;
 
 pub use face::*;

crates/league-toolkit/src/core/mesh/static/read.rs

@@ -46,7 +46,7 @@ impl StaticMesh {
             true => {
                 let mut v = Vec::with_capacity(vertex_count as usize);
                 for _ in 0..vertex_count {
-                    v.push(reader.read_color::<LE>()?);
+                    v.push(reader.read_color_f32::<LE>()?);
                 }
                 Some(v)
             }

crates/league-toolkit/src/core/meta/bin_tree/mod.rs

@@ -0,0 +1,41 @@
+use std::collections::HashMap;
+
+mod object;
+use super::error::ParseError;
+pub use object::*;
+
+pub mod read;
+pub mod write;
+
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[derive(Debug, PartialEq)]
+pub struct BinTree {
+    pub is_override: bool,
+    pub version: u32,
+
+    pub objects: HashMap<u32, BinTreeObject>,
+    /// List of other property bins we depend on.
+    ///
+    /// Property bins can depend on other property bins in a similar fashion to importing code libraries
+    pub dependencies: Vec<String>,
+
+    data_overrides: Vec<()>,
+}
+
+impl BinTree {
+    pub fn new(
+        objects: impl IntoIterator<Item = BinTreeObject>,
+        dependencies: impl IntoIterator<Item = String>,
+    ) -> Self {
+        Self {
+            version: 3,
+            is_override: false,
+            objects: objects
+                .into_iter()
+                .map(|o: BinTreeObject| (o.path_hash, o))
+                .collect(),
+            dependencies: dependencies.into_iter().collect(),
+            data_overrides: Vec::new(),
+        }
+    }
+}

crates/league-toolkit/src/core/meta/bin_tree/object.rs

@@ -0,0 +1,69 @@
+use std::{collections::HashMap, io};
+
+use crate::util::{measure, window};
+
+use super::{super::BinProperty, ParseError};
+use byteorder::{ReadBytesExt as _, WriteBytesExt as _, LE};
+
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[derive(Debug, PartialEq)]
+pub struct BinTreeObject {
+    pub path_hash: u32,
+    pub class_hash: u32,
+    pub properties: HashMap<u32, BinProperty>,
+}
+
+impl BinTreeObject {
+    pub fn from_reader<R: io::Read + io::Seek + ?Sized>(
+        reader: &mut R,
+        class_hash: u32,
+        legacy: bool,
+    ) -> Result<Self, ParseError> {
+        let size = reader.read_u32::<LE>()?;
+        let (real_size, value) = measure(reader, |reader| {
+            let path_hash = reader.read_u32::<LE>()?;
+
+            let prop_count = reader.read_u16::<LE>()? as usize;
+            let mut properties = HashMap::with_capacity(prop_count);
+            for _ in 0..prop_count {
+                let prop = BinProperty::from_reader(reader, legacy)?;
+                properties.insert(prop.name_hash, prop);
+            }
+
+            Ok::<_, ParseError>(Self {
+                path_hash,
+                class_hash,
+                properties,
+            })
+        })?;
+
+        if size as u64 != real_size {
+            return Err(ParseError::InvalidSize(size as _, real_size));
+        }
+        Ok(value)
+    }
+
+    pub fn to_writer<W: io::Write + io::Seek + ?Sized>(
+        &self,
+        writer: &mut W,
+        legacy: bool,
+    ) -> io::Result<()> {
+        if legacy {
+            unimplemented!("legacy BinTreeObject write");
+        }
+        let size_pos = writer.stream_position()?;
+        writer.write_u32::<LE>(0)?;
+
+        let (size, _) = measure(writer, |writer| {
+            writer.write_u32::<LE>(self.path_hash)?;
+            writer.write_u16::<LE>(self.properties.len() as _)?;
+            for prop in self.properties.values() {
+                prop.to_writer(writer)?;
+            }
+            Ok::<_, io::Error>(())
+        })?;
+
+        window(writer, size_pos, |writer| writer.write_u32::<LE>(size as _))?;
+        Ok(())
+    }
+}

crates/league-toolkit/src/core/meta/bin_tree/read.rs

@@ -0,0 +1,113 @@
+use std::{collections::HashMap, io};
+
+use crate::core::meta::ParseError;
+
+use super::{BinTree, BinTreeObject};
+use crate::util::ReaderExt as _;
+use byteorder::{ReadBytesExt as _, LE};
+
+impl BinTree {
+    pub const PROP: u32 = u32::from_le_bytes(*b"PROP");
+    pub const PTCH: u32 = u32::from_le_bytes(*b"PTCH");
+    pub fn from_reader<R: io::Read + std::io::Seek + ?Sized>(
+        reader: &mut R,
+    ) -> Result<Self, ParseError> {
+        let magic = reader.read_u32::<LE>()?;
+        let is_override = match magic {
+            Self::PROP => false,
+            Self::PTCH => {
+                let override_version = reader.read_u32::<LE>()?;
+                if override_version != 1 {
+                    return Err(ParseError::InvalidFileVersion(override_version));
+                }
+
+                // It might be possible to create an override property bin
+                // and set the original file as a dependency.
+                // This seems to be the object count of the override section
+                let _maybe_override_object_count = reader.read_u32::<LE>()?;
+
+                let magic = reader.read_u32::<LE>()?;
+                if magic != Self::PROP {
+                    // TODO (alan): repr this in the error
+                    log::error!(
+                        "Expected PROP ({:#x}) section after PTCH ({:#x}), got '{:#x}'",
+                        Self::PROP,
+                        Self::PTCH,
+                        magic
+                    );
+                    return Err(ParseError::InvalidFileSignature);
+                }
+                true
+            }
+            _ => return Err(ParseError::InvalidFileSignature),
+        };
+
+        let version = reader.read_u32::<LE>()?;
+        if !matches!(version, 1..=3) {
+            // TODO (alan): distinguish override/non-override version
+            return Err(ParseError::InvalidFileVersion(version));
+        }
+
+        let dependencies = match version {
+            2.. => {
+                let dep_count = reader.read_u32::<LE>()?;
+                let mut dependencies = Vec::with_capacity(dep_count as _);
+                for _ in 0..dep_count {
+                    dependencies.push(reader.read_len_prefixed_string::<LE>()?);
+                }
+                dependencies
+            }
+            _ => Vec::new(),
+        };
+
+        let obj_count = reader.read_u32::<LE>()? as usize;
+        let mut obj_classes = Vec::with_capacity(obj_count);
+        for _ in 0..obj_count {
+            obj_classes.push(reader.read_u32::<LE>()?);
+        }
+
+        let mut objects = HashMap::with_capacity(obj_count);
+        match Self::try_read_objects(reader, &obj_classes, &mut objects, false) {
+            Ok(_) => {}
+            Err(ParseError::InvalidPropertyTypePrimitive(kind)) => {
+                log::warn!("Invalid prop type {kind}. Trying reading objects as legacy.");
+                Self::try_read_objects(reader, &obj_classes, &mut objects, true)?;
+            }
+            e => e?,
+        }
+
+        let data_overrides = match (is_override, version) {
+            (true, 3..) => {
+                let count = reader.read_u32::<LE>()?;
+                let mut v = Vec::with_capacity(count as _);
+                for _ in 0..count {
+                    v.push(()); // TODO: impl data overrides
+                }
+                v
+            }
+            _ => Vec::new(),
+        };
+
+        Ok(Self {
+            version,
+            is_override,
+            objects,
+            dependencies,
+            data_overrides,
+        })
+    }
+
+    fn try_read_objects<R: io::Read + std::io::Seek + ?Sized>(
+        reader: &mut R,
+        obj_classes: &[u32],
+        objects: &mut HashMap<u32, BinTreeObject>,
+        legacy: bool,
+    ) -> Result<(), ParseError> {
+        objects.clear();
+        for &class_hash in obj_classes {
+            let tree_obj = BinTreeObject::from_reader(reader, class_hash, legacy)?;
+            objects.insert(tree_obj.path_hash, tree_obj);
+        }
+        Ok(())
+    }
+}

crates/league-toolkit/src/core/meta/bin_tree/write.rs

@@ -0,0 +1,57 @@
+use std::io;
+
+use super::BinTree;
+use crate::util::WriterExt as _;
+use byteorder::{WriteBytesExt as _, LE};
+
+impl BinTree {
+    pub fn to_writer<W: io::Write + io::Seek + ?Sized>(
+        &self,
+        writer: &mut W,
+        legacy: bool,
+    ) -> io::Result<()> {
+        match self.is_override {
+            true => todo!("implement is_override BinTree write"),
+            false => {
+                writer.write_u32::<LE>(Self::PROP)?;
+            }
+        }
+
+        writer.write_u32::<LE>(self.version)?;
+
+        if !self.dependencies.is_empty() && self.version < 2 {
+            // FIXME: move this assertion to object creation
+            panic!(
+                "cannot write BinTree with dependencies @ version {}",
+                self.version
+            );
+        }
+
+        if self.version >= 2 {
+            writer.write_u32::<LE>(self.dependencies.len() as _)?;
+            for dep in &self.dependencies {
+                writer.write_len_prefixed_string::<LE, _>(dep)?;
+            }
+        }
+
+        writer.write_u32::<LE>(self.objects.len() as _)?;
+        for obj in self.objects.values() {
+            writer.write_u32::<LE>(obj.class_hash)?;
+        }
+        for obj in self.objects.values() {
+            obj.to_writer(writer, legacy)?;
+        }
+
+        if self.is_override {
+            if self.version < 3 {
+                panic!("cannot write data overrides @ version {}", self.version);
+            }
+            writer.write_u32::<LE>(self.data_overrides.len() as _)?;
+            // TODO: impl data overrides
+            //for o in &self.data_overrides {
+            //}
+        }
+
+        Ok(())
+    }
+}

crates/league-toolkit/src/core/meta/error.rs

@@ -0,0 +1,29 @@
+use miette::Diagnostic;
+
+use super::property::BinPropertyKind;
+
+#[derive(Debug, thiserror::Error, Diagnostic)]
+pub enum ParseError {
+    #[error("Invalid file signature")]
+    InvalidFileSignature,
+    #[error("Invalid file version '{0}'")]
+    InvalidFileVersion(u32),
+    #[error("Invalid '{0}' - got '{1}'")]
+    InvalidField(&'static str, String),
+    #[error("Invalid property kind - {0}")]
+    InvalidPropertyTypePrimitive(#[from] num_enum::TryFromPrimitiveError<BinPropertyKind>),
+    #[error("Invalid size - expected {0}, got {1} bytes")]
+    InvalidSize(u64, u64),
+
+    #[error("Container type {0:?} cannot be nested!")]
+    InvalidNesting(BinPropertyKind),
+    #[error("Invalid map key type {0:?}, only primitive types can be used as keys.")]
+    InvalidKeyType(BinPropertyKind),
+
+    #[error(transparent)]
+    ReaderError(#[from] crate::util::ReaderError),
+    #[error("IO Error - {0}")]
+    IOError(#[from] std::io::Error),
+    #[error("UTF-8 Error - {0}")]
+    Utf8Error(#[from] std::str::Utf8Error),
+}

crates/league-toolkit/src/core/meta/mod.rs

@@ -0,0 +1,10 @@
+pub mod property;
+pub use property::BinProperty;
+
+mod bin_tree;
+pub use bin_tree::*;
+
+pub mod error;
+pub use error::*;
+
+pub mod traits;