String encodings continue to be one of the hardest things

CMakeLists.txt

@@ -8,7 +8,7 @@ if(PROJECT_SOURCE_DIR STREQUAL PROJECT_BINARY_DIR)
 endif()
 
 set(NAME "SoulsyHUD")
-set(VERSION 0.16.1.0)
+set(VERSION 0.16.2.0)
 
 project(
     ${NAME}

Cargo.toml

@@ -7,13 +7,15 @@ license      = "GPL-3.0"
 name         = "soulsy"
 readme       = "README.md"
 rust-version = "1.71.1"
-version      = "0.16.1"
+version      = "0.16.2"
 
 [lib]
 crate-type = ["staticlib"]
 
 [dependencies]
 bincode   = "2.0.0-rc.3"
+byte-slice-cast = "1.2.2"
+chardet = "0.2.4"
 cxx       = { version = "1.0.111", features = ["c++20"] }
 enumset   = "1.1.3"
 eyre      = "0.6.9"
@@ -28,6 +30,7 @@ strfmt    = "0.2.4"
 strum     = { version = "0.25.0", features = ["derive"] }
 textcode  = "0.2.2"
 toml      = "0.8.6"
+ucs2 = "0.3.2"
 
 [build-dependencies]
 cxx-build = "1.0.111"

justfile

@@ -97,20 +97,8 @@ translations:
 # check that all $ strings in config have matching translation strings
 [unix]
 check-translations:
-    #!/usr/bin/env bash
-    converted=$(iconv -f utf-16 -t utf-8 data/Interface/Translations/SoulsyHUD_english.txt > tmp.txt)
-
-    # I am too lazy to figure out how to get jq to do all of it.
-    keys=$(cat data/mcm/config/SoulsyHUD/config.json | jq '.pages[] | .content[] | .[]' -r | grep "\\$" | tr -d '," $' | sort | uniq)
-    for k in $keys; do
-        cmd="grep $k tmp.txt"
-        suppressed=$(sh -c "$cmd")
-        exit=$?
-        if [ $exit != '0' ]; then
-            echo "missing translation: $k"
-        fi
-    done
-    rm tmp.txt
+	mcm-meta-helper --moddir installer/core check all
+
 
 # Create a mod archive and 7zip it. Requires bash.
 [unix]

src/controller/facade.rs

@@ -59,6 +59,7 @@ pub fn toggle_item(key: u32, #[allow(clippy::boxed_local)] menu_item: Box<HudIte
     control::get().handle_toggle_item(action, *menu_item)
 }
 
+/// Pass along menu events to the controller.
 pub fn handle_menu_event(key: u32, button: &ButtonEvent) -> bool {
     control::get().handle_menu_event(key, button)
 }
@@ -68,6 +69,7 @@ pub fn entry_to_show_in_slot(element: HudElement) -> Box<HudItem> {
     control::get().entry_to_show_in_slot(element)
 }
 
+/// Refresh our view of what's needs to be in the HUD right now.
 pub fn refresh_hud_items() {
     control::get().refresh_hud_items();
 }
@@ -108,6 +110,7 @@ pub fn handle_item_equipped(
     control::get().handle_item_equipped(equipped, form_spec, right, left)
 }
 
+/// Pass along a CGO grip-change event to the controller.
 pub fn handle_grip_change(use_alt_grip: bool) {
     control::get().handle_grip_change(use_alt_grip);
 }
@@ -117,6 +120,7 @@ pub fn handle_inventory_changed(form_spec: &String, count: u32) {
     control::get().handle_inventory_changed(form_spec, count);
 }
 
+/// Handle an item being favorited.
 pub fn handle_favorite_event(
     button: &ButtonEvent,
     is_favorite: bool,
@@ -125,6 +129,7 @@ pub fn handle_favorite_event(
     control::get().handle_favorite_event(button, is_favorite, *item);
 }
 
+/// Ask the control to refresh settings.
 pub fn refresh_user_settings() {
     if let Some(e) = UserSettings::refresh().err() {
         log::warn!("Failed to read user settings! using defaults; {e:#}");
@@ -133,6 +138,7 @@ pub fn refresh_user_settings() {
     control::get().apply_settings();
 }
 
+/// Clear all cycles. MCM -> this function -> controller.
 pub fn clear_cycles() {
     control::get().clear_cycles();
 }
@@ -251,45 +257,3 @@ pub fn set_equipset_icon(id: u32, itemname: String) -> bool {
 pub fn look_up_equipset_by_name(name: String) -> u32 {
     control::get().cycles.equipset_by_name(name)
 }
-
-pub fn show_ui() -> bool {
-    control::get().cycles.hud_visible()
-}
-
-// ----------- windows character shenanigans
-
-use textcode::iso8859_15;
-
-/// C++ calls this version.
-pub fn string_to_utf8(bytes_ffi: &CxxVector<u8>) -> String {
-    let bytes: Vec<u8> = bytes_ffi.iter().copied().collect();
-    convert_to_utf8_doggedly(bytes)
-}
-
-// To test in game: install daegon
-// player.additem 4c2b15f4 1
-// Sacrÿfev Tëliimi
-
-/// Get a valid Rust representation of this Windows codepage string data by hook or by crook.
-pub fn convert_to_utf8_doggedly(input: Vec<u8>) -> String {
-    let bytes = if input.ends_with(&[0]) {
-        let chopped = input.len() - 1;
-        let mut tmp = input.clone();
-        tmp.truncate(chopped);
-        tmp
-    } else {
-        input.clone()
-    };
-    if bytes.is_empty() {
-        return String::new();
-    }
-
-    // Maybe it's the easy case and we're done!
-    if let Ok(utf8string) = String::from_utf8(bytes.clone()) {
-        return utf8string;
-    }
-
-    let mut dst = String::new();
-    iso8859_15::decode(bytes.as_slice(), &mut dst);
-    dst
-}

src/controller/mod.rs

@@ -14,7 +14,9 @@ pub mod facade;
 pub mod keys;
 pub mod logs;
 pub mod settings;
+pub mod strings;
 
 pub use facade::*;
 pub use logs::*;
 pub use settings::UserSettings;
+pub use strings::*;

src/controller/strings.rs

@@ -0,0 +1,108 @@
+//! Character encoding shenanigans. Bethesda is very bad at utf-8, I am told.
+
+use byte_slice_cast::AsSliceOf;
+use cxx::CxxVector;
+use textcode::{iso8859_15, iso8859_9};
+
+// To test in game: install daegon
+// player.additem 4c2b15f4 1
+// Sacrÿfev Tëliimi
+
+/// C++ should use this for std::string conversions.
+pub fn string_to_utf8(bytes_ffi: &CxxVector<u8>) -> String {
+    let bytes: Vec<u8> = bytes_ffi.iter().copied().collect();
+    convert_to_utf8(bytes)
+}
+
+/// Use this for null-terminated C strings.
+pub fn cstr_to_utf8(bytes_ffi: &CxxVector<u8>) -> String {
+    let bytes: Vec<u8> = bytes_ffi.iter().copied().collect();
+    let bytes = if bytes.ends_with(&[0]) {
+        let chopped = bytes.len() - 1;
+        let mut tmp = bytes.clone();
+        tmp.truncate(chopped);
+        tmp
+    } else {
+        bytes
+    };
+    convert_to_utf8(bytes)
+}
+
+/// Get a valid Rust representation of this Windows codepage string data by hook or by crook.
+pub fn convert_to_utf8(bytes: Vec<u8>) -> String {
+    if bytes.is_empty() {
+        return String::new();
+    }
+
+    let (encoding, _confidence, _language) = chardet::detect(&bytes);
+    match encoding.as_str() {
+        "utf-8" => String::from_utf8(bytes.clone())
+            .unwrap_or_else(|_| String::from_utf8_lossy(&bytes).to_string()),
+        "ISO-8859-9" => {
+            let mut dst = String::new();
+            iso8859_9::decode(bytes.as_slice(), &mut dst);
+            dst
+        }
+        "ISO-8859-15" => {
+            let mut dst = String::new();
+            iso8859_15::decode(bytes.as_slice(), &mut dst);
+            dst
+        }
+        _ => {
+            let Ok(widebytes) = bytes.as_slice_of::<u16>() else {
+                return String::from_utf8_lossy(bytes.as_slice()).to_string();
+            };
+            let mut utf8bytes: Vec<u8> = vec![0; widebytes.len()];
+            let Ok(_c) = ucs2::decode(widebytes, &mut utf8bytes) else {
+                return String::from_utf8_lossy(bytes.as_slice()).to_string();
+            };
+            String::from_utf8(utf8bytes.clone())
+                .unwrap_or_else(|_| String::from_utf8_lossy(utf8bytes.as_slice()).to_string())
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn utf8_data_is_untouched() {
+        let example = "Sacrÿfev Tëliimi";
+        let converted = convert_to_utf8(example.as_bytes().to_vec());
+        assert_eq!(converted, example);
+        let ex2 = "おはよう";
+        let convert2 = convert_to_utf8(ex2.as_bytes().to_vec());
+        assert_eq!(convert2, ex2);
+        let ex3 = "Zażółć gęślą jaźń";
+        let convert3 = convert_to_utf8(ex3.as_bytes().to_vec());
+        assert_eq!(convert3, ex3);
+    }
+
+    #[test]
+    fn iso8859_is_decoded() {
+        // This is the example above (from the Daegon mod), in its expression
+        // as windows codepage bytes. This test is the equivalent of me testing
+        // that the textcode mod works, but I am feeling timid.
+        let bytes: Vec<u8> = vec![
+            0x53, 0x61, 0x63, 0x72, 0xff, 0x66, 0x65, 0x76, 0x20, 0x54, 0xeb, 0x6c, 0x69, 0x69,
+            0x6d, 0x69,
+        ];
+        assert!(String::from_utf8(bytes.clone()).is_err());
+        let utf8_version = "Sacrÿfev Tëliimi".to_string();
+        let converted = convert_to_utf8(bytes.clone());
+        assert_eq!(converted, utf8_version);
+    }
+
+    #[test]
+    fn utf16le_is_decoded() {
+        let bytes = vec![
+            36, 0, 83, 0, 111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 72, 0, 85, 0, 68, 0, 9, 0, 83, 0,
+            111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 32, 0, 72, 0, 85, 0, 68, 0,
+        ];
+        assert_eq!(bytes.len(), 42);
+        let converted = convert_to_utf8(bytes.clone());
+        assert_eq!(converted.len(), bytes.len() / 2);
+        assert_eq!(converted, "$SoulsyHUD	Soulsy HUD");
+    }
+}

src/data/huditem.rs

@@ -222,12 +222,13 @@ impl HudItem {
     }
 
     pub fn fmtstr(&self, fmt: String) -> String {
-        // This implementation caches nothing. It might be fast enough?
-        // needs measurement
         match strfmt(&fmt, &self.format_vars) {
             Ok(v) => v,
             Err(e) => {
-                log::debug!("Failed to render format string for HUD item; error: {e:#}");
+                log::debug!(
+                    "Failed to render format string for HUD item; formspec='{}'; error: {e:#}",
+                    self.form_string
+                );
                 "".to_string()
             }
         }