From 6fc18bfc39d435602417949cd97d2aea1642b91f Mon Sep 17 00:00:00 2001 From: C J Silverio Date: Sun, 7 Jan 2024 20:41:27 -0800 Subject: [PATCH] String encodings continue to be one of the hardest things (#103) When we get a string for an item name from the game, it might be in one of several encodings: - ascii - Latin-1 1252, aka ISO-8859 most likely the 9 variation - UCS2, which is almost like UTF-16 LE except it's fixed width We must take this varied input and turn it into valid UTF-8 strings for Rust usage & logging, and for display in the HUD via Imgui. We must also not disturb any already-valid strings by mangling them while attempting to decode them into a modern encoding. So, we do the following. First, use `chardet::detect()` to guess at an encoding. If it's utf-8 already or one of the ISO-8859s, we can decode immediately and return. If it is not one of those, we make the _assumption_ that it is UCS2. The string might be almost-correctly detected as UTF-16le if it uses the second byte for any characters, but if the characters it is holding are plain old ascii, it'll be reported as ASCII and that is wrong. So we guess, and then decode the UCS2. Broke the string manglers out into their own file and wrote some basic tests. Changed the helper wrappers to use cstr null-terminated version for item name handling, since those come from the game with null termination. If I discover the null termination is also double-width I guess I'll have a bug to fix. Put mcm-meta-helper to work on the task for which it was invented. Added better logging for when fmtstr fails on huditems. --- CMakeLists.txt | 2 +- Cargo.lock | 36 +++++++++++-- Cargo.toml | 5 +- justfile | 16 +----- src/controller/facade.rs | 48 +++-------------- src/controller/mod.rs | 2 + src/controller/strings.rs | 108 ++++++++++++++++++++++++++++++++++++++ src/data/huditem.rs | 7 +-- src/lib.rs | 10 ++-- src/util/helpers.cpp | 4 +- 10 files changed, 168 insertions(+), 70 deletions(-) create mode 100644 src/controller/strings.rs diff --git a/CMakeLists.txt b/CMakeLists.txt index 2553424b..f9ca1380 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ if(PROJECT_SOURCE_DIR STREQUAL PROJECT_BINARY_DIR) endif() set(NAME "SoulsyHUD") -set(VERSION 0.16.1.0) +set(VERSION 0.16.2.0) project( ${NAME} diff --git a/Cargo.lock b/Cargo.lock index 05999045..4715f432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,6 +62,12 @@ dependencies = [ "virtue", ] +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -74,6 +80,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + [[package]] name = "bytemuck" version = "1.14.0" @@ -95,6 +107,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chardet" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a48563284b67c003ba0fb7243c87fab68885e1532c605704228a80238512e31" + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -471,9 +489,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" @@ -771,9 +789,11 @@ checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "soulsy" -version = "0.16.1" +version = "0.16.2" dependencies = [ "bincode", + "byte-slice-cast", + "chardet", "cxx", "cxx-build", "enumset", @@ -791,6 +811,7 @@ dependencies = [ "strum", "textcode", "toml", + "ucs2", ] [[package]] @@ -986,6 +1007,15 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" +[[package]] +name = "ucs2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad643914094137d475641b6bab89462505316ec2ce70907ad20102d28a79ab8" +dependencies = [ + "bit_field", +] + [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/Cargo.toml b/Cargo.toml index e64cbb9c..51398bd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,13 +7,15 @@ license = "GPL-3.0" name = "soulsy" readme = "README.md" rust-version = "1.71.1" -version = "0.16.1" +version = "0.16.2" [lib] crate-type = ["staticlib"] [dependencies] bincode = "2.0.0-rc.3" +byte-slice-cast = "1.2.2" +chardet = "0.2.4" cxx = { version = "1.0.111", features = ["c++20"] } enumset = "1.1.3" eyre = "0.6.9" @@ -28,6 +30,7 @@ strfmt = "0.2.4" strum = { version = "0.25.0", features = ["derive"] } textcode = "0.2.2" toml = "0.8.6" +ucs2 = "0.3.2" [build-dependencies] cxx-build = "1.0.111" diff --git a/justfile b/justfile index 4c5d6226..73bb8762 100644 --- a/justfile +++ b/justfile @@ -97,20 +97,8 @@ translations: # check that all $ strings in config have matching translation strings [unix] check-translations: - #!/usr/bin/env bash - converted=$(iconv -f utf-16 -t utf-8 data/Interface/Translations/SoulsyHUD_english.txt > tmp.txt) - - # I am too lazy to figure out how to get jq to do all of it. - keys=$(cat data/mcm/config/SoulsyHUD/config.json | jq '.pages[] | .content[] | .[]' -r | grep "\\$" | tr -d '," $' | sort | uniq) - for k in $keys; do - cmd="grep $k tmp.txt" - suppressed=$(sh -c "$cmd") - exit=$? - if [ $exit != '0' ]; then - echo "missing translation: $k" - fi - done - rm tmp.txt + mcm-meta-helper --moddir installer/core check all + # Create a mod archive and 7zip it. Requires bash. [unix] diff --git a/src/controller/facade.rs b/src/controller/facade.rs index a0a39d14..264f0744 100644 --- a/src/controller/facade.rs +++ b/src/controller/facade.rs @@ -59,6 +59,7 @@ pub fn toggle_item(key: u32, #[allow(clippy::boxed_local)] menu_item: Box bool { control::get().handle_menu_event(key, button) } @@ -68,6 +69,7 @@ pub fn entry_to_show_in_slot(element: HudElement) -> Box { control::get().entry_to_show_in_slot(element) } +/// Refresh our view of what's needs to be in the HUD right now. pub fn refresh_hud_items() { control::get().refresh_hud_items(); } @@ -108,6 +110,7 @@ pub fn handle_item_equipped( control::get().handle_item_equipped(equipped, form_spec, right, left) } +/// Pass along a CGO grip-change event to the controller. pub fn handle_grip_change(use_alt_grip: bool) { control::get().handle_grip_change(use_alt_grip); } @@ -117,6 +120,7 @@ pub fn handle_inventory_changed(form_spec: &String, count: u32) { control::get().handle_inventory_changed(form_spec, count); } +/// Handle an item being favorited. pub fn handle_favorite_event( button: &ButtonEvent, is_favorite: bool, @@ -125,6 +129,7 @@ pub fn handle_favorite_event( control::get().handle_favorite_event(button, is_favorite, *item); } +/// Ask the control to refresh settings. pub fn refresh_user_settings() { if let Some(e) = UserSettings::refresh().err() { log::warn!("Failed to read user settings! using defaults; {e:#}"); @@ -133,6 +138,7 @@ pub fn refresh_user_settings() { control::get().apply_settings(); } +/// Clear all cycles. MCM -> this function -> controller. pub fn clear_cycles() { control::get().clear_cycles(); } @@ -251,45 +257,3 @@ pub fn set_equipset_icon(id: u32, itemname: String) -> bool { pub fn look_up_equipset_by_name(name: String) -> u32 { control::get().cycles.equipset_by_name(name) } - -pub fn show_ui() -> bool { - control::get().cycles.hud_visible() -} - -// ----------- windows character shenanigans - -use textcode::iso8859_15; - -/// C++ calls this version. -pub fn string_to_utf8(bytes_ffi: &CxxVector) -> String { - let bytes: Vec = bytes_ffi.iter().copied().collect(); - convert_to_utf8_doggedly(bytes) -} - -// To test in game: install daegon -// player.additem 4c2b15f4 1 -// Sacrÿfev Tëliimi - -/// Get a valid Rust representation of this Windows codepage string data by hook or by crook. -pub fn convert_to_utf8_doggedly(input: Vec) -> String { - let bytes = if input.ends_with(&[0]) { - let chopped = input.len() - 1; - let mut tmp = input.clone(); - tmp.truncate(chopped); - tmp - } else { - input.clone() - }; - if bytes.is_empty() { - return String::new(); - } - - // Maybe it's the easy case and we're done! - if let Ok(utf8string) = String::from_utf8(bytes.clone()) { - return utf8string; - } - - let mut dst = String::new(); - iso8859_15::decode(bytes.as_slice(), &mut dst); - dst -} diff --git a/src/controller/mod.rs b/src/controller/mod.rs index 26c17870..ffa6b964 100644 --- a/src/controller/mod.rs +++ b/src/controller/mod.rs @@ -14,7 +14,9 @@ pub mod facade; pub mod keys; pub mod logs; pub mod settings; +pub mod strings; pub use facade::*; pub use logs::*; pub use settings::UserSettings; +pub use strings::*; diff --git a/src/controller/strings.rs b/src/controller/strings.rs new file mode 100644 index 00000000..c5eb28b0 --- /dev/null +++ b/src/controller/strings.rs @@ -0,0 +1,108 @@ +//! Character encoding shenanigans. Bethesda is very bad at utf-8, I am told. + +use byte_slice_cast::AsSliceOf; +use cxx::CxxVector; +use textcode::{iso8859_15, iso8859_9}; + +// To test in game: install daegon +// player.additem 4c2b15f4 1 +// Sacrÿfev Tëliimi + +/// C++ should use this for std::string conversions. +pub fn string_to_utf8(bytes_ffi: &CxxVector) -> String { + let bytes: Vec = bytes_ffi.iter().copied().collect(); + convert_to_utf8(bytes) +} + +/// Use this for null-terminated C strings. +pub fn cstr_to_utf8(bytes_ffi: &CxxVector) -> String { + let bytes: Vec = bytes_ffi.iter().copied().collect(); + let bytes = if bytes.ends_with(&[0]) { + let chopped = bytes.len() - 1; + let mut tmp = bytes.clone(); + tmp.truncate(chopped); + tmp + } else { + bytes + }; + convert_to_utf8(bytes) +} + +/// Get a valid Rust representation of this Windows codepage string data by hook or by crook. +pub fn convert_to_utf8(bytes: Vec) -> String { + if bytes.is_empty() { + return String::new(); + } + + let (encoding, _confidence, _language) = chardet::detect(&bytes); + match encoding.as_str() { + "utf-8" => String::from_utf8(bytes.clone()) + .unwrap_or_else(|_| String::from_utf8_lossy(&bytes).to_string()), + "ISO-8859-9" => { + let mut dst = String::new(); + iso8859_9::decode(bytes.as_slice(), &mut dst); + dst + } + "ISO-8859-15" => { + let mut dst = String::new(); + iso8859_15::decode(bytes.as_slice(), &mut dst); + dst + } + _ => { + let Ok(widebytes) = bytes.as_slice_of::() else { + return String::from_utf8_lossy(bytes.as_slice()).to_string(); + }; + let mut utf8bytes: Vec = vec![0; widebytes.len()]; + let Ok(_c) = ucs2::decode(widebytes, &mut utf8bytes) else { + return String::from_utf8_lossy(bytes.as_slice()).to_string(); + }; + String::from_utf8(utf8bytes.clone()) + .unwrap_or_else(|_| String::from_utf8_lossy(utf8bytes.as_slice()).to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn utf8_data_is_untouched() { + let example = "Sacrÿfev Tëliimi"; + let converted = convert_to_utf8(example.as_bytes().to_vec()); + assert_eq!(converted, example); + let ex2 = "おはよう"; + let convert2 = convert_to_utf8(ex2.as_bytes().to_vec()); + assert_eq!(convert2, ex2); + let ex3 = "Zażółć gęślą jaźń"; + let convert3 = convert_to_utf8(ex3.as_bytes().to_vec()); + assert_eq!(convert3, ex3); + } + + #[test] + fn iso8859_is_decoded() { + // This is the example above (from the Daegon mod), in its expression + // as windows codepage bytes. This test is the equivalent of me testing + // that the textcode mod works, but I am feeling timid. + let bytes: Vec = vec![ + 0x53, 0x61, 0x63, 0x72, 0xff, 0x66, 0x65, 0x76, 0x20, 0x54, 0xeb, 0x6c, 0x69, 0x69, + 0x6d, 0x69, + ]; + assert!(String::from_utf8(bytes.clone()).is_err()); + let utf8_version = "Sacrÿfev Tëliimi".to_string(); + let converted = convert_to_utf8(bytes.clone()); + assert_eq!(converted, utf8_version); + } + + #[test] + fn utf16le_is_decoded() { + let bytes = vec![ + 36, 0, 83, 0, 111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 72, 0, 85, 0, 68, 0, 9, 0, 83, 0, + 111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 32, 0, 72, 0, 85, 0, 68, 0, + ]; + assert_eq!(bytes.len(), 42); + let converted = convert_to_utf8(bytes.clone()); + assert_eq!(converted.len(), bytes.len() / 2); + assert_eq!(converted, "$SoulsyHUD Soulsy HUD"); + } +} diff --git a/src/data/huditem.rs b/src/data/huditem.rs index ff9a1286..c9661fb9 100644 --- a/src/data/huditem.rs +++ b/src/data/huditem.rs @@ -222,12 +222,13 @@ impl HudItem { } pub fn fmtstr(&self, fmt: String) -> String { - // This implementation caches nothing. It might be fast enough? - // needs measurement match strfmt(&fmt, &self.format_vars) { Ok(v) => v, Err(e) => { - log::debug!("Failed to render format string for HUD item; error: {e:#}"); + log::debug!( + "Failed to render format string for HUD item; formspec='{}'; error: {e:#}", + self.form_string + ); "".to_string() } } diff --git a/src/lib.rs b/src/lib.rs index bf972e92..0c8f2a0c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -282,14 +282,16 @@ pub mod plugin { /// Log at trace level. Use this level for debugging programming problems. fn log_trace(message: String); - /// Decode a vector of bytes from a windows codepage string to a utf-8 string. + // So... ucs2 is a fixed-width format, and it might well end with a 0 if the + // data is ascii, so we cannot guess if something is null-terminated or not. + /// Decode a vector of bytes from a std::string (no null termination) to a utf-8 string. fn string_to_utf8(bytes: &CxxVector) -> String; + /// Decode a null-terminated C string from whatever it is to utf-8. + fn cstr_to_utf8(bytes_ffi: &CxxVector) -> String; /// Trigger rust to read config, figure out what the player has equipped, /// and figure out what it should draw. fn initialize_hud(); - /// Check if the user wants the HUD visible right now or not. - fn show_ui() -> bool; /// Get cycle data for cosave. fn serialize_cycles() -> Vec; /// Serialization format version. @@ -420,7 +422,7 @@ pub mod plugin { has_time_left: bool, max_time: f32, time_left: f32, - ) -> Box; + ) -> Box; /// Call this to get the fallback-aware key for an icon. fn get_icon_key(name: String) -> String; diff --git a/src/util/helpers.cpp b/src/util/helpers.cpp index 4025e479..a59df552 100644 --- a/src/util/helpers.cpp +++ b/src/util/helpers.cpp @@ -38,7 +38,7 @@ namespace helpers // It is called by bound object finder functions. auto name = form->GetName(); // this use is required auto chonker = helpers::chars_to_vec(name); - auto safename = std::string(string_to_utf8(chonker)); + auto safename = std::string(cstr_to_utf8(chonker)); return safename; } @@ -47,7 +47,7 @@ namespace helpers // Do not call this from bound object finder functions. auto name = gear::displayName(form); auto chonker = helpers::chars_to_vec(name); - auto safename = std::string(string_to_utf8(chonker)); + auto safename = std::string(cstr_to_utf8(chonker)); return safename; }