diff --git a/CMakeLists.txt b/CMakeLists.txt index 2553424b..f9ca1380 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ if(PROJECT_SOURCE_DIR STREQUAL PROJECT_BINARY_DIR) endif() set(NAME "SoulsyHUD") -set(VERSION 0.16.1.0) +set(VERSION 0.16.2.0) project( ${NAME} diff --git a/Cargo.lock b/Cargo.lock index 05999045..4715f432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,6 +62,12 @@ dependencies = [ "virtue", ] +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -74,6 +80,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + [[package]] name = "bytemuck" version = "1.14.0" @@ -95,6 +107,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chardet" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a48563284b67c003ba0fb7243c87fab68885e1532c605704228a80238512e31" + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -471,9 +489,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" @@ -771,9 +789,11 @@ checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "soulsy" -version = "0.16.1" +version = "0.16.2" dependencies = [ "bincode", + "byte-slice-cast", + "chardet", "cxx", "cxx-build", "enumset", @@ -791,6 +811,7 @@ dependencies = [ "strum", "textcode", "toml", + "ucs2", ] [[package]] @@ -986,6 +1007,15 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" +[[package]] +name = "ucs2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad643914094137d475641b6bab89462505316ec2ce70907ad20102d28a79ab8" +dependencies = [ + "bit_field", +] + [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/Cargo.toml b/Cargo.toml index e64cbb9c..51398bd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,13 +7,15 @@ license = "GPL-3.0" name = "soulsy" readme = "README.md" rust-version = "1.71.1" -version = "0.16.1" +version = "0.16.2" [lib] crate-type = ["staticlib"] [dependencies] bincode = "2.0.0-rc.3" +byte-slice-cast = "1.2.2" +chardet = "0.2.4" cxx = { version = "1.0.111", features = ["c++20"] } enumset = "1.1.3" eyre = "0.6.9" @@ -28,6 +30,7 @@ strfmt = "0.2.4" strum = { version = "0.25.0", features = ["derive"] } textcode = "0.2.2" toml = "0.8.6" +ucs2 = "0.3.2" [build-dependencies] cxx-build = "1.0.111" diff --git a/justfile b/justfile index 4c5d6226..73bb8762 100644 --- a/justfile +++ b/justfile @@ -97,20 +97,8 @@ translations: # check that all $ strings in config have matching translation strings [unix] check-translations: - #!/usr/bin/env bash - converted=$(iconv -f utf-16 -t utf-8 data/Interface/Translations/SoulsyHUD_english.txt > tmp.txt) - - # I am too lazy to figure out how to get jq to do all of it. - keys=$(cat data/mcm/config/SoulsyHUD/config.json | jq '.pages[] | .content[] | .[]' -r | grep "\\$" | tr -d '," $' | sort | uniq) - for k in $keys; do - cmd="grep $k tmp.txt" - suppressed=$(sh -c "$cmd") - exit=$? - if [ $exit != '0' ]; then - echo "missing translation: $k" - fi - done - rm tmp.txt + mcm-meta-helper --moddir installer/core check all + # Create a mod archive and 7zip it. Requires bash. [unix] diff --git a/src/controller/facade.rs b/src/controller/facade.rs index a0a39d14..264f0744 100644 --- a/src/controller/facade.rs +++ b/src/controller/facade.rs @@ -59,6 +59,7 @@ pub fn toggle_item(key: u32, #[allow(clippy::boxed_local)] menu_item: Box bool { control::get().handle_menu_event(key, button) } @@ -68,6 +69,7 @@ pub fn entry_to_show_in_slot(element: HudElement) -> Box { control::get().entry_to_show_in_slot(element) } +/// Refresh our view of what's needs to be in the HUD right now. pub fn refresh_hud_items() { control::get().refresh_hud_items(); } @@ -108,6 +110,7 @@ pub fn handle_item_equipped( control::get().handle_item_equipped(equipped, form_spec, right, left) } +/// Pass along a CGO grip-change event to the controller. pub fn handle_grip_change(use_alt_grip: bool) { control::get().handle_grip_change(use_alt_grip); } @@ -117,6 +120,7 @@ pub fn handle_inventory_changed(form_spec: &String, count: u32) { control::get().handle_inventory_changed(form_spec, count); } +/// Handle an item being favorited. pub fn handle_favorite_event( button: &ButtonEvent, is_favorite: bool, @@ -125,6 +129,7 @@ pub fn handle_favorite_event( control::get().handle_favorite_event(button, is_favorite, *item); } +/// Ask the control to refresh settings. pub fn refresh_user_settings() { if let Some(e) = UserSettings::refresh().err() { log::warn!("Failed to read user settings! using defaults; {e:#}"); @@ -133,6 +138,7 @@ pub fn refresh_user_settings() { control::get().apply_settings(); } +/// Clear all cycles. MCM -> this function -> controller. pub fn clear_cycles() { control::get().clear_cycles(); } @@ -251,45 +257,3 @@ pub fn set_equipset_icon(id: u32, itemname: String) -> bool { pub fn look_up_equipset_by_name(name: String) -> u32 { control::get().cycles.equipset_by_name(name) } - -pub fn show_ui() -> bool { - control::get().cycles.hud_visible() -} - -// ----------- windows character shenanigans - -use textcode::iso8859_15; - -/// C++ calls this version. -pub fn string_to_utf8(bytes_ffi: &CxxVector) -> String { - let bytes: Vec = bytes_ffi.iter().copied().collect(); - convert_to_utf8_doggedly(bytes) -} - -// To test in game: install daegon -// player.additem 4c2b15f4 1 -// Sacrÿfev Tëliimi - -/// Get a valid Rust representation of this Windows codepage string data by hook or by crook. -pub fn convert_to_utf8_doggedly(input: Vec) -> String { - let bytes = if input.ends_with(&[0]) { - let chopped = input.len() - 1; - let mut tmp = input.clone(); - tmp.truncate(chopped); - tmp - } else { - input.clone() - }; - if bytes.is_empty() { - return String::new(); - } - - // Maybe it's the easy case and we're done! - if let Ok(utf8string) = String::from_utf8(bytes.clone()) { - return utf8string; - } - - let mut dst = String::new(); - iso8859_15::decode(bytes.as_slice(), &mut dst); - dst -} diff --git a/src/controller/mod.rs b/src/controller/mod.rs index 26c17870..ffa6b964 100644 --- a/src/controller/mod.rs +++ b/src/controller/mod.rs @@ -14,7 +14,9 @@ pub mod facade; pub mod keys; pub mod logs; pub mod settings; +pub mod strings; pub use facade::*; pub use logs::*; pub use settings::UserSettings; +pub use strings::*; diff --git a/src/controller/strings.rs b/src/controller/strings.rs new file mode 100644 index 00000000..c5eb28b0 --- /dev/null +++ b/src/controller/strings.rs @@ -0,0 +1,108 @@ +//! Character encoding shenanigans. Bethesda is very bad at utf-8, I am told. + +use byte_slice_cast::AsSliceOf; +use cxx::CxxVector; +use textcode::{iso8859_15, iso8859_9}; + +// To test in game: install daegon +// player.additem 4c2b15f4 1 +// Sacrÿfev Tëliimi + +/// C++ should use this for std::string conversions. +pub fn string_to_utf8(bytes_ffi: &CxxVector) -> String { + let bytes: Vec = bytes_ffi.iter().copied().collect(); + convert_to_utf8(bytes) +} + +/// Use this for null-terminated C strings. +pub fn cstr_to_utf8(bytes_ffi: &CxxVector) -> String { + let bytes: Vec = bytes_ffi.iter().copied().collect(); + let bytes = if bytes.ends_with(&[0]) { + let chopped = bytes.len() - 1; + let mut tmp = bytes.clone(); + tmp.truncate(chopped); + tmp + } else { + bytes + }; + convert_to_utf8(bytes) +} + +/// Get a valid Rust representation of this Windows codepage string data by hook or by crook. +pub fn convert_to_utf8(bytes: Vec) -> String { + if bytes.is_empty() { + return String::new(); + } + + let (encoding, _confidence, _language) = chardet::detect(&bytes); + match encoding.as_str() { + "utf-8" => String::from_utf8(bytes.clone()) + .unwrap_or_else(|_| String::from_utf8_lossy(&bytes).to_string()), + "ISO-8859-9" => { + let mut dst = String::new(); + iso8859_9::decode(bytes.as_slice(), &mut dst); + dst + } + "ISO-8859-15" => { + let mut dst = String::new(); + iso8859_15::decode(bytes.as_slice(), &mut dst); + dst + } + _ => { + let Ok(widebytes) = bytes.as_slice_of::() else { + return String::from_utf8_lossy(bytes.as_slice()).to_string(); + }; + let mut utf8bytes: Vec = vec![0; widebytes.len()]; + let Ok(_c) = ucs2::decode(widebytes, &mut utf8bytes) else { + return String::from_utf8_lossy(bytes.as_slice()).to_string(); + }; + String::from_utf8(utf8bytes.clone()) + .unwrap_or_else(|_| String::from_utf8_lossy(utf8bytes.as_slice()).to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn utf8_data_is_untouched() { + let example = "Sacrÿfev Tëliimi"; + let converted = convert_to_utf8(example.as_bytes().to_vec()); + assert_eq!(converted, example); + let ex2 = "おはよう"; + let convert2 = convert_to_utf8(ex2.as_bytes().to_vec()); + assert_eq!(convert2, ex2); + let ex3 = "Zażółć gęślą jaźń"; + let convert3 = convert_to_utf8(ex3.as_bytes().to_vec()); + assert_eq!(convert3, ex3); + } + + #[test] + fn iso8859_is_decoded() { + // This is the example above (from the Daegon mod), in its expression + // as windows codepage bytes. This test is the equivalent of me testing + // that the textcode mod works, but I am feeling timid. + let bytes: Vec = vec![ + 0x53, 0x61, 0x63, 0x72, 0xff, 0x66, 0x65, 0x76, 0x20, 0x54, 0xeb, 0x6c, 0x69, 0x69, + 0x6d, 0x69, + ]; + assert!(String::from_utf8(bytes.clone()).is_err()); + let utf8_version = "Sacrÿfev Tëliimi".to_string(); + let converted = convert_to_utf8(bytes.clone()); + assert_eq!(converted, utf8_version); + } + + #[test] + fn utf16le_is_decoded() { + let bytes = vec![ + 36, 0, 83, 0, 111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 72, 0, 85, 0, 68, 0, 9, 0, 83, 0, + 111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 32, 0, 72, 0, 85, 0, 68, 0, + ]; + assert_eq!(bytes.len(), 42); + let converted = convert_to_utf8(bytes.clone()); + assert_eq!(converted.len(), bytes.len() / 2); + assert_eq!(converted, "$SoulsyHUD Soulsy HUD"); + } +} diff --git a/src/data/huditem.rs b/src/data/huditem.rs index ff9a1286..c9661fb9 100644 --- a/src/data/huditem.rs +++ b/src/data/huditem.rs @@ -222,12 +222,13 @@ impl HudItem { } pub fn fmtstr(&self, fmt: String) -> String { - // This implementation caches nothing. It might be fast enough? - // needs measurement match strfmt(&fmt, &self.format_vars) { Ok(v) => v, Err(e) => { - log::debug!("Failed to render format string for HUD item; error: {e:#}"); + log::debug!( + "Failed to render format string for HUD item; formspec='{}'; error: {e:#}", + self.form_string + ); "".to_string() } } diff --git a/src/lib.rs b/src/lib.rs index bf972e92..0c8f2a0c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -282,14 +282,16 @@ pub mod plugin { /// Log at trace level. Use this level for debugging programming problems. fn log_trace(message: String); - /// Decode a vector of bytes from a windows codepage string to a utf-8 string. + // So... ucs2 is a fixed-width format, and it might well end with a 0 if the + // data is ascii, so we cannot guess if something is null-terminated or not. + /// Decode a vector of bytes from a std::string (no null termination) to a utf-8 string. fn string_to_utf8(bytes: &CxxVector) -> String; + /// Decode a null-terminated C string from whatever it is to utf-8. + fn cstr_to_utf8(bytes_ffi: &CxxVector) -> String; /// Trigger rust to read config, figure out what the player has equipped, /// and figure out what it should draw. fn initialize_hud(); - /// Check if the user wants the HUD visible right now or not. - fn show_ui() -> bool; /// Get cycle data for cosave. fn serialize_cycles() -> Vec; /// Serialization format version. @@ -420,7 +422,7 @@ pub mod plugin { has_time_left: bool, max_time: f32, time_left: f32, - ) -> Box; + ) -> Box; /// Call this to get the fallback-aware key for an icon. fn get_icon_key(name: String) -> String; diff --git a/src/util/helpers.cpp b/src/util/helpers.cpp index 4025e479..a59df552 100644 --- a/src/util/helpers.cpp +++ b/src/util/helpers.cpp @@ -38,7 +38,7 @@ namespace helpers // It is called by bound object finder functions. auto name = form->GetName(); // this use is required auto chonker = helpers::chars_to_vec(name); - auto safename = std::string(string_to_utf8(chonker)); + auto safename = std::string(cstr_to_utf8(chonker)); return safename; } @@ -47,7 +47,7 @@ namespace helpers // Do not call this from bound object finder functions. auto name = gear::displayName(form); auto chonker = helpers::chars_to_vec(name); - auto safename = std::string(string_to_utf8(chonker)); + auto safename = std::string(cstr_to_utf8(chonker)); return safename; }