-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
String encodings continue to be one of the hardest things (#103)
When we get a string for an item name from the game, it might be in one of several encodings: - ascii - Latin-1 1252, aka ISO-8859 most likely the 9 variation - UCS2, which is almost like UTF-16 LE except it's fixed width We must take this varied input and turn it into valid UTF-8 strings for Rust usage & logging, and for display in the HUD via Imgui. We must also not disturb any already-valid strings by mangling them while attempting to decode them into a modern encoding. So, we do the following. First, use `chardet::detect()` to guess at an encoding. If it's utf-8 already or one of the ISO-8859s, we can decode immediately and return. If it is not one of those, we make the _assumption_ that it is UCS2. The string might be almost-correctly detected as UTF-16le if it uses the second byte for any characters, but if the characters it is holding are plain old ascii, it'll be reported as ASCII and that is wrong. So we guess, and then decode the UCS2. Broke the string manglers out into their own file and wrote some basic tests. Changed the helper wrappers to use cstr null-terminated version for item name handling, since those come from the game with null termination. If I discover the null termination is also double-width I guess I'll have a bug to fix. Put mcm-meta-helper to work on the task for which it was invented. Added better logging for when fmtstr fails on huditems.
- Loading branch information
Showing
10 changed files
with
168 additions
and
70 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| //! Character encoding shenanigans. Bethesda is very bad at utf-8, I am told. | ||
| use byte_slice_cast::AsSliceOf; | ||
| use cxx::CxxVector; | ||
| use textcode::{iso8859_15, iso8859_9}; | ||
|
|
||
| // To test in game: install daegon | ||
| // player.additem 4c2b15f4 1 | ||
| // Sacrÿfev Tëliimi | ||
|
|
||
| /// C++ should use this for std::string conversions. | ||
| pub fn string_to_utf8(bytes_ffi: &CxxVector<u8>) -> String { | ||
| let bytes: Vec<u8> = bytes_ffi.iter().copied().collect(); | ||
| convert_to_utf8(bytes) | ||
| } | ||
|
|
||
| /// Use this for null-terminated C strings. | ||
| pub fn cstr_to_utf8(bytes_ffi: &CxxVector<u8>) -> String { | ||
| let bytes: Vec<u8> = bytes_ffi.iter().copied().collect(); | ||
| let bytes = if bytes.ends_with(&[0]) { | ||
| let chopped = bytes.len() - 1; | ||
| let mut tmp = bytes.clone(); | ||
| tmp.truncate(chopped); | ||
| tmp | ||
| } else { | ||
| bytes | ||
| }; | ||
| convert_to_utf8(bytes) | ||
| } | ||
|
|
||
| /// Get a valid Rust representation of this Windows codepage string data by hook or by crook. | ||
| pub fn convert_to_utf8(bytes: Vec<u8>) -> String { | ||
| if bytes.is_empty() { | ||
| return String::new(); | ||
| } | ||
|
|
||
| let (encoding, _confidence, _language) = chardet::detect(&bytes); | ||
| match encoding.as_str() { | ||
| "utf-8" => String::from_utf8(bytes.clone()) | ||
| .unwrap_or_else(|_| String::from_utf8_lossy(&bytes).to_string()), | ||
| "ISO-8859-9" => { | ||
| let mut dst = String::new(); | ||
| iso8859_9::decode(bytes.as_slice(), &mut dst); | ||
| dst | ||
| } | ||
| "ISO-8859-15" => { | ||
| let mut dst = String::new(); | ||
| iso8859_15::decode(bytes.as_slice(), &mut dst); | ||
| dst | ||
| } | ||
| _ => { | ||
| let Ok(widebytes) = bytes.as_slice_of::<u16>() else { | ||
| return String::from_utf8_lossy(bytes.as_slice()).to_string(); | ||
| }; | ||
| let mut utf8bytes: Vec<u8> = vec![0; widebytes.len()]; | ||
| let Ok(_c) = ucs2::decode(widebytes, &mut utf8bytes) else { | ||
| return String::from_utf8_lossy(bytes.as_slice()).to_string(); | ||
| }; | ||
| String::from_utf8(utf8bytes.clone()) | ||
| .unwrap_or_else(|_| String::from_utf8_lossy(utf8bytes.as_slice()).to_string()) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| use super::*; | ||
|
|
||
| #[test] | ||
| fn utf8_data_is_untouched() { | ||
| let example = "Sacrÿfev Tëliimi"; | ||
| let converted = convert_to_utf8(example.as_bytes().to_vec()); | ||
| assert_eq!(converted, example); | ||
| let ex2 = "おはよう"; | ||
| let convert2 = convert_to_utf8(ex2.as_bytes().to_vec()); | ||
| assert_eq!(convert2, ex2); | ||
| let ex3 = "Zażółć gęślą jaźń"; | ||
| let convert3 = convert_to_utf8(ex3.as_bytes().to_vec()); | ||
| assert_eq!(convert3, ex3); | ||
| } | ||
|
|
||
| #[test] | ||
| fn iso8859_is_decoded() { | ||
| // This is the example above (from the Daegon mod), in its expression | ||
| // as windows codepage bytes. This test is the equivalent of me testing | ||
| // that the textcode mod works, but I am feeling timid. | ||
| let bytes: Vec<u8> = vec![ | ||
| 0x53, 0x61, 0x63, 0x72, 0xff, 0x66, 0x65, 0x76, 0x20, 0x54, 0xeb, 0x6c, 0x69, 0x69, | ||
| 0x6d, 0x69, | ||
| ]; | ||
| assert!(String::from_utf8(bytes.clone()).is_err()); | ||
| let utf8_version = "Sacrÿfev Tëliimi".to_string(); | ||
| let converted = convert_to_utf8(bytes.clone()); | ||
| assert_eq!(converted, utf8_version); | ||
| } | ||
|
|
||
| #[test] | ||
| fn utf16le_is_decoded() { | ||
| let bytes = vec![ | ||
| 36, 0, 83, 0, 111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 72, 0, 85, 0, 68, 0, 9, 0, 83, 0, | ||
| 111, 0, 117, 0, 108, 0, 115, 0, 121, 0, 32, 0, 72, 0, 85, 0, 68, 0, | ||
| ]; | ||
| assert_eq!(bytes.len(), 42); | ||
| let converted = convert_to_utf8(bytes.clone()); | ||
| assert_eq!(converted.len(), bytes.len() / 2); | ||
| assert_eq!(converted, "$SoulsyHUD Soulsy HUD"); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.