Fix encoding/decoding of non-english TLK files

NickHugi · Nov 8, 2023 · 6fb09df · 6fb09df
1 parent 0fe53e1
commit 6fb09df
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/pykotor/common/stream.py b/pykotor/common/stream.py
@@ -8,7 +8,7 @@
 from typing import TYPE_CHECKING, BinaryIO
 
 from pykotor.common.geometry import Vector2, Vector3, Vector4
-from pykotor.common.language import LocalizedString
+from pykotor.common.language import Language, LocalizedString
 from pykotor.common.misc import encode_bytes_with_fallback
 from pykotor.tools.path import Path
 
@@ -1408,8 +1408,8 @@ def write_string(
             while len(value) < string_length:
                 value += padding
             value = value[:string_length]
-
-        self._stream.write(value.encode(encoding))
+        errors = "ignore" if encoding in [Language.CHINESE_SIMPLIFIED.get_encoding(), Language.CHINESE_TRADITIONAL.get_encoding()] else "strict"
+        self._stream.write(value.encode(encoding, errors=errors))
 
     def write_line(
         self,
@@ -1904,7 +1904,8 @@ def write_line(
         self._encode_val_and_update_position(line, "ascii")
 
     def _encode_val_and_update_position(self, value: str, encoding: str):
-        encoded = value.encode(encoding)
+        errors = "ignore" if encoding in [Language.CHINESE_SIMPLIFIED.get_encoding(), Language.CHINESE_TRADITIONAL.get_encoding()] else "strict"
+        encoded = value.encode(encoding, errors=errors)
         self._ba[self._position : self._position + len(encoded)] = encoded
         self._position += len(encoded)
 

diff --git a/pykotor/resource/formats/tlk/io_tlk.py b/pykotor/resource/formats/tlk/io_tlk.py
@@ -91,7 +91,7 @@ def _load_text(
         text_header = self._text_headers[stringref]
 
         self._reader.seek(text_header.offset + self._texts_offset)
-        text = self._reader.read_string(text_header.length)
+        text = self._reader.read_string(text_header.length, encoding=self._tlk.language.get_encoding())
 
         self._tlk.entries[stringref].text = text
 
@@ -116,9 +116,8 @@ def write(
         for entry in self._tlk.entries:
             self._write_entry(entry, text_offset)
 
-        encoding = self._tlk.language.get_encoding()
         for entry in self._tlk.entries:
-            self._writer.write_string(entry.text, encoding)
+            self._writer.write_string(entry.text, self._tlk.language.get_encoding())
 
     def _calculate_entries_offset(
         self,