Merge pull request #6 from ShoukanLabs/dev

Bump version, better check all punctuation
ShoukanLabs · Oct 9, 2024 · 2b1ab6d · 2b1ab6d
2 parents 285fac4 + 4274aa3
commit 2b1ab6d
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 6 deletions.
diff --git a/VoPho/langtokenizers/multicoded.py b/VoPho/langtokenizers/multicoded.py
@@ -146,12 +146,14 @@ def detect_writing_system(self, text):
 
     def is_punctuation(self, char):
         if len(char) > 1:
-            valid_chars = set(string.punctuation + ' ')
-            return all(c in valid_chars for c in char)
+            # Valid punctuation characters, including space
+            return all(self.is_punctuation(c) for c in char)  # Check each character individually
         else:
-            return not char.isalnum() and not char.isspace() and not self.is_writing_system(char,
-                                                                                            self.detect_writing_system(
-                                                                                                char))
+            # Single character check (as per your original logic)
+            return (not char.isalnum()  # Is not alphanumeric
+                    and not char.isspace()  # Is not whitespace
+                    and not self.is_writing_system(char, self.detect_writing_system(char)))  # Is not in a writing system                                                                             self.detect_writing_system(
+
 
     def split_text_by_writing_system(self, text):
         segments = []

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "VoPho"
-version = "0.0.4"
+version = "0.0.5"
 description = "An easy to use Multilingual phonemization meta-library"
 readme = "README.md"
 authors = [