Merge pull request #838 from PrimozGodec/fix-normalization

[FIX] Normalize - fix unpickling for Normalizers before caching was implemented

orangecontrib/text/preprocess/normalize.py

@@ -48,6 +48,12 @@ def __getstate__(self):
         d["_normalization_cache"] = {}
         return d
 
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        # support old pickles (before caching was implemented) that are missing
+        # _normalization_cache
+        self._normalization_cache = {}
+
 
 class WordNetLemmatizer(BaseNormalizer):
     name = 'WordNet Lemmatizer'
@@ -201,7 +207,7 @@ def __setstate__(self, state):
 
         Note: __model will be loaded on __call__
         """
-        self.__dict__.update(state)
+        super().__setstate__(state)
         self.models = UDPipeModels()
 
 

orangecontrib/text/tests/test_preprocess.py

@@ -340,6 +340,16 @@ def test_cache(self):
         loaded_normalizer = pickle.loads(pickle.dumps(normalizer))
         self.assertEqual(0, len(loaded_normalizer._normalization_cache))
 
+    def test_nocache_normalizer_restorable(self):
+        """
+        Pickled normalizers made before implementing cache in normalizer must
+        load correctly
+        """
+        test_folder = os.path.dirname(os.path.abspath(__file__))
+        with open(os.path.join(test_folder, "normalizer-v1.pkl"), "rb") as f:
+            loaded_normalizer = pickle.load(f)
+        loaded_normalizer(self.corpus)
+
 
 class UDPipeModelsTests(unittest.TestCase):
     def test_label_transform(self):