diff --git a/orangecontrib/text/preprocess/preprocess.py b/orangecontrib/text/preprocess/preprocess.py index f37dbd1d7..90fb19f08 100644 --- a/orangecontrib/text/preprocess/preprocess.py +++ b/orangecontrib/text/preprocess/preprocess.py @@ -131,7 +131,10 @@ def __str__(self): return '\n'.join(['{}: {}'.format(name, value) for name, value in self.report()]) def report(self): - return ( + if getattr(self.normalizer, 'use_tokenizer', False): + self.tokenizer = \ + 'UDPipe Tokenizer ({})'.format(self.normalizer.language) + rep = ( ('Transformers', ', '.join(str(tr) for tr in self.transformers) if self.transformers else None), ('Tokenizer', str(self.tokenizer) if self.tokenizer else None), @@ -144,7 +147,8 @@ def report(self): else None), ('Pos tagger', str(self.pos_tagger) if self.pos_tagger else None), ) - + del self.tokenizer + return rep base_preprocessor = Preprocessor(transformers=BASE_TRANSFORMERS, tokenizer=BASE_TOKENIZER)