From f82969d9d56a03dfd9438664abcf4f4b6b67d9a8 Mon Sep 17 00:00:00 2001 From: Elena Merdjanovska Date: Tue, 23 Jul 2024 22:59:06 +0200 Subject: [PATCH 001/333] add intial noisebench dataset class --- flair/datasets/sequence_labeling.py | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 38ca75e94b..044f2a8609 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -4973,6 +4973,57 @@ def _write_instances(cls, version, base_path, split, data): out_file.write("\n") +class NER_NOISEBENCH(ColumnCorpus): + label_url = "https://github.com/elenamer/NoiseBench/tree/8a32da1e06f2239afc95b3f9dc5274abc25cc46d/data/annotations" + def __init__( + self, + noise: str = None, + base_path: Optional[Union[str, Path]] = None, + in_memory: bool = True, + **corpusargs, + ) -> None: + """Initialize the NoiseBench corpus. + + Args: + noise (string): Chooses the labelset for the data. + clean (default): Clean labels + crowd,crowdbest,expert,distant,weak,llm : Different kinds of noisy labelsets (details: ...) + base_path (Optional[Union[str, Path]]): Path to the data. + Default is None, meaning the corpus gets automatically downloaded and saved. + You can override this by passing a path to a directory containing the unprocessed files but typically this + should not be necessary. + in_memory (bool): If True the dataset is kept in memory achieving speedups in training. + **corpusargs: The arguments propagated to :meth:'flair.datasets.ColumnCorpus.__init__'. + """ + if noise not in ['clean', None, 'crowd','crowdbest','expert','distant','weak','llm']: + raise Exception( + "Please choose a valid version" + ) + + base_path = self._set_path(base_path) + + filename = 'clean' if noise in ['clean',None] else f'noise_{noise}' + + cached_path(f"{self.label_url}/{filename}.traindev", base_path) + cached_path(f"{self.label_url}/index.txt", base_path) + + super().__init__( + data_folder=base_path, + train_file=f"{filename}.train", + dev_file=f"{filename}.dev", + test_file=f"clean.test", # test set is always clean (without noise) + column_format={0: "text", 1: "ner"}, + in_memory=in_memory, + column_delimiter="\t", + document_separator = "-DOCSTART-", + **corpusargs, + ) + + @classmethod + def _set_path(cls, base_path) -> Path: + base_path = flair.cache_root / "datasets" / "noisebench" if not base_path else Path(base_path) + return base_path + class MASAKHA_POS(MultiCorpus): def __init__( self, From 2ede7190217ec28320fa537f5f2d615d1c12bcf4 Mon Sep 17 00:00:00 2001 From: Elena Merdjanovska Date: Wed, 24 Jul 2024 09:34:56 +0200 Subject: [PATCH 002/333] add downloading of cleanconll --- flair/datasets/sequence_labeling.py | 32 ++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 044f2a8609..0c89793463 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -5003,9 +5003,35 @@ def __init__( base_path = self._set_path(base_path) filename = 'clean' if noise in ['clean',None] else f'noise_{noise}' - - cached_path(f"{self.label_url}/{filename}.traindev", base_path) - cached_path(f"{self.label_url}/index.txt", base_path) + file_paths = [base_path / f'{filename}.train', base_path / f'{filename}.dev', base_path / 'clean.test'] + files_exist = [path.exists() for path in file_paths] + cleanconll_base_path = flair.cache_root / "datasets" / "cleanconll" + + if not all(files_exist): + cached_path(f"{self.label_url}/{filename}.traindev", base_path / 'annotations_only') + cached_path(f"{self.label_url}/index.txt", base_path / 'annotations_only') + + cleanconll_files_exist = [Path(f'{cleanconll_base_path}/cleanconll.{split}').exists() for split in ['train','dev','test']] + if not all(cleanconll_files_exist): + # download cleanconll + + clone = f"git clone https://github.com/flairNLP/CleanCoNLL.git {cleanconll_base_path}/CleanCoNLL" + os.system(clone) # Cloning + cwd = os.getcwd() + + os.chdir(f"{cleanconll_base_path}/CleanCoNLL") + chmod = f"chmod u+x create_cleanconll_from_conll03.sh" + os.system(chmod) + create = f"bash create_cleanconll_from_conll03.sh" + + os.system(create) + os.chdir(cwd) + + shutil.move(f'{cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.train', cleanconll_base_path) + shutil.move(f'{cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.dev', cleanconll_base_path) + shutil.move(f'{cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.test', cleanconll_base_path) + + # create dataset files from index and train/test splits super().__init__( data_folder=base_path, From a34d2b2e5c7c60c8efc7d6ea0cd28c46cb901a47 Mon Sep 17 00:00:00 2001 From: Elena Merdjanovska Date: Wed, 24 Jul 2024 09:35:11 +0200 Subject: [PATCH 003/333] update __init__ --- flair/datasets/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flair/datasets/__init__.py b/flair/datasets/__init__.py index 2837e017c0..5a7ccf7e77 100644 --- a/flair/datasets/__init__.py +++ b/flair/datasets/__init__.py @@ -197,6 +197,7 @@ NER_ENGLISH_WIKIGOLD, NER_ENGLISH_WNUT_2020, NER_ESTONIAN_NOISY, + NER_NOISEBENCH, NER_FINNISH, NER_GERMAN_BIOFID, NER_GERMAN_EUROPARL, From 2684e93136fa7e61059aa9340cb23e5aa460fc0a Mon Sep 17 00:00:00 2001 From: Elena Merdjanovska Date: Wed, 24 Jul 2024 14:22:41 +0200 Subject: [PATCH 004/333] add processing of noisebench label sets --- flair/datasets/sequence_labeling.py | 144 ++++++++++++++++++++++++---- 1 file changed, 126 insertions(+), 18 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 0c89793463..9a30c05ee4 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -4974,7 +4974,9 @@ def _write_instances(cls, version, base_path, split, data): class NER_NOISEBENCH(ColumnCorpus): - label_url = "https://github.com/elenamer/NoiseBench/tree/8a32da1e06f2239afc95b3f9dc5274abc25cc46d/data/annotations" + label_url = "https://raw.githubusercontent.com/elenamer/NoiseBench/main/data/annotations/" + SAVE_TRAINDEV_FILE = False + def __init__( self, noise: str = None, @@ -5000,26 +5002,26 @@ def __init__( "Please choose a valid version" ) - base_path = self._set_path(base_path) + self._set_path(base_path) filename = 'clean' if noise in ['clean',None] else f'noise_{noise}' - file_paths = [base_path / f'{filename}.train', base_path / f'{filename}.dev', base_path / 'clean.test'] + file_paths = [self.base_path / f'{filename}.train', self.base_path / f'{filename}.dev', self.base_path / 'clean.test'] files_exist = [path.exists() for path in file_paths] - cleanconll_base_path = flair.cache_root / "datasets" / "cleanconll" + self.cleanconll_base_path = flair.cache_root / "datasets" / "cleanconll" if not all(files_exist): - cached_path(f"{self.label_url}/{filename}.traindev", base_path / 'annotations_only') - cached_path(f"{self.label_url}/index.txt", base_path / 'annotations_only') + cached_path(f"{self.label_url}/{filename}.traindev", self.base_path / 'annotations_only') + cached_path(f"{self.label_url}/index.txt", self.base_path / 'annotations_only') - cleanconll_files_exist = [Path(f'{cleanconll_base_path}/cleanconll.{split}').exists() for split in ['train','dev','test']] + cleanconll_files_exist = [Path(f'{self.cleanconll_base_path}/cleanconll.{split}').exists() for split in ['train','dev','test']] if not all(cleanconll_files_exist): # download cleanconll - clone = f"git clone https://github.com/flairNLP/CleanCoNLL.git {cleanconll_base_path}/CleanCoNLL" + clone = f"git clone https://github.com/flairNLP/CleanCoNLL.git {self.cleanconll_base_path}/CleanCoNLL" os.system(clone) # Cloning cwd = os.getcwd() - os.chdir(f"{cleanconll_base_path}/CleanCoNLL") + os.chdir(f"{self.cleanconll_base_path}/CleanCoNLL") chmod = f"chmod u+x create_cleanconll_from_conll03.sh" os.system(chmod) create = f"bash create_cleanconll_from_conll03.sh" @@ -5027,29 +5029,135 @@ def __init__( os.system(create) os.chdir(cwd) - shutil.move(f'{cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.train', cleanconll_base_path) - shutil.move(f'{cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.dev', cleanconll_base_path) - shutil.move(f'{cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.test', cleanconll_base_path) + shutil.move(f'{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.train', self.cleanconll_base_path) + shutil.move(f'{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.dev', self.cleanconll_base_path) + shutil.move(f'{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.test', self.cleanconll_base_path) + + shutil.rmtree(self.cleanconll_base_path / 'CleanCoNLL') # create dataset files from index and train/test splits + self.generate_data_files(filename,) super().__init__( - data_folder=base_path, + data_folder=self.base_path, train_file=f"{filename}.train", dev_file=f"{filename}.dev", test_file=f"clean.test", # test set is always clean (without noise) column_format={0: "text", 1: "ner"}, in_memory=in_memory, column_delimiter="\t", - document_separator = "-DOCSTART-", + document_separator_token = "-DOCSTART-", **corpusargs, ) - @classmethod - def _set_path(cls, base_path) -> Path: - base_path = flair.cache_root / "datasets" / "noisebench" if not base_path else Path(base_path) - return base_path + def _set_path(self, base_path) -> Path: + self.base_path = flair.cache_root / "datasets" / "noisebench" if not base_path else Path(base_path) + + @staticmethod + def read_column_file(filename): + raw = open(filename, 'r', errors='replace') + raw = raw.readlines() + all_x = [] + point = [] + for line in raw: + if '\t' in line.strip(): + stripped_line = line.strip().split('\t') + else: + stripped_line = line.strip().split(' ') + point.append(stripped_line) + if line.strip() == '': + if len(point[:-1]) > 0: + all_x.append(point[:-1]) + point = [] + + if len(point) > 0: + all_x.append(point) + + all_x = all_x + return all_x + + @staticmethod + def save_to_column_file(filename, list): + with open(filename, "w") as f: + for sentence in list: + for token in sentence: + f.write('\t'.join(token)) + f.write('\n') + f.write('\n') + + def _create_train_dev_splits(self, filename, all_sentences = None, datestring = '1996-08-24'): + if not all_sentences: + all_sentences = self.read_column_file(filename) + + train_sentences = [] + dev_sentences = [] + for i, s in enumerate(all_sentences): + if 'DOCSTART' in s[0][0]: + assert i+3 < len(all_sentences) # last document is too short + + # news date is usually in 3rd or 4th sentence of each article + if datestring in all_sentences[i+2][-1][0] or datestring in all_sentences[i+3][-1][0]: + save_to_dev = True + else: + save_to_dev = False + + if save_to_dev: + dev_sentences.append(s) + else: + train_sentences.append(s) + + self.save_to_column_file(os.sep.join(filename.split(os.sep)[:-1])+os.sep+filename.split(os.sep)[-1].split('.')[0]+'.dev',dev_sentences) + self.save_to_column_file(os.sep.join(filename.split(os.sep)[:-1])+os.sep+filename.split(os.sep)[-1].split('.')[0]+'.train',train_sentences) + + + def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): + # generate NoiseBench dataset variants, given CleanCoNLL, noisy label files and index file + + noisy_labels = self.read_column_file(os.path.join(self.base_path,'annotations_only',f'{corpus}.traindev')) + #print(noisy_labels) + #print(token_indices) + for index, sentence in zip(token_indices, noisy_labels): + + if index.strip() == 'docstart': + assert len(sentence) == 1 + sentence[0][0] = '-DOCSTART-' + continue + clean_sentence = all_clean_sentences[int(index.strip())] + + assert len(clean_sentence) == len(sentence) # this means indexing is wrong + + for token, label in zip(clean_sentence, sentence): + label[0] = token[0] # token[0] -> text, token[1] -> BIO label + if self.SAVE_TRAINDEV_FILE: + self.save_to_column_file(os.path.join(self.base_path,f'{corpus}.traindev'),noisy_labels) + return noisy_labels + + + def generate_data_files(self, filename): + + index_file = open(os.path.join(self.base_path,'annotations_only','index.txt')) + token_indices = index_file.readlines() + + all_clean_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path,'cleanconll.train')) + + #os.makedirs(os.path.join('data','noisebench'), exist_ok=True) + + noisy_sentences = self._merge_tokens_labels(filename, all_clean_sentences, token_indices) + self._create_train_dev_splits(all_sentences=noisy_sentences,filename=os.path.join(self.base_path,f'{filename}.traindev')) + + # copy test set + all_clean_test_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path,'cleanconll.test')) + + test_sentences = [] + for s in all_clean_test_sentences: + new_s = [] + for token in s: + new_s.append([token[0],token[4]]) + test_sentences.append(new_s) + + self.save_to_column_file(os.path.join(self.base_path,f'clean.test'),test_sentences) + class MASAKHA_POS(MultiCorpus): def __init__( self, From 0e8183d7a75276e6c9188d0d91f6211e961c54cb Mon Sep 17 00:00:00 2001 From: Elena Merdjanovska Date: Wed, 24 Jul 2024 14:57:35 +0200 Subject: [PATCH 005/333] fix formatting --- flair/datasets/__init__.py | 3 +- flair/datasets/sequence_labeling.py | 172 +++++++++++++++------------- 2 files changed, 97 insertions(+), 78 deletions(-) diff --git a/flair/datasets/__init__.py b/flair/datasets/__init__.py index 5a7ccf7e77..b38d1bd761 100644 --- a/flair/datasets/__init__.py +++ b/flair/datasets/__init__.py @@ -197,7 +197,6 @@ NER_ENGLISH_WIKIGOLD, NER_ENGLISH_WNUT_2020, NER_ESTONIAN_NOISY, - NER_NOISEBENCH, NER_FINNISH, NER_GERMAN_BIOFID, NER_GERMAN_EUROPARL, @@ -217,6 +216,7 @@ NER_MULTI_WIKINER, NER_MULTI_XTREME, NER_NERMUD, + NER_NOISEBENCH, NER_SWEDISH, NER_TURKU, NER_UKRAINIAN, @@ -495,6 +495,7 @@ "NER_GERMAN_MOBIE", "NER_GERMAN_POLITICS", "NER_HIPE_2022", + "NER_NOISEBENCH", "NER_HUNGARIAN", "NER_ICDAR_EUROPEANA", "NER_ICELANDIC", diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 9a30c05ee4..7cc06c8a0a 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -4976,10 +4976,10 @@ def _write_instances(cls, version, base_path, split, data): class NER_NOISEBENCH(ColumnCorpus): label_url = "https://raw.githubusercontent.com/elenamer/NoiseBench/main/data/annotations/" SAVE_TRAINDEV_FILE = False - + def __init__( self, - noise: str = None, + noise: str = "clean", base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, **corpusargs, @@ -4997,78 +4997,90 @@ def __init__( in_memory (bool): If True the dataset is kept in memory achieving speedups in training. **corpusargs: The arguments propagated to :meth:'flair.datasets.ColumnCorpus.__init__'. """ - if noise not in ['clean', None, 'crowd','crowdbest','expert','distant','weak','llm']: - raise Exception( - "Please choose a valid version" - ) + if noise not in ["clean", "crowd", "crowdbest", "expert", "distant", "weak", "llm"]: + raise Exception("Please choose a valid version") self._set_path(base_path) - filename = 'clean' if noise in ['clean',None] else f'noise_{noise}' - file_paths = [self.base_path / f'{filename}.train', self.base_path / f'{filename}.dev', self.base_path / 'clean.test'] + filename = "clean" if noise == "clean" else f"noise_{noise}" + file_paths = [ + self.base_path / f"{filename}.train", + self.base_path / f"{filename}.dev", + self.base_path / "clean.test", + ] files_exist = [path.exists() for path in file_paths] self.cleanconll_base_path = flair.cache_root / "datasets" / "cleanconll" if not all(files_exist): - cached_path(f"{self.label_url}/{filename}.traindev", self.base_path / 'annotations_only') - cached_path(f"{self.label_url}/index.txt", self.base_path / 'annotations_only') - - cleanconll_files_exist = [Path(f'{self.cleanconll_base_path}/cleanconll.{split}').exists() for split in ['train','dev','test']] + cached_path(f"{self.label_url}/{filename}.traindev", self.base_path / "annotations_only") + cached_path(f"{self.label_url}/index.txt", self.base_path / "annotations_only") + + cleanconll_files_exist = [ + Path(f"{self.cleanconll_base_path}/cleanconll.{split}").exists() for split in ["train", "dev", "test"] + ] if not all(cleanconll_files_exist): # download cleanconll - clone = f"git clone https://github.com/flairNLP/CleanCoNLL.git {self.cleanconll_base_path}/CleanCoNLL" - os.system(clone) # Cloning + clone = f"git clone https://github.com/flairNLP/CleanCoNLL.git {self.cleanconll_base_path}/CleanCoNLL" + os.system(clone) # Cloning cwd = os.getcwd() os.chdir(f"{self.cleanconll_base_path}/CleanCoNLL") - chmod = f"chmod u+x create_cleanconll_from_conll03.sh" + chmod = "chmod u+x create_cleanconll_from_conll03.sh" os.system(chmod) - create = f"bash create_cleanconll_from_conll03.sh" - + create = "bash create_cleanconll_from_conll03.sh" + os.system(create) os.chdir(cwd) - - shutil.move(f'{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.train', self.cleanconll_base_path) - shutil.move(f'{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.dev', self.cleanconll_base_path) - shutil.move(f'{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.test', self.cleanconll_base_path) - - shutil.rmtree(self.cleanconll_base_path / 'CleanCoNLL') + + shutil.move( + f"{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.train", + self.cleanconll_base_path, + ) + shutil.move( + f"{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.dev", self.cleanconll_base_path + ) + shutil.move( + f"{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.test", self.cleanconll_base_path + ) + + shutil.rmtree(self.cleanconll_base_path / "CleanCoNLL") # create dataset files from index and train/test splits - self.generate_data_files(filename,) + self.generate_data_files( + filename, + ) super().__init__( data_folder=self.base_path, train_file=f"{filename}.train", dev_file=f"{filename}.dev", - test_file=f"clean.test", # test set is always clean (without noise) + test_file="clean.test", # test set is always clean (without noise) column_format={0: "text", 1: "ner"}, in_memory=in_memory, column_delimiter="\t", - document_separator_token = "-DOCSTART-", + document_separator_token="-DOCSTART-", **corpusargs, ) - def _set_path(self, base_path) -> Path: + def _set_path(self, base_path): self.base_path = flair.cache_root / "datasets" / "noisebench" if not base_path else Path(base_path) - - @staticmethod + + @staticmethod def read_column_file(filename): - raw = open(filename, 'r', errors='replace') - raw = raw.readlines() - all_x = [] - point = [] - for line in raw: - if '\t' in line.strip(): - stripped_line = line.strip().split('\t') - else: - stripped_line = line.strip().split(' ') - point.append(stripped_line) - if line.strip() == '': - if len(point[:-1]) > 0: - all_x.append(point[:-1]) - point = [] + with open(filename, errors="replace") as file: + lines = file.readlines() + all_x = [] + point = [] + for line in lines: + if "\t" in line.strip(): + stripped_line = line.strip().split("\t") if "\t" in line.strip() else line.strip().split(" ") + + point.append(stripped_line) + if line.strip() == "": + if len(point[:-1]) > 0: + all_x.append(point[:-1]) + point = [] if len(point) > 0: all_x.append(point) @@ -5081,22 +5093,22 @@ def save_to_column_file(filename, list): with open(filename, "w") as f: for sentence in list: for token in sentence: - f.write('\t'.join(token)) - f.write('\n') - f.write('\n') + f.write("\t".join(token)) + f.write("\n") + f.write("\n") - def _create_train_dev_splits(self, filename, all_sentences = None, datestring = '1996-08-24'): + def _create_train_dev_splits(self, filename, all_sentences=None, datestring="1996-08-24"): if not all_sentences: all_sentences = self.read_column_file(filename) - train_sentences = [] - dev_sentences = [] + train_sentences = [] + dev_sentences = [] for i, s in enumerate(all_sentences): - if 'DOCSTART' in s[0][0]: - assert i+3 < len(all_sentences) # last document is too short - + if "DOCSTART" in s[0][0]: + assert i + 3 < len(all_sentences) # last document is too short + # news date is usually in 3rd or 4th sentence of each article - if datestring in all_sentences[i+2][-1][0] or datestring in all_sentences[i+3][-1][0]: + if datestring in all_sentences[i + 2][-1][0] or datestring in all_sentences[i + 3][-1][0]: save_to_dev = True else: save_to_dev = False @@ -5106,57 +5118,63 @@ def _create_train_dev_splits(self, filename, all_sentences = None, datestring = else: train_sentences.append(s) - self.save_to_column_file(os.sep.join(filename.split(os.sep)[:-1])+os.sep+filename.split(os.sep)[-1].split('.')[0]+'.dev',dev_sentences) - self.save_to_column_file(os.sep.join(filename.split(os.sep)[:-1])+os.sep+filename.split(os.sep)[-1].split('.')[0]+'.train',train_sentences) - + self.save_to_column_file( + os.sep.join(filename.split(os.sep)[:-1]) + os.sep + filename.split(os.sep)[-1].split(".")[0] + ".dev", + dev_sentences, + ) + self.save_to_column_file( + os.sep.join(filename.split(os.sep)[:-1]) + os.sep + filename.split(os.sep)[-1].split(".")[0] + ".train", + train_sentences, + ) def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): # generate NoiseBench dataset variants, given CleanCoNLL, noisy label files and index file - noisy_labels = self.read_column_file(os.path.join(self.base_path,'annotations_only',f'{corpus}.traindev')) - #print(noisy_labels) - #print(token_indices) + noisy_labels = self.read_column_file(os.path.join(self.base_path, "annotations_only", f"{corpus}.traindev")) + # print(noisy_labels) + # print(token_indices) for index, sentence in zip(token_indices, noisy_labels): - if index.strip() == 'docstart': + if index.strip() == "docstart": assert len(sentence) == 1 - sentence[0][0] = '-DOCSTART-' + sentence[0][0] = "-DOCSTART-" continue clean_sentence = all_clean_sentences[int(index.strip())] - assert len(clean_sentence) == len(sentence) # this means indexing is wrong + assert len(clean_sentence) == len(sentence) # this means indexing is wrong for token, label in zip(clean_sentence, sentence): - label[0] = token[0] # token[0] -> text, token[1] -> BIO label + label[0] = token[0] # token[0] -> text, token[1] -> BIO label if self.SAVE_TRAINDEV_FILE: - self.save_to_column_file(os.path.join(self.base_path,f'{corpus}.traindev'),noisy_labels) + self.save_to_column_file(os.path.join(self.base_path, f"{corpus}.traindev"), noisy_labels) return noisy_labels - def generate_data_files(self, filename): - index_file = open(os.path.join(self.base_path,'annotations_only','index.txt')) - token_indices = index_file.readlines() + with open(os.path.join(self.base_path, "annotations_only", "index.txt")) as index_file: + token_indices = index_file.readlines() - all_clean_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path,'cleanconll.train')) + all_clean_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, "cleanconll.train")) - #os.makedirs(os.path.join('data','noisebench'), exist_ok=True) + # os.makedirs(os.path.join('data','noisebench'), exist_ok=True) - noisy_sentences = self._merge_tokens_labels(filename, all_clean_sentences, token_indices) - self._create_train_dev_splits(all_sentences=noisy_sentences,filename=os.path.join(self.base_path,f'{filename}.traindev')) - + noisy_sentences = self._merge_tokens_labels(filename, all_clean_sentences, token_indices) + self._create_train_dev_splits( + all_sentences=noisy_sentences, filename=os.path.join(self.base_path, f"{filename}.traindev") + ) + + # copy test set + all_clean_test_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, "cleanconll.test")) - # copy test set - all_clean_test_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path,'cleanconll.test')) - test_sentences = [] for s in all_clean_test_sentences: new_s = [] for token in s: - new_s.append([token[0],token[4]]) + new_s.append([token[0], token[4]]) test_sentences.append(new_s) - self.save_to_column_file(os.path.join(self.base_path,f'clean.test'),test_sentences) + self.save_to_column_file(os.path.join(self.base_path, "clean.test"), test_sentences) + class MASAKHA_POS(MultiCorpus): def __init__( From 588279f00bdad2302630fb26a8bf3959dacb610d Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 6 Sep 2024 14:54:36 +0200 Subject: [PATCH 006/333] fix T5 tokenizer loading --- flair/embeddings/transformer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index f3492178f9..1e88787deb 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -26,6 +26,7 @@ LayoutLMv2FeatureExtractor, PretrainedConfig, PreTrainedTokenizer, + T5TokenizerFast, ) from transformers.tokenization_utils_base import LARGE_INTEGER from transformers.utils import PaddingStrategy @@ -444,7 +445,7 @@ def _tokenizer_from_bytes(cls, zip_data: BytesIO) -> PreTrainedTokenizer: zip_obj = zipfile.ZipFile(zip_data) with tempfile.TemporaryDirectory() as temp_dir: zip_obj.extractall(temp_dir) - return AutoTokenizer.from_pretrained(temp_dir, add_prefix_space=True) + return AutoTokenizer.from_pretrained(temp_dir) @classmethod def _feature_extractor_from_bytes(cls, zip_data: Optional[BytesIO]) -> Optional[FeatureExtractionMixin]: @@ -458,7 +459,13 @@ def _feature_extractor_from_bytes(cls, zip_data: Optional[BytesIO]) -> Optional[ def __tokenizer_bytes(self): with tempfile.TemporaryDirectory() as temp_dir: files = list(self.tokenizer.save_pretrained(temp_dir)) - if self.tokenizer.is_fast and self.tokenizer.slow_tokenizer_class: + if ( + self.tokenizer.is_fast + and self.tokenizer.slow_tokenizer_class + and not isinstance( + self.tokenizer, T5TokenizerFast + ) # do not remove slow files for T5, as it can only be created from slow tokenizer with prefix space + ): vocab_files = self.tokenizer.slow_tokenizer_class.vocab_files_names.values() files = [f for f in files if all(v not in f for v in vocab_files)] zip_data = BytesIO() From caea8bb58467f9b84b4025d21906c8af9b1fdb64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Karlo=20Do=C5=A1ilovi=C4=87?= Date: Sat, 19 Oct 2024 12:11:28 +0200 Subject: [PATCH 007/333] Initial commit. --- flair/embeddings/transformer.py | 27 ++++++++----------- .../test_transformer_document_embeddings.py | 19 +++++++++++++ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index 1e88787deb..3205aeabc6 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -8,18 +8,18 @@ from abc import abstractmethod from io import BytesIO from pathlib import Path -from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union, cast +from typing import Any, cast, Dict, List, Literal, Optional, Tuple, Type, Union import torch import transformers from packaging.version import Version from torch.jit import ScriptModule from transformers import ( - CONFIG_MAPPING, AutoConfig, AutoFeatureExtractor, AutoModel, AutoTokenizer, + CONFIG_MAPPING, FeatureExtractionMixin, LayoutLMTokenizer, LayoutLMTokenizerFast, @@ -32,13 +32,8 @@ from transformers.utils import PaddingStrategy import flair -from flair.data import Sentence, Token, log -from flair.embeddings.base import ( - DocumentEmbeddings, - Embeddings, - TokenEmbeddings, - register_embeddings, -) +from flair.data import log, Sentence, Token +from flair.embeddings.base import DocumentEmbeddings, Embeddings, register_embeddings, TokenEmbeddings SENTENCE_BOUNDARY_TAG: str = "[FLERT]" @@ -198,7 +193,7 @@ def fill_mean_token_embeddings( @torch.jit.script_if_tracing -def document_mean_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: torch.Tensor): +def document_mean_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: torch.Tensor) -> torch.Tensor: result = torch.zeros( sentence_hidden_states.shape[0], sentence_hidden_states.shape[2], dtype=sentence_hidden_states.dtype ) @@ -206,9 +201,11 @@ def document_mean_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths for i in torch.arange(sentence_hidden_states.shape[0]): result[i] = sentence_hidden_states[i, : sentence_lengths[i]].mean(dim=0) + return result + @torch.jit.script_if_tracing -def document_max_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: torch.Tensor): +def document_max_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: torch.Tensor) -> torch.Tensor: result = torch.zeros( sentence_hidden_states.shape[0], sentence_hidden_states.shape[2], dtype=sentence_hidden_states.dtype ) @@ -216,6 +213,8 @@ def document_max_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: for i in torch.arange(sentence_hidden_states.shape[0]): result[i], _ = sentence_hidden_states[i, : sentence_lengths[i]].max(dim=0) + return result + def _legacy_reconstruct_word_ids( embedding: "TransformerBaseEmbeddings", flair_tokens: List[List[str]] @@ -1127,11 +1126,7 @@ def is_supported_t5_model(config: PretrainedConfig) -> bool: if peft_config is not None: # add adapters for finetuning try: - from peft import ( - TaskType, - get_peft_model, - prepare_model_for_kbit_training, - ) + from peft import get_peft_model, prepare_model_for_kbit_training, TaskType except ImportError: log.error("You cannot use the PEFT finetuning without peft being installed") raise diff --git a/tests/embeddings/test_transformer_document_embeddings.py b/tests/embeddings/test_transformer_document_embeddings.py index 1a65a96fdb..f253d2d8d8 100644 --- a/tests/embeddings/test_transformer_document_embeddings.py +++ b/tests/embeddings/test_transformer_document_embeddings.py @@ -1,7 +1,10 @@ +import pytest + from flair.data import Dictionary from flair.embeddings import TransformerDocumentEmbeddings from flair.models import TextClassifier from flair.nn import Classifier + from tests.embedding_test_utils import BaseEmbeddingsTest @@ -37,3 +40,19 @@ def test_if_loaded_embeddings_have_all_attributes(tasks_base_path): # check that context_length and use_context_separator is the same for both assert model.embeddings.context_length == loaded_single_task.embeddings.context_length assert model.embeddings.use_context_separator == loaded_single_task.embeddings.use_context_separator + + +@pytest.mark.parametrize("cls_pooling", ["cls", "mean", "max"]) +def test_cls_pooling(cls_pooling): + from flair.data import Sentence + from flair.embeddings import TransformerDocumentEmbeddings + + embeddings = TransformerDocumentEmbeddings( + model="xlm-roberta-base", + layers="-1", + cls_pooling=cls_pooling, + allow_long_sentences=True, + ) + sentence = Sentence("Today is a good day.") + embeddings.embed(sentence) + assert sentence.embedding is not None From 43fc96a5370fa0260c62018d0bd4f87579a2e987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Karlo=20Do=C5=A1ilovi=C4=87?= Date: Sat, 19 Oct 2024 12:16:52 +0200 Subject: [PATCH 008/333] Remove imports from the test function. --- tests/embeddings/test_transformer_document_embeddings.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/embeddings/test_transformer_document_embeddings.py b/tests/embeddings/test_transformer_document_embeddings.py index f253d2d8d8..bf202110b1 100644 --- a/tests/embeddings/test_transformer_document_embeddings.py +++ b/tests/embeddings/test_transformer_document_embeddings.py @@ -1,6 +1,6 @@ import pytest -from flair.data import Dictionary +from flair.data import Dictionary, Sentence from flair.embeddings import TransformerDocumentEmbeddings from flair.models import TextClassifier from flair.nn import Classifier @@ -44,9 +44,6 @@ def test_if_loaded_embeddings_have_all_attributes(tasks_base_path): @pytest.mark.parametrize("cls_pooling", ["cls", "mean", "max"]) def test_cls_pooling(cls_pooling): - from flair.data import Sentence - from flair.embeddings import TransformerDocumentEmbeddings - embeddings = TransformerDocumentEmbeddings( model="xlm-roberta-base", layers="-1", From fdb49952c61abe3f671b0a39d1eb2685a8aff9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Karlo=20Do=C5=A1ilovi=C4=87?= Date: Sat, 19 Oct 2024 13:12:25 +0200 Subject: [PATCH 009/333] Refactor cls pooling into a function. --- flair/embeddings/transformer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index 3205aeabc6..8a1e76cd72 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -191,6 +191,10 @@ def fill_mean_token_embeddings( return all_token_embeddings +@torch.jit.script_if_tracing +def document_cls_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: torch.Tensor) -> torch.Tensor: + return sentence_hidden_states[torch.arange(sentence_hidden_states.shape[0]), sentence_lengths - 1] + @torch.jit.script_if_tracing def document_mean_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: torch.Tensor) -> torch.Tensor: @@ -1436,9 +1440,7 @@ def forward( else: assert sub_token_lengths is not None if self.cls_pooling == "cls": - document_embeddings = sentence_hidden_states[ - torch.arange(sentence_hidden_states.shape[0]), sub_token_lengths - 1 - ] + document_embeddings = document_cls_pooling(sentence_hidden_states, sub_token_lengths) elif self.cls_pooling == "mean": document_embeddings = document_mean_pooling(sentence_hidden_states, sub_token_lengths) elif self.cls_pooling == "max": From 4202333d43536e3329cd44810f39b2b5763e6d24 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 25 Oct 2024 14:51:37 +0200 Subject: [PATCH 010/333] drop python 3.8 --- .github/workflows/ci.yml | 4 +- .github/workflows/publish-docs.yml | 2 +- CONTRIBUTING.md | 4 +- README.md | 2 +- docs/contributing/local_development.md | 4 +- docs/tutorial/intro.md | 2 +- flair/class_utils.py | 11 +- flair/data.py | 167 ++++++------- flair/datasets/base.py | 12 +- flair/datasets/biomedical.py | 230 +++++++++--------- flair/datasets/document_classification.py | 132 +++++----- flair/datasets/entity_linking.py | 109 +++++---- flair/datasets/ocr.py | 8 +- flair/datasets/relation_extraction.py | 30 +-- flair/datasets/sequence_labeling.py | 207 ++++++++-------- flair/datasets/text_image.py | 11 +- flair/datasets/text_text.py | 48 ++-- flair/datasets/treebanks.py | 10 +- flair/embeddings/base.py | 24 +- flair/embeddings/document.py | 62 ++--- flair/embeddings/image.py | 16 +- flair/embeddings/legacy.py | 22 +- flair/embeddings/token.py | 67 +++-- flair/embeddings/transformer.py | 70 +++--- flair/file_utils.py | 11 +- flair/inference_utils.py | 4 +- flair/models/entity_linker_model.py | 27 +- flair/models/entity_mention_linking.py | 95 ++++---- flair/models/language_model.py | 14 +- flair/models/lemmatizer_model.py | 36 +-- flair/models/multitask_model.py | 32 +-- flair/models/pairwise_classification_model.py | 5 +- flair/models/pairwise_regression_model.py | 29 +-- flair/models/prefixed_tagger.py | 36 +-- flair/models/regexp_tagger.py | 22 +- flair/models/relation_classifier_model.py | 67 +++-- flair/models/relation_extractor_model.py | 10 +- flair/models/sequence_tagger_model.py | 32 +-- flair/models/sequence_tagger_utils/viterbi.py | 12 +- flair/models/tars_model.py | 44 ++-- flair/models/text_classification_model.py | 6 +- flair/models/text_regression_model.py | 28 +-- flair/models/triple_classification_model.py | 5 +- flair/models/word_tagger_model.py | 8 +- flair/nn/decoder.py | 6 +- flair/nn/model.py | 70 +++--- flair/nn/multitask.py | 17 +- flair/samplers.py | 3 +- flair/splitter.py | 18 +- flair/tokenization.py | 34 +-- flair/trainers/language_model_trainer.py | 17 +- flair/trainers/plugins/base.py | 19 +- .../plugins/functional/anneal_on_plateau.py | 4 +- .../plugins/functional/checkpoints.py | 4 +- .../plugins/functional/linear_scheduler.py | 4 +- .../functional/reduce_transformer_vocab.py | 3 +- .../plugins/functional/weight_extractor.py | 4 +- flair/trainers/plugins/loggers/log_file.py | 4 +- flair/trainers/plugins/loggers/loss_file.py | 8 +- .../plugins/loggers/metric_history.py | 7 +- flair/trainers/plugins/loggers/tensorboard.py | 4 +- flair/trainers/plugins/loggers/wandb.py | 4 +- flair/trainers/plugins/metric_records.py | 5 +- flair/trainers/trainer.py | 32 +-- flair/training_utils.py | 18 +- flair/visual/ner_html.py | 4 +- flair/visual/training_curves.py | 8 +- flair/visual/tree_printer.py | 6 +- pyproject.toml | 4 +- requirements-dev.txt | 2 +- resources/docs/EXPERIMENTS.md | 12 +- resources/docs/HUNFLAIR2.md | 2 +- .../KOR_docs/TUTORIAL_7_TRAINING_A_MODEL.md | 2 +- .../KOR_docs/TUTORIAL_8_MODEL_OPTIMIZATION.md | 2 +- setup.py | 2 +- tests/embedding_test_utils.py | 16 +- ...test_document_transform_word_embeddings.py | 8 +- tests/embeddings/test_word_embeddings.py | 4 +- tests/model_test_utils.py | 10 +- tests/models/test_relation_classifier.py | 12 +- tests/test_datasets_biomedical.py | 4 +- tests/test_labels.py | 38 ++- tests/test_tokenize_sentence.py | 4 +- 83 files changed, 1084 insertions(+), 1118 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b3633d93e..5633ac850f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,11 +12,11 @@ jobs: FLAIR_CACHE_ROOT: ./cache/flair steps: - uses: actions/checkout@v3 - - name: Set up Python 3.8 + - name: Set up Python 3.9 id: setup-python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - name: Install Torch cpu run: pip install torch --index-url https://download.pytorch.org/whl/cpu - name: Install Flair dependencies diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 24a424adba..f752b1324c 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -10,7 +10,7 @@ jobs: name: Build the docs using Sphinx and push to gh-pages runs-on: ubuntu-latest env: - python-version: 3.8 + python-version: 3.9 steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b40ddfe77e..b44927f17a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,8 +24,8 @@ the code should hopefully be easy. ### Setup -Flair requires python-3.8 or higher. To make sure your code also runs on the oldest supported -python version, it is recommended to use python-3.8.x for flair development. +Flair requires python-3.9 or higher. To make sure your code also runs on the oldest supported +python version, it is recommended to use python-3.9.x for flair development. Create a python environment of your preference and run: ```bash diff --git a/README.md b/README.md index 92502c972b..fdf4130124 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ In your favorite virtual environment, simply do: pip install flair ``` -Flair requires Python 3.8+. +Flair requires Python 3.9+. ### Example 1: Tag Entities in Text diff --git a/docs/contributing/local_development.md b/docs/contributing/local_development.md index 87439439f2..9c7413703e 100644 --- a/docs/contributing/local_development.md +++ b/docs/contributing/local_development.md @@ -6,8 +6,8 @@ the code should hopefully be easy. ## Setup -Flair requires python-3.8 or higher. To make sure our code also runs on the oldest supported -python version, it is recommended to use python-3.8.x for flair development. +Flair requires python-3.9 or higher. To make sure our code also runs on the oldest supported +python version, it is recommended to use python-3.9.x for flair development. Create a python environment of your preference and run: ```bash diff --git a/docs/tutorial/intro.md b/docs/tutorial/intro.md index e652583f76..b8af9b5667 100644 --- a/docs/tutorial/intro.md +++ b/docs/tutorial/intro.md @@ -16,7 +16,7 @@ In your favorite virtual environment, simply do: pip install flair ``` -Flair requires Python 3.8+. +Flair requires Python 3.9+. ## Example 1: Tag Entities in Text diff --git a/flair/class_utils.py b/flair/class_utils.py index 9aa95cd1ee..7e01f4ff42 100644 --- a/flair/class_utils.py +++ b/flair/class_utils.py @@ -1,12 +1,13 @@ import importlib import inspect +from collections.abc import Iterable from types import ModuleType -from typing import Any, Iterable, List, Optional, Type, TypeVar, Union, overload +from typing import Any, Optional, TypeVar, Union, overload T = TypeVar("T") -def get_non_abstract_subclasses(cls: Type[T]) -> Iterable[Type[T]]: +def get_non_abstract_subclasses(cls: type[T]) -> Iterable[type[T]]: for subclass in cls.__subclasses__(): yield from get_non_abstract_subclasses(subclass) if inspect.isabstract(subclass): @@ -14,7 +15,7 @@ def get_non_abstract_subclasses(cls: Type[T]) -> Iterable[Type[T]]: yield subclass -def get_state_subclass_by_name(cls: Type[T], cls_name: Optional[str]) -> Type[T]: +def get_state_subclass_by_name(cls: type[T], cls_name: Optional[str]) -> type[T]: for sub_cls in get_non_abstract_subclasses(cls): if sub_cls.__name__ == cls_name: return sub_cls @@ -26,12 +27,12 @@ def lazy_import(group: str, module: str, first_symbol: None) -> ModuleType: ... @overload -def lazy_import(group: str, module: str, first_symbol: str, *symbols: str) -> List[Any]: ... +def lazy_import(group: str, module: str, first_symbol: str, *symbols: str) -> list[Any]: ... def lazy_import( group: str, module: str, first_symbol: Optional[str] = None, *symbols: str -) -> Union[List[Any], ModuleType]: +) -> Union[list[Any], ModuleType]: try: imported_module = importlib.import_module(module) except ImportError: diff --git a/flair/data.py b/flair/data.py index 69d85baf92..56622b249c 100644 --- a/flair/data.py +++ b/flair/data.py @@ -4,10 +4,11 @@ import typing from abc import ABC, abstractmethod from collections import Counter, defaultdict +from collections.abc import Iterable from operator import itemgetter from os import PathLike from pathlib import Path -from typing import Any, DefaultDict, Dict, Iterable, List, NamedTuple, Optional, Tuple, Union, cast +from typing import Any, NamedTuple, Optional, Union, cast import torch from deprecated.sphinx import deprecated @@ -52,8 +53,8 @@ class Dictionary: def __init__(self, add_unk: bool = True) -> None: # init dictionaries - self.item2idx: Dict[bytes, int] = {} - self.idx2item: List[bytes] = [] + self.item2idx: dict[bytes, int] = {} + self.idx2item: list[bytes] = [] self.add_unk = add_unk self.multi_label = False self.span_labels = False @@ -101,7 +102,7 @@ def get_idx_for_item(self, item: str) -> int: ) raise IndexError - def get_idx_for_items(self, items: List[str]) -> List[int]: + def get_idx_for_items(self, items: list[str]) -> list[int]: """Returns the IDs for each item of the list of string, otherwise 0 if not found. Args: @@ -120,7 +121,7 @@ def get_idx_for_items(self, items: List[str]) -> List[int]: return [results] return list(results) - def get_items(self) -> List[str]: + def get_items(self) -> list[str]: items = [] for item in self.idx2item: items.append(item.decode("UTF-8")) @@ -151,7 +152,7 @@ def save(self, savefile: PathLike): mappings = {"idx2item": self.idx2item, "item2idx": self.item2idx} pickle.dump(mappings, f) - def __setstate__(self, d: Dict) -> None: + def __setstate__(self, d: dict) -> None: self.__dict__ = d # set 'add_unk' if the dictionary was created with a version of Flair older than 0.9 if "add_unk" not in self.__dict__: @@ -281,9 +282,9 @@ class DataPoint: """ def __init__(self) -> None: - self.annotation_layers: Dict[str, List[Label]] = {} - self._embeddings: Dict[str, torch.Tensor] = {} - self._metadata: Dict[str, Any] = {} + self.annotation_layers: dict[str, list[Label]] = {} + self._embeddings: dict[str, torch.Tensor] = {} + self._metadata: dict[str, Any] = {} @property @abstractmethod @@ -293,7 +294,7 @@ def embedding(self) -> torch.Tensor: def set_embedding(self, name: str, vector: torch.Tensor): self._embeddings[name] = vector - def get_embedding(self, names: Optional[List[str]] = None) -> torch.Tensor: + def get_embedding(self, names: Optional[list[str]] = None) -> torch.Tensor: # if one embedding name, directly return it if names and len(names) == 1: if names[0] in self._embeddings: @@ -308,7 +309,7 @@ def get_embedding(self, names: Optional[List[str]] = None) -> torch.Tensor: else: return torch.tensor([], device=flair.device) - def get_each_embedding(self, embedding_names: Optional[List[str]] = None) -> List[torch.Tensor]: + def get_each_embedding(self, embedding_names: Optional[list[str]] = None) -> list[torch.Tensor]: embeddings = [] for embed_name in sorted(self._embeddings.keys()): if embedding_names and embed_name not in embedding_names: @@ -325,7 +326,7 @@ def to(self, device: str, pin_memory: bool = False) -> None: else: self._embeddings[name] = vector.to(device, non_blocking=True) - def clear_embeddings(self, embedding_names: Optional[List[str]] = None) -> None: + def clear_embeddings(self, embedding_names: Optional[list[str]] = None) -> None: if embedding_names is None: self._embeddings = {} else: @@ -368,14 +369,14 @@ def get_label(self, label_type: Optional[str] = None, zero_tag_value: str = "O") return Label(self, zero_tag_value) return self.get_labels(label_type)[0] - def get_labels(self, typename: Optional[str] = None) -> List[Label]: + def get_labels(self, typename: Optional[str] = None) -> list[Label]: if typename is None: return self.labels return self.annotation_layers.get(typename, []) @property - def labels(self) -> List[Label]: + def labels(self) -> list[Label]: all_labels = [] for key in self.annotation_layers: all_labels.extend(self.annotation_layers[key]) @@ -447,8 +448,8 @@ def __init__( concept_id: str, concept_name: str, database_name: str, - additional_ids: Optional[List[str]] = None, - synonyms: Optional[List[str]] = None, + additional_ids: Optional[list[str]] = None, + synonyms: Optional[list[str]] = None, description: Optional[str] = None, ): """A Concept as part of a knowledgebase or ontology. @@ -483,7 +484,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return str(self) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "concept_id": self.concept_id, "concept_name": self.concept_name, @@ -550,8 +551,8 @@ def __init__( self._start_position = start_position - self._embeddings: Dict = {} - self.tags_proba_dist: Dict[str, List[Label]] = {} + self._embeddings: dict[str, torch.Tensor] = {} + self.tags_proba_dist: dict[str, list[Label]] = {} @property def idx(self) -> int: @@ -568,10 +569,10 @@ def text(self) -> str: def unlabeled_identifier(self) -> str: return f'Token[{self.idx - 1}]: "{self.text}"' - def add_tags_proba_dist(self, tag_type: str, tags: List[Label]) -> None: + def add_tags_proba_dist(self, tag_type: str, tags: list[Label]) -> None: self.tags_proba_dist[tag_type] = tags - def get_tags_proba_dist(self, tag_type: str) -> List[Label]: + def get_tags_proba_dist(self, tag_type: str) -> list[Label]: if tag_type in self.tags_proba_dist: return self.tags_proba_dist[tag_type] return [] @@ -617,7 +618,7 @@ def set_label(self, typename: str, value: str, score: float = 1.0, **metadata): else: DataPoint.set_label(self, typename=typename, value=value, score=score, **metadata) - def to_dict(self, tag_type: Optional[str] = None) -> Dict[str, Any]: + def to_dict(self, tag_type: Optional[str] = None) -> dict[str, Any]: return { "text": self.text, "start_pos": self.start_position, @@ -629,7 +630,7 @@ def to_dict(self, tag_type: Optional[str] = None) -> Dict[str, Any]: class Span(_PartOfSentence): """This class represents one textual span consisting of Tokens.""" - def __new__(self, tokens: List[Token]): + def __new__(self, tokens: list[Token]): # check if the span already exists. If so, return it unlabeled_identifier = self._make_unlabeled_identifier(tokens) if unlabeled_identifier in tokens[0].sentence._known_spans: @@ -643,7 +644,7 @@ def __new__(self, tokens: List[Token]): tokens[0].sentence._known_spans[unlabeled_identifier] = span return span - def __init__(self, tokens: List[Token]) -> None: + def __init__(self, tokens: list[Token]) -> None: if not self.initialized: super().__init__(tokens[0].sentence) self.tokens = tokens @@ -662,7 +663,7 @@ def text(self) -> str: return "".join([t.text + t.whitespace_after * " " for t in self.tokens]).strip() @staticmethod - def _make_unlabeled_identifier(tokens: List[Token]): + def _make_unlabeled_identifier(tokens: list[Token]): text = "".join([t.text + t.whitespace_after * " " for t in tokens]).strip() return f'Span[{tokens[0].idx - 1}:{tokens[-1].idx}]: "{text}"' @@ -769,7 +770,7 @@ class Sentence(DataPoint): def __init__( self, - text: Union[str, List[str], List[Token]], + text: Union[str, list[str], list[Token]], use_tokenizer: Union[bool, Tokenizer] = True, language_code: Optional[str] = None, start_position: int = 0, @@ -790,10 +791,10 @@ def __init__( """ super().__init__() - self.tokens: List[Token] = [] + self.tokens: list[Token] = [] # private field for all known spans - self._known_spans: Dict[str, _PartOfSentence] = {} + self._known_spans: dict[str, _PartOfSentence] = {} self.language_code: Optional[str] = language_code @@ -818,7 +819,7 @@ def __init__( self._previous_sentence: Optional[Sentence] = None self._has_context: bool = False self._next_sentence: Optional[Sentence] = None - self._position_in_dataset: Optional[typing.Tuple[Dataset, int]] = None + self._position_in_dataset: Optional[tuple[Dataset, int]] = None # if text is passed, instantiate sentence with tokens (words) if isinstance(text, str): @@ -830,7 +831,7 @@ def __init__( self.tokens[-1].whitespace_after = 0 return else: - words = cast(List[str], text) + words = cast(list[str], text) text = " ".join(words) # determine token positions and whitespace_after flag @@ -861,15 +862,15 @@ def __init__( def unlabeled_identifier(self): return f'Sentence[{len(self)}]: "{self.text}"' - def get_relations(self, label_type: Optional[str] = None) -> List[Relation]: - relations: List[Relation] = [] + def get_relations(self, label_type: Optional[str] = None) -> list[Relation]: + relations: list[Relation] = [] for label in self.get_labels(label_type): if isinstance(label.data_point, Relation): relations.append(label.data_point) return relations - def get_spans(self, label_type: Optional[str] = None) -> List[Span]: - spans: List[Span] = [] + def get_spans(self, label_type: Optional[str] = None) -> list[Span]: + spans: list[Span] = [] for potential_span in self._known_spans.values(): if isinstance(potential_span, Span) and (label_type is None or potential_span.has_label(label_type)): spans.append(potential_span) @@ -922,16 +923,16 @@ def to(self, device: str, pin_memory: bool = False): for token in self: token.to(device, pin_memory) - def clear_embeddings(self, embedding_names: Optional[List[str]] = None): + def clear_embeddings(self, embedding_names: Optional[list[str]] = None): super().clear_embeddings(embedding_names) # clear token embeddings for token in self: token.clear_embeddings(embedding_names) - def left_context(self, context_length: int, respect_document_boundaries: bool = True) -> List[Token]: + def left_context(self, context_length: int, respect_document_boundaries: bool = True) -> list[Token]: sentence = self - left_context: List[Token] = [] + left_context: list[Token] = [] while len(left_context) < context_length: sentence = sentence.previous_sentence() if sentence is None: @@ -943,9 +944,9 @@ def left_context(self, context_length: int, respect_document_boundaries: bool = left_context = sentence.tokens + left_context return left_context[-context_length:] - def right_context(self, context_length: int, respect_document_boundaries: bool = True) -> List[Token]: + def right_context(self, context_length: int, respect_document_boundaries: bool = True) -> list[Token]: sentence = self - right_context: List[Token] = [] + right_context: list[Token] = [] while len(right_context) < context_length: sentence = sentence.next_sentence() if sentence is None: @@ -1037,7 +1038,7 @@ def to_original_text(self) -> str: [t.text + t.whitespace_after * " " for t in self.tokens] ).strip() - def to_dict(self, tag_type: Optional[str] = None) -> Dict[str, Any]: + def to_dict(self, tag_type: Optional[str] = None) -> dict[str, Any]: return { "text": self.to_original_text(), "labels": [label.to_dict() for label in self.get_labels(tag_type) if label.data_point is self], @@ -1180,7 +1181,7 @@ def copy_context_from_sentence(self, sentence: "Sentence") -> None: self._position_in_dataset = sentence._position_in_dataset @classmethod - def set_context_for_sentences(cls, sentences: List["Sentence"]) -> None: + def set_context_for_sentences(cls, sentences: list["Sentence"]) -> None: previous_sentence = None for sentence in sentences: if sentence.is_context_set(): @@ -1231,7 +1232,7 @@ def to(self, device: str, pin_memory: bool = False): self.first.to(device, pin_memory) self.second.to(device, pin_memory) - def clear_embeddings(self, embedding_names: Optional[List[str]] = None): + def clear_embeddings(self, embedding_names: Optional[list[str]] = None): self.first.clear_embeddings(embedding_names) self.second.clear_embeddings(embedding_names) if self.concatenated_data is not None: @@ -1276,7 +1277,7 @@ def to(self, device: str, pin_memory: bool = False): self.second.to(device, pin_memory) self.third.to(device, pin_memory) - def clear_embeddings(self, embedding_names: Optional[List[str]] = None): + def clear_embeddings(self, embedding_names: Optional[list[str]] = None): self.first.clear_embeddings(embedding_names) self.second.clear_embeddings(embedding_names) self.third.clear_embeddings(embedding_names) @@ -1313,7 +1314,7 @@ def __init__(self, data=None, imageURL=None): super().__init__() self.data = data - self._embeddings: Dict = {} + self._embeddings: dict[str, torch.Tensor] = {} self.imageURL = imageURL @property @@ -1497,17 +1498,17 @@ def make_vocab_dictionary(self, max_tokens: int = -1, min_freq: int = 1) -> Dict return vocab_dictionary - def _get_most_common_tokens(self, max_tokens: int, min_freq: int) -> List[str]: + def _get_most_common_tokens(self, max_tokens: int, min_freq: int) -> list[str]: tokens_and_frequencies = Counter(self._get_all_tokens()) - tokens: List[str] = [] + tokens: list[str] = [] for token, freq in tokens_and_frequencies.most_common(): if (min_freq != -1 and freq < min_freq) or (max_tokens != -1 and len(tokens) == max_tokens): break tokens.append(token) return tokens - def _get_all_tokens(self) -> List[str]: + def _get_all_tokens(self) -> list[str]: assert self.train tokens = [s.tokens for s in _iter_dataset(self.train)] tokens = [token for sublist in tokens for token in sublist] @@ -1544,13 +1545,8 @@ def _obtain_statistics_for(sentences, name, tag_type) -> dict: tags_to_count = Corpus._count_token_labels(sentences, tag_type) tokens_per_sentence = Corpus._get_tokens_per_sentence(sentences) - label_size_dict = {} - for label, c in classes_to_count.items(): - label_size_dict[label] = c - - tag_size_dict = {} - for tag, c in tags_to_count.items(): - tag_size_dict[tag] = c + label_size_dict = dict(classes_to_count) + tag_size_dict = dict(tags_to_count) return { "dataset": name, @@ -1566,20 +1562,20 @@ def _obtain_statistics_for(sentences, name, tag_type) -> dict: } @staticmethod - def _get_tokens_per_sentence(sentences: Iterable[Sentence]) -> List[int]: + def _get_tokens_per_sentence(sentences: Iterable[Sentence]) -> list[int]: return [len(x.tokens) for x in sentences] @staticmethod - def _count_sentence_labels(sentences: Iterable[Sentence]) -> DefaultDict[str, int]: - label_count: DefaultDict[str, int] = defaultdict(lambda: 0) + def _count_sentence_labels(sentences: Iterable[Sentence]) -> defaultdict[str, int]: + label_count: defaultdict[str, int] = defaultdict(lambda: 0) for sent in sentences: for label in sent.labels: label_count[label.value] += 1 return label_count @staticmethod - def _count_token_labels(sentences: Iterable[Sentence], label_type: str) -> DefaultDict[str, int]: - label_count: DefaultDict[str, int] = defaultdict(lambda: 0) + def _count_token_labels(sentences: Iterable[Sentence], label_type: str) -> defaultdict[str, int]: + label_count: defaultdict[str, int] = defaultdict(lambda: 0) for sent in sentences: for token in sent.tokens: if label_type in token.annotation_layers: @@ -1623,7 +1619,7 @@ def make_label_dictionary( sentence_label_type_counter: typing.Counter[str] = Counter() label_value_counter: typing.Counter[str] = Counter() - all_sentence_labels: List[str] = [] + all_sentence_labels: list[str] = [] # first, determine the datapoint type by going through dataset until first label is found datapoint_type = None @@ -1687,10 +1683,10 @@ def make_label_dictionary( def add_label_noise( self, label_type: str, - labels: List[str], + labels: list[str], noise_share: float = 0.2, split: str = "train", - noise_transition_matrix: Optional[Dict[str, List[float]]] = None, + noise_transition_matrix: Optional[dict[str, list[float]]] = None, ): """Generates uniform label noise distribution in the chosen dataset split. @@ -1817,12 +1813,12 @@ def make_tag_dictionary(self, tag_type: str) -> Dictionary: class MultiCorpus(Corpus): def __init__( self, - corpora: List[Corpus], - task_ids: Optional[List[str]] = None, + corpora: list[Corpus], + task_ids: Optional[list[str]] = None, name: str = "multicorpus", **corpusargs, ) -> None: - self.corpora: List[Corpus] = corpora + self.corpora: list[Corpus] = corpora ids = task_ids if task_ids else [f"Task_{i}" for i in range(len(corpora))] @@ -1871,8 +1867,8 @@ class ConcatFlairDataset(Dataset): datasets (sequence): List of datasets to be concatenated """ - datasets: List[Dataset] - cumulative_sizes: List[int] + datasets: list[Dataset] + cumulative_sizes: list[int] @staticmethod def cumsum(sequence): @@ -1907,36 +1903,13 @@ def __getitem__(self, idx: int) -> Sentence: return sentence @property - def cummulative_sizes(self) -> List[int]: + def cummulative_sizes(self) -> list[int]: return self.cumulative_sizes -def iob2(tags: List) -> bool: - """Converts the tags to the IOB2 format. - - Check that tags have a valid IOB format. - Tags in IOB1 format are converted to IOB2. - """ - for i, tag in enumerate(tags): - if tag.value == "O": - continue - split = tag.value.split("-") - if len(split) != 2 or split[0] not in ["I", "B"]: - return False - if split[0] == "B": - continue - elif i == 0 or tags[i - 1].value == "O": # conversion IOB1 to IOB2 - tags[i].value = "B" + tag.value[1:] - elif tags[i - 1].value[1:] == tag.value[1:]: - continue - else: # conversion IOB1 to IOB2 - tags[i].value = "B" + tag.value[1:] - return True - - def randomly_split_into_two_datasets( dataset: Dataset, length_of_first: int, random_seed: Optional[int] = None -) -> Tuple[Subset, Subset]: +) -> tuple[Subset, Subset]: """Shuffles a dataset and splits into two subsets. The length of the first is specified and the remaining samples go into the second subset. @@ -1959,17 +1932,17 @@ def randomly_split_into_two_datasets( def get_spans_from_bio( - bioes_tags: List[str], bioes_scores: Optional[List[float]] = None -) -> List[typing.Tuple[List[int], float, str]]: + bioes_tags: list[str], bioes_scores: Optional[list[float]] = None +) -> list[tuple[list[int], float, str]]: # add a dummy "O" to close final prediction bioes_tags.append("O") # return complex list found_spans = [] # internal variables - current_tag_weights: Dict[str, float] = {} + current_tag_weights: dict[str, float] = {} previous_tag = "O-" - current_span: List[int] = [] - current_span_scores: List[float] = [] + current_span: list[int] = [] + current_span_scores: list[float] = [] for idx, bioes_tag in enumerate(bioes_tags): # non-set tags are OUT tags if bioes_tag == "" or bioes_tag == "O" or bioes_tag == "_": diff --git a/flair/datasets/base.py b/flair/datasets/base.py index a38d0b1321..0737b4660d 100644 --- a/flair/datasets/base.py +++ b/flair/datasets/base.py @@ -1,7 +1,7 @@ import logging from abc import abstractmethod from pathlib import Path -from typing import Generic, List, Optional, Union +from typing import Generic, Optional, Union import torch.utils.data.dataloader from deprecated.sphinx import deprecated @@ -41,7 +41,7 @@ def __init__( class FlairDatapointDataset(FlairDataset, Generic[DT]): """A simple Dataset object to wrap a List of Datapoints, for example Sentences.""" - def __init__(self, datapoints: Union[DT, List[DT]]) -> None: + def __init__(self, datapoints: Union[DT, list[DT]]) -> None: """Instantiate FlairDatapointDataset. Args: @@ -64,7 +64,7 @@ def __getitem__(self, index: int = 0) -> DT: class SentenceDataset(FlairDatapointDataset): @deprecated(version="0.11", reason="The 'SentenceDataset' class was renamed to 'FlairDatapointDataset'") - def __init__(self, sentences: Union[Sentence, List[Sentence]]) -> None: + def __init__(self, sentences: Union[Sentence, list[Sentence]]) -> None: super().__init__(sentences) @@ -73,7 +73,7 @@ class StringDataset(FlairDataset): def __init__( self, - texts: Union[str, List[str]], + texts: Union[str, list[str]], use_tokenizer: Union[bool, Tokenizer] = SpaceTokenizer(), ) -> None: """Instantiate StringDataset. @@ -111,7 +111,7 @@ def __init__( database: str, collection: str, text_field: str, - categories_field: Optional[List[str]] = None, + categories_field: Optional[list[str]] = None, max_tokens_per_doc: int = -1, max_chars_per_doc: int = -1, tokenizer: Tokenizer = SegtokTokenizer(), @@ -195,7 +195,7 @@ def __init__( def _parse_document_to_sentence( self, text: str, - labels: List[str], + labels: list[str], tokenizer: Union[bool, Tokenizer], ): if self.max_chars_per_doc > 0: diff --git a/flair/datasets/biomedical.py b/flair/datasets/biomedical.py index 28f4aca98b..e99a71ccf7 100644 --- a/flair/datasets/biomedical.py +++ b/flair/datasets/biomedical.py @@ -7,6 +7,7 @@ import sys from abc import ABC, abstractmethod from collections import defaultdict, deque +from collections.abc import Iterable from copy import copy from operator import attrgetter from pathlib import Path @@ -18,7 +19,7 @@ StreamError, TarError, ) -from typing import Dict, Iterable, List, NamedTuple, Optional, Tuple, Union +from typing import NamedTuple, Optional, Union from zipfile import BadZipFile, LargeZipFile import ftfy @@ -56,7 +57,7 @@ class Entity: text as well as the type of entity (e.g. Chemical, Gene, and so on). """ - def __init__(self, char_span: Tuple[int, int], entity_type: str) -> None: + def __init__(self, char_span: tuple[int, int], entity_type: str) -> None: assert char_span[0] < char_span[1] self.char_span = range(*char_span) self.type = entity_type @@ -98,9 +99,9 @@ class InternalBioNerDataset: def __init__( self, - documents: Dict[str, str], - entities_per_document: Dict[str, List[Entity]], - entity_types: List[str] = [], + documents: dict[str, str], + entities_per_document: dict[str, list[Entity]], + entity_types: list[str] = [], ): self.documents = documents self.entities_per_document = entities_per_document @@ -134,7 +135,7 @@ def merge_datasets(data_sets: Iterable[InternalBioNerDataset]): def filter_and_map_entities( - dataset: InternalBioNerDataset, entity_type_to_canonical: Dict[str, str] + dataset: InternalBioNerDataset, entity_type_to_canonical: dict[str, str] ) -> InternalBioNerDataset: mapped_entities_per_document = {} entity_types = list(entity_type_to_canonical.values()) @@ -223,7 +224,7 @@ def bioc_to_internal(bioc_file: Path): for document in Tqdm.tqdm(documents, desc="Converting to internal"): document_id = document.xpath("./id")[0].text - texts: List[str] = [] + texts: list[str] = [] entities = [] for passage in document.xpath("passage"): @@ -358,7 +359,7 @@ def __init__( """ self.sentence_splitter = sentence_splitter - def process_dataset(self, datasets: Dict[str, InternalBioNerDataset], out_dir: Path): + def process_dataset(self, datasets: dict[str, InternalBioNerDataset], out_dir: Path): if "train" in datasets: self.write_to_conll(datasets["train"], out_dir / (self.sentence_splitter.name + "_train.conll")) if "dev" in datasets: @@ -450,7 +451,7 @@ def to_internal(self, data_folder: Path) -> InternalBioNerDataset: @staticmethod @abstractmethod - def split_url() -> Union[str, List[str]]: + def split_url() -> Union[str, list[str]]: raise NotImplementedError def get_corpus_sentence_splitter(self) -> Optional[SentenceSplitter]: @@ -596,8 +597,8 @@ def download_dataset(cls, data_dir: Path) -> Path: @classmethod def parse_dataset(cls, original_file: Path): - documents: Dict[str, str] = {} - entities_per_document: Dict[str, List[Entity]] = {} + documents: dict[str, str] = {} + entities_per_document: dict[str, list[Entity]] = {} tree = etree.parse(str(original_file)) sentence_elems = tree.xpath("//sentence") @@ -647,7 +648,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return merge_datasets([train_data, test_data]) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -726,14 +727,14 @@ def download_and_prepare_test(cls, data_folder: Path, sentence_tag: str) -> Inte @classmethod def read_file(cls, input_iob_file: Path, sentence_tag: str) -> InternalBioNerDataset: - documents: Dict[str, str] = {} - entities_per_document: Dict[str, List[Entity]] = defaultdict(list) + documents: dict[str, str] = {} + entities_per_document: dict[str, list[Entity]] = defaultdict(list) with open(str(input_iob_file), encoding="utf8") as file_reader: document_id: Optional[str] = None document_text: Optional[str] = None - entities: List[Entity] = [] + entities: list[Entity] = [] entity_type: Optional[str] = None entity_start = 0 @@ -818,7 +819,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return merge_datasets([train_data, test_data]) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -994,7 +995,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @staticmethod - def split_url() -> List[str]: + def split_url() -> list[str]: split_urls = [ "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/cellfinder_cellline", "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/cellfinder_species", @@ -1009,7 +1010,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -1176,7 +1177,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return merge_datasets([train_data, test_data]) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -1566,7 +1567,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(dataset, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -1747,8 +1748,8 @@ def download_dataset(data_dir: Path): @classmethod def parse_dataset(cls, original_file: Path): - documents: Dict[str, str] = {} - entities_per_document: Dict[str, List[Entity]] = {} + documents: dict[str, str] = {} + entities_per_document: dict[str, list[Entity]] = {} tree = etree.parse(str(original_file)) document_elems = tree.xpath("//document") @@ -1905,7 +1906,7 @@ def split_url() -> str: def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return LINNEAUS.download_and_parse_dataset(data_dir) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -1995,7 +1996,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return all_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2021,7 +2022,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return all_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2033,7 +2034,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @staticmethod - def split_url() -> List[str]: + def split_url() -> list[str]: split_urls = [ "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/CDRDisease", "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/CDRChem", @@ -2052,7 +2053,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return all_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2167,7 +2168,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return all_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2190,7 +2191,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return all_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2213,7 +2214,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return all_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2230,7 +2231,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @staticmethod - def split_url() -> List[str]: + def split_url() -> list[str]: split_urls = [ "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/variome_gene", "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/variome_disease", @@ -2247,7 +2248,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return all_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2343,7 +2344,7 @@ def parse_input_file(input_file: Path): with open(str(input_file), encoding="utf8") as file: document_id = "" document_text = "" - entities: List[Entity] = [] + entities: list[Entity] = [] c = 1 for line in file: @@ -2406,7 +2407,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return merge_datasets([train_data, dev_data, test_data]) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2455,13 +2456,13 @@ def download_corpus(self, data_folder: Path) -> Path: @staticmethod def parse_input_file(input_file: Path): - documents: Dict[str, str] = {} - entities_per_document: Dict[str, List[Entity]] = {} + documents: dict[str, str] = {} + entities_per_document: dict[str, list[Entity]] = {} with open(str(input_file), encoding="iso-8859-1") as file: document_id = None document_text = "" - entities: List[Entity] = [] + entities: list[Entity] = [] entity_type = None entity_start = 0 @@ -2584,7 +2585,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(corpus, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2605,7 +2606,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(corpus, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2628,7 +2629,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @staticmethod - def split_url() -> List[str]: + def split_url() -> list[str]: split_urls = [ "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/scai_chemicals", "https://raw.githubusercontent.com/hu-ner/huner/master/ner_scripts/splits/scai_disease", @@ -2641,7 +2642,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(corpus, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2763,7 +2764,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(corpus, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2863,7 +2864,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -2945,7 +2946,7 @@ def download_dev_corpus(cls, data_dir) -> Path: @staticmethod def parse_input_file(text_file: Path, ann_file: Path) -> InternalBioNerDataset: documents = {} - entities_per_document: Dict[str, List[Entity]] = {} + entities_per_document: dict[str, list[Entity]] = {} document_title_length = {} @@ -3010,7 +3011,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return merge_datasets([train_data, dev_data]) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -3071,8 +3072,8 @@ def download_corpus(cls, data_dir: Path) -> Path: @staticmethod def parse_corpus(text_dir: Path, gold_file: Path) -> InternalBioNerDataset: - documents: Dict[str, str] = {} - entities_per_document: Dict[str, List[Entity]] = {} + documents: dict[str, str] = {} + entities_per_document: dict[str, list[Entity]] = {} text_files = [file for file in os.listdir(str(text_dir)) if not file.startswith(".")] @@ -3122,7 +3123,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return DECA.parse_corpus(text_dir, gold_file) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -3221,7 +3222,7 @@ def parse_corpus(corpus_dir: Path, sentence_separator: str) -> InternalBioNerDat akt_pos += len(words[i]) + 1 sentences += [tmp_sentence] - pre_entities: List[List[Tuple[int, int, str]]] = [[] for _ in sentences] + pre_entities: list[list[tuple[int, int, str]]] = [[] for _ in sentences] for protein in protein_tree: for span in protein.get("span").split(","): start = word_to_id[span.split("..")[0]] @@ -3287,7 +3288,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(corpus, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -3450,8 +3451,8 @@ def parse_dataset(data_dir: Path) -> InternalBioNerDataset: ] text_files = sorted(text_files) - documents: Dict[str, str] = {} - entities_per_document: Dict[str, List[Entity]] = {} + documents: dict[str, str] = {} + entities_per_document: dict[str, list[Entity]] = {} for text_file in sorted(text_files): document_id = os.path.basename(text_file).split("_")[0] @@ -3590,7 +3591,7 @@ def parse_test_dataset(cls, data_folder: Path) -> InternalBioNerDataset: @staticmethod def parse_dataset(text_file: Path, ann_file: Path) -> InternalBioNerDataset: documents = {} - entities_per_document: Dict[str, List[Entity]] = {} + entities_per_document: dict[str, list[Entity]] = {} with open(str(text_file), encoding="utf8") as text_file_reader: for line in text_file_reader: @@ -3733,7 +3734,7 @@ def download_dev_corpus(cls, data_dir) -> Path: @staticmethod def parse_input_file(text_file: Path, ann_file: Path) -> InternalBioNerDataset: documents = {} - entities_per_document: Dict[str, List[Entity]] = {} + entities_per_document: dict[str, list[Entity]] = {} document_abstract_length = {} with open(str(text_file), encoding="utf8") as text_reader: @@ -3806,7 +3807,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: dataset = merge_datasets([train_data, dev_data]) return filter_and_map_entities(dataset, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -3945,7 +3946,7 @@ def to_internal(self, data_dir: Path, annotator: int = 0) -> InternalBioNerDatas dataset = CHEBI.parse_dataset(corpus_dir, annotator=annotator) return filter_and_map_entities(dataset, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -4038,7 +4039,7 @@ def __init__( @staticmethod @abstractmethod - def download_corpus(data_folder: Path) -> Tuple[Path, Path, Path]: + def download_corpus(data_folder: Path) -> tuple[Path, Path, Path]: pass @staticmethod @@ -4083,7 +4084,7 @@ class BIONLP2013_PC(BioNLPCorpus): """ @staticmethod - def download_corpus(download_folder: Path) -> Tuple[Path, Path, Path]: + def download_corpus(download_folder: Path) -> tuple[Path, Path, Path]: train_url = "http://2013.bionlp-st.org/tasks/BioNLP-ST_2013_PC_training_data.tar.gz" dev_url = "http://2013.bionlp-st.org/tasks/BioNLP-ST_2013_PC_development_data.tar.gz" test_url = "http://2013.bionlp-st.org/tasks/BioNLP-ST_2013_PC_test_data.tar.gz" @@ -4125,7 +4126,7 @@ class BIONLP2013_CG(BioNLPCorpus): """ @staticmethod - def download_corpus(download_folder: Path) -> Tuple[Path, Path, Path]: + def download_corpus(download_folder: Path) -> tuple[Path, Path, Path]: url = "https://github.com/openbiocorpora/bionlp-st-2013-cg/archive/refs/heads/master.zip" cached_path(url, download_folder) @@ -4292,9 +4293,10 @@ def download_corpora(download_dir: Path): @staticmethod def convert_and_write(download_folder, data_folder, tag_type): data_folder.mkdir(parents=True, exist_ok=True) - with (download_folder / "train.tsv").open(encoding="utf8") as f_in, (data_folder / "train.conll").open( - "w", encoding="utf8" - ) as f_out: + with ( + (download_folder / "train.tsv").open(encoding="utf8") as f_in, + (data_folder / "train.conll").open("w", encoding="utf8") as f_out, + ): for line in f_in: if not line.strip(): f_out.write("\n") @@ -4305,9 +4307,10 @@ def convert_and_write(download_folder, data_folder, tag_type): tag = tag + "-" + tag_type f_out.write(f"{token} {tag}\n") - with (download_folder / "devel.tsv").open(encoding="utf8") as f_in, (data_folder / "dev.conll").open( - "w", encoding="utf8" - ) as f_out: + with ( + (download_folder / "devel.tsv").open(encoding="utf8") as f_in, + (data_folder / "dev.conll").open("w", encoding="utf8") as f_out, + ): for line in f_in: if not line.strip(): f_out.write("\n") @@ -4317,9 +4320,10 @@ def convert_and_write(download_folder, data_folder, tag_type): tag = tag + "-" + tag_type f_out.write(f"{token} {tag}\n") - with (download_folder / "test.tsv").open(encoding="utf8") as f_in, (data_folder / "test.conll").open( - "w", encoding="utf8" - ) as f_out: + with ( + (download_folder / "test.tsv").open(encoding="utf8") as f_in, + (data_folder / "test.conll").open("w", encoding="utf8") as f_out, + ): for line in f_in: if not line.strip(): f_out.write("\n") @@ -4638,7 +4642,7 @@ def download_corpus(cls, data_dir: Path) -> Path: @staticmethod def prepare_splits( data_dir: Path, corpus: InternalBioNerDataset - ) -> Tuple[InternalBioNerDataset, InternalBioNerDataset, InternalBioNerDataset]: + ) -> tuple[InternalBioNerDataset, InternalBioNerDataset, InternalBioNerDataset]: splits_dir = data_dir / "splits" os.makedirs(str(splits_dir), exist_ok=True) @@ -4734,7 +4738,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(corpus, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -4792,7 +4796,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return filter_and_map_entities(corpus, self.entity_type_mapping) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -4896,7 +4900,7 @@ def parse_corpus(input_file: Path) -> InternalBioNerDataset: prev_sentence_id: Optional[str] = None document_text: Optional[str] = None - entities: List[Entity] = [] + entities: list[Entity] = [] offset: Optional[int] = None for line in azdz_reader: @@ -5014,7 +5018,7 @@ def to_internal(self, data_dir: Path) -> InternalBioNerDataset: return corpus_data - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return self.entity_type_mapping @@ -5221,7 +5225,7 @@ def __init__( sample_missing_splits=True, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: """Return the mapping of entity type given in the dataset to canonical types. Note, if a entity type is not present in the map it is discarded. @@ -5279,8 +5283,8 @@ def build_corpus_directory_name(self, dataset_name: str) -> str: def to_internal_dataset(self, dataset, split: str) -> InternalBioNerDataset: """Converts a dataset given in hugging datasets format to our internal corpus representation.""" - id_to_text: Dict[str, str] = {} - id_to_entities: Dict[str, list] = {} + id_to_text: dict[str, str] = {} + id_to_entities: dict[str, list] = {} entity_type_set = set() for document in dataset[split]: document_id = document["document_id"] @@ -5331,10 +5335,10 @@ def to_internal_dataset(self, dataset, split: str) -> InternalBioNerDataset: def bin_search_passage( self, - passages: List[Tuple[str, List[Tuple[int, int]]]], + passages: list[tuple[str, list[tuple[int, int]]]], low: int, high: int, - entity: Dict, + entity: dict, ): """Helper methods to find the passage to a given entity mention (incl. offset). @@ -5381,7 +5385,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return { "Gene": GENE_TAG, "GENERIF": GENE_TAG, @@ -5414,7 +5418,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"GENE-N": GENE_TAG, "GENE-Y": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5441,7 +5445,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"CHEMICAL": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5452,7 +5456,7 @@ class HUNER_ALL_DRUGPROT(BIGBIO_NER_CORPUS): def __init__(self, *args, **kwargs): super().__init__(*args, dataset_name="drugprot", **kwargs) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"GENE-N": GENE_TAG, "GENE-Y": GENE_TAG, "CHEMICAL": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5479,7 +5483,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"GeneOrGeneProduct": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5506,7 +5510,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"ChemicalEntity": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5533,7 +5537,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"DiseaseOrPhenotypicFeature": DISEASE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5560,7 +5564,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"OrganismTaxon": SPECIES_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5587,7 +5591,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"CellLine": CELL_LINE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5598,7 +5602,7 @@ class HUNER_ALL_BIORED(BIGBIO_NER_CORPUS): def __init__(self, *args, **kwargs): super().__init__(*args, dataset_name="biored", **kwargs) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return { "GeneOrGeneProduct": GENE_TAG, "ChemicalEntity": CHEMICAL_TAG, @@ -5631,7 +5635,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"protein": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5658,7 +5662,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"compound": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5669,7 +5673,7 @@ class HUNER_ALL_CPI(BIGBIO_NER_CORPUS): def __init__(self, *args, **kwargs): super().__init__(*args, dataset_name="cpi", **kwargs) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"protein": GENE_TAG, "compound": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5696,7 +5700,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Gene_or_gene_product": GENE_TAG, "Complex": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5723,7 +5727,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Simple_chemical": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5734,7 +5738,7 @@ class HUNER_ALL_BIONLP_ST_2013_PC(BIGBIO_NER_CORPUS): def __init__(self, *args, **kwargs): super().__init__(*args, dataset_name="bionlp_st_2013_pc", **kwargs) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return { "Gene_or_gene_product": GENE_TAG, "Complex": GENE_TAG, @@ -5765,7 +5769,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"protein": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5792,7 +5796,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Protein": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5819,7 +5823,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Protein": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5846,7 +5850,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Chemical": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5873,7 +5877,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Organism": SPECIES_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5884,7 +5888,7 @@ class HUNER_ALL_BIONLP_ST_2011_ID(BIGBIO_NER_CORPUS): def __init__(self, *args, **kwargs): super().__init__(*args, dataset_name="bionlp_st_2011_id", **kwargs) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return { "Protein": GENE_TAG, "Chemical": CHEMICAL_TAG, @@ -5915,7 +5919,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Protein": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5942,7 +5946,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Protein": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5969,7 +5973,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Microorganism": SPECIES_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -5996,7 +6000,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"gene": GENE_TAG, "protein": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -6023,7 +6027,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"chemical": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -6050,7 +6054,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"species": SPECIES_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -6077,7 +6081,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: # TODO whether cell or cell line is the correct tag return {"cellline": CELL_LINE_TAG} @@ -6089,7 +6093,7 @@ class HUNER_ALL_BIOID(BIGBIO_NER_CORPUS): def __init__(self, *args, **kwargs): super().__init__(*args, dataset_name="bioid", **kwargs) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: # TODO whether cell or cell line is the correct tag return { "gene": GENE_TAG, @@ -6123,7 +6127,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Gene": GENE_TAG, "FamilyName": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -6155,7 +6159,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"progene_text": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -6182,7 +6186,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Chemical": CHEMICAL_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -6209,7 +6213,7 @@ def __init__( test_split_name=test_split_name, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Gene": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: @@ -6224,7 +6228,7 @@ def __init__(self, *args, **kwargs): **kwargs, ) - def get_entity_type_mapping(self) -> Optional[Dict]: + def get_entity_type_mapping(self) -> Optional[dict]: return {"Gene": GENE_TAG} def build_corpus_directory_name(self, dataset_name: str) -> str: diff --git a/flair/datasets/document_classification.py b/flair/datasets/document_classification.py index 0bbc471818..363c84e561 100644 --- a/flair/datasets/document_classification.py +++ b/flair/datasets/document_classification.py @@ -2,8 +2,9 @@ import json import logging import os +import tarfile from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Optional, Union import flair from flair.data import ( @@ -36,8 +37,8 @@ def __init__( filter_if_longer_than: int = -1, tokenizer: Union[bool, Tokenizer] = SegtokTokenizer(), memory_mode: str = "partial", - label_name_map: Optional[Dict[str, str]] = None, - skip_labels: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + skip_labels: Optional[list[str]] = None, allow_examples_without_labels=False, sample_missing_splits: bool = True, encoding: str = "utf-8", @@ -131,8 +132,8 @@ def __init__( filter_if_longer_than: int = -1, tokenizer: Union[bool, Tokenizer] = SegtokTokenizer(), memory_mode: str = "partial", - label_name_map: Optional[Dict[str, str]] = None, - skip_labels: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + skip_labels: Optional[list[str]] = None, allow_examples_without_labels=False, encoding: str = "utf-8", ) -> None: @@ -277,11 +278,7 @@ def _parse_line_to_sentence(self, line: str, label_prefix: str, tokenizer: Union return None def is_in_memory(self) -> bool: - if self.memory_mode == "disk": - return False - if self.memory_mode == "partial": - return False - return True + return self.memory_mode not in ["disk", "partial"] def __len__(self) -> int: return self.total_sentence_count @@ -309,7 +306,7 @@ class CSVClassificationCorpus(Corpus): def __init__( self, data_folder: Union[str, Path], - column_name_map: Dict[int, str], + column_name_map: dict[int, str], label_type: str, name: str = "csv_corpus", train_file=None, @@ -404,7 +401,7 @@ class CSVClassificationDataset(FlairDataset): def __init__( self, path_to_file: Union[str, Path], - column_name_map: Dict[int, str], + column_name_map: dict[int, str], label_type: str, max_tokens_per_doc: int = -1, max_chars_per_doc: int = -1, @@ -453,8 +450,8 @@ def __init__( self.total_sentence_count: int = 0 # most data sets have the token text in the first column, if not, pass 'text' as column - self.text_columns: List[int] = [] - self.pair_columns: List[int] = [] + self.text_columns: list[int] = [] + self.pair_columns: list[int] = [] for column in column_name_map: if column_name_map[column] == "text": self.text_columns.append(column) @@ -567,7 +564,7 @@ class AMAZON_REVIEWS(ClassificationCorpus): def __init__( self, split_max: int = 30000, - label_name_map: Dict[str, str] = { + label_name_map: dict[str, str] = { "1.0": "NEGATIVE", "2.0": "NEGATIVE", "3.0": "NEGATIVE", @@ -955,9 +952,10 @@ def __init__( original_filenames = original_filenames[:-1] if not data_file.is_file(): for original_filename, new_filename in zip(original_filenames, new_filenames): - with open(data_folder / "original" / original_filename, encoding="utf-8") as open_fp, open( - data_folder / new_filename, "w", encoding="utf-8" - ) as write_fp: + with ( + open(data_folder / "original" / original_filename, encoding="utf-8") as open_fp, + open(data_folder / new_filename, "w", encoding="utf-8") as write_fp, + ): csv_reader = csv.reader( open_fp, quotechar='"', delimiter=",", quoting=csv.QUOTE_ALL, skipinitialspace=True ) @@ -1048,9 +1046,10 @@ def __init__( label_list.append(labels[int(line) - 1]) # handle data file - with (data_path / "original" / "title_StackOverflow.txt").open(encoding="latin1") as open_fp, ( - data_folder / "train.txt" - ).open("w", encoding="utf-8") as write_fp: + with ( + (data_path / "original" / "title_StackOverflow.txt").open(encoding="latin1") as open_fp, + (data_folder / "train.txt").open("w", encoding="utf-8") as write_fp, + ): for idx, line in enumerate(open_fp): line = line.rstrip() @@ -1104,9 +1103,10 @@ def __init__( os.makedirs(data_folder) # create train.txt file from CSV - with open(data_folder / "train.txt", "w") as train_file, open( - senteval_folder / "training.1600000.processed.noemoticon.csv", encoding="latin-1" - ) as csv_train: + with ( + open(data_folder / "train.txt", "w") as train_file, + open(senteval_folder / "training.1600000.processed.noemoticon.csv", encoding="latin-1") as csv_train, + ): csv_reader = csv.reader(csv_train) for row in csv_reader: @@ -1115,9 +1115,10 @@ def __init__( train_file.write(f"__label__{label} {text}\n") # create test.txt file from CSV - with (data_folder / "test.txt").open("w", encoding="utf-8") as train_file, ( - senteval_folder / "testdata.manual.2009.06.14.csv" - ).open(encoding="latin-1") as csv_train: + with ( + (data_folder / "test.txt").open("w", encoding="utf-8") as train_file, + (senteval_folder / "testdata.manual.2009.06.14.csv").open(encoding="latin-1") as csv_train, + ): csv_reader = csv.reader(csv_train) for row in csv_reader: @@ -1384,9 +1385,10 @@ def __init__( # create train dev and test files in fasttext format for new_filename, original_filename in zip(new_filenames, original_filenames): - with open(data_folder / new_filename, "a") as out_file, open( - data_folder / "raw" / original_filename - ) as in_file: + with ( + open(data_folder / new_filename, "a") as out_file, + open(data_folder / "raw" / original_filename) as in_file, + ): for line in in_file: fields = line.split("\t") label = "POSITIVE" if fields[1].rstrip() == "1" else "NEGATIVE" @@ -1437,9 +1439,10 @@ def __init__( # convert to FastText format for split in ["train", "dev", "test"]: - with (data_folder / f"{split}.txt").open("w", encoding="utf-8") as train_file, ( - data_folder / "raw" / f"stsa.fine.{split}" - ).open(encoding="latin1") as file: + with ( + (data_folder / f"{split}.txt").open("w", encoding="utf-8") as train_file, + (data_folder / "raw" / f"stsa.fine.{split}").open(encoding="latin1") as file, + ): for line in file: train_file.write(f"__label__{line[0]} {line[2:]}") @@ -1496,9 +1499,10 @@ def __init__( # create train and dev splits in fasttext format for split in ["train", "dev"]: - with open(data_folder / "CoLA" / (split + ".txt"), "a") as out_file, open( - data_folder / "CoLA" / "original" / (split + ".tsv") - ) as in_file: + with ( + open(data_folder / "CoLA" / (split + ".txt"), "a") as out_file, + open(data_folder / "CoLA" / "original" / (split + ".tsv")) as in_file, + ): for line in in_file: fields = line.rstrip().split("\t") label = int(fields[1]) @@ -1506,9 +1510,10 @@ def __init__( out_file.write(f"__label__{label_map[label]} {sentence}\n") # create eval_dataset file with no labels - with open(data_folder / "CoLA" / "eval_dataset.txt", "a") as out_file, open( - data_folder / "CoLA" / "original" / "test.tsv" - ) as in_file: + with ( + open(data_folder / "CoLA" / "eval_dataset.txt", "a") as out_file, + open(data_folder / "CoLA" / "original" / "test.tsv") as in_file, + ): for line in in_file: fields = line.rstrip().split("\t") sentence = fields[1] @@ -1702,9 +1707,10 @@ def __init__( data_path = flair.cache_root / "datasets" / dataset_name / "raw" # create correctly formated txt files for name in ["train", "test", "dev"]: - with (data_folder / (name + ".txt")).open("w", encoding="utf-8") as txt_file, ( - data_path / (name + ".tsv") - ).open(encoding="utf-8") as tsv_file: + with ( + (data_folder / (name + ".txt")).open("w", encoding="utf-8") as txt_file, + (data_path / (name + ".tsv")).open(encoding="utf-8") as tsv_file, + ): lines = tsv_file.readlines() for line in lines: row = line.split("\t") @@ -1764,9 +1770,10 @@ def __init__( if not data_file.is_file(): for original_filename, new_filename in zip(original_filenames, new_filenames): - with (data_folder / "original" / original_filename).open(encoding="latin1") as open_fp, ( - data_folder / new_filename - ).open("w", encoding="utf-8") as write_fp: + with ( + (data_folder / "original" / original_filename).open(encoding="latin1") as open_fp, + (data_folder / new_filename).open("w", encoding="utf-8") as write_fp, + ): for line in open_fp: line = line.rstrip() fields = line.split() @@ -1820,9 +1827,10 @@ def __init__( if not data_file.is_file(): for original_filename, new_filename in zip(original_filenames, new_filenames): - with (data_folder / "original" / original_filename).open(encoding="latin1") as open_fp, ( - data_folder / new_filename - ).open("w", encoding="utf-8") as write_fp: + with ( + (data_folder / "original" / original_filename).open(encoding="latin1") as open_fp, + (data_folder / new_filename).open("w", encoding="utf-8") as write_fp, + ): for line in open_fp: line = line.rstrip() fields = line.split() @@ -1887,21 +1895,20 @@ def __init__( if not (data_folder / "train.txt").is_file(): cached_path(url, original) - import tarfile - - tar = tarfile.open(original / "yahoo_answers_csv.tgz", "r:gz") - members = [] + with tarfile.open(original / "yahoo_answers_csv.tgz", "r:gz") as tar: + members = [] - for member in tar.getmembers(): - if "test.csv" in member.name or "train.csv" in member.name: - members.append(member) + for member in tar.getmembers(): + if "test.csv" in member.name or "train.csv" in member.name: + members.append(member) - tar.extractall(original, members=members) + tar.extractall(original, members=members) for name in ["train", "test"]: - with (original / "yahoo_answers_csv" / (name + ".csv")).open(encoding="utf-8") as file, ( - data_folder / (name + ".txt") - ).open("w", encoding="utf-8") as writer: + with ( + (original / "yahoo_answers_csv" / (name + ".csv")).open(encoding="utf-8") as file, + (data_folder / (name + ".txt")).open("w", encoding="utf-8") as writer, + ): reader = csv.reader(file) for row in reader: writer.write("__label__" + label_map[row[0]] + " " + row[1] + "\n") @@ -1963,9 +1970,10 @@ def __init__( if not data_file.is_file(): for original_filename, new_filename in zip(original_filenames, new_filenames): - with (data_folder / "original" / original_filename).open(encoding="utf-8") as open_fp, ( - data_folder / task_setting / new_filename - ).open("w", encoding="utf-8") as write_fp: + with ( + (data_folder / "original" / original_filename).open(encoding="utf-8") as open_fp, + (data_folder / task_setting / new_filename).open("w", encoding="utf-8") as write_fp, + ): for line in open_fp: line = line.rstrip() fields = line.split("\t") diff --git a/flair/datasets/entity_linking.py b/flair/datasets/entity_linking.py index 20f2caefdd..f74bad7092 100644 --- a/flair/datasets/entity_linking.py +++ b/flair/datasets/entity_linking.py @@ -4,8 +4,9 @@ import logging import os import re +from collections.abc import Iterable, Iterator from pathlib import Path -from typing import Any, Dict, Iterable, Iterator, List, Optional, Union +from typing import Any, Optional, Union import requests from bioc import biocxml, pubtator @@ -47,7 +48,7 @@ def __init__( self._idx_to_candidates = {candidate.concept_id: candidate for candidate in candidates} # one name can map to multiple concepts - self._text_to_index: Dict[str, List[str]] = {} + self._text_to_index: dict[str, list[str]] = {} for candidate in candidates: for text in [candidate.concept_name, *candidate.synonyms]: if text not in self._text_to_index: @@ -60,11 +61,11 @@ def database_name(self) -> str: return self._dataset_name @property - def text_to_index(self) -> Dict[str, List[str]]: + def text_to_index(self) -> dict[str, list[str]]: return self._text_to_index @property - def candidates(self) -> List[EntityCandidate]: + def candidates(self) -> list[EntityCandidate]: return list(self._idx_to_candidates.values()) def __getitem__(self, item: str) -> EntityCandidate: @@ -80,18 +81,18 @@ def to_in_memory_dictionary(self) -> "InMemoryEntityLinkingDictionary": # NOTE: EntityLinkingDictionary are lazy-loaded from a preprocessed file. # Use this class to load into memory all candidates class InMemoryEntityLinkingDictionary(EntityLinkingDictionary): - def __init__(self, candidates: List[EntityCandidate], dataset_name: str): + def __init__(self, candidates: list[EntityCandidate], dataset_name: str): self._dataset_name = dataset_name super().__init__(candidates, dataset_name=dataset_name) - def to_state(self) -> Dict[str, Any]: + def to_state(self) -> dict[str, Any]: return { "dataset_name": self._dataset_name, "candidates": [candidate.to_dict() for candidate in self._idx_to_candidates.values()], } @classmethod - def from_state(cls, state: Dict[str, Any]) -> "InMemoryEntityLinkingDictionary": + def from_state(cls, state: dict[str, Any]) -> "InMemoryEntityLinkingDictionary": return cls( dataset_name=state["dataset_name"], candidates=[EntityCandidate(**candidate) for candidate in state["candidates"]], @@ -488,7 +489,7 @@ def __init__( to point to a different folder but typically this should not be necessary. in_memory: bool If True, keeps dataset in memory giving speedups in training. - column_format: Dict[int, str] + column_format: dict[int, str] The column-format to specify which columns correspond to the text or label types. """ base_path = flair.cache_root / "datasets" if not base_path else Path(base_path) @@ -776,9 +777,10 @@ def __init__( wiki_language + "_dev.tsv", ], ): - with open(doc_path, encoding="utf-8") as read, open( - data_folder / file_name, "w", encoding="utf-8" - ) as write: + with ( + open(doc_path, encoding="utf-8") as read, + open(data_folder / file_name, "w", encoding="utf-8") as write, + ): # ignore first line read.readline() line = read.readline() @@ -1208,9 +1210,10 @@ def __init__( if not parsed_dataset.exists(): original_file_path = cached_path(f"{tweeki_gold_el_path}", Path("datasets") / dataset_name) - with open(original_file_path, encoding="utf-8") as read, open( - parsed_dataset, "w", encoding="utf-8" - ) as write: + with ( + open(original_file_path, encoding="utf-8") as read, + open(parsed_dataset, "w", encoding="utf-8") as write, + ): line = read.readline() while line: if line.startswith("#"): @@ -1274,9 +1277,10 @@ def __init__( with open(data_folder / corpus_file_name, "w", encoding="utf-8") as txtout: # First parse the post titles - with open(data_folder / "posts.tsv", encoding="utf-8") as tsvin1, open( - data_folder / "gold_post_annotations.tsv", encoding="utf-8" - ) as tsvin2: + with ( + open(data_folder / "posts.tsv", encoding="utf-8") as tsvin1, + open(data_folder / "gold_post_annotations.tsv", encoding="utf-8") as tsvin2, + ): posts = csv.reader(tsvin1, delimiter="\t") self.post_annotations = csv.reader(tsvin2, delimiter="\t") self.curr_annot = next(self.post_annotations) @@ -1312,13 +1316,14 @@ def __init__( ) # Then parse the comments - with open(data_folder / "comments.tsv", encoding="utf-8") as tsvin3, open( - data_folder / "gold_comment_annotations.tsv", encoding="utf-8" - ) as tsvin4: + with ( + open(data_folder / "comments.tsv", encoding="utf-8") as tsvin3, + open(data_folder / "gold_comment_annotations.tsv", encoding="utf-8") as tsvin4, + ): self.comments = csv.reader(tsvin3, delimiter="\t") self.comment_annotations = csv.reader(tsvin4, delimiter="\t") self.curr_annot = next(self.comment_annotations) - self.curr_row: Optional[List[str]] = next(self.comments) + self.curr_row: Optional[list[str]] = next(self.comments) self.stop_iter = False # Iterate over the comments.tsv file, until the end is reached @@ -1545,7 +1550,7 @@ def make_line(word, begin_or_inside, attributes): return line - def split_span(word_fields: List[str], datasetname: str): + def split_span(word_fields: list[str], datasetname: str): """Function that splits a word if necessary, i.e. if it is a multiple-word-span. Parameters @@ -1646,12 +1651,12 @@ def determine_tsv_file(filename: str, data_folder: Path, cut_multisense: bool = class WSD_UFSAC(MultiCorpus): def __init__( self, - filenames: Union[str, List[str]] = ["masc", "semcor"], + filenames: Union[str, list[str]] = ["masc", "semcor"], base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, cut_multisense: bool = True, columns={0: "text", 3: "sense"}, - banned_sentences: Optional[List[str]] = None, + banned_sentences: Optional[list[str]] = None, sample_missing_splits_in_multicorpus: Union[bool, str] = True, sample_missing_splits_in_each_corpus: Union[bool, str] = True, use_raganato_ALL_as_test_data: bool = False, @@ -1713,7 +1718,7 @@ def __init__( if isinstance(filenames, str): filenames = [filenames] - corpora: List[Corpus] = [] + corpora: list[Corpus] = [] log.info("Transforming data into column format and creating corpora...") @@ -1784,8 +1789,8 @@ def __init__( base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, columns={0: "text", 3: "sense"}, - label_name_map: Optional[Dict[str, str]] = None, - banned_sentences: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + banned_sentences: Optional[list[str]] = None, sample_missing_splits: bool = True, cut_multisense: bool = True, ) -> None: @@ -1847,8 +1852,8 @@ def __init__( base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, columns={0: "text", 3: "sense"}, - label_name_map: Optional[Dict[str, str]] = None, - banned_sentences: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + banned_sentences: Optional[list[str]] = None, sample_missing_splits: Union[bool, str] = True, cut_multisense: bool = True, use_raganato_ALL_as_test_data: bool = False, @@ -1922,8 +1927,8 @@ def __init__( base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, columns={0: "text", 3: "sense"}, - label_name_map: Optional[Dict[str, str]] = None, - banned_sentences: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + banned_sentences: Optional[list[str]] = None, sample_missing_splits: Union[bool, str] = True, use_raganato_ALL_as_test_data: bool = False, ) -> None: @@ -1994,8 +1999,8 @@ def __init__( base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, columns={0: "text", 3: "sense"}, - label_name_map: Optional[Dict[str, str]] = None, - banned_sentences: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + banned_sentences: Optional[list[str]] = None, sample_missing_splits: Union[bool, str] = True, cut_multisense: bool = True, use_raganato_ALL_as_test_data: bool = False, @@ -2070,8 +2075,8 @@ def __init__( base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, columns={0: "text", 3: "sense"}, - label_name_map: Optional[Dict[str, str]] = None, - banned_sentences: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + banned_sentences: Optional[list[str]] = None, sample_missing_splits: Union[bool, str] = True, cut_multisense: bool = True, use_raganato_ALL_as_test_data: bool = False, @@ -2147,8 +2152,8 @@ def __init__( base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, columns={0: "text", 3: "sense"}, - label_name_map: Optional[Dict[str, str]] = None, - banned_sentences: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + banned_sentences: Optional[list[str]] = None, sample_missing_splits: Union[bool, str] = True, use_raganato_ALL_as_test_data: bool = False, ) -> None: @@ -2230,7 +2235,7 @@ def __init__( self, base_path: Optional[Union[str, Path]] = None, label_type: str = "el", - norm_keys: List[str] = ["db_name", "db_id"], + norm_keys: list[str] = ["db_name", "db_id"], **kwargs, ) -> None: self.label_type = label_type @@ -2250,14 +2255,14 @@ def __init__( ) @abc.abstractmethod - def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Path]]]: + def _download_dataset(self, data_folder: Path) -> dict[str, Union[Path, list[Path]]]: pass @abc.abstractmethod - def _file_to_dicts(self, filepath: Path) -> Iterator[Dict[str, Any]]: + def _file_to_dicts(self, filepath: Path) -> Iterator[dict[str, Any]]: pass - def _dict_to_sentences(self, entry: Dict[str, Any]) -> List[Sentence]: + def _dict_to_sentences(self, entry: dict[str, Any]) -> list[Sentence]: entities = [entity for entity in entry["entities"] if entity["normalized"]] tokenized_passages = [ @@ -2326,7 +2331,7 @@ def _dict_to_sentences(self, entry: Dict[str, Any]) -> List[Sentence]: sent_s[start_token_idx : end_token_idx + 1].add_label(self.label_type, mention_id) return passage_sentences - def _files_to_dataset(self, paths: Union[Path, List[Path]]) -> FlairDatapointDataset: + def _files_to_dataset(self, paths: Union[Path, list[Path]]) -> FlairDatapointDataset: if isinstance(paths, Path): paths = [paths] all_sentences = [] @@ -2347,7 +2352,7 @@ class BIGBIO_EL_NCBI_DISEASE(BigBioEntityLinkingCorpus): def __init__(self, base_path: Optional[Union[str, Path]] = None, label_type: str = "el-diseases", **kwargs) -> None: super().__init__(base_path, label_type, **kwargs) - def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Path]]]: + def _download_dataset(self, data_folder: Path) -> dict[str, Union[Path, list[Path]]]: download_urls = { "train": ( "NCBItrainset_corpus.txt", @@ -2362,7 +2367,7 @@ def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Pat "https://www.ncbi.nlm.nih.gov/CBBresearch/Dogan/DISEASE/NCBItestset_corpus.zip", ), } - results_files: Dict[str, Union[Path, List[Path]]] = {} + results_files: dict[str, Union[Path, list[Path]]] = {} for split, (filename, url) in download_urls.items(): result_path = data_folder / filename @@ -2376,7 +2381,7 @@ def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Pat return results_files - def _file_to_dicts(self, filepath: Path) -> Iterator[Dict[str, Any]]: + def _file_to_dicts(self, filepath: Path) -> Iterator[dict[str, Any]]: with open(filepath) as f: for doc in pubtator.iterparse(f): unified_example = { @@ -2449,7 +2454,7 @@ class BIGBIO_EL_BC5CDR_CHEMICAL(BigBioEntityLinkingCorpus): def __init__(self, base_path: Optional[Union[str, Path]] = None, label_type: str = "el-chemical", **kwargs) -> None: super().__init__(base_path, label_type, **kwargs) - def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Path]]]: + def _download_dataset(self, data_folder: Path) -> dict[str, Union[Path, list[Path]]]: url = "https://huggingface.co/datasets/bigbio/bc5cdr/resolve/main/CDR_Data.zip" path = cached_path(url, data_folder) @@ -2458,7 +2463,7 @@ def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Pat unpack_file(path, data_folder) assert data_folder.exists() - results_files: Dict[str, Union[Path, List[Path]]] = { + results_files: dict[str, Union[Path, list[Path]]] = { "train": data_path / "CDR_TrainingSet.BioC.xml", "dev": data_path / "CDR_DevelopmentSet.BioC.xml", "test": data_path / "CDR_TestSet.BioC.xml", @@ -2497,7 +2502,7 @@ def _get_bioc_entity(self, span, db_id_key="MESH"): "normalized": normalized, } - def _file_to_dicts(self, filepath: Path) -> Iterator[Dict[str, Any]]: + def _file_to_dicts(self, filepath: Path) -> Iterator[dict[str, Any]]: reader = biocxml.BioCXMLDocumentReader(str(filepath)) for i, xdoc in enumerate(reader): @@ -2542,7 +2547,7 @@ def __init__(self, base_path: Optional[Union[str, Path]] = None, label_type: str self._re_tax_id = re.compile(r"(?P\d+)\([tT]ax:(?P\d+)\)") super().__init__(base_path, label_type, norm_keys=["db_id"], **kwargs) - def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Path]]]: + def _download_dataset(self, data_folder: Path) -> dict[str, Union[Path, list[Path]]]: url = "https://www.ncbi.nlm.nih.gov/CBBresearch/Lu/Demo/tmTools/download/GNormPlus/GNormPlusCorpus.zip" path = cached_path(url, data_folder) @@ -2551,7 +2556,7 @@ def _download_dataset(self, data_folder: Path) -> Dict[str, Union[Path, List[Pat unpack_file(path, data_folder) assert data_folder.exists() - results_files: Dict[str, Union[Path, List[Path]]] = { + results_files: dict[str, Union[Path, list[Path]]] = { "train": [data_path / "BC2GNtrain.BioC.xml", data_path / "NLMIAT.BioC.xml"], "test": data_path / "BC2GNtest.BioC.xml", } @@ -2595,7 +2600,7 @@ def _parse_bioc_entity(self, span, db_id_key="NCBIGene", insert_tax_id=False): "normalized": normalized, } - def _adjust_entity_offsets(self, text: str, entities: List[Dict]): + def _adjust_entity_offsets(self, text: str, entities: list[dict]): for entity in entities: start, end = entity["offsets"][0] entity_mention = entity["text"][0] @@ -2605,7 +2610,7 @@ def _adjust_entity_offsets(self, text: str, entities: List[Dict]): elif text[start : end - 1] == entity_mention: entity["offsets"] = [(start, end - 1)] - def _file_to_dicts(self, filepath: Path) -> Iterator[Dict[str, Any]]: + def _file_to_dicts(self, filepath: Path) -> Iterator[dict[str, Any]]: with filepath.open("r") as f: collection = biocxml.load(f) diff --git a/flair/datasets/ocr.py b/flair/datasets/ocr.py index bf60b2b0d6..4a58e4e7d3 100644 --- a/flair/datasets/ocr.py +++ b/flair/datasets/ocr.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import Dict, Optional, Union +from typing import Optional, Union import gdown.download_folder import PIL @@ -20,7 +20,7 @@ def __init__( encoding: str = "utf-8", load_images: bool = False, normalize_coords_to_thousands: bool = True, - label_name_map: Optional[Dict[str, str]] = None, + label_name_map: Optional[dict[str, str]] = None, ) -> None: """Instantiates a Dataset from a OCR-Json format. @@ -132,7 +132,7 @@ def __init__( in_memory: bool = True, load_images: bool = False, normalize_coords_to_thousands: bool = True, - label_name_map: Optional[Dict[str, str]] = None, + label_name_map: Optional[dict[str, str]] = None, **corpusargs, ) -> None: """Instantiates a Corpus from a OCR-Json format. @@ -205,7 +205,7 @@ def __init__( in_memory: bool = True, load_images: bool = False, normalize_coords_to_thousands: bool = True, - label_name_map: Optional[Dict[str, str]] = None, + label_name_map: Optional[dict[str, str]] = None, **corpusargs, ) -> None: """Instantiates the SROIE corpus with perfect ocr boxes. diff --git a/flair/datasets/relation_extraction.py b/flair/datasets/relation_extraction.py index 30709a14c4..871811abc2 100644 --- a/flair/datasets/relation_extraction.py +++ b/flair/datasets/relation_extraction.py @@ -5,8 +5,9 @@ import os import re from collections import defaultdict +from collections.abc import Iterable from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Optional, Union import conllu import gdown @@ -279,7 +280,7 @@ def extract_and_convert_to_conllu(self, data_file, data_folder): token_list = self._tacred_example_to_token_list(example) target_file.write(token_list.serialize()) - def _tacred_example_to_token_list(self, example: Dict[str, Any]) -> conllu.TokenList: + def _tacred_example_to_token_list(self, example: dict[str, Any]) -> conllu.TokenList: id_ = example["id"] tokens = example["token"] ner = example["stanford_ner"] @@ -379,7 +380,7 @@ def _parse_incr(self, source_file) -> Iterable[conllu.TokenList]: } metadata_parsers = {"__fallback__": lambda k, v: tuple(k.split())} - lines: List[str] = [] + lines: list[str] = [] for index, line in enumerate(source_file): if index > 0 and line.startswith("#"): source_str = "".join(lines) @@ -416,9 +417,10 @@ def convert_to_conllu(self, source_data_folder: Path, data_folder): ] for source_filename, target_filename in zip(source_filenames, target_filenames): - with (source_data_folder / source_filename).open(encoding="utf-8") as source_file, ( - data_folder / target_filename - ).open("w", encoding="utf-8") as target_file: + with ( + (source_data_folder / source_filename).open(encoding="utf-8") as source_file, + (data_folder / target_filename).open("w", encoding="utf-8") as target_file, + ): # write CoNLL-U Plus header target_file.write("# global.columns = id form ner\n") @@ -426,7 +428,7 @@ def convert_to_conllu(self, source_data_folder: Path, data_folder): token_list = self._src_token_list_to_token_list(src_token_list) target_file.write(token_list.serialize()) - def _bio_tags_to_spans(self, tags: List[str]) -> List[Tuple[int, int]]: + def _bio_tags_to_spans(self, tags: list[str]) -> list[tuple[int, int]]: spans = [] span_start = 0 span_end = 0 @@ -590,7 +592,7 @@ def extract_and_convert_to_conllu(self, data_file, data_folder): ent2 = arg2.split(":")[1] pmid_to_relations[pmid].add((rel_type, ent1, ent2)) - tokenlists: List[conllu.TokenList] = [] + tokenlists: list[conllu.TokenList] = [] with zip_file.open( f"drugprot-gs-training-development/{split}/drugprot_{split}_abstracs.tsv" ) as abstracts_file: @@ -652,13 +654,13 @@ def has_overlap(self, a, b): def drugprot_document_to_tokenlists( self, pmid: str, - title_sentences: List[Sentence], - abstract_sentences: List[Sentence], + title_sentences: list[Sentence], + abstract_sentences: list[Sentence], abstract_offset: int, - entities: Dict[str, Tuple[str, int, int, str]], - relations: Set[Tuple[str, str, str]], - ) -> List[conllu.TokenList]: - tokenlists: List[conllu.TokenList] = [] + entities: dict[str, tuple[str, int, int, str]], + relations: set[tuple[str, str, str]], + ) -> list[conllu.TokenList]: + tokenlists: list[conllu.TokenList] = [] sentence_id = 1 for offset, sents in [ (0, title_sentences), diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 6699548498..55e50723d1 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -4,17 +4,13 @@ import os import re import shutil +import tarfile from collections import defaultdict +from collections.abc import Iterable, Iterator from pathlib import Path from typing import ( Any, - DefaultDict, - Dict, - Iterable, - Iterator, - List, Optional, - Tuple, Union, cast, ) @@ -224,7 +220,7 @@ def __init__( self.label_type = label_type self.path_to_json_file = path_to_json_file - self.sentences: List[Sentence] = [] + self.sentences: list[Sentence] = [] with path_to_json_file.open(encoding=encoding) as jsonl_fp: for line in jsonl_fp: current_line = json.loads(line) @@ -238,7 +234,7 @@ def __init__( self.sentences.append(current_sentence) - def _add_labels_to_sentence(self, raw_text: str, sentence: Sentence, labels: List[List[Any]]): + def _add_labels_to_sentence(self, raw_text: str, sentence: Sentence, labels: list[list[Any]]): # Add tags for each annotated span for label in labels: self._add_label_to_sentence(raw_text, sentence, label[0], label[1], label[2]) @@ -288,7 +284,7 @@ def _add_label_to_sentence(self, text: str, sentence: Sentence, start: int, end: sentence[start_idx : end_idx + 1].add_label(self.label_type, label) - def _add_metadatas_to_sentence(self, sentence: Sentence, metadatas: List[Tuple[str, str]]): + def _add_metadatas_to_sentence(self, sentence: Sentence, metadatas: list[tuple[str, str]]): # Add metadatas for sentence for metadata in metadatas: self._add_metadata_to_sentence(sentence, metadata[0], metadata[1]) @@ -313,7 +309,7 @@ def __getitem__(self, index: int) -> Sentence: class MultiFileColumnCorpus(Corpus): def __init__( self, - column_format: Dict[int, str], + column_format: dict[int, str], train_files=None, test_files=None, dev_files=None, @@ -323,8 +319,8 @@ def __init__( document_separator_token: Optional[str] = None, skip_first_line: bool = False, in_memory: bool = True, - label_name_map: Optional[Dict[str, str]] = None, - banned_sentences: Optional[List[str]] = None, + label_name_map: Optional[dict[str, str]] = None, + banned_sentences: Optional[list[str]] = None, default_whitespace_after: int = 1, **corpusargs, ) -> None: @@ -424,7 +420,7 @@ class ColumnCorpus(MultiFileColumnCorpus): def __init__( self, data_folder: Union[str, Path], - column_format: Dict[int, str], + column_format: dict[int, str], train_file=None, test_file=None, dev_file=None, @@ -475,15 +471,15 @@ class ColumnDataset(FlairDataset): def __init__( self, path_to_column_file: Union[str, Path], - column_name_map: Dict[int, str], + column_name_map: dict[int, str], column_delimiter: str = r"\s+", comment_symbol: Optional[str] = None, - banned_sentences: Optional[List[str]] = None, + banned_sentences: Optional[list[str]] = None, in_memory: bool = True, document_separator_token: Optional[str] = None, encoding: str = "utf-8", skip_first_line: bool = False, - label_name_map: Optional[Dict[str, str]] = None, + label_name_map: Optional[dict[str, str]] = None, default_whitespace_after: int = 1, ) -> None: r"""Instantiates a column dataset. @@ -537,7 +533,7 @@ def __init__( # option 1: keep Sentence objects in memory if self.in_memory: - self.sentences: List[Sentence] = [] + self.sentences: list[Sentence] = [] # pointer to previous previous_sentence = None @@ -579,7 +575,7 @@ def __init__( # option 2: keep source data in memory if not self.in_memory: - self.sentences_raw: List[List[str]] = [] + self.sentences_raw: list[list[str]] = [] while True: # read lines for next sentence, but don't parse @@ -679,10 +675,10 @@ def _read_next_sentence(self, file): return lines def _convert_lines_to_sentence( - self, lines, word_level_tag_columns: Dict[int, str], span_level_tag_columns: Optional[Dict[int, str]] = None + self, lines, word_level_tag_columns: dict[int, str], span_level_tag_columns: Optional[dict[int, str]] = None ): token: Optional[Token] = None - tokens: List[Token] = [] + tokens: list[Token] = [] filtered_lines = [] comments = [] for line in lines: @@ -749,9 +745,9 @@ def _convert_lines_to_sentence( return sentence return None - def _parse_token(self, line: str, column_name_map: Dict[int, str], last_token: Optional[Token] = None) -> Token: + def _parse_token(self, line: str, column_name_map: dict[int, str], last_token: Optional[Token] = None) -> Token: # get fields from line - fields: List[str] = self.column_delimiter.split(line.rstrip()) + fields: list[str] = self.column_delimiter.split(line.rstrip()) field_count = len(fields) # get head_id if exists (only in dependency parses) head_id = int(fields[self.head_id_column]) if self.head_id_column else None @@ -855,7 +851,7 @@ def __init__( base_path: Optional[Union[str, Path]] = None, version: str = "v4", language: str = "english", - domain: Union[None, str, List[str], Dict[str, Union[None, str, List[str]]]] = None, + domain: Union[None, str, list[str], dict[str, Union[None, str, list[str]]]] = None, in_memory: bool = True, **corpusargs, ) -> None: @@ -893,7 +889,7 @@ def get_available_domains( version: str = "v4", language: str = "english", split: str = "train", - ) -> List[str]: + ) -> list[str]: processed_data_path = cls._ensure_data_processed(base_path=base_path, language=language, version=version) processed_split_path = processed_data_path / "splits" / version / language / split @@ -907,7 +903,7 @@ def _get_processed_file_paths( split: str = "train", version: str = "v4", language: str = "english", - domain: Optional[Union[str, List[str], Dict[str, Union[None, str, List[str]]]]] = None, + domain: Optional[Union[str, list[str], dict[str, Union[None, str, list[str]]]]] = None, ) -> Iterable[Path]: processed_split_path = processed_data_path / "splits" / version / language / split @@ -1009,8 +1005,8 @@ def _process_coref_span_annotations_for_word( cls, label: str, word_index: int, - clusters: DefaultDict[int, List[Tuple[int, int]]], - coref_stacks: DefaultDict[int, List[int]], + clusters: defaultdict[int, list[tuple[int, int]]], + coref_stacks: defaultdict[int, list[int]], ) -> None: """For a given coref label, add it to a currently open span(s), complete a span(s) or ignore it, if it is outside of all spans. @@ -1048,9 +1044,9 @@ def _process_coref_span_annotations_for_word( @classmethod def _process_span_annotations_for_word( cls, - annotations: List[str], - span_labels: List[List[str]], - current_span_labels: List[Optional[str]], + annotations: list[str], + span_labels: list[list[str]], + current_span_labels: list[Optional[str]], ) -> None: for annotation_index, annotation in enumerate(annotations): # strip all bracketing information to @@ -1076,33 +1072,33 @@ def _process_span_annotations_for_word( current_span_labels[annotation_index] = None @classmethod - def _conll_rows_to_sentence(cls, conll_rows: List[str]) -> Dict: + def _conll_rows_to_sentence(cls, conll_rows: list[str]) -> dict: document_id: str sentence_id: int # The words in the sentence. - sentence: List[str] = [] + sentence: list[str] = [] # The pos tags of the words in the sentence. - pos_tags: List[str] = [] + pos_tags: list[str] = [] # the pieces of the parse tree. - parse_pieces: List[Optional[str]] = [] + parse_pieces: list[Optional[str]] = [] # The lemmatised form of the words in the sentence which # have SRL or word sense information. - predicate_lemmas: List[Optional[str]] = [] + predicate_lemmas: list[Optional[str]] = [] # The FrameNet ID of the predicate. - predicate_framenet_ids: List[Optional[str]] = [] + predicate_framenet_ids: list[Optional[str]] = [] # The sense of the word, if available. - word_senses: List[Optional[float]] = [] + word_senses: list[Optional[float]] = [] # The current speaker, if available. - speakers: List[Optional[str]] = [] + speakers: list[Optional[str]] = [] - verbal_predicates: List[str] = [] - span_labels: List[List[str]] = [] - current_span_labels: List[Optional[str]] = [] + verbal_predicates: list[str] = [] + span_labels: list[list[str]] = [] + current_span_labels: list[Optional[str]] = [] # Cluster id -> List of (start_index, end_index) spans. - clusters: DefaultDict[int, List[Tuple[int, int]]] = defaultdict(list) + clusters: defaultdict[int, list[tuple[int, int]]] = defaultdict(list) # Cluster id -> List of start_indices which are open for this id. - coref_stacks: DefaultDict[int, List[int]] = defaultdict(list) + coref_stacks: defaultdict[int, list[int]] = defaultdict(list) for index, row in enumerate(conll_rows): conll_components = row.split() @@ -1178,7 +1174,7 @@ def _conll_rows_to_sentence(cls, conll_rows: List[str]) -> Dict: srl_frames = list(zip(verbal_predicates, span_labels[1:])) # this would not be reached if parse_pieces contained None, hence the cast - parse_tree = "".join(cast(List[str], parse_pieces)) if all(parse_pieces) else None + parse_tree = "".join(cast(list[str], parse_pieces)) if all(parse_pieces) else None coref_span_tuples = {(cluster_id, span) for cluster_id, span_list in clusters.items() for span in span_list} return { @@ -1197,7 +1193,7 @@ def _conll_rows_to_sentence(cls, conll_rows: List[str]) -> Dict: } @classmethod - def dataset_document_iterator(cls, file_path: Union[Path, str]) -> Iterator[List]: + def dataset_document_iterator(cls, file_path: Union[Path, str]) -> Iterator[list[dict]]: """An iterator over CONLL formatted files which yields documents, regardless of the number of document annotations in a particular file. This is useful for conll data which has been preprocessed, such @@ -1206,7 +1202,7 @@ def dataset_document_iterator(cls, file_path: Union[Path, str]) -> Iterator[List """ with open(file_path, encoding="utf8") as open_file: conll_rows = [] - document: List = [] + document: list[dict] = [] for line in open_file: line = line.strip() if line != "" and not line.startswith("#"): @@ -1456,17 +1452,22 @@ def __init__( cached_path(f"{conll_2000_path}train.txt.gz", Path("datasets") / dataset_name) cached_path(f"{conll_2000_path}test.txt.gz", Path("datasets") / dataset_name) import gzip - import shutil - with gzip.open(flair.cache_root / "datasets" / dataset_name / "train.txt.gz", "rb") as f_in, open( - flair.cache_root / "datasets" / dataset_name / "train.txt", - "wb", - ) as f_out: + with ( + gzip.open(flair.cache_root / "datasets" / dataset_name / "train.txt.gz", "rb") as f_in, + open( + flair.cache_root / "datasets" / dataset_name / "train.txt", + "wb", + ) as f_out, + ): shutil.copyfileobj(f_in, f_out) - with gzip.open(flair.cache_root / "datasets" / dataset_name / "test.txt.gz", "rb") as f_in, open( - flair.cache_root / "datasets" / dataset_name / "test.txt", - "wb", - ) as f_out: + with ( + gzip.open(flair.cache_root / "datasets" / dataset_name / "test.txt.gz", "rb") as f_in, + open( + flair.cache_root / "datasets" / dataset_name / "test.txt", + "wb", + ) as f_out, + ): shutil.copyfileobj(f_in, f_out) super().__init__( @@ -1735,8 +1736,6 @@ def __init__( data_file = data_path / "named_ent_eu.train" if not data_file.is_file(): cached_path(f"{ner_basque_path}/eiec_v1.0.tgz", Path("datasets") / dataset_name) - import shutil - import tarfile with tarfile.open( flair.cache_root / "datasets" / dataset_name / "eiec_v1.0.tgz", @@ -2247,15 +2246,13 @@ def __init__( if not base_path: base_path = Path(flair.cache_root) / "datasets" data_folder = base_path / dataset_name - import tarfile if not os.path.isfile(data_folder / "webpages_ner.txt"): # # download zip tar_file = "https://cogcomp.seas.upenn.edu/Data/NERWebpagesColumns.tgz" webpages_ner_path = cached_path(tar_file, Path("datasets") / dataset_name) - tf = tarfile.open(webpages_ner_path) - tf.extractall(data_folder) - tf.close() + with tarfile.open(webpages_ner_path) as tf: + tf.extractall(data_folder) outputfile = os.path.abspath(data_folder) # merge the files in one as the zip is containing multiples files @@ -2538,7 +2535,7 @@ def _add_IOB_tags(self, data_file: Union[str, Path], encoding: str = "utf8", ner Specifies the ner-tagged column. The default is 1 (the second column). """ - def add_I_prefix(current_line: List[str], ner: int, tag: str): + def add_I_prefix(current_line: list[str], ner: int, tag: str): for i in range(len(current_line)): if i == 0: f.write(line_list[i]) @@ -2779,9 +2776,11 @@ def _create_datasets(self, data_file: Union[str, Path], data_folder: Path): train_len = round(num_lines * 0.8) test_len = round(num_lines * 0.1) - with (data_folder / "train.txt").open("w", encoding="utf-8") as train, (data_folder / "test.txt").open( - "w", encoding="utf-8" - ) as test, (data_folder / "dev.txt").open("w", encoding="utf-8") as dev: + with ( + (data_folder / "train.txt").open("w", encoding="utf-8") as train, + (data_folder / "test.txt").open("w", encoding="utf-8") as test, + (data_folder / "dev.txt").open("w", encoding="utf-8") as dev, + ): for k, line in enumerate(file.readlines(), start=1): if k <= train_len: train.write(line) @@ -2972,7 +2971,7 @@ def __prepare_jap_wikinews_corpus(file_in: Union[str, Path], file_out: Union[str class NER_MASAKHANE(MultiCorpus): def __init__( self, - languages: Union[str, List[str]] = "luo", + languages: Union[str, list[str]] = "luo", version: str = "v2", base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, @@ -3056,7 +3055,7 @@ def __init__( if languages == ["all"]: languages = list(language_to_code.values()) - corpora: List[Corpus] = [] + corpora: list[Corpus] = [] for language in languages: if language in language_to_code: language = language_to_code[language] @@ -3239,7 +3238,7 @@ def __init__( class NER_MULTI_WIKIANN(MultiCorpus): def __init__( self, - languages: Union[str, List[str]] = "en", + languages: Union[str, list[str]] = "en", base_path: Optional[Union[str, Path]] = None, in_memory: bool = False, **corpusargs, @@ -3251,7 +3250,7 @@ def __init__( Parameters ---------- - languages : Union[str, List[str]] + languages : Union[str, list[str]] Should be an abbreviation of a language ("en", "de",..) or a list of abbreviations. The datasets of all passed languages will be saved in one MultiCorpus. (Note that, even though listed on https://elisa-ie.github.io/wikiann/ some datasets are empty. @@ -3282,7 +3281,7 @@ def __init__( # this list is handed to the multicorpus # list that contains the columncopora - corpora: List[Corpus] = [] + corpora: list[Corpus] = [] google_drive_path = "https://drive.google.com/uc?id=" # download data if necessary @@ -3294,8 +3293,6 @@ def __init__( # if language not downloaded yet, download it if not language_folder.exists(): if first: - import tarfile - import gdown first = False @@ -3310,10 +3307,8 @@ def __init__( # unzip log.info("Extracting data...") - tar = tarfile.open(str(language_folder / language) + ".tar.gz", "r:gz") - # tar.extractall(language_folder,members=[tar.getmember(file_name)]) - tar.extract(file_name, str(language_folder)) - tar.close() + with tarfile.open(str(language_folder / language) + ".tar.gz", "r:gz") as tar: + tar.extract(file_name, str(language_folder)) log.info("...done.") # transform data into required format @@ -3342,9 +3337,10 @@ def __init__( ) def _silver_standard_to_simple_ner_annotation(self, data_file: Union[str, Path]): - with open(data_file, encoding="utf-8") as f_read, open( - str(data_file) + "_new", "w+", encoding="utf-8" - ) as f_write: + with ( + open(data_file, encoding="utf-8") as f_read, + open(str(data_file) + "_new", "w+", encoding="utf-8") as f_write, + ): while True: line = f_read.readline() if line: @@ -3660,7 +3656,7 @@ def _google_drive_id_from_language_name(self, language): class NER_MULTI_XTREME(MultiCorpus): def __init__( self, - languages: Union[str, List[str]] = "en", + languages: Union[str, list[str]] = "en", base_path: Optional[Union[str, Path]] = None, in_memory: bool = False, **corpusargs, @@ -3672,7 +3668,7 @@ def __init__( Parameters ---------- - languages : Union[str, List[str]], optional + languages : Union[str, list[str]], optional Specify the languages you want to load. Provide an empty list or string to select all languages. base_path : Union[str, Path], optional Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this to point to a different folder but typically this should not be necessary. @@ -3743,7 +3739,7 @@ def __init__( # This list is handed to the multicorpus # list that contains the columncopora - corpora: List[Corpus] = [] + corpora: list[Corpus] = [] hu_path = "https://nlp.informatik.hu-berlin.de/resources/datasets/panx_dataset" @@ -3765,12 +3761,10 @@ def __init__( # unzip log.info("Extracting data...") - import tarfile - tar = tarfile.open(str(temp_file), "r:gz") - for part in ["train", "test", "dev"]: - tar.extract(part, str(language_folder)) - tar.close() + with tarfile.open(str(temp_file), "r:gz") as tar: + for part in ["train", "test", "dev"]: + tar.extract(part, str(language_folder)) log.info("...done.") # transform data into required format @@ -3809,7 +3803,7 @@ def _xtreme_to_simple_ner_annotation(self, data_file: Union[str, Path]): class NER_MULTI_WIKINER(MultiCorpus): def __init__( self, - languages: Union[str, List[str]] = "en", + languages: Union[str, list[str]] = "en", base_path: Optional[Union[str, Path]] = None, in_memory: bool = False, **corpusargs, @@ -3828,7 +3822,7 @@ def __init__( data_folder = base_path / dataset_name - corpora: List[Corpus] = [] + corpora: list[Corpus] = [] for language in languages: language_folder = data_folder / language @@ -3868,11 +3862,14 @@ def _download_wikiner(self, language_code: str, dataset_name: str): flair.cache_root / "datasets" / dataset_name / f"aij-wikiner-{lc}-wp3.bz2", "rb", ) - with bz_file as f, open( - flair.cache_root / "datasets" / dataset_name / f"aij-wikiner-{lc}-wp3.train", - "w", - encoding="utf-8", - ) as out: + with ( + bz_file as f, + open( + flair.cache_root / "datasets" / dataset_name / f"aij-wikiner-{lc}-wp3.train", + "w", + encoding="utf-8", + ) as out, + ): for lineb in f: line = lineb.decode("utf-8") words = line.split(" ") @@ -4740,7 +4737,7 @@ def __init__( class NER_NERMUD(MultiCorpus): def __init__( self, - domains: Union[str, List[str]] = "all", + domains: Union[str, list[str]] = "all", base_path: Optional[Union[str, Path]] = None, in_memory: bool = False, **corpusargs, @@ -4779,7 +4776,7 @@ def __init__( data_folder = base_path / dataset_name - corpora: List[Corpus] = [] + corpora: list[Corpus] = [] github_path = "https://raw.githubusercontent.com/dhfbk/KIND/main/evalita-2023" @@ -4923,7 +4920,7 @@ def _set_path(cls, base_path) -> Path: return base_path @classmethod - def _load_features(cls, base_path) -> List[List[str]]: + def _load_features(cls, base_path) -> list[list[str]]: print(base_path) unpack_file(cached_path(cls.data_url, base_path), base_path, "zip", False) with open(f"{base_path}/estner.cnll", encoding="utf-8") as in_file: @@ -4932,17 +4929,17 @@ def _load_features(cls, base_path) -> List[List[str]]: return features @classmethod - def _process_clean_labels(cls, features) -> List[List[str]]: + def _process_clean_labels(cls, features) -> list[list[str]]: preinstances = [[instance[0], instance[len(instance) - 1]] for instance in features] return preinstances @classmethod - def _rmv_clean_labels(cls, features) -> List[str]: + def _rmv_clean_labels(cls, features) -> list[str]: rdcd_features = [feature[:-1] for feature in features] return rdcd_features @classmethod - def _load_noisy_labels(cls, version, base_path) -> List[str]: + def _load_noisy_labels(cls, version, base_path) -> list[str]: file_name = f"NoisyNER_labelset{version}.labels" cached_path(f"{cls.label_url}/{file_name}", base_path) with open(f"{base_path}/{file_name}", encoding="utf-8") as in_file: @@ -4950,7 +4947,7 @@ def _load_noisy_labels(cls, version, base_path) -> List[str]: return labels @classmethod - def _process_noisy_labels(cls, rdcd_features, labels) -> List[List[str]]: + def _process_noisy_labels(cls, rdcd_features, labels) -> list[list[str]]: instances = [] label_idx = 0 for feature in rdcd_features: @@ -4965,7 +4962,7 @@ def _process_noisy_labels(cls, rdcd_features, labels) -> List[List[str]]: return instances @classmethod - def _delete_empty_labels(cls, version, preinstances) -> List[str]: + def _delete_empty_labels(cls, version, preinstances) -> list[str]: instances = [] if version == 0: for instance in preinstances: @@ -4978,7 +4975,7 @@ def _delete_empty_labels(cls, version, preinstances) -> List[str]: return instances @classmethod - def _split_data(cls, instances) -> Tuple[List[str], List[str], List[str]]: + def _split_data(cls, instances) -> tuple[list[str], list[str], list[str]]: train = instances[:185708] dev = instances[185708:208922] test = instances[208922:] @@ -4996,7 +4993,7 @@ def _write_instances(cls, version, base_path, split, data): class MASAKHA_POS(MultiCorpus): def __init__( self, - languages: Union[str, List[str]] = "bam", + languages: Union[str, list[str]] = "bam", version: str = "v1", base_path: Optional[Union[str, Path]] = None, in_memory: bool = True, @@ -5063,7 +5060,7 @@ def __init__( if languages == ["all"]: languages = supported_languages - corpora: List[Corpus] = [] + corpora: list[Corpus] = [] for language in languages: if language not in supported_languages: log.error(f"Language '{language}' is not in list of supported languages!") diff --git a/flair/datasets/text_image.py b/flair/datasets/text_image.py index f7baf72be9..676b078d7f 100644 --- a/flair/datasets/text_image.py +++ b/flair/datasets/text_image.py @@ -3,7 +3,6 @@ import os import urllib from pathlib import Path -from typing import List import numpy as np import torch.utils.data.dataloader @@ -40,13 +39,13 @@ def __init__(self, **kwargs) -> None: feidegger_dataset: Dataset = FeideggerDataset(dataset_info, **kwargs) - train_indices = list(np.where(np.in1d(feidegger_dataset.split, list(range(8))))[0]) # type: ignore[attr-defined] + train_indices = list(np.where(np.isin(feidegger_dataset.split, list(range(8))))[0]) # type: ignore[attr-defined] train = torch.utils.data.dataset.Subset(feidegger_dataset, train_indices) - dev_indices = list(np.where(np.in1d(feidegger_dataset.split, [8]))[0]) # type: ignore[attr-defined] + dev_indices = list(np.where(np.isin(feidegger_dataset.split, [8]))[0]) # type: ignore[attr-defined] dev = torch.utils.data.dataset.Subset(feidegger_dataset, dev_indices) - test_indices = list(np.where(np.in1d(feidegger_dataset.split, [9]))[0]) # type: ignore[attr-defined] + test_indices = list(np.where(np.isin(feidegger_dataset.split, [9]))[0]) # type: ignore[attr-defined] test = torch.utils.data.dataset.Subset(feidegger_dataset, test_indices) super().__init__(train, dev, test, name="feidegger") @@ -56,8 +55,8 @@ class FeideggerDataset(FlairDataset): def __init__(self, dataset_info, **kwargs) -> None: super().__init__() - self.data_points: List[DataPair] = [] - self.split: List[int] = [] + self.data_points: list[DataPair] = [] + self.split: list[int] = [] def identity(x): return x diff --git a/flair/datasets/text_text.py b/flair/datasets/text_text.py index 0bf0e91020..58a40d62c9 100644 --- a/flair/datasets/text_text.py +++ b/flair/datasets/text_text.py @@ -1,7 +1,7 @@ import logging import os from pathlib import Path -from typing import List, Optional, Union +from typing import Optional, Union import flair from flair.data import ( @@ -144,14 +144,15 @@ def __init__( self.total_sentence_count: int = 0 if self.in_memory: - self.bi_sentences: List[DataPair] = [] + self.bi_sentences: list[DataPair] = [] else: - self.source_lines: List[str] = [] - self.target_lines: List[str] = [] + self.source_lines: list[str] = [] + self.target_lines: list[str] = [] - with open(str(path_to_source), encoding="utf-8") as source_file, open( - str(path_to_target), encoding="utf-8" - ) as target_file: + with ( + open(str(path_to_source), encoding="utf-8") as source_file, + open(str(path_to_target), encoding="utf-8") as target_file, + ): source_line = source_file.readline() target_line = target_file.readline() @@ -204,7 +205,7 @@ class DataPairCorpus(Corpus): def __init__( self, data_folder: Union[str, Path], - columns: List[int] = [0, 1, 2], + columns: list[int] = [0, 1, 2], train_file=None, test_file=None, dev_file=None, @@ -318,7 +319,7 @@ class DataPairDataset(FlairDataset): def __init__( self, path_to_data: Union[str, Path], - columns: List[int] = [0, 1, 2], + columns: list[int] = [0, 1, 2], max_tokens_per_doc=-1, max_chars_per_doc=-1, use_tokenizer=True, @@ -368,11 +369,11 @@ def __init__( self.total_data_count: int = 0 if self.in_memory: - self.data_pairs: List[DataPair] = [] + self.data_pairs: list[DataPair] = [] else: - self.first_elements: List[str] = [] - self.second_elements: List[str] = [] - self.labels: List[Optional[str]] = [] + self.first_elements: list[str] = [] + self.second_elements: list[str] = [] + self.labels: list[Optional[str]] = [] with open(str(path_to_data), encoding=encoding) as source_file: source_line = source_file.readline() @@ -448,7 +449,7 @@ class DataTripleCorpus(Corpus): def __init__( self, data_folder: Union[str, Path], - columns: List[int] = [0, 1, 2, 3], + columns: list[int] = [0, 1, 2, 3], train_file=None, test_file=None, dev_file=None, @@ -563,7 +564,7 @@ class DataTripleDataset(FlairDataset): def __init__( self, path_to_data: Union[str, Path], - columns: List[int] = [0, 1, 2, 3], + columns: list[int] = [0, 1, 2, 3], max_tokens_per_doc=-1, max_chars_per_doc=-1, use_tokenizer=True, @@ -614,12 +615,12 @@ def __init__( self.total_data_count: int = 0 if self.in_memory: - self.data_triples: List[DataTriple] = [] + self.data_triples: list[DataTriple] = [] else: - self.first_elements: List[str] = [] - self.second_elements: List[str] = [] - self.third_elements: List[str] = [] - self.labels: List[Optional[str]] = [] + self.first_elements: list[str] = [] + self.second_elements: list[str] = [] + self.third_elements: list[str] = [] + self.labels: list[Optional[str]] = [] with open(str(path_to_data), encoding=encoding) as source_file: source_line = source_file.readline() @@ -828,9 +829,10 @@ def __init__( str(data_folder / "MNLI" / temp_file), ) - with open(data_folder / "MNLI" / dev_filename, "a", encoding="utf-8") as out_file, open( - data_folder / "MNLI" / temp_file, encoding="utf-8" - ) as in_file: + with ( + open(data_folder / "MNLI" / dev_filename, "a", encoding="utf-8") as out_file, + open(data_folder / "MNLI" / temp_file, encoding="utf-8") as in_file, + ): for line in in_file: fields = line.split("\t") reordered_columns = "\t".join(fields[column_id] for column_id in range(11)) diff --git a/flair/datasets/treebanks.py b/flair/datasets/treebanks.py index ed0f0135cd..21ae327691 100644 --- a/flair/datasets/treebanks.py +++ b/flair/datasets/treebanks.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import List, Optional, Union +from typing import Optional, Union import flair from flair.data import Corpus, FlairDataset, Sentence, Token @@ -82,7 +82,7 @@ def __init__( with open(str(self.path_to_conll_file), encoding="utf-8") as file: # option 1: read only sentence boundaries as offset positions if not self.in_memory: - self.indices: List[int] = [] + self.indices: list[int] = [] line = file.readline() position = 0 @@ -97,7 +97,7 @@ def __init__( # option 2: keep everything in memory if self.in_memory: - self.sentences: List[Sentence] = [] + self.sentences: list[Sentence] = [] while True: sentence = self._read_next_sentence(file) @@ -129,7 +129,7 @@ def __getitem__(self, index: int = 0) -> Sentence: def _read_next_sentence(self, file) -> Optional[Sentence]: line = file.readline() - tokens: List[Token] = [] + tokens: list[Token] = [] # current token ID token_idx = 0 @@ -143,7 +143,7 @@ def _read_next_sentence(self, file) -> Optional[Sentence]: newline_reached = False while line: line = line.strip() - fields: List[str] = re.split("\t+", line) + fields: list[str] = re.split("\t+", line) # end of sentence if line == "": diff --git a/flair/embeddings/base.py b/flair/embeddings/base.py index 154f2600be..294b41fac8 100644 --- a/flair/embeddings/base.py +++ b/flair/embeddings/base.py @@ -1,7 +1,8 @@ import inspect import logging from abc import abstractmethod -from typing import Any, Dict, Generic, List, Sequence, Type, Union +from collections.abc import Sequence +from typing import Any, Generic, Union import torch from torch.nn import Parameter, ParameterList @@ -37,7 +38,7 @@ def embedding_length(self) -> int: def embedding_type(self) -> str: raise NotImplementedError - def embed(self, data_points: Union[DT, List[DT]]) -> List[DT]: + def embed(self, data_points: Union[DT, list[DT]]) -> list[DT]: """Add embeddings to all words in a list of sentences. If embeddings are already added, updates only if embeddings are non-static. @@ -55,10 +56,10 @@ def _everything_embedded(self, data_points: Sequence[DT]) -> bool: return all(self.name in data_point._embeddings for data_point in data_points) @abstractmethod - def _add_embeddings_internal(self, sentences: List[DT]): + def _add_embeddings_internal(self, sentences: list[DT]): """Private method for adding embeddings to all words in a list of sentences.""" - def get_names(self) -> List[str]: + def get_names(self) -> list[str]: """Returns a list of embedding names. In most cases, it is just a list with one item, namely the name of @@ -67,9 +68,6 @@ def get_names(self) -> List[str]: """ return [self.name] - def get_named_embeddings_dict(self) -> Dict: - return {self.name: self} - @staticmethod def get_instance_parameters(locals: dict) -> dict: class_definition = locals.get("__class__") @@ -84,14 +82,14 @@ def get_instance_parameters(locals: dict) -> dict: return instance_parameters @classmethod - def from_params(cls, params: Dict[str, Any]) -> "Embeddings": + def from_params(cls, params: dict[str, Any]) -> "Embeddings": raise NotImplementedError - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: raise NotImplementedError @classmethod - def load_embedding(cls, params: Dict[str, Any]): + def load_embedding(cls, params: dict[str, Any]): state_dict = params.pop("state_dict", None) embedding = cls.from_params(params) @@ -155,7 +153,7 @@ def __init__(self, mixture_size: int, trainable: bool = False) -> None: requires_grad=trainable, ) - def forward(self, tensors: List[torch.Tensor]) -> torch.Tensor: + def forward(self, tensors: list[torch.Tensor]) -> torch.Tensor: """Forward pass of scalar mix. Computes a weighted average of the ``tensors``. The input tensors an be any shape @@ -203,7 +201,7 @@ def _everything_embedded(self, data_points: Sequence[Sentence]) -> bool: return True -EMBEDDING_CLASSES: Dict[str, Type[Embeddings]] = {} +EMBEDDING_CLASSES: dict[str, type[Embeddings]] = {} def register_embeddings(*args): @@ -225,7 +223,7 @@ def _register(cls): return _register -def load_embeddings(params: Dict[str, Any]) -> Embeddings: +def load_embeddings(params: dict[str, Any]) -> Embeddings: cls_name = params.pop("__cls__") cls = EMBEDDING_CLASSES[cls_name] return cls.load_embedding(params) diff --git a/flair/embeddings/document.py b/flair/embeddings/document.py index c1e73442e6..28867d889a 100644 --- a/flair/embeddings/document.py +++ b/flair/embeddings/document.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, Optional, Union, cast import torch from sklearn.feature_extraction.text import TfidfVectorizer @@ -67,7 +67,7 @@ def create_from_state(cls, **state): class DocumentPoolEmbeddings(DocumentEmbeddings): def __init__( self, - embeddings: Union[TokenEmbeddings, List[TokenEmbeddings]], + embeddings: Union[TokenEmbeddings, list[TokenEmbeddings]], fine_tune_mode: str = "none", pooling: str = "mean", ) -> None: @@ -114,7 +114,7 @@ def __init__( def embedding_length(self) -> int: return self.__embedding_length - def embed(self, sentences: Union[List[Sentence], Sentence]): + def embed(self, sentences: Union[list[Sentence], Sentence]): """Add embeddings to every sentence in the given list of sentences. If embeddings are already added, updates only if embeddings are non-static. @@ -146,18 +146,18 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): sentence.set_embedding(self.name, pooled_embedding) - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): pass def extra_repr(self): return f"fine_tune_mode={self.fine_tune_mode}, pooling={self.pooling}" @classmethod - def from_params(cls, params: Dict[str, Any]) -> "DocumentPoolEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "DocumentPoolEmbeddings": embeddings = cast(StackedEmbeddings, load_embeddings(params.pop("embeddings"))).embeddings return cls(embeddings=embeddings, **params) - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: return { "pooling": self.pooling, "fine_tune_mode": self.fine_tune_mode, @@ -169,7 +169,7 @@ def to_params(self) -> Dict[str, Any]: class DocumentTFIDFEmbeddings(DocumentEmbeddings): def __init__( self, - train_dataset: List[Sentence], + train_dataset: list[Sentence], vectorizer: Optional[TfidfVectorizer] = None, **vectorizer_params, ) -> None: @@ -203,7 +203,7 @@ def __init__( def embedding_length(self) -> int: return self.__embedding_length - def embed(self, sentences: Union[List[Sentence], Sentence]): + def embed(self, sentences: Union[list[Sentence], Sentence]): """Add embeddings to every sentence in the given list of sentences.""" # if only one sentence is passed, convert to list of sentence if isinstance(sentences, Sentence): @@ -215,14 +215,14 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): for sentence_id, sentence in enumerate(sentences): sentence.set_embedding(self.name, tfidf_vectors[sentence_id]) - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): pass @classmethod - def from_params(cls, params: Dict[str, Any]) -> "DocumentTFIDFEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "DocumentTFIDFEmbeddings": return cls(train_dataset=[], vectorizer=params["vectorizer"]) - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: return { "vectorizer": self.vectorizer, } @@ -232,7 +232,7 @@ def to_params(self) -> Dict[str, Any]: class DocumentRNNEmbeddings(DocumentEmbeddings): def __init__( self, - embeddings: List[TokenEmbeddings], + embeddings: list[TokenEmbeddings], hidden_size=128, rnn_layers=1, reproject_words: bool = True, @@ -317,7 +317,7 @@ def __init__( def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): """Add embeddings to all sentences in the given list of sentences. If embeddings are already added, update only if embeddings are non-static. @@ -332,7 +332,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): # embed words in the sentence self.embeddings.embed(sentences) - lengths: List[int] = [len(sentence.tokens) for sentence in sentences] + lengths: list[int] = [len(sentence.tokens) for sentence in sentences] longest_token_sequence_in_batch: int = max(lengths) pre_allocated_zero_tensor = torch.zeros( @@ -341,7 +341,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): device=flair.device, ) - all_embs: List[torch.Tensor] = [] + all_embs: list[torch.Tensor] = [] for sentence in sentences: all_embs += [emb for token in sentence for emb in token.get_each_embedding()] nb_padding_tokens = longest_token_sequence_in_batch - len(sentence) @@ -436,7 +436,7 @@ def to_params(self): return model_state @classmethod - def from_params(cls, params: Dict[str, Any]) -> "DocumentRNNEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "DocumentRNNEmbeddings": stacked_embeddings = load_embeddings(params["embeddings"]) assert isinstance(stacked_embeddings, StackedEmbeddings) return cls( @@ -484,7 +484,7 @@ def __setstate__(self, d): @register_embeddings class DocumentLMEmbeddings(DocumentEmbeddings): - def __init__(self, flair_embeddings: List[FlairEmbeddings]) -> None: + def __init__(self, flair_embeddings: list[FlairEmbeddings]) -> None: super().__init__() self.embeddings = flair_embeddings @@ -503,7 +503,7 @@ def __init__(self, flair_embeddings: List[FlairEmbeddings]) -> None: def embedding_length(self) -> int: return self._embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): for embedding in self.embeddings: embedding.embed(sentences) @@ -520,17 +520,17 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): return sentences - def get_names(self) -> List[str]: + def get_names(self) -> list[str]: if "__names" not in self.__dict__: self.__names = [name for embedding in self.embeddings for name in embedding.get_names()] return self.__names - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: return {"flair_embeddings": [embedding.save_embeddings(False) for embedding in self.embeddings]} @classmethod - def from_params(cls, params: Dict[str, Any]) -> "DocumentLMEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "DocumentLMEmbeddings": return cls([cast(FlairEmbeddings, load_embeddings(embedding)) for embedding in params["flair_embeddings"]]) @@ -566,7 +566,7 @@ def __init__( self.static_embeddings = True self.eval() - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: sentence_batches = [ sentences[i * self.batch_size : (i + 1) * self.batch_size] for i in range((len(sentences) + self.batch_size - 1) // self.batch_size) @@ -577,7 +577,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: return sentences - def _add_embeddings_to_sentences(self, sentences: List[Sentence]): + def _add_embeddings_to_sentences(self, sentences: list[Sentence]): # convert to plain strings, embedded in a list for the encode function sentences_plain_text = [sentence.to_plain_string() for sentence in sentences] @@ -591,10 +591,10 @@ def embedding_length(self) -> int: return self.model.get_sentence_embedding_dimension() @classmethod - def from_params(cls, params: Dict[str, Any]) -> "SentenceTransformerDocumentEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "SentenceTransformerDocumentEmbeddings": return cls(**params) - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: return { "model": self.model_name, "batch_size": self.batch_size, @@ -605,7 +605,7 @@ def to_params(self) -> Dict[str, Any]: class DocumentCNNEmbeddings(DocumentEmbeddings): def __init__( self, - embeddings: List[TokenEmbeddings], + embeddings: list[TokenEmbeddings], kernels=((100, 3), (100, 4), (100, 5)), reproject_words: bool = True, reproject_words_dimension: Optional[int] = None, @@ -673,7 +673,7 @@ def __init__( def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): """Add embeddings to all sentences in the given list of sentences. If embeddings are already added, update only if embeddings are non-static. @@ -689,7 +689,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): # embed words in the sentence self.embeddings.embed(sentences) - lengths: List[int] = [len(sentence.tokens) for sentence in sentences] + lengths: list[int] = [len(sentence.tokens) for sentence in sentences] padding_length: int = max(max(lengths), self.min_sequence_length) pre_allocated_zero_tensor = torch.zeros( @@ -698,7 +698,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): device=flair.device, ) - all_embs: List[torch.Tensor] = [] + all_embs: list[torch.Tensor] = [] for sentence in sentences: all_embs += [emb for token in sentence for emb in token.get_each_embedding()] nb_padding_tokens = padding_length - len(sentence) @@ -757,11 +757,11 @@ def _apply(self, fn): child_module._apply(fn) @classmethod - def from_params(cls, params: Dict[str, Any]) -> "DocumentCNNEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "DocumentCNNEmbeddings": embeddings = cast(StackedEmbeddings, load_embeddings(params.pop("embeddings"))).embeddings return cls(embeddings=embeddings, **params) - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: return { "embeddings": self.embeddings.save_embeddings(False), "kernels": self.kernels, diff --git a/flair/embeddings/image.py b/flair/embeddings/image.py index df6d1fadd9..5d79a04390 100644 --- a/flair/embeddings/image.py +++ b/flair/embeddings/image.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict, List, Optional +from typing import Any, Optional import torch import torch.nn.functional as F @@ -29,12 +29,12 @@ class ImageEmbeddings(Embeddings[Image]): def embedding_type(self) -> str: return "image-level" - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: # legacy pickle-like saving for image embeddings, as implementation details are not obvious return self.__getstate__() @classmethod - def from_params(cls, params: Dict[str, Any]) -> "Embeddings": + def from_params(cls, params: dict[str, Any]) -> "Embeddings": # legacy pickle-like loading for image embeddings, as implementation details are not obvious embedding = cls.__new__(cls) embedding.__setstate__(params) @@ -53,7 +53,7 @@ def __init__(self, transforms) -> None: self.static_embeddings = True super().__init__() - def _add_embeddings_internal(self, images: List[Image]): + def _add_embeddings_internal(self, images: list[Image]): for image in images: image_data = self.PIL.Image.open(image.imageURL) image_data.load() @@ -77,7 +77,7 @@ def __init__(self, url2tensor_dict, name) -> None: self.static_embeddings = True super().__init__() - def _add_embeddings_internal(self, images: List[Image]): + def _add_embeddings_internal(self, images: list[Image]): for image in images: if image.imageURL in self.url2tensor_dict: image.set_embedding(self.name, self.url2tensor_dict[image.imageURL]) @@ -137,7 +137,7 @@ def __init__(self, name, pretrained=True, transforms=None) -> None: else: raise Exception(f"Image embeddings {name} not available.") - def _add_embeddings_internal(self, images: List[Image]): + def _add_embeddings_internal(self, images: list[Image]): image_tensor = torch.stack([self.transforms(image.data) for image in images]) image_embeddings = self.features(image_tensor) image_embeddings = ( @@ -163,7 +163,7 @@ def __init__(self, feats_in, convnet_parms, posnet_parms, transformer_parms) -> adaptive_pool_func_map = {"max": AdaptiveMaxPool2d, "avg": AdaptiveAvgPool2d} - convnet_arch: List[Any] = [] if convnet_parms["dropout"][0] <= 0 else [Dropout2d(convnet_parms["dropout"][0])] + convnet_arch: list[Any] = [] if convnet_parms["dropout"][0] <= 0 else [Dropout2d(convnet_parms["dropout"][0])] convnet_arch.extend( [ Conv2d( @@ -266,7 +266,7 @@ def forward(self, x): return x - def _add_embeddings_internal(self, images: List[Image]): + def _add_embeddings_internal(self, images: list[Image]): image_tensor = torch.stack([image.data for image in images]) image_embeddings = self.forward(image_tensor) for image_id, image in enumerate(images): diff --git a/flair/embeddings/legacy.py b/flair/embeddings/legacy.py index b2658e2d2f..4b3d2a9517 100644 --- a/flair/embeddings/legacy.py +++ b/flair/embeddings/legacy.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import List, Optional, Union +from typing import Optional, Union import torch from deprecated.sphinx import deprecated @@ -110,12 +110,12 @@ def use_layers_top(self, x): def use_layers_average(self, x): return torch.mean(torch.stack(x), 0) - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: # ELMoEmbeddings before Release 0.5 did not set self.embedding_mode_fn if not getattr(self, "embedding_mode_fn", None): self.embedding_mode_fn = self.use_layers_all - sentence_words: List[List[str]] = [] + sentence_words: list[list[str]] = [] for sentence in sentences: sentence_words.append([token.text for token in sentence]) @@ -394,7 +394,7 @@ def __getstate__(self): def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: # if cache is used, try setting embeddings from cache first if "cache" in self.__dict__ and self.cache is not None: # try populating embeddings from cache @@ -463,7 +463,7 @@ class DocumentMeanEmbeddings(DocumentEmbeddings): version="0.3.1", reason="The functionality of this class is moved to 'DocumentPoolEmbeddings'", ) - def __init__(self, token_embeddings: List[TokenEmbeddings]) -> None: + def __init__(self, token_embeddings: list[TokenEmbeddings]) -> None: """The constructor takes a list of embeddings to be combined.""" super().__init__() @@ -478,7 +478,7 @@ def __init__(self, token_embeddings: List[TokenEmbeddings]) -> None: def embedding_length(self) -> int: return self.__embedding_length - def embed(self, sentences: Union[List[Sentence], Sentence]): + def embed(self, sentences: Union[list[Sentence], Sentence]): """Add embeddings to every sentence in the given list of sentences. If embeddings are already added, updates only if embeddings are non-static. """ @@ -506,7 +506,7 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): sentence.set_embedding(self.name, mean_embedding) - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): pass @@ -517,7 +517,7 @@ class DocumentLSTMEmbeddings(DocumentEmbeddings): ) def __init__( self, - embeddings: List[TokenEmbeddings], + embeddings: list[TokenEmbeddings], hidden_size=128, rnn_layers=1, reproject_words: bool = True, @@ -587,7 +587,7 @@ def __init__( def embedding_length(self) -> int: return self.__embedding_length - def embed(self, sentences: Union[List[Sentence], Sentence]): + def embed(self, sentences: Union[list[Sentence], Sentence]): """Add embeddings to all sentences in the given list of sentences. If embeddings are already added, update only if embeddings are non-static. """ @@ -604,7 +604,7 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): longest_token_sequence_in_batch: int = len(sentences[0]) all_sentence_tensors = [] - lengths: List[int] = [] + lengths: list[int] = [] # go through each sentence in batch for _i, sentence in enumerate(sentences): @@ -669,5 +669,5 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): sentence = sentences[sentence_no] sentence.set_embedding(self.name, embedding) - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): pass diff --git a/flair/embeddings/token.py b/flair/embeddings/token.py index b068305800..700eaf4c45 100644 --- a/flair/embeddings/token.py +++ b/flair/embeddings/token.py @@ -4,7 +4,7 @@ import tempfile from collections import Counter from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import Any, Optional, Union import numpy as np import torch @@ -64,7 +64,7 @@ def create_from_state(cls, **state): class StackedEmbeddings(TokenEmbeddings): """A stack of embeddings, used if you need to combine several different embedding types.""" - def __init__(self, embeddings: List[TokenEmbeddings], overwrite_names: bool = True) -> None: + def __init__(self, embeddings: list[TokenEmbeddings], overwrite_names: bool = True) -> None: """The constructor takes a list of embeddings to be combined.""" super().__init__() @@ -88,7 +88,7 @@ def __init__(self, embeddings: List[TokenEmbeddings], overwrite_names: bool = Tr self.__embedding_length += embedding.embedding_length self.eval() - def embed(self, sentences: Union[Sentence, List[Sentence]], static_embeddings: bool = True): + def embed(self, sentences: Union[Sentence, list[Sentence]], static_embeddings: bool = True): # if only one sentence is passed, convert to list of sentence if type(sentences) is Sentence: sentences = [sentences] @@ -104,7 +104,7 @@ def embedding_type(self) -> str: def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: for embedding in self.embeddings: embedding._add_embeddings_internal(sentences) @@ -113,7 +113,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: def __str__(self) -> str: return f'StackedEmbeddings [{",".join([str(e) for e in self.embeddings])}]' - def get_names(self) -> List[str]: + def get_names(self) -> list[str]: """Returns a list of embedding names. In most cases, it is just a list with one item, namely the name of this embedding. But in some cases, the @@ -126,13 +126,6 @@ def get_names(self) -> List[str]: return self.__names - def get_named_embeddings_dict(self) -> Dict: - named_embeddings_dict = {} - for embedding in self.embeddings: - named_embeddings_dict.update(embedding.get_named_embeddings_dict()) - - return named_embeddings_dict - @classmethod def from_params(cls, params): embeddings = [load_embeddings(p) for p in params["embeddings"]] @@ -154,7 +147,7 @@ def __init__( force_cpu: bool = True, stable: bool = False, no_header: bool = False, - vocab: Optional[Dict[str, int]] = None, + vocab: Optional[dict[str, int]] = None, embedding_length: Optional[int] = None, name: Optional[str] = None, ) -> None: @@ -334,10 +327,10 @@ def get_cached_token_index(self, word: str) -> int: else: return len(self.vocab) # token - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: tokens = [token for sentence in sentences for token in sentence.tokens] - word_indices: List[int] = [] + word_indices: list[int] = [] for token in tokens: word = token.text if self.field is None else token.get_label(self.field).value word_indices.append(self.get_cached_token_index(word)) @@ -386,7 +379,7 @@ def __getattribute__(self, item): return None return super().__getattribute__(item) - def __setstate__(self, state: Dict[str, Any]): + def __setstate__(self, state: dict[str, Any]): state.pop("get_cached_vec", None) state.setdefault("embeddings", state["name"]) state.setdefault("force_cpu", True) @@ -416,10 +409,10 @@ def __setstate__(self, state: Dict[str, Any]): super().__setstate__(state) @classmethod - def from_params(cls, params: Dict[str, Any]) -> "WordEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "WordEmbeddings": return cls(embeddings=None, **params) - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: return { "vocab": self.vocab, "stable": self.stable, @@ -487,7 +480,7 @@ def __init__( def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): for sentence in sentences: tokens_char_indices = [] @@ -544,10 +537,10 @@ def __str__(self) -> str: return self.name @classmethod - def from_params(cls, params: Dict[str, Any]) -> "CharacterEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "CharacterEmbeddings": return cls(**params) - def to_params(self) -> Dict[str, Any]: + def to_params(self) -> dict[str, Any]: return { "path_to_char_dict": self.char_dictionary, "char_embedding_dim": self.char_embedding_dim, @@ -793,7 +786,7 @@ def train(self, mode=True): def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: # gradients are enable if fine-tuning is enabled gradient_context = torch.enable_grad() if self.fine_tune else torch.no_grad() @@ -885,7 +878,7 @@ def from_params(cls, params): lm = LanguageModel(**model_params) return cls(lm, **params) - def __setstate__(self, d: Dict[str, Any]): + def __setstate__(self, d: dict[str, Any]): # make compatible with old models d.setdefault("fine_tune", False) d.setdefault("chars_per_chunk", 512) @@ -920,8 +913,8 @@ def __init__( self.name = self.context_embeddings.name + "-context" # these fields are for the embedding memory - self.word_embeddings: Dict[str, torch.Tensor] = {} - self.word_count: Dict[str, int] = {} + self.word_embeddings: dict[str, torch.Tensor] = {} + self.word_count: dict[str, int] = {} # whether to add only capitalized words to memory (faster runtime and lower memory consumption) self.only_capitalized = only_capitalized @@ -940,7 +933,7 @@ def train(self, mode=True): self.word_embeddings = {} self.word_count = {} - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: self.context_embeddings.embed(sentences) # if we keep a pooling, it needs to be updated continuously @@ -989,10 +982,10 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: def embedding_length(self) -> int: return self.__embedding_length - def get_names(self) -> List[str]: + def get_names(self) -> list[str]: return [self.name, self.context_embeddings.name] - def __setstate__(self, d: Dict[str, Any]): + def __setstate__(self, d: dict[str, Any]): super().__setstate__(d) if flair.device.type != "cpu": @@ -1073,7 +1066,7 @@ def get_cached_vec(self, word: str) -> torch.Tensor: word_embedding = torch.tensor(word_embedding.tolist(), device=flair.device, dtype=torch.float) return word_embedding - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: for sentence in sentences: for token in sentence.tokens: word = token.text if self.field is None else token.get_label(self.field).value @@ -1152,7 +1145,7 @@ def __init__( def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: tokens = [t for sentence in sentences for t in sentence.tokens] if self.field == "text": @@ -1240,7 +1233,7 @@ def num_embeddings(self) -> int: def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): def get_idx_for_item(text): hash_function = hashlib.new(self.__hash_method) hash_function.update(bytes(str(text), "utf-8")) @@ -1282,7 +1275,7 @@ def __init__( self.name: str = "muse-crosslingual" self.static_embeddings = True self.__embedding_length: int = 300 - self.language_embeddings: Dict[str, Any] = {} + self.language_embeddings: dict[str, Any] = {} (KeyedVectors,) = lazy_import("word-embeddings", "gensim.models", "KeyedVectors") self.kv = KeyedVectors super().__init__() @@ -1304,7 +1297,7 @@ def get_cached_vec(self, language_code: str, word: str) -> torch.Tensor: word_embedding = torch.tensor(word_embedding, device=flair.device, dtype=torch.float) return word_embedding - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: for _i, sentence in enumerate(sentences): language_code = sentence.get_language_code() supported = [ @@ -1465,10 +1458,10 @@ def _preprocess(self, text: str) -> str: def embedding_length(self) -> int: return self.__embedding_length - def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + def _add_embeddings_internal(self, sentences: list[Sentence]) -> list[Sentence]: tokens = [token for sentence in sentences for token in sentence.tokens] - word_indices: List[List[int]] = [] + word_indices: list[list[int]] = [] for token in tokens: word = token.text if self.field is None else token.get_label(self.field).value @@ -1601,13 +1594,13 @@ def __init__(self, embeddings: str, model: str = "skip", size: int = 100) -> Non else: embeddings_path = embeddings - log.info("Reading embeddings from %s" % embeddings_path) + log.info("Reading embeddings from %s", embeddings_path) super().__init__( embeddings=str(extract_single_zip_file(embeddings_path, cache_dir=cache_dir)), name="NILC-" + embeddings ) @classmethod - def from_params(cls, params: Dict[str, Any]) -> "WordEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "WordEmbeddings": # no need to recreate as NILCEmbeddings return WordEmbeddings(embeddings=None, **params) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index 1e88787deb..d09ed33699 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -8,7 +8,7 @@ from abc import abstractmethod from io import BytesIO from pathlib import Path -from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union, cast +from typing import Any, Literal, Optional, Union, cast import torch import transformers @@ -44,7 +44,7 @@ @torch.jit.script_if_tracing -def pad_sequence_embeddings(all_hidden_states: List[torch.Tensor]) -> torch.Tensor: +def pad_sequence_embeddings(all_hidden_states: list[torch.Tensor]) -> torch.Tensor: embedding_length = all_hidden_states[0].shape[1] longest_token_sequence_in_batch = 0 for hidden_states in all_hidden_states: @@ -218,13 +218,13 @@ def document_max_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: def _legacy_reconstruct_word_ids( - embedding: "TransformerBaseEmbeddings", flair_tokens: List[List[str]] -) -> List[List[Optional[int]]]: + embedding: "TransformerBaseEmbeddings", flair_tokens: list[list[str]] +) -> list[list[Optional[int]]]: word_ids_list = [] max_len = 0 for tokens in flair_tokens: token_texts = embedding.tokenizer.tokenize(" ".join(tokens), is_split_into_words=True) - token_ids = cast(List[int], embedding.tokenizer.convert_tokens_to_ids(token_texts)) + token_ids = cast(list[int], embedding.tokenizer.convert_tokens_to_ids(token_texts)) expanded_token_ids = embedding.tokenizer.build_inputs_with_special_tokens(token_ids) j = 0 for _i, token_id in enumerate(token_ids): @@ -264,10 +264,10 @@ def _get_processed_token_text(tokenizer, token: str) -> str: return token_text.strip() -def _reconstruct_word_ids_from_subtokens(embedding, tokens: List[str], subtokens: List[str]): +def _reconstruct_word_ids_from_subtokens(embedding, tokens: list[str], subtokens: list[str]): word_iterator = iter(enumerate(_get_processed_token_text(embedding.tokenizer, token) for token in tokens)) token_id, token_text = next(word_iterator) - word_ids: List[Optional[int]] = [] + word_ids: list[Optional[int]] = [] reconstructed_token = "" subtoken_count = 0 processed_first_token = False @@ -504,10 +504,10 @@ def embedding_type(self) -> str: return "word-level" if self.token_embedding else "sentence-level" @abstractmethod - def _forward_tensors(self, tensors) -> Dict[str, torch.Tensor]: + def _forward_tensors(self, tensors) -> dict[str, torch.Tensor]: return self(**tensors) - def prepare_tensors(self, sentences: List[Sentence], device: Optional[torch.device] = None): + def prepare_tensors(self, sentences: list[Sentence], device: Optional[torch.device] = None): if device is None: device = flair.device flair_tokens, offsets, lengths = self.__gather_flair_tokens(sentences) @@ -535,13 +535,13 @@ def prepare_tensors(self, sentences: List[Sentence], device: Optional[torch.devi def __build_transformer_model_inputs( self, - sentences: List[Sentence], - offsets: List[int], - sentence_lengths: List[int], - flair_tokens: List[List[Token]], + sentences: list[Sentence], + offsets: list[int], + sentence_lengths: list[int], + flair_tokens: list[list[Token]], device: torch.device, ): - tokenizer_kwargs: Dict[str, Any] = {} + tokenizer_kwargs: dict[str, Any] = {} if self.tokenizer_needs_ocr_boxes: tokenizer_kwargs["boxes"] = [[t.get_metadata("bbox") for t in tokens] for tokens in flair_tokens] else: @@ -662,7 +662,7 @@ def __build_transformer_model_inputs( return model_kwargs - def __gather_flair_tokens(self, sentences: List[Sentence]) -> Tuple[List[List[Token]], List[int], List[int]]: + def __gather_flair_tokens(self, sentences: list[Sentence]) -> tuple[list[list[Token]], list[int], list[int]]: offsets = [] lengths = [] if self.context_length > 0: @@ -686,7 +686,7 @@ def __gather_flair_tokens(self, sentences: List[Sentence]) -> Tuple[List[List[To lengths.append(len(sentence)) return sentence_tokens, offsets, lengths - def _expand_sentence_with_context(self, sentence) -> Tuple[List[Token], int]: + def _expand_sentence_with_context(self, sentence) -> tuple[list[Token], int]: # fields to store left and right context left_context = [] right_context = [] @@ -722,7 +722,7 @@ def __extract_token_embeddings(self, sentence_embeddings, sentences): for token_embedding, token in zip(token_embeddings, sentence): token.set_embedding(self.name, token_embedding) - def _add_embeddings_internal(self, sentences: List[Sentence]): + def _add_embeddings_internal(self, sentences: list[Sentence]): tensors = self.prepare_tensors(sentences, device=self.force_device) gradient_context = torch.enable_grad() if (self.fine_tune and self.training) else torch.no_grad() with gradient_context: @@ -739,7 +739,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): @register_embeddings class TransformerOnnxEmbeddings(TransformerBaseEmbeddings): - def __init__(self, onnx_model: str, providers: List = [], session_options: Optional[Dict] = None, **kwargs) -> None: + def __init__(self, onnx_model: str, providers: list = [], session_options: Optional[dict] = None, **kwargs) -> None: # onnx prepares numpy arrays, no mather if it runs on gpu or cpu, the input is on cpu first. super().__init__(**kwargs, force_device=torch.device("cpu")) self.onnx_model = onnx_model @@ -756,7 +756,7 @@ def to_params(self): return params @classmethod - def from_params(cls, params: Dict[str, Any]) -> "TransformerOnnxEmbeddings": + def from_params(cls, params: dict[str, Any]) -> "TransformerOnnxEmbeddings": params["tokenizer"] = cls._tokenizer_from_bytes(params.pop("tokenizer_data")) params["feature_extractor"] = cls._feature_extractor_from_bytes(params.pop("feature_extractor_data", None)) return cls(**params) @@ -812,7 +812,7 @@ def quantize_model(self, quantize_model_path, use_external_data_format: bool = F self.onnx_model = quantize_model_path self.create_session() - def _forward_tensors(self, tensors) -> Dict[str, torch.Tensor]: + def _forward_tensors(self, tensors) -> dict[str, torch.Tensor]: input_array = {k: v.numpy() for k, v in tensors.items()} embeddings = self.session.run([], input_array) @@ -854,9 +854,9 @@ def export_from_embedding( cls, path: Union[str, Path], embedding: "TransformerEmbeddings", - example_sentences: List[Sentence], + example_sentences: list[Sentence], opset_version: int = 14, - providers: Optional[List] = None, + providers: Optional[list] = None, session_options: Optional[dict] = None, ): path = str(path) @@ -903,7 +903,7 @@ def export_from_embedding( @register_embeddings class TransformerJitEmbeddings(TransformerBaseEmbeddings): - def __init__(self, jit_model: Union[bytes, ScriptModule], param_names: List[str], **kwargs) -> None: + def __init__(self, jit_model: Union[bytes, ScriptModule], param_names: list[str], **kwargs) -> None: super().__init__(**kwargs) if isinstance(jit_model, bytes): buffer = BytesIO(jit_model) @@ -925,12 +925,12 @@ def to_params(self): return state @classmethod - def from_params(cls, params: Dict[str, Any]) -> "Embeddings": + def from_params(cls, params: dict[str, Any]) -> "Embeddings": params["tokenizer"] = cls._tokenizer_from_bytes(params.pop("tokenizer_data")) params["feature_extractor"] = cls._feature_extractor_from_bytes(params.pop("feature_extractor_data", None)) return cls(**params) - def _forward_tensors(self, tensors) -> Dict[str, torch.Tensor]: + def _forward_tensors(self, tensors) -> dict[str, torch.Tensor]: parameters = [] for param in self.param_names: parameters.append(tensors[param]) @@ -945,13 +945,13 @@ def _forward_tensors(self, tensors) -> Dict[str, torch.Tensor]: raise ValueError("either 'token_embedding' or 'document_embedding' needs to be set.") @classmethod - def create_from_embedding(cls, module: ScriptModule, embedding: "TransformerEmbeddings", param_names: List[str]): + def create_from_embedding(cls, module: ScriptModule, embedding: "TransformerEmbeddings", param_names: list[str]): return cls(jit_model=module, param_names=param_names, **embedding.to_args()) @classmethod def parameter_to_list( - cls, embedding: "TransformerEmbeddings", wrapper: torch.nn.Module, sentences: List[Sentence] - ) -> Tuple[List[str], List[torch.Tensor]]: + cls, embedding: "TransformerEmbeddings", wrapper: torch.nn.Module, sentences: list[Sentence] + ) -> tuple[list[str], list[torch.Tensor]]: tensors = embedding.prepare_tensors(sentences) param_names = list(inspect.signature(wrapper.forward).parameters.keys()) params = [] @@ -998,7 +998,7 @@ def __init__( @register_embeddings class TransformerEmbeddings(TransformerBaseEmbeddings): - onnx_cls: Type[TransformerOnnxEmbeddings] = TransformerOnnxEmbeddings + onnx_cls: type[TransformerOnnxEmbeddings] = TransformerOnnxEmbeddings def __init__( self, @@ -1021,11 +1021,11 @@ def __init__( force_max_length: bool = False, needs_manual_ocr: Optional[bool] = None, use_context_separator: bool = True, - transformers_tokenizer_kwargs: Dict[str, Any] = {}, - transformers_config_kwargs: Dict[str, Any] = {}, - transformers_model_kwargs: Dict[str, Any] = {}, + transformers_tokenizer_kwargs: dict[str, Any] = {}, + transformers_config_kwargs: dict[str, Any] = {}, + transformers_model_kwargs: dict[str, Any] = {}, peft_config=None, - peft_gradient_checkpointing_kwargs: Optional[Dict[str, Any]] = {}, + peft_gradient_checkpointing_kwargs: Optional[dict[str, Any]] = {}, **kwargs, ) -> None: """Instantiate transformers embeddings. @@ -1503,11 +1503,11 @@ def forward( result["token_embeddings"] = all_token_embeddings return result - def _forward_tensors(self, tensors) -> Dict[str, torch.Tensor]: + def _forward_tensors(self, tensors) -> dict[str, torch.Tensor]: return self.forward(**tensors) def export_onnx( - self, path: Union[str, Path], example_sentences: List[Sentence], **kwargs + self, path: Union[str, Path], example_sentences: list[Sentence], **kwargs ) -> TransformerOnnxEmbeddings: """Export TransformerEmbeddings to OnnxFormat. diff --git a/flair/file_utils.py b/flair/file_utils.py index 7a0118822b..518d69e809 100644 --- a/flair/file_utils.py +++ b/flair/file_utils.py @@ -12,8 +12,9 @@ import typing import warnings import zipfile +from collections.abc import Sequence from pathlib import Path -from typing import Optional, Sequence, Tuple, Union, cast +from typing import Optional, Union, cast from urllib.parse import urlparse import boto3 @@ -28,10 +29,10 @@ logger = logging.getLogger("flair") -url_proxies: Optional[typing.Dict[str, str]] = None +url_proxies: Optional[dict[str, str]] = None -def set_proxies(proxies: typing.Dict[str, str]) -> None: +def set_proxies(proxies: dict[str, str]) -> None: r"""Allows for data downloaded from urls to be forwarded to a proxy. see https://requests.readthedocs.io/en/latest/user/advanced/#proxies @@ -74,7 +75,7 @@ def url_to_filename(url: str, etag: Optional[str] = None) -> str: return decoded -def filename_to_url(filename: str) -> Tuple[str, Optional[str]]: +def filename_to_url(filename: str) -> tuple[str, Optional[str]]: """Recovers the the url from the encoded filename. Returns it and the ETag (which may be ``None``) @@ -374,7 +375,7 @@ def create_cache(self, *args, **kwargs): return decorator -def load_torch_state(model_file: str) -> typing.Dict[str, typing.Any]: +def load_torch_state(model_file: str) -> dict[str, typing.Any]: with warnings.catch_warnings(): warnings.filterwarnings("ignore") # load_big_file is a workaround byhttps://github.com/highway11git diff --git a/flair/inference_utils.py b/flair/inference_utils.py index 0310671534..c811bf39a1 100644 --- a/flair/inference_utils.py +++ b/flair/inference_utils.py @@ -126,7 +126,7 @@ def create_stores(model, backend="sqlite"): Also deletes the original vectors to save memory. """ for embedding in WordEmbeddingsStore._word_embeddings(model): - if type(embedding) == WordEmbeddings: + if isinstance(embedding, WordEmbeddings): WordEmbeddingsStore(embedding, backend) del embedding.precomputed_word_embeddings @@ -135,7 +135,7 @@ def load_stores(model, backend="sqlite"): """Loads the db versions of all word embeddings in the model.""" embeds = WordEmbeddingsStore._word_embeddings(model) for i, embedding in enumerate(embeds): - if type(embedding) == WordEmbeddings: + if isinstance(embedding, WordEmbeddings): embeds[i] = WordEmbeddingsStore(embedding, backend) @staticmethod diff --git a/flair/models/entity_linker_model.py b/flair/models/entity_linker_model.py index 9f516a703c..0f1c916bfb 100644 --- a/flair/models/entity_linker_model.py +++ b/flair/models/entity_linker_model.py @@ -2,7 +2,7 @@ import re from functools import lru_cache from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Set, Union, cast +from typing import Any, Callable, Optional, Union, cast from unicodedata import category import torch @@ -19,9 +19,9 @@ class CandidateGenerator: """Given a string, the CandidateGenerator returns possible target classes as candidates.""" - def __init__(self, candidates: Union[str, Dict[str, List[str]]], backoff: bool = True) -> None: + def __init__(self, candidates: Union[str, dict[str, list[str]]], backoff: bool = True) -> None: # internal candidate lists of generator - self.mention_to_candidates_map: Dict = {} + self.mention_to_candidates_map: dict[str, list[str]] = {} # load Zelda candidates if so passed if isinstance(candidates, str) and candidates.lower() == "zelda": @@ -39,16 +39,15 @@ def __init__(self, candidates: Union[str, Dict[str, List[str]]], backoff: bool = self.mention_to_candidates_map = candidate_lists - elif isinstance(candidates, Dict): + elif isinstance(candidates, dict): self.mention_to_candidates_map = candidates else: raise ValueError(f"'{candidates}' could not be loaded.") - self.mention_to_candidates_map = cast(Dict[str, List[str]], self.mention_to_candidates_map) # if lower casing is enabled, create candidate lists of lower cased versions self.backoff = backoff if self.backoff: # create a new dictionary for lower cased mentions - lowercased_mention_to_candidates_map: Dict = {} + lowercased_mention_to_candidates_map: dict[str, list[str]] = {} # go through each mention and its candidates for mention, candidates_list in self.mention_to_candidates_map.items(): @@ -56,8 +55,8 @@ def __init__(self, candidates: Union[str, Dict[str, List[str]]], backoff: bool = # check if backoff mention already seen. If so, add candidates. Else, create new entry. if backoff_mention in lowercased_mention_to_candidates_map: current_candidates = lowercased_mention_to_candidates_map[backoff_mention] - lowercased_mention_to_candidates_map[backoff_mention] = set(current_candidates).union( - candidates_list + lowercased_mention_to_candidates_map[backoff_mention] = list( + set(current_candidates).union(candidates_list) ) else: lowercased_mention_to_candidates_map[backoff_mention] = candidates_list @@ -72,7 +71,7 @@ def _make_backoff_string(self, mention: str) -> str: backoff_mention = re.sub(" +", " ", backoff_mention) return backoff_mention - def get_candidates(self, mention: str) -> Set[str]: + def get_candidates(self, mention: str) -> set[str]: """Given a mention, this method returns a set of candidate classes.""" if self.backoff: mention = self._make_backoff_string(mention) @@ -125,7 +124,7 @@ def __init__( self._label_type = label_type self._span_label_type = span_label_type - cases: Dict[str, Callable[[Span, List[str]], torch.Tensor]] = { + cases: dict[str, Callable[[Span, list[str]], torch.Tensor]] = { "average": self.emb_mean, "first": self.emb_first, "last": self.emb_last, @@ -155,7 +154,7 @@ def emb_firstAndLast(self, span: Span, embedding_names): def emb_mean(self, span, embedding_names): return torch.mean(torch.stack([token.get_embedding(embedding_names) for token in span], 0), 0) - def _get_data_points_from_sentence(self, sentence: Sentence) -> List[Span]: + def _get_data_points_from_sentence(self, sentence: Sentence) -> list[Span]: if self._span_label_type is not None: spans = sentence.get_spans(self._span_label_type) # only use span label type if there are predictions, otherwise search for output label type (training labels) @@ -223,7 +222,7 @@ def _init_model_with_state_dict(cls, state, **kwargs): def label_type(self): return self._label_type - def _mask_scores(self, scores: torch.Tensor, data_points: List[Span]): + def _mask_scores(self, scores: torch.Tensor, data_points: list[Span]): if not self.candidates: return scores @@ -242,9 +241,7 @@ def _mask_scores(self, scores: torch.Tensor, data_points: List[Span]): return masked_scores @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "SpanClassifier": - from typing import cast - + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "SpanClassifier": return cast("SpanClassifier", super().load(model_path=model_path)) diff --git a/flair/models/entity_mention_linking.py b/flair/models/entity_mention_linking.py index cecf2d9e57..5a1382dd60 100644 --- a/flair/models/entity_mention_linking.py +++ b/flair/models/entity_mention_linking.py @@ -4,9 +4,10 @@ import re import string from abc import ABC, abstractmethod +from collections.abc import Sequence from enum import Enum, auto from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Type, Union, cast +from typing import Any, Optional, Union, cast import numpy as np import torch @@ -89,7 +90,7 @@ "chemical": "ctd-chemicals", } -BIOMEDICAL_DICTIONARIES: Dict[str, Type] = { +BIOMEDICAL_DICTIONARIES: dict[str, type] = { "ctd-diseases": CTD_DISEASES_DICTIONARY, "ctd-chemicals": CTD_CHEMICALS_DICTIONARY, "ncbi-gene": NCBI_GENE_HUMAN_DICTIONARY, @@ -151,7 +152,7 @@ def load_dictionary( class EntityPreprocessor(ABC): """A pre-processor used to transform / clean both entity mentions and entity names.""" - def initialize(self, sentences: List[Sentence]) -> None: + def initialize(self, sentences: list[Sentence]) -> None: """Initializes the pre-processor for a batch of sentences. This may be necessary for more sophisticated transformations. @@ -187,14 +188,14 @@ def process_entity_name(self, entity_name: str) -> str: """ @classmethod - def _from_state(cls, state_dict: Dict[str, Any]) -> "EntityPreprocessor": + def _from_state(cls, state_dict: dict[str, Any]) -> "EntityPreprocessor": if inspect.isabstract(cls): cls_name = state_dict.pop("__cls__", None) return get_state_subclass_by_name(cls, cls_name)._from_state(state_dict) else: return cls(**state_dict) - def _get_state(self) -> Dict[str, Any]: + def _get_state(self) -> dict[str, Any]: return {"__cls__": self.__class__.__name__} @@ -237,7 +238,7 @@ def process_entity_name(self, entity_name: str) -> str: return entity_name - def _get_state(self) -> Dict[str, Any]: + def _get_state(self) -> dict[str, Any]: return { **super()._get_state(), "lowercase": self.lowercase, @@ -270,9 +271,9 @@ def __init__( self.ab3p = pyab3p.Ab3p() self.preprocessor = preprocessor - self.abbreviation_dict: Dict[str, Dict[str, str]] = {} + self.abbreviation_dict: dict[str, dict[str, str]] = {} - def initialize(self, sentences: List[Sentence]) -> None: + def initialize(self, sentences: list[Sentence]) -> None: self.abbreviation_dict = self._build_abbreviation_dict(sentences) def process_mention(self, entity_mention: str, sentence: Optional[Sentence] = None) -> str: @@ -303,7 +304,7 @@ def process_entity_name(self, entity_name: str) -> str: return entity_name - def _build_abbreviation_dict(self, sentences: List[flair.data.Sentence]) -> Dict[str, Dict[str, str]]: + def _build_abbreviation_dict(self, sentences: list[flair.data.Sentence]) -> dict[str, dict[str, str]]: """Processes the given sentences with the Ab3P tool. The function returns a (nested) dictionary containing the abbreviations found for each sentence, e.g.: @@ -321,7 +322,7 @@ def _build_abbreviation_dict(self, sentences: List[flair.data.Sentence]) -> Dict Returns: abbreviation_dict: abbreviations and their resolution detected in each input sentence """ - abbreviation_dict: Dict[str, Dict[str, str]] = {} + abbreviation_dict: dict[str, dict[str, str]] = {} for sentence in sentences: sentence_text = sentence.to_original_text() @@ -331,14 +332,14 @@ def _build_abbreviation_dict(self, sentences: List[flair.data.Sentence]) -> Dict return abbreviation_dict - def _get_state(self) -> Dict[str, Any]: + def _get_state(self) -> dict[str, Any]: return { **super()._get_state(), "preprocessor": None if self.preprocessor is None else self.preprocessor._get_state(), } @classmethod - def _from_state(cls, state_dict: Dict[str, Any]) -> "EntityPreprocessor": + def _from_state(cls, state_dict: dict[str, Any]) -> "EntityPreprocessor": return cls( preprocessor=( None @@ -364,7 +365,7 @@ def index(self, dictionary: EntityLinkingDictionary, preprocessor: Optional[Enti """ @abstractmethod - def search(self, entity_mentions: List[str], top_k: int) -> List[List[Tuple[str, float]]]: + def search(self, entity_mentions: list[str], top_k: int) -> list[list[tuple[str, float]]]: """Returns the top-k entity / concept identifiers for each entity mention. Args: @@ -376,14 +377,14 @@ def search(self, entity_mentions: List[str], top_k: int) -> List[List[Tuple[str, """ @classmethod - def _from_state(cls, state_dict: Dict[str, Any]) -> "CandidateSearchIndex": + def _from_state(cls, state_dict: dict[str, Any]) -> "CandidateSearchIndex": if inspect.isabstract(cls): cls_name = state_dict.pop("__cls__", None) return get_state_subclass_by_name(cls, cls_name)._from_state(state_dict) else: return cls(**state_dict) - def _get_state(self) -> Dict[str, Any]: + def _get_state(self) -> dict[str, Any]: return {"__cls__": self.__class__.__name__} @@ -396,7 +397,7 @@ def __init__(self): Args: name_to_id_index: internal state, should only be set when loading an initialized index. """ - self.name_to_id_index: Dict[str, str] = {} + self.name_to_id_index: dict[str, str] = {} def index(self, dictionary: EntityLinkingDictionary, preprocessor: Optional[EntityPreprocessor] = None) -> None: def p(text: str) -> str: @@ -407,8 +408,8 @@ def p(text: str) -> str: for synonym in candidate.synonyms: self.name_to_id_index[p(synonym)] = candidate.concept_id - def search(self, entity_mentions: List[str], top_k: int) -> List[List[Tuple[str, float]]]: - results: List[List[Tuple[str, float]]] = [] + def search(self, entity_mentions: list[str], top_k: int) -> list[list[tuple[str, float]]]: + results: list[list[tuple[str, float]]] = [] for mention in entity_mentions: dict_entry = self.name_to_id_index.get(mention) if dict_entry is None: @@ -419,12 +420,12 @@ def search(self, entity_mentions: List[str], top_k: int) -> List[List[Tuple[str, return results @classmethod - def _from_state(cls, state_dict: Dict[str, Any]) -> "CandidateSearchIndex": + def _from_state(cls, state_dict: dict[str, Any]) -> "CandidateSearchIndex": index = cls() index.name_to_id_index = state_dict["name_to_id_index"] return index - def _get_state(self) -> Dict[str, Any]: + def _get_state(self) -> dict[str, Any]: return { **super()._get_state(), "name_to_id_index": self.name_to_id_index, @@ -436,7 +437,7 @@ class SemanticCandidateSearchIndex(CandidateSearchIndex): def __init__( self, - embeddings: Dict[str, DocumentEmbeddings], + embeddings: dict[str, DocumentEmbeddings], hybrid_search: bool, similarity_metric: SimilarityMetric = SimilarityMetric.INNER_PRODUCT, sparse_weight: float = DEFAULT_SPARSE_WEIGHT, @@ -460,8 +461,8 @@ def __init__( self.show_progress = show_progress self.batch_size = batch_size - self.ids: List[str] = [] - self._precomputed_embeddings: Dict[str, np.ndarray] = {"sparse": np.array([]), "dense": np.array([])} + self.ids: list[str] = [] + self._precomputed_embeddings: dict[str, np.ndarray] = {"sparse": np.array([]), "dense": np.array([])} @classmethod def bi_encoder( @@ -479,7 +480,7 @@ def bi_encoder( if model_name_or_path in PRETRAINED_MODELS: similarity_metric = PRETRAINED_MODEL_TO_SIMILARITY_METRIC[model_name_or_path] - embeddings: Dict[str, DocumentEmbeddings] = {"dense": TransformerDocumentEmbeddings(model_name_or_path)} + embeddings: dict[str, DocumentEmbeddings] = {"dense": TransformerDocumentEmbeddings(model_name_or_path)} if hybrid_search: if dictionary is None: @@ -515,7 +516,7 @@ def index(self, dictionary: EntityLinkingDictionary, preprocessor: Optional[Enti def p(text: str) -> str: return preprocessor.process_entity_name(text) if preprocessor is not None else text - texts: List[str] = [] + texts: list[str] = [] self.ids = [] for candidate in dictionary.candidates: texts.append(p(candidate.concept_name)) @@ -564,8 +565,8 @@ def p(text: str) -> str: sent.clear_embeddings() self._precomputed_embeddings["sparse"] = np.stack(sparse_embs, axis=0) - def embed(self, entity_mentions: List[str]) -> Dict[str, np.ndarray]: - query_embeddings: Dict[str, List] = {"dense": []} + def embed(self, entity_mentions: list[str]) -> dict[str, np.ndarray]: + query_embeddings: dict[str, list[np.ndarray]] = {"dense": []} inputs = [Sentence(name) for name in entity_mentions] @@ -600,7 +601,7 @@ def embed(self, entity_mentions: List[str]) -> Dict[str, np.ndarray]: return {k: np.stack(v, axis=0) for k, v in query_embeddings.items()} - def search(self, entity_mentions: List[str], top_k: int) -> List[List[Tuple[str, float]]]: + def search(self, entity_mentions: list[str], top_k: int) -> list[list[tuple[str, float]]]: """Returns the top-k entity / concept identifiers for each entity mention. Args: @@ -634,10 +635,10 @@ def search(self, entity_mentions: List[str], top_k: int) -> List[List[Tuple[str, return results @classmethod - def _from_state(cls, state_dict: Dict[str, Any]) -> "SemanticCandidateSearchIndex": + def _from_state(cls, state_dict: dict[str, Any]) -> "SemanticCandidateSearchIndex": index = cls( embeddings=cast( - Dict[str, DocumentEmbeddings], {k: load_embeddings(emb) for k, emb in state_dict["embeddings"].items()} + dict[str, DocumentEmbeddings], {k: load_embeddings(emb) for k, emb in state_dict["embeddings"].items()} ), similarity_metric=SimilarityMetric(state_dict["similarity_metric"]), sparse_weight=state_dict["sparse_weight"], @@ -649,7 +650,7 @@ def _from_state(cls, state_dict: Dict[str, Any]) -> "SemanticCandidateSearchInde index._precomputed_embeddings = state_dict["precomputed_embeddings"] return index - def _get_state(self) -> Dict[str, Any]: + def _get_state(self) -> dict[str, Any]: return { **super()._get_state(), "embeddings": {k: emb.save_embeddings() for k, emb in self.embeddings.items()}, @@ -670,7 +671,7 @@ def __init__( self, candidate_generator: CandidateSearchIndex, preprocessor: EntityPreprocessor, - entity_label_types: Union[str, Sequence[str], Dict[str, Set[str]]], + entity_label_types: Union[str, Sequence[str], dict[str, set[str]]], label_type: str, dictionary: EntityLinkingDictionary, batch_size: int = 1024, @@ -698,8 +699,8 @@ def __init__( super().__init__() def get_entity_label_types( - self, entity_label_types: Union[str, Sequence[str], Dict[str, Set[str]]] - ) -> Dict[str, Set[str]]: + self, entity_label_types: Union[str, Sequence[str], dict[str, set[str]]] + ) -> dict[str, set[str]]: """Find out what NER labels to extract from sentence. Args: @@ -709,9 +710,9 @@ def get_entity_label_types( To use all labels from 'ner', pass 'ner' """ if isinstance(entity_label_types, str): - entity_label_types = cast(Dict[str, Set[str]], {entity_label_types: {}}) + entity_label_types = cast(dict[str, set[str]], {entity_label_types: {}}) elif isinstance(entity_label_types, Sequence): - entity_label_types = cast(Dict[str, Set[str]], {label: {} for label in entity_label_types}) + entity_label_types = cast(dict[str, set[str]], {label: {} for label in entity_label_types}) entity_label_types = { label: {normalize_entity_type(e) for e in entity_types} @@ -728,9 +729,9 @@ def label_type(self): def dictionary(self) -> EntityLinkingDictionary: return self._dictionary - def extract_entities_mentions(self, sentence: Sentence, entity_label_types: Dict[str, Set[str]]) -> List[Label]: + def extract_entities_mentions(self, sentence: Sentence, entity_label_types: dict[str, set[str]]) -> list[Label]: """Extract tagged mentions from sentences.""" - entities_mentions: List[Label] = [] + entities_mentions: list[Label] = [] # NOTE: This is a hacky workaround for the fact that # the `label_type`s in `Classifier.load('hunflair)` are @@ -762,10 +763,10 @@ def extract_entities_mentions(self, sentence: Sentence, entity_label_types: Dict def predict( self, - sentences: Union[List[Sentence], Sentence], + sentences: Union[list[Sentence], Sentence], top_k: int = 1, pred_label_type: Optional[str] = None, - entity_label_types: Optional[Union[str, Sequence[str], Dict[str, Set[str]]]] = None, + entity_label_types: Optional[Union[str, Sequence[str], dict[str, set[str]]]] = None, batch_size: Optional[int] = None, ) -> None: """Predicts the best matching top-k entity / concept identifiers of all named entities annotated with tag input_entity_annotation_layer. @@ -859,7 +860,7 @@ def _fetch_model(model_name: str) -> str: return hf_download(model_name) @classmethod - def _init_model_with_state_dict(cls, state: Dict[str, Any], **kwargs) -> "EntityMentionLinker": + def _init_model_with_state_dict(cls, state: dict[str, Any], **kwargs) -> "EntityMentionLinker": candidate_generator = CandidateSearchIndex._from_state(state["candidate_search_index"]) preprocessor = EntityPreprocessor._from_state(state["entity_preprocessor"]) entity_label_types = state["entity_label_types"] @@ -961,7 +962,7 @@ def __get_model_path_and_entity_type( model_name_or_path: str, entity_type: Optional[str] = None, hybrid_search: bool = False, - ) -> Tuple[str, str]: + ) -> tuple[str, str]: """Try to figure out what model the user wants.""" if model_name_or_path not in MODELS and model_name_or_path not in ENTITY_TYPES: raise ValueError( @@ -1039,24 +1040,24 @@ def __get_dictionary_path( return dictionary_name_or_path - def forward_loss(self, data_points: List[DT]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, data_points: list[DT]) -> tuple[torch.Tensor, int]: raise NotImplementedError("The EntityLinker cannot be trained") @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "EntityMentionLinker": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "EntityMentionLinker": from typing import cast return cast("EntityMentionLinker", super().load(model_path=model_path)) def evaluate( self, - data_points: Union[List[Sentence], Dataset], + data_points: Union[list[Sentence], Dataset], gold_label_type: str, out_path: Optional[Union[str, Path]] = None, embedding_storage_mode: str = "none", mini_batch_size: int = 32, - main_evaluation_metric: Tuple[str, str] = ("accuracy", "f1-score"), - exclude_labels: Optional[List[str]] = None, + main_evaluation_metric: tuple[str, str] = ("accuracy", "f1-score"), + exclude_labels: Optional[list[str]] = None, gold_label_dictionary: Optional[Dictionary] = None, return_loss: bool = True, k: int = 1, diff --git a/flair/models/language_model.py b/flair/models/language_model.py index ed417f2434..d85db2fb93 100644 --- a/flair/models/language_model.py +++ b/flair/models/language_model.py @@ -1,6 +1,6 @@ import math from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import Optional, Union import torch from torch import logsumexp, nn @@ -111,7 +111,7 @@ def init_hidden(self, bsz): def get_representation( self, - strings: List[str], + strings: list[str], start_marker: str, end_marker: str, chars_per_chunk: int = 512, @@ -119,7 +119,7 @@ def get_representation( len_longest_str: int = len(max(strings, key=len)) # pad strings with whitespaces to longest sentence - padded_strings: List[str] = [] + padded_strings: list[str] = [] for string in strings: if not self.is_forward_lm: @@ -141,11 +141,11 @@ def get_representation( padding_char_index = self.dictionary.get_idx_for_item(" ") - batches: List[torch.Tensor] = [] + batches: list[torch.Tensor] = [] # push each chunk through the RNN language model for chunk in chunks: len_longest_chunk: int = len(max(chunk, key=len)) - sequences_as_char_indices: List[List[int]] = [] + sequences_as_char_indices: list[list[int]] = [] for string in chunk: char_indices = self.dictionary.get_idx_for_items(list(string)) char_indices += [padding_char_index] * (len_longest_chunk - len(string)) @@ -176,7 +176,7 @@ def get_output(self, text: str): def repackage_hidden(self, h): """Wraps hidden states in new Variables, to detach them from their history.""" - if type(h) == torch.Tensor: + if isinstance(h, torch.Tensor): return h.clone().detach() else: return tuple(self.repackage_hidden(v) for v in h) @@ -296,7 +296,7 @@ def generate_text( number_of_characters: int = 1000, temperature: float = 1.0, break_on_suffix=None, - ) -> Tuple[str, float]: + ) -> tuple[str, float]: if prefix == "": prefix = "\n" diff --git a/flair/models/lemmatizer_model.py b/flair/models/lemmatizer_model.py index 6f0854d4b5..55fa34698c 100644 --- a/flair/models/lemmatizer_model.py +++ b/flair/models/lemmatizer_model.py @@ -1,6 +1,6 @@ import logging from math import inf -from typing import List, Optional, Tuple, Union +from typing import Optional, Union import torch from torch import nn @@ -159,7 +159,7 @@ def label_type(self): def words_to_char_indices( self, - tokens: List[str], + tokens: list[str], end_symbol=True, start_symbol=False, padding_in_front=False, @@ -202,7 +202,7 @@ def words_to_char_indices( return tensor - def forward_pass(self, sentences: Union[List[Sentence], Sentence]): + def forward_pass(self, sentences: Union[list[Sentence], Sentence]): if isinstance(sentences, Sentence): sentences = [sentences] @@ -247,7 +247,7 @@ def decode(self, decoder_input_indices, initial_hidden_states, all_encoder_outpu output_vectors = self.character_decoder(output) return output_vectors, hidden - def _prepare_tensors(self, sentences: List[Sentence]) -> Tuple[Optional[torch.Tensor], ...]: + def _prepare_tensors(self, sentences: list[Sentence]) -> tuple[Optional[torch.Tensor], ...]: # get all tokens tokens = [token for sentence in sentences for token in sentence] @@ -290,7 +290,7 @@ def forward( encoder_input_indices: Optional[torch.Tensor], lengths: Optional[torch.Tensor], token_embedding_hidden: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: # variable to store initial hidden states for decoder initial_hidden_for_decoder = [] @@ -340,7 +340,7 @@ def forward( return initial_hidden, all_encoder_outputs - def encode(self, sentences: List[Sentence]): + def encode(self, sentences: list[Sentence]): tensors = self._prepare_tensors(sentences) return self.forward(*tensors) @@ -396,14 +396,14 @@ def _calculate_loss(self, scores, labels): return self.loss(scores_in_correct_format, target), len(labels) - def forward_loss(self, sentences: Union[List[Sentence], Sentence]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, sentences: Union[list[Sentence], Sentence]) -> tuple[torch.Tensor, int]: scores, labels = self.forward_pass(sentences) return self._calculate_loss(scores, labels) def predict( self, - sentences: Union[List[Sentence], Sentence], + sentences: Union[list[Sentence], Sentence], mini_batch_size: int = 16, return_probabilities_for_all_classes: bool = False, verbose: bool = False, @@ -474,7 +474,7 @@ def predict( # option 1: greedy decoding if self.beam_size == 1: # predictions - predicted: List[List[Union[int, float]]] = [[] for _ in range(number_tokens)] + predicted: list[list[Union[int, float]]] = [[] for _ in range(number_tokens)] for _decode_step in range(max_length): # decode next character @@ -525,7 +525,7 @@ def predict( # keep track of how many hypothesis were completed for each token n_completed = [0 for _ in range(number_tokens)] # cpu - final_candidates: List[List[Tuple[torch.Tensor, float]]] = [[] for _ in range(number_tokens)] # cpu + final_candidates: list[list[tuple[torch.Tensor, float]]] = [[] for _ in range(number_tokens)] # cpu # if all_encoder_outputs returned, expand them to beam size (otherwise keep this as None) batched_encoding_output = ( @@ -552,24 +552,24 @@ def predict( # check if an end symbol has been predicted and, in that case, set hypothesis aside end_symbols = (index_candidates == self.end_index).nonzero(as_tuple=False) - for tuple in end_symbols: + for row in end_symbols: # if the sequence is already ended, do not record as candidate - if sequences[tuple[0], -1].item() == self.end_index: + if sequences[row[0], -1].item() == self.end_index: continue # index of token in in list tokens_in_batch - token_number = torch.div(tuple[0], self.beam_size, rounding_mode="trunc") + token_number = torch.div(row[0], self.beam_size, rounding_mode="trunc") # print(token_number) - seq = sequences[tuple[0], :] # hypothesis sequence + seq = sequences[row[0], :] # hypothesis sequence # hypothesis score - score = (scores[tuple[0]] + log_probabilities[tuple[0], tuple[1]]) / (len(seq) + 1) + score = (scores[row[0]] + log_probabilities[row[0], row[1]]) / (len(seq) + 1) final_candidates[token_number].append((seq, score.item())) # TODO: remove token if number of completed hypothesis exceeds given value n_completed[token_number] += 1 # set score of corresponding entry to -inf so it will not be expanded - log_probabilities[tuple[0], tuple[1]] = -inf + log_probabilities[row[0], row[1]] = -inf # get leading_indices for next expansion # find highest scoring hypothesis among beam_size*beam_size possible ones for each token @@ -594,8 +594,8 @@ def predict( # a list of length beam_size*batch_size # where the first three inidices belong to the first token, the next three to the second token, # and so on - beam_numbers: List[int] = [] - seq_numbers: List[int] = [] + beam_numbers: list[int] = [] + seq_numbers: list[int] = [] for i, row in enumerate(indices_per_token): beam_numbers.extend(i * self.beam_size + index.item() // self.beam_size for index in row) diff --git a/flair/models/multitask_model.py b/flair/models/multitask_model.py index 733751eff7..414eb46197 100644 --- a/flair/models/multitask_model.py +++ b/flair/models/multitask_model.py @@ -2,7 +2,7 @@ import random import typing from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import torch @@ -27,9 +27,9 @@ class MultitaskModel(flair.nn.Classifier): def __init__( self, - models: List[flair.nn.Classifier], - task_ids: Optional[List[str]] = None, - loss_factors: Optional[List[float]] = None, + models: list[flair.nn.Classifier], + task_ids: Optional[list[str]] = None, + loss_factors: Optional[list[float]] = None, use_all_tasks: bool = False, ) -> None: """Instantiates the MultiTaskModel. @@ -42,10 +42,10 @@ def __init__( """ super().__init__() - task_ids_internal: List[str] = task_ids if task_ids else [f"Task_{i}" for i in range(len(models))] + task_ids_internal: list[str] = task_ids if task_ids else [f"Task_{i}" for i in range(len(models))] - self.tasks: Dict[str, flair.nn.Classifier] = {} - self.loss_factors: Dict[str, float] = {} + self.tasks: dict[str, flair.nn.Classifier] = {} + self.loss_factors: dict[str, float] = {} self.use_all_tasks = use_all_tasks if not loss_factors: @@ -63,10 +63,10 @@ def __init__( def forward(self, *args) -> torch.Tensor: raise NotImplementedError("`forward` is not used for multitask learning") - def _prepare_tensors(self, data_points: List[DT]) -> Tuple[torch.Tensor, ...]: + def _prepare_tensors(self, data_points: list[DT]) -> tuple[torch.Tensor, ...]: raise NotImplementedError("`_prepare_tensors` is not used for multitask learning") - def forward_loss(self, sentences: Union[List[Sentence], Sentence]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, sentences: Union[list[Sentence], Sentence]) -> tuple[torch.Tensor, int]: """Calls the respective forward loss of each model and sums them weighted by their loss factors. Args: @@ -92,7 +92,9 @@ def predict( task.predict(sentences, **predictargs) @staticmethod - def split_batch_to_task_ids(sentences: Union[List[Sentence], Sentence], all_tasks: bool = False) -> Dict: + def split_batch_to_task_ids( + sentences: Union[list[Sentence], Sentence], all_tasks: bool = False + ) -> dict[str, list[int]]: """Splits a batch of sentences to its respective model. If single sentence is assigned to several tasks (i.e. same corpus but different tasks), then the model @@ -104,7 +106,7 @@ def split_batch_to_task_ids(sentences: Union[List[Sentence], Sentence], all_task Returns: Key-value pairs as (task_id, list of sentences ids in batch) """ - batch_to_task_mapping: Dict[str, List[int]] = {} + batch_to_task_mapping: dict[str, list[int]] = {} for sentence_id, sentence in enumerate(sentences): if all_tasks: multitask_ids = sentence.get_labels("multitask_id") @@ -122,7 +124,7 @@ def evaluate( # type: ignore[override] data_points, gold_label_type: str, out_path: Optional[Union[str, Path]] = None, - main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"), + main_evaluation_metric: tuple[str, str] = ("micro avg", "f1-score"), evaluate_all: bool = True, **evalargs, ) -> Result: @@ -161,7 +163,7 @@ def evaluate( # type: ignore[override] loss = torch.tensor(0.0, device=flair.device) main_score = 0.0 all_detailed_results = "" - all_classification_report: Dict[str, Dict[str, Any]] = {} + all_classification_report: dict[str, dict[str, Any]] = {} for task_id, split in batch_split.items(): result = self.tasks[task_id].evaluate( @@ -203,7 +205,7 @@ def evaluate( # type: ignore[override] def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: for model in self.tasks.values(): yield from model.get_used_tokens(corpus, context_length, respect_document_boundaries) @@ -272,7 +274,7 @@ def _fetch_model(model_name) -> str: return model_name @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "MultitaskModel": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "MultitaskModel": from typing import cast return cast("MultitaskModel", super().load(model_path=model_path)) diff --git a/flair/models/pairwise_classification_model.py b/flair/models/pairwise_classification_model.py index 262fd08cb5..6308573edc 100644 --- a/flair/models/pairwise_classification_model.py +++ b/flair/models/pairwise_classification_model.py @@ -1,5 +1,4 @@ import typing -from typing import List import torch @@ -69,7 +68,7 @@ def __init__( def label_type(self): return self._label_type - def _get_data_points_from_sentence(self, sentence: TextPair) -> List[TextPair]: + def _get_data_points_from_sentence(self, sentence: TextPair) -> list[TextPair]: return [sentence] def _get_embedding_for_data_point(self, prediction_data_point: TextPair) -> torch.Tensor: @@ -119,7 +118,7 @@ def _init_model_with_state_dict(cls, state, **kwargs): def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: for sentence_pair in _iter_dataset(corpus.get_all_sentences()): yield [t.text for t in sentence_pair.first] yield [t.text for t in sentence_pair.first.left_context(context_length, respect_document_boundaries)] diff --git a/flair/models/pairwise_regression_model.py b/flair/models/pairwise_regression_model.py index c3f34e0f69..9a1c2704be 100644 --- a/flair/models/pairwise_regression_model.py +++ b/flair/models/pairwise_regression_model.py @@ -1,5 +1,6 @@ +from collections.abc import Iterable from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Optional, Union import torch from torch import nn @@ -90,7 +91,7 @@ def label_type(self): def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> Iterable[List[str]]: + ) -> Iterable[list[str]]: for sentence_pair in _iter_dataset(corpus.get_all_sentences()): yield [t.text for t in sentence_pair.first] yield [t.text for t in sentence_pair.first.left_context(context_length, respect_document_boundaries)] @@ -99,14 +100,14 @@ def get_used_tokens( yield [t.text for t in sentence_pair.second.left_context(context_length, respect_document_boundaries)] yield [t.text for t in sentence_pair.second.right_context(context_length, respect_document_boundaries)] - def forward_loss(self, pairs: List[TextPair]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, pairs: list[TextPair]) -> tuple[torch.Tensor, int]: loss, num = self._forward_loss_and_scores(pairs=pairs, return_num=True, return_scores=False) assert isinstance(loss, torch.Tensor) assert isinstance(num, int) return loss, num - def _forward_loss_and_scores(self, pairs: List[TextPair], return_num=True, return_scores=True) -> Tuple: + def _forward_loss_and_scores(self, pairs: list[TextPair], return_num=True, return_scores=True) -> tuple: # make a forward pass to produce embedded data points and labels pairs = [pair for pair in pairs if self._filter_data_point(pair)] @@ -128,7 +129,7 @@ def _forward_loss_and_scores(self, pairs: List[TextPair], return_num=True, retur # calculate the loss loss, num = self._calculate_loss(scores, target_tensor) - return_value: Tuple[Any, ...] = (loss,) + return_value: tuple[Any, ...] = (loss,) if return_num: return_value += (num,) @@ -138,10 +139,10 @@ def _forward_loss_and_scores(self, pairs: List[TextPair], return_num=True, retur return return_value - def _calculate_loss(self, scores: torch.Tensor, target_tensor: torch.Tensor) -> Tuple[torch.Tensor, int]: + def _calculate_loss(self, scores: torch.Tensor, target_tensor: torch.Tensor) -> tuple[torch.Tensor, int]: return self.loss_function(scores, target_tensor), target_tensor.size(0) - def _prepare_target_tensor(self, pairs: List[TextPair]): + def _prepare_target_tensor(self, pairs: list[TextPair]): target_values = [ torch.tensor([float(label.value) for label in pair.get_labels(self.label_name)], dtype=torch.float) for pair in pairs @@ -152,7 +153,7 @@ def _prepare_target_tensor(self, pairs: List[TextPair]): def _filter_data_point(self, pair: TextPair) -> bool: return len(pair) > 0 - def _encode_data_points(self, data_points: List[TextPair]) -> torch.Tensor: + def _encode_data_points(self, data_points: list[TextPair]) -> torch.Tensor: # get a tensor of data points data_point_tensor = torch.stack([self._get_embedding_for_data_point(data_point) for data_point in data_points]) @@ -203,7 +204,7 @@ def _get_state_dict(self): return model_state @classmethod - def _init_model_with_state_dict(cls, state: Dict[str, Any], **kwargs): + def _init_model_with_state_dict(cls, state: dict[str, Any], **kwargs): """Initializes a TextPairRegressor model from a state dictionary (exported by _get_state_dict). Requires keys 'state_dict', 'document_embeddings', and 'label_type' in the state dictionary. @@ -227,12 +228,12 @@ def _init_model_with_state_dict(cls, state: Dict[str, Any], **kwargs): def predict( self, - pairs: Union[TextPair, List[TextPair]], + pairs: Union[TextPair, list[TextPair]], mini_batch_size: int = 32, verbose: bool = False, label_name: Optional[str] = None, embedding_storage_mode="none", - ) -> List[TextPair]: + ) -> list[TextPair]: if label_name is None: label_name = self.label_name if self.label_name is not None else "label" @@ -278,13 +279,13 @@ def predict( def evaluate( self, - data_points: Union[List[TextPair], Dataset], + data_points: Union[list[TextPair], Dataset], gold_label_type: str, out_path: Union[str, Path, None] = None, embedding_storage_mode: EmbeddingStorageMode = "none", mini_batch_size: int = 32, - main_evaluation_metric: Tuple[str, str] = ("correlation", "pearson"), - exclude_labels: Optional[List[str]] = None, + main_evaluation_metric: tuple[str, str] = ("correlation", "pearson"), + exclude_labels: Optional[list[str]] = None, gold_label_dictionary: Optional[Dictionary] = None, return_loss: bool = True, **kwargs, diff --git a/flair/models/prefixed_tagger.py b/flair/models/prefixed_tagger.py index 05d8fa8c34..b001653bdc 100644 --- a/flair/models/prefixed_tagger.py +++ b/flair/models/prefixed_tagger.py @@ -1,7 +1,7 @@ import logging from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Optional, Union, cast import torch from torch.utils.data import Dataset @@ -26,7 +26,7 @@ class SentenceAugmentationStrategy(ABC): @abstractmethod def augment_sentence( - self, sentence: Sentence, annotation_layers: Optional[Union[str, List[str]]] = None + self, sentence: Sentence, annotation_layers: Optional[Union[str, list[str]]] = None ) -> PrefixedSentence: """Augments the given sentence text with additional instructions for working / predicting the task on the given annotations. @@ -64,7 +64,7 @@ def _init_strategy_with_state_dict(cls, state, **kwargs): """Initializes the strategy from the given state.""" def augment_dataset( - self, dataset: Dataset[Sentence], annotation_layers: Optional[Union[str, List[str]]] = None + self, dataset: Dataset[Sentence], annotation_layers: Optional[Union[str, list[str]]] = None ) -> FlairDatapointDataset[PrefixedSentence]: """Transforms a dataset into a dataset containing augmented sentences specific to the `PrefixedSequenceTagger`. @@ -78,14 +78,14 @@ def augment_dataset( Returns: A dataset of augmented sentences specific to the `PrefixedSequenceTagger` """ data_loader: DataLoader = DataLoader(dataset, batch_size=1) - original_sentences: List[Sentence] = [batch[0] for batch in iter(data_loader)] + original_sentences: list[Sentence] = [batch[0] for batch in iter(data_loader)] augmented_sentences = [self.augment_sentence(sentence, annotation_layers) for sentence in original_sentences] return FlairDatapointDataset(augmented_sentences) def augment_corpus( - self, corpus: Corpus[Sentence], annotation_layers: Optional[Union[str, List[str]]] = None + self, corpus: Corpus[Sentence], annotation_layers: Optional[Union[str, list[str]]] = None ) -> Corpus[PrefixedSentence]: """Transforms a corpus into a corpus containing augmented sentences specific to the `PrefixedSequenceTagger`. @@ -120,7 +120,7 @@ class EntityTypeTaskPromptAugmentationStrategy(SentenceAugmentationStrategy): "[Tag gene and disease] Mutations in the TP53 tumour suppressor gene are found in ~50% of human cancers" """ - def __init__(self, entity_types: List[str]): + def __init__(self, entity_types: list[str]): if len(entity_types) <= 0: raise AssertionError @@ -128,7 +128,7 @@ def __init__(self, entity_types: List[str]): self.task_prompt = self._build_tag_prompt_prefix(entity_types) def augment_sentence( - self, sentence: Sentence, annotation_layers: Optional[Union[str, List[str]]] = None + self, sentence: Sentence, annotation_layers: Optional[Union[str, list[str]]] = None ) -> PrefixedSentence: # Prepend the task description prompt to the sentence text augmented_sentence = PrefixedSentence( @@ -182,7 +182,7 @@ def apply_predictions( ] orig_span.add_label(target_annotation_layer, label.value, label.score) - def _build_tag_prompt_prefix(self, entity_types: List[str]) -> List[str]: + def _build_tag_prompt_prefix(self, entity_types: list[str]) -> list[str]: if len(self.entity_types) == 1: prompt = f"[ Tag {entity_types[0]} ]" else: @@ -219,29 +219,29 @@ def _init_model_with_state_dict(cls, state, **kwargs): return super()._init_model_with_state_dict(state, augmentation_strategy=strategy, **kwargs) @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "PrefixedSequenceTagger": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "PrefixedSequenceTagger": from typing import cast return cast("PrefixedSequenceTagger", super().load(model_path=model_path)) - def forward_loss(self, sentences: Union[List[Sentence], List[PrefixedSentence]]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, sentences: Union[list[Sentence], list[PrefixedSentence]]) -> tuple[torch.Tensor, int]: # If all sentences are not augmented -> augment them if all(isinstance(sentence, Sentence) for sentence in sentences): # mypy does not infer the type of "sentences" restricted by the if statement - sentences = cast(List[Sentence], sentences) + sentences = cast(list[Sentence], sentences) sentences = self.augment_sentences(sentences=sentences, annotation_layers=self.tag_type) elif not all(isinstance(sentence, PrefixedSentence) for sentence in sentences): raise ValueError("All passed sentences must be either uniformly augmented or not.") # mypy does not infer the type of "sentences" restricted by code above - sentences = cast(List[Sentence], sentences) + sentences = cast(list[Sentence], sentences) return super().forward_loss(sentences) def predict( self, - sentences: Union[List[Sentence], Sentence, List[PrefixedSentence], PrefixedSentence], + sentences: Union[list[Sentence], Sentence, list[PrefixedSentence], PrefixedSentence], mini_batch_size: int = 32, return_probabilities_for_all_classes: bool = False, verbose: bool = False, @@ -260,7 +260,7 @@ def predict( # If all sentences are already augmented (i.e. compatible with this class), just forward the sentences if all(isinstance(sentence, PrefixedSentence) for sentence in sentences): # mypy does not infer the type of "sentences" restricted by the if statement - sentences = cast(List[Sentence], sentences) + sentences = cast(list[Sentence], sentences) return super().predict( sentences, @@ -280,12 +280,12 @@ def predict( for sentence in sentences: sentence.remove_labels(prediction_label_type) - sentences = cast(List[Sentence], sentences) + sentences = cast(list[Sentence], sentences) # Augment sentences - copy all annotation of the given tag type augmented_sentences = self.augment_sentences(sentences, self.tag_type) - mypy_safe_augmented_sentences = cast(List[Sentence], augmented_sentences) + mypy_safe_augmented_sentences = cast(list[Sentence], augmented_sentences) # Predict on augmented sentence and store it in an internal annotation layer / label loss_and_count = super().predict( @@ -312,8 +312,8 @@ def predict( return loss_and_count def augment_sentences( - self, sentences: Union[Sentence, List[Sentence]], annotation_layers: Optional[Union[str, List[str]]] = None - ) -> List[PrefixedSentence]: + self, sentences: Union[Sentence, list[Sentence]], annotation_layers: Optional[Union[str, list[str]]] = None + ) -> list[PrefixedSentence]: if not isinstance(sentences, list) and not isinstance(sentences, flair.data.Dataset): sentences = [sentences] diff --git a/flair/models/regexp_tagger.py b/flair/models/regexp_tagger.py index 35c244d960..e41981c899 100644 --- a/flair/models/regexp_tagger.py +++ b/flair/models/regexp_tagger.py @@ -1,7 +1,7 @@ import re import typing from dataclasses import dataclass, field -from typing import Dict, List, Tuple, Union +from typing import Union from flair.data import Sentence, Span, Token @@ -15,8 +15,8 @@ class TokenCollection: """ sentence: Sentence - __tokens_start_pos: List[int] = field(init=False, default_factory=list) - __tokens_end_pos: List[int] = field(init=False, default_factory=list) + __tokens_start_pos: list[int] = field(init=False, default_factory=list) + __tokens_end_pos: list[int] = field(init=False, default_factory=list) def __post_init__(self): for token in self.tokens: @@ -24,10 +24,10 @@ def __post_init__(self): self.__tokens_end_pos.append(token.end_position) @property - def tokens(self) -> List[Token]: + def tokens(self) -> list[Token]: return list(self.sentence) - def get_token_span(self, span: Tuple[int, int]) -> Span: + def get_token_span(self, span: tuple[int, int]) -> Span: """Find a span by the token character positions. Given an interval specified with start and end pos as tuple, this function returns a Span object @@ -45,7 +45,7 @@ def get_token_span(self, span: Tuple[int, int]) -> Span: class RegexpTagger: - def __init__(self, mapping: Union[List[Tuple[str, str]], Tuple[str, str]]) -> None: + def __init__(self, mapping: Union[list[tuple[str, str]], tuple[str, str]]) -> None: r"""This tagger is capable of tagging sentence objects with given regexp -> label mappings. I.e: The tuple (r'(["\'])(?:(?=(\\?))\2.)*?\1', 'QUOTE') maps every match of the regexp to @@ -58,14 +58,14 @@ def __init__(self, mapping: Union[List[Tuple[str, str]], Tuple[str, str]]) -> No Args: mapping: A list of tuples or a single tuple representing a mapping as regexp -> label """ - self._regexp_mapping: Dict[str, typing.Pattern] = {} + self._regexp_mapping: dict[str, typing.Pattern] = {} self.register_labels(mapping=mapping) @property def registered_labels(self): return self._regexp_mapping - def register_labels(self, mapping: Union[List[Tuple[str, str]], Tuple[str, str]]): + def register_labels(self, mapping: Union[list[tuple[str, str]], tuple[str, str]]): """Register a regexp -> label mapping. Args: @@ -81,7 +81,7 @@ def register_labels(self, mapping: Union[List[Tuple[str, str]], Tuple[str, str]] f"Couldn't compile regexp '{regexp}' for label '{label}'. Aborted with error: '{err.msg}'" ) - def remove_labels(self, labels: Union[List[str], str]): + def remove_labels(self, labels: Union[list[str], str]): """Remove a registered regexp -> label mapping given by label. Args: @@ -101,7 +101,7 @@ def _listify(element: object) -> list: else: return element - def predict(self, sentences: Union[List[Sentence], Sentence]) -> List[Sentence]: + def predict(self, sentences: Union[list[Sentence], Sentence]) -> list[Sentence]: """Predict the given sentences according to the registered mappings.""" if not isinstance(sentences, list): sentences = [sentences] @@ -122,7 +122,7 @@ def _label(self, sentence: Sentence): for label, pattern in self._regexp_mapping.items(): for match in pattern.finditer(sentence.to_original_text()): - span: Tuple[int, int] = match.span() + span: tuple[int, int] = match.span() try: token_span = collection.get_token_span(span) except ValueError: diff --git a/flair/models/relation_classifier_model.py b/flair/models/relation_classifier_model.py index 53ccabac36..9c6c69577f 100644 --- a/flair/models/relation_classifier_model.py +++ b/flair/models/relation_classifier_model.py @@ -2,17 +2,12 @@ import logging import typing from abc import ABC, abstractmethod +from collections.abc import Iterator, Sequence from pathlib import Path from typing import ( Any, - Dict, - Iterator, - List, NamedTuple, Optional, - Sequence, - Set, - Tuple, Union, cast, ) @@ -50,7 +45,7 @@ class EncodedSentence(Sentence): class EncodingStrategy(ABC): """The encoding of the head and tail entities in a sentence with a relation annotation.""" - special_tokens: Set[str] = set() + special_tokens: set[str] = set() def __init__(self, add_special_tokens: bool = False) -> None: self.add_special_tokens = add_special_tokens @@ -84,7 +79,7 @@ class EntityMask(EncodingStrategy): - "Larry Page and [TAIL] founded [HEAD]" -> Relation(head='Google', tail='Sergey Brin'). """ - special_tokens: Set[str] = {"[HEAD]", "[TAIL]"} + special_tokens: set[str] = {"[HEAD]", "[TAIL]"} def encode_head(self, head_span: Span, label: Label) -> str: return "[HEAD]" @@ -126,7 +121,7 @@ class EntityMarker(EncodingStrategy): -> Relation(head='Google', tail='Sergey Brin'). """ - special_tokens: Set[str] = {"[HEAD]", "[/HEAD]", "[TAIL]", "[/TAIL]"} + special_tokens: set[str] = {"[HEAD]", "[/HEAD]", "[TAIL]", "[/TAIL]"} def encode_head(self, head: Span, label: Label) -> str: space_tokenized_text: str = " ".join(token.text for token in head) @@ -254,8 +249,8 @@ def __init__( embeddings: DocumentEmbeddings, label_dictionary: Dictionary, label_type: str, - entity_label_types: Union[str, Sequence[str], Dict[str, Optional[Set[str]]]], - entity_pair_labels: Optional[Set[Tuple[str, str]]] = None, + entity_label_types: Union[str, Sequence[str], dict[str, Optional[set[str]]]], + entity_pair_labels: Optional[set[tuple[str, str]]] = None, entity_threshold: Optional[float] = None, cross_augmentation: bool = True, encoding_strategy: EncodingStrategy = TypedEntityMarker(), @@ -298,7 +293,7 @@ def __init__( ) if isinstance(entity_label_types, str): - self.entity_label_types: Dict[str, Optional[Set[str]]] = {entity_label_types: None} + self.entity_label_types: dict[str, Optional[set[str]]] = {entity_label_types: None} elif isinstance(entity_label_types, Sequence): self.entity_label_types = {entity_label_type: None for entity_label_type in entity_label_types} else: @@ -316,7 +311,7 @@ def __init__( and self.encoding_strategy.special_tokens and isinstance(self.embeddings, TransformerDocumentEmbeddings) ): - special_tokens: List[str] = list(self.encoding_strategy.special_tokens) + special_tokens: list[str] = list(self.encoding_strategy.special_tokens) tokenizer = self.embeddings.tokenizer tokenizer.add_special_tokens({"additional_special_tokens": special_tokens}) self.embeddings.model.resize_token_embeddings(len(tokenizer)) @@ -355,7 +350,7 @@ def _valid_entities(self, sentence: Sentence) -> Iterator[_Entity]: def _entity_pair_permutations( self, sentence: Sentence, - ) -> Iterator[Tuple[_Entity, _Entity, Optional[str]]]: + ) -> Iterator[tuple[_Entity, _Entity, Optional[str]]]: """Yields all valid entity pair permutations (relation candidates). If the passed sentence contains relation annotations, @@ -370,10 +365,10 @@ def _entity_pair_permutations( Yields: Tuples of (HEAD, TAIL, gold_label): The head and tail `_Entity`s` have span references to the passed sentence. """ - valid_entities: List[_Entity] = list(self._valid_entities(sentence)) + valid_entities: list[_Entity] = list(self._valid_entities(sentence)) # Use a dictionary to find gold relation annotations for a given entity pair - relation_to_gold_label: Dict[str, str] = { + relation_to_gold_label: dict[str, str] = { relation.unlabeled_identifier: relation.get_label(self.label_type, zero_tag_value=self.zero_tag_value).value for relation in sentence.get_relations(self.label_type) } @@ -420,13 +415,13 @@ def _encode_sentence( assert original_sentence is tail.span.sentence, "The head and tail need to come from the same sentence." # Pre-compute non-leading head and tail tokens for entity masking - non_leading_head_tokens: List[Token] = head.span.tokens[1:] - non_leading_tail_tokens: List[Token] = tail.span.tokens[1:] + non_leading_head_tokens: list[Token] = head.span.tokens[1:] + non_leading_tail_tokens: list[Token] = tail.span.tokens[1:] # We can not use the plaintext of the head/tail span in the sentence as the mask/marker # since there may be multiple occurrences of the same entity mentioned in the sentence. # Therefore, we use the span's position in the sentence. - encoded_sentence_tokens: List[str] = [] + encoded_sentence_tokens: list[str] = [] for token in original_sentence: if token is head.span[0]: encoded_sentence_tokens.append(self.encoding_strategy.encode_head(head.span, head.label)) @@ -456,7 +451,7 @@ def _encode_sentence( def _encode_sentence_for_inference( self, sentence: Sentence, - ) -> Iterator[Tuple[EncodedSentence, Relation]]: + ) -> Iterator[tuple[EncodedSentence, Relation]]: """Create Encoded Sentences and Relation pairs for Inference. Yields encoded sentences annotated with their gold relation and @@ -505,7 +500,7 @@ def _encode_sentence_for_training(self, sentence: Sentence) -> Iterator[EncodedS yield masked_sentence - def transform_sentence(self, sentences: Union[Sentence, List[Sentence]]) -> List[EncodedSentence]: + def transform_sentence(self, sentences: Union[Sentence, list[Sentence]]) -> list[EncodedSentence]: """Transforms sentences into encoded sentences specific to the `RelationClassifier`. For more information on the internal sentence transformation procedure, @@ -541,7 +536,7 @@ def transform_dataset(self, dataset: Dataset[Sentence]) -> FlairDatapointDataset Returns: A dataset of encoded sentences specific to the `RelationClassifier` """ data_loader: DataLoader = DataLoader(dataset, batch_size=1) - original_sentences: List[Sentence] = [batch[0] for batch in iter(data_loader)] + original_sentences: list[Sentence] = [batch[0] for batch in iter(data_loader)] return FlairDatapointDataset(self.transform_sentence(original_sentences)) def transform_corpus(self, corpus: Corpus[Sentence]) -> Corpus[EncodedSentence]: @@ -568,10 +563,10 @@ def transform_corpus(self, corpus: Corpus[Sentence]) -> Corpus[EncodedSentence]: ) def _get_embedding_for_data_point(self, prediction_data_point: EncodedSentence) -> torch.Tensor: - embedding_names: List[str] = self.embeddings.get_names() + embedding_names: list[str] = self.embeddings.get_names() return prediction_data_point.get_embedding(embedding_names) - def _get_data_points_from_sentence(self, sentence: EncodedSentence) -> List[EncodedSentence]: + def _get_data_points_from_sentence(self, sentence: EncodedSentence) -> list[EncodedSentence]: """Returns the encoded sentences to which labels are added. To encode sentences, use the `transform` function of the `RelationClassifier`. @@ -597,14 +592,14 @@ def _get_data_points_from_sentence(self, sentence: EncodedSentence) -> List[Enco def predict( self, - sentences: Union[List[Sentence], List[EncodedSentence], Sentence, EncodedSentence], + sentences: Union[list[Sentence], list[EncodedSentence], Sentence, EncodedSentence], mini_batch_size: int = 32, return_probabilities_for_all_classes: bool = False, verbose: bool = False, label_name: Optional[str] = None, return_loss: bool = False, embedding_storage_mode: EmbeddingStorageMode = "none", - ) -> Optional[Tuple[torch.Tensor, int]]: + ) -> Optional[tuple[torch.Tensor, int]]: """Predicts the class labels for the given sentence(s). Standard `Sentence` objects and `EncodedSentences` specific to the `RelationClassifier` are allowed as input. @@ -626,14 +621,14 @@ def predict( if not isinstance(sentences, list): sentences = [sentences] - loss: Optional[Tuple[torch.Tensor, int]] - encoded_sentences: List[EncodedSentence] + loss: Optional[tuple[torch.Tensor, int]] + encoded_sentences: list[EncodedSentence] if all(isinstance(sentence, EncodedSentence) for sentence in sentences): # Deal with the case where all sentences are encoded sentences # mypy does not infer the type of "sentences" restricted by the if statement - encoded_sentences = cast(List[EncodedSentence], sentences) + encoded_sentences = cast(list[EncodedSentence], sentences) loss = super().predict( encoded_sentences, mini_batch_size=mini_batch_size, @@ -646,8 +641,8 @@ def predict( elif all(not isinstance(sentence, EncodedSentence) for sentence in sentences): # Deal with the case where all sentences are standard (non-encoded) sentences - Sentence.set_context_for_sentences(cast(List[Sentence], sentences)) - sentences_with_relation_reference: List[Tuple[EncodedSentence, Relation]] = list( + Sentence.set_context_for_sentences(cast(list[Sentence], sentences)) + sentences_with_relation_reference: list[tuple[EncodedSentence, Relation]] = list( itertools.chain.from_iterable(self._encode_sentence_for_inference(sentence) for sentence in sentences) ) @@ -672,8 +667,8 @@ def predict( return loss if return_loss else None - def _get_state_dict(self) -> Dict[str, Any]: - model_state: Dict[str, Any] = { + def _get_state_dict(self) -> dict[str, Any]: + model_state: dict[str, Any] = { **super()._get_state_dict(), "embeddings": self.embeddings.save_embeddings(use_state_dict=False), "label_dictionary": self.label_dictionary, @@ -689,7 +684,7 @@ def _get_state_dict(self) -> Dict[str, Any]: return model_state @classmethod - def _init_model_with_state_dict(cls, state: Dict[str, Any], **kwargs): + def _init_model_with_state_dict(cls, state: dict[str, Any], **kwargs): return super()._init_model_with_state_dict( state, embeddings=state["embeddings"], @@ -719,7 +714,7 @@ def allow_unk_tag(self) -> bool: def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: yield from super().get_used_tokens(corpus, context_length, respect_document_boundaries) for sentence in _iter_dataset(corpus.get_all_sentences()): for span in sentence.get_spans(self.label_type): @@ -727,7 +722,7 @@ def get_used_tokens( yield self.encoding_strategy.encode_tail(span, span.get_label(self.label_type)).split(" ") @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "RelationClassifier": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "RelationClassifier": from typing import cast return cast("RelationClassifier", super().load(model_path=model_path)) diff --git a/flair/models/relation_extractor_model.py b/flair/models/relation_extractor_model.py index 795e8a517f..0c56abf5bd 100644 --- a/flair/models/relation_extractor_model.py +++ b/flair/models/relation_extractor_model.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Optional, Union import torch @@ -18,7 +18,7 @@ def __init__( embeddings: flair.embeddings.TokenEmbeddings, label_type: str, entity_label_type: str, - entity_pair_filters: Optional[List[Tuple[str, str]]] = None, + entity_pair_filters: Optional[list[tuple[str, str]]] = None, pooling_operation: str = "first_last", train_on_gold_pairs_only: bool = False, **classifierargs, @@ -56,13 +56,13 @@ def __init__( # whether to use gold entity pairs, and whether to filter entity pairs by type if entity_pair_filters is not None: - self.entity_pair_filters: Optional[Set[Tuple[str, str]]] = set(entity_pair_filters) + self.entity_pair_filters: Optional[set[tuple[str, str]]] = set(entity_pair_filters) else: self.entity_pair_filters = None self.to(flair.device) - def _get_data_points_from_sentence(self, sentence: Sentence) -> List[Relation]: + def _get_data_points_from_sentence(self, sentence: Sentence) -> list[Relation]: entity_pairs = [] entity_spans = sentence.get_spans(self.entity_label_type) @@ -172,7 +172,7 @@ def _fetch_model(model_name) -> str: return model_name @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "RelationExtractor": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "RelationExtractor": from typing import cast return cast("RelationExtractor", super().load(model_path=model_path)) diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index 16f20a0ddf..2c0fc00ebb 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -1,7 +1,7 @@ import logging import tempfile from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Optional, Union, cast import torch import torch.nn @@ -40,7 +40,7 @@ def __init__( word_dropout: float = 0.05, locked_dropout: float = 0.5, train_initial_hidden_state: bool = False, - loss_weights: Optional[Dict[str, float]] = None, + loss_weights: Optional[dict[str, float]] = None, init_from_state_dict: bool = False, allow_unk_predictions: bool = False, ) -> None: @@ -204,7 +204,7 @@ def __init__( def label_type(self): return self.tag_type - def _init_loss_weights(self, loss_weights: Dict[str, float]) -> torch.Tensor: + def _init_loss_weights(self, loss_weights: dict[str, float]) -> torch.Tensor: """Initializes the loss weights based on given dictionary. Args: @@ -267,7 +267,7 @@ def RNN( return RNN - def forward_loss(self, sentences: List[Sentence]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, sentences: list[Sentence]) -> tuple[torch.Tensor, int]: # if there are no sentences, there is no loss if len(sentences) == 0: return torch.tensor(0.0, dtype=torch.float, device=flair.device, requires_grad=True), 0 @@ -281,7 +281,7 @@ def forward_loss(self, sentences: List[Sentence]) -> Tuple[torch.Tensor, int]: # calculate loss given scores and labels return self._calculate_loss(scores, gold_labels) - def _prepare_tensors(self, data_points: Union[List[Sentence], Sentence]) -> Tuple[torch.Tensor, torch.LongTensor]: + def _prepare_tensors(self, data_points: Union[list[Sentence], Sentence]) -> tuple[torch.Tensor, torch.LongTensor]: sentences = [data_points] if not isinstance(data_points, list) else data_points self.embeddings.embed(sentences) @@ -331,15 +331,15 @@ def forward(self, sentence_tensor: torch.Tensor, lengths: torch.LongTensor): return scores - def _calculate_loss(self, scores: torch.Tensor, labels: torch.LongTensor) -> Tuple[torch.Tensor, int]: + def _calculate_loss(self, scores: torch.Tensor, labels: torch.LongTensor) -> tuple[torch.Tensor, int]: if labels.size(0) == 0: return torch.tensor(0.0, requires_grad=True, device=flair.device), 1 return self.loss_function(scores, labels), len(labels) - def _make_padded_tensor_for_batch(self, sentences: List[Sentence]) -> Tuple[torch.LongTensor, torch.Tensor]: + def _make_padded_tensor_for_batch(self, sentences: list[Sentence]) -> tuple[torch.LongTensor, torch.Tensor]: names = self.embeddings.get_names() - lengths: List[int] = [len(sentence.tokens) for sentence in sentences] + lengths: list[int] = [len(sentence.tokens) for sentence in sentences] longest_token_sequence_in_batch: int = max(lengths) pre_allocated_zero_tensor = torch.zeros( self.embeddings.embedding_length * longest_token_sequence_in_batch, @@ -382,7 +382,7 @@ def _get_scores_from_features(features: torch.Tensor, lengths: torch.Tensor): return scores - def _get_gold_labels(self, sentences: List[Sentence]) -> List[str]: + def _get_gold_labels(self, sentences: list[Sentence]) -> list[str]: """Extracts gold labels from each sentence. Args: @@ -419,7 +419,7 @@ def _get_gold_labels(self, sentences: List[Sentence]) -> List[str]: return labels - def _prepare_label_tensor(self, sentences: List[Sentence]): + def _prepare_label_tensor(self, sentences: list[Sentence]): gold_labels = self._get_gold_labels(sentences) labels = torch.tensor( [self.label_dictionary.get_idx_for_item(label) for label in gold_labels], @@ -430,7 +430,7 @@ def _prepare_label_tensor(self, sentences: List[Sentence]): def predict( self, - sentences: Union[List[Sentence], Sentence], + sentences: Union[list[Sentence], Sentence], mini_batch_size: int = 32, return_probabilities_for_all_classes: bool = False, verbose: bool = False, @@ -462,7 +462,7 @@ def predict( if not isinstance(sentences, list) and not isinstance(sentences, flair.data.Dataset): sentences = [sentences] - Sentence.set_context_for_sentences(cast(List[Sentence], sentences)) + Sentence.set_context_for_sentences(cast(list[Sentence], sentences)) # filter empty sentences sentences = [sentence for sentence in sentences if len(sentence) > 0] @@ -542,7 +542,7 @@ def predict( return overall_loss, label_count return None - def _standard_inference(self, features: torch.Tensor, batch: List[Sentence], probabilities_for_all_classes: bool): + def _standard_inference(self, features: torch.Tensor, batch: list[Sentence], probabilities_for_all_classes: bool): """Softmax over emission scores from forward propagation. Args: @@ -573,7 +573,7 @@ def _standard_inference(self, features: torch.Tensor, batch: List[Sentence], pro return predictions, all_tags - def _all_scores_for_token(self, sentences: List[Sentence], score_tensor: torch.Tensor, lengths: List[int]): + def _all_scores_for_token(self, sentences: list[Sentence], score_tensor: torch.Tensor, lengths: list[int]): """Returns all scores for each tag in tag dictionary.""" scores = score_tensor.numpy() tokens = [token for sentence in sentences for token in sentence] @@ -861,7 +861,7 @@ def push_to_hub( return repo_url @staticmethod - def _filter_empty_sentences(sentences: List[Sentence]) -> List[Sentence]: + def _filter_empty_sentences(sentences: list[Sentence]) -> list[Sentence]: filtered_sentences = [sentence for sentence in sentences if sentence.tokens] if len(sentences) != len(filtered_sentences): log.warning(f"Ignore {len(sentences) - len(filtered_sentences)} sentence(s) with no tokens.") @@ -919,7 +919,7 @@ def _print_predictions(self, batch, gold_label_type): return lines @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "SequenceTagger": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "SequenceTagger": from typing import cast return cast("SequenceTagger", super().load(model_path=model_path)) diff --git a/flair/models/sequence_tagger_utils/viterbi.py b/flair/models/sequence_tagger_utils/viterbi.py index ed84d1a6b7..5c87f49f0c 100644 --- a/flair/models/sequence_tagger_utils/viterbi.py +++ b/flair/models/sequence_tagger_utils/viterbi.py @@ -1,5 +1,3 @@ -from typing import Tuple - import numpy as np import torch import torch.nn @@ -7,7 +5,7 @@ from torch.nn.utils.rnn import pack_padded_sequence import flair -from flair.data import Dictionary, Label, List, Sentence +from flair.data import Dictionary, Label, Sentence START_TAG: str = "" STOP_TAG: str = "" @@ -141,8 +139,8 @@ def __init__(self, tag_dictionary: Dictionary) -> None: self.stop_tag = tag_dictionary.get_idx_for_item(STOP_TAG) def decode( - self, features_tuple: tuple, probabilities_for_all_classes: bool, sentences: List[Sentence] - ) -> Tuple[List, List]: + self, features_tuple: tuple, probabilities_for_all_classes: bool, sentences: list[Sentence] + ) -> tuple[list[list[tuple[str, float]]], list[list[list[Label]]]]: """Decoding function returning the most likely sequence of tags. Args: @@ -211,7 +209,7 @@ def decode( scores = softmax(scores_upto_t, dim=2) confidences = torch.max(scores, dim=2) - tags = [] + tags: list[list[tuple[str, float]]] = [] for tag_seq, tag_seq_conf, length_seq in zip(decoded, confidences.values, lengths): tags.append( [ @@ -230,7 +228,7 @@ def _all_scores_for_token( score_tensor: torch.Tensor, tag_sequences: torch.Tensor, lengths: torch.IntTensor, - sentences: List[Sentence], + sentences: list[Sentence], ): """Returns all scores for each tag in tag dictionary.""" scores = score_tensor.numpy() diff --git a/flair/models/tars_model.py b/flair/models/tars_model.py index a7a41bdb5d..4f5cb85731 100644 --- a/flair/models/tars_model.py +++ b/flair/models/tars_model.py @@ -3,7 +3,7 @@ from abc import ABC from collections import OrderedDict from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Optional, Union import numpy as np import torch @@ -30,14 +30,14 @@ class FewshotClassifier(flair.nn.Classifier[Sentence], ABC): def __init__(self) -> None: self._current_task = None - self._task_specific_attributes: Dict[str, Dict[str, Any]] = {} + self._task_specific_attributes: dict[str, dict[str, Any]] = {} self.label_nearest_map = None self.tars_model: flair.nn.Classifier[Sentence] self.separator: str super().__init__() - def forward_loss(self, data_points: Union[List[Sentence], Sentence]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, data_points: Union[list[Sentence], Sentence]) -> tuple[torch.Tensor, int]: if not isinstance(data_points, list): data_points = [data_points] @@ -54,7 +54,7 @@ def tars_embeddings(self): def _get_tars_formatted_sentence(self, label, sentence): raise NotImplementedError - def _get_tars_formatted_sentences(self, sentences: List[Sentence]): + def _get_tars_formatted_sentences(self, sentences: list[Sentence]): label_text_pairs = [] all_labels = [label.decode("utf-8") for label in self.get_current_label_dictionary().idx2item] for sentence in sentences: @@ -173,7 +173,7 @@ def is_current_task_multi_label(self): def add_and_switch_to_new_task( self, task_name: str, - label_dictionary: Union[List, Set, Dictionary, str], + label_dictionary: Union[list, set, Dictionary, str], label_type: str, multi_label: bool = True, force_switch: bool = False, @@ -219,7 +219,7 @@ def add_and_switch_to_new_task( self.switch_to_task(task_name) - def list_existing_tasks(self) -> Set[str]: + def list_existing_tasks(self) -> set[str]: """Lists existing tasks in the loaded TARS model on the console.""" return set(self._task_specific_attributes.keys()) @@ -246,7 +246,7 @@ def _drop_task(self, task_name): log.warning("No task exists with the name `%s`.", task_name) @staticmethod - def _filter_empty_sentences(sentences: List[Sentence]) -> List[Sentence]: + def _filter_empty_sentences(sentences: list[Sentence]) -> list[Sentence]: filtered_sentences = [sentence for sentence in sentences if sentence.tokens] if len(sentences) != len(filtered_sentences): log.warning(f"Ignore {len(sentences) - len(filtered_sentences)} sentence(s) with no tokens.") @@ -258,8 +258,8 @@ def label_type(self): def predict_zero_shot( self, - sentences: Union[List[Sentence], Sentence], - candidate_label_set: Union[List[str], Set[str], str], + sentences: Union[list[Sentence], Sentence], + candidate_label_set: Union[list[str], set[str], str], multi_label: bool = True, ): """Make zero shot predictions from the TARS model. @@ -307,14 +307,14 @@ def predict_zero_shot( def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: yield from super().get_used_tokens(corpus, context_length, respect_document_boundaries) for label in self.get_current_label_dictionary().idx2item: yield [label.decode("utf-8")] yield [self.separator] @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "FewshotClassifier": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "FewshotClassifier": from typing import cast return cast("FewshotClassifier", super().load(model_path=model_path)) @@ -472,7 +472,7 @@ def tars_embeddings(self): def predict( self, - sentences: Union[List[Sentence], Sentence], + sentences: Union[list[Sentence], Sentence], mini_batch_size=32, return_probabilities_for_all_classes: bool = False, verbose: bool = False, @@ -532,12 +532,12 @@ def predict( if not batch: continue - tars_sentences: List[Sentence] = [] - all_labels_to_sentence: List[Dict[str, Sentence]] = [] + tars_sentences: list[Sentence] = [] + all_labels_to_sentence: list[dict[str, Sentence]] = [] for sentence in batch: # always remove tags first sentence.remove_labels(label_name) - labels_to_sentence: Dict[str, Sentence] = {} + labels_to_sentence: dict[str, Sentence] = {} for label in all_labels: tars_sentence = self._get_tars_formatted_sentence(label, sentence) tars_sentences.append(tars_sentence) @@ -570,7 +570,7 @@ def predict( if most_probable_first: import operator - already_set_indices: List[int] = [] + already_set_indices: list[int] = [] sorted_x = sorted(all_detected.items(), key=operator.itemgetter(1)) sorted_x.reverse() @@ -648,7 +648,7 @@ def _print_predictions(self, batch, gold_label_type): return lines @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "TARSTagger": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "TARSTagger": from typing import cast return cast("TARSTagger", super().load(model_path=model_path)) @@ -832,7 +832,7 @@ def tars_embeddings(self): def predict( self, - sentences: Union[List[Sentence], Sentence], + sentences: Union[list[Sentence], Sentence], mini_batch_size=32, return_probabilities_for_all_classes: bool = False, verbose: bool = False, @@ -907,12 +907,12 @@ def predict( if not batch: continue - tars_sentences: List[Sentence] = [] - all_labels_to_sentence: List[Dict[str, Sentence]] = [] + tars_sentences: list[Sentence] = [] + all_labels_to_sentence: list[dict[str, Sentence]] = [] for sentence in batch: # always remove tags first sentence.remove_labels(label_name) - labels_to_sentence: Dict[str, Sentence] = {} + labels_to_sentence: dict[str, Sentence] = {} for label in all_labels: tars_sentence = self._get_tars_formatted_sentence(label, sentence) tars_sentences.append(tars_sentence) @@ -972,7 +972,7 @@ def predict( return None @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "TARSClassifier": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "TARSClassifier": from typing import cast return cast("TARSClassifier", super().load(model_path=model_path)) diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py index 1b330a0da3..7f4e00d2c4 100644 --- a/flair/models/text_classification_model.py +++ b/flair/models/text_classification_model.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Any, Dict, List, Union +from typing import Any, Union import torch @@ -56,7 +56,7 @@ def _get_embedding_for_data_point(self, prediction_data_point: Sentence) -> torc embedding_names = self.embeddings.get_names() return prediction_data_point.get_embedding(embedding_names) - def _get_data_points_from_sentence(self, sentence: Sentence) -> List[Sentence]: + def _get_data_points_from_sentence(self, sentence: Sentence) -> list[Sentence]: return [sentence] def _get_state_dict(self): @@ -133,7 +133,7 @@ def label_type(self): return self._label_type @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "TextClassifier": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "TextClassifier": from typing import cast return cast("TextClassifier", super().load(model_path=model_path)) diff --git a/flair/models/text_regression_model.py b/flair/models/text_regression_model.py index 894ce3087e..d1ad98d4e0 100644 --- a/flair/models/text_regression_model.py +++ b/flair/models/text_regression_model.py @@ -1,7 +1,7 @@ import logging import typing from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import torch from torch import nn @@ -43,7 +43,7 @@ def __init__( def label_type(self): return self.label_name - def _prepare_tensors(self, sentences: List[Sentence]) -> Tuple[torch.Tensor]: + def _prepare_tensors(self, sentences: list[Sentence]) -> tuple[torch.Tensor]: self.document_embeddings.embed(sentences) embedding_names = self.document_embeddings.get_names() text_embedding_list = [sentence.get_embedding(embedding_names).unsqueeze(0) for sentence in sentences] @@ -55,14 +55,14 @@ def forward(self, *args: torch.Tensor) -> torch.Tensor: label_scores = self.decoder(text_embedding_tensor) return label_scores - def forward_loss(self, sentences: List[Sentence]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, sentences: list[Sentence]) -> tuple[torch.Tensor, int]: labels = self._labels_to_tensor(sentences) text_embedding_tensor = self._prepare_tensors(sentences) scores = self.forward(*text_embedding_tensor) return self.loss_function(scores.squeeze(1), labels), len(sentences) - def _labels_to_tensor(self, sentences: List[Sentence]): + def _labels_to_tensor(self, sentences: list[Sentence]): indices = [ torch.tensor([float(label.value) for label in sentence.get_labels(self.label_name)], dtype=torch.float) for sentence in sentences @@ -74,12 +74,12 @@ def _labels_to_tensor(self, sentences: List[Sentence]): def predict( self, - sentences: Union[Sentence, List[Sentence]], + sentences: Union[Sentence, list[Sentence]], mini_batch_size: int = 32, verbose: bool = False, label_name: Optional[str] = None, embedding_storage_mode: EmbeddingStorageMode = "none", - ) -> List[Sentence]: + ) -> list[Sentence]: if label_name is None: label_name = self.label_name if self.label_name is not None else "label" @@ -123,7 +123,7 @@ def predict( return sentences - def forward_labels_and_loss(self, sentences: List[Sentence]) -> Tuple[torch.Tensor, torch.Tensor]: + def forward_labels_and_loss(self, sentences: list[Sentence]) -> tuple[torch.Tensor, torch.Tensor]: labels = self._labels_to_tensor(sentences) text_embedding_tensor = self._prepare_tensors(sentences) scores = self.forward(*text_embedding_tensor) @@ -132,13 +132,13 @@ def forward_labels_and_loss(self, sentences: List[Sentence]) -> Tuple[torch.Tens def evaluate( self, - data_points: Union[List[Sentence], Dataset], + data_points: Union[list[Sentence], Dataset], gold_label_type: str, out_path: Optional[Union[str, Path]] = None, embedding_storage_mode: EmbeddingStorageMode = "none", mini_batch_size: int = 32, - main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"), - exclude_labels: Optional[List[str]] = None, + main_evaluation_metric: tuple[str, str] = ("micro avg", "f1-score"), + exclude_labels: Optional[list[str]] = None, gold_label_dictionary: Optional[Dictionary] = None, return_loss: bool = True, **kwargs, @@ -154,7 +154,7 @@ def evaluate( metric = MetricRegression("Evaluation") - lines: List[str] = [] + lines: list[str] = [] total_count = 0 for batch in data_loader: if isinstance(batch, Sentence): @@ -227,21 +227,21 @@ def _init_model_with_state_dict(cls, state, **kwargs): ) @staticmethod - def _filter_empty_sentences(sentences: List[Sentence]) -> List[Sentence]: + def _filter_empty_sentences(sentences: list[Sentence]) -> list[Sentence]: filtered_sentences = [sentence for sentence in sentences if sentence.tokens] if len(sentences) != len(filtered_sentences): log.warning(f"Ignore {len(sentences) - len(filtered_sentences)} sentence(s) with no tokens.") return filtered_sentences @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "TextRegressor": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "TextRegressor": from typing import cast return cast("TextRegressor", super().load(model_path=model_path)) def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: for sentence in _iter_dataset(corpus.get_all_sentences()): yield [t.text for t in sentence] yield [t.text for t in sentence.left_context(context_length, respect_document_boundaries)] diff --git a/flair/models/triple_classification_model.py b/flair/models/triple_classification_model.py index 1c1337f9b0..9f1a57a23e 100644 --- a/flair/models/triple_classification_model.py +++ b/flair/models/triple_classification_model.py @@ -1,5 +1,4 @@ import typing -from typing import List import torch @@ -69,7 +68,7 @@ def __init__( def label_type(self): return self._label_type - def _get_data_points_from_sentence(self, sentence: TextTriple) -> List[TextTriple]: + def _get_data_points_from_sentence(self, sentence: TextTriple) -> list[TextTriple]: return [sentence] def _get_embedding_for_data_point(self, prediction_data_point: TextTriple) -> torch.Tensor: @@ -121,7 +120,7 @@ def _init_model_with_state_dict(cls, state, **kwargs): def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: for sentence_pair in _iter_dataset(corpus.get_all_sentences()): yield [t.text for t in sentence_pair.first] yield [t.text for t in sentence_pair.first.left_context(context_length, respect_document_boundaries)] diff --git a/flair/models/word_tagger_model.py b/flair/models/word_tagger_model.py index 2d32a54b06..5040a63728 100644 --- a/flair/models/word_tagger_model.py +++ b/flair/models/word_tagger_model.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Any, Dict, List, Union +from typing import Any, Union import torch from deprecated.sphinx import deprecated @@ -99,7 +99,7 @@ def _get_embedding_for_data_point(self, prediction_data_point: Token) -> torch.T names = self.embeddings.get_names() return prediction_data_point.get_embedding(names) - def _get_data_points_from_sentence(self, sentence: Sentence) -> List[Token]: + def _get_data_points_from_sentence(self, sentence: Sentence) -> list[Token]: # special handling during training if this is a span prediction problem if self.training and self.span_prediction_problem: for token in sentence.tokens: @@ -125,7 +125,7 @@ def _post_process_batch_after_prediction(self, batch, label_name): for sentence in batch: # internal variables previous_tag = "O-" - current_span: List[Token] = [] + current_span: list[Token] = [] for token in sentence: bioes_tag = token.get_label(label_name).value @@ -222,7 +222,7 @@ def _print_predictions(self, batch, gold_label_type): return lines @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "TokenClassifier": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "TokenClassifier": from typing import cast return cast("TokenClassifier", super().load(model_path=model_path)) diff --git a/flair/nn/decoder.py b/flair/nn/decoder.py index 65f802148a..48cdbf39b0 100644 --- a/flair/nn/decoder.py +++ b/flair/nn/decoder.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional +from typing import Optional import torch @@ -151,11 +151,11 @@ class LabelVerbalizerDecoder(torch.nn.Module): def __init__(self, label_embedding: Embeddings, label_dictionary: Dictionary): super().__init__() self.label_embedding = label_embedding - self.verbalized_labels: List[Sentence] = self.verbalize_labels(label_dictionary) + self.verbalized_labels: list[Sentence] = self.verbalize_labels(label_dictionary) self.to(flair.device) @staticmethod - def verbalize_labels(label_dictionary: Dictionary) -> List[Sentence]: + def verbalize_labels(label_dictionary: Dictionary) -> list[Sentence]: """Takes a label dictionary and returns a list of sentences with verbalized labels. Args: diff --git a/flair/nn/model.py b/flair/nn/model.py index eeb5b7c84a..bf13baf2f1 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from collections import Counter from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Optional, Union import torch.nn from torch import Tensor @@ -31,7 +31,7 @@ class Model(torch.nn.Module, typing.Generic[DT], ABC): Every new type of model must implement these methods. """ - model_card: Optional[Dict[str, Any]] = None + model_card: Optional[dict[str, Any]] = None @property @abstractmethod @@ -40,7 +40,7 @@ def label_type(self) -> str: raise NotImplementedError @abstractmethod - def forward_loss(self, data_points: List[DT]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, data_points: list[DT]) -> tuple[torch.Tensor, int]: """Performs a forward pass and returns a loss tensor for backpropagation. Implement this to enable training. @@ -50,13 +50,13 @@ def forward_loss(self, data_points: List[DT]) -> Tuple[torch.Tensor, int]: @abstractmethod def evaluate( self, - data_points: Union[List[DT], Dataset], + data_points: Union[list[DT], Dataset], gold_label_type: str, out_path: Optional[Union[str, Path]] = None, embedding_storage_mode: EmbeddingStorageMode = "none", mini_batch_size: int = 32, - main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"), - exclude_labels: Optional[List[str]] = None, + main_evaluation_metric: tuple[str, str] = ("micro avg", "f1-score"), + exclude_labels: Optional[list[str]] = None, gold_label_dictionary: Optional[Dictionary] = None, return_loss: bool = True, **kwargs, @@ -84,7 +84,7 @@ def evaluate( exclude_labels = exclude_labels if exclude_labels is not None else [] raise NotImplementedError - def _get_state_dict(self) -> Dict: + def _get_state_dict(self) -> dict: """Returns the state dictionary for this model.""" # Always include the name of the Model class for which the state dict holds state_dict = {"state_dict": self.state_dict(), "__cls__": self.__class__.__name__} @@ -92,7 +92,7 @@ def _get_state_dict(self) -> Dict: return state_dict @classmethod - def _init_model_with_state_dict(cls, state: Dict[str, Any], **kwargs): + def _init_model_with_state_dict(cls, state: dict[str, Any], **kwargs): """Initialize the model from a state dictionary.""" if "embeddings" in kwargs: embeddings = kwargs.pop("embeddings") @@ -128,7 +128,7 @@ def save(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: torch.save(model_state, str(model_file), pickle_protocol=4) @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "Model": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "Model": """Loads the model from the given file. Args: @@ -238,7 +238,7 @@ class ReduceTransformerVocabMixin(ABC): @abstractmethod def get_used_tokens( self, corpus: Corpus, context_lenth: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: pass @@ -251,13 +251,13 @@ class Classifier(Model[DT], typing.Generic[DT], ReduceTransformerVocabMixin, ABC def evaluate( self, - data_points: Union[List[DT], Dataset], + data_points: Union[list[DT], Dataset], gold_label_type: str, out_path: Optional[Union[str, Path]] = None, embedding_storage_mode: EmbeddingStorageMode = "none", mini_batch_size: int = 32, - main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"), - exclude_labels: Optional[List[str]] = None, + main_evaluation_metric: tuple[str, str] = ("micro avg", "f1-score"), + exclude_labels: Optional[list[str]] = None, gold_label_dictionary: Optional[Dictionary] = None, return_loss: bool = True, **kwargs, @@ -281,10 +281,10 @@ def evaluate( average_over = 0 # variables for printing - lines: List[str] = [] + lines: list[str] = [] # variables for computing scores - all_spans: Set[str] = set() + all_spans: set[str] = set() all_true_values = {} all_predicted_values = {} @@ -476,7 +476,7 @@ def evaluate( ) # Create and populate score object for logging with all evaluation values, plus the loss - scores: Dict[Union[Tuple[str, ...], str], Any] = {} + scores: dict[Union[tuple[str, ...], str], Any] = {} for avg_type in ("micro avg", "macro avg"): for metric_type in ("f1-score", "precision", "recall"): @@ -514,7 +514,7 @@ def evaluate( @abstractmethod def predict( self, - sentences: Union[List[DT], DT], + sentences: Union[list[DT], DT], mini_batch_size: int = 32, return_probabilities_for_all_classes: bool = False, verbose: bool = False, @@ -537,7 +537,7 @@ def predict( """ raise NotImplementedError - def _print_predictions(self, batch: List[DT], gold_label_type: str) -> List[str]: + def _print_predictions(self, batch: list[DT], gold_label_type: str) -> list[str]: lines = [] for datapoint in batch: # check if there is a label mismatch @@ -557,14 +557,14 @@ def _print_predictions(self, batch: List[DT], gold_label_type: str) -> List[str] def get_used_tokens( self, corpus: Corpus, context_length: int = 0, respect_document_boundaries: bool = True - ) -> typing.Iterable[List[str]]: + ) -> typing.Iterable[list[str]]: for sentence in _iter_dataset(corpus.get_all_sentences()): yield [t.text for t in sentence] yield [t.text for t in sentence.left_context(context_length, respect_document_boundaries)] yield [t.text for t in sentence.right_context(context_length, respect_document_boundaries)] @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "Classifier": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "Classifier": from typing import cast return cast("Classifier", super().load(model_path=model_path)) @@ -589,7 +589,7 @@ def __init__( word_dropout: float = 0.0, multi_label: bool = False, multi_label_threshold: float = 0.5, - loss_weights: Optional[Dict[str, float]] = None, + loss_weights: Optional[dict[str, float]] = None, decoder: Optional[torch.nn.Module] = None, inverse_model: bool = False, train_on_gold_pairs_only: bool = False, @@ -663,21 +663,21 @@ def _get_embedding_for_data_point(self, prediction_data_point: DT2) -> torch.Ten raise NotImplementedError @abstractmethod - def _get_data_points_from_sentence(self, sentence: DT) -> List[DT2]: + def _get_data_points_from_sentence(self, sentence: DT) -> list[DT2]: """Returns the data_points to which labels are added. The results should be of any type that inherits from DataPoint (Sentence, Span, Token, ... objects). """ raise NotImplementedError - def _get_data_points_for_batch(self, sentences: List[DT]) -> List[DT2]: + def _get_data_points_for_batch(self, sentences: list[DT]) -> list[DT2]: """Returns the data_points to which labels are added. The results should be of any type that inherits from DataPoint (Sentence, Span, Token, ... objects). """ return [data_point for sentence in sentences for data_point in self._get_data_points_from_sentence(sentence)] - def _get_label_of_datapoint(self, data_point: DT2) -> List[str]: + def _get_label_of_datapoint(self, data_point: DT2) -> list[str]: """Extracts the labels from the data points. Each data point might return a list of strings, representing multiple labels. @@ -701,7 +701,7 @@ def multi_label_threshold(self, x): # setter method else: self._multi_label_threshold = {"default": x} - def _prepare_label_tensor(self, prediction_data_points: List[DT2]) -> torch.Tensor: + def _prepare_label_tensor(self, prediction_data_points: list[DT2]) -> torch.Tensor: labels = [self._get_label_of_datapoint(dp) for dp in prediction_data_points] if self.multi_label: return torch.tensor( @@ -726,7 +726,7 @@ def _prepare_label_tensor(self, prediction_data_points: List[DT2]) -> torch.Tens device=flair.device, ) - def _encode_data_points(self, sentences: List[DT], data_points: List[DT2]) -> Tensor: + def _encode_data_points(self, sentences: list[DT], data_points: list[DT2]) -> Tensor: # embed sentences if self.should_embed_sentence: self.embeddings.embed(sentences) @@ -747,7 +747,7 @@ def _mask_scores(self, scores: Tensor, data_points) -> Tensor: """Classes that inherit from DefaultClassifier may optionally mask scores.""" return scores - def forward_loss(self, sentences: List[DT]) -> Tuple[torch.Tensor, int]: + def forward_loss(self, sentences: list[DT]) -> tuple[torch.Tensor, int]: # make a forward pass to produce embedded data points and labels sentences = [sentence for sentence in sentences if self._filter_data_point(sentence)] @@ -773,10 +773,10 @@ def forward_loss(self, sentences: List[DT]) -> Tuple[torch.Tensor, int]: # calculate the loss return self._calculate_loss(scores, label_tensor) - def _calculate_loss(self, scores: torch.Tensor, labels: torch.Tensor) -> Tuple[torch.Tensor, int]: + def _calculate_loss(self, scores: torch.Tensor, labels: torch.Tensor) -> tuple[torch.Tensor, int]: return self.loss_function(scores, labels), labels.size(0) - def _sort_data(self, data_points: List[DT]) -> List[DT]: + def _sort_data(self, data_points: list[DT]) -> list[DT]: if len(data_points) == 0: return [] @@ -784,16 +784,16 @@ def _sort_data(self, data_points: List[DT]) -> List[DT]: return data_points # filter empty sentences - sentences = [sentence for sentence in typing.cast(List[Sentence], data_points) if len(sentence) > 0] + sentences = [sentence for sentence in typing.cast(list[Sentence], data_points) if len(sentence) > 0] # reverse sort all sequences by their length reordered_sentences = sorted(sentences, key=len, reverse=True) - return typing.cast(List[DT], reordered_sentences) + return typing.cast(list[DT], reordered_sentences) def predict( self, - sentences: Union[List[DT], DT], + sentences: Union[list[DT], DT], mini_batch_size: int = 32, return_probabilities_for_all_classes: bool = False, verbose: bool = False, @@ -824,7 +824,7 @@ def predict( sentences = [sentences] if isinstance(sentences[0], Sentence): - Sentence.set_context_for_sentences(typing.cast(List[Sentence], sentences)) + Sentence.set_context_for_sentences(typing.cast(list[Sentence], sentences)) reordered_sentences = self._sort_data(sentences) @@ -832,7 +832,7 @@ def predict( return sentences if len(reordered_sentences) > mini_batch_size: - batches: Union[DataLoader, List[List[DT]]] = DataLoader( + batches: Union[DataLoader, list[list[DT]]] = DataLoader( dataset=FlairDatapointDataset(reordered_sentences), batch_size=mini_batch_size, ) @@ -990,7 +990,7 @@ def _get_state_dict(self): return state @classmethod - def load(cls, model_path: Union[str, Path, Dict[str, Any]]) -> "DefaultClassifier": + def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "DefaultClassifier": from typing import cast return cast("DefaultClassifier", super().load(model_path=model_path)) diff --git a/flair/nn/multitask.py b/flair/nn/multitask.py index 6fa2f20c02..42c5665141 100644 --- a/flair/nn/multitask.py +++ b/flair/nn/multitask.py @@ -1,4 +1,5 @@ -from typing import Iterable, Tuple, Union +from collections.abc import Iterable +from typing import Union from flair.data import Corpus, MultiCorpus from flair.models import MultitaskModel @@ -6,18 +7,18 @@ def make_multitask_model_and_corpus( - mapping: Iterable[Union[Tuple[Classifier, Corpus], Tuple[Classifier, Corpus, float]]] -) -> Tuple[Model, Corpus]: + mapping: Iterable[Union[tuple[Classifier, Corpus], tuple[Classifier, Corpus, float]]] +) -> tuple[Model, Corpus]: models = [] corpora = [] loss_factors = [] ids = [] - for task_id, map in enumerate(mapping): - models.append(map[0]) - corpora.append(map[1]) - if len(map) == 3: - loss_factors.append(map[2]) + for task_id, _map in enumerate(mapping): + models.append(_map[0]) + corpora.append(_map[1]) + if len(_map) == 3: + loss_factors.append(_map[2]) else: loss_factors.append(1.0) diff --git a/flair/samplers.py b/flair/samplers.py index 135dfb3310..53ad40c4c5 100644 --- a/flair/samplers.py +++ b/flair/samplers.py @@ -1,7 +1,6 @@ import logging import random from collections import defaultdict -from typing import Dict import torch from torch.utils.data.sampler import Sampler @@ -36,7 +35,7 @@ def set_dataset(self, data_source): self.indices = list(range(len(data_source))) # first determine the distribution of classes in the dataset - label_count: Dict[str, int] = defaultdict(int) + label_count: dict[str, int] = defaultdict(int) for sentence in data_source: for label in sentence.labels: label_count[label.value] += 1 diff --git a/flair/splitter.py b/flair/splitter.py index 9f7e502c87..2b6c90cd7f 100644 --- a/flair/splitter.py +++ b/flair/splitter.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, List, Optional, Union +from typing import Any, Optional, Union from segtok.segmenter import split_multi @@ -25,7 +25,7 @@ class SentenceSplitter(ABC): the sentence splitter's configuration. """ - def split(self, text: str, link_sentences: Optional[bool] = True) -> List[Sentence]: + def split(self, text: str, link_sentences: Optional[bool] = True) -> list[Sentence]: sentences = self._perform_split(text) if not link_sentences: return sentences @@ -34,7 +34,7 @@ def split(self, text: str, link_sentences: Optional[bool] = True) -> List[Senten return sentences @abstractmethod - def _perform_split(self, text: str) -> List[Sentence]: + def _perform_split(self, text: str) -> list[Sentence]: raise NotImplementedError @property @@ -62,11 +62,11 @@ def __init__(self, tokenizer: Tokenizer = SegtokTokenizer()) -> None: super().__init__() self._tokenizer = tokenizer - def _perform_split(self, text: str) -> List[Sentence]: - plain_sentences: List[str] = split_multi(text) + def _perform_split(self, text: str) -> list[Sentence]: + plain_sentences: list[str] = split_multi(text) sentence_offset = 0 - sentences: List[Sentence] = [] + sentences: list[Sentence] = [] for sentence in plain_sentences: try: sentence_offset = text.index(sentence, sentence_offset) @@ -133,7 +133,7 @@ def __init__(self, model: Union[Any, str], tokenizer: Optional[Tokenizer] = None else: self._tokenizer = tokenizer - def _perform_split(self, text: str) -> List[Sentence]: + def _perform_split(self, text: str) -> list[Sentence]: document = self.model(text) sentences = [ @@ -192,7 +192,7 @@ def __init__(self, tag: str, tokenizer: Tokenizer = SegtokTokenizer()) -> None: self._tokenizer = tokenizer self.tag = tag - def _perform_split(self, text: str) -> List[Sentence]: + def _perform_split(self, text: str) -> list[Sentence]: plain_sentences = text.split(self.tag) sentences = [] @@ -252,7 +252,7 @@ def __init__(self, tokenizer: Tokenizer = SegtokTokenizer()) -> None: super().__init__() self._tokenizer = tokenizer - def _perform_split(self, text: str) -> List[Sentence]: + def _perform_split(self, text: str) -> list[Sentence]: return [Sentence(text=text, use_tokenizer=self._tokenizer, start_position=0)] @property diff --git a/flair/tokenization.py b/flair/tokenization.py index 185e944d3b..b377c419e6 100644 --- a/flair/tokenization.py +++ b/flair/tokenization.py @@ -1,7 +1,7 @@ import logging import sys from abc import ABC, abstractmethod -from typing import Callable, List +from typing import Callable from segtok.segmenter import split_single from segtok.tokenizer import split_contractions, word_tokenizer @@ -20,7 +20,7 @@ class Tokenizer(ABC): """ @abstractmethod - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: raise NotImplementedError @property @@ -57,11 +57,11 @@ def __init__(self, model) -> None: "spacy model or the name of the model to load." ) - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: from spacy.tokens.doc import Doc doc: Doc = self.model.make_doc(text) - words: List[str] = [] + words: list[str] = [] for word in doc: if len(word.text.strip()) == 0: continue @@ -82,12 +82,12 @@ class SegtokTokenizer(Tokenizer): def __init__(self) -> None: super().__init__() - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: return SegtokTokenizer.run_tokenize(text) @staticmethod - def run_tokenize(text: str) -> List[str]: - words: List[str] = [] + def run_tokenize(text: str) -> list[str]: + words: list[str] = [] sentences = split_single(text) for sentence in sentences: @@ -105,12 +105,12 @@ class SpaceTokenizer(Tokenizer): def __init__(self) -> None: super().__init__() - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: return SpaceTokenizer.run_tokenize(text) @staticmethod - def run_tokenize(text: str) -> List[str]: - tokens: List[str] = [] + def run_tokenize(text: str) -> list[str]: + tokens: list[str] = [] word = "" index = -1 for index, char in enumerate(text): @@ -166,8 +166,8 @@ def __init__(self, tokenizer: str, sudachi_mode: str = "A") -> None: self.sentence_tokenizer = konoha.SentenceTokenizer() self.word_tokenizer = konoha.WordTokenizer(tokenizer, mode=sudachi_mode) - def tokenize(self, text: str) -> List[str]: - words: List[str] = [] + def tokenize(self, text: str) -> list[str]: + words: list[str] = [] sentences = self.sentence_tokenizer.tokenize(text) for sentence in sentences: @@ -184,11 +184,11 @@ def name(self) -> str: class TokenizerWrapper(Tokenizer): """Helper class to wrap tokenizer functions to the class-based tokenizer interface.""" - def __init__(self, tokenizer_func: Callable[[str], List[str]]) -> None: + def __init__(self, tokenizer_func: Callable[[str], list[str]]) -> None: super().__init__() self.tokenizer_func = tokenizer_func - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: return self.tokenizer_func(text) @property @@ -225,7 +225,7 @@ def __init__(self) -> None: " Note that the scispacy version and the version of the model must match to work properly!" ) - def combined_rule_prefixes() -> List[str]: + def combined_rule_prefixes() -> list[str]: """Helper function that returns the prefix pattern for the tokenizer. It is a helper function to accommodate spacy tests that only test prefixes. @@ -270,9 +270,9 @@ def combined_rule_prefixes() -> List[str]: self.model.tokenizer.prefix_search = prefix_re.search self.model.tokenizer.infix_finditer = infix_re.finditer - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: sentence = self.model(text) - words: List[str] = [] + words: list[str] = [] for word in sentence: words.append(word.text) return words diff --git a/flair/trainers/language_model_trainer.py b/flair/trainers/language_model_trainer.py index eb374ed75d..341cead776 100644 --- a/flair/trainers/language_model_trainer.py +++ b/flair/trainers/language_model_trainer.py @@ -3,8 +3,9 @@ import math import random import time +from collections.abc import Iterable from pathlib import Path -from typing import Any, Dict, Iterable, Optional, Type, Union +from typing import Any, Optional, Union import torch from torch import cuda @@ -155,16 +156,16 @@ def __init__( self, model: LanguageModel, corpus: TextCorpus, - optimizer: Type[Optimizer] = SGD, + optimizer: type[Optimizer] = SGD, test_mode: bool = False, epoch: int = 0, split: int = 0, loss: float = 10000, - optimizer_state: Optional[Dict[str, Any]] = None, - scaler_state: Optional[Dict[str, Any]] = None, + optimizer_state: Optional[dict[str, Any]] = None, + scaler_state: Optional[dict[str, Any]] = None, ) -> None: self.model: LanguageModel = model - self.optimizer: Type[Optimizer] = optimizer + self.optimizer: type[Optimizer] = optimizer self.corpus: TextCorpus = corpus self.test_mode: bool = test_mode @@ -362,7 +363,7 @@ def train( ) with open(loss_txt, "a") as myfile: - myfile.write("%s\n" % summary) + myfile.write(f"{summary}\n") log.info(summary) log.info("-" * 89) @@ -386,7 +387,7 @@ def train( summary = f"TEST: valid loss {test_loss:5.4f} | valid ppl {math.exp(test_loss):8.4f}" with open(loss_txt, "a") as myfile: - myfile.write("%s\n" % summary) + myfile.write(f"{summary}\n") log.info(summary) log.info("-" * 89) @@ -440,7 +441,7 @@ def _repackage_hidden(h): def load_checkpoint( checkpoint_file: Union[str, Path], corpus: TextCorpus, - optimizer: Type[Optimizer] = SGD, + optimizer: type[Optimizer] = SGD, ): if isinstance(checkpoint_file, str): checkpoint_file = Path(checkpoint_file) diff --git a/flair/trainers/plugins/base.py b/flair/trainers/plugins/base.py index 958d57b785..663a78d6d3 100644 --- a/flair/trainers/plugins/base.py +++ b/flair/trainers/plugins/base.py @@ -1,19 +1,14 @@ import logging from collections import defaultdict +from collections.abc import Iterator, Sequence from inspect import isclass, signature from itertools import count from queue import Queue from typing import ( Any, Callable, - Dict, - Iterator, - List, NewType, Optional, - Sequence, - Set, - Type, Union, cast, ) @@ -21,7 +16,7 @@ log = logging.getLogger("flair") -PluginArgument = Union["BasePlugin", Type["BasePlugin"]] +PluginArgument = Union["BasePlugin", type["BasePlugin"]] HookHandleId = NewType("HookHandleId", int) EventIdenifier = str @@ -34,7 +29,7 @@ class TrainingInterrupt(Exception): class Pluggable: """Dispatches events which attached plugins can react to.""" - valid_events: Optional[Set[EventIdenifier]] = None + valid_events: Optional[set[EventIdenifier]] = None def __init__(self, *, plugins: Sequence[PluginArgument] = []) -> None: """Initialize a `Pluggable`. @@ -42,11 +37,11 @@ def __init__(self, *, plugins: Sequence[PluginArgument] = []) -> None: Args: plugins: Plugins which should be attached to this `Pluggable`. """ - self._hook_handles: Dict[EventIdenifier, Dict[HookHandleId, HookHandle]] = defaultdict(dict) + self._hook_handles: dict[EventIdenifier, dict[HookHandleId, HookHandle]] = defaultdict(dict) self._hook_handle_id_counter = count() - self._plugins: List[BasePlugin] = [] + self._plugins: list[BasePlugin] = [] # This flag tracks, whether an event is currently being processed (otherwise it is added to the queue) self._processing_events = False @@ -181,7 +176,7 @@ class BasePlugin: def __init__(self) -> None: """Initialize the base plugin.""" - self._hook_handles: List[HookHandle] = [] + self._hook_handles: list[HookHandle] = [] self._pluggable: Optional[Pluggable] = None def attach_to(self, pluggable: Pluggable): @@ -260,7 +255,7 @@ def pluggable(self) -> Optional[Pluggable]: def __str__(self) -> str: return self.__class__.__name__ - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return {"__cls__": f"{self.__module__}.{self.__class__.__name__}"} diff --git a/flair/trainers/plugins/functional/anneal_on_plateau.py b/flair/trainers/plugins/functional/anneal_on_plateau.py index d62b21fba1..ccd330bf0f 100644 --- a/flair/trainers/plugins/functional/anneal_on_plateau.py +++ b/flair/trainers/plugins/functional/anneal_on_plateau.py @@ -1,6 +1,6 @@ import logging import os -from typing import Any, Dict +from typing import Any from flair.trainers.plugins.base import TrainerPlugin, TrainingInterrupt from flair.trainers.plugins.metric_records import MetricRecord @@ -108,7 +108,7 @@ def __str__(self) -> str: f"min_learning_rate: '{self.min_learning_rate}'" ) - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "base_path": str(self.base_path), diff --git a/flair/trainers/plugins/functional/checkpoints.py b/flair/trainers/plugins/functional/checkpoints.py index 75ecb9bd98..1936177835 100644 --- a/flair/trainers/plugins/functional/checkpoints.py +++ b/flair/trainers/plugins/functional/checkpoints.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict +from typing import Any from flair.trainers.plugins.base import TrainerPlugin @@ -29,7 +29,7 @@ def after_training_epoch(self, epoch, **kw): model_name = "model_epoch_" + str(epoch) + ".pt" self.model.save(self.base_path / model_name, checkpoint=self.save_optimizer_state) - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "base_path": str(self.base_path), diff --git a/flair/trainers/plugins/functional/linear_scheduler.py b/flair/trainers/plugins/functional/linear_scheduler.py index 1000be6dd7..2258844129 100644 --- a/flair/trainers/plugins/functional/linear_scheduler.py +++ b/flair/trainers/plugins/functional/linear_scheduler.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict +from typing import Any from flair.optim import LinearSchedulerWithWarmup from flair.trainers.plugins.base import TrainerPlugin @@ -62,7 +62,7 @@ def after_training_batch(self, optimizer_was_run: bool, **kwargs): def __str__(self) -> str: return f"LinearScheduler | warmup_fraction: '{self.warmup_fraction}'" - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "warmup_fraction": self.warmup_fraction, diff --git a/flair/trainers/plugins/functional/reduce_transformer_vocab.py b/flair/trainers/plugins/functional/reduce_transformer_vocab.py index 162c667f88..86759c2fe2 100644 --- a/flair/trainers/plugins/functional/reduce_transformer_vocab.py +++ b/flair/trainers/plugins/functional/reduce_transformer_vocab.py @@ -1,6 +1,5 @@ import logging from pathlib import Path -from typing import List from transformer_smaller_training_vocab import reduce_train_vocab @@ -57,7 +56,7 @@ def save_model_at_the_end(self, **kw): self.model.save(self.base_path / "final-model.pt", checkpoint=self.save_optimizer_state) -def get_transformer_embeddings(model: Model) -> List[TransformerEmbeddings]: +def get_transformer_embeddings(model: Model) -> list[TransformerEmbeddings]: embeddings = model.tars_embeddings if isinstance(model, FewshotClassifier) else getattr(model, "embeddings", None) if embeddings is None: diff --git a/flair/trainers/plugins/functional/weight_extractor.py b/flair/trainers/plugins/functional/weight_extractor.py index ef5afe081e..4ba7c07621 100644 --- a/flair/trainers/plugins/functional/weight_extractor.py +++ b/flair/trainers/plugins/functional/weight_extractor.py @@ -1,4 +1,4 @@ -from typing import Any, Dict +from typing import Any from flair.trainers.plugins.base import TrainerPlugin from flair.training_utils import WeightExtractor @@ -21,7 +21,7 @@ def after_training_batch(self, batch_no, epoch, total_number_of_batches, **kw): if (iteration + 1) % modulo == 0: self.weight_extractor.extract_weights(self.model.state_dict(), iteration) - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "base_path": str(self.base_path), diff --git a/flair/trainers/plugins/loggers/log_file.py b/flair/trainers/plugins/loggers/log_file.py index a9b7453a09..21a8c54632 100644 --- a/flair/trainers/plugins/loggers/log_file.py +++ b/flair/trainers/plugins/loggers/log_file.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Any, Dict +from typing import Any from flair.trainers.plugins.base import TrainerPlugin from flair.training_utils import add_file_handler @@ -21,5 +21,5 @@ def close_file_handler(self, **kw): self.log_handler.close() log.removeHandler(self.log_handler) - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return {**super().get_state(), "base_path": str(self.base_path)} diff --git a/flair/trainers/plugins/loggers/loss_file.py b/flair/trainers/plugins/loggers/loss_file.py index 29c42fc930..b53a23a956 100644 --- a/flair/trainers/plugins/loggers/loss_file.py +++ b/flair/trainers/plugins/loggers/loss_file.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Optional, Union from flair.trainers.plugins.base import TrainerPlugin from flair.trainers.plugins.metric_records import MetricName @@ -10,7 +10,7 @@ class LossFilePlugin(TrainerPlugin): """Plugin that manages the loss.tsv file output.""" def __init__( - self, base_path, epoch: int, metrics_to_collect: Optional[Dict[Union[Tuple, str], str]] = None + self, base_path, epoch: int, metrics_to_collect: Optional[dict[Union[tuple, str], str]] = None ) -> None: super().__init__() @@ -56,9 +56,9 @@ def __init__( self.headers[metric_name] = f"{prefix.upper()}_{header}" # initialize the first log line - self.current_row: Optional[Dict[MetricName, str]] = None + self.current_row: Optional[dict[MetricName, str]] = None - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "base_path": str(self.base_path), diff --git a/flair/trainers/plugins/loggers/metric_history.py b/flair/trainers/plugins/loggers/metric_history.py index 8d7c946e8d..a22cf1b0e7 100644 --- a/flair/trainers/plugins/loggers/metric_history.py +++ b/flair/trainers/plugins/loggers/metric_history.py @@ -1,5 +1,6 @@ import logging -from typing import Any, Dict, Mapping +from collections.abc import Mapping +from typing import Any from flair.trainers.plugins.base import TrainerPlugin @@ -17,7 +18,7 @@ class MetricHistoryPlugin(TrainerPlugin): def __init__(self, metrics_to_collect: Mapping = default_metrics_to_collect) -> None: super().__init__() - self.metric_history: Dict[str, list] = {} + self.metric_history: dict[str, list] = {} self.metrics_to_collect: Mapping = metrics_to_collect for target in self.metrics_to_collect.values(): self.metric_history[target] = [] @@ -33,7 +34,7 @@ def after_training(self, **kw): """Returns metric history.""" self.trainer.return_values.update(self.metric_history) - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "metrics_to_collect": dict(self.metrics_to_collect), diff --git a/flair/trainers/plugins/loggers/tensorboard.py b/flair/trainers/plugins/loggers/tensorboard.py index 59bba9f2e9..a7af50a521 100644 --- a/flair/trainers/plugins/loggers/tensorboard.py +++ b/flair/trainers/plugins/loggers/tensorboard.py @@ -1,6 +1,6 @@ import logging import os -from typing import Any, Dict +from typing import Any from flair.trainers.plugins.base import TrainerPlugin from flair.training_utils import log_line @@ -59,7 +59,7 @@ def _training_finally(self, **kw): assert self.writer is not None self.writer.close() - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "log_dir": str(self.log_dir) if self.log_dir is not None else None, diff --git a/flair/trainers/plugins/loggers/wandb.py b/flair/trainers/plugins/loggers/wandb.py index 8608fcdbd9..0f8dc89f73 100644 --- a/flair/trainers/plugins/loggers/wandb.py +++ b/flair/trainers/plugins/loggers/wandb.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict +from typing import Any from flair.trainers.plugins.base import TrainerPlugin @@ -72,7 +72,7 @@ def metric_recorded(self, record): def _training_finally(self, **kw): self.writer.close() - def get_state(self) -> Dict[str, Any]: + def get_state(self) -> dict[str, Any]: return { **super().get_state(), "emit_alerts": self.emit_alerts, diff --git a/flair/trainers/plugins/metric_records.py b/flair/trainers/plugins/metric_records.py index 034c021854..548b54fccd 100644 --- a/flair/trainers/plugins/metric_records.py +++ b/flair/trainers/plugins/metric_records.py @@ -1,14 +1,15 @@ import time +from collections.abc import Iterable, Iterator from dataclasses import dataclass from enum import Enum -from typing import Any, Iterable, Iterator, Optional, Tuple, Union +from typing import Any, Optional, Union RecordType = Enum("RecordType", ["scalar", "image", "histogram", "string", "scalar_list"]) class MetricName: def __init__(self, name) -> None: - self.parts: Tuple[str, ...] + self.parts: tuple[str, ...] if isinstance(name, str): self.parts = tuple(name.split("/")) diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 03e6edc083..2f32b54c01 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -7,7 +7,7 @@ import warnings from inspect import signature from pathlib import Path -from typing import List, Optional, Tuple, Type, Union +from typing import Optional, Union import torch from torch.optim.sgd import SGD @@ -128,7 +128,7 @@ def train( base_path, anneal_factor: float = 0.5, patience: int = 3, - min_learning_rate: Union[float, List[float]] = 0.0001, + min_learning_rate: Union[float, list[float]] = 0.0001, initial_extra_patience: int = 0, anneal_with_restarts: bool = False, learning_rate: float = 0.1, @@ -137,17 +137,17 @@ def train( eval_batch_size: int = 64, mini_batch_chunk_size: Optional[int] = None, max_epochs: int = 100, - optimizer: Type[torch.optim.Optimizer] = torch.optim.SGD, + optimizer: type[torch.optim.Optimizer] = torch.optim.SGD, train_with_dev: bool = False, train_with_test: bool = False, reduce_transformer_vocab: bool = False, # evaluation and monitoring - main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"), + main_evaluation_metric: tuple[str, str] = ("micro avg", "f1-score"), monitor_test: bool = False, monitor_train_sample: float = 0.0, use_final_model_for_eval: bool = False, gold_label_dictionary_for_eval: Optional[Dictionary] = None, - exclude_labels: Optional[List[str]] = None, + exclude_labels: Optional[list[str]] = None, # sampling and shuffling sampler=None, shuffle: bool = True, @@ -164,7 +164,7 @@ def train( create_loss_file: bool = True, write_weights: bool = False, # plugins - plugins: Optional[List[TrainerPlugin]] = None, + plugins: Optional[list[TrainerPlugin]] = None, attach_default_scheduler: bool = True, **kwargs, ): @@ -211,17 +211,17 @@ def fine_tune( eval_batch_size: int = 16, mini_batch_chunk_size: Optional[int] = None, max_epochs: int = 10, - optimizer: Type[torch.optim.Optimizer] = torch.optim.AdamW, + optimizer: type[torch.optim.Optimizer] = torch.optim.AdamW, train_with_dev: bool = False, train_with_test: bool = False, reduce_transformer_vocab: bool = False, # evaluation and monitoring - main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"), + main_evaluation_metric: tuple[str, str] = ("micro avg", "f1-score"), monitor_test: bool = False, monitor_train_sample: float = 0.0, use_final_model_for_eval: bool = True, gold_label_dictionary_for_eval: Optional[Dictionary] = None, - exclude_labels: Optional[List[str]] = None, + exclude_labels: Optional[list[str]] = None, # sampling and shuffling sampler=None, shuffle: bool = True, @@ -240,7 +240,7 @@ def fine_tune( # amp use_amp: bool = False, # plugins - plugins: Optional[List[TrainerPlugin]] = None, + plugins: Optional[list[TrainerPlugin]] = None, attach_default_scheduler: bool = True, **kwargs, ): @@ -304,20 +304,20 @@ def train_custom( eval_batch_size: int = 64, mini_batch_chunk_size: Optional[int] = None, max_epochs: int = 100, - optimizer: Type[torch.optim.Optimizer] = SGD, + optimizer: type[torch.optim.Optimizer] = SGD, train_with_dev: bool = False, train_with_test: bool = False, max_grad_norm: Optional[float] = 5.0, reduce_transformer_vocab: bool = False, # evaluation and monitoring - main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"), + main_evaluation_metric: tuple[str, str] = ("micro avg", "f1-score"), monitor_test: bool = False, monitor_train_sample: float = 0.0, use_final_model_for_eval: bool = False, gold_label_dictionary_for_eval: Optional[Dictionary] = None, - exclude_labels: Optional[List[str]] = None, + exclude_labels: Optional[list[str]] = None, # sampling and shuffling - sampler: Optional[FlairSampler] = None, + sampler: Optional[Union[FlairSampler, type[FlairSampler]]] = None, shuffle: bool = True, shuffle_first_epoch: bool = True, # evaluation and monitoring @@ -334,7 +334,7 @@ def train_custom( # amp use_amp: bool = False, # plugins - plugins: Optional[List[TrainerPlugin]] = None, + plugins: Optional[list[TrainerPlugin]] = None, **kwargs, ) -> dict: """Trains any class that implements the flair.nn.Model interface. @@ -475,7 +475,7 @@ def train_custom( # initialize sampler if provided if sampler is not None: # init with default values if only class is provided - if inspect.isclass(sampler): + if isinstance(sampler, type): sampler = sampler() # set dataset to sample from sampler.set_dataset(train_data) diff --git a/flair/training_utils.py b/flair/training_utils.py index 0b4ef91cbf..9b38ec1ddb 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -5,7 +5,7 @@ from functools import reduce from math import inf from pathlib import Path -from typing import Dict, List, Literal, Optional, Union +from typing import Literal, Optional, Union from scipy.stats import pearsonr, spearmanr from sklearn.metrics import mean_absolute_error, mean_squared_error @@ -25,7 +25,7 @@ def __init__( main_score: float, detailed_results: str, classification_report: Optional[dict] = None, - scores: Optional[Dict] = None, + scores: Optional[dict] = None, ) -> None: classification_report = classification_report if classification_report is not None else {} assert scores is not None and "loss" in scores, "No loss provided." @@ -47,8 +47,8 @@ class MetricRegression: def __init__(self, name) -> None: self.name = name - self.true: List[float] = [] - self.pred: List[float] = [] + self.true: list[float] = [] + self.pred: list[float] = [] def mean_squared_error(self): return mean_squared_error(self.true, self.pred) @@ -98,7 +98,7 @@ def __init__(self, directory: Union[str, Path], number_of_weights: int = 10) -> if isinstance(directory, str): directory = Path(directory) self.weights_file = init_output_file(directory, "weights.txt") - self.weights_dict: Dict[str, Dict[int, List[float]]] = defaultdict(lambda: defaultdict(list)) + self.weights_dict: dict[str, dict[int, list[float]]] = defaultdict(lambda: defaultdict(list)) self.number_of_weights = number_of_weights def extract_weights(self, state_dict, iteration): @@ -338,7 +338,7 @@ def init_output_file(base_path: Union[str, Path], file_name: str) -> Path: return file -def convert_labels_to_one_hot(label_list: List[List[str]], label_dict: Dictionary) -> List[List[int]]: +def convert_labels_to_one_hot(label_list: list[list[str]], label_dict: Dictionary) -> list[list[int]]: """Convert list of labels to a one hot list. Args: @@ -365,9 +365,9 @@ def add_file_handler(log, output_file): def store_embeddings( - data_points: Union[List[DT], Dataset], + data_points: Union[list[DT], Dataset], storage_mode: EmbeddingStorageMode, - dynamic_embeddings: Optional[List[str]] = None, + dynamic_embeddings: Optional[list[str]] = None, ): if isinstance(data_points, Dataset): data_points = list(_iter_dataset(data_points)) @@ -391,7 +391,7 @@ def store_embeddings( data_point.to("cpu", pin_memory=pin_memory) -def identify_dynamic_embeddings(data_points: List[DT]) -> Optional[List]: +def identify_dynamic_embeddings(data_points: list[DT]) -> Optional[list[str]]: dynamic_embeddings = [] all_embeddings = [] for data_point in data_points: diff --git a/flair/visual/ner_html.py b/flair/visual/ner_html.py index c71e108379..5b691a9e60 100644 --- a/flair/visual/ner_html.py +++ b/flair/visual/ner_html.py @@ -1,5 +1,5 @@ import html -from typing import List, Union +from typing import Union from flair.data import Sentence @@ -41,7 +41,7 @@ def split_to_spans(s: Sentence, label_name="ner"): def render_ner_html( - sentences: Union[List[Sentence], Sentence], + sentences: Union[list[Sentence], Sentence], title: str = "Flair", colors={ "PER": "#F7FF53", diff --git a/flair/visual/training_curves.py b/flair/visual/training_curves.py index 1fd856b669..32947c3348 100644 --- a/flair/visual/training_curves.py +++ b/flair/visual/training_curves.py @@ -3,7 +3,7 @@ import math from collections import defaultdict from pathlib import Path -from typing import Dict, List, Union +from typing import Union import matplotlib.pyplot as plt import numpy as np @@ -27,7 +27,7 @@ class Plotter: def _extract_evaluation_data(file_name: Union[str, Path], score: str = "F1") -> dict: file_name = Path(file_name) - training_curves: Dict[str, Dict[str, List[float]]] = { + training_curves: dict[str, dict[str, list[float]]] = { "train": {"loss": [], "score": []}, "test": {"loss": [], "score": []}, "dev": {"loss": [], "score": []}, @@ -70,7 +70,7 @@ def _extract_weight_data(file_name: Union[str, Path]) -> dict: if isinstance(file_name, str): file_name = Path(file_name) - weights: Dict[str, Dict[str, List[float]]] = defaultdict(lambda: defaultdict(list)) + weights: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) with open(file_name) as f: tsvin = csv.reader(f, delimiter="\t") @@ -151,7 +151,7 @@ def plot_weights(self, file_name: Union[str, Path]): log.info(f"Weights plots are saved in {path}") # to let user know the path of the save plots plt.close(fig) - def plot_training_curves(self, file_name: Union[str, Path], plot_values: List[str] = ["loss", "F1"]): + def plot_training_curves(self, file_name: Union[str, Path], plot_values: list[str] = ["loss", "F1"]): file_name = Path(file_name) fig = plt.figure(figsize=(15, 10)) diff --git a/flair/visual/tree_printer.py b/flair/visual/tree_printer.py index fc461d9f81..9753a37a09 100644 --- a/flair/visual/tree_printer.py +++ b/flair/visual/tree_printer.py @@ -1,5 +1,3 @@ -from typing import List - from pptree import print_tree from flair.data import Sentence, Token @@ -9,7 +7,7 @@ class NodeToken: def __init__(self, token: Token, tag_type: str) -> None: self.token: Token = token self.tag_type: str = tag_type - self.children: List[NodeToken] = [] + self.children: list[NodeToken] = [] def set_haed(self, parent): parent.children.append(self) @@ -19,7 +17,7 @@ def __str__(self) -> str: def tree_printer(sentence: Sentence, tag_type: str): - tree: List[NodeToken] = [NodeToken(token, tag_type) for token in sentence] + tree: list[NodeToken] = [NodeToken(token, tag_type) for token in sentence] for x in tree: if x.token.head_id != 0: head_token = x.token.get_head() diff --git a/pyproject.toml b/pyproject.toml index 78d1692a09..9f4c5a7535 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 120 -target-version = ['py37'] +target-version = ['py39'] exclude = ''' ( /( @@ -49,7 +49,7 @@ ignore_errors = true [tool.ruff] line-length = 120 -target-version = "py38" +target-version = "py39" [tool.ruff.lint] #select = ["ALL"] # Uncommit to autofix all the things diff --git a/requirements-dev.txt b/requirements-dev.txt index 61d45acf8c..3b8fbde79c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ pytest-black-ng==0.4.* pytest-github-actions-annotate-failures>=0.1.8 pytest-mypy>=0.10.3 pytest-ruff==0.3.* -ruff==0.3.* +ruff==0.7.* types-dataclasses>=0.6.6 types-Deprecated>=1.2.9.2 types-requests>=2.28.11.17 diff --git a/resources/docs/EXPERIMENTS.md b/resources/docs/EXPERIMENTS.md index 69f1a5fbf2..c6bbe72a1c 100644 --- a/resources/docs/EXPERIMENTS.md +++ b/resources/docs/EXPERIMENTS.md @@ -55,7 +55,7 @@ tag_type = 'ner' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) # initialize embeddings -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ # GloVe embeddings WordEmbeddings('glove'), @@ -124,7 +124,7 @@ tag_type = 'ner' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) # initialize embeddings -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ WordEmbeddings('de'), PooledFlairEmbeddings('german-forward'), PooledFlairEmbeddings('german-backward'), @@ -225,7 +225,7 @@ tag_type = 'ner' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) # initialize embeddings -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ WordEmbeddings('crawl'), WordEmbeddings('twitter'), FlairEmbeddings('news-forward'), @@ -292,7 +292,7 @@ tag_type = 'ner' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) # initialize embeddings -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ WordEmbeddings('crawl'), FlairEmbeddings('news-forward'), FlairEmbeddings('news-backward'), @@ -361,7 +361,7 @@ tag_type = 'pos' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) # initialize embeddings -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ WordEmbeddings('extvec'), FlairEmbeddings('news-forward'), FlairEmbeddings('news-backward'), @@ -416,7 +416,7 @@ tag_type = 'np' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) # initialize embeddings -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ WordEmbeddings('extvec'), FlairEmbeddings('news-forward'), FlairEmbeddings('news-backward'), diff --git a/resources/docs/HUNFLAIR2.md b/resources/docs/HUNFLAIR2.md index 6f2c1474b3..032b4fe075 100644 --- a/resources/docs/HUNFLAIR2.md +++ b/resources/docs/HUNFLAIR2.md @@ -14,7 +14,7 @@ NER tools on unseen corpora. ## Quick Start #### Requirements and Installation -*HunFlair2* is based on Flair 0.14+ and Python 3.8+. If you do not have Python 3.8, install it first. +*HunFlair2* is based on Flair 0.14+ and Python 3.9+. If you do not have Python 3.9, install it first. Then, in your favorite virtual environment, simply do: ``` pip install flair diff --git a/resources/docs/KOR_docs/TUTORIAL_7_TRAINING_A_MODEL.md b/resources/docs/KOR_docs/TUTORIAL_7_TRAINING_A_MODEL.md index cd839acc15..382600d5be 100644 --- a/resources/docs/KOR_docs/TUTORIAL_7_TRAINING_A_MODEL.md +++ b/resources/docs/KOR_docs/TUTORIAL_7_TRAINING_A_MODEL.md @@ -313,7 +313,7 @@ label_type = 'ner' # 3. 말뭉치에서 레이블 사전 만들기 label_dict = corpus.make_label_dictionary(label_type=label_type, add_unk=False) # 4. 임베딩 초기화하기 -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ WordEmbeddings('glove') ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) diff --git a/resources/docs/KOR_docs/TUTORIAL_8_MODEL_OPTIMIZATION.md b/resources/docs/KOR_docs/TUTORIAL_8_MODEL_OPTIMIZATION.md index e821dc2d17..ec3affe947 100644 --- a/resources/docs/KOR_docs/TUTORIAL_8_MODEL_OPTIMIZATION.md +++ b/resources/docs/KOR_docs/TUTORIAL_8_MODEL_OPTIMIZATION.md @@ -95,7 +95,7 @@ tag_type = 'ner' tag_dictionary = corpus.make_label_dictionary(label_type=tag_type, add_unk=False) print(tag_dictionary.idx2item) # 4. 임베딩 초기화하기 -embedding_types: List[TokenEmbeddings] = [ +embedding_types: list[TokenEmbeddings] = [ WordEmbeddings('glove'), ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) diff --git a/setup.py b/setup.py index 3c1cc06018..0573896c19 100644 --- a/setup.py +++ b/setup.py @@ -20,5 +20,5 @@ "word-embeddings": ["gensim>=4.2.0", "bpemb>=0.3.5"], }, include_package_data=True, - python_requires=">=3.8", + python_requires=">=3.9", ) diff --git a/tests/embedding_test_utils.py b/tests/embedding_test_utils.py index 554ef32777..c1a0b1a791 100644 --- a/tests/embedding_test_utils.py +++ b/tests/embedding_test_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Type +from typing import Any, Optional import pytest import torch @@ -9,15 +9,15 @@ class BaseEmbeddingsTest: - embedding_cls: Type[Embeddings[Sentence]] + embedding_cls: type[Embeddings[Sentence]] is_token_embedding: bool is_document_embedding: bool - default_args: Dict[str, Any] - valid_args: List[Dict[str, Any]] = [] - invalid_args: List[Dict[str, Any]] = [] - invalid_names: List[str] = [] + default_args: dict[str, Any] + valid_args: list[dict[str, Any]] = [] + invalid_args: list[dict[str, Any]] = [] + invalid_names: list[str] = [] name_field: Optional[str] = None - weired_texts: List[str] = [ + weired_texts: list[str] = [ "Hybrid mesons , qq ̄ states with an admixture", "typical proportionalities of \u223C 1nmV \u2212 1 [ 3,4 ] .", "🤟 🤟 🤟 hüllo", @@ -33,7 +33,7 @@ def create_embedding_from_name(self, name: str): kwargs.pop(self.name_field) return self.embedding_cls(name, **kwargs) # type: ignore[call-arg] - def create_embedding_with_args(self, args: Dict[str, Any]): + def create_embedding_with_args(self, args: dict[str, Any]): kwargs = dict(self.default_args) for k, v in args.items(): kwargs[k] = v diff --git a/tests/embeddings/test_document_transform_word_embeddings.py b/tests/embeddings/test_document_transform_word_embeddings.py index 6a06372723..73567ffbeb 100644 --- a/tests/embeddings/test_document_transform_word_embeddings.py +++ b/tests/embeddings/test_document_transform_word_embeddings.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any from flair.embeddings import ( DocumentCNNEmbeddings, @@ -19,7 +19,7 @@ class BaseDocumentsViaWordEmbeddingsTest(BaseEmbeddingsTest): is_document_embedding = True is_token_embedding = False - base_embeddings: List[TokenEmbeddings] = [word, flair_embedding] + base_embeddings: list[TokenEmbeddings] = [word, flair_embedding] def create_embedding_from_name(self, name: str): """Overwrite this method if it is more complex to load an embedding by name.""" @@ -28,7 +28,7 @@ def create_embedding_from_name(self, name: str): kwargs.pop(self.name_field) return self.embedding_cls(name, **kwargs) # type: ignore[call-arg] - def create_embedding_with_args(self, args: Dict[str, Any]): + def create_embedding_with_args(self, args: dict[str, Any]): kwargs = dict(self.default_args) for k, v in args.items(): kwargs[k] = v @@ -63,4 +63,4 @@ class TestDocumentCNNEmbeddings(BaseDocumentsViaWordEmbeddingsTest): class TestDocumentLMEmbeddings(BaseDocumentsViaWordEmbeddingsTest): embedding_cls = DocumentLMEmbeddings base_embeddings = [flair_embedding, flair_embedding_back] - default_args: Dict[str, Any] = {} + default_args: dict[str, Any] = {} diff --git a/tests/embeddings/test_word_embeddings.py b/tests/embeddings/test_word_embeddings.py index 34d0b3b9f7..87f56fec4f 100644 --- a/tests/embeddings/test_word_embeddings.py +++ b/tests/embeddings/test_word_embeddings.py @@ -1,4 +1,4 @@ -from typing import Any, Dict +from typing import Any from flair.embeddings import MuseCrosslingualEmbeddings, NILCEmbeddings, WordEmbeddings from tests.embedding_test_utils import BaseEmbeddingsTest @@ -18,7 +18,7 @@ class TestMuseCrosslingualEmbeddings(BaseEmbeddingsTest): embedding_cls = MuseCrosslingualEmbeddings is_token_embedding = True is_document_embedding = False - default_args: Dict[str, Any] = {} + default_args: dict[str, Any] = {} class TestNILCEmbeddings(BaseEmbeddingsTest): diff --git a/tests/model_test_utils.py b/tests/model_test_utils.py index 10aab0831f..b5afd81bfe 100644 --- a/tests/model_test_utils.py +++ b/tests/model_test_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Type +from typing import Any, Optional import pytest @@ -11,13 +11,13 @@ class BaseModelTest: - model_cls: Type[Model] + model_cls: type[Model] pretrained_model: Optional[str] = None empty_sentence = Sentence(" ") train_label_type: str - multiclass_prediction_labels: List[str] - model_args: Dict[str, Any] = {} - training_args: Dict[str, Any] = {} + multiclass_prediction_labels: list[str] + model_args: dict[str, Any] = {} + training_args: dict[str, Any] = {} finetune_instead_of_train: bool = False @pytest.fixture() diff --git a/tests/models/test_relation_classifier.py b/tests/models/test_relation_classifier.py index c0ca34bce5..da4de52bfc 100644 --- a/tests/models/test_relation_classifier.py +++ b/tests/models/test_relation_classifier.py @@ -1,5 +1,5 @@ from operator import itemgetter -from typing import Dict, List, Optional, Set, Tuple +from typing import Optional import pytest from torch.utils.data import Dataset @@ -20,7 +20,7 @@ ) from tests.model_test_utils import BaseModelTest -encoding_strategies: Dict[EncodingStrategy, List[Tuple[str, str]]] = { +encoding_strategies: dict[EncodingStrategy, list[tuple[str, str]]] = { EntityMask(): [("[HEAD]", "[TAIL]") for _ in range(7)], TypedEntityMask(): [ ("[HEAD-ORG]", "[TAIL-PER]"), @@ -140,7 +140,7 @@ def train_test_sentence(self): return sentence def assert_training_example(self, predicted_training_example): - relations: List[Relation] = predicted_training_example.get_relations("relation") + relations: list[Relation] = predicted_training_example.get_relations("relation") assert len(relations) == 2 # Intel ----founded_by---> Gordon Moore @@ -164,7 +164,7 @@ def assert_training_example(self, predicted_training_example): @staticmethod def check_transformation_correctness( split: Optional[Dataset], - ground_truth: Set[Tuple[str, Tuple[str, ...]]], + ground_truth: set[tuple[str, tuple[str, ...]]], ) -> None: # Ground truth is a set of tuples of (, ) assert split is not None @@ -190,7 +190,7 @@ def test_transform_corpus( embeddings: TransformerDocumentEmbeddings, cross_augmentation: bool, encoding_strategy: EncodingStrategy, - encoded_entity_pairs: List[Tuple[str, str]], + encoded_entity_pairs: list[tuple[str, str]], ) -> None: label_dictionary = corpus.make_label_dictionary("relation") model: RelationClassifier = self.build_model( @@ -200,7 +200,7 @@ def test_transform_corpus( # Check sentence masking and relation label annotation on # training, validation and test dataset (in this test the splits are the same) - ground_truth: Set[Tuple[str, Tuple[str, ...]]] = { + ground_truth: set[tuple[str, tuple[str, ...]]] = { # Entity pair permutations of: "Larry Page and Sergey Brin founded Google ." (f"{encoded_entity_pairs[0][1]} and Sergey Brin founded {encoded_entity_pairs[0][0]} .", ("founded_by",)), (f"Larry Page and {encoded_entity_pairs[1][1]} founded {encoded_entity_pairs[1][0]} .", ("founded_by",)), diff --git a/tests/test_datasets_biomedical.py b/tests/test_datasets_biomedical.py index 0264b08394..c15674eb6b 100644 --- a/tests/test_datasets_biomedical.py +++ b/tests/test_datasets_biomedical.py @@ -2,7 +2,7 @@ import os import tempfile from pathlib import Path -from typing import List, Optional +from typing import Optional from flair.datasets.biomedical import ( CoNLLWriter, @@ -84,7 +84,7 @@ def test_conll_writer_one_token_multiple_entities2(): def assert_conll_writer_output( dataset: InternalBioNerDataset, - expected_output: List[str], + expected_output: list[str], sentence_splitter: Optional[SentenceSplitter] = None, ): fd, outfile_path = tempfile.mkstemp() diff --git a/tests/test_labels.py b/tests/test_labels.py index 210a215889..099484162c 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -1,5 +1,3 @@ -from typing import List - from flair.data import Label, Relation, Sentence, Span @@ -14,7 +12,7 @@ def test_token_tags(): sentence[0].add_label("pos", "pronoun") # check if there are three POS labels with correct text and values - labels: List[Label] = sentence.get_labels("pos") + labels: list[Label] = sentence.get_labels("pos") assert len(labels) == 3 assert labels[0].data_point.text == "I" assert labels[0].value == "pronoun" @@ -24,7 +22,7 @@ def test_token_tags(): assert labels[2].value == "proper noun" # check if there are is one SENTIMENT label with correct text and values - labels: List[Label] = sentence.get_labels("sentiment") + labels: list[Label] = sentence.get_labels("sentiment") assert len(labels) == 1 assert labels[0].data_point.text == "love" assert labels[0].value == "positive" @@ -45,7 +43,7 @@ def test_token_tags(): # remove the pos label from the last word sentence[2].remove_labels("pos") # there should be 2 POS labels left - labels: List[Label] = sentence.get_labels("pos") + labels: list[Label] = sentence.get_labels("pos") assert len(labels) == 2 assert len(sentence[0].get_labels("pos")) == 1 assert len(sentence[1].get_labels("pos")) == 1 @@ -72,7 +70,7 @@ def test_span_tags(): sentence[7:8].add_label("ner", "City") # check if there are three labels with correct text and values - labels: List[Label] = sentence.get_labels("ner") + labels: list[Label] = sentence.get_labels("ner") assert len(labels) == 3 assert labels[0].data_point.text == "Humboldt Universität zu Berlin" assert labels[0].value == "Organization" @@ -82,7 +80,7 @@ def test_span_tags(): assert labels[2].value == "City" # check if there are two spans with correct text and values - spans: List[Span] = sentence.get_spans("ner") + spans: list[Span] = sentence.get_spans("ner") assert len(spans) == 2 assert spans[0].text == "Humboldt Universität zu Berlin" assert len(spans[0].get_labels("ner")) == 2 @@ -92,12 +90,12 @@ def test_span_tags(): # now delete the NER tags of "Humboldt-Universität zu Berlin" sentence[0:4].remove_labels("ner") # should be only one NER label left - labels: List[Label] = sentence.get_labels("ner") + labels: list[Label] = sentence.get_labels("ner") assert len(labels) == 1 assert labels[0].data_point.text == "Berlin" assert labels[0].value == "City" # and only one NER span - spans: List[Span] = sentence.get_spans("ner") + spans: list[Span] = sentence.get_spans("ner") assert len(spans) == 1 assert spans[0].text == "Berlin" assert spans[0].get_label("ner").value == "City" @@ -111,7 +109,7 @@ def test_different_span_tags(): sentence[7:8].add_label("ner", "City") # check if there are three labels with correct text and values - labels: List[Label] = sentence.get_labels("ner") + labels: list[Label] = sentence.get_labels("ner") assert len(labels) == 2 assert labels[0].data_point.text == "Humboldt Universität zu Berlin" assert labels[0].value == "Organization" @@ -119,7 +117,7 @@ def test_different_span_tags(): assert labels[1].value == "City" # check if there are two spans with correct text and values - spans: List[Span] = sentence.get_spans("ner") + spans: list[Span] = sentence.get_spans("ner") assert len(spans) == 2 assert spans[0].text == "Humboldt Universität zu Berlin" assert spans[0].get_label("ner").value == "Organization" @@ -131,22 +129,22 @@ def test_different_span_tags(): # now delete the NER tags of "Humboldt-Universität zu Berlin" sentence[0:4].remove_labels("ner") # should be only one NER label left - labels: List[Label] = sentence.get_labels("ner") + labels: list[Label] = sentence.get_labels("ner") assert len(labels) == 1 assert labels[0].data_point.text == "Berlin" assert labels[0].value == "City" # and only one NER span - spans: List[Span] = sentence.get_spans("ner") + spans: list[Span] = sentence.get_spans("ner") assert len(spans) == 1 assert spans[0].text == "Berlin" assert spans[0].get_label("ner").value == "City" # but there is also one orgtype span and label - labels: List[Label] = sentence.get_labels("orgtype") + labels: list[Label] = sentence.get_labels("orgtype") assert len(labels) == 1 assert labels[0].data_point.text == "Humboldt Universität zu Berlin" assert labels[0].value == "University" # and only one NER span - spans: List[Span] = sentence.get_spans("orgtype") + spans: list[Span] = sentence.get_spans("orgtype") assert len(spans) == 1 assert spans[0].text == "Humboldt Universität zu Berlin" assert spans[0].get_label("orgtype").value == "University" @@ -154,7 +152,7 @@ def test_different_span_tags(): # let's add the NER tag back sentence[0:4].add_label("ner", "Organization") # check if there are three labels with correct text and values - labels: List[Label] = sentence.get_labels("ner") + labels: list[Label] = sentence.get_labels("ner") print(labels) assert len(labels) == 2 assert labels[0].data_point.text == "Humboldt Universität zu Berlin" @@ -163,7 +161,7 @@ def test_different_span_tags(): assert labels[1].value == "City" # check if there are two spans with correct text and values - spans: List[Span] = sentence.get_spans("ner") + spans: list[Span] = sentence.get_spans("ner") assert len(spans) == 2 assert spans[0].text == "Humboldt Universität zu Berlin" assert spans[0].get_label("ner").value == "Organization" @@ -194,17 +192,17 @@ def test_relation_tags(): Relation(sentence[0:2], sentence[3:4]).add_label("syntactic", "apposition") # there should be two relation labels - labels: List[Label] = sentence.get_labels("rel") + labels: list[Label] = sentence.get_labels("rel") assert len(labels) == 2 assert labels[0].value == "located in" assert labels[1].value == "university of" # there should be one syntactic labels - labels: List[Label] = sentence.get_labels("syntactic") + labels: list[Label] = sentence.get_labels("syntactic") assert len(labels) == 1 # there should be two relations, one with two and one with one label - relations: List[Relation] = sentence.get_relations("rel") + relations: list[Relation] = sentence.get_relations("rel") assert len(relations) == 2 assert len(relations[0].labels) == 1 assert len(relations[1].labels) == 2 diff --git a/tests/test_tokenize_sentence.py b/tests/test_tokenize_sentence.py index fd049b642e..7fd03ac6ba 100644 --- a/tests/test_tokenize_sentence.py +++ b/tests/test_tokenize_sentence.py @@ -1,5 +1,3 @@ -from typing import List - import pytest import flair @@ -492,5 +490,5 @@ def test_line_separator_is_ignored(): assert Sentence(with_separator).to_original_text() == Sentence(without_separator).to_original_text() -def no_op_tokenizer(text: str) -> List[str]: +def no_op_tokenizer(text: str) -> list[str]: return [text] From a30fd78bfbcdd2eb8d203629dd7b827e700bdbd6 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 25 Oct 2024 15:44:04 +0200 Subject: [PATCH 011/333] fix unrequred type ignore statements --- flair/embeddings/document.py | 2 +- flair/embeddings/token.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flair/embeddings/document.py b/flair/embeddings/document.py index 28867d889a..8f66a198ed 100644 --- a/flair/embeddings/document.py +++ b/flair/embeddings/document.py @@ -371,7 +371,7 @@ def _add_embeddings_internal(self, sentences: list[Sentence]): sentence_tensor = self.word_reprojection_map(sentence_tensor) # push through RNN - packed = pack_padded_sequence(sentence_tensor, lengths, enforce_sorted=False, batch_first=True) # type: ignore[arg-type] + packed = pack_padded_sequence(sentence_tensor, lengths, enforce_sorted=False, batch_first=True) rnn_out, hidden = self.rnn(packed) outputs, output_lengths = pad_packed_sequence(rnn_out, batch_first=True) diff --git a/flair/embeddings/token.py b/flair/embeddings/token.py index 700eaf4c45..3d95c8ee0b 100644 --- a/flair/embeddings/token.py +++ b/flair/embeddings/token.py @@ -513,7 +513,7 @@ def _add_embeddings_internal(self, sentences: list[Sentence]): character_embeddings = self.char_embedding(chars).transpose(0, 1) - packed = torch.nn.utils.rnn.pack_padded_sequence(character_embeddings, chars2_length) # type: ignore[arg-type] + packed = torch.nn.utils.rnn.pack_padded_sequence(character_embeddings, chars2_length) lstm_out, self.hidden = self.char_rnn(packed) From 783ebc7c5db43f2d0e3ca0d6473556da34a4e70e Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 25 Oct 2024 19:00:09 +0200 Subject: [PATCH 012/333] fix transformers switching attention implementation --- flair/embeddings/transformer.py | 5 +++++ pyproject.toml | 1 + 2 files changed, 6 insertions(+) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index d09ed33699..245d528e5d 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -1353,6 +1353,11 @@ def from_params(cls, params): def to_params(self): config_dict = self.model.config.to_dict() + + # do not switch the attention implementation upon reload. + config_dict["attn_implementation"] = self.model.config._attn_implementation + del config_dict["_attn_implementation_autoset"] + super_params = super().to_params() # those parameters are only from the super class and will be recreated in the constructor. diff --git a/pyproject.toml b/pyproject.toml index 9f4c5a7535..9711794abb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ filterwarnings = [ 'ignore:`resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.', # transformers calls deprecated hf_hub "ignore:`torch.cuda.amp.GradScaler", # GradScaler changes in torch 2.3.0 but we want to be backwards compatible. "ignore:`clean_up_tokenization_spaces` was not set", # Default behavior changes in transformers v4.45, raising irrelevant FutureWarning for serialized models. + "ignore:1Torch was not compiled with flash attention", # You might want to install flash attention, but you don't have to. ] markers = [ "integration", From d932baf83cad1081d8eb8f2ee9715d947a6c466a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Karlo=20Do=C5=A1ilovi=C4=87?= Date: Fri, 25 Oct 2024 23:05:48 +0200 Subject: [PATCH 013/333] Fix formatting errors. --- flair/embeddings/transformer.py | 11 ++++++----- .../test_transformer_document_embeddings.py | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index 8a1e76cd72..8ba17b1fec 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -8,18 +8,18 @@ from abc import abstractmethod from io import BytesIO from pathlib import Path -from typing import Any, cast, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union, cast import torch import transformers from packaging.version import Version from torch.jit import ScriptModule from transformers import ( + CONFIG_MAPPING, AutoConfig, AutoFeatureExtractor, AutoModel, AutoTokenizer, - CONFIG_MAPPING, FeatureExtractionMixin, LayoutLMTokenizer, LayoutLMTokenizerFast, @@ -32,8 +32,8 @@ from transformers.utils import PaddingStrategy import flair -from flair.data import log, Sentence, Token -from flair.embeddings.base import DocumentEmbeddings, Embeddings, register_embeddings, TokenEmbeddings +from flair.data import Sentence, Token, log +from flair.embeddings.base import DocumentEmbeddings, Embeddings, TokenEmbeddings, register_embeddings SENTENCE_BOUNDARY_TAG: str = "[FLERT]" @@ -191,6 +191,7 @@ def fill_mean_token_embeddings( return all_token_embeddings + @torch.jit.script_if_tracing def document_cls_pooling(sentence_hidden_states: torch.Tensor, sentence_lengths: torch.Tensor) -> torch.Tensor: return sentence_hidden_states[torch.arange(sentence_hidden_states.shape[0]), sentence_lengths - 1] @@ -1130,7 +1131,7 @@ def is_supported_t5_model(config: PretrainedConfig) -> bool: if peft_config is not None: # add adapters for finetuning try: - from peft import get_peft_model, prepare_model_for_kbit_training, TaskType + from peft import TaskType, get_peft_model, prepare_model_for_kbit_training except ImportError: log.error("You cannot use the PEFT finetuning without peft being installed") raise diff --git a/tests/embeddings/test_transformer_document_embeddings.py b/tests/embeddings/test_transformer_document_embeddings.py index bf202110b1..7402b2b467 100644 --- a/tests/embeddings/test_transformer_document_embeddings.py +++ b/tests/embeddings/test_transformer_document_embeddings.py @@ -4,7 +4,6 @@ from flair.embeddings import TransformerDocumentEmbeddings from flair.models import TextClassifier from flair.nn import Classifier - from tests.embedding_test_utils import BaseEmbeddingsTest From 3c3620061c4a3578a536bf9c25ea78c745e2f19b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Karlo=20Do=C5=A1ilovi=C4=87?= Date: Tue, 29 Oct 2024 21:01:30 +0100 Subject: [PATCH 014/333] Use torch.allclose for comparing tensors in BaseEmbeddingsTest. --- tests/embedding_test_utils.py | 28 +++++++++++++++---- .../test_transformer_document_embeddings.py | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/embedding_test_utils.py b/tests/embedding_test_utils.py index 554ef32777..933dafad73 100644 --- a/tests/embedding_test_utils.py +++ b/tests/embedding_test_utils.py @@ -19,11 +19,11 @@ class BaseEmbeddingsTest: name_field: Optional[str] = None weired_texts: List[str] = [ "Hybrid mesons , qq ̄ states with an admixture", - "typical proportionalities of \u223C 1nmV \u2212 1 [ 3,4 ] .", + "typical proportionalities of \u223c 1nmV \u2212 1 [ 3,4 ] .", "🤟 🤟 🤟 hüllo", "🤟hallo 🤟 🤟 🤟 🤟", "🤟", - "\uF8F9", + "\uf8f9", ] def create_embedding_from_name(self, name: str): @@ -150,9 +150,17 @@ def test_embeddings_stay_the_same_after_saving_and_loading(self, args): if self.is_token_embedding: for token_old, token_new in zip(sentence_old, sentence_new): - assert (token_old.get_embedding(names_old) == token_new.get_embedding(names_new)).all() + assert torch.allclose( + token_old.get_embedding(names_old), + token_new.get_embedding(names_new), + atol=1e-6, + ) if self.is_document_embedding: - assert (sentence_old.get_embedding(names_old) == sentence_new.get_embedding(names_new)).all() + assert torch.allclose( + sentence_old.get_embedding(names_old), + sentence_new.get_embedding(names_new), + atol=1e-6, + ) def test_default_embeddings_stay_the_same_after_saving_and_loading(self): embeddings = self.create_embedding_with_args(self.default_args) @@ -176,9 +184,17 @@ def test_default_embeddings_stay_the_same_after_saving_and_loading(self): if self.is_token_embedding: for token_old, token_new in zip(sentence_old, sentence_new): - assert (token_old.get_embedding(names_old) == token_new.get_embedding(names_new)).all() + assert torch.allclose( + token_old.get_embedding(names_old), + token_new.get_embedding(names_new), + atol=1e-6, + ) if self.is_document_embedding: - assert (sentence_old.get_embedding(names_old) == sentence_new.get_embedding(names_new)).all() + assert torch.allclose( + sentence_old.get_embedding(names_old), + sentence_new.get_embedding(names_new), + atol=1e-6, + ) def test_embeddings_load_in_eval_mode(self): embeddings = self.create_embedding_with_args(self.default_args) diff --git a/tests/embeddings/test_transformer_document_embeddings.py b/tests/embeddings/test_transformer_document_embeddings.py index 7402b2b467..f0f6389b7d 100644 --- a/tests/embeddings/test_transformer_document_embeddings.py +++ b/tests/embeddings/test_transformer_document_embeddings.py @@ -44,7 +44,7 @@ def test_if_loaded_embeddings_have_all_attributes(tasks_base_path): @pytest.mark.parametrize("cls_pooling", ["cls", "mean", "max"]) def test_cls_pooling(cls_pooling): embeddings = TransformerDocumentEmbeddings( - model="xlm-roberta-base", + model="distilbert-base-uncased", layers="-1", cls_pooling=cls_pooling, allow_long_sentences=True, From e71797c4f77f71519de72f304a6615c712b39f9c Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Mon, 23 Sep 2024 22:54:48 -0600 Subject: [PATCH 015/333] GH-3429: Add experimental multi GPU support --- flair/__init__.py | 6 + flair/distributed_utils.py | 49 +++++++++ flair/nn/model.py | 3 + flair/trainers/trainer.py | 220 ++++++++++++++++++++----------------- 4 files changed, 176 insertions(+), 102 deletions(-) create mode 100644 flair/distributed_utils.py diff --git a/flair/__init__.py b/flair/__init__.py index 341f630e43..55bd6d00cf 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -33,6 +33,12 @@ else: device = torch.device("cpu") +distributed = False +"""Experimental flag to indicate multiple GPUs are in use. + +Set by `launch_distributed` -- do not set manually. +""" + # global variable: version __version__ = "0.14.0" """The current version of the flair library installed.""" diff --git a/flair/distributed_utils.py b/flair/distributed_utils.py new file mode 100644 index 0000000000..596570b7ce --- /dev/null +++ b/flair/distributed_utils.py @@ -0,0 +1,49 @@ +import logging +import os + +import torch +import torch.multiprocessing as mp +from torch.distributed import destroy_process_group, init_process_group + +import flair + +log = logging.getLogger("flair") + + +def launch_distributed(fp, *args): + """Executes the function fp(*args) on multiple GPUs (all local GPUs)""" + world_size = torch.cuda.device_count() + log.info(f"Launching {world_size} distributed processes") + mp.spawn(entrypoint, args=(world_size, fp, *args), nprocs=world_size) + + +def entrypoint(rank, world_size, fp, *args): + ddp_setup(rank, world_size) + fp(*args) + destroy_process_group() + + +def ddp_setup(rank: int, world_size: int) -> None: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12355" + init_process_group(backend="nccl", rank=rank, world_size=world_size) + flair.distributed = True + flair.device = torch.device(rank) + torch.cuda.set_device(flair.device) + + +def is_main_process() -> bool: + """True for exactly 1 process, regardless of whether being run on CPU/single-GPU/multi-gpu""" + if flair.distributed: + return flair.device.index == 0 + else: + return True + + +class DistributedModel(torch.nn.parallel.DistributedDataParallel): + """DistributedDataParallel, but redirects access to methods and attributes to the original Model""" + def __getattr__(self, name): + try: + return super().__getattr__(name) + except AttributeError: + return getattr(self.module, name) diff --git a/flair/nn/model.py b/flair/nn/model.py index bf13baf2f1..1710e32c76 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -17,6 +17,7 @@ from flair.class_utils import get_non_abstract_subclasses from flair.data import DT, DT2, Corpus, Dictionary, Sentence, _iter_dataset from flair.datasets import DataLoader, FlairDatapointDataset +from flair.distributed_utils import is_main_process from flair.embeddings import Embeddings from flair.embeddings.base import load_embeddings from flair.file_utils import Tqdm, load_torch_state @@ -118,6 +119,8 @@ def save(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: model_file: the model file checkpoint: currently unused. """ + if not is_main_process(): + return model_state = self._get_state_dict() # write out a "model card" if one is set diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 2f32b54c01..05c376ec20 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -11,12 +11,14 @@ import torch from torch.optim.sgd import SGD +from torch.utils.data import DistributedSampler from torch.utils.data.dataset import ConcatDataset import flair import flair.nn from flair.data import Corpus, Dictionary, _len_dataset from flair.datasets import DataLoader +from flair.distributed_utils import DistributedModel, is_main_process from flair.samplers import FlairSampler from flair.trainers.plugins import ( AnnealingPlugin, @@ -420,6 +422,8 @@ def train_custom( base_path=base_path, ).attach_to(self) + if flair.distributed: + self.model = DistributedModel(self.model, device_ids=[flair.device.index]) # === END BLOCK: ACTIVATE PLUGINS === # # derive parameters the function was called with (or defaults) @@ -509,36 +513,37 @@ def train_custom( else "model from best epoch (best-model.pt)" ) - log_line(log) - log.info(f'Model: "{self.model}"') - log_line(log) - log.info(f"{self.corpus}") - log_line(log) - log.info(f"Train: {len(train_data)} sentences") - log.info(f" (train_with_dev={train_with_dev}, train_with_test={train_with_test})") - log_line(log) - log.info("Training Params:") - log.info( - f' - learning_rate: "{learning_rate}" ' - f'{"(decoder: " + str(decoder_learning_rate) + ")" if decoder_learning_rate else ""}' - ) - log.info(f' - mini_batch_size: "{mini_batch_size}"') - log.info(f' - max_epochs: "{max_epochs}"') - log.info(f' - shuffle: "{shuffle}"') - log_line(log) - log.info("Plugins:") - for plugin in plugins: - log.info(" - " + str(plugin)) - log_line(log) - log.info(f"Final evaluation on {final_eval_info}") - log.info(f' - metric: "{main_evaluation_metric}"') - log_line(log) - log.info("Computation:") - log.info(f" - compute on device: {flair.device}") - log.info(f" - embedding storage: {embeddings_storage_mode}") - log_line(log) - log.info(f'Model training base path: "{base_path}"') - log_line(log) + if is_main_process(): + log_line(log) + log.info(f'Model: "{self.model}"') + log_line(log) + log.info(f"{self.corpus}") + log_line(log) + log.info(f"Train: {len(train_data)} sentences") + log.info(f" (train_with_dev={train_with_dev}, train_with_test={train_with_test})") + log_line(log) + log.info("Training Params:") + log.info( + f' - learning_rate: "{learning_rate}" ' + f'{"(decoder: " + str(decoder_learning_rate) + ")" if decoder_learning_rate else ""}' + ) + log.info(f' - mini_batch_size: "{mini_batch_size}"') + log.info(f' - max_epochs: "{max_epochs}"') + log.info(f' - shuffle: "{shuffle}"') + log_line(log) + log.info("Plugins:") + for plugin in plugins: + log.info(" - " + str(plugin)) + log_line(log) + log.info(f"Final evaluation on {final_eval_info}") + log.info(f' - metric: "{main_evaluation_metric}"') + log_line(log) + log.info("Computation:") + log.info(f" - compute on device: {flair.device}") + log.info(f" - embedding storage: {embeddings_storage_mode}") + log_line(log) + log.info(f'Model training base path: "{base_path}"') + log_line(log) # At any point you can hit Ctrl + C to break out of training early. try: @@ -560,12 +565,21 @@ def train_custom( if not shuffle_first_epoch and epoch == 1: shuffle_data_this_epoch = False - batch_loader = DataLoader( - train_data, - batch_size=mini_batch_size, - shuffle=shuffle_data_this_epoch, - sampler=sampler, - ) + if flair.distributed: + batch_loader = DataLoader( + train_data, + batch_size=mini_batch_size, + shuffle=False, + sampler=DistributedSampler(train_data, shuffle=shuffle_data_this_epoch), + ) + batch_loader.sampler.set_epoch(epoch) + else: + batch_loader = DataLoader( + train_data, + batch_size=mini_batch_size, + shuffle=shuffle_data_this_epoch, + sampler=sampler, + ) self.model.train() @@ -682,49 +696,50 @@ def train_custom( # Determine if this is the best model or if we need to anneal current_epoch_has_best_model_so_far = False - validation_scores: tuple - - for evaluation_split, evaluation_split_data in evaluation_splits.items(): - eval_result = self.model.evaluate( - evaluation_split_data, - out_path=base_path / f"{evaluation_split}.tsv", - mini_batch_size=eval_batch_size, - exclude_labels=exclude_labels, - main_evaluation_metric=main_evaluation_metric, - gold_label_dictionary=gold_label_dictionary_for_eval, - embedding_storage_mode=embeddings_storage_mode, - gold_label_type=self.model.label_type, - gold_label_dictionary_for_eval=gold_label_dictionary_for_eval, - ) + validation_scores = () + + if is_main_process(): + for evaluation_split, evaluation_split_data in evaluation_splits.items(): + eval_result = self.model.evaluate( + evaluation_split_data, + out_path=base_path / f"{evaluation_split}.tsv", + mini_batch_size=eval_batch_size, + exclude_labels=exclude_labels, + main_evaluation_metric=main_evaluation_metric, + gold_label_dictionary=gold_label_dictionary_for_eval, + embedding_storage_mode=embeddings_storage_mode, + gold_label_type=self.model.label_type, + gold_label_dictionary_for_eval=gold_label_dictionary_for_eval, + ) - # log results - log.info( - f"{evaluation_split.upper()} : loss {eval_result.loss}" - f" - {main_evaluation_metric[1]}" - f" ({main_evaluation_metric[0]})" - f" {round(eval_result.main_score, 4)}" - ) + # log results + log.info( + f"{evaluation_split.upper()} : loss {eval_result.loss}" + f" - {main_evaluation_metric[1]}" + f" ({main_evaluation_metric[0]})" + f" {round(eval_result.main_score, 4)}" + ) - # depending on memory mode, embeddings are moved to CPU, GPU or deleted - store_embeddings(evaluation_split_data, embeddings_storage_mode) + # depending on memory mode, embeddings are moved to CPU, GPU or deleted + store_embeddings(evaluation_split_data, embeddings_storage_mode) - self._publish_eval_result(eval_result, evaluation_split, global_step=epoch) + self._publish_eval_result(eval_result, evaluation_split, global_step=epoch) - # use DEV split to determine if this is the best model so far - if determine_best_epoch_using_dev_score and evaluation_split == "dev": - validation_scores = eval_result.main_score, eval_result.loss + # use DEV split to determine if this is the best model so far + if determine_best_epoch_using_dev_score and evaluation_split == "dev": + validation_scores = eval_result.main_score, eval_result.loss - if eval_result.main_score > best_epoch_score: - current_epoch_has_best_model_so_far = True - best_epoch_score = eval_result.main_score + if eval_result.main_score > best_epoch_score: + current_epoch_has_best_model_so_far = True + best_epoch_score = eval_result.main_score - # if not using DEV score, determine best model using train loss - if not determine_best_epoch_using_dev_score: - validation_scores = (train_loss,) + # if not using DEV score, determine best model using train loss + if not determine_best_epoch_using_dev_score: + validation_scores = (train_loss,) - if epoch_train_loss < best_epoch_score: - current_epoch_has_best_model_so_far = True - best_epoch_score = train_loss + if epoch_train_loss < best_epoch_score: + current_epoch_has_best_model_so_far = True + best_epoch_score = train_loss # - LossFilePlugin -> somehow prints all relevant metrics # - AnnealPlugin -> scheduler step @@ -776,41 +791,42 @@ def train_custom( self.dispatch("_training_finally") # test best model if test data is present - if self.corpus.test and not train_with_test: - log_line(log) + if is_main_process(): + if self.corpus.test and not train_with_test: + log_line(log) - self.model.eval() + self.model.eval() - if (base_path / "best-model.pt").exists(): - log.info("Loading model from best epoch ...") - self.model.load_state_dict(self.model.load(base_path / "best-model.pt").state_dict()) - else: - log.info("Testing using last state of model ...") - - test_results = self.model.evaluate( - self.corpus.test, - gold_label_type=self.model.label_type, - mini_batch_size=eval_batch_size, - out_path=base_path / "test.tsv", - embedding_storage_mode="none", - main_evaluation_metric=main_evaluation_metric, - gold_label_dictionary=gold_label_dictionary_for_eval, - exclude_labels=exclude_labels, - return_loss=False, - ) + if (base_path / "best-model.pt").exists(): + log.info("Loading model from best epoch ...") + self.model.load_state_dict(self.model.load(base_path / "best-model.pt").state_dict()) + else: + log.info("Testing using last state of model ...") + + test_results = self.model.evaluate( + self.corpus.test, + gold_label_type=self.model.label_type, + mini_batch_size=eval_batch_size, + out_path=base_path / "test.tsv", + embedding_storage_mode="none", + main_evaluation_metric=main_evaluation_metric, + gold_label_dictionary=gold_label_dictionary_for_eval, + exclude_labels=exclude_labels, + return_loss=False, + ) - log.info(test_results.detailed_results) - log_line(log) + log.info(test_results.detailed_results) + log_line(log) - # get and return the final test score of best model - self.return_values["test_score"] = test_results.main_score + # get and return the final test score of best model + self.return_values["test_score"] = test_results.main_score - else: - if (base_path / "best-model.pt").exists(): - log.info("Loading model from best epoch ...") - self.model.load_state_dict(self.model.load(base_path / "best-model.pt").state_dict()) - self.return_values["test_score"] = 0 - log.info("Test data not provided setting final score to 0") + else: + if (base_path / "best-model.pt").exists(): + log.info("Loading model from best epoch ...") + self.model.load_state_dict(self.model.load(base_path / "best-model.pt").state_dict()) + self.return_values["test_score"] = 0 + log.info("Test data not provided setting final score to 0") # MetricHistoryPlugin -> stores the loss history in return_values self.dispatch("after_training") From f777cc6dac88fc6db3c277756257ef175ebdd08f Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Mon, 30 Sep 2024 16:45:25 -0600 Subject: [PATCH 016/333] GH-3429: Move process spawning inside `.train`; WIP sync gradients --- flair/__init__.py | 6 - flair/distributed_utils.py | 58 ++++--- flair/nn/model.py | 7 +- flair/trainers/trainer.py | 328 +++++++++++++++++++------------------ 4 files changed, 212 insertions(+), 187 deletions(-) diff --git a/flair/__init__.py b/flair/__init__.py index 55bd6d00cf..341f630e43 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -33,12 +33,6 @@ else: device = torch.device("cpu") -distributed = False -"""Experimental flag to indicate multiple GPUs are in use. - -Set by `launch_distributed` -- do not set manually. -""" - # global variable: version __version__ = "0.14.0" """The current version of the flair library installed.""" diff --git a/flair/distributed_utils.py b/flair/distributed_utils.py index 596570b7ce..39ec8c4ce0 100644 --- a/flair/distributed_utils.py +++ b/flair/distributed_utils.py @@ -1,49 +1,63 @@ import logging import os +from multiprocessing.connection import Connection +from typing import Callable +import numpy as np import torch import torch.multiprocessing as mp from torch.distributed import destroy_process_group, init_process_group import flair +from flair.class_utils import T log = logging.getLogger("flair") -def launch_distributed(fp, *args): - """Executes the function fp(*args) on multiple GPUs (all local GPUs)""" - world_size = torch.cuda.device_count() - log.info(f"Launching {world_size} distributed processes") - mp.spawn(entrypoint, args=(world_size, fp, *args), nprocs=world_size) - +def launch_distributed(fn, *args, **kwargs): + """Executes the function fn(*args, **kwargs) on multiple processes (one for each local GPU). -def entrypoint(rank, world_size, fp, *args): - ddp_setup(rank, world_size) - fp(*args) + Returns: the return value of the function fp(*args, **kwargs) from the rank 0 process + """ + world_size = torch.cuda.device_count() + log.info(f"Launching {world_size} processes") + parent_conn, child_conn = mp.Pipe() + mp.spawn(_entrypoint, args=(world_size, child_conn, fn, args, kwargs), nprocs=world_size) + return_value = parent_conn.recv() + return return_value + + +def _entrypoint(rank: int, world_size: int, child_conn: Connection, fn: Callable, args: tuple, kwargs: dict) -> None: + """Lifecycle of a process -- setup, run, cleanup.""" + log.info(f"Started process on rank={rank}") + _ddp_setup(rank, world_size) + return_value = fn(*args, **kwargs) + if is_main_process(): + child_conn.send(return_value) destroy_process_group() -def ddp_setup(rank: int, world_size: int) -> None: +def _ddp_setup(rank: int, world_size: int) -> None: os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = "12355" - init_process_group(backend="nccl", rank=rank, world_size=world_size) - flair.distributed = True flair.device = torch.device(rank) torch.cuda.set_device(flair.device) + init_process_group(backend="nccl", rank=rank, world_size=world_size) def is_main_process() -> bool: - """True for exactly 1 process, regardless of whether being run on CPU/single-GPU/multi-gpu""" - if flair.distributed: - return flair.device.index == 0 + """True for exactly 1 process, regardless of whether being run on CPU/single-GPU/multi-gpu.""" + if torch.distributed.is_initialized(): + return torch.distributed.get_rank() == 0 else: return True -class DistributedModel(torch.nn.parallel.DistributedDataParallel): - """DistributedDataParallel, but redirects access to methods and attributes to the original Model""" - def __getattr__(self, name): - try: - return super().__getattr__(name) - except AttributeError: - return getattr(self.module, name) +def aggregate_if_distributed(value: T, aggregation_fn: Callable = np.mean) -> T: + """Gathers value from each process and returns the aggregated value according to the supplied function.""" + if torch.distributed.is_initialized(): + gathered_values = [None for _ in range(torch.distributed.get_world_size())] + torch.distributed.all_gather_object(gathered_values, value) + return aggregation_fn(gathered_values) + else: + return value diff --git a/flair/nn/model.py b/flair/nn/model.py index 1710e32c76..c903c4000a 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -17,7 +17,6 @@ from flair.class_utils import get_non_abstract_subclasses from flair.data import DT, DT2, Corpus, Dictionary, Sentence, _iter_dataset from flair.datasets import DataLoader, FlairDatapointDataset -from flair.distributed_utils import is_main_process from flair.embeddings import Embeddings from flair.embeddings.base import load_embeddings from flair.file_utils import Tqdm, load_torch_state @@ -48,6 +47,10 @@ def forward_loss(self, data_points: list[DT]) -> tuple[torch.Tensor, int]: """ raise NotImplementedError + def forward(self, data_points: List[DT]) -> Tuple[torch.Tensor, int]: + """Wraps forward_loss to maintain compatibility with hooks.""" + return self.forward_loss(data_points) + @abstractmethod def evaluate( self, @@ -119,8 +122,6 @@ def save(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: model_file: the model file checkpoint: currently unused. """ - if not is_main_process(): - return model_state = self._get_state_dict() # write out a "model card" if one is set diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 05c376ec20..ae5e30c712 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -7,9 +7,12 @@ import warnings from inspect import signature from pathlib import Path -from typing import Optional, Union +from queue import Queue +from typing import Optional, Tuple, Type, Union +import numpy as np import torch +from torch.nn.parallel import DistributedDataParallel from torch.optim.sgd import SGD from torch.utils.data import DistributedSampler from torch.utils.data.dataset import ConcatDataset @@ -18,7 +21,7 @@ import flair.nn from flair.data import Corpus, Dictionary, _len_dataset from flair.datasets import DataLoader -from flair.distributed_utils import DistributedModel, is_main_process +from flair.distributed_utils import aggregate_if_distributed, is_main_process, launch_distributed from flair.samplers import FlairSampler from flair.trainers.plugins import ( AnnealingPlugin, @@ -165,6 +168,8 @@ def train( create_file_logs: bool = True, create_loss_file: bool = True, write_weights: bool = False, + # scaling + multi_gpu: bool = False, # plugins plugins: Optional[list[TrainerPlugin]] = None, attach_default_scheduler: bool = True, @@ -200,7 +205,12 @@ def train( "kwargs", ]: local_variables.pop(var) - return self.train_custom(**local_variables, **kwargs) + + if multi_gpu: + self._event_queue = None # Each process will make its own queue rather than share + return launch_distributed(self.train_custom, **local_variables, **kwargs) + else: + return self.train_custom(**local_variables, **kwargs) def fine_tune( self, @@ -239,8 +249,9 @@ def fine_tune( create_file_logs: bool = True, create_loss_file: bool = True, write_weights: bool = False, - # amp + # scaling use_amp: bool = False, + multi_gpu: bool = False, # plugins plugins: Optional[list[TrainerPlugin]] = None, attach_default_scheduler: bool = True, @@ -254,47 +265,21 @@ def fine_tune( if attach_default_scheduler: plugins.append(LinearSchedulerPlugin(warmup_fraction=warmup_fraction)) - return self.train_custom( - base_path=base_path, - # training parameters - learning_rate=learning_rate, - decoder_learning_rate=decoder_learning_rate, - mini_batch_size=mini_batch_size, - eval_batch_size=eval_batch_size, - mini_batch_chunk_size=mini_batch_chunk_size, - max_epochs=max_epochs, - optimizer=optimizer, - train_with_dev=train_with_dev, - train_with_test=train_with_test, - reduce_transformer_vocab=reduce_transformer_vocab, - # evaluation and monitoring - main_evaluation_metric=main_evaluation_metric, - monitor_test=monitor_test, - monitor_train_sample=monitor_train_sample, - use_final_model_for_eval=use_final_model_for_eval, - gold_label_dictionary_for_eval=gold_label_dictionary_for_eval, - exclude_labels=exclude_labels, - # sampling and shuffling - sampler=sampler, - shuffle=shuffle, - shuffle_first_epoch=shuffle_first_epoch, - # evaluation and monitoring - embeddings_storage_mode=embeddings_storage_mode, - epoch=epoch, - # when and what to save - save_final_model=save_final_model, - save_optimizer_state=save_optimizer_state, - save_model_each_k_epochs=save_model_each_k_epochs, - # logging parameters - create_file_logs=create_file_logs, - create_loss_file=create_loss_file, - write_weights=write_weights, - # amp - use_amp=use_amp, - # plugins - plugins=plugins, - **kwargs, - ) + # call self.train_custom with all parameters (minus the ones specific to the LinearSchedulerPlugin) + local_variables = locals() + for var in [ + "self", + "warmup_fraction", + "attach_default_scheduler", + "kwargs", + ]: + local_variables.pop(var) + + if multi_gpu: + self._event_queue = None + return launch_distributed(self.train_custom, **local_variables, **kwargs) + else: + return self.train_custom(**local_variables, **kwargs) def train_custom( self, @@ -333,8 +318,9 @@ def train_custom( create_file_logs: bool = True, create_loss_file: bool = True, write_weights: bool = False, - # amp + # scaling use_amp: bool = False, + multi_gpu: bool = False, # plugins plugins: Optional[list[TrainerPlugin]] = None, **kwargs, @@ -377,6 +363,7 @@ def train_custom( create_file_logs: If True, logging output is written to a file create_loss_file: If True, a loss file logging output is created use_amp: If True, uses the torch automatic mixed precision + multi_gpu: If True, uses all available GPUs write_weights: If True, write weights to weights.txt on each batch logging event. plugins: Any additional plugins you want to pass to the trainer **kwargs: Additional arguments, for instance for the optimizer @@ -422,8 +409,11 @@ def train_custom( base_path=base_path, ).attach_to(self) - if flair.distributed: - self.model = DistributedModel(self.model, device_ids=[flair.device.index]) + if multi_gpu: + self.model.to(flair.device) + self.ddp_model = DistributedDataParallel(self.model, device_ids=[flair.device.index]) + self._event_queue = Queue() # Each process uses its own queue rather than share + log.disabled = not is_main_process() # Disable logging in distributed mode for all but the main process # === END BLOCK: ACTIVATE PLUGINS === # # derive parameters the function was called with (or defaults) @@ -512,38 +502,38 @@ def train_custom( if use_final_model_for_eval else "model from best epoch (best-model.pt)" ) - - if is_main_process(): - log_line(log) - log.info(f'Model: "{self.model}"') - log_line(log) - log.info(f"{self.corpus}") - log_line(log) - log.info(f"Train: {len(train_data)} sentences") - log.info(f" (train_with_dev={train_with_dev}, train_with_test={train_with_test})") - log_line(log) - log.info("Training Params:") - log.info( - f' - learning_rate: "{learning_rate}" ' - f'{"(decoder: " + str(decoder_learning_rate) + ")" if decoder_learning_rate else ""}' - ) - log.info(f' - mini_batch_size: "{mini_batch_size}"') - log.info(f' - max_epochs: "{max_epochs}"') - log.info(f' - shuffle: "{shuffle}"') - log_line(log) - log.info("Plugins:") - for plugin in plugins: - log.info(" - " + str(plugin)) - log_line(log) - log.info(f"Final evaluation on {final_eval_info}") - log.info(f' - metric: "{main_evaluation_metric}"') - log_line(log) - log.info("Computation:") - log.info(f" - compute on device: {flair.device}") - log.info(f" - embedding storage: {embeddings_storage_mode}") - log_line(log) - log.info(f'Model training base path: "{base_path}"') - log_line(log) + computation_device_info = f"{torch.cuda.device_count()} GPUs" if multi_gpu else flair.device + + log_line(log) + log.info(f'Model: "{self.model}"') + log_line(log) + log.info(f"{self.corpus}") + log_line(log) + log.info(f"Train: {len(train_data)} sentences") + log.info(f" (train_with_dev={train_with_dev}, train_with_test={train_with_test})") + log_line(log) + log.info("Training Params:") + log.info( + f' - learning_rate: "{learning_rate}" ' + f'{"(decoder: " + str(decoder_learning_rate) + ")" if decoder_learning_rate else ""}' + ) + log.info(f' - mini_batch_size: "{mini_batch_size}"') + log.info(f' - max_epochs: "{max_epochs}"') + log.info(f' - shuffle: "{shuffle}"') + log_line(log) + log.info("Plugins:") + for plugin in plugins: + log.info(" - " + str(plugin)) + log_line(log) + log.info(f"Final evaluation on {final_eval_info}") + log.info(f' - metric: "{main_evaluation_metric}"') + log_line(log) + log.info("Computation:") + log.info(f" - compute on device: {computation_device_info}") + log.info(f" - embedding storage: {embeddings_storage_mode}") + log_line(log) + log.info(f'Model training base path: "{base_path}"') + log_line(log) # At any point you can hit Ctrl + C to break out of training early. try: @@ -565,14 +555,14 @@ def train_custom( if not shuffle_first_epoch and epoch == 1: shuffle_data_this_epoch = False - if flair.distributed: + if multi_gpu: batch_loader = DataLoader( train_data, batch_size=mini_batch_size, shuffle=False, sampler=DistributedSampler(train_data, shuffle=shuffle_data_this_epoch), ) - batch_loader.sampler.set_epoch(epoch) + batch_loader.sampler.set_epoch(epoch - 1) else: batch_loader = DataLoader( train_data, @@ -617,7 +607,10 @@ def train_custom( for batch_step in batch_steps: # forward pass with torch.autocast(device_type=flair.device.type, enabled=use_amp): - loss, datapoint_count = self.model.forward_loss(batch_step) + if multi_gpu: + loss, datapoint_count = self.ddp_model(batch_step) + else: + loss, datapoint_count = self.model.forward_loss(batch_step) batch_train_samples += datapoint_count batch_train_loss += loss.item() @@ -663,8 +656,11 @@ def train_custom( if epoch_train_samples > 0 else epoch_train_samples / (batch_no + 1) ) + intermittent_loss = aggregate_if_distributed(intermittent_loss) current_time = time.time() + samples_per_second = epoch_train_samples / (current_time - epoch_start_time) + samples_per_second = aggregate_if_distributed(samples_per_second, np.sum) lr_info, momentum_info = self._get_current_lr_and_momentum(batch_count) log.info( @@ -672,7 +668,7 @@ def train_custom( f" - iter {batch_no + 1}/{len(batch_loader)}" f" - loss {intermittent_loss:.8f}" f" - time (sec): {(current_time - epoch_start_time):.2f}" - f" - samples/sec: {epoch_train_samples / (current_time - epoch_start_time):.2f}" + f" - samples/sec: {samples_per_second:.2f}" f"{lr_info}{momentum_info}" ) @@ -681,6 +677,7 @@ def train_custom( self.dispatch("after_training_batch", **batch_kw) train_loss = epoch_train_loss / epoch_train_samples + train_loss = aggregate_if_distributed(train_loss) self._record(MetricRecord.scalar(("train", "loss"), train_loss, epoch)) total_train_samples += epoch_train_samples @@ -696,50 +693,49 @@ def train_custom( # Determine if this is the best model or if we need to anneal current_epoch_has_best_model_so_far = False - validation_scores = () - - if is_main_process(): - for evaluation_split, evaluation_split_data in evaluation_splits.items(): - eval_result = self.model.evaluate( - evaluation_split_data, - out_path=base_path / f"{evaluation_split}.tsv", - mini_batch_size=eval_batch_size, - exclude_labels=exclude_labels, - main_evaluation_metric=main_evaluation_metric, - gold_label_dictionary=gold_label_dictionary_for_eval, - embedding_storage_mode=embeddings_storage_mode, - gold_label_type=self.model.label_type, - gold_label_dictionary_for_eval=gold_label_dictionary_for_eval, - ) + validation_scores: tuple = () + + for evaluation_split, evaluation_split_data in evaluation_splits.items(): + eval_result = self.model.evaluate( + evaluation_split_data, + out_path=base_path / f"{evaluation_split}.tsv", + mini_batch_size=eval_batch_size, + exclude_labels=exclude_labels, + main_evaluation_metric=main_evaluation_metric, + gold_label_dictionary=gold_label_dictionary_for_eval, + embedding_storage_mode=embeddings_storage_mode, + gold_label_type=self.model.label_type, + gold_label_dictionary_for_eval=gold_label_dictionary_for_eval, + ) - # log results - log.info( - f"{evaluation_split.upper()} : loss {eval_result.loss}" - f" - {main_evaluation_metric[1]}" - f" ({main_evaluation_metric[0]})" - f" {round(eval_result.main_score, 4)}" - ) + # log results + log.info( + f"{evaluation_split.upper()} : loss {eval_result.loss}" + f" - {main_evaluation_metric[1]}" + f" ({main_evaluation_metric[0]})" + f" {round(eval_result.main_score, 4)}" + ) - # depending on memory mode, embeddings are moved to CPU, GPU or deleted - store_embeddings(evaluation_split_data, embeddings_storage_mode) + # depending on memory mode, embeddings are moved to CPU, GPU or deleted + store_embeddings(evaluation_split_data, embeddings_storage_mode) - self._publish_eval_result(eval_result, evaluation_split, global_step=epoch) + self._publish_eval_result(eval_result, evaluation_split, global_step=epoch) - # use DEV split to determine if this is the best model so far - if determine_best_epoch_using_dev_score and evaluation_split == "dev": - validation_scores = eval_result.main_score, eval_result.loss + # use DEV split to determine if this is the best model so far + if determine_best_epoch_using_dev_score and evaluation_split == "dev": + validation_scores = eval_result.main_score, eval_result.loss - if eval_result.main_score > best_epoch_score: - current_epoch_has_best_model_so_far = True - best_epoch_score = eval_result.main_score + if eval_result.main_score > best_epoch_score: + current_epoch_has_best_model_so_far = True + best_epoch_score = eval_result.main_score - # if not using DEV score, determine best model using train loss - if not determine_best_epoch_using_dev_score: - validation_scores = (train_loss,) + # if not using DEV score, determine best model using train loss + if not determine_best_epoch_using_dev_score: + validation_scores = (train_loss,) - if epoch_train_loss < best_epoch_score: - current_epoch_has_best_model_so_far = True - best_epoch_score = train_loss + if train_loss < best_epoch_score: + current_epoch_has_best_model_so_far = True + best_epoch_score = train_loss # - LossFilePlugin -> somehow prints all relevant metrics # - AnnealPlugin -> scheduler step @@ -752,14 +748,14 @@ def train_custom( if save_best_model and current_epoch_has_best_model_so_far: log.info("saving best model") - self.model.save(base_path / "best-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "best-model.pt", checkpoint=save_optimizer_state) # - SWAPlugin -> restores SGD weights from SWA self.dispatch("after_training_loop") # if we do not use dev data for model selection, save final model if save_final_model: - self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) except KeyboardInterrupt: log_line(log) @@ -769,7 +765,7 @@ def train_custom( if save_final_model: log.info("Saving model ...") - self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) log.info("Done.") except TrainingInterrupt as exc: @@ -780,7 +776,7 @@ def train_custom( if save_final_model: log.info("Saving model ...") - self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) log.info("Done.") except Exception: @@ -791,42 +787,41 @@ def train_custom( self.dispatch("_training_finally") # test best model if test data is present - if is_main_process(): - if self.corpus.test and not train_with_test: - log_line(log) + if self.corpus.test and not train_with_test: + log_line(log) - self.model.eval() + self.model.eval() - if (base_path / "best-model.pt").exists(): - log.info("Loading model from best epoch ...") - self.model.load_state_dict(self.model.load(base_path / "best-model.pt").state_dict()) - else: - log.info("Testing using last state of model ...") - - test_results = self.model.evaluate( - self.corpus.test, - gold_label_type=self.model.label_type, - mini_batch_size=eval_batch_size, - out_path=base_path / "test.tsv", - embedding_storage_mode="none", - main_evaluation_metric=main_evaluation_metric, - gold_label_dictionary=gold_label_dictionary_for_eval, - exclude_labels=exclude_labels, - return_loss=False, - ) + if (base_path / "best-model.pt").exists(): + log.info("Loading model from best epoch ...") + self._load_model(base_path / "best-model.pt") + else: + log.info("Testing using last state of model ...") + + test_results = self.model.evaluate( + self.corpus.test, + gold_label_type=self.model.label_type, + mini_batch_size=eval_batch_size, + out_path=base_path / "test.tsv", + embedding_storage_mode="none", + main_evaluation_metric=main_evaluation_metric, + gold_label_dictionary=gold_label_dictionary_for_eval, + exclude_labels=exclude_labels, + return_loss=False, + ) - log.info(test_results.detailed_results) - log_line(log) + log.info(test_results.detailed_results) + log_line(log) - # get and return the final test score of best model - self.return_values["test_score"] = test_results.main_score + # get and return the final test score of best model + self.return_values["test_score"] = test_results.main_score - else: - if (base_path / "best-model.pt").exists(): - log.info("Loading model from best epoch ...") - self.model.load_state_dict(self.model.load(base_path / "best-model.pt").state_dict()) - self.return_values["test_score"] = 0 - log.info("Test data not provided setting final score to 0") + else: + if (base_path / "best-model.pt").exists(): + log.info("Loading model from best epoch ...") + self._load_model(base_path / "best-model.pt") + self.return_values["test_score"] = 0 + log.info("Test data not provided setting final score to 0") # MetricHistoryPlugin -> stores the loss history in return_values self.dispatch("after_training") @@ -840,7 +835,9 @@ def train_custom( def _get_current_lr_and_momentum(self, batch_count): current_learning_rate = [group["lr"] for group in self.optimizer.param_groups] + current_learning_rate = [aggregate_if_distributed(m) for m in current_learning_rate] momentum = [group.get("momentum", 0) for group in self.optimizer.param_groups] + momentum = [aggregate_if_distributed(m) for m in momentum] lr_info = " - lr: " + ",".join([f"{m:.6f}" for m in current_learning_rate]) momentum_info = " - momentum: " + ",".join([f"{m:.6f}" for m in momentum]) self._record(MetricRecord.scalar_list("learning_rate", current_learning_rate, batch_count)) @@ -921,3 +918,22 @@ def _initialize_model_card(self, **training_parameters): def _record(self, metric): self.dispatch("metric_recorded", metric) + + def _save_model(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: + """Saves the current model. Safe to call from a distributed context. + + Args: + model_file: the model file + checkpoint: currently unused. + """ + if is_main_process(): + self.model.save(model_file, checkpoint) + + if torch.distributed.is_initialized(): + torch.distributed.barrier() # Prevent any process from loading a model until writing is complete + + def _load_model(self, model_file: Union[str, Path]) -> None: + """Loads the model from the given file into the current state. Safe to call from a distributed context.""" + self.model.load_state_dict(self.model.load(model_file).state_dict()) + if torch.distributed.is_initialized(): + self.ddp_model = DistributedDataParallel(self.model, device_ids=[flair.device.index]) From c91809379402f2c359e1aff22b51adfb7539d111 Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Fri, 25 Oct 2024 02:32:13 -0700 Subject: [PATCH 017/333] GH-3429: Redirect forward_loss; User calls launch; Handle plugins --- flair/distributed_utils.py | 31 +++-- flair/nn/model.py | 7 +- flair/trainers/plugins/base.py | 9 ++ .../plugins/functional/checkpoints.py | 4 + .../plugins/functional/linear_scheduler.py | 5 +- .../functional/reduce_transformer_vocab.py | 4 + .../plugins/functional/weight_extractor.py | 4 + .../plugins/loggers/clearml_logger.py | 4 + flair/trainers/plugins/loggers/log_file.py | 4 + flair/trainers/plugins/loggers/loss_file.py | 4 + .../plugins/loggers/metric_history.py | 4 + flair/trainers/plugins/loggers/tensorboard.py | 4 + flair/trainers/plugins/loggers/wandb.py | 4 + flair/trainers/trainer.py | 114 ++++++++++++------ 14 files changed, 146 insertions(+), 56 deletions(-) diff --git a/flair/distributed_utils.py b/flair/distributed_utils.py index 39ec8c4ce0..d391251b31 100644 --- a/flair/distributed_utils.py +++ b/flair/distributed_utils.py @@ -9,7 +9,6 @@ from torch.distributed import destroy_process_group, init_process_group import flair -from flair.class_utils import T log = logging.getLogger("flair") @@ -17,24 +16,30 @@ def launch_distributed(fn, *args, **kwargs): """Executes the function fn(*args, **kwargs) on multiple processes (one for each local GPU). + If training with multi_gpu=True, launch_distributed should wrap your code that calls .train or .fine_tune. + Returns: the return value of the function fp(*args, **kwargs) from the rank 0 process """ world_size = torch.cuda.device_count() log.info(f"Launching {world_size} processes") parent_conn, child_conn = mp.Pipe() - mp.spawn(_entrypoint, args=(world_size, child_conn, fn, args, kwargs), nprocs=world_size) + mp.spawn(_process_entrypoint, args=(world_size, child_conn, fn, args, kwargs), nprocs=world_size) return_value = parent_conn.recv() return return_value -def _entrypoint(rank: int, world_size: int, child_conn: Connection, fn: Callable, args: tuple, kwargs: dict) -> None: - """Lifecycle of a process -- setup, run, cleanup.""" +def _process_entrypoint( + rank: int, world_size: int, child_conn: Connection, fn: Callable, args: tuple, kwargs: dict +) -> None: + """Lifecycle of a distributed process -- setup, run, cleanup.""" log.info(f"Started process on rank={rank}") - _ddp_setup(rank, world_size) - return_value = fn(*args, **kwargs) - if is_main_process(): - child_conn.send(return_value) - destroy_process_group() + try: + _ddp_setup(rank, world_size) + return_value = fn(*args, **kwargs) + if is_main_process(): + child_conn.send(return_value) + finally: + destroy_process_group() def _ddp_setup(rank: int, world_size: int) -> None: @@ -53,11 +58,11 @@ def is_main_process() -> bool: return True -def aggregate_if_distributed(value: T, aggregation_fn: Callable = np.mean) -> T: - """Gathers value from each process and returns the aggregated value according to the supplied function.""" +def aggregate(value, aggregation_fn=np.mean): + """Gather `value` from all processes and send to `aggregation_fn` to get a single return value.""" if torch.distributed.is_initialized(): gathered_values = [None for _ in range(torch.distributed.get_world_size())] torch.distributed.all_gather_object(gathered_values, value) - return aggregation_fn(gathered_values) else: - return value + gathered_values = [value] + return aggregation_fn(gathered_values) diff --git a/flair/nn/model.py b/flair/nn/model.py index c903c4000a..f670c969a0 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -17,6 +17,7 @@ from flair.class_utils import get_non_abstract_subclasses from flair.data import DT, DT2, Corpus, Dictionary, Sentence, _iter_dataset from flair.datasets import DataLoader, FlairDatapointDataset +from flair.distributed_utils import is_main_process from flair.embeddings import Embeddings from flair.embeddings.base import load_embeddings from flair.file_utils import Tqdm, load_torch_state @@ -47,10 +48,6 @@ def forward_loss(self, data_points: list[DT]) -> tuple[torch.Tensor, int]: """ raise NotImplementedError - def forward(self, data_points: List[DT]) -> Tuple[torch.Tensor, int]: - """Wraps forward_loss to maintain compatibility with hooks.""" - return self.forward_loss(data_points) - @abstractmethod def evaluate( self, @@ -295,7 +292,7 @@ def evaluate( loader = DataLoader(data_points, batch_size=mini_batch_size) sentence_id = 0 - for batch in Tqdm.tqdm(loader): + for batch in Tqdm.tqdm(loader, disable=not is_main_process()): # remove any previously predicted labels for datapoint in batch: datapoint.remove_labels("predicted") diff --git a/flair/trainers/plugins/base.py b/flair/trainers/plugins/base.py index 663a78d6d3..dcf7240a83 100644 --- a/flair/trainers/plugins/base.py +++ b/flair/trainers/plugins/base.py @@ -13,6 +13,8 @@ cast, ) +from flair.distributed_utils import is_main_process + log = logging.getLogger("flair") @@ -184,6 +186,8 @@ def attach_to(self, pluggable: Pluggable): assert self._pluggable is None assert len(self._hook_handles) == 0 + if not is_main_process() and not self.attach_to_all_processes: + return self._pluggable = pluggable pluggable.append_plugin(self) @@ -252,6 +256,11 @@ def decorator_func(func: Callable): def pluggable(self) -> Optional[Pluggable]: return self._pluggable + @property + def attach_to_all_processes(self) -> bool: + """If set, the plugin will be attached to all processes when distributed, not just the main process.""" + return True + def __str__(self) -> str: return self.__class__.__name__ diff --git a/flair/trainers/plugins/functional/checkpoints.py b/flair/trainers/plugins/functional/checkpoints.py index 1936177835..cf1a21468a 100644 --- a/flair/trainers/plugins/functional/checkpoints.py +++ b/flair/trainers/plugins/functional/checkpoints.py @@ -29,6 +29,10 @@ def after_training_epoch(self, epoch, **kw): model_name = "model_epoch_" + str(epoch) + ".pt" self.model.save(self.base_path / model_name, checkpoint=self.save_optimizer_state) + @property + def attach_to_all_processes(self) -> bool: + return False + def get_state(self) -> dict[str, Any]: return { **super().get_state(), diff --git a/flair/trainers/plugins/functional/linear_scheduler.py b/flair/trainers/plugins/functional/linear_scheduler.py index 2258844129..fdf4752bde 100644 --- a/flair/trainers/plugins/functional/linear_scheduler.py +++ b/flair/trainers/plugins/functional/linear_scheduler.py @@ -1,6 +1,8 @@ import logging from typing import Any +import torch.distributed + from flair.optim import LinearSchedulerWithWarmup from flair.trainers.plugins.base import TrainerPlugin @@ -34,7 +36,8 @@ def after_setup( ): """Initialize different schedulers, including anneal target for AnnealOnPlateau, batch_growth_annealing, loading schedulers.""" # calculate warmup steps - steps_per_epoch = (dataset_size + mini_batch_size - 1) / mini_batch_size + num_processes = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1 + steps_per_epoch = (dataset_size + mini_batch_size - 1) / mini_batch_size / num_processes num_train_steps = int(steps_per_epoch * max_epochs) num_warmup_steps = int(num_train_steps * self.warmup_fraction) diff --git a/flair/trainers/plugins/functional/reduce_transformer_vocab.py b/flair/trainers/plugins/functional/reduce_transformer_vocab.py index 86759c2fe2..eed6d7f1b2 100644 --- a/flair/trainers/plugins/functional/reduce_transformer_vocab.py +++ b/flair/trainers/plugins/functional/reduce_transformer_vocab.py @@ -55,6 +55,10 @@ def save_model_at_the_end(self, **kw): elif (self.base_path / "final-model.pt").exists(): self.model.save(self.base_path / "final-model.pt", checkpoint=self.save_optimizer_state) + @property + def attach_to_all_processes(self) -> bool: + return False + def get_transformer_embeddings(model: Model) -> list[TransformerEmbeddings]: embeddings = model.tars_embeddings if isinstance(model, FewshotClassifier) else getattr(model, "embeddings", None) diff --git a/flair/trainers/plugins/functional/weight_extractor.py b/flair/trainers/plugins/functional/weight_extractor.py index 4ba7c07621..5c9bd4c4ac 100644 --- a/flair/trainers/plugins/functional/weight_extractor.py +++ b/flair/trainers/plugins/functional/weight_extractor.py @@ -21,6 +21,10 @@ def after_training_batch(self, batch_no, epoch, total_number_of_batches, **kw): if (iteration + 1) % modulo == 0: self.weight_extractor.extract_weights(self.model.state_dict(), iteration) + @property + def attach_to_all_processes(self) -> bool: + return False + def get_state(self) -> dict[str, Any]: return { **super().get_state(), diff --git a/flair/trainers/plugins/loggers/clearml_logger.py b/flair/trainers/plugins/loggers/clearml_logger.py index 891b9f9244..18228d2db6 100644 --- a/flair/trainers/plugins/loggers/clearml_logger.py +++ b/flair/trainers/plugins/loggers/clearml_logger.py @@ -40,3 +40,7 @@ def metric_recorded(self, record: MetricRecord) -> None: self.logger.report_text(record.value, print_console=False) elif record.is_histogram: self.logger.report_histogram(record_name, record_name, record.value, record.global_step) + + @property + def attach_to_all_processes(self) -> bool: + return False diff --git a/flair/trainers/plugins/loggers/log_file.py b/flair/trainers/plugins/loggers/log_file.py index 21a8c54632..9bf22a284d 100644 --- a/flair/trainers/plugins/loggers/log_file.py +++ b/flair/trainers/plugins/loggers/log_file.py @@ -21,5 +21,9 @@ def close_file_handler(self, **kw): self.log_handler.close() log.removeHandler(self.log_handler) + @property + def attach_to_all_processes(self) -> bool: + return False + def get_state(self) -> dict[str, Any]: return {**super().get_state(), "base_path": str(self.base_path)} diff --git a/flair/trainers/plugins/loggers/loss_file.py b/flair/trainers/plugins/loggers/loss_file.py index b53a23a956..bfe938b72d 100644 --- a/flair/trainers/plugins/loggers/loss_file.py +++ b/flair/trainers/plugins/loggers/loss_file.py @@ -113,3 +113,7 @@ def after_evaluation(self, epoch, **kw): f.write("\t".join([str(self.current_row[col]) for col in self.headers]) + "\n") self.current_row = {} + + @property + def attach_to_all_processes(self) -> bool: + return False diff --git a/flair/trainers/plugins/loggers/metric_history.py b/flair/trainers/plugins/loggers/metric_history.py index a22cf1b0e7..426e055186 100644 --- a/flair/trainers/plugins/loggers/metric_history.py +++ b/flair/trainers/plugins/loggers/metric_history.py @@ -34,6 +34,10 @@ def after_training(self, **kw): """Returns metric history.""" self.trainer.return_values.update(self.metric_history) + @property + def attach_to_all_processes(self) -> bool: + return False + def get_state(self) -> dict[str, Any]: return { **super().get_state(), diff --git a/flair/trainers/plugins/loggers/tensorboard.py b/flair/trainers/plugins/loggers/tensorboard.py index a7af50a521..bf2dfcc29d 100644 --- a/flair/trainers/plugins/loggers/tensorboard.py +++ b/flair/trainers/plugins/loggers/tensorboard.py @@ -59,6 +59,10 @@ def _training_finally(self, **kw): assert self.writer is not None self.writer.close() + @property + def attach_to_all_processes(self) -> bool: + return False + def get_state(self) -> dict[str, Any]: return { **super().get_state(), diff --git a/flair/trainers/plugins/loggers/wandb.py b/flair/trainers/plugins/loggers/wandb.py index 0f8dc89f73..410d0377ba 100644 --- a/flair/trainers/plugins/loggers/wandb.py +++ b/flair/trainers/plugins/loggers/wandb.py @@ -72,6 +72,10 @@ def metric_recorded(self, record): def _training_finally(self, **kw): self.writer.close() + @property + def attach_to_all_processes(self) -> bool: + return False + def get_state(self) -> dict[str, Any]: return { **super().get_state(), diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index ae5e30c712..36c4f14295 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -21,7 +21,7 @@ import flair.nn from flair.data import Corpus, Dictionary, _len_dataset from flair.datasets import DataLoader -from flair.distributed_utils import aggregate_if_distributed, is_main_process, launch_distributed +from flair.distributed_utils import aggregate, is_main_process from flair.samplers import FlairSampler from flair.trainers.plugins import ( AnnealingPlugin, @@ -205,12 +205,7 @@ def train( "kwargs", ]: local_variables.pop(var) - - if multi_gpu: - self._event_queue = None # Each process will make its own queue rather than share - return launch_distributed(self.train_custom, **local_variables, **kwargs) - else: - return self.train_custom(**local_variables, **kwargs) + return self.train_custom(**local_variables, **kwargs) def fine_tune( self, @@ -265,21 +260,48 @@ def fine_tune( if attach_default_scheduler: plugins.append(LinearSchedulerPlugin(warmup_fraction=warmup_fraction)) - # call self.train_custom with all parameters (minus the ones specific to the LinearSchedulerPlugin) - local_variables = locals() - for var in [ - "self", - "warmup_fraction", - "attach_default_scheduler", - "kwargs", - ]: - local_variables.pop(var) - - if multi_gpu: - self._event_queue = None - return launch_distributed(self.train_custom, **local_variables, **kwargs) - else: - return self.train_custom(**local_variables, **kwargs) + return self.train_custom( + base_path=base_path, + # training parameters + learning_rate=learning_rate, + decoder_learning_rate=decoder_learning_rate, + mini_batch_size=mini_batch_size, + eval_batch_size=eval_batch_size, + mini_batch_chunk_size=mini_batch_chunk_size, + max_epochs=max_epochs, + optimizer=optimizer, + train_with_dev=train_with_dev, + train_with_test=train_with_test, + reduce_transformer_vocab=reduce_transformer_vocab, + # evaluation and monitoring + main_evaluation_metric=main_evaluation_metric, + monitor_test=monitor_test, + monitor_train_sample=monitor_train_sample, + use_final_model_for_eval=use_final_model_for_eval, + gold_label_dictionary_for_eval=gold_label_dictionary_for_eval, + exclude_labels=exclude_labels, + # sampling and shuffling + sampler=sampler, + shuffle=shuffle, + shuffle_first_epoch=shuffle_first_epoch, + # evaluation and monitoring + embeddings_storage_mode=embeddings_storage_mode, + epoch=epoch, + # when and what to save + save_final_model=save_final_model, + save_optimizer_state=save_optimizer_state, + save_model_each_k_epochs=save_model_each_k_epochs, + # logging parameters + create_file_logs=create_file_logs, + create_loss_file=create_loss_file, + write_weights=write_weights, + # scaling + use_amp=use_amp, + multi_gpu=multi_gpu, + # plugins + plugins=plugins, + **kwargs, + ) def train_custom( self, @@ -363,7 +385,7 @@ def train_custom( create_file_logs: If True, logging output is written to a file create_loss_file: If True, a loss file logging output is created use_amp: If True, uses the torch automatic mixed precision - multi_gpu: If True, uses all available GPUs + multi_gpu: If True, distributes training across local GPUs write_weights: If True, write weights to weights.txt on each batch logging event. plugins: Any additional plugins you want to pass to the trainer **kwargs: Additional arguments, for instance for the optimizer @@ -409,11 +431,6 @@ def train_custom( base_path=base_path, ).attach_to(self) - if multi_gpu: - self.model.to(flair.device) - self.ddp_model = DistributedDataParallel(self.model, device_ids=[flair.device.index]) - self._event_queue = Queue() # Each process uses its own queue rather than share - log.disabled = not is_main_process() # Disable logging in distributed mode for all but the main process # === END BLOCK: ACTIVATE PLUGINS === # # derive parameters the function was called with (or defaults) @@ -475,6 +492,16 @@ def train_custom( sampler.set_dataset(train_data) shuffle = False + # configure special behavior to use multiple GPUs + if multi_gpu: + if not torch.distributed.is_initialized(): + raise RuntimeError("multi_gpu=True can only used inside flair.distributed_utils.launch_distributed()") + self.ddp_model = DistributedDataParallel( + self.model, device_ids=[flair.device.index], find_unused_parameters=True + ) + log.disabled = not is_main_process() # Only print logs once + original_forward = self.model.forward + # this field stores the names of all dynamic embeddings in the model (determined after first forward pass) dynamic_embeddings = None @@ -502,7 +529,9 @@ def train_custom( if use_final_model_for_eval else "model from best epoch (best-model.pt)" ) - computation_device_info = f"{torch.cuda.device_count()} GPUs" if multi_gpu else flair.device + computation_device_info = aggregate( + flair.device, lambda devices: ", ".join([str(device) for device in devices]) + ) log_line(log) log.info(f'Model: "{self.model}"') @@ -556,13 +585,16 @@ def train_custom( shuffle_data_this_epoch = False if multi_gpu: + distributed_sampler: DistributedSampler = DistributedSampler( + train_data, shuffle=shuffle_data_this_epoch + ) + distributed_sampler.set_epoch(epoch - 1) batch_loader = DataLoader( train_data, batch_size=mini_batch_size, shuffle=False, - sampler=DistributedSampler(train_data, shuffle=shuffle_data_this_epoch), + sampler=distributed_sampler, ) - batch_loader.sampler.set_epoch(epoch - 1) else: batch_loader = DataLoader( train_data, @@ -608,6 +640,14 @@ def train_custom( # forward pass with torch.autocast(device_type=flair.device.type, enabled=use_amp): if multi_gpu: + # We need to __call__ ddp_model() because this triggers hooks that sync gradients. + # But that calls forward rather than forward_loss. So we patch forward to redirect + # to forward_loss. Then undo the patch in case forward_loss itself calls forward. + def wrapped_forward_loss(*args, **kwargs2): + self.model.forward = original_forward + return self.model.forward_loss(*args, **kwargs2) + + self.model.forward = wrapped_forward_loss loss, datapoint_count = self.ddp_model(batch_step) else: loss, datapoint_count = self.model.forward_loss(batch_step) @@ -656,11 +696,11 @@ def train_custom( if epoch_train_samples > 0 else epoch_train_samples / (batch_no + 1) ) - intermittent_loss = aggregate_if_distributed(intermittent_loss) + intermittent_loss = aggregate(intermittent_loss) current_time = time.time() samples_per_second = epoch_train_samples / (current_time - epoch_start_time) - samples_per_second = aggregate_if_distributed(samples_per_second, np.sum) + samples_per_second = aggregate(samples_per_second, np.sum) lr_info, momentum_info = self._get_current_lr_and_momentum(batch_count) log.info( @@ -677,7 +717,7 @@ def train_custom( self.dispatch("after_training_batch", **batch_kw) train_loss = epoch_train_loss / epoch_train_samples - train_loss = aggregate_if_distributed(train_loss) + train_loss = aggregate(train_loss) self._record(MetricRecord.scalar(("train", "loss"), train_loss, epoch)) total_train_samples += epoch_train_samples @@ -835,9 +875,7 @@ def train_custom( def _get_current_lr_and_momentum(self, batch_count): current_learning_rate = [group["lr"] for group in self.optimizer.param_groups] - current_learning_rate = [aggregate_if_distributed(m) for m in current_learning_rate] momentum = [group.get("momentum", 0) for group in self.optimizer.param_groups] - momentum = [aggregate_if_distributed(m) for m in momentum] lr_info = " - lr: " + ",".join([f"{m:.6f}" for m in current_learning_rate]) momentum_info = " - momentum: " + ",".join([f"{m:.6f}" for m in momentum]) self._record(MetricRecord.scalar_list("learning_rate", current_learning_rate, batch_count)) @@ -936,4 +974,6 @@ def _load_model(self, model_file: Union[str, Path]) -> None: """Loads the model from the given file into the current state. Safe to call from a distributed context.""" self.model.load_state_dict(self.model.load(model_file).state_dict()) if torch.distributed.is_initialized(): - self.ddp_model = DistributedDataParallel(self.model, device_ids=[flair.device.index]) + self.ddp_model = DistributedDataParallel( + self.model, device_ids=[flair.device.index], find_unused_parameters=True + ) From 0cbeb9fb786e27bffbca034131d4f09bf120c646 Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Mon, 28 Oct 2024 18:11:00 -0700 Subject: [PATCH 018/333] fix transformers switching attention implementation --- flair/embeddings/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index 245d528e5d..c43258a193 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -1356,7 +1356,7 @@ def to_params(self): # do not switch the attention implementation upon reload. config_dict["attn_implementation"] = self.model.config._attn_implementation - del config_dict["_attn_implementation_autoset"] + config_dict.pop("_attn_implementation_autoset", None) super_params = super().to_params() From a590cd6bb02c57f6a700179390650a2ca4329447 Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Mon, 28 Oct 2024 18:11:08 -0700 Subject: [PATCH 019/333] GH-3429: Rename parameter group --- flair/trainers/trainer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 36c4f14295..86a0ce3527 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -168,7 +168,7 @@ def train( create_file_logs: bool = True, create_loss_file: bool = True, write_weights: bool = False, - # scaling + # acceleration multi_gpu: bool = False, # plugins plugins: Optional[list[TrainerPlugin]] = None, @@ -244,7 +244,7 @@ def fine_tune( create_file_logs: bool = True, create_loss_file: bool = True, write_weights: bool = False, - # scaling + # acceleration use_amp: bool = False, multi_gpu: bool = False, # plugins @@ -295,7 +295,7 @@ def fine_tune( create_file_logs=create_file_logs, create_loss_file=create_loss_file, write_weights=write_weights, - # scaling + # acceleration use_amp=use_amp, multi_gpu=multi_gpu, # plugins @@ -340,7 +340,7 @@ def train_custom( create_file_logs: bool = True, create_loss_file: bool = True, write_weights: bool = False, - # scaling + # acceleration use_amp: bool = False, multi_gpu: bool = False, # plugins From ffb8e297a70f77f57405dffaa5c61cdb79f6ab39 Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Fri, 8 Nov 2024 02:45:23 -0800 Subject: [PATCH 020/333] GH-3429: Add example to docs; Guard against corpus being different on each process --- examples/README.md | 5 +-- examples/multi_gpu/README.md | 32 +++++++++++++++++ examples/multi_gpu/__init__.py | 0 examples/multi_gpu/run_multi_gpu.py | 54 +++++++++++++++++++++++++++++ flair/data.py | 5 +++ flair/distributed_utils.py | 19 ++++++++++ flair/nn/model.py | 6 +++- flair/trainers/trainer.py | 30 ++++------------ tests/test_sentence.py | 10 ++++-- 9 files changed, 133 insertions(+), 28 deletions(-) create mode 100644 examples/multi_gpu/README.md create mode 100644 examples/multi_gpu/__init__.py create mode 100644 examples/multi_gpu/run_multi_gpu.py diff --git a/examples/README.md b/examples/README.md index 53e11ffe21..1221fbf49f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,6 +4,7 @@ This folder contains actively maintained examples of use of Flair, organized alo ## Table of Tasks -| Task | Documentation -| ----------------------------- | ------------- +| Task | Documentation +| ------------------------------ | ------------- | Named Entity Recognition (NER) | [Here](ner/) +| Multi GPU | [Here](multi_gpu/) diff --git a/examples/multi_gpu/README.md b/examples/multi_gpu/README.md new file mode 100644 index 0000000000..4c12b9d0bd --- /dev/null +++ b/examples/multi_gpu/README.md @@ -0,0 +1,32 @@ +# Multi GPU + +Training can be distributed across multiple GPUs on a local machine when using +[`ModelTrainer`](#flair.trainers.trainer.ModelTrainer). + +## Example + +See the script `run_multi_gpu.py` and its comments. + +## Tutorial + +There are 2 changes that are always required, as well as a few things to consider + +Always Required: +1) Pass the argument `multi_gpu=True` to your [`.train()`](#flair.trainers.trainer.ModelTrainer.train) or `.fine_tune()` +2) Wrap your code in [`launch_distributed`](#flair.distributed_utils.launch_distributed), e.g. + `launch_distributed(main, *args)`. This spawns multiple processes, each driving a GPU + +Other considerations: +- The corpus and other preprocessing must be the same on all processes. For example, if corpus initialization involves + anything random, you should either + - Set the random seed before initializing the corpus (e.g. [`flair.set_seed(42)`) OR + - Initialize the corpus before calling `launch_distributed` and pass the corpus as an argument so it's serialized to + all processes +- The effective batch size will be larger by a factor of num_gpus + - Each GPU will now process `mini_batch_size` examples before the optimizer steps, resulting in fewer total steps + taken relative to training with a single device. To obtain comparable results between single/multi gpu, + both mathematically, and in terms of wall time, consider the method in the example script. +- Large batch sizes may be necessary to see faster runs, otherwise the communication overhead may dominate + +Only the parameter updates in the training process will be distributed across multiple GPUs. Evaluation and prediction +are still done on a single device. diff --git a/examples/multi_gpu/__init__.py b/examples/multi_gpu/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/multi_gpu/run_multi_gpu.py b/examples/multi_gpu/run_multi_gpu.py new file mode 100644 index 0000000000..bba529b1b2 --- /dev/null +++ b/examples/multi_gpu/run_multi_gpu.py @@ -0,0 +1,54 @@ +import torch + +import flair +from flair.datasets import IMDB +from flair.distributed_utils import launch_distributed +from flair.embeddings import TransformerDocumentEmbeddings +from flair.models import TextClassifier +from flair.trainers import ModelTrainer + + +def main(multi_gpu): + # Note: Multi-GPU can affect corpus loading + # This code will run multiple times -- each GPU gets its own process and each process runs this code. We need to + # ensure that the corpus has the same elements and order on all processes, despite sampling. We do that by using + # the same seed on all processes. + flair.set_seed(1336) + + corpus = IMDB() + corpus.downsample(0.01) + label_type = "sentiment" + label_dictionary = corpus.make_label_dictionary(label_type) + + embeddings = TransformerDocumentEmbeddings(model="distilbert-base-uncased") + model = TextClassifier(embeddings, label_type, label_dictionary=label_dictionary) + + # Note: Multi-GPU can affect choice of batch size. + # In order to compare batch updates fairly between single and multi-GPU training, we should: + # 1) Step the optimizer after the same number of examples to achieve com + # 2) Process the same number of examples in each forward pass + mini_batch_chunk_size = 32 # Make this as large as possible without running out of GPU-memory to pack device + num_devices_when_distributing = max(torch.cuda.device_count(), 1) + mini_batch_size = mini_batch_chunk_size if multi_gpu else mini_batch_chunk_size * num_devices_when_distributing + # e.g. Suppose your machine has 2 GPUs. If multi_gpu=False, the first gpu will process 32 examples, then the + # first gpu will process another 32 examples, then the optimizer will step. If multi_gpu=True, each gpu will + # process 32 examples at the same time, then the optimizer will step. + + trainer = ModelTrainer(model, corpus) + trainer.train( + "resources/taggers/multi-gpu", + multi_gpu=multi_gpu, # Required for multi-gpu + max_epochs=2, + mini_batch_chunk_size=mini_batch_chunk_size, + mini_batch_size=mini_batch_size, + ) + + +if __name__ == "__main__": + """Minimal example demonstrating how to train a model on multiple GPUs.""" + multi_gpu = True + + if multi_gpu: + launch_distributed(main, multi_gpu) # Required for multi-gpu + else: + main(multi_gpu) diff --git a/flair/data.py b/flair/data.py index 56622b249c..ce97a4ce91 100644 --- a/flair/data.py +++ b/flair/data.py @@ -1072,6 +1072,11 @@ def __len__(self) -> int: def __repr__(self) -> str: return self.__str__() + def __eq__(self, o: object) -> bool: + if not isinstance(o, Sentence): + return False + return self.to_dict() == o.to_dict() + @property def start_position(self) -> int: return self._start_position diff --git a/flair/distributed_utils.py b/flair/distributed_utils.py index d391251b31..f3296eda58 100644 --- a/flair/distributed_utils.py +++ b/flair/distributed_utils.py @@ -1,5 +1,6 @@ import logging import os +import random from multiprocessing.connection import Connection from typing import Callable @@ -7,8 +8,10 @@ import torch import torch.multiprocessing as mp from torch.distributed import destroy_process_group, init_process_group +from torch.utils.data import Dataset import flair +from flair.data import Corpus, _len_dataset log = logging.getLogger("flair") @@ -66,3 +69,19 @@ def aggregate(value, aggregation_fn=np.mean): else: gathered_values = [value] return aggregation_fn(gathered_values) + + +def validate_corpus_same_across_processes(corpus: Corpus) -> None: + for dataset in [corpus.train, corpus.dev, corpus.test]: + if dataset is not None: + validate_dataset_same_across_processes(dataset) + + +def validate_dataset_same_across_processes(dataset: Dataset, sample_size: int = 10) -> None: + """Sanity checks a few examples to catch datasets that are obviously different, but not exhaustive to save time.""" + random_indices = random.sample(range(_len_dataset(dataset)), min(sample_size, _len_dataset(dataset))) + for i in random_indices: + example = dataset[i] + examples = aggregate(example, list) + if not all(example == examples[0] for example in examples): + raise ValueError("Dataset must be the same on each process") diff --git a/flair/nn/model.py b/flair/nn/model.py index f670c969a0..40761b5f52 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -126,7 +126,11 @@ def save(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: model_state["model_card"] = self.model_card # save model - torch.save(model_state, str(model_file), pickle_protocol=4) + if is_main_process(): + torch.save(model_state, str(model_file), pickle_protocol=4) + + if torch.distributed.is_initialized(): + torch.distributed.barrier() # Prevent any process from loading a model until writing is complete @classmethod def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "Model": diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 86a0ce3527..25c33491a1 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -21,7 +21,7 @@ import flair.nn from flair.data import Corpus, Dictionary, _len_dataset from flair.datasets import DataLoader -from flair.distributed_utils import aggregate, is_main_process +from flair.distributed_utils import aggregate, is_main_process, validate_corpus_same_across_processes from flair.samplers import FlairSampler from flair.trainers.plugins import ( AnnealingPlugin, @@ -496,6 +496,8 @@ def train_custom( if multi_gpu: if not torch.distributed.is_initialized(): raise RuntimeError("multi_gpu=True can only used inside flair.distributed_utils.launch_distributed()") + # Guard against each process initializing corpus differently due to e.g. different random seeds + validate_corpus_same_across_processes(self.corpus) self.ddp_model = DistributedDataParallel( self.model, device_ids=[flair.device.index], find_unused_parameters=True ) @@ -788,14 +790,14 @@ def wrapped_forward_loss(*args, **kwargs2): if save_best_model and current_epoch_has_best_model_so_far: log.info("saving best model") - self._save_model(base_path / "best-model.pt", checkpoint=save_optimizer_state) + self.model.save(base_path / "best-model.pt", checkpoint=save_optimizer_state) # - SWAPlugin -> restores SGD weights from SWA self.dispatch("after_training_loop") # if we do not use dev data for model selection, save final model if save_final_model: - self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) except KeyboardInterrupt: log_line(log) @@ -805,7 +807,7 @@ def wrapped_forward_loss(*args, **kwargs2): if save_final_model: log.info("Saving model ...") - self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) log.info("Done.") except TrainingInterrupt as exc: @@ -816,7 +818,7 @@ def wrapped_forward_loss(*args, **kwargs2): if save_final_model: log.info("Saving model ...") - self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) log.info("Done.") except Exception: @@ -957,23 +959,5 @@ def _initialize_model_card(self, **training_parameters): def _record(self, metric): self.dispatch("metric_recorded", metric) - def _save_model(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: - """Saves the current model. Safe to call from a distributed context. - - Args: - model_file: the model file - checkpoint: currently unused. - """ - if is_main_process(): - self.model.save(model_file, checkpoint) - - if torch.distributed.is_initialized(): - torch.distributed.barrier() # Prevent any process from loading a model until writing is complete - def _load_model(self, model_file: Union[str, Path]) -> None: - """Loads the model from the given file into the current state. Safe to call from a distributed context.""" self.model.load_state_dict(self.model.load(model_file).state_dict()) - if torch.distributed.is_initialized(): - self.ddp_model = DistributedDataParallel( - self.model, device_ids=[flair.device.index], find_unused_parameters=True - ) diff --git a/tests/test_sentence.py b/tests/test_sentence.py index 3e31422648..90f7b2f204 100644 --- a/tests/test_sentence.py +++ b/tests/test_sentence.py @@ -13,9 +13,15 @@ def test_sentence_context(): def test_equality(): assert Sentence("Guten Tag!") != Sentence("Good day!") assert Sentence("Guten Tag!", use_tokenizer=True) != Sentence("Guten Tag!", use_tokenizer=False) + sentence1 = Sentence("This sentence will be labeled") + sentence1[1].set_label("ner", "B-subject") + sentence2 = Sentence("This sentence will be labeled") + sentence2[1].set_label("ner", "B-object") + assert sentence1 != sentence2 - # TODO: is this desirable? Or should two sentences with same text be considered same objects? - assert Sentence("Guten Tag!") != Sentence("Guten Tag!") + assert Sentence("Guten Tag!") == Sentence("Guten Tag!") + sentence2[1].set_label("ner", "B-subject") + assert sentence1 == sentence2 def test_token_labeling(): From 9a82b0eb9cbd6ed65996f23e7d3f710b53b0b197 Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Thu, 14 Nov 2024 11:58:34 -0700 Subject: [PATCH 021/333] GH-3429: Remove serializability requirement --- examples/multi_gpu/run_multi_gpu.py | 6 +++--- flair/data.py | 5 ----- flair/distributed_utils.py | 11 ++++++----- flair/trainers/trainer.py | 7 +++---- tests/test_sentence.py | 10 ++-------- 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/examples/multi_gpu/run_multi_gpu.py b/examples/multi_gpu/run_multi_gpu.py index bba529b1b2..f7d059111b 100644 --- a/examples/multi_gpu/run_multi_gpu.py +++ b/examples/multi_gpu/run_multi_gpu.py @@ -13,10 +13,10 @@ def main(multi_gpu): # This code will run multiple times -- each GPU gets its own process and each process runs this code. We need to # ensure that the corpus has the same elements and order on all processes, despite sampling. We do that by using # the same seed on all processes. - flair.set_seed(1336) + flair.set_seed(42) corpus = IMDB() - corpus.downsample(0.01) + corpus.downsample(0.1) label_type = "sentiment" label_dictionary = corpus.make_label_dictionary(label_type) @@ -35,7 +35,7 @@ def main(multi_gpu): # process 32 examples at the same time, then the optimizer will step. trainer = ModelTrainer(model, corpus) - trainer.train( + trainer.fine_tune( "resources/taggers/multi-gpu", multi_gpu=multi_gpu, # Required for multi-gpu max_epochs=2, diff --git a/flair/data.py b/flair/data.py index ce97a4ce91..56622b249c 100644 --- a/flair/data.py +++ b/flair/data.py @@ -1072,11 +1072,6 @@ def __len__(self) -> int: def __repr__(self) -> str: return self.__str__() - def __eq__(self, o: object) -> bool: - if not isinstance(o, Sentence): - return False - return self.to_dict() == o.to_dict() - @property def start_position(self) -> int: return self._start_position diff --git a/flair/distributed_utils.py b/flair/distributed_utils.py index f3296eda58..e774084009 100644 --- a/flair/distributed_utils.py +++ b/flair/distributed_utils.py @@ -71,17 +71,18 @@ def aggregate(value, aggregation_fn=np.mean): return aggregation_fn(gathered_values) -def validate_corpus_same_across_processes(corpus: Corpus) -> None: +def validate_corpus_same_each_process(corpus: Corpus) -> None: + """Catches most cases in which a corpus is not the same on each process. However, there is no guarantee for two + reasons: 1) It uses a sample for speed 2) It compares strings to avoid requiring the datasets to be serializable""" for dataset in [corpus.train, corpus.dev, corpus.test]: if dataset is not None: - validate_dataset_same_across_processes(dataset) + _validate_dataset_same_each_process(dataset) -def validate_dataset_same_across_processes(dataset: Dataset, sample_size: int = 10) -> None: - """Sanity checks a few examples to catch datasets that are obviously different, but not exhaustive to save time.""" +def _validate_dataset_same_each_process(dataset: Dataset, sample_size: int = 10) -> None: random_indices = random.sample(range(_len_dataset(dataset)), min(sample_size, _len_dataset(dataset))) for i in random_indices: - example = dataset[i] + example = str(dataset[i]) examples = aggregate(example, list) if not all(example == examples[0] for example in examples): raise ValueError("Dataset must be the same on each process") diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 25c33491a1..37c63f5d33 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -7,8 +7,7 @@ import warnings from inspect import signature from pathlib import Path -from queue import Queue -from typing import Optional, Tuple, Type, Union +from typing import Optional, Union import numpy as np import torch @@ -21,7 +20,7 @@ import flair.nn from flair.data import Corpus, Dictionary, _len_dataset from flair.datasets import DataLoader -from flair.distributed_utils import aggregate, is_main_process, validate_corpus_same_across_processes +from flair.distributed_utils import aggregate, is_main_process, validate_corpus_same_each_process from flair.samplers import FlairSampler from flair.trainers.plugins import ( AnnealingPlugin, @@ -497,7 +496,7 @@ def train_custom( if not torch.distributed.is_initialized(): raise RuntimeError("multi_gpu=True can only used inside flair.distributed_utils.launch_distributed()") # Guard against each process initializing corpus differently due to e.g. different random seeds - validate_corpus_same_across_processes(self.corpus) + validate_corpus_same_each_process(self.corpus) self.ddp_model = DistributedDataParallel( self.model, device_ids=[flair.device.index], find_unused_parameters=True ) diff --git a/tests/test_sentence.py b/tests/test_sentence.py index 90f7b2f204..3e31422648 100644 --- a/tests/test_sentence.py +++ b/tests/test_sentence.py @@ -13,15 +13,9 @@ def test_sentence_context(): def test_equality(): assert Sentence("Guten Tag!") != Sentence("Good day!") assert Sentence("Guten Tag!", use_tokenizer=True) != Sentence("Guten Tag!", use_tokenizer=False) - sentence1 = Sentence("This sentence will be labeled") - sentence1[1].set_label("ner", "B-subject") - sentence2 = Sentence("This sentence will be labeled") - sentence2[1].set_label("ner", "B-object") - assert sentence1 != sentence2 - assert Sentence("Guten Tag!") == Sentence("Guten Tag!") - sentence2[1].set_label("ner", "B-subject") - assert sentence1 == sentence2 + # TODO: is this desirable? Or should two sentences with same text be considered same objects? + assert Sentence("Guten Tag!") != Sentence("Guten Tag!") def test_token_labeling(): From 952e9fa445464bcd70a654008b3ced5136376da9 Mon Sep 17 00:00:00 2001 From: Jeff Picard Date: Thu, 21 Nov 2024 16:38:59 -0700 Subject: [PATCH 022/333] GH-3429: Fix checkpointing --- flair/nn/model.py | 6 +----- flair/trainers/plugins/functional/checkpoints.py | 4 ++++ flair/trainers/trainer.py | 14 ++++++++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/flair/nn/model.py b/flair/nn/model.py index 40761b5f52..f670c969a0 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -126,11 +126,7 @@ def save(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: model_state["model_card"] = self.model_card # save model - if is_main_process(): - torch.save(model_state, str(model_file), pickle_protocol=4) - - if torch.distributed.is_initialized(): - torch.distributed.barrier() # Prevent any process from loading a model until writing is complete + torch.save(model_state, str(model_file), pickle_protocol=4) @classmethod def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "Model": diff --git a/flair/trainers/plugins/functional/checkpoints.py b/flair/trainers/plugins/functional/checkpoints.py index cf1a21468a..a8179edbc5 100644 --- a/flair/trainers/plugins/functional/checkpoints.py +++ b/flair/trainers/plugins/functional/checkpoints.py @@ -1,6 +1,8 @@ import logging from typing import Any +import torch + from flair.trainers.plugins.base import TrainerPlugin log = logging.getLogger("flair") @@ -28,6 +30,8 @@ def after_training_epoch(self, epoch, **kw): ) model_name = "model_epoch_" + str(epoch) + ".pt" self.model.save(self.base_path / model_name, checkpoint=self.save_optimizer_state) + if torch.distributed.is_initialized(): + torch.distributed.barrier() # Prevent any process from loading a model until writing is complete @property def attach_to_all_processes(self) -> bool: diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 37c63f5d33..cf3bb80eb6 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -789,14 +789,14 @@ def wrapped_forward_loss(*args, **kwargs2): if save_best_model and current_epoch_has_best_model_so_far: log.info("saving best model") - self.model.save(base_path / "best-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "best-model.pt", checkpoint=save_optimizer_state) # - SWAPlugin -> restores SGD weights from SWA self.dispatch("after_training_loop") # if we do not use dev data for model selection, save final model if save_final_model: - self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) except KeyboardInterrupt: log_line(log) @@ -806,7 +806,7 @@ def wrapped_forward_loss(*args, **kwargs2): if save_final_model: log.info("Saving model ...") - self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) log.info("Done.") except TrainingInterrupt as exc: @@ -817,7 +817,7 @@ def wrapped_forward_loss(*args, **kwargs2): if save_final_model: log.info("Saving model ...") - self.model.save(base_path / "final-model.pt", checkpoint=save_optimizer_state) + self._save_model(base_path / "final-model.pt", checkpoint=save_optimizer_state) log.info("Done.") except Exception: @@ -960,3 +960,9 @@ def _record(self, metric): def _load_model(self, model_file: Union[str, Path]) -> None: self.model.load_state_dict(self.model.load(model_file).state_dict()) + + def _save_model(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: + if is_main_process(): + self.model.save(model_file, checkpoint) + if torch.distributed.is_initialized(): + torch.distributed.barrier() # Prevent any process from loading a model until writing is complete From 83238458c44333a97e751925289cc4c94a21b575 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Fri, 22 Nov 2024 16:21:25 +0100 Subject: [PATCH 023/333] GH-3561: Update SECURITY.md with current contact --- SECURITY.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 5b9338fe9d..7c578c0b3f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,8 +1,4 @@ We acknowledge that every line of code that we write may potentially contain security issues. We are trying to deal with it responsibly and provide patches as quickly as possible. -We host our bug bounty program on HackerOne, it is currently private, therefore if you would like to report a vulnerability and get rewarded for it, please ask to join our program by filling this form: - -https://corporate.zalando.com/en/services-and-contact#security-form - -You can also send you report via this form if you do not want to join our bug bounty program and just want to report a vulnerability or security issue. +Please report any issues to [Alan Akbik](http://alanakbik.github.io/). From f8696b31e3b89432c251bb021b3128c820d0b7f9 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 22 Nov 2024 16:21:43 +0100 Subject: [PATCH 024/333] remove clustering support --- flair/models/__init__.py | 2 - flair/models/clustering.py | 120 --------------- resources/docs/TUTORIAL_12_CLUSTERING.md | 180 ----------------------- 3 files changed, 302 deletions(-) delete mode 100644 flair/models/clustering.py delete mode 100644 resources/docs/TUTORIAL_12_CLUSTERING.md diff --git a/flair/models/__init__.py b/flair/models/__init__.py index 8357cc47ea..e75daf074b 100644 --- a/flair/models/__init__.py +++ b/flair/models/__init__.py @@ -1,4 +1,3 @@ -from .clustering import ClusteringModel from .entity_linker_model import SpanClassifier from .entity_mention_linking import EntityMentionLinker from .language_model import LanguageModel @@ -37,6 +36,5 @@ "TARSTagger", "TextClassifier", "TextRegressor", - "ClusteringModel", "MultitaskModel", ] diff --git a/flair/models/clustering.py b/flair/models/clustering.py deleted file mode 100644 index e9902f6f67..0000000000 --- a/flair/models/clustering.py +++ /dev/null @@ -1,120 +0,0 @@ -import logging -import pickle -from collections import OrderedDict -from pathlib import Path -from typing import Optional, Union - -import joblib -from sklearn.base import BaseEstimator, ClusterMixin -from sklearn.metrics import normalized_mutual_info_score -from tqdm import tqdm - -from flair.data import Corpus, _iter_dataset -from flair.datasets import DataLoader -from flair.embeddings import DocumentEmbeddings - -log = logging.getLogger("flair") - - -class ClusteringModel: - """A wrapper class to apply sklearn clustering models on DocumentEmbeddings.""" - - def __init__(self, model: Union[ClusterMixin, BaseEstimator], embeddings: DocumentEmbeddings) -> None: - """Instantiate the ClusteringModel. - - Args: - model: the clustering algorithm from sklearn this wrapper will use. - embeddings: the flair DocumentEmbedding this wrapper uses to calculate a vector for each sentence. - """ - self.model = model - self.embeddings = embeddings - - def fit(self, corpus: Corpus, **kwargs): - """Trains the model. - - Args: - corpus: the flair corpus this wrapper will use for fitting the model. - **kwargs: parameters propagated to the models `.fit()` method. - """ - X = self._convert_dataset(corpus) - - log.info("Start clustering " + str(self.model) + " with " + str(len(X)) + " Datapoints.") - self.model.fit(X, **kwargs) - log.info("Finished clustering.") - - def predict(self, corpus: Corpus): - """Predict labels given a list of sentences and returns the respective class indices. - - Args: - corpus: the flair corpus this wrapper will use for predicting the labels. - """ - X = self._convert_dataset(corpus) - log.info("Start the prediction " + str(self.model) + " with " + str(len(X)) + " Datapoints.") - predict = self.model.predict(X) - - for idx, sentence in enumerate(_iter_dataset(corpus.get_all_sentences())): - sentence.set_label("cluster", str(predict[idx])) - - log.info("Finished prediction and labeled all sentences.") - return predict - - def save(self, model_file: Union[str, Path]): - """Saves current model. - - Args: - model_file: path where to save the model. - """ - joblib.dump(pickle.dumps(self), str(model_file)) - - log.info("Saved the model to: " + str(model_file)) - - @staticmethod - def load(model_file: Union[str, Path]): - """Loads a model from a given path. - - Args: - model_file: path to the file where the model is saved. - """ - log.info("Loading model from: " + str(model_file)) - return pickle.loads(joblib.load(str(model_file))) - - def _convert_dataset( - self, corpus, label_type: Optional[str] = None, batch_size: int = 32, return_label_dict: bool = False - ): - """Makes a flair-corpus sklearn compatible. - - Turns the corpora into X, y datasets as required for most sklearn clustering models. - Ref.: https://scikit-learn.org/stable/modules/classes.html#module-sklearn.cluster - """ - log.info("Embed sentences...") - sentences = [] - for batch in tqdm(DataLoader(corpus.get_all_sentences(), batch_size=batch_size)): - self.embeddings.embed(batch) - sentences.extend(batch) - - X = [sentence.embedding.cpu().detach().numpy() for sentence in sentences] - - if label_type is None: - return X - - labels = [sentence.get_labels(label_type)[0].value for sentence in sentences] - label_dict = {v: k for k, v in enumerate(OrderedDict.fromkeys(labels))} - y = [label_dict.get(label) for label in labels] - - if return_label_dict: - return X, y, label_dict - - return X, y - - def evaluate(self, corpus: Corpus, label_type: str): - """This method calculates some evaluation metrics for the clustering. - - Also, the result of the evaluation is logged. - - Args: - corpus: the flair corpus this wrapper will use for evaluation. - label_type: the label from the sentence will be used for the evaluation. - """ - X, Y = self._convert_dataset(corpus, label_type=label_type) - predict = self.model.predict(X) - log.info("NMI - Score: " + str(normalized_mutual_info_score(predict, Y))) diff --git a/resources/docs/TUTORIAL_12_CLUSTERING.md b/resources/docs/TUTORIAL_12_CLUSTERING.md deleted file mode 100644 index 376e5d5639..0000000000 --- a/resources/docs/TUTORIAL_12_CLUSTERING.md +++ /dev/null @@ -1,180 +0,0 @@ -Text Clustering in flair ----------- - -In this package text clustering is implemented. This module has the following -clustering algorithms implemented: -- k-Means -- BIRCH -- Expectation Maximization - -Each of the implemented algorithm needs to have an instanced DocumentEmbedding. This embedding will -transform each text/document to a vector. With these vectors the clustering algorithm can be performed. - ---------------------------- - -k-Means ------- -k-Means is a classical and well known clustering algorithm. k-Means is a partitioning-based Clustering algorithm. -The user defines with the parameter *k* how many clusters the given data has. -So the choice of *k* is very important. -More about k-Means can be read on the official [sklearn](https://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html). - - -```python -from flair.models import ClusteringModel -from flair.datasets import TREC_6 -from flair.embeddings import SentenceTransformerDocumentEmbeddings -from sklearn.cluster import KMeans - -embeddings = SentenceTransformerDocumentEmbeddings() - -# store all embeddings in memory which is required to perform clustering -corpus = TREC_6(memory_mode='full').downsample(0.05) - -model = KMeans(n_clusters=6) - -clustering_model = ClusteringModel( - model=model, - embeddings=embeddings -) - -# fit the model on a corpus -clustering_model.fit(corpus) - -# evaluate the model on a corpus with the given label -clustering_model.evaluate(corpus, label_type="question_class") -``` - -BIRCH ---------- -BIRCH (Balanced Iterative Reducing and Clustering using Hierarchies) is a hierarchical clustering algorithm. -BIRCH is specialized to handle large amounts of data. BIRCH scans the data a single time and builds an internal data -structure. This data structure contains the data but in a compressed way. -More about BIRCH can be read on the official [sklearn](https://scikit-learn.org/stable/modules/generated/sklearn.cluster.Birch.html). - -```python -from sklearn.cluster import Birch -from flair.datasets import TREC_6 -from flair.embeddings import SentenceTransformerDocumentEmbeddings -from flair.models import ClusteringModel - -embeddings = SentenceTransformerDocumentEmbeddings() - -# store all embeddings in memory which is required to perform clustering -corpus = TREC_6(memory_mode='full').downsample(0.05) - -model = Birch(n_clusters=6) - -clustering_model = ClusteringModel( - model=model, - embeddings=embeddings -) - -# fit the model on a corpus -clustering_model.fit(corpus) - -# evaluate the model on a corpus with the given label -clustering_model.evaluate(corpus, label_type="question_class") -``` - - -Expectation Maximization --------------------------- -Expectation Maximization (EM) is a different class of clustering algorithms called soft clustering algorithms. -Here each point isn't directly assigned to a cluster by a hard decision. -Each data point has a probability to which cluster the data point belongs. The Expectation Maximization (EM) -algorithm is a soft clustering algorithm. -More about EM can be read on the official [sklearn](https://scikit-learn.org/stable/modules/generated/sklearn.mixture.GaussianMixture.html). - - -```python -from sklearn.mixture import GaussianMixture -from flair.datasets import TREC_6 -from flair.embeddings import SentenceTransformerDocumentEmbeddings -from flair.models import ClusteringModel - -embeddings = SentenceTransformerDocumentEmbeddings() - -# store all embeddings in memory which is required to perform clustering -corpus = TREC_6(memory_mode='full').downsample(0.05) - -model = GaussianMixture(n_components=6) - -clustering_model = ClusteringModel( - model=model, - embeddings=embeddings -) - -# fit the model on a corpus -clustering_model.fit(corpus) - -# evaluate the model on a corpus with the given label -clustering_model.evaluate(corpus, label_type="question_class") -``` - ---------------------------- - -Loading/Saving the model ------------ - -The model can be saved and loaded. The code below shows how to save a model. -```python -from flair.models import ClusteringModel -from flair.datasets import TREC_6 -from flair.embeddings import SentenceTransformerDocumentEmbeddings -from sklearn.cluster import KMeans - -embeddings = SentenceTransformerDocumentEmbeddings() - -# store all embeddings in memory which is required to perform clustering -corpus = TREC_6(memory_mode='full').downsample(0.05) - -model = KMeans(n_clusters=6) - -clustering_model = ClusteringModel( - model=model, - embeddings=embeddings -) - -# fit the model on a corpus -clustering_model.fit(corpus) - -# save the model -clustering_model.save(model_file="clustering_model.pt") -``` - -The code for loading a model. - -````python -# load saved clustering model -model = ClusteringModel.load(model_file="clustering_model.pt") - -# load a corpus -corpus = TREC_6(memory_mode='full').downsample(0.05) - -# predict the corpus -model.predict(corpus) -```` - ---------------------- - -Evaluation ---------- -The result of the clustering can be evaluated. For this we will use the -[NMI](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.normalized_mutual_info_score.html). -(Normalized Mutual Info) score. - -````python -# need to fit() the model first -# evaluate the model on a corpus with the given label -clustering_model.evaluate(corpus, label_type="question_class") -```` - -The result of the evaluation can be seen below with the SentenceTransformerDocumentEmbeddings: - - -| Clustering Algorithm | Dataset | NMI | -|--------------------------|:-------------:|--------:| -| k Means | StackOverflow | ~0.2122 | -| BIRCH | StackOverflow | ~0,2424 | -| Expectation Maximization | 20News group | ~0,2222 | From 8f934a4c444ab53020a7b6daf9a99c5cee699228 Mon Sep 17 00:00:00 2001 From: Matt Buchovecky Date: Fri, 22 Nov 2024 12:15:37 -0800 Subject: [PATCH 025/333] perf: optimize dictionary items by providing function to check for presence of label in constant time, rather than requiring the creation of a list to be checked. update slow uses of get_items() to use this new function. --- flair/data.py | 14 +++++++------- flair/models/lemmatizer_model.py | 2 +- flair/nn/model.py | 4 +--- tests/test_corpus_dictionary.py | 18 +++++++++--------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/flair/data.py b/flair/data.py index 56622b249c..8a76a82c1a 100644 --- a/flair/data.py +++ b/flair/data.py @@ -122,18 +122,18 @@ def get_idx_for_items(self, items: list[str]) -> list[int]: return list(results) def get_items(self) -> list[str]: - items = [] - for item in self.idx2item: - items.append(item.decode("UTF-8")) - return items + return [item.decode("UTF-8") for item in self.idx2item] def __len__(self) -> int: return len(self.idx2item) - def get_item_for_index(self, idx): + def get_item_for_index(self, idx: int) -> str: return self.idx2item[idx].decode("UTF-8") - def set_start_stop_tags(self): + def has_item(self, item: str) -> bool: + return item.encode("utf-8") in self.item2idx + + def set_start_stop_tags(self) -> None: self.add_item("") self.add_item("") @@ -1659,7 +1659,7 @@ def make_label_dictionary( unked_count += count if len(label_dictionary.idx2item) == 0 or ( - len(label_dictionary.idx2item) == 1 and "" in label_dictionary.get_items() + len(label_dictionary.idx2item) == 1 and label_dictionary.has_item("") ): log.error(f"ERROR: You specified label_type='{label_type}' which is not in this dataset!") contained_labels = ", ".join( diff --git a/flair/models/lemmatizer_model.py b/flair/models/lemmatizer_model.py index 55fa34698c..16ec734941 100644 --- a/flair/models/lemmatizer_model.py +++ b/flair/models/lemmatizer_model.py @@ -491,7 +491,7 @@ def predict( for t_id, token in enumerate(tokens_in_batch): predicted_lemma = "".join( - self.char_dictionary.get_item_for_index(idx) if idx != self.end_index else "" + self.char_dictionary.get_item_for_index(int(idx)) if idx != self.end_index else "" for idx in predicted[t_id] ) token.set_label(typename=label_name, value=predicted_lemma) diff --git a/flair/nn/model.py b/flair/nn/model.py index f670c969a0..8f9d0aa09f 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -877,9 +877,7 @@ def predict( filtered_indices = [] has_unknown_label = False for idx, dp in enumerate(data_points): - if all( - label in self.label_dictionary.get_items() for label in self._get_label_of_datapoint(dp) - ): + if all(self.label_dictionary.has_item(label) for label in self._get_label_of_datapoint(dp)): filtered_indices.append(idx) else: has_unknown_label = True diff --git a/tests/test_corpus_dictionary.py b/tests/test_corpus_dictionary.py index cc65019b67..d107c0b62e 100644 --- a/tests/test_corpus_dictionary.py +++ b/tests/test_corpus_dictionary.py @@ -110,9 +110,9 @@ def test_tagged_corpus_make_vocab_dictionary(): vocab = corpus.make_vocab_dictionary(max_tokens=2, min_freq=-1) assert len(vocab) == 3 - assert "" in vocab.get_items() - assert "training" in vocab.get_items() - assert "." in vocab.get_items() + assert vocab.has_item("") + assert vocab.has_item("training") + assert vocab.has_item(".") vocab = corpus.make_vocab_dictionary(max_tokens=-1, min_freq=-1) @@ -121,9 +121,9 @@ def test_tagged_corpus_make_vocab_dictionary(): vocab = corpus.make_vocab_dictionary(max_tokens=-1, min_freq=2) assert len(vocab) == 3 - assert "" in vocab.get_items() - assert "training" in vocab.get_items() - assert "." in vocab.get_items() + assert vocab.has_item("") + assert vocab.has_item("training") + assert vocab.has_item(".") def test_label_set_confidence(): @@ -153,9 +153,9 @@ def test_tagged_corpus_make_label_dictionary(): label_dict = corpus.make_label_dictionary("label", add_unk=True) assert len(label_dict) == 3 - assert "" in label_dict.get_items() - assert "class_1" in label_dict.get_items() - assert "class_2" in label_dict.get_items() + assert label_dict.has_item("") + assert label_dict.has_item("class_1") + assert label_dict.has_item("class_2") with pytest.warns(DeprecationWarning): # test to make sure the warning comes, but function works corpus.make_tag_dictionary("label") From 1f831e08a03e8570af463f38c060a3ef68fac7ca Mon Sep 17 00:00:00 2001 From: MdMotahar Date: Wed, 1 May 2024 16:18:57 +0000 Subject: [PATCH 026/333] fixed _all_scores_for_token method to correctly calculate token probability --- flair/models/sequence_tagger_model.py | 2 +- tests/models/test_sequence_tagger.py | 35 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index 2c0fc00ebb..68e8e95b57 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -589,7 +589,7 @@ def _all_scores_for_token(self, sentences: list[Sentence], score_tensor: torch.T previous = 0 for length in lengths: prob_tags_per_sentence.append(prob_all_tags[previous : previous + length]) - previous = length + previous += length return prob_tags_per_sentence def _get_state_dict(self): diff --git a/tests/models/test_sequence_tagger.py b/tests/models/test_sequence_tagger.py index 67bf976899..13cc15ddc1 100644 --- a/tests/models/test_sequence_tagger.py +++ b/tests/models/test_sequence_tagger.py @@ -1,4 +1,6 @@ import pytest +import torch +import torch.nn.functional as F import flair from flair.embeddings import FlairEmbeddings, WordEmbeddings @@ -121,3 +123,36 @@ def test_train_load_use_tagger_disjunct_tags( loaded_model.predict([example_sentence, self.empty_sentence]) loaded_model.predict([self.empty_sentence]) del loaded_model + + @pytest.mark.integration() + def test_all_token_prob_distribution(self, embeddings, corpus): + tag_dictionary = corpus.make_label_dictionary("ner", add_unk=False) + model = self.build_model(embeddings, tag_dictionary) + + # get features from forward propagation + sentences = [corpus.train[i] for i in range(len(corpus.train))] + + # reverse sort all sequences by their length + sentences = sorted(sentences, key=len, reverse=True) + + with torch.no_grad(): + sentence_tensor, lengths = model._prepare_tensors(sentences) + features = model.forward(sentence_tensor, lengths) + + # remove previously predicted labels of this type + for sentence in sentences: + sentence.remove_labels(model.label_type) + + softmax_batch = F.softmax(features, dim=1).cpu() + lengths = [len(sentence) for sentence in sentences] + all_tokens_prob_distrib = model._all_scores_for_token(sentences, softmax_batch, lengths) + + for i, sen_tokens_prob_distribution in enumerate(all_tokens_prob_distrib): + assert len(sen_tokens_prob_distribution) == lengths[i] + for token_prob_distrib, token in zip(sen_tokens_prob_distribution, sentences[i]): + assert len(token_prob_distrib) == len(model.label_dictionary) + score_sum = 0.0 + for token_label in token_prob_distrib: + assert token_label.data_point == token + score_sum += token_label.score + assert abs(score_sum - 1.0) < 1.0e-5 From 2c053af584ea48ff1fb6a7d9f287ebb8598ab8dc Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 10:55:18 +0200 Subject: [PATCH 027/333] changes --- docs/_static/css/header.css | 89 +++++++++++++++++++++++++++ docs/_static/css/main.css | 118 ++++++++++++++++++++++++++++++++++++ docs/_static/flair_logo.svg | 101 ++++++++++++++++++++++++++++++ docs/conf.py | 10 ++- docs/glossary/index.rst | 7 --- 5 files changed, 317 insertions(+), 8 deletions(-) create mode 100644 docs/_static/css/header.css create mode 100644 docs/_static/css/main.css create mode 100755 docs/_static/flair_logo.svg delete mode 100644 docs/glossary/index.rst diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css new file mode 100644 index 0000000000..2c086e1a40 --- /dev/null +++ b/docs/_static/css/header.css @@ -0,0 +1,89 @@ +.bd-header { + width: 100%; + height: var(--header-height); + background: var(--white-blue) !important; + .header-wrapper { + margin: 0 calc(10% + 10px); + height: 100%; + display: flex; + justify-content: space-between; + nav { + height: 100%; + ul { + display: flex; + height: 100%; + li { + height: 100%; + display: flex; + align-items: center; + a { + font-size: 1.1rem; + color: var(--orange-white); + svg { + height: 50%; + flex-grow: 1; + } + &.header-logo { + height: 100%; + display: flex; + align-items: center; + } + &.header-hover-underline { + position: relative; + &::after { + transition: width 150ms cubic-bezier(.17,.67,0,1); + content: ""; + width: 0; + height: 3px; + background-color: var(--orange-white); + position: absolute; + left: 50%; + transform: translateX(-50%); + bottom: -1rem; + } + &:hover::after { + width: 1rem; + } + } + } + margin-right: 3em; + } + } + } + .header-controls { + height: 100%; + ul { + height: 100%; + display: flex; + li { + margin-left: 3rem; + height: 100%; + display: flex; + align-items: center; + &:not(:last-of-type) { + cursor: pointer; + } + &:nth-of-type(2) { + svg { + &:first-of-type { + :root .dark-mode & { + display: none; + } + } + &:nth-of-type(2) { + display: none; + :root .dark-mode & { + display: block; + } + } + } + } + svg { + height: 35%; + width: auto; + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css new file mode 100644 index 0000000000..0903efca63 --- /dev/null +++ b/docs/_static/css/main.css @@ -0,0 +1,118 @@ +@import url('https://fonts.googleapis.com/css2?family=Afacad:ital,wght@0,400..700;1,400..700&display=swap'); + +:root { + --flair-orange: #f79910; + --flair-orange-light: #ffc063; + --flair-orange-transparent: rgba(247, 153, 16, .5); + --white-transparent: rgba(255, 255, 255, .5); + --flair-blue: #2f2e41; + --white-blue: white; + --blue-white: var(--flair-blue); + --orange-white: var(--flair-orange); + --orange-white-transparent: var(--flair-orange-transparent); + --orange-blue: var(--flair-orange); + --blue-orange: var(--flair-blue); + --header-height: 90px; + --footer-height: 60px; + font-family: "Afacad", sans-serif; + font-optical-sizing: auto; + font-weight: 400; + font-size: 20px; + font-style: normal; +} + +:root[data-theme=dark] { + --white-blue: var(--flair-blue); + --blue-white: white; + --orange-white: white; + --orange-blue: var(--flair-blue); + --blue-orange: var(--flair-orange); + --orange-white-transparent: var(--white-transparent); +} + +.fill-white-blue { + fill: var(--white-blue); + stroke: none; +} + + +.fill-orange { + fill: var(--flair-orange); + stroke: none; +} + +.stroke-orange-white { + stroke: var(--orange-white); + fill: none; +} + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, +article, aside, canvas, details, embed, +figure, figcaption, footer, header, hgroup, +menu, nav, output, ruby, section, summary, +time, mark, audio, video { + margin: 0; + padding: 0; + border: 0; + font: inherit; + vertical-align: baseline; + line-height: 1.2em; +} + +button { + font-family: inherit; +} + +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} + +body { + line-height: 1; + background-color: var(--white-blue); +} + +main { + min-height: calc(100vh - var(--header-height) - var(--footer-height)); +} + +ol, ul { + list-style: none; +} + +blockquote, q { + quotes: none; +} + +table { + border-collapse: collapse; + border-spacing: 0; +} + +a { + color: var(--flair-orange); + text-decoration: none; + transition: color 200ms cubic-bezier(0,.35,.08,.89); + &:hover { + color: var(--flair-orange-light); + } +} + +::selection { + background: var(--flair-orange); + color: var(--white-blue); +} + +.inv-sel::selection { + background: var(--white-blue); + color: var(--flair-orange); +} \ No newline at end of file diff --git a/docs/_static/flair_logo.svg b/docs/_static/flair_logo.svg new file mode 100755 index 0000000000..e37b4f5a45 --- /dev/null +++ b/docs/_static/flair_logo.svg @@ -0,0 +1,101 @@ + + + + diff --git a/docs/conf.py b/docs/conf.py index 64624043e0..1664a3bb0f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -83,6 +83,14 @@ def linkcode_resolve(*args): # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +html_css_files = [ + 'css/main.css', + 'css/header.css', +] + +html_logo = "_static/flair_logo.svg" +html_show_sphinx = False + # Napoleon settings napoleon_include_init_with_doc = True napoleon_include_private_with_doc = True @@ -116,7 +124,7 @@ def linkcode_resolve(*args): smv_tag_whitelist = r"^v\d+\.\d+\.\d+$" # Whitelist pattern for branches (set to None to ignore all branches) -smv_branch_whitelist = r"^master$" +smv_branch_whitelist = r"^master|documentation$" # Whitelist pattern for remotes (set to None to use local branches only) smv_remote_whitelist = r"^origin$" diff --git a/docs/glossary/index.rst b/docs/glossary/index.rst deleted file mode 100644 index c732a1a121..0000000000 --- a/docs/glossary/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Glossary -======== - -.. glossary:: - - Sentence - a sentence is a text-unit consisting of tokens, labels and possibly metadata. Notice that a sentence is not limited in size, hence a Sentence itself could hold either a full document, a paragraph, a simple phrase or a linguistic \ No newline at end of file From 2cf6097bc3623cf91c3a5e9aa83a4ce56297ce81 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 10:58:03 +0200 Subject: [PATCH 028/333] changes --- docs/index.rst | 85 ++------------------------------------------------ 1 file changed, 2 insertions(+), 83 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 3cff769118..b7adde6b62 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,92 +4,11 @@ flair .. _flair_docs_mainpage: -**Version**: |version| - -**Useful links**: -`Getting started `_ | -`Source Repository `_ | -`Issue Tracker `_ | - -Flair is a very simple framework for state-of-the-art Natural Language Processing (NLP) - -.. grid:: 2 - - .. grid-item-card:: - :img-top: ./_static/tutorial.svg - - Tutorial - ^^^^^^^^ - - New to Flair? Check out the Tutorials. It contains an introduction to Flair's main concepts. - - +++ - - .. button-ref:: tutorial/index - :expand: - :color: secondary - :click-parent: - - To the tutorials - - .. grid-item-card:: - :img-top: ./_static/api.svg - - API-docs - ^^^^^^^^ - - The API-docs provides in-depth information on the classes and functions designed for public use. - - +++ - - .. button-ref:: api/index - :expand: - :color: secondary - :click-parent: - - To the API docs - - .. grid-item-card:: - :img-top: ./_static/contributing.svg - - Contributor's Guide - ^^^^^^^^^^^^^^^^^^^ - - Want to add to the codebase? Can help add to the - documentation? The contributing guidelines will guide you through the - process of improving Flair. - - +++ - - .. button-ref:: contributing/index - :expand: - :color: secondary - :click-parent: - - To the contributor's guide - - .. grid-item-card:: - :img-top: ./_static/glossary.svg - - Glossary - ^^^^^^^^ - - Not sure what the exact meaning of certain terms is? Find their definition in the Glossary. - - +++ - - .. button-ref:: glossary/index - :expand: - :color: secondary - :click-parent: - - To the glossary .. toctree:: :maxdepth: 3 :hidden: Tutorials - API reference - Contributing - Glossary \ No newline at end of file + API + Contributing \ No newline at end of file From 4354525c1d7b5991eaad685b048567a925d477f5 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:01:42 +0200 Subject: [PATCH 029/333] changes --- docs/_static/html/landing_page_header_styles.html | 5 +++++ docs/index.rst | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/_static/html/landing_page_header_styles.html diff --git a/docs/_static/html/landing_page_header_styles.html b/docs/_static/html/landing_page_header_styles.html new file mode 100644 index 0000000000..d2fa50bbfd --- /dev/null +++ b/docs/_static/html/landing_page_header_styles.html @@ -0,0 +1,5 @@ + \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index b7adde6b62..269af35b37 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,8 @@ flair .. _flair_docs_mainpage: - +.. raw:: html + :file: html/landing_page_header_styles.html .. toctree:: :maxdepth: 3 From 097e9f6ddd3743e57ac3935489abd31019a2a99a Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:03:41 +0200 Subject: [PATCH 030/333] changes --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 269af35b37..be769eca7a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ flair .. _flair_docs_mainpage: .. raw:: html - :file: html/landing_page_header_styles.html + :file: _static/html/landing_page_header_styles.html .. toctree:: :maxdepth: 3 From 0322720d861cfbf09e30623f2bb3c8f17108a5cb Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:15:11 +0200 Subject: [PATCH 031/333] changes --- docs/_static/css/header.css | 2 +- docs/_static/flair_logo.svg | 4 ++-- docs/_static/html/landing_page_header_styles.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 2c086e1a40..efb28a6436 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -1,7 +1,7 @@ .bd-header { width: 100%; height: var(--header-height); - background: var(--white-blue) !important; + background: var(--flair-orange) !important; .header-wrapper { margin: 0 calc(10% + 10px); height: 100%; diff --git a/docs/_static/flair_logo.svg b/docs/_static/flair_logo.svg index e37b4f5a45..aad189d1b8 100755 --- a/docs/_static/flair_logo.svg +++ b/docs/_static/flair_logo.svg @@ -89,12 +89,12 @@ d="m 224.21,3.67 c -0.28,0.04 -0.3,0.33 0.01,0.42 1.21,0.42 2.32,1.25 3.11,2.44 1.92,2.92 1.11,6.77 -1.88,8.18 -0.18,0.09 -0.38,-0.08 -0.32,-0.27 0.16,-0.45 0.25,-0.94 0.25,-1.45 0,-1.57 -0.85,-2.95 -2.11,-3.69 -0.18,-0.11 -0.39,0.08 -0.32,0.28 0.22,0.66 0.16,1.46 -0.23,2.18 -0.41,0.77 -0.75,1.55 -2.6,1.77 -1.61,0.22 -2.69,0.45 -4.08,1.46 -0.03,0.02 -0.05,0.04 -0.08,0.06 -0.26,0.19 -0.51,0.4 -0.75,0.61 -0.06,0.05 -0.12,0.11 -0.17,0.16 -0.21,0.2 -0.42,0.42 -0.62,0.64 -0.05,0.06 -0.11,0.12 -0.16,0.18 -0.21,0.26 -0.42,0.52 -0.61,0.8 -0.02,0.03 -0.04,0.05 -0.06,0.08 -0.21,0.31 -0.4,0.64 -0.58,0.98 -0.02,0.04 -0.04,0.09 -0.06,0.13 -0.14,0.28 -0.27,0.57 -0.39,0.87 -0.04,0.11 -0.08,0.22 -0.12,0.32 -0.09,0.24 -0.16,0.49 -0.23,0.74 -0.03,0.12 -0.07,0.25 -0.09,0.38 -0.06,0.26 -0.1,0.52 -0.14,0.78 -0.02,0.12 -0.04,0.23 -0.05,0.35 -0.04,0.38 -0.07,0.77 -0.07,1.16 0,3.95 2.21,7.37 5.43,9.02 0.05,0.03 0.12,0.06 0.19,0.1 0.17,0.08 0.34,0.17 0.52,0.24 1.82,0.79 3.83,1.23 5.94,1.23 3.49,0 6.7,-1.18 9.26,-3.17 3.55,-2.76 5.84,-7.07 5.84,-11.92 0,-8.23 -6.62,-14.92 -14.83,-15.06 z m 7.46,18.53 c -0.82,2.9 -3.19,4.93 -5.96,5.47 -0.97,0.27 -2.06,0.29 -3.16,-0.02 -0.08,-0.02 -0.15,-0.05 -0.23,-0.08 -0.07,-0.02 -0.13,-0.03 -0.2,-0.05 V 27.5 c -2.48,-0.9 -3.96,-3.24 -3.35,-5.4 0.65,-2.29 3.39,-3.51 6.13,-2.74 1.28,0.36 2.33,1.1 3.02,2.02 1.38,-0.59 2.48,-1.77 2.92,-3.32 0.16,-0.55 0.21,-1.1 0.19,-1.64 0.93,1.71 1.22,3.76 0.64,5.78 z" id="path21" inkscape:connector-curvature="0" - style="fill:#f79910" /> \ No newline at end of file From 7211f2c8945762fa7a030909a2950e36c84d5be9 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:22:27 +0200 Subject: [PATCH 032/333] changes --- docs/_static/{flair_logo.svg => flair_logo_white.svg} | 0 docs/_static/html/landing_page_header_styles.html | 6 ++++++ docs/conf.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) rename docs/_static/{flair_logo.svg => flair_logo_white.svg} (100%) diff --git a/docs/_static/flair_logo.svg b/docs/_static/flair_logo_white.svg similarity index 100% rename from docs/_static/flair_logo.svg rename to docs/_static/flair_logo_white.svg diff --git a/docs/_static/html/landing_page_header_styles.html b/docs/_static/html/landing_page_header_styles.html index e539264515..6b8883796d 100644 --- a/docs/_static/html/landing_page_header_styles.html +++ b/docs/_static/html/landing_page_header_styles.html @@ -1,5 +1,11 @@ \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 1664a3bb0f..929358d3af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,7 +88,7 @@ def linkcode_resolve(*args): 'css/header.css', ] -html_logo = "_static/flair_logo.svg" +html_logo = "_static/flair_logo_white.svg" html_show_sphinx = False # Napoleon settings From 33dbba3f743966f2ed16aca8ef38ccc30674367b Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:26:59 +0200 Subject: [PATCH 033/333] changes --- docs/_static/flair_logo_orange.svg | 101 ++++++++++++++++++ .../html/landing_page_header_styles.html | 4 +- 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100755 docs/_static/flair_logo_orange.svg diff --git a/docs/_static/flair_logo_orange.svg b/docs/_static/flair_logo_orange.svg new file mode 100755 index 0000000000..aad189d1b8 --- /dev/null +++ b/docs/_static/flair_logo_orange.svg @@ -0,0 +1,101 @@ + + + + diff --git a/docs/_static/html/landing_page_header_styles.html b/docs/_static/html/landing_page_header_styles.html index 6b8883796d..b9a064d87d 100644 --- a/docs/_static/html/landing_page_header_styles.html +++ b/docs/_static/html/landing_page_header_styles.html @@ -5,7 +5,9 @@ img { visibility: hidden; } - background: red; + background-image: url("../flair_logo_orange.svg"); + background-size: contain; + background-repeat: no-repeat; } } \ No newline at end of file From fb4ee203b5b5879ee30f21dd37fd125fde0fa4c3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:29:04 +0200 Subject: [PATCH 034/333] changes --- docs/_static/flair_logo_orange.svg | 4 ++-- docs/_static/html/landing_page_header_styles.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_static/flair_logo_orange.svg b/docs/_static/flair_logo_orange.svg index aad189d1b8..e37b4f5a45 100755 --- a/docs/_static/flair_logo_orange.svg +++ b/docs/_static/flair_logo_orange.svg @@ -89,12 +89,12 @@ d="m 224.21,3.67 c -0.28,0.04 -0.3,0.33 0.01,0.42 1.21,0.42 2.32,1.25 3.11,2.44 1.92,2.92 1.11,6.77 -1.88,8.18 -0.18,0.09 -0.38,-0.08 -0.32,-0.27 0.16,-0.45 0.25,-0.94 0.25,-1.45 0,-1.57 -0.85,-2.95 -2.11,-3.69 -0.18,-0.11 -0.39,0.08 -0.32,0.28 0.22,0.66 0.16,1.46 -0.23,2.18 -0.41,0.77 -0.75,1.55 -2.6,1.77 -1.61,0.22 -2.69,0.45 -4.08,1.46 -0.03,0.02 -0.05,0.04 -0.08,0.06 -0.26,0.19 -0.51,0.4 -0.75,0.61 -0.06,0.05 -0.12,0.11 -0.17,0.16 -0.21,0.2 -0.42,0.42 -0.62,0.64 -0.05,0.06 -0.11,0.12 -0.16,0.18 -0.21,0.26 -0.42,0.52 -0.61,0.8 -0.02,0.03 -0.04,0.05 -0.06,0.08 -0.21,0.31 -0.4,0.64 -0.58,0.98 -0.02,0.04 -0.04,0.09 -0.06,0.13 -0.14,0.28 -0.27,0.57 -0.39,0.87 -0.04,0.11 -0.08,0.22 -0.12,0.32 -0.09,0.24 -0.16,0.49 -0.23,0.74 -0.03,0.12 -0.07,0.25 -0.09,0.38 -0.06,0.26 -0.1,0.52 -0.14,0.78 -0.02,0.12 -0.04,0.23 -0.05,0.35 -0.04,0.38 -0.07,0.77 -0.07,1.16 0,3.95 2.21,7.37 5.43,9.02 0.05,0.03 0.12,0.06 0.19,0.1 0.17,0.08 0.34,0.17 0.52,0.24 1.82,0.79 3.83,1.23 5.94,1.23 3.49,0 6.7,-1.18 9.26,-3.17 3.55,-2.76 5.84,-7.07 5.84,-11.92 0,-8.23 -6.62,-14.92 -14.83,-15.06 z m 7.46,18.53 c -0.82,2.9 -3.19,4.93 -5.96,5.47 -0.97,0.27 -2.06,0.29 -3.16,-0.02 -0.08,-0.02 -0.15,-0.05 -0.23,-0.08 -0.07,-0.02 -0.13,-0.03 -0.2,-0.05 V 27.5 c -2.48,-0.9 -3.96,-3.24 -3.35,-5.4 0.65,-2.29 3.39,-3.51 6.13,-2.74 1.28,0.36 2.33,1.1 3.02,2.02 1.38,-0.59 2.48,-1.77 2.92,-3.32 0.16,-0.55 0.21,-1.1 0.19,-1.64 0.93,1.71 1.22,3.76 0.64,5.78 z" id="path21" inkscape:connector-curvature="0" - style="fill:#ffffff" /> \ No newline at end of file diff --git a/docs/_templates/landing-page-banner.html b/docs/_templates/landing-page-banner.html new file mode 100644 index 0000000000..8130a83d52 --- /dev/null +++ b/docs/_templates/landing-page-banner.html @@ -0,0 +1,353 @@ +
+ +
\ No newline at end of file diff --git a/docs/_templates/landing-page-illustrations.html b/docs/_templates/landing-page-illustrations.html new file mode 100644 index 0000000000..3b4069cb75 --- /dev/null +++ b/docs/_templates/landing-page-illustrations.html @@ -0,0 +1,30 @@ +
+
+ + + Easy to Use +

+ State-of-the-art NLP with just a few lines of code! Find entities, detect sentiment, and more. + Check out our demo! +

+
+
+ + + Huge Community +

+ With a community of ~200 contributors, Flair is used in hundreds of companies, + over 2,000 open source projects, and + 2,000+ papers! +

+
+
+ + + Open Source and Free +

+ Flair is completely free and open source, making it accessible for everyone to use + and report issues. +

+
+
\ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index be769eca7a..6a6d6b795f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,6 +6,12 @@ flair .. raw:: html :file: _static/html/landing_page_header_styles.html +.. raw:: html + :file: _templates/landing-page-banner.html + +.. raw:: html + :file: _templates/landing-page-illustrations.html + .. toctree:: :maxdepth: 3 :hidden: From c92f1b5e48cdad5759c8122c8c0a65a634a6e71a Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:36:33 +0200 Subject: [PATCH 036/333] changes --- docs/_static/css/landing-page-banner.css | 71 ----------- .../css/landing-page-illustrations.css | 38 ------ docs/_static/html/landing_page_styles.html | 111 +++++++++++++++++- 3 files changed, 109 insertions(+), 111 deletions(-) delete mode 100644 docs/_static/css/landing-page-banner.css delete mode 100644 docs/_static/css/landing-page-illustrations.css diff --git a/docs/_static/css/landing-page-banner.css b/docs/_static/css/landing-page-banner.css deleted file mode 100644 index ae3a78525c..0000000000 --- a/docs/_static/css/landing-page-banner.css +++ /dev/null @@ -1,71 +0,0 @@ -.landing-page { - .banner { - display: flex; - height: 510px; - background: var(--flair-orange); - .left-side { - align-items: flex-start; - display: flex; - flex-direction: column; - height: 100%; - justify-content: center; - width: 50%; - padding-left: 10%; - padding-right: 10%; - box-sizing: border-box; - svg { - height: 20%; - width: auto; - } - & > span { - font-size: 1.5rem; - margin: 1.3em 0; - line-height: 1.2em; - color: var(--white-blue); - } - #get-started { - background: var(--white-blue); - border: none; - padding: 0.2em 0.9em; - font-size: 1.5rem; - color: var(--blue-white); - font-weight: 600; - border-radius: .5em; - transition: transform 200ms cubic-bezier(0,.35,.08,.89); - &:hover { - transform: scale(1.1); - } - } - } - .right-side { - display: flex; - justify-content: flex-start; - align-items: center; - width: 50%; - .card { - margin-left: 3rem; - border-radius: 1rem; - width: 20%; - display: flex; - justify-content: center; - align-items: center; - flex-direction: column; - transition: transform 200ms cubic-bezier(0,.35,.08,.89); - background: var(--white-blue); - color: var(--blue-white); - min-height: 42%; - &:hover { - transform: scale(1.1); - } - svg { - height: 40%; - width: auto; - } - & > span { - color: var(--blue-white); - margin-top: 1em; - } - } - } - } -} \ No newline at end of file diff --git a/docs/_static/css/landing-page-illustrations.css b/docs/_static/css/landing-page-illustrations.css deleted file mode 100644 index 28c58dcb7a..0000000000 --- a/docs/_static/css/landing-page-illustrations.css +++ /dev/null @@ -1,38 +0,0 @@ -.landing-page-illustrations { - display: flex; - padding: 5rem; - background: var(--white-blue); - .item { - flex: 1 1 0; - display: flex; - flex-direction: column; - justify-content: flex-start; - align-items: flex-start; - padding: 0 5%; - &:not(:last-of-type) { - border-right: #ccc 2px solid; - } - svg { - height: 12vw; - width: auto; - & + span { - margin-top: 2.5em; - margin-bottom: 0.75em; - font-size: 1.5rem; - font-weight: 600; - } - &:nth-of-type(2) { - display: none; - :root .dark-mode & { - display: block; - } - } - :root .dark-mode & { - display: none; - } - } - span, p { - color: var(--blue-white); - } - } -} \ No newline at end of file diff --git a/docs/_static/html/landing_page_styles.html b/docs/_static/html/landing_page_styles.html index c9ca7a8e5c..2cfc618bdc 100644 --- a/docs/_static/html/landing_page_styles.html +++ b/docs/_static/html/landing_page_styles.html @@ -1,6 +1,4 @@ \ No newline at end of file From 72fa60296d292e0cd22f0b3c7651e018be4fb6fe Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:38:40 +0200 Subject: [PATCH 037/333] changes --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 6a6d6b795f..47876aea55 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ flair .. _flair_docs_mainpage: .. raw:: html - :file: _static/html/landing_page_header_styles.html + :file: _static/html/landing_page_styles.html .. raw:: html :file: _templates/landing-page-banner.html From dbdf9d247ac6c4dea878c1aca2c238813283f55d Mon Sep 17 00:00:00 2001 From: konstantin Date: Sun, 4 Aug 2024 11:44:42 +0200 Subject: [PATCH 038/333] changes --- docs/_static/html/landing_page_styles.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/_static/html/landing_page_styles.html b/docs/_static/html/landing_page_styles.html index 2cfc618bdc..fe99d482d7 100644 --- a/docs/_static/html/landing_page_styles.html +++ b/docs/_static/html/landing_page_styles.html @@ -1,4 +1,10 @@ \ No newline at end of file From da3ee908aaa0cecc8ab64420f49558471014afc6 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 12:34:33 +0200 Subject: [PATCH 060/333] changes --- docs/_static/html/landing_page_styles.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/html/landing_page_styles.html b/docs/_static/html/landing_page_styles.html index 2a42b60e9d..35661ab101 100644 --- a/docs/_static/html/landing_page_styles.html +++ b/docs/_static/html/landing_page_styles.html @@ -159,7 +159,7 @@ a.list-group-item { background: var(--white-blue); &:hover { - background: var(--white-transparent); + background: var(--orange-white-transparent); color: var(--orange-white); } span { From 4960ad748c8b1bac3aa6f647896038083617a950 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 12:35:45 +0200 Subject: [PATCH 061/333] changes --- docs/_static/html/landing_page_styles.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/html/landing_page_styles.html b/docs/_static/html/landing_page_styles.html index 35661ab101..9871541a87 100644 --- a/docs/_static/html/landing_page_styles.html +++ b/docs/_static/html/landing_page_styles.html @@ -160,7 +160,7 @@ background: var(--white-blue); &:hover { background: var(--orange-white-transparent); - color: var(--orange-white); + color: var(--white-blue); } span { color: var(--orange-white); From ac06833edba1f4582882d3a574aee6e8c13115dd Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 12:38:07 +0200 Subject: [PATCH 062/333] changes --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 112a1aec01..dc117caf74 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,7 +27,7 @@ } # dummy value that sphinx-github-style won't crash when run in temp folder. html_theme_options = { - "navbar_end": ["theme-switcher", "version-switcher", "navbar-icon-links"] + "navbar_end": ["version-switcher", "navbar-icon-links"] } From ebf02556ab91cfb7b21dc8ee9921bbe065bc3557 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 12:40:58 +0200 Subject: [PATCH 063/333] changes --- docs/_templates/darkmode-toggle.html | 138 +++++++++++++++++++++++++++ docs/conf.py | 2 +- 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 docs/_templates/darkmode-toggle.html diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html new file mode 100644 index 0000000000..9610954b69 --- /dev/null +++ b/docs/_templates/darkmode-toggle.html @@ -0,0 +1,138 @@ +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ diff --git a/docs/conf.py b/docs/conf.py index dc117caf74..f24ec214ac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,7 +27,7 @@ } # dummy value that sphinx-github-style won't crash when run in temp folder. html_theme_options = { - "navbar_end": ["version-switcher", "navbar-icon-links"] + "navbar_end": ["darkmode-toggle", "version-switcher", "navbar-icon-links"] } From 665d495ebd06d18036a910d99365388547744549 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 12:47:38 +0200 Subject: [PATCH 064/333] changes --- docs/_static/css/main.css | 18 +++++++++++++++++- docs/_templates/darkmode-toggle.html | 5 ----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index 5c28405bbf..67b489b6ae 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -21,7 +21,7 @@ font-style: normal; } -:root[data-theme=dark] { +:root.dark-mode { --white-blue: var(--flair-blue); --blue-white: white; --orange-white: white; @@ -119,4 +119,20 @@ a { .inv-sel::selection { background: var(--white-blue); color: var(--flair-orange); +} + +.fill-white-blue { + fill: var(--white-blue); + stroke: none; +} + + +.fill-orange { + fill: var(--flair-orange); + stroke: none; +} + +.stroke-orange-white { + stroke: var(--orange-white); + fill: none; } \ No newline at end of file diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index 9610954b69..a5851fb072 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -117,11 +117,6 @@ id="circle1" style="stroke:#ffffff;stroke-width:0.50868;stroke-dasharray:none;stroke-opacity:1" /> - + From d9314fad3fb42cd33da8de9929ba4682bc8dd936 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 13:28:44 +0200 Subject: [PATCH 070/333] changes --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index f24ec214ac..8352d92de1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,7 +27,8 @@ } # dummy value that sphinx-github-style won't crash when run in temp folder. html_theme_options = { - "navbar_end": ["darkmode-toggle", "version-switcher", "navbar-icon-links"] + "navbar_end": ["darkmode-toggle", "version-switcher", "navbar-icon-links"], + "show_prev_next": False } From 711baafc0f1fc0ab3bd0dfab5eba12ac1e5560ed Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 13:33:43 +0200 Subject: [PATCH 071/333] changes --- docs/_static/css/header.css | 112 ++++++++++++++++++++++++++ docs/_templates/version-switcher.html | 2 +- 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 55fc362886..16157e3360 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -3,6 +3,19 @@ height: var(--header-height); background: var(--flair-orange) !important; box-shadow: none; + .bd-header__inner { + margin: 0 10%; + padding: 0; + height: 100%; + display: flex; + justify-content: space-between; + .navbar-header-items__start { + padding: 0; + height: 55%;.bd-header { + width: 100%; + height: var(--header-height); + background: var(--flair-orange) !important; + box-shadow: none; .bd-header__inner { margin: 0 10%; padding: 0; @@ -69,6 +82,105 @@ } } } + .navbar-header-items__end { + + .navbar-item:first-of-type { + button.search-button { + background-size: contain; + background-repeat: no-repeat; + background-position: center; + border-radius: 0; + height: 1.75rem; + width: 1.75rem; + i { + visibility: hidden; + &:focus, &, &:active { + border: none; + outline: none; + } + } + } + } + .navbar-item:nth-of-type(2) { + svg { + cursor: pointer; + &:first-of-type { + :root .dark-mode & { + display: none; + } + } + &:nth-of-type(2) { + display: none; + :root .dark-mode & { + display: block; + } + } + } + } + svg { + height: 35%; + width: auto; + } + } + } +} + .navbar-item { + height: 100%; + .navbar-brand.logo { + height: 100%; + padding: 0; + display: block; + .logo__image { + height: 100%; + width: auto; + } + } + } + } + nav { + height: 100%; + ul.bd-navbar-elements { + display: flex; + height: 100%; + li.nav-item { + height: auto; + display: flex; + align-items: center; + margin-right: 0; + margin-left: 3rem; + + &.active a { + font-weight: inherit; + &::after { + width: 100%; + } + } + a { + position: relative; + font-size: 1.1rem; + padding: 0; + color: white; + &:hover { + color: white; + &::after { + width: 100%; + } + } + &::after { + transition: width 200ms cubic-bezier(0,.35,.08,.89); + content: ""; + width: 0; + height: 3px; + background-color: white; + position: absolute; + bottom: -.5rem; + left: 50%; + transform: translateX(-50%); + } + } + } + } + } .navbar-header-items__end { .navbar-item:first-of-type { diff --git a/docs/_templates/version-switcher.html b/docs/_templates/version-switcher.html index 6b598954e5..a90f974b89 100644 --- a/docs/_templates/version-switcher.html +++ b/docs/_templates/version-switcher.html @@ -43,7 +43,7 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { - background-image: url("_static/magnifying_glass.svg"); + background-image: url("{{ pathto(_static/magnifying_glass.svg) }}"); } } } From cc658b1874264ea22830b314b175b6ffd9d691dd Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 13:37:03 +0200 Subject: [PATCH 072/333] changes --- docs/_templates/version-switcher.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/version-switcher.html b/docs/_templates/version-switcher.html index a90f974b89..85f7f7efc1 100644 --- a/docs/_templates/version-switcher.html +++ b/docs/_templates/version-switcher.html @@ -43,7 +43,7 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { - background-image: url("{{ pathto(_static/magnifying_glass.svg) }}"); + background-image: url("{{ pathto(magnifying_glass.svg) }}"); } } } From cc1f12c52a72ded58603d8c663afd28be3e8972e Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 13:38:17 +0200 Subject: [PATCH 073/333] changes --- docs/_templates/version-switcher.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/version-switcher.html b/docs/_templates/version-switcher.html index 85f7f7efc1..b033e82285 100644 --- a/docs/_templates/version-switcher.html +++ b/docs/_templates/version-switcher.html @@ -43,7 +43,7 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { - background-image: url("{{ pathto(magnifying_glass.svg) }}"); + background-image: url("{{ pathto(../_static/magnifying_glass.svg) }}"); } } } From c7ec7f113ac683d2261f3459926e91ed9ad498af Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 13:52:03 +0200 Subject: [PATCH 074/333] changes --- docs/_static/css/header.css | 113 --------------------- docs/_static/html/landing_page_styles.html | 15 ++- docs/_static/magnifying_glass.svg | 10 +- docs/_static/magnifying_glass_dark.svg | 56 ++++++++++ docs/_templates/version-switcher.html | 2 +- 5 files changed, 76 insertions(+), 120 deletions(-) create mode 100644 docs/_static/magnifying_glass_dark.svg diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 16157e3360..b927575b82 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -3,19 +3,6 @@ height: var(--header-height); background: var(--flair-orange) !important; box-shadow: none; - .bd-header__inner { - margin: 0 10%; - padding: 0; - height: 100%; - display: flex; - justify-content: space-between; - .navbar-header-items__start { - padding: 0; - height: 55%;.bd-header { - width: 100%; - height: var(--header-height); - background: var(--flair-orange) !important; - box-shadow: none; .bd-header__inner { margin: 0 10%; padding: 0; @@ -123,104 +110,4 @@ } } } -} - .navbar-item { - height: 100%; - .navbar-brand.logo { - height: 100%; - padding: 0; - display: block; - .logo__image { - height: 100%; - width: auto; - } - } - } - } - nav { - height: 100%; - ul.bd-navbar-elements { - display: flex; - height: 100%; - li.nav-item { - height: auto; - display: flex; - align-items: center; - margin-right: 0; - margin-left: 3rem; - - &.active a { - font-weight: inherit; - &::after { - width: 100%; - } - } - a { - position: relative; - font-size: 1.1rem; - padding: 0; - color: white; - &:hover { - color: white; - &::after { - width: 100%; - } - } - &::after { - transition: width 200ms cubic-bezier(0,.35,.08,.89); - content: ""; - width: 0; - height: 3px; - background-color: white; - position: absolute; - bottom: -.5rem; - left: 50%; - transform: translateX(-50%); - } - } - } - } - } - .navbar-header-items__end { - - .navbar-item:first-of-type { - button.search-button { - background-size: contain; - background-repeat: no-repeat; - background-position: center; - border-radius: 0; - background-color: red; - height: 55%; - width: auto; - i { - visibility: hidden; - &:focus, &, &:active { - border: none; - outline: none; - } - } - } - } - .navbar-item:nth-of-type(2) { - svg { - cursor: pointer; - &:first-of-type { - :root .dark-mode & { - display: none; - } - } - &:nth-of-type(2) { - display: none; - :root .dark-mode & { - display: block; - } - } - } - } - svg { - height: 35%; - width: auto; - } - } - } } \ No newline at end of file diff --git a/docs/_static/html/landing_page_styles.html b/docs/_static/html/landing_page_styles.html index 014a148462..2815b9573a 100644 --- a/docs/_static/html/landing_page_styles.html +++ b/docs/_static/html/landing_page_styles.html @@ -172,4 +172,17 @@ } } } - \ No newline at end of file + .bd-header { + .bd-header__inner { + .navbar-header-items__end { + .navbar-item:first-of-type { + button.search-button { + :root .dark-mode & { + background-image: url("{{ pathto('_static/magnifying_glass_dark.svg', 1) }}"); + } + } + } + } + } + } + diff --git a/docs/_static/magnifying_glass.svg b/docs/_static/magnifying_glass.svg index 5350d0e540..3048ce30c6 100644 --- a/docs/_static/magnifying_glass.svg +++ b/docs/_static/magnifying_glass.svg @@ -14,7 +14,7 @@ xmlns:svg="http://www.w3.org/2000/svg"> + style="fill:none;stroke:#f79910;stroke-opacity:1"> + style="fill:none;stroke:#f79910;stroke-opacity:1"> + style="fill:none;stroke:#f79910;stroke-opacity:1" /> + style="fill:none;stroke:#f79910;stroke-opacity:1" /> diff --git a/docs/_static/magnifying_glass_dark.svg b/docs/_static/magnifying_glass_dark.svg new file mode 100644 index 0000000000..5350d0e540 --- /dev/null +++ b/docs/_static/magnifying_glass_dark.svg @@ -0,0 +1,56 @@ + + + + + + + + + diff --git a/docs/_templates/version-switcher.html b/docs/_templates/version-switcher.html index b033e82285..a908b0044e 100644 --- a/docs/_templates/version-switcher.html +++ b/docs/_templates/version-switcher.html @@ -43,7 +43,7 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { - background-image: url("{{ pathto(../_static/magnifying_glass.svg) }}"); + background-image: url("{{ pathto('_static/magnifying_glass.svg', 1) }}"); } } } From 413317d318dfaf41530da337a0900a3e494d5c56 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 13:54:32 +0200 Subject: [PATCH 075/333] changes --- docs/_static/html/landing_page_styles.html | 13 ------------- docs/index.rst | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/_static/html/landing_page_styles.html b/docs/_static/html/landing_page_styles.html index 2815b9573a..0b43d7bb72 100644 --- a/docs/_static/html/landing_page_styles.html +++ b/docs/_static/html/landing_page_styles.html @@ -172,17 +172,4 @@ } } } - .bd-header { - .bd-header__inner { - .navbar-header-items__end { - .navbar-item:first-of-type { - button.search-button { - :root .dark-mode & { - background-image: url("{{ pathto('_static/magnifying_glass_dark.svg', 1) }}"); - } - } - } - } - } - } diff --git a/docs/index.rst b/docs/index.rst index c8ddac1c9f..08241759ef 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,24 @@ .. raw:: html :file: _templates/landing-page-illustrations.html +.. raw:: html + + + .. toctree:: :maxdepth: 3 :hidden: From 7335d3dfd1ffb4d48ef9701a603838e659a1afdc Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 13:56:48 +0200 Subject: [PATCH 076/333] changes --- .../landing_page_styles.html | 13 ++++++++++++ docs/index.rst | 20 +------------------ 2 files changed, 14 insertions(+), 19 deletions(-) rename docs/{_static/html => _templates}/landing_page_styles.html (92%) diff --git a/docs/_static/html/landing_page_styles.html b/docs/_templates/landing_page_styles.html similarity index 92% rename from docs/_static/html/landing_page_styles.html rename to docs/_templates/landing_page_styles.html index 0b43d7bb72..3c9efe3ce1 100644 --- a/docs/_static/html/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -172,4 +172,17 @@ } } } + .bd-header { + .bd-header__inner { + .navbar-header-items__end { + .navbar-item:first-of-type { + button.search-button { + :root .dark-mode & { + background-image: url("{{ pathto('_static/magnifying_glass_dark.svg', 1) }}"); + } + } + } + } + } + } diff --git a/docs/index.rst b/docs/index.rst index 08241759ef..c39010a2c7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,7 @@ .. title:: Home .. raw:: html - :file: _static/html/landing_page_styles.html + :file: _templates/landing_page_styles.html .. raw:: html :file: _templates/landing-page-banner.html @@ -11,24 +11,6 @@ .. raw:: html :file: _templates/landing-page-illustrations.html -.. raw:: html - - - .. toctree:: :maxdepth: 3 :hidden: From 82c73beab26207a89b14a976f0992d61e8a19cde Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 14:01:48 +0200 Subject: [PATCH 077/333] changes --- docs/_templates/landing_page_styles.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 3c9efe3ce1..d9bf936818 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -178,7 +178,7 @@ .navbar-item:first-of-type { button.search-button { :root .dark-mode & { - background-image: url("{{ pathto('_static/magnifying_glass_dark.svg', 1) }}"); + background-image: url("_static/magnifying_glass_dark.svg"); } } } From 081db63cb8a5166f4d19a1c87bfa086cfee7b92d Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 14:04:50 +0200 Subject: [PATCH 078/333] changes --- docs/_templates/version-switcher.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/version-switcher.html b/docs/_templates/version-switcher.html index a908b0044e..9fb4bc511e 100644 --- a/docs/_templates/version-switcher.html +++ b/docs/_templates/version-switcher.html @@ -43,7 +43,7 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { - background-image: url("{{ pathto('_static/magnifying_glass.svg', 1) }}"); + background-image: url("{{ pathto('_static/magnifying_glass_dark.svg', 1) }}"); } } } From 0de42e8804d18ef30e32f368f39890e912a2a70a Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 14:07:23 +0200 Subject: [PATCH 079/333] changes --- docs/_templates/landing_page_styles.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index d9bf936818..ca4c834d8b 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -177,6 +177,7 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { + background-image: url("{{ pathto('_static/magnifying_glass.svg', 1) }}"); :root .dark-mode & { background-image: url("_static/magnifying_glass_dark.svg"); } From 056bdfffa04b01b7819e67b696c55ca193b737f4 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 14:07:39 +0200 Subject: [PATCH 080/333] changes --- docs/_templates/landing_page_styles.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index ca4c834d8b..f88a033b3f 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -177,7 +177,7 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { - background-image: url("{{ pathto('_static/magnifying_glass.svg', 1) }}"); + background-image: url("_static/magnifying_glass.svg"); :root .dark-mode & { background-image: url("_static/magnifying_glass_dark.svg"); } From 1d5f3c73bb738965b8e5274e5963e7d28ee610a1 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 14:13:07 +0200 Subject: [PATCH 081/333] changes --- docs/_static/css/header.css | 15 ++++++++++++ docs/_static/css/main.css | 16 ------------- docs/_templates/landing_page_styles.html | 30 ++++++++++++++---------- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index b927575b82..6ef02d0faf 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -110,4 +110,19 @@ } } } +} + +.fill-white-blue { + fill: var(--white-blue); + stroke: none; +} + +.fill-orange { + fill: var(--flair-orange); + stroke: none; +} + +.stroke-moon { + stroke: white; + fill: none; } \ No newline at end of file diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index 1b5007b26e..1c24bfc662 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -119,20 +119,4 @@ a { .inv-sel::selection { background: var(--white-blue); color: var(--flair-orange); -} - -.fill-white-blue { - fill: var(--white-blue); - stroke: none; -} - - -.fill-orange { - fill: var(--flair-orange); - stroke: none; -} - -.stroke-orange-white { - stroke: var(--orange-white); - fill: none; } \ No newline at end of file diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index f88a033b3f..b9fcebe573 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -173,17 +173,21 @@ } } .bd-header { - .bd-header__inner { - .navbar-header-items__end { - .navbar-item:first-of-type { - button.search-button { - background-image: url("_static/magnifying_glass.svg"); - :root .dark-mode & { - background-image: url("_static/magnifying_glass_dark.svg"); - } - } - } - } - } - } + .bd-header__inner { + .navbar-header-items__end { + .navbar-item:first-of-type { + button.search-button { + background-image: url("_static/magnifying_glass.svg"); + :root .dark-mode & { + background-image: url("_static/magnifying_glass_dark.svg"); + } + } + } + } + } + } + .stroke-moon { + stroke: var(--orange-white); + fill: none; + } From 9c25250143c9a258c4da6568bbac1bd24a80cfc3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 14:13:30 +0200 Subject: [PATCH 082/333] changes --- docs/_templates/darkmode-toggle.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index a5851fb072..da11101150 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -1,7 +1,7 @@
- + Date: Wed, 7 Aug 2024 14:15:47 +0200 Subject: [PATCH 083/333] changes --- docs/_static/css/header.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 6ef02d0faf..3011bab05c 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -70,7 +70,9 @@ } } .navbar-header-items__end { - + .navbar-item:not(:last-of-type) { + margin-right: 2rem; + } .navbar-item:first-of-type { button.search-button { background-size: contain; From 7a74d52f5abefe8cd036d41502efc6559d72a6b2 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 15:00:37 +0200 Subject: [PATCH 084/333] changes --- docs/_static/css/footer.css | 12 ++++++++++++ docs/conf.py | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 docs/_static/css/footer.css diff --git a/docs/_static/css/footer.css b/docs/_static/css/footer.css new file mode 100644 index 0000000000..a28e39b799 --- /dev/null +++ b/docs/_static/css/footer.css @@ -0,0 +1,12 @@ +.bd-footer { + border: none; + background: var(--blue-orange); + .bd-footer__inner { + padding: 1rem 10%; + margin: 0; + box-sizing: border-box; + .footer-item * { + color: white; + } + } +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 8352d92de1..c9b02310b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,8 @@ html_theme_options = { "navbar_end": ["darkmode-toggle", "version-switcher", "navbar-icon-links"], - "show_prev_next": False + "show_prev_next": False, + "footer_end": [] } @@ -80,6 +81,7 @@ def linkcode_resolve(*args): html_css_files = [ 'css/main.css', 'css/header.css', + 'css/footer.css', 'css/version-switcher.css', ] From e067a437351996fd463fc242ea50f963c6fd79fa Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 15:17:31 +0200 Subject: [PATCH 085/333] changes --- docs/_templates/footer-links.html | 5 +++++ docs/_templates/landing-page-illustrations.html | 6 +++--- docs/_templates/version-switcher.html | 1 + docs/conf.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 docs/_templates/footer-links.html diff --git a/docs/_templates/footer-links.html b/docs/_templates/footer-links.html new file mode 100644 index 0000000000..074731dfb5 --- /dev/null +++ b/docs/_templates/footer-links.html @@ -0,0 +1,5 @@ + \ No newline at end of file diff --git a/docs/_templates/landing-page-illustrations.html b/docs/_templates/landing-page-illustrations.html index 3b4069cb75..7a2185a4e4 100644 --- a/docs/_templates/landing-page-illustrations.html +++ b/docs/_templates/landing-page-illustrations.html @@ -5,7 +5,7 @@ Easy to Use

State-of-the-art NLP with just a few lines of code! Find entities, detect sentiment, and more. - Check out our demo! + Check out our demo!

@@ -14,7 +14,7 @@ Huge Community

With a community of ~200 contributors, Flair is used in hundreds of companies, - over 2,000 open source projects, and + over 2,000 open source projects, and 2,000+ papers!

@@ -24,7 +24,7 @@ Open Source and Free

Flair is completely free and open source, making it accessible for everyone to use - and report issues. + and report issues.

\ No newline at end of file diff --git a/docs/_templates/version-switcher.html b/docs/_templates/version-switcher.html index 9fb4bc511e..642039f919 100644 --- a/docs/_templates/version-switcher.html +++ b/docs/_templates/version-switcher.html @@ -36,6 +36,7 @@ `); + document.querySelector('footer.bd-footer') Open Source and Free

From 627a0d69bf22c30b450f245c342d30ea931aa589 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 16:09:30 +0200 Subject: [PATCH 094/333] changes --- docs/_templates/landing-page-illustrations.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/landing-page-illustrations.html b/docs/_templates/landing-page-illustrations.html index fa6dc03869..7a2185a4e4 100644 --- a/docs/_templates/landing-page-illustrations.html +++ b/docs/_templates/landing-page-illustrations.html @@ -19,7 +19,7 @@

- + Open Source and Free

From a801fc205f3847af4dcd060ccbd05cf5bba471b0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 16:10:33 +0200 Subject: [PATCH 095/333] changes --- docs/_templates/landing-page-illustrations.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/landing-page-illustrations.html b/docs/_templates/landing-page-illustrations.html index 7a2185a4e4..b6b55230a5 100644 --- a/docs/_templates/landing-page-illustrations.html +++ b/docs/_templates/landing-page-illustrations.html @@ -20,7 +20,7 @@

- + Open Source and Free

Flair is completely free and open source, making it accessible for everyone to use From e6128a47acb886081842fd881430611b5416a38b Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 20:22:52 +0200 Subject: [PATCH 096/333] changes --- docs/_static/css/header.css | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 7ddca94db2..8cc42304f4 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -116,6 +116,28 @@ } } +.search-button__wrapper.show { + .search-button__overlay { + background: var(--flair-blue); + } + form.bd-search { + i { + display: none; + } + input { + border: 2px var(--flair-orange) solid; + color: var(--flair-blue); + box-shadow: none; + padding: .25em 5.5em .25em .75em; + border-radius: 1rem; + font-size: 1.2rem; + } + .search-button__kbd-shortcut { + align-items: center; + } + } +} + .fill-white-blue { fill: var(--white-blue); stroke: none; From 78c22a349f9f6ee21e0c6378db3af4d63381acef Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 20:32:21 +0200 Subject: [PATCH 097/333] changes --- docs/_static/css/main.css | 4 ++++ docs/conf.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index 1c24bfc662..cb2312234c 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -119,4 +119,8 @@ a { .inv-sel::selection { background: var(--white-blue); color: var(--flair-orange); +} + +.bd-container__inner { + max-width: 100%; } \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index ee93d563c9..b607ec474c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,8 @@ html_theme_options = { "navbar_end": ["darkmode-toggle", "version-switcher", "navbar-icon-links"], "show_prev_next": False, - "footer_end": ["footer-links/legal-notice.html", "footer-links/x.html", "footer-links/linkedin.html"] + "footer_end": ["footer-links/legal-notice.html", "footer-links/x.html", "footer-links/linkedin.html"], + "secondary_sidebar_items": [] } From 37da6d5b4ba486d0292a284e87fd7beb7da91b96 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 20:41:24 +0200 Subject: [PATCH 098/333] changes --- docs/_static/css/sidebar.css | 5 +++++ docs/conf.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/_static/css/sidebar.css diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css new file mode 100644 index 0000000000..b72a05b072 --- /dev/null +++ b/docs/_static/css/sidebar.css @@ -0,0 +1,5 @@ +.bd-sidebar-primary.bd-sidebar { + h3 { + display: none; + } +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index b607ec474c..b421b261e4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -84,6 +84,7 @@ def linkcode_resolve(*args): 'css/header.css', 'css/footer.css', 'css/version-switcher.css', + 'css/sidebar.css', ] html_logo = "_static/flair_logo_white.svg" @@ -111,7 +112,6 @@ def linkcode_resolve(*args): "**": [ "globaltoc.html", "searchbox.html", - "versioning.html", ], "index": [], } From 8e6e0d7becac2bb6e5144e5d602880c59e0ee75c Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 20:51:26 +0200 Subject: [PATCH 099/333] changes --- docs/_static/css/sidebar.css | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css index b72a05b072..38eba96a5c 100644 --- a/docs/_static/css/sidebar.css +++ b/docs/_static/css/sidebar.css @@ -1,5 +1,29 @@ .bd-sidebar-primary.bd-sidebar { + padding: 2rem; + width: auto; h3 { display: none; } + * { + color: var(--flair-blue); + } + .toctree-l1 { + font-size: 1.5rem; + font-weight: 600; + margin-top: .8rem; + } + .toctree-l2 { + font-size: 1rem; + font-weight: 400; + margin-top: .8rem; + ul { + margin-left: 1rem; + } + } + .toctree-l3 { + margin-top: .8rem; + } + .current { + color: var(--flair-orange); + } } \ No newline at end of file From 86cb934433ee374d7ee63ca5c2d960520a05ab7b Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 21:34:23 +0200 Subject: [PATCH 100/333] changes --- docs/_static/css/main.css | 20 +++++++++++++ docs/_static/css/sidebar.css | 36 ++++++++++++++++++++++-- docs/_templates/landing_page_styles.html | 3 ++ docs/conf.py | 3 +- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index cb2312234c..ee52630e51 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -123,4 +123,24 @@ a { .bd-container__inner { max-width: 100%; +} + + /* width */ +::-webkit-scrollbar { + width: 10px; +} + +/* Track */ +::-webkit-scrollbar-track { + background: #2f2e41; +} + +/* Handle */ +::-webkit-scrollbar-thumb { + background: #888; +} + +/* Handle on hover */ +::-webkit-scrollbar-thumb:hover { + background: #555; } \ No newline at end of file diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css index 38eba96a5c..4a219be4f9 100644 --- a/docs/_static/css/sidebar.css +++ b/docs/_static/css/sidebar.css @@ -1,11 +1,35 @@ .bd-sidebar-primary.bd-sidebar { - padding: 2rem; - width: auto; + top: 90px; + width: 350px; + padding: 0; + border: none; + overflow: initial; + background: var(--white-blue); + #rtd-footer-container, .sidebar-primary-items__end { + display: none; + } + &::after { + content: ""; + background: linear-gradient(90deg, rgba(0,0,0,.2) 0%, rgba(0,0,0,0) 100%); + height: 100%; + width: 7px; + position: absolute; + right: 0; + transform: translateX(100%); + top: 0; + } h3 { display: none; } * { - color: var(--flair-blue); + color: var(--blue-white); + } + .sidebar-primary-items__start { + overflow-y: auto; + padding: 2rem; + } + .sidebar-primary-item { + padding: 0; } .toctree-l1 { font-size: 1.5rem; @@ -26,4 +50,10 @@ .current { color: var(--flair-orange); } + code { + padding: 0; + background: transparent; + border: none; + font-weight: 400; + } } \ No newline at end of file diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index b9fcebe573..042cca785c 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -190,4 +190,7 @@ stroke: var(--orange-white); fill: none; } + #searchbox { + display: none; + } diff --git a/docs/conf.py b/docs/conf.py index b421b261e4..9aa061a52b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -110,8 +110,7 @@ def linkcode_resolve(*args): html_sidebars = { "**": [ - "globaltoc.html", - "searchbox.html", + "globaltoc.html" ], "index": [], } From 1834e12d5bd65d06158318910aeeb748d2a920c3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 21:38:59 +0200 Subject: [PATCH 101/333] changes --- docs/_static/css/header.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 8cc42304f4..4f075f5fc7 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -60,7 +60,7 @@ transition: width 200ms cubic-bezier(0,.35,.08,.89); content: ""; width: 0; - height: 3px; + height: 2px; background-color: white; position: absolute; bottom: -.5rem; From e788605c40c9a32d2cd0e0bcf2f77ea1adbaa7cd Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 21:49:49 +0200 Subject: [PATCH 102/333] changes --- docs/_static/css/main.css | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index ee52630e51..5b68d02269 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -14,6 +14,9 @@ --blue-orange: var(--flair-blue); --header-height: 90px; --footer-height: 60px; + --track-color: #dddddd; + --track-thumb: #bbbbbb; + --track-thumb-hover: #999999; font-family: "Afacad", sans-serif; font-optical-sizing: auto; font-weight: 400; @@ -28,6 +31,9 @@ --orange-blue: var(--flair-blue); --blue-orange: var(--flair-orange); --orange-white-transparent: var(--white-transparent); + --track-color: #38374a; + --track-thumb: #4c4b5c; + --track-thumb-hover: #5c5b69; } .fill-white-blue { @@ -125,22 +131,18 @@ a { max-width: 100%; } - /* width */ ::-webkit-scrollbar { width: 10px; } -/* Track */ ::-webkit-scrollbar-track { - background: #2f2e41; + background: var(--track-color); } -/* Handle */ ::-webkit-scrollbar-thumb { - background: #888; + background: var(--track-thumb); } -/* Handle on hover */ ::-webkit-scrollbar-thumb:hover { - background: #555; + background: var(--track-thumb-hover); } \ No newline at end of file From 509f6847190c8afb6b72abe4345e69656cb7ed59 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 21:56:10 +0200 Subject: [PATCH 103/333] changes --- docs/_static/css/main.css | 17 +++++++++-------- docs/_static/css/sidebar.css | 3 +++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index 5b68d02269..8968068092 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -131,18 +131,19 @@ a { max-width: 100%; } -::-webkit-scrollbar { - width: 10px; +body::-webkit-scrollbar, ::-webkit-scrollbar { + width: 10px; } -::-webkit-scrollbar-track { - background: var(--track-color); +body::-webkit-scrollbar-track, ::-webkit-scrollbar-track { + background: var(--track-color); } -::-webkit-scrollbar-thumb { - background: var(--track-thumb); +body::-webkit-scrollbar-thumb, ::-webkit-scrollbar-thumb { + background: var(--track-thumb); + border-radius: 0; } -::-webkit-scrollbar-thumb:hover { - background: var(--track-thumb-hover); +body::-webkit-scrollbar-thumb:hover, ::-webkit-scrollbar-thumb:hover { + background: var(--track-thumb-hover); } \ No newline at end of file diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css index 4a219be4f9..749122db8c 100644 --- a/docs/_static/css/sidebar.css +++ b/docs/_static/css/sidebar.css @@ -24,6 +24,9 @@ * { color: var(--blue-white); } + a:hover { + color: var(--flair-orange); + } .sidebar-primary-items__start { overflow-y: auto; padding: 2rem; From 694cbba3a6d21affc800a8c7a5ace58bd9b113ae Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 7 Aug 2024 21:58:47 +0200 Subject: [PATCH 104/333] changes --- docs/_static/css/sidebar.css | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css index 749122db8c..598be603cd 100644 --- a/docs/_static/css/sidebar.css +++ b/docs/_static/css/sidebar.css @@ -26,6 +26,12 @@ } a:hover { color: var(--flair-orange); + * { + color: var(--flair-orange); + } + } + a * { + transition: color 200ms cubic-bezier(0,.35,.08,.89); } .sidebar-primary-items__start { overflow-y: auto; From 6bbc2bbfc23abda65e347bc6685786b5aaa4edca Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 09:04:48 +0200 Subject: [PATCH 105/333] changes --- docs/_static/css/tutorial.css | 33 +++++++++++++++++++++++++++++++++ docs/conf.py | 1 + 2 files changed, 34 insertions(+) create mode 100644 docs/_static/css/tutorial.css diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css new file mode 100644 index 0000000000..16b47118d1 --- /dev/null +++ b/docs/_static/css/tutorial.css @@ -0,0 +1,33 @@ +ul.bd-breadcrumbs { + li.breadcrumb-home .fa-home { + &::before { + content: "Home"; + font-family: "Afacad", sans-serif; + font-size: 1rem; + font-weight: 400; + color: var(--blue-white); + } + } + li.breadcrumb-item:not(.breadcrumb-home) { + &::before { + content: ""; + width: 1px; + height: 1rem; + background: white; + display: inline-block; + padding: 0; + transform: rotate(30deg); + margin: 0 .6rem; + } + a { + color: var(--blue-white); + font-size: 1rem; + font-weight: 400; + } + &.active { + color: var(--blue-white); + font-size: 1rem; + font-weight: 400; + } + } +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 9aa061a52b..f5e6d6c6a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,6 +85,7 @@ def linkcode_resolve(*args): 'css/footer.css', 'css/version-switcher.css', 'css/sidebar.css', + 'css/tutorial.css' ] html_logo = "_static/flair_logo_white.svg" From 60e2b72f76085b2c3a6f1d70d264603361e94bd0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 09:13:27 +0200 Subject: [PATCH 106/333] changes --- docs/_static/css/tutorial.css | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 16b47118d1..d0a0462e48 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -1,3 +1,21 @@ +.bd-header-article { + margin-bottom: 2rem; +} + +.bd-main .bd-content .bd-article-container{ + max-width: initial; + padding: 4rem 5rem; + box-sizing: border-box; +} + +.bd-main .bd-content .bd-article-container .bd-article { + padding: 0; +} + +.header-article-items.header-article__inner { + padding: 0; +} + ul.bd-breadcrumbs { li.breadcrumb-home .fa-home { &::before { From a5b859863d6e65f78753ac496c1823ec79d1c8ec Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 09:21:51 +0200 Subject: [PATCH 107/333] changes --- docs/_static/css/main.css | 2 +- docs/_static/css/tutorial.css | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index 8968068092..1778597b8b 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -20,7 +20,7 @@ font-family: "Afacad", sans-serif; font-optical-sizing: auto; font-weight: 400; - font-size: 20px; + font-size: 21px; font-style: normal; } diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index d0a0462e48..9d700f7fd9 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -1,7 +1,3 @@ -.bd-header-article { - margin-bottom: 2rem; -} - .bd-main .bd-content .bd-article-container{ max-width: initial; padding: 4rem 5rem; @@ -10,6 +6,22 @@ .bd-main .bd-content .bd-article-container .bd-article { padding: 0; + h1 { + font-size: 3rem; + color: var(--blue-white); + font-weight: 600; + margin-top: 0.5em; + } + h2 { + font-size: 1.6rem; + color: var(--blue-white); + font-weight: 600; + margin-top: 2em; + } + p { + color: var(--blue-white); + margin: 0.5em 0 1em 0; + } } .header-article-items.header-article__inner { From bd558be123d38316c2686738034ed720f095c49d Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 09:31:04 +0200 Subject: [PATCH 108/333] changes --- docs/_static/css/main.css | 4 ++++ docs/_static/css/tutorial.css | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index 1778597b8b..bf73dbb939 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -6,12 +6,15 @@ --flair-orange-transparent: rgba(247, 153, 16, .5); --white-transparent: rgba(255, 255, 255, .5); --flair-blue: #2f2e41; + --flair-dark-blue: #262635; + --light-gray: #e6e6e6; --white-blue: white; --blue-white: var(--flair-blue); --orange-white: var(--flair-orange); --orange-white-transparent: var(--flair-orange-transparent); --orange-blue: var(--flair-orange); --blue-orange: var(--flair-blue); + --gray-dark-blue: var(--light-gray); --header-height: 90px; --footer-height: 60px; --track-color: #dddddd; @@ -31,6 +34,7 @@ --orange-blue: var(--flair-blue); --blue-orange: var(--flair-orange); --orange-white-transparent: var(--white-transparent); + --gray-dark-blue: var(--flair-dark-blue); --track-color: #38374a; --track-thumb: #4c4b5c; --track-thumb-hover: #5c5b69; diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 9d700f7fd9..a09ba9974a 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -1,6 +1,6 @@ .bd-main .bd-content .bd-article-container{ max-width: initial; - padding: 4rem 5rem; + padding: 4rem 30% 4rem 5rem; box-sizing: border-box; } @@ -20,7 +20,18 @@ } p { color: var(--blue-white); - margin: 0.5em 0 1em 0; + margin: 0.4em 0 0.9em 0; + } + .highlight-python { + .highlight { + background: transparent; + pre { + background: var(--gray-dark-blue); + .nn, .n { + color: var(--blue-white); + } + } + } } } From 6d49182704225eea5185f2cda4ee55fa50d9a412 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 09:42:52 +0200 Subject: [PATCH 109/333] changes --- docs/_static/css/tutorial.css | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index a09ba9974a..fde1236f2a 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -26,10 +26,18 @@ .highlight { background: transparent; pre { + padding: 1.1em 1.5em; + border-radius: 1rem; background: var(--gray-dark-blue); .nn, .n { color: var(--blue-white); } + .c1 { + color: #999; + :root .dark-mode & { + color: #696991; + } + } } } } From 7373488224b51980c85729a987c5feadfb615009 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 10:07:59 +0200 Subject: [PATCH 110/333] changes --- docs/_static/css/main.css | 2 +- docs/_static/css/tutorial.css | 21 +++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index bf73dbb939..e2b94342c0 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -1,4 +1,4 @@ -@import url('https://fonts.googleapis.com/css2?family=Afacad:ital,wght@0,400..700;1,400..700&display=swap'); +@import url('https://fonts.googleapis.com/css2?family=Afacad:ital,wght@0,400..700;1,400..700&family=Source+Code+Pro:ital,wght@0,200..900;1,200..900&display=swap'); :root { --flair-orange: #f79910; diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index fde1236f2a..56241ee3f5 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -26,17 +26,30 @@ .highlight { background: transparent; pre { + font-family: "Source Code Pro", monospace; padding: 1.1em 1.5em; border-radius: 1rem; background: var(--gray-dark-blue); - .nn, .n { + .nn, .n, .o, .p { color: var(--blue-white); } + .kn { + color: #CF8E6D; + } + .kp { + color: #57AAF7; + } + .nb { + color: #8888C6; + } .c1 { - color: #999; - :root .dark-mode & { + color: #5F826B; + /*:root .dark-mode & { color: #696991; - } + }*/ + } + .s1 { + color : #6AAB73; } } } From ac2ec964c71eb53d976648b23be289352a5d6394 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 10:37:24 +0200 Subject: [PATCH 111/333] changes --- docs/_static/css/tutorial.css | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 56241ee3f5..c03fbf9758 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -54,6 +54,52 @@ } } } + .highlight-console { + .highlight { + background: transparent; + } + pre { + font-family: "Source Code Pro", monospace; + border-radius: 1rem; + padding: 1.1em 1.5em; + background: var(--gray-dark-blue); + color: var(--blue-white); + .go { + font-size: inherit; + padding-right: 1.1em; + } + } + } + .admonition, div.admonition, .admonition.note, div.admonition.note { + box-shadow: none !important; + background: linear-gradient(180deg, var(--flair-orange) 0, var(--flair-orange) 2rem, var(--white-blue) 2rem, var(--white-blue) 100%); + border: 2px var(--flair-orange) solid; + border-radius: 1rem; + padding: 0; + box-sizing: border-box; + p.admonition-title { + background: var(--flair-orange); + border-radius: 0 0 1rem 1rem; + margin: 0; + font-size: 1.5rem; + font-weight: 400; + padding: 0.5em 2.2em; + position: relative; + line-height: 1em; + &::before { + content: none; + } + &::after { + color: white; + margin-left: 0.4em; + font-size: 0.9em; + } + } + p:not(.admonition-title) { + padding: 0.5em 2.2em 0.25em 3.3em; + line-height: 1em; + } + } } .header-article-items.header-article__inner { From 730cb18104394355c6e8bf2cafd24bca530af2fe Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 10:55:27 +0200 Subject: [PATCH 112/333] changes --- docs/_static/css/tutorial.css | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index c03fbf9758..3c4f5d53a8 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -138,4 +138,34 @@ ul.bd-breadcrumbs { font-weight: 400; } } +} + +table.table { + tr { + &:not(:last-of-type), &:has(th) { + border-bottom: 2px var(--blue-white) solid; + } + th { + font-weight: 600; + } + th, td { + padding: .5em .2em .15em .2em; + &:first-of-type { + padding-left: 0; + } + &:last-of-type { + padding-right: 0; + } + } + } +} + +.reference .docutils { + border: 2px var(--flair-orange) solid; + background: transparent; + border-radius: 0.5em; + font-weight: 400; + font-family: "Source Code Pro", monospace; + font-size: 0.8em; + color: var(--blue-white); } \ No newline at end of file From 4010213ada6c24f5d529d7a184f9b63daa583b9a Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 11:10:54 +0200 Subject: [PATCH 113/333] changes --- docs/_static/css/tutorial.css | 46 ++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 3c4f5d53a8..dd6de62e0b 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -18,6 +18,12 @@ font-weight: 600; margin-top: 2em; } + h3 { + font-size: 1.2rem; + color: var(--blue-white); + font-weight: 600; + margin-top: 2em; + } p { color: var(--blue-white); margin: 0.4em 0 0.9em 0; @@ -30,31 +36,31 @@ padding: 1.1em 1.5em; border-radius: 1rem; background: var(--gray-dark-blue); - .nn, .n, .o, .p { + .nn, .n, .o, .p, .kp { color: var(--blue-white); } - .kn { + .kn, .k, .ow { color: #CF8E6D; } - .kp { - color: #57AAF7; - } .nb { color: #8888C6; } .c1 { - color: #5F826B; - /*:root .dark-mode & { - color: #696991; - }*/ + color: #7A7E85; } - .s1 { + .s1, .sa, .si { color : #6AAB73; } + .mi { + color: #2AACB8; + } } } } - .highlight-console { + .highlight-console, .highlight-default { + * { + color: var(--blue-white); + } .highlight { background: transparent; } @@ -97,7 +103,9 @@ } p:not(.admonition-title) { padding: 0.5em 2.2em 0.25em 3.3em; - line-height: 1em; + } + .highlight-python, .highlight-default, .highlight-console, .highlight { + margin-bottom: 0.7rem; } } } @@ -160,12 +168,22 @@ table.table { } } -.reference .docutils { - border: 2px var(--flair-orange) solid; +.docutils, .docutils.literal { + border: 2px var(--blue-white) solid; background: transparent; border-radius: 0.5em; font-weight: 400; font-family: "Source Code Pro", monospace; font-size: 0.8em; color: var(--blue-white); +} +.reference { + .docutils, .docutils.literal { + border: 2px var(--flair-orange) solid; + transition: color, background-color 200ms ease-in-out; + &:hover { + background: var(--flair-orange); + color: white; + } + } } \ No newline at end of file From b0014ebb394c0dea507d2a71ea96a44c8c7b3d65 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 11:21:40 +0200 Subject: [PATCH 114/333] changes --- docs/_static/css/tutorial.css | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index dd6de62e0b..bc07a9001b 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -148,7 +148,9 @@ ul.bd-breadcrumbs { } } -table.table { +table.table {border-collapse: collapse; + border-radius: 1rem; + box-shadow: 0 0 0 2px var(--blue-white); tr { &:not(:last-of-type), &:has(th) { border-bottom: 2px var(--blue-white) solid; @@ -157,12 +159,12 @@ table.table { font-weight: 600; } th, td { - padding: .5em .2em .15em .2em; + padding: .4em .2em 0 .2em; &:first-of-type { - padding-left: 0; + padding-left: .5em; } &:last-of-type { - padding-right: 0; + padding-right: .5em; } } } From 4a76a8be036b442714f2f57a256fa30dfe770cb7 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 11:30:18 +0200 Subject: [PATCH 115/333] changes --- docs/_static/css/tutorial.css | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index bc07a9001b..728e2b7363 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -48,7 +48,7 @@ .c1 { color: #7A7E85; } - .s1, .sa, .si { + .s1, .s2, .sa, .si { color : #6AAB73; } .mi { @@ -171,21 +171,40 @@ table.table {border-collapse: collapse; } .docutils, .docutils.literal { - border: 2px var(--blue-white) solid; + border: none; background: transparent; border-radius: 0.5em; font-weight: 400; font-family: "Source Code Pro", monospace; font-size: 0.8em; color: var(--blue-white); + position: relative; + z-index: 1; + &::before { + content: ""; + width: 100%; + height: 90%; + top: 50%; + left: 0; + transform: translateY(-50%); + border-radius: 0.5em; + border: 2px var(--blue-white) solid; + position: absolute; + } } .reference { .docutils, .docutils.literal { - border: 2px var(--flair-orange) solid; - transition: color, background-color 200ms ease-in-out; + transition: color 200ms ease-in-out; &:hover { - background: var(--flair-orange); + &::before { + background: var(--flair-orange); + } color: white; } + &::before { + z-index: -1; + border-color: var(--flair-orange); + transition: background-color 200ms ease-in-out; + } } } \ No newline at end of file From 9fe360a8780cdd2eb8712e0306fbba1b83b840ec Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 11:43:14 +0200 Subject: [PATCH 116/333] changes --- docs/_static/css/tutorial.css | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 728e2b7363..26c9ad3d2d 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -6,6 +6,9 @@ .bd-main .bd-content .bd-article-container .bd-article { padding: 0; + .headerlink { + display: none; + } h1 { font-size: 3rem; color: var(--blue-white); @@ -28,6 +31,13 @@ color: var(--blue-white); margin: 0.4em 0 0.9em 0; } + ul { + list-style: initial; + margin-left: 1rem; + ::marker { + color: var(--blue-white); + } + } .highlight-python { .highlight { background: transparent; @@ -51,7 +61,7 @@ .s1, .s2, .sa, .si { color : #6AAB73; } - .mi { + .mi, .mf { color: #2AACB8; } } @@ -122,14 +132,18 @@ ul.bd-breadcrumbs { font-size: 1rem; font-weight: 400; color: var(--blue-white); + transition: color 200ms ease-in-out; } } + li.breadcrumb-home a:hover .fa-home::before { + color: var(--flair-orange); + } li.breadcrumb-item:not(.breadcrumb-home) { &::before { content: ""; width: 1px; height: 1rem; - background: white; + background: var(--blue-white); display: inline-block; padding: 0; transform: rotate(30deg); @@ -139,6 +153,9 @@ ul.bd-breadcrumbs { color: var(--blue-white); font-size: 1rem; font-weight: 400; + &:hover { + color: var(--flair-orange); + } } &.active { color: var(--blue-white); From 2a25cadddd45c5fb8f99e4eee84c5325f366f2de Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 11:49:30 +0200 Subject: [PATCH 117/333] changes --- docs/_static/css/tutorial.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 26c9ad3d2d..cf66b9f027 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -67,7 +67,7 @@ } } } - .highlight-console, .highlight-default { + [class^="highlight-"]:not(.highlight-python) { * { color: var(--blue-white); } From f81ef2ce0489fbc0725a7d8e22f32e066c4ef81e Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 12:05:52 +0200 Subject: [PATCH 118/333] changes --- docs/_static/css/tutorial.css | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index cf66b9f027..cd2dc7e9ba 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -49,12 +49,15 @@ .nn, .n, .o, .p, .kp { color: var(--blue-white); } - .kn, .k, .ow { + .kn, .k, .ow, .kc { color: #CF8E6D; } .nb { color: #8888C6; } + .nf { + color: #56A8F5; + } .c1 { color: #7A7E85; } @@ -224,4 +227,14 @@ table.table {border-collapse: collapse; transition: background-color 200ms ease-in-out; } } +} + +#flair-tutorials, [id^="tutorial"] { + li { + list-style-type: "»"; + padding-left: 0.25em; + ::marker { + color: var(--blue-white); + } + } } \ No newline at end of file From cedca6d981f11540fc62de1099d09484d9d17dcb Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 12:22:26 +0200 Subject: [PATCH 119/333] changes --- docs/_static/css/api.css | 0 docs/conf.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 docs/_static/css/api.css diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/conf.py b/docs/conf.py index f5e6d6c6a1..b1238878bc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,7 +85,8 @@ def linkcode_resolve(*args): 'css/footer.css', 'css/version-switcher.css', 'css/sidebar.css', - 'css/tutorial.css' + 'css/tutorial.css', + 'css/api.css' ] html_logo = "_static/flair_logo_white.svg" From 3feaa21e12f1e29ef8224cde604170423d7f418f Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 12:49:54 +0200 Subject: [PATCH 120/333] changes --- docs/_static/css/api.css | 12 ++++++++++++ docs/_static/css/tutorial.css | 27 ++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index e69de29bb2..b25b39cd2f 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -0,0 +1,12 @@ +.sidebar-primary-item { + .docutils { + &::before { + content: none; + } + .pre { + font-family: "Afacad", sans-serif; + font-size: 1rem; + text-overflow: ellipsis; + } + } +} \ No newline at end of file diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index cd2dc7e9ba..0c48a3a79f 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -229,12 +229,37 @@ table.table {border-collapse: collapse; } } -#flair-tutorials, [id^="tutorial"] { +#flair-tutorials, #api-docs, [id^="tutorial"], [id^="flair"] { li { list-style-type: "»"; padding-left: 0.25em; + margin-top: .5em; ::marker { color: var(--blue-white); + font-size: 1.2rem; + } + } + .toctree-wrapper li[class^="toctree-l"] { + margin-top: .5em; + &::marker { + font-size: 1.2rem; + } + & > a { + font-size: 1.2rem; + } + } + .docutils { + color: var(--flair-orange); + &:hover { + color: var(--flair-orange-light); + } + &::before { + content: none; + } + .pre { + font-family: "Afacad", sans-serif; + text-overflow: ellipsis; + font-size: 1.2rem; } } } \ No newline at end of file From 59024414f238a661865be8c0452191ce00ac07f6 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 12:54:29 +0200 Subject: [PATCH 121/333] changes --- docs/_static/css/tutorial.css | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 0c48a3a79f..fa960786e2 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -32,10 +32,14 @@ margin: 0.4em 0 0.9em 0; } ul { - list-style: initial; - margin-left: 1rem; + list-style-type: "»"; + margin-left: .6em; + li { + padding-left: .3em; + } ::marker { color: var(--blue-white); + font-size: inherit; } } .highlight-python { From ec80d59c49202750d8cf9c6bc802e6fba5f37f41 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 13:26:50 +0200 Subject: [PATCH 122/333] changes --- docs/_static/css/api.css | 13 +++++++++++++ docs/_static/octocat.svg | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 docs/_static/octocat.svg diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index b25b39cd2f..39d05de307 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -9,4 +9,17 @@ text-overflow: ellipsis; } } +} + +.sig { + color: var(--blue-white); + font-family: "Source Code Pro", monospace; + .linkcode-link { + .pre { + display: none; + } + &::after { + content: url("_static/octocat.svg"); + } + } } \ No newline at end of file diff --git a/docs/_static/octocat.svg b/docs/_static/octocat.svg new file mode 100644 index 0000000000..ce2cf1d99d --- /dev/null +++ b/docs/_static/octocat.svg @@ -0,0 +1,41 @@ + + + + + + From 5491a45ba2050a383385f402badb33bc03f8c1fc Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 13:28:59 +0200 Subject: [PATCH 123/333] changes --- docs/_static/css/api.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 39d05de307..378063ee37 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -19,7 +19,7 @@ display: none; } &::after { - content: url("_static/octocat.svg"); + content: url("../../_static/octocat.svg"); } } } \ No newline at end of file From 00f4e1bf1fca567d0875fba09a124a27cd2cc2a0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 13:58:02 +0200 Subject: [PATCH 124/333] changes --- docs/_static/css/api.css | 46 ++++++++++++++++++++++++++++++++++++++- docs/_static/css/main.css | 2 ++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 378063ee37..dc3f3a5435 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -15,11 +15,55 @@ color: var(--blue-white); font-family: "Source Code Pro", monospace; .linkcode-link { + padding: 0; + display: inline-block; + width: 1.5rem; + height: 1.5rem; + transform: translateY(25%); .pre { display: none; } &::after { - content: url("../../_static/octocat.svg"); + content:""; + background-image: url('../../_static/octocat.svg'); + background-size: cover; + display: inline-block; + width: 100%; + height: 100%; + } + } +} + +.sig-name.descname { + color: var(--flair-orange); +} + +.class, .method { + .sig { + margin-top: 2rem; + } + dd { + margin-left: 0 !important; + padding-left: 3rem !important; + border-left: 2px var(--gray-white) solid; + dl.field-list { + background-color: transparent; + border: 2px var(--flair-orange) solid; + border-radius: 1rem; + padding: 1rem 1.5rem; + dt { + background-color: transparent !important; + color: var(--blue-white); + font-weight: 600; + font-size: 1.5rem; + padding: 0 !important; + margin: 0 !important; + } + dd { + margin: 0 !important; + padding: 0 !important; + border-left: none; + } } } } \ No newline at end of file diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index e2b94342c0..d3c6242c41 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -15,6 +15,7 @@ --orange-blue: var(--flair-orange); --blue-orange: var(--flair-blue); --gray-dark-blue: var(--light-gray); + --gray-white: var(--light-gray); --header-height: 90px; --footer-height: 60px; --track-color: #dddddd; @@ -35,6 +36,7 @@ --blue-orange: var(--flair-orange); --orange-white-transparent: var(--white-transparent); --gray-dark-blue: var(--flair-dark-blue); + --gray-white: white; --track-color: #38374a; --track-thumb: #4c4b5c; --track-thumb-hover: #5c5b69; From f4805474da2dbb741e768405913a0fead318f316 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 14:06:00 +0200 Subject: [PATCH 125/333] changes --- docs/_static/css/api.css | 3 ++- docs/_static/css/tutorial.css | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index dc3f3a5435..0a9bba2b67 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -38,9 +38,10 @@ color: var(--flair-orange); } -.class, .method { +.class, .method, .function { .sig { margin-top: 2rem; + margin-bottom: 1rem; } dd { margin-left: 0 !important; diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index fa960786e2..c9ec7d4f64 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -252,7 +252,7 @@ table.table {border-collapse: collapse; font-size: 1.2rem; } } - .docutils { + .docutils:not(.py) { color: var(--flair-orange); &:hover { color: var(--flair-orange-light); From 55fcb22a311dc7c5282c986bf51710eada5f3340 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 14:13:47 +0200 Subject: [PATCH 126/333] changes --- docs/_static/css/api.css | 35 +++++++++++++++++++++++++++++++++++ docs/_static/css/main.css | 1 + docs/_static/css/tutorial.css | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 0a9bba2b67..e0640a06ef 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -67,4 +67,39 @@ } } } +} + +div.deprecated { + box-shadow: none !important; + background: linear-gradient(180deg, var(--error-red) 0, var(--error-red) 2rem, var(--white-blue) 2rem, var(--white-blue) 100%); + border: 2px var(--error-red) solid; + border-radius: 1rem; + padding: 0; + box-sizing: border-box; + p { + &::before { + content: none; + } + } + /* p { + background: var(--flair-orange); + border-radius: 0 0 1rem 1rem; + margin: 0; + font-size: 1.5rem; + font-weight: 400; + padding: 0.5em 2.2em; + position: relative; + line-height: 1em; + &::before { + content: none; + } + &::after { + color: white; + margin-left: 0.4em; + font-size: 0.9em; + } + } + p:not(.admonition-title) { + padding: 0.5em 2.2em 0.25em 3.3em; + }*/ } \ No newline at end of file diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index d3c6242c41..12a8a7abcb 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -16,6 +16,7 @@ --blue-orange: var(--flair-blue); --gray-dark-blue: var(--light-gray); --gray-white: var(--light-gray); + --error-red: #e32239; --header-height: 90px; --footer-height: 60px; --track-color: #dddddd; diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index c9ec7d4f64..58880393a8 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -121,7 +121,7 @@ p:not(.admonition-title) { padding: 0.5em 2.2em 0.25em 3.3em; } - .highlight-python, .highlight-default, .highlight-console, .highlight { + [class^="highlight-"] { margin-bottom: 0.7rem; } } From 719c740c33b58476b628ca41a4f888c31891f1c1 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 14:25:40 +0200 Subject: [PATCH 127/333] changes --- docs/_static/css/api.css | 44 +++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index e0640a06ef..16c6f2777b 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -41,7 +41,12 @@ .class, .method, .function { .sig { margin-top: 2rem; - margin-bottom: 1rem; + margin-bottom: .5em; + padding-bottom: .5em; + } + dt:target { + background: var(--flair-orange-transparent); + border-radius: 10px; } dd { margin-left: 0 !important; @@ -71,35 +76,28 @@ div.deprecated { box-shadow: none !important; - background: linear-gradient(180deg, var(--error-red) 0, var(--error-red) 2rem, var(--white-blue) 2rem, var(--white-blue) 100%); + background: var(--error-red); border: 2px var(--error-red) solid; border-radius: 1rem; - padding: 0; + padding: 0.5em 2.2em 0 2.2em; box-sizing: border-box; p { - &::before { - content: none; + span.versionmodified.deprecated::before { + color: white; } - } - /* p { - background: var(--flair-orange); - border-radius: 0 0 1rem 1rem; - margin: 0; - font-size: 1.5rem; - font-weight: 400; - padding: 0.5em 2.2em; - position: relative; - line-height: 1em; &::before { content: none; } - &::after { - color: white; - margin-left: 0.4em; - font-size: 0.9em; - } } - p:not(.admonition-title) { - padding: 0.5em 2.2em 0.25em 3.3em; - }*/ +} + +a.github::before { + content: ""; + height: 1em; + width: 1em; + display: inline-block; + background-image: url("../../_static/octocat.svg"); + background-size: contain; + background-position: center; + transform: translateY(25%); } \ No newline at end of file From 34d76c86492461749f49faa572da8b9a7b6570fc Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 14:42:42 +0200 Subject: [PATCH 128/333] changes --- docs/_static/css/api.css | 24 +++++++++++++----------- docs/_static/css/tutorial.css | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 16c6f2777b..fad83624d7 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -17,9 +17,9 @@ .linkcode-link { padding: 0; display: inline-block; - width: 1.5rem; - height: 1.5rem; - transform: translateY(25%); + width: 1rem; + height: 1rem; + transform: translateY(10%) scale(1.5); .pre { display: none; } @@ -38,16 +38,18 @@ color: var(--flair-orange); } -.class, .method, .function { - .sig { - margin-top: 2rem; - margin-bottom: .5em; - padding-bottom: .5em; - } - dt:target { +.sig { + margin-top: 2rem; + margin-bottom: .5em; + padding-bottom: .5em; + padding-top: .5em; + &:target { background: var(--flair-orange-transparent); - border-radius: 10px; + border-radius: .5rem; } +} + +.class, .method, .function, .data { dd { margin-left: 0 !important; padding-left: 3rem !important; diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 58880393a8..8b1e0d72e9 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -233,7 +233,7 @@ table.table {border-collapse: collapse; } } -#flair-tutorials, #api-docs, [id^="tutorial"], [id^="flair"] { +#flair-tutorials, #api-docs, #contributing, [id^="tutorial"], [id^="flair"] { li { list-style-type: "»"; padding-left: 0.25em; From fabfec107979e6f37f8f033eac9116519ccf1b1b Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 15:08:49 +0200 Subject: [PATCH 129/333] changes --- docs/_static/css/api.css | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index fad83624d7..1dd843d54c 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -14,6 +14,7 @@ .sig { color: var(--blue-white); font-family: "Source Code Pro", monospace; + line-height: 2em; .linkcode-link { padding: 0; display: inline-block; @@ -59,6 +60,9 @@ border: 2px var(--flair-orange) solid; border-radius: 1rem; padding: 1rem 1.5rem; + ul { + padding-left: 0 !important; + } dt { background-color: transparent !important; color: var(--blue-white); @@ -74,6 +78,12 @@ } } } + .n { + color: #2AACB8; + } + .default_value { + color: #8888C6; + } } div.deprecated { @@ -102,4 +112,18 @@ a.github::before { background-size: contain; background-position: center; transform: translateY(25%); +} + +blockquote { + font-family: "Source Code Pro", monospace; + border-radius: 1rem; + padding: 1.1em 1.5em; + background: var(--gray-dark-blue); + color: var(--blue-white); + &::before { + content: none; + } + p { + margin: 0 !important; + } } \ No newline at end of file From 4fbdcd272c3e19f9fdf37c7519efa67aeb2f34b6 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 14 Aug 2024 15:12:43 +0200 Subject: [PATCH 130/333] changes --- docs/_static/css/api.css | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 1dd843d54c..5fd8ef8c89 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -45,9 +45,12 @@ padding-bottom: .5em; padding-top: .5em; &:target { - background: var(--flair-orange-transparent); + background: rgba(47, 46, 65, 0.15); border-radius: .5rem; } + :root .dark-mode &:target { + background: rgba(255, 255, 255, 0.15); + } } .class, .method, .function, .data { From 0422de10e73bf797e93417d4c32f710196755087 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:16:32 +0200 Subject: [PATCH 131/333] changes --- docs/legal-notice/index.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 docs/legal-notice/index.rst diff --git a/docs/legal-notice/index.rst b/docs/legal-notice/index.rst new file mode 100644 index 0000000000..08b272d915 --- /dev/null +++ b/docs/legal-notice/index.rst @@ -0,0 +1,12 @@ +.. _flair_legal-notice: + +.. title:: Legal notice + +.. raw:: html + :file: _templates/landing_page_styles.html + +.. raw:: html + :file: _templates/landing-page-banner.html + +.. raw:: html + :file: _templates/landing-page-illustrations.html \ No newline at end of file From 79d664fafc9d01b96d8e3b83f06bbe8ae6f393cd Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:21:24 +0200 Subject: [PATCH 132/333] changes --- docs/_templates/footer-links.html | 5 ----- docs/_templates/footer-links/legal-notice.html | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 docs/_templates/footer-links.html diff --git a/docs/_templates/footer-links.html b/docs/_templates/footer-links.html deleted file mode 100644 index 074731dfb5..0000000000 --- a/docs/_templates/footer-links.html +++ /dev/null @@ -1,5 +0,0 @@ -

\ No newline at end of file diff --git a/docs/_templates/footer-links/legal-notice.html b/docs/_templates/footer-links/legal-notice.html index 44658a6eba..ff1eed5ca9 100644 --- a/docs/_templates/footer-links/legal-notice.html +++ b/docs/_templates/footer-links/legal-notice.html @@ -1 +1 @@ -Legal notice \ No newline at end of file +Legal notice \ No newline at end of file From 3b3379da6f564aea40174572af9029e72390752c Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:24:51 +0200 Subject: [PATCH 133/333] changes --- docs/_templates/footer-links/legal-notice.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/footer-links/legal-notice.html b/docs/_templates/footer-links/legal-notice.html index ff1eed5ca9..30553d5aed 100644 --- a/docs/_templates/footer-links/legal-notice.html +++ b/docs/_templates/footer-links/legal-notice.html @@ -1 +1 @@ -Legal notice \ No newline at end of file +Legal notice \ No newline at end of file From 1cadc5b898ea4ba66fd688dc7b6754598b375a6b Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:30:07 +0200 Subject: [PATCH 134/333] changes --- docs/_templates/footer-links/legal-notice.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/footer-links/legal-notice.html b/docs/_templates/footer-links/legal-notice.html index 30553d5aed..ab3b0dd4ac 100644 --- a/docs/_templates/footer-links/legal-notice.html +++ b/docs/_templates/footer-links/legal-notice.html @@ -1 +1 @@ -Legal notice \ No newline at end of file +Legal notice \ No newline at end of file From c95f8598d8b5cb479f38bf74d6135d59e5718f28 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:32:07 +0200 Subject: [PATCH 135/333] changes --- docs/_templates/legal-notice-content.html | 3 +++ docs/legal-notice/index.rst | 8 +------- 2 files changed, 4 insertions(+), 7 deletions(-) create mode 100644 docs/_templates/legal-notice-content.html diff --git a/docs/_templates/legal-notice-content.html b/docs/_templates/legal-notice-content.html new file mode 100644 index 0000000000..7b25d9a135 --- /dev/null +++ b/docs/_templates/legal-notice-content.html @@ -0,0 +1,3 @@ +
+ HELLO; WORLD; +
\ No newline at end of file diff --git a/docs/legal-notice/index.rst b/docs/legal-notice/index.rst index 08b272d915..812268cebb 100644 --- a/docs/legal-notice/index.rst +++ b/docs/legal-notice/index.rst @@ -3,10 +3,4 @@ .. title:: Legal notice .. raw:: html - :file: _templates/landing_page_styles.html - -.. raw:: html - :file: _templates/landing-page-banner.html - -.. raw:: html - :file: _templates/landing-page-illustrations.html \ No newline at end of file + :file: _templates/legal-notice-content-html \ No newline at end of file From 5e63d976b2b0f47647760875d74aa3ef6899b655 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:33:48 +0200 Subject: [PATCH 136/333] changes --- docs/legal-notice/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/legal-notice/index.rst b/docs/legal-notice/index.rst index 812268cebb..7eafaeeb3f 100644 --- a/docs/legal-notice/index.rst +++ b/docs/legal-notice/index.rst @@ -3,4 +3,4 @@ .. title:: Legal notice .. raw:: html - :file: _templates/legal-notice-content-html \ No newline at end of file + :file: _templates/legal-notice-content.html \ No newline at end of file From fda3778b10b24a144715575ff051bc1bfb42a8f6 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:35:32 +0200 Subject: [PATCH 137/333] changes --- docs/legal-notice/index.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/legal-notice/index.rst b/docs/legal-notice/index.rst index 7eafaeeb3f..70779f7ce0 100644 --- a/docs/legal-notice/index.rst +++ b/docs/legal-notice/index.rst @@ -3,4 +3,12 @@ .. title:: Legal notice .. raw:: html - :file: _templates/legal-notice-content.html \ No newline at end of file + :file: _templates/legal-notice-content.html + +.. toctree:: + :maxdepth: 3 + :hidden: + + Tutorials + API + Contributing \ No newline at end of file From 197e58fa5412dc0ebed39aa4336e4971beeaea66 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:39:12 +0200 Subject: [PATCH 138/333] changes --- docs/legal-notice/index.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/legal-notice/index.rst b/docs/legal-notice/index.rst index 70779f7ce0..54b24b860c 100644 --- a/docs/legal-notice/index.rst +++ b/docs/legal-notice/index.rst @@ -1,6 +1,7 @@ -.. _flair_legal-notice: +Legal Notice +============ -.. title:: Legal notice +.. title:: Legal Notice .. raw:: html :file: _templates/legal-notice-content.html From 2a61780dcc8e0cda682683cb0cc5bf3427d536f7 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:42:50 +0200 Subject: [PATCH 139/333] changes --- docs/legal-notice/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/legal-notice/index.rst b/docs/legal-notice/index.rst index 54b24b860c..5da0252fd4 100644 --- a/docs/legal-notice/index.rst +++ b/docs/legal-notice/index.rst @@ -4,7 +4,7 @@ Legal Notice .. title:: Legal Notice .. raw:: html - :file: _templates/legal-notice-content.html + :file: ../_templates/legal-notice-content.html .. toctree:: :maxdepth: 3 From da9a507659c19841bd8b1096962dd3f98736a979 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:49:29 +0200 Subject: [PATCH 140/333] changes --- docs/_static/css/legal-notice.css | 0 docs/_templates/legal-notice-content.html | 35 ++++++++++++++++++++++- docs/conf.py | 3 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 docs/_static/css/legal-notice.css diff --git a/docs/_static/css/legal-notice.css b/docs/_static/css/legal-notice.css new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/_templates/legal-notice-content.html b/docs/_templates/legal-notice-content.html index 7b25d9a135..7e4951ef01 100644 --- a/docs/_templates/legal-notice-content.html +++ b/docs/_templates/legal-notice-content.html @@ -1,3 +1,36 @@
- HELLO; WORLD; +
+
+
+

Contact

+

+ In case of questions, feel free to open a + Github Issue + or write me an email: + alan.akbik@hu-berlin.de. +

+

+ + Flair NLP is maintained by: + +
+ Alan Akbik
+ Humboldt-Universität zu Berlin
+ Institut für Informatik / Lehrstuhl Maschinelles Lernen
+ Unter den Linden 6
+ 10099 Berlin
+ Germany +

+

+ Website +

+

+ Privacy Policy
+ The webserver / web hosting company might collect certain log files to prevent abuse of services. + These log files can include: IP address, URL, date and time.
+ We do not use any tracking services or cookies to track or re-identify visitors. +

+
+
+
\ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index b1238878bc..f5f0541ab1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -86,7 +86,8 @@ def linkcode_resolve(*args): 'css/version-switcher.css', 'css/sidebar.css', 'css/tutorial.css', - 'css/api.css' + 'css/api.css', + 'css/legal-notice.css', ] html_logo = "_static/flair_logo_white.svg" From 033edd96eb8fe4c7278fdff5c48f3e6ae83b7e36 Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:56:57 +0200 Subject: [PATCH 141/333] changes --- docs/_static/css/legal-notice.css | 6 ++++++ docs/_templates/legal-notice-content.html | 7 +++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/_static/css/legal-notice.css b/docs/_static/css/legal-notice.css index e69de29bb2..e9794bb7b4 100644 --- a/docs/_static/css/legal-notice.css +++ b/docs/_static/css/legal-notice.css @@ -0,0 +1,6 @@ +.legal-notice { + h2 { + font-size: 1.75rem; + font-weight: 600; + } +} \ No newline at end of file diff --git a/docs/_templates/legal-notice-content.html b/docs/_templates/legal-notice-content.html index 7e4951ef01..15a5a0ac8a 100644 --- a/docs/_templates/legal-notice-content.html +++ b/docs/_templates/legal-notice-content.html @@ -1,8 +1,7 @@ -
-
-
+
+
+
-

Contact

In case of questions, feel free to open a Github Issue From b1d0067d094d58cb8b9e8e2eeb35375daa04b73b Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 11:58:46 +0200 Subject: [PATCH 142/333] changes --- docs/_static/css/legal-notice.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/legal-notice.css b/docs/_static/css/legal-notice.css index e9794bb7b4..de85cd6252 100644 --- a/docs/_static/css/legal-notice.css +++ b/docs/_static/css/legal-notice.css @@ -1,5 +1,5 @@ .legal-notice { - h2 { + strong { font-size: 1.75rem; font-weight: 600; } From 5730f85a14c8600e604c3ed2ca20719c07693b3e Mon Sep 17 00:00:00 2001 From: konstantin Date: Sat, 17 Aug 2024 12:14:11 +0200 Subject: [PATCH 143/333] changes --- docs/_static/css/footer.css | 9 ++++++++- docs/_static/css/header.css | 4 ++++ docs/_templates/landing_page_styles.html | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/footer.css b/docs/_static/css/footer.css index 5c92ef4a8b..451e1081c6 100644 --- a/docs/_static/css/footer.css +++ b/docs/_static/css/footer.css @@ -1,8 +1,15 @@ .bd-footer { border: none; background: var(--blue-orange); + height: var(--footer-height); + display: flex; + align-items: center; + *::selection { + background: var(--white-blue); + color: var(--flair-orange); + } .bd-footer__inner { - padding: 1rem 10%; + padding: 0 10%; width: 100%; max-width: initial; margin: 0; diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 4f075f5fc7..fe1c29e1fb 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -3,6 +3,10 @@ height: var(--header-height); background: var(--flair-orange) !important; box-shadow: none; + *::selection { + background: var(--white-blue); + color: var(--flair-orange); + } .bd-header__inner { margin: 0 10%; padding: 0; diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 042cca785c..65cd48af87 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -14,6 +14,10 @@ } .bd-header { background: var(--white-blue) !important; + *::selection { + background: var(--flair-orange); + color: var(--white-blue); + } .navbar-brand.logo { img { visibility: hidden; From c187a7892e3ffe406e3ce2658b265ea6aea6738f Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 07:58:19 +0200 Subject: [PATCH 144/333] changes --- docs/api/flair.data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index 00dd67a521..310d43395d 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -1,4 +1,4 @@ flair.data ========== -.. automodule:: flair.data \ No newline at end of file +.. autoclass:: flair.data.BoundingBox \ No newline at end of file From 1c432a68e8f111890f907fc8c7c2e7ae91b3afa3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 07:59:34 +0200 Subject: [PATCH 145/333] changes --- docs/api/flair.data.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index 310d43395d..d8912d8a1b 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -1,4 +1,5 @@ flair.data ========== -.. autoclass:: flair.data.BoundingBox \ No newline at end of file +.. autoclass:: flair.data.BoundingBox +.. autoclass:: flair.data.Dictionary \ No newline at end of file From 2b8e2dc932c12aa9be6cb07925b5a39446e9e6bd Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:01:04 +0200 Subject: [PATCH 146/333] changes --- docs/api/data/flair.data.rst | 4 ++++ docs/api/flair.data.rst | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 docs/api/data/flair.data.rst delete mode 100644 docs/api/flair.data.rst diff --git a/docs/api/data/flair.data.rst b/docs/api/data/flair.data.rst new file mode 100644 index 0000000000..310d43395d --- /dev/null +++ b/docs/api/data/flair.data.rst @@ -0,0 +1,4 @@ +flair.data +========== + +.. autoclass:: flair.data.BoundingBox \ No newline at end of file diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst deleted file mode 100644 index d8912d8a1b..0000000000 --- a/docs/api/flair.data.rst +++ /dev/null @@ -1,5 +0,0 @@ -flair.data -========== - -.. autoclass:: flair.data.BoundingBox -.. autoclass:: flair.data.Dictionary \ No newline at end of file From aa7c30db7af2a0d382f6e879440e8aa636c6f6cb Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:06:11 +0200 Subject: [PATCH 147/333] changes --- docs/api/data/flair.data.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/data/flair.data.rst b/docs/api/data/flair.data.rst index 310d43395d..5905a0741c 100644 --- a/docs/api/data/flair.data.rst +++ b/docs/api/data/flair.data.rst @@ -1,4 +1,4 @@ -flair.data -========== +flair.data.BoundingBox +====================== .. autoclass:: flair.data.BoundingBox \ No newline at end of file From 597fe29695bf5dcee104a2687ee64f56a93329d9 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:08:45 +0200 Subject: [PATCH 148/333] changes --- docs/api/data/{flair.data.rst => BoundingBox.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/api/data/{flair.data.rst => BoundingBox.rst} (100%) diff --git a/docs/api/data/flair.data.rst b/docs/api/data/BoundingBox.rst similarity index 100% rename from docs/api/data/flair.data.rst rename to docs/api/data/BoundingBox.rst From 6b90498582963a875f4ee792a3ab5c8be9c9d4c7 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:14:02 +0200 Subject: [PATCH 149/333] changes --- docs/api/data/BoundingBox.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api/data/BoundingBox.rst b/docs/api/data/BoundingBox.rst index 5905a0741c..bb02b2067a 100644 --- a/docs/api/data/BoundingBox.rst +++ b/docs/api/data/BoundingBox.rst @@ -1,4 +1,5 @@ flair.data.BoundingBox ====================== -.. autoclass:: flair.data.BoundingBox \ No newline at end of file +.. autoclass:: flair.data.BoundingBox + :members: \ No newline at end of file From 827ca94ad293448cfdfcb62fa4e784fb802800bb Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:17:01 +0200 Subject: [PATCH 150/333] changes --- docs/api/data/BoundingBox.rst | 5 ----- docs/api/flair.data.rst | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 docs/api/data/BoundingBox.rst create mode 100644 docs/api/flair.data.rst diff --git a/docs/api/data/BoundingBox.rst b/docs/api/data/BoundingBox.rst deleted file mode 100644 index bb02b2067a..0000000000 --- a/docs/api/data/BoundingBox.rst +++ /dev/null @@ -1,5 +0,0 @@ -flair.data.BoundingBox -====================== - -.. autoclass:: flair.data.BoundingBox - :members: \ No newline at end of file diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst new file mode 100644 index 0000000000..340436d660 --- /dev/null +++ b/docs/api/flair.data.rst @@ -0,0 +1,4 @@ +flair.data +========== + +.. autoclass:: flair.data \ No newline at end of file From 28cc6d9e3a0ddbeb0818aa05fe5e8a1f42a7a04e Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:18:09 +0200 Subject: [PATCH 151/333] changes --- docs/api/flair.data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index 340436d660..00dd67a521 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -1,4 +1,4 @@ flair.data ========== -.. autoclass:: flair.data \ No newline at end of file +.. automodule:: flair.data \ No newline at end of file From b017f1090719435c3cb56dbc32c0e28b29458684 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:19:47 +0200 Subject: [PATCH 152/333] changes --- docs/api/data/BoundingBox.rst | 4 ++++ docs/api/flair.data.rst | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 docs/api/data/BoundingBox.rst diff --git a/docs/api/data/BoundingBox.rst b/docs/api/data/BoundingBox.rst new file mode 100644 index 0000000000..f1592b254c --- /dev/null +++ b/docs/api/data/BoundingBox.rst @@ -0,0 +1,4 @@ +flair.datasets.BoundingBox +========================== + +.. autoclass:: flair.datasets.BoundingBox \ No newline at end of file diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index 00dd67a521..c124497c2f 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -1,4 +1,8 @@ flair.data ========== -.. automodule:: flair.data \ No newline at end of file +.. toctree:: + :glob: + :maxdepth: 2 + + data/* From 2e2c4166e85fb742ce9b0e8ad29ab0fa0ca42e5f Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:20:28 +0200 Subject: [PATCH 153/333] changes --- docs/api/data/BoundingBox.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/data/BoundingBox.rst b/docs/api/data/BoundingBox.rst index f1592b254c..09a27814f9 100644 --- a/docs/api/data/BoundingBox.rst +++ b/docs/api/data/BoundingBox.rst @@ -1,4 +1,4 @@ -flair.datasets.BoundingBox +flair.data.BoundingBox ========================== -.. autoclass:: flair.datasets.BoundingBox \ No newline at end of file +.. autoclass:: flair.data.BoundingBox \ No newline at end of file From 9ea8a5713b61c01a7aa424d0dea06939f8b94b38 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:24:27 +0200 Subject: [PATCH 154/333] changes --- docs/api/data/DataPoint.rst | 4 ++++ docs/api/data/Dictionary.rst | 4 ++++ docs/api/data/EntityCandidate.rst | 4 ++++ docs/api/data/Label.rst | 4 ++++ 4 files changed, 16 insertions(+) create mode 100644 docs/api/data/DataPoint.rst create mode 100644 docs/api/data/Dictionary.rst create mode 100644 docs/api/data/EntityCandidate.rst create mode 100644 docs/api/data/Label.rst diff --git a/docs/api/data/DataPoint.rst b/docs/api/data/DataPoint.rst new file mode 100644 index 0000000000..e57f7ad264 --- /dev/null +++ b/docs/api/data/DataPoint.rst @@ -0,0 +1,4 @@ +flair.data.DataPoint +==================== + +.. autoclass:: flair.data.DataPoint \ No newline at end of file diff --git a/docs/api/data/Dictionary.rst b/docs/api/data/Dictionary.rst new file mode 100644 index 0000000000..4a5534b96a --- /dev/null +++ b/docs/api/data/Dictionary.rst @@ -0,0 +1,4 @@ +flair.data.Dictionary +===================== + +.. autoclass:: flair.data.Dictionary \ No newline at end of file diff --git a/docs/api/data/EntityCandidate.rst b/docs/api/data/EntityCandidate.rst new file mode 100644 index 0000000000..898f992aa2 --- /dev/null +++ b/docs/api/data/EntityCandidate.rst @@ -0,0 +1,4 @@ +flair.data.EntityCandidate +========================== + +.. autoclass:: flair.data.EntityCandidate \ No newline at end of file diff --git a/docs/api/data/Label.rst b/docs/api/data/Label.rst new file mode 100644 index 0000000000..9d97babe40 --- /dev/null +++ b/docs/api/data/Label.rst @@ -0,0 +1,4 @@ +flair.data.Label +================ + +.. autoclass:: flair.data.Label \ No newline at end of file From d18a9ea4c5d880107efc2106767afc36c203dfee Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:27:14 +0200 Subject: [PATCH 155/333] changes --- docs/api/flair.rst | 6 +++++- docs/api/flair/cache_root.rst | 4 ++++ docs/api/flair/device.rst | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docs/api/flair/cache_root.rst create mode 100644 docs/api/flair/device.rst diff --git a/docs/api/flair.rst b/docs/api/flair.rst index 4e12a03829..3c68468400 100644 --- a/docs/api/flair.rst +++ b/docs/api/flair.rst @@ -1,4 +1,8 @@ flair ===== -.. automodule:: flair \ No newline at end of file +.. toctree:: + :glob: + :maxdepth: 2 + + flair/* diff --git a/docs/api/flair/cache_root.rst b/docs/api/flair/cache_root.rst new file mode 100644 index 0000000000..ddaa73672e --- /dev/null +++ b/docs/api/flair/cache_root.rst @@ -0,0 +1,4 @@ +flair.cache_root +================ + +.. autoclass:: flair.cache_root \ No newline at end of file diff --git a/docs/api/flair/device.rst b/docs/api/flair/device.rst new file mode 100644 index 0000000000..79ff231e60 --- /dev/null +++ b/docs/api/flair/device.rst @@ -0,0 +1,4 @@ +flair.device +============ + +.. autoclass:: flair.device \ No newline at end of file From 745699bc6c577ebbf00780a79a7acb7f7720f0a6 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:29:41 +0200 Subject: [PATCH 156/333] changes --- docs/api/flair.data.rst | 2 +- docs/api/flair.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index c124497c2f..50bf8f0254 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -3,6 +3,6 @@ flair.data .. toctree:: :glob: - :maxdepth: 2 + :maxdepth: 1 data/* diff --git a/docs/api/flair.rst b/docs/api/flair.rst index 3c68468400..f2e9f5a9ab 100644 --- a/docs/api/flair.rst +++ b/docs/api/flair.rst @@ -3,6 +3,6 @@ flair .. toctree:: :glob: - :maxdepth: 2 + :maxdepth: 1 flair/* From 740f839e132db99a47b707d7a9973f4d3737f997 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:32:09 +0200 Subject: [PATCH 157/333] changes --- docs/api/flair/cache_root.rst | 2 +- docs/api/flair/device.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/flair/cache_root.rst b/docs/api/flair/cache_root.rst index ddaa73672e..ee6b8a806d 100644 --- a/docs/api/flair/cache_root.rst +++ b/docs/api/flair/cache_root.rst @@ -1,4 +1,4 @@ flair.cache_root ================ -.. autoclass:: flair.cache_root \ No newline at end of file +.. autodata:: flair.cache_root \ No newline at end of file diff --git a/docs/api/flair/device.rst b/docs/api/flair/device.rst index 79ff231e60..80a29b5c53 100644 --- a/docs/api/flair/device.rst +++ b/docs/api/flair/device.rst @@ -1,4 +1,4 @@ flair.device ============ -.. autoclass:: flair.device \ No newline at end of file +.. autoattribute:: flair.device \ No newline at end of file From 79ff1c2d22b2fc7a4b24d7ef8ed745bf9076f9e0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 08:34:13 +0200 Subject: [PATCH 158/333] changes --- docs/api/flair/device.rst | 2 +- docs/api/flair/logger.rst | 4 ++++ docs/api/flair/set_proxies.rst | 4 ++++ docs/api/flair/set_seed.rst | 4 ++++ 4 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docs/api/flair/logger.rst create mode 100644 docs/api/flair/set_proxies.rst create mode 100644 docs/api/flair/set_seed.rst diff --git a/docs/api/flair/device.rst b/docs/api/flair/device.rst index 80a29b5c53..a172113a31 100644 --- a/docs/api/flair/device.rst +++ b/docs/api/flair/device.rst @@ -1,4 +1,4 @@ flair.device ============ -.. autoattribute:: flair.device \ No newline at end of file +.. autodata:: flair.device \ No newline at end of file diff --git a/docs/api/flair/logger.rst b/docs/api/flair/logger.rst new file mode 100644 index 0000000000..8e14d4304f --- /dev/null +++ b/docs/api/flair/logger.rst @@ -0,0 +1,4 @@ +flair.logger +============ + +.. autodata:: flair.logger \ No newline at end of file diff --git a/docs/api/flair/set_proxies.rst b/docs/api/flair/set_proxies.rst new file mode 100644 index 0000000000..56b883fbf3 --- /dev/null +++ b/docs/api/flair/set_proxies.rst @@ -0,0 +1,4 @@ +flair.set_proxies +================= + +.. autofunction:: flair.set_proxies \ No newline at end of file diff --git a/docs/api/flair/set_seed.rst b/docs/api/flair/set_seed.rst new file mode 100644 index 0000000000..bbdac8bac6 --- /dev/null +++ b/docs/api/flair/set_seed.rst @@ -0,0 +1,4 @@ +flair.set_seed +============== + +.. autofunction:: flair.set_seed \ No newline at end of file From b8d3734b0d08182c8b602f8478c0903cc494fb11 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 09:41:28 +0200 Subject: [PATCH 159/333] changes --- docs/_static/css/search.css | 5 +++++ docs/conf.py | 1 + 2 files changed, 6 insertions(+) create mode 100644 docs/_static/css/search.css diff --git a/docs/_static/css/search.css b/docs/_static/css/search.css new file mode 100644 index 0000000000..588ba5cbfa --- /dev/null +++ b/docs/_static/css/search.css @@ -0,0 +1,5 @@ +.bd-search-container { + h1 { + color: white; + } +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index f5f0541ab1..752ee2f5b0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,6 +88,7 @@ def linkcode_resolve(*args): 'css/tutorial.css', 'css/api.css', 'css/legal-notice.css', + 'css/search.css', ] html_logo = "_static/flair_logo_white.svg" From 0e43045be8b93594163a784820a98fabd29d7de3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 09:55:15 +0200 Subject: [PATCH 160/333] changes --- docs/_static/css/search.css | 49 ++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/search.css b/docs/_static/css/search.css index 588ba5cbfa..08318970fd 100644 --- a/docs/_static/css/search.css +++ b/docs/_static/css/search.css @@ -1,5 +1,52 @@ .bd-search-container { h1 { - color: white; + color: var(--blue-white); + font-size: 3rem; + font-weight: 600; + } + h2 { + color: var(--blue-white); + font-weight: 600; + font-size: 1.5rem; + } + form.bd-search { + i { + display: none; + } + input { + border: 2px var(--flair-orange) solid; + color: var(--flair-blue); + box-shadow: none; + padding: .25em 5.5em .25em .75em; + border-radius: 1rem; + font-size: 1.2rem; + } + .search-button__kbd-shortcut { + align-items: center; + } + } + p.search-summary { + color: var(--blue-white) !important; + } + ul.search { + li { + border-top: 2px solid var(--blue-white); + margin: 0 0 .05em 0; + &:first-of-type { + margin-top: 1rem; + border: none; + } + a { + font-weight: 400; + } + span { + color: var(--blue-white); + } + .highlighted { + background: var(--flair-orange); + color: white; + border-radius: 5px; + } + } } } \ No newline at end of file From c2da5d031c1465d5714770b723d8a6566e3c4e93 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 10:05:56 +0200 Subject: [PATCH 161/333] changes --- docs/_static/css/search.css | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/_static/css/search.css b/docs/_static/css/search.css index 08318970fd..62fd54bb4a 100644 --- a/docs/_static/css/search.css +++ b/docs/_static/css/search.css @@ -42,10 +42,30 @@ span { color: var(--blue-white); } - .highlighted { - background: var(--flair-orange); - color: white; - border-radius: 5px; + } + } +} +.highlighted { + background: var(--flair-orange) !important; + color: white !important; + border-radius: 5px !important; +} +div#searchbox { + p.highlight-link { + box-shadow: none !important; + margin: 1rem 0; + a { + background: var(--white-blue) !important; + border: 2px var(--flair-orange) solid; + color: var(--flair-orange); + transition: all 200ms ease-in-out; + &:hover { + background: var(--flair-orange) !important; + color: white !important; + box-shadow: none !important; + } + &::before { + content: none; } } } From 69f70a028f48df17b91f00fa7269f5f82a56d64b Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 10:27:06 +0200 Subject: [PATCH 162/333] changes --- docs/_templates/landing_page_styles.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 65cd48af87..f06475f654 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -197,4 +197,17 @@ #searchbox { display: none; } + + + @media screen and (max-width: 1600px) { + .landing-page { + .banner { + .right-side { + .card { + background: red; + } + } + } + } + } From 6f00704d2e2d21d18f741a73cd953e313a9000ca Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 10:40:34 +0200 Subject: [PATCH 163/333] changes --- docs/_templates/landing_page_styles.html | 70 +++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index f06475f654..88b9229fc0 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -204,7 +204,75 @@ .banner { .right-side { .card { - background: red; + margin-left: 0; + margin-right: 2rem; + width: 25%; + } + } + } + } + } + @media screen and (max-width: 1250px) { + .landing-page { + .banner { + .right-side { + .card { + margin-right: 1rem; + width: 28%; + } + } + } + } + } + @media screen and (max-width: 1100px) { + .landing-page { + .banner { + height: auto; + flex-direction: column; + .left-side { + margin-top: 3rem; + width: 100%; + svg { + width: 50%; + } + } + .right-side { + width: 100%; + padding: 0 10%; + justify-content: space-between; + .card { + margin: 0; + width: 23vw; + height: 28vw; + } + margin-bottom: 5rem; + margin-top: 2rem; + } + } + } + } + @media screen and (max-width: 700px) { + .landing-page { + .banner { + .right-side { + .card { + span { + display: none; + } + } + } + } + } + } + @media screen and (max-width: 700px) { + .landing-page { + .banner { + .left-side { + svg { + width: 75%; + } + span { + font-size: 1.2rem; } } } From 176df4b4f8b52b7649cba80b5146b3e1d83b01eb Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 10:55:24 +0200 Subject: [PATCH 164/333] changes --- docs/_templates/landing_page_styles.html | 41 ++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 88b9229fc0..b8a732ef36 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -250,17 +250,25 @@ } } } - } - @media screen and (max-width: 700px) { - .landing-page { - .banner { - .right-side { - .card { - span { - display: none; - } + .landing-page-illustrations { + flex-direction: column; + .item { + align-items: center; + padding: 2rem 0 5rem 0; + svg { + width: 50%; + height: auto; + & + span { + margin: 2rem 0 0 0; } } + &:not(:last-of-type) { + border-right: none; + } + p { + text-align: center; + max-width: 500px; + } } } } @@ -275,6 +283,21 @@ font-size: 1.2rem; } } + .right-side { + .card { + span { + display: none; + } + } + } + } + } + .landing-page-illustrations { + padding: 3rem; + .item { + svg { + width: 100%; + } } } } From 398bdd563bc5c7edafc27f486b7f3dc5bba7b2f9 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 10:56:34 +0200 Subject: [PATCH 165/333] changes --- docs/_templates/landing_page_styles.html | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index b8a732ef36..6e569b58aa 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -301,4 +301,14 @@ } } } + @media screen and (max-width: 400px) { + .landing-page-illustrations { + padding: 2rem; + .item { + svg { + width: 100%; + } + } + } + } From a11c933b96dce6d4d46a3286fb4a2fd5f1c45472 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 12:09:05 +0200 Subject: [PATCH 166/333] changes --- docs/_static/css/header.css | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index fe1c29e1fb..6870dcc13e 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -155,4 +155,23 @@ .stroke-moon { stroke: white; fill: none; +} + +@media screen and (max-width: 1250px) { + .bd-header { + .bd-header__inner { + nav { + ul.bd-navbar-elements { + li.nav-item { + margin-left: 2rem; + } + } + } + .navbar-header-items__end { + .navbar-item:not(:last-of-type) { + margin-right: 1rem; + } + } + } + } } \ No newline at end of file From 438028636925d345377637b9825059dba430eb84 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 12:11:39 +0200 Subject: [PATCH 167/333] changes --- docs/_templates/version-switcher.html | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/_templates/version-switcher.html b/docs/_templates/version-switcher.html index 642039f919..ad1676b4d8 100644 --- a/docs/_templates/version-switcher.html +++ b/docs/_templates/version-switcher.html @@ -39,15 +39,7 @@ document.querySelector('footer.bd-footer') From be4ce85e5e2b719fef17777101c823e803432e05 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 12:17:20 +0200 Subject: [PATCH 168/333] changes --- docs/_static/css/header.css | 41 +++++++++++++----------- docs/_templates/landing_page_styles.html | 5 +++ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 6870dcc13e..68f44283f2 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -79,23 +79,6 @@ .navbar-item:not(:last-of-type) { margin-right: 2rem; } - .navbar-item:first-of-type { - button.search-button { - background-size: contain; - background-repeat: no-repeat; - background-position: center; - border-radius: 0; - height: 1.85rem; - width: 1.85rem; - i { - visibility: hidden; - &:focus, &, &:active { - border: none; - outline: none; - } - } - } - } .navbar-item:nth-of-type(2) { svg { cursor: pointer; @@ -157,6 +140,28 @@ fill: none; } +button.search-button { + background-size: contain; + background-repeat: no-repeat; + background-position: center; + border-radius: 0; + height: 1.85rem; + width: 1.85rem; + i { + visibility: hidden; + &:focus, &, &:active { + border: none; + outline: none; + } + } +} + +.sidebar-toggle { + span { + color: white; + } +} + @media screen and (max-width: 1250px) { .bd-header { .bd-header__inner { @@ -174,4 +179,4 @@ } } } -} \ No newline at end of file +} diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 6e569b58aa..f7ac2306d8 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -197,6 +197,11 @@ #searchbox { display: none; } + .sidebar-toggle { + span { + color: var(--blue-white); + } + } @media screen and (max-width: 1600px) { From 7a1c2bef04c2187ccb560ec8ed8c3a35df5ccf5c Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 12:28:52 +0200 Subject: [PATCH 169/333] changes --- docs/_static/css/header.css | 44 ++++++++++++++++++---------- docs/_templates/darkmode-toggle.html | 2 +- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 68f44283f2..fc3e0b6c7d 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -79,22 +79,6 @@ .navbar-item:not(:last-of-type) { margin-right: 2rem; } - .navbar-item:nth-of-type(2) { - svg { - cursor: pointer; - &:first-of-type { - :root .dark-mode & { - display: none; - } - } - &:nth-of-type(2) { - display: none; - :root .dark-mode & { - display: block; - } - } - } - } svg { height: 35%; width: auto; @@ -159,6 +143,23 @@ button.search-button { .sidebar-toggle { span { color: white; + font-size: 2rem; + margin-right: 1rem; + } +} + +.dark-mode-toggle svg { + cursor: pointer; + &:first-of-type { + :root .dark-mode & { + display: none; + } + } + &:nth-of-type(2) { + display: none; + :root .dark-mode & { + display: block; + } } } @@ -180,3 +181,14 @@ button.search-button { } } } + + +@media screen and (max-width: 959px) { + .bd-sidebar-primary.bd-sidebar.hide-on-wide { + width: 100%; + max-width: initial; + .bd-navbar-elements { + padding: 0 10%; + } + } +} diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index 749081b8e3..eccabccba3 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -1,4 +1,4 @@ -

+
From a308bc9d11785ad00b0a4b25718f998dfa59daad Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 13:13:42 +0200 Subject: [PATCH 170/333] changes --- docs/_static/css/footer.css | 45 ++++++++++++++++++++++++++++++++++++- docs/_static/css/header.css | 14 +++++++++--- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/_static/css/footer.css b/docs/_static/css/footer.css index 451e1081c6..b97ae22cd4 100644 --- a/docs/_static/css/footer.css +++ b/docs/_static/css/footer.css @@ -28,4 +28,47 @@ } } } -} \ No newline at end of file +} + +@media screen and (max-width: 959px) { + .bd-footer { + font-size: 0.8rem; + .bd-footer__inner { + .footer-items__end { + .footer-item:not(:last-of-type) { + margin-right: 2rem; + } + } + } + } +} + + + +@media screen and (max-width: 700px) { + .bd-footer { + font-size: 1rem; + .bd-footer__inner { + .footer-items__start { + display: none; + } + .footer-items__end { + width: 100%; + justify-content: space-between; + .footer-item:not(:last-of-type) { + margin-right: 1rem; + } + } + } + } +} + + + + +@media screen and (max-width: 435px) { + .bd-footer { + font-size: 0.8rem; + } +} + diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index fc3e0b6c7d..3a734417d2 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -185,10 +185,18 @@ button.search-button { @media screen and (max-width: 959px) { .bd-sidebar-primary.bd-sidebar.hide-on-wide { - width: 100%; + width: 100vw; max-width: initial; - .bd-navbar-elements { - padding: 0 10%; + padding: 10vw; + box-sizing: border-box; + .sidebar-header-items { + flex-direction: row; + max-width: 100%; + justify-content: space-between; + align-items: flex-start; + .sidebar-header-items__title, .nav-item .nav-link { + color: var(--orange-white); + } } } } From 15c65173e828c99ec1d2deb32ffe4211bd78fe05 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 14:30:11 +0200 Subject: [PATCH 171/333] changes --- docs/_static/css/header.css | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index 3a734417d2..ea0ef898ad 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -184,19 +184,39 @@ button.search-button { @media screen and (max-width: 959px) { - .bd-sidebar-primary.bd-sidebar.hide-on-wide { - width: 100vw; - max-width: initial; - padding: 10vw; - box-sizing: border-box; + .bd-sidebar-primary.bd-sidebar { + width: 100vw !important; + background: var(--flair-orange) !important; + height: calc(100vh - var(--header-height)); + max-width: initial !important; + display: block; + overflow-y: auto !important; + overflow-x: hidden !important; .sidebar-header-items { flex-direction: row; max-width: 100%; justify-content: space-between; align-items: flex-start; .sidebar-header-items__title, .nav-item .nav-link { - color: var(--orange-white); + color: white; } + padding: 10vw 10vw 5vw 10vw !important; + } + .sidebar-primary-items__start { + padding: 5vw 10vw 10vw 10vw !important; + border-top: 2px solid white; + } + .sidebar-header-items__end { + gap: 0; + } + .current { + color: var(--flair-blue) !important; + } + a:hover { + color: var(--flair-blue) !important; + } + * { + color: white !important; } } } From 104aa7da70fb79ca7ce61d5704aafc1729b73055 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 14:47:53 +0200 Subject: [PATCH 172/333] changes --- docs/_static/css/tutorial.css | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 8b1e0d72e9..c553fbcf6b 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -266,4 +266,26 @@ table.table {border-collapse: collapse; font-size: 1.2rem; } } -} \ No newline at end of file +} + +@media screen and (max-width: 700px) { + .bd-article-container { + padding: 3rem !important; + } +} + +@media screen and (max-width: 500px) { + #flair-tutorials, #api-docs, #contributing, [id^="tutorial"], [id^="flair"] { + & .toctree-wrapper li[class^="toctree-l"] { + & > a { + font-size: 1rem; + } + } + } + .toctree-wrapper li[class^="toctree-l"] > ul { + padding-inline-start: 0; + } + .bd-article-container { + padding: 2rem !important; + } +} From eac0d925391b7d59ac834fabd05cc410b89f5db1 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 15:14:59 +0200 Subject: [PATCH 173/333] changes --- docs/_static/css/api.css | 16 ++++++++++++++++ docs/_static/css/tutorial.css | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 5fd8ef8c89..841462d145 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -15,6 +15,14 @@ color: var(--blue-white); font-family: "Source Code Pro", monospace; line-height: 2em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; + span.pre { + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; + } .linkcode-link { padding: 0; display: inline-block; @@ -129,4 +137,12 @@ blockquote { p { margin: 0 !important; } +} + +@media screen and (max-width: 500px) { + .class, .method, .function, .data { + & dd { + padding-left: 1.5rem !important; + } + } } \ No newline at end of file diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index c553fbcf6b..12ebbb4fcd 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -14,18 +14,27 @@ color: var(--blue-white); font-weight: 600; margin-top: 0.5em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; } h2 { font-size: 1.6rem; color: var(--blue-white); font-weight: 600; margin-top: 2em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; } h3 { font-size: 1.2rem; color: var(--blue-white); font-weight: 600; margin-top: 2em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; } p { color: var(--blue-white); From 9adc796a714787cbd622066fa43f4028863a7966 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 15:30:23 +0200 Subject: [PATCH 174/333] changes --- docs/_static/css/api.css | 2 +- docs/_static/css/tutorial.css | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 841462d145..822caee761 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -139,7 +139,7 @@ blockquote { } } -@media screen and (max-width: 500px) { +@media screen and (max-width: 1200px) { .class, .method, .function, .data { & dd { padding-left: 1.5rem !important; diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 12ebbb4fcd..00aaf40860 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -298,3 +298,23 @@ table.table {border-collapse: collapse; padding: 2rem !important; } } + +@media screen and (max-width: 1200px) { + .bd-main .bd-content .bd-article-container .bd-article { + + h1 { + font-size: 2rem; + } + + h2 { + font-size: 1.2rem; + } + + h3 { + font-size: 1rem; + } + } + .bd-article-container { + padding: 4rem; + } +} \ No newline at end of file From 8906c194fbba91932313668f683e0187a45207cd Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 15:50:35 +0200 Subject: [PATCH 175/333] changes --- docs/_static/css/api.css | 35 ++++++++++++++++++++++++++++++++--- docs/_static/css/tutorial.css | 31 +++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 822caee761..55d1f11b0f 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -19,9 +19,9 @@ overflow-wrap: break-word; white-space: normal; span.pre { - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; } .linkcode-link { padding: 0; @@ -145,4 +145,33 @@ blockquote { padding-left: 1.5rem !important; } } +} + +@media screen and (max-width: 900px) { + .sig, .sig * { + font-size: .8rem; + } + .class, .method, .function, .data { + & dd { + & dl.field-list { + padding: 1rem; + } + } + } +} + +@media screen and (max-width: 400px) { + dl { + * { + font-size: .8rem !important; + } + code { + font-size: .6rem !important; + } + } + .class, .method, .function, .data { + & dd { + padding-left: 1rem !important; + } + } } \ No newline at end of file diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 00aaf40860..742faf3438 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -6,6 +6,11 @@ .bd-main .bd-content .bd-article-container .bd-article { padding: 0; + * { + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; + } .headerlink { display: none; } @@ -141,6 +146,9 @@ } ul.bd-breadcrumbs { + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; li.breadcrumb-home .fa-home { &::before { content: "Home"; @@ -317,4 +325,27 @@ table.table {border-collapse: collapse; .bd-article-container { padding: 4rem; } +} + +@media screen and (max-width: 900px) { + .bd-main .bd-content .bd-article-container .bd-article { + + h1 { + font-size: 1.6rem; + } + + h2 { + font-size: 1rem; + } + + h3 { + font-size: 1rem; + } + } + .bd-article-container { + padding: 4rem; + } + .bd-header-article { + margin-bottom: 3rem; + } } \ No newline at end of file From 014e012a0690ed7963f8322f95f767b63313a5fb Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 21 Aug 2024 15:53:35 +0200 Subject: [PATCH 176/333] changes --- docs/_static/css/tutorial.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 742faf3438..a4f12d66e7 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -348,4 +348,7 @@ table.table {border-collapse: collapse; .bd-header-article { margin-bottom: 3rem; } + pre, pre * { + font-size: 0.8rem; + } } \ No newline at end of file From 680320eb0e44423febe1f833446d2ad3533b850c Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 17:54:26 +0200 Subject: [PATCH 177/333] changes --- docs/_templates/landing_page_styles.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index f7ac2306d8..8300c8825e 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -278,6 +278,9 @@ } } @media screen and (max-width: 700px) { + .bd-article-container { + padding: 0; + } .landing-page { .banner { .left-side { From 4ac2ae836588a1856a32f89d0b245c5cbe7d78c4 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 17:56:14 +0200 Subject: [PATCH 178/333] changes --- docs/_templates/landing_page_styles.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 8300c8825e..15fce16b21 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -279,7 +279,7 @@ } @media screen and (max-width: 700px) { .bd-article-container { - padding: 0; + padding: 0 !important; } .landing-page { .banner { From 247266a67a64960aada213ba3960aaef2efda8d5 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 18:07:01 +0200 Subject: [PATCH 179/333] changes --- docs/_templates/landing_page_styles.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 15fce16b21..c32096693a 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -281,6 +281,9 @@ .bd-article-container { padding: 0 !important; } + .bd-header-article { + margin: 0 !important; + } .landing-page { .banner { .left-side { From e35e41ff73952e5ad00ba5def4b829abc68fc334 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 18:08:37 +0200 Subject: [PATCH 180/333] changes --- docs/_templates/landing_page_styles.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index c32096693a..c7495a62aa 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -282,7 +282,7 @@ padding: 0 !important; } .bd-header-article { - margin: 0 !important; + margin-bottom: 0 !important; } .landing-page { .banner { From 210cc56cd39033be62f58183cb69e85e48ae9183 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 18:09:41 +0200 Subject: [PATCH 181/333] changes --- docs/_templates/landing_page_styles.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index c7495a62aa..5eea8126aa 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -277,13 +277,15 @@ } } } + @media screen and (max-width: 900px) { + .bd-header-article { + margin-bottom: 0 !important; + } + } @media screen and (max-width: 700px) { .bd-article-container { padding: 0 !important; } - .bd-header-article { - margin-bottom: 0 !important; - } .landing-page { .banner { .left-side { From e945a6410f43867abcc76e123b0f3aa3ae10e0cb Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 18:35:26 +0200 Subject: [PATCH 182/333] changes --- docs/_static/css/tutorial.css | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index a4f12d66e7..7b738a0378 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -289,13 +289,34 @@ table.table {border-collapse: collapse; .bd-article-container { padding: 3rem !important; } + .admonition, div.admonition, .admonition.note, div.admonition.note { + p { + padding-left: 1rem !important; + &.admonition-title { + padding-left: 1rem !important; + &::after { + position: relative; + left: -0.1rem; + } + } + } + } + .deprecated { + padding-left: 1rem !important; + .deprecated { + padding-left: 0 !important; + } + &::before { + position: relative !important; + } + } } @media screen and (max-width: 500px) { #flair-tutorials, #api-docs, #contributing, [id^="tutorial"], [id^="flair"] { - & .toctree-wrapper li[class^="toctree-l"] { - & > a { - font-size: 1rem; + & .toctree-wrapper li[class^="toctree-l"]{ + & > a, & > a span { + font-size: 1rem !important; } } } From 62aa8f752037782772caaf681a5587354fc2239d Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 18:47:38 +0200 Subject: [PATCH 183/333] changes --- docs/_templates/darkmode-toggle.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index eccabccba3..c83af9e93d 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -123,6 +123,15 @@ if (window.matchMedia("(prefers-color-scheme: dark)").matches) { document.body.classList.add("dark-mode"); } + const matchPreferredScheme = (e) => { + if (e.matches) { + document.body.classList.add("dark-mode"); + } else { + document.body.classList.remove("dark-mode"); + } + } + + window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', matchPreferredScheme); const toggleDarkMode = () => { if (document.body.classList.contains("dark-mode")) { document.body.classList.remove("dark-mode"); From 92b669ec40aff55b62dd8ea19648dfe489080823 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 18:57:01 +0200 Subject: [PATCH 184/333] changes --- docs/_templates/darkmode-toggle.html | 42 +++++++++++++++++----------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index c83af9e93d..bb89e2ac1f 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -120,23 +120,33 @@
From 608c57fe331e33005ff2fb4cf721409420e0dbd6 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:00:04 +0200 Subject: [PATCH 185/333] changes --- docs/_templates/darkmode-toggle.html | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index bb89e2ac1f..2545e708ca 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -143,10 +143,7 @@ window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', matchPreferredScheme); const toggleDarkMode = () => { - if (document.body.classList.contains("dark-mode")) { - document.body.classList.remove("dark-mode"); - } else { - document.body.classList.add("dark-mode"); - } + localStorage.setItem("theme", document.body.classList.contains("dark-mode") ? "dark" : "light"); + applyTheme(); } From 23802e83779a6b8b15c25e5c60c51d206ad8c190 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:01:01 +0200 Subject: [PATCH 186/333] changes --- docs/_templates/darkmode-toggle.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index 2545e708ca..ab0b50d81a 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -143,7 +143,7 @@ window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', matchPreferredScheme); const toggleDarkMode = () => { - localStorage.setItem("theme", document.body.classList.contains("dark-mode") ? "dark" : "light"); + localStorage.setItem("theme", document.body.classList.contains("dark-mode") ? "light" : "dark"); applyTheme(); } From b6c0f2d72cdadc86fad770e0c89d477c0b4aa60f Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:03:02 +0200 Subject: [PATCH 187/333] changes --- docs/_templates/darkmode-toggle.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index ab0b50d81a..40133f856b 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -143,7 +143,8 @@ window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', matchPreferredScheme); const toggleDarkMode = () => { - localStorage.setItem("theme", document.body.classList.contains("dark-mode") ? "light" : "dark"); + const theme = localStorage.getItem("theme"); + localStorage.setItem("theme", theme === "dark" ? "light" : "dark"); applyTheme(); } From 6da53bb8ab0b5a881d8be0b6ca74079a389eb137 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:04:17 +0200 Subject: [PATCH 188/333] changes --- docs/_templates/darkmode-toggle.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index 40133f856b..250b4feda2 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -122,6 +122,7 @@ From b189d13f57e87bf0ddfd78132a07a297f2a6e699 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:23:33 +0200 Subject: [PATCH 196/333] changes --- docs/_templates/darkmode-toggle.html | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index fe71c722cd..131e66a454 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -121,7 +121,7 @@
From db1179bab6ab66fc96e15b0f6102c6000006c52d Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:25:20 +0200 Subject: [PATCH 197/333] changes --- docs/_templates/darkmode-toggle.html | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/_templates/darkmode-toggle.html b/docs/_templates/darkmode-toggle.html index 131e66a454..f7e0658f94 100644 --- a/docs/_templates/darkmode-toggle.html +++ b/docs/_templates/darkmode-toggle.html @@ -135,7 +135,6 @@ localStorage.setItem("theme-color", "light"); } } - console.log(theme) } function matchPreferredScheme (e) { From 72e66bf3fbc0695706316e98f16d54a74003c7f8 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:28:12 +0200 Subject: [PATCH 198/333] changes --- docs/_templates/landing_page_styles.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 5eea8126aa..67ed6ff0a1 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -181,9 +181,9 @@ .navbar-header-items__end { .navbar-item:first-of-type { button.search-button { - background-image: url("_static/magnifying_glass.svg"); + background-image: url("_static/magnifying_glass.svg") !important; :root .dark-mode & { - background-image: url("_static/magnifying_glass_dark.svg"); + background-image: url("_static/magnifying_glass_dark.svg") !important; } } } From 1f5376aca20ab196395eb2b922c8a3b9cf4d282d Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:31:23 +0200 Subject: [PATCH 199/333] changes --- docs/_templates/landing_page_styles.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 67ed6ff0a1..428dd40d17 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -190,6 +190,14 @@ } } } + .navbar-persistent--mobile { + button.search-button { + background-image: url("_static/magnifying_glass.svg") !important; + :root .dark-mode & { + background-image: url("_static/magnifying_glass_dark.svg") !important; + } + } + } .stroke-moon { stroke: var(--orange-white); fill: none; From 00da17a4f1b959022b2be12560f4e3b59bb5e04a Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:35:39 +0200 Subject: [PATCH 200/333] changes --- docs/_templates/landing_page_styles.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 428dd40d17..24b84b1fcd 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -198,6 +198,11 @@ } } } + .dark-mode-toggle { + svg:first-of-type { + display: none; + } + } .stroke-moon { stroke: var(--orange-white); fill: none; @@ -210,6 +215,10 @@ color: var(--blue-white); } } + .stroke-moon { + stroke: white !important; + fill: none !important; + } @media screen and (max-width: 1600px) { From df624f675b41ebbbb9569e8487600c4e5060a4b4 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:37:12 +0200 Subject: [PATCH 201/333] changes --- docs/_templates/landing_page_styles.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index 24b84b1fcd..f457bb6301 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -198,11 +198,6 @@ } } } - .dark-mode-toggle { - svg:first-of-type { - display: none; - } - } .stroke-moon { stroke: var(--orange-white); fill: none; From 1ecbe1c523fa6478204b2e5ceaefa9de152c5171 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:39:58 +0200 Subject: [PATCH 202/333] changes --- docs/_templates/landing_page_styles.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/landing_page_styles.html b/docs/_templates/landing_page_styles.html index f457bb6301..d98569463a 100644 --- a/docs/_templates/landing_page_styles.html +++ b/docs/_templates/landing_page_styles.html @@ -210,7 +210,7 @@ color: var(--blue-white); } } - .stroke-moon { + .sidebar-primary__section .stroke-moon { stroke: white !important; fill: none !important; } From accd03cc372429a50515d43ad3f37370fb778d8c Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:43:23 +0200 Subject: [PATCH 203/333] changes --- docs/_static/css/api.css | 300 +++++------ docs/_static/css/footer.css | 118 ++--- docs/_static/css/header.css | 399 +++++++------- docs/_static/css/legal-notice.css | 8 +- docs/_static/css/main.css | 39 +- docs/_static/css/search.css | 142 ++--- docs/_static/css/sidebar.css | 147 +++--- docs/_static/css/tutorial.css | 735 ++++++++++++++------------ docs/_static/css/version-switcher.css | 142 ++--- 9 files changed, 1056 insertions(+), 974 deletions(-) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 55d1f11b0f..4666de16ea 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -1,177 +1,179 @@ -.sidebar-primary-item { - .docutils { - &::before { - content: none; - } - .pre { - font-family: "Afacad", sans-serif; - font-size: 1rem; - text-overflow: ellipsis; - } - } +.sidebar-primary-item .docutils::before { + content: none; +} + +.sidebar-primary-item .docutils .pre { + font-family: "Afacad", sans-serif; + font-size: 1rem; + text-overflow: ellipsis; } .sig { - color: var(--blue-white); - font-family: "Source Code Pro", monospace; - line-height: 2em; - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; - span.pre { - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; - } - .linkcode-link { - padding: 0; - display: inline-block; - width: 1rem; - height: 1rem; - transform: translateY(10%) scale(1.5); - .pre { - display: none; - } - &::after { - content:""; - background-image: url('../../_static/octocat.svg'); - background-size: cover; - display: inline-block; - width: 100%; - height: 100%; - } - } + color: var(--blue-white); + font-family: "Source Code Pro", monospace; + line-height: 2em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; +} + +.sig span.pre { + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; +} + +.sig .linkcode-link { + padding: 0; + display: inline-block; + width: 1rem; + height: 1rem; + transform: translateY(10%) scale(1.5); +} + +.sig .linkcode-link .pre { + display: none; +} + +.sig .linkcode-link::after { + content: ""; + background-image: url("../../_static/octocat.svg"); + background-size: cover; + display: inline-block; + width: 100%; + height: 100%; } .sig-name.descname { - color: var(--flair-orange); + color: var(--flair-orange); } .sig { - margin-top: 2rem; - margin-bottom: .5em; - padding-bottom: .5em; - padding-top: .5em; - &:target { - background: rgba(47, 46, 65, 0.15); - border-radius: .5rem; - } - :root .dark-mode &:target { - background: rgba(255, 255, 255, 0.15); - } -} - -.class, .method, .function, .data { - dd { - margin-left: 0 !important; - padding-left: 3rem !important; - border-left: 2px var(--gray-white) solid; - dl.field-list { - background-color: transparent; - border: 2px var(--flair-orange) solid; - border-radius: 1rem; - padding: 1rem 1.5rem; - ul { - padding-left: 0 !important; - } - dt { - background-color: transparent !important; - color: var(--blue-white); - font-weight: 600; - font-size: 1.5rem; - padding: 0 !important; - margin: 0 !important; - } - dd { - margin: 0 !important; - padding: 0 !important; - border-left: none; - } - } - } - .n { - color: #2AACB8; - } - .default_value { - color: #8888C6; - } + margin-top: 2rem; + margin-bottom: .5em; + padding-bottom: .5em; + padding-top: .5em; +} + +.sig:target { + background: rgba(47, 46, 65, 0.15); + border-radius: .5rem; +} + +:root .dark-mode .sig:target { + background: rgba(255, 255, 255, 0.15); +} + +.class dd, .method dd, .function dd, .data dd { + margin-left: 0 !important; + padding-left: 3rem !important; + border-left: 2px var(--gray-white) solid; +} + +.class dd dl.field-list, .method dd dl.field-list, .function dd dl.field-list, .data dd dl.field-list { + background-color: transparent; + border: 2px var(--flair-orange) solid; + border-radius: 1rem; + padding: 1rem 1.5rem; +} + +.class dd dl.field-list ul, .method dd dl.field-list ul, .function dd dl.field-list ul, .data dd dl.field-list ul { + padding-left: 0 !important; +} + +.class dd dl.field-list dt, .method dd dl.field-list dt, .function dd dl.field-list dt, .data dd dl.field-list dt { + background-color: transparent !important; + color: var(--blue-white); + font-weight: 600; + font-size: 1.5rem; + padding: 0 !important; + margin: 0 !important; +} + +.class dd dl.field-list dd, .method dd dl.field-list dd, .function dd dl.field-list dd, .data dd dl.field-list dd { + margin: 0 !important; + padding: 0 !important; + border-left: none; +} + +.class .n, .method .n, .function .n, .data .n { + color: #2AACB8; +} + +.class .default_value, .method .default_value, .function .default_value, .data .default_value { + color: #8888C6; } div.deprecated { - box-shadow: none !important; - background: var(--error-red); - border: 2px var(--error-red) solid; - border-radius: 1rem; - padding: 0.5em 2.2em 0 2.2em; - box-sizing: border-box; - p { - span.versionmodified.deprecated::before { - color: white; - } - &::before { - content: none; - } - } + box-shadow: none !important; + background: var(--error-red); + border: 2px var(--error-red) solid; + border-radius: 1rem; + padding: 0.5em 2.2em 0 2.2em; + box-sizing: border-box; +} + +div.deprecated p span.versionmodified.deprecated::before { + color: white; +} + +div.deprecated p::before { + content: none; } a.github::before { - content: ""; - height: 1em; - width: 1em; - display: inline-block; - background-image: url("../../_static/octocat.svg"); - background-size: contain; - background-position: center; - transform: translateY(25%); + content: ""; + height: 1em; + width: 1em; + display: inline-block; + background-image: url("../../_static/octocat.svg"); + background-size: contain; + background-position: center; + transform: translateY(25%); } blockquote { - font-family: "Source Code Pro", monospace; - border-radius: 1rem; - padding: 1.1em 1.5em; - background: var(--gray-dark-blue); - color: var(--blue-white); - &::before { - content: none; - } - p { - margin: 0 !important; - } + font-family: "Source Code Pro", monospace; + border-radius: 1rem; + padding: 1.1em 1.5em; + background: var(--gray-dark-blue); + color: var(--blue-white); +} + +blockquote::before { + content: none; +} + +blockquote p { + margin: 0 !important; } @media screen and (max-width: 1200px) { - .class, .method, .function, .data { - & dd { - padding-left: 1.5rem !important; - } - } + .class dd, .method dd, .function dd, .data dd { + padding-left: 1.5rem !important; + } } @media screen and (max-width: 900px) { - .sig, .sig * { - font-size: .8rem; - } - .class, .method, .function, .data { - & dd { - & dl.field-list { - padding: 1rem; - } - } - } + .sig, .sig * { + font-size: .8rem; + } + + .class dd dl.field-list, .method dd dl.field-list, .function dd dl.field-list, .data dd dl.field-list { + padding: 1rem; + } } @media screen and (max-width: 400px) { - dl { - * { - font-size: .8rem !important; - } - code { - font-size: .6rem !important; - } - } - .class, .method, .function, .data { - & dd { - padding-left: 1rem !important; - } - } + dl * { + font-size: .8rem !important; + } + + dl code { + font-size: .6rem !important; + } + + .class dd, .method dd, .function dd, .data dd { + padding-left: 1rem !important; + } } \ No newline at end of file diff --git a/docs/_static/css/footer.css b/docs/_static/css/footer.css index b97ae22cd4..a5c39f7e40 100644 --- a/docs/_static/css/footer.css +++ b/docs/_static/css/footer.css @@ -1,74 +1,72 @@ .bd-footer { - border: none; - background: var(--blue-orange); - height: var(--footer-height); - display: flex; - align-items: center; - *::selection { - background: var(--white-blue); - color: var(--flair-orange); - } - .bd-footer__inner { - padding: 0 10%; - width: 100%; - max-width: initial; - margin: 0; - box-sizing: border-box; - .footer-item * { - color: white; - } - a:hover { - color: var(--orange-blue); - } - .footer-items__end { - display: flex; - flex-direction: row; - .footer-item:not(:last-of-type) { - margin-right: 3rem; - } - } - } + border: none; + background: var(--blue-orange); + height: var(--footer-height); + display: flex; + align-items: center; } -@media screen and (max-width: 959px) { - .bd-footer { - font-size: 0.8rem; - .bd-footer__inner { - .footer-items__end { - .footer-item:not(:last-of-type) { - margin-right: 2rem; - } - } - } - } +.bd-footer *::selection { + background: var(--white-blue); + color: var(--flair-orange); } +.bd-footer .bd-footer__inner { + padding: 0 10%; + width: 100%; + max-width: initial; + margin: 0; + box-sizing: border-box; +} +.bd-footer .bd-footer__inner .footer-item * { + color: white; +} -@media screen and (max-width: 700px) { - .bd-footer { - font-size: 1rem; - .bd-footer__inner { - .footer-items__start { - display: none; - } - .footer-items__end { - width: 100%; - justify-content: space-between; - .footer-item:not(:last-of-type) { - margin-right: 1rem; - } - } - } - } +.bd-footer .bd-footer__inner a:hover { + color: var(--orange-blue); } +.bd-footer .bd-footer__inner .footer-items__end { + display: flex; + flex-direction: row; +} +.bd-footer .bd-footer__inner .footer-items__end .footer-item:not(:last-of-type) { + margin-right: 3rem; +} +@media screen and (max-width: 959px) { + .bd-footer { + font-size: 0.8rem; + } -@media screen and (max-width: 435px) { - .bd-footer { - font-size: 0.8rem; - } + .bd-footer .bd-footer__inner .footer-items__end .footer-item:not(:last-of-type) { + margin-right: 2rem; + } } +@media screen and (max-width: 700px) { + .bd-footer { + font-size: 1rem; + } + + .bd-footer .bd-footer__inner .footer-items__start { + display: none; + } + + .bd-footer .bd-footer__inner .footer-items__end { + width: 100%; + justify-content: space-between; + } + + .bd-footer .bd-footer__inner .footer-items__end .footer-item:not(:last-of-type) { + margin-right: 1rem; + } +} + +@media screen and (max-width: 435px) { + .bd-footer { + font-size: 0.8rem; + } +} \ No newline at end of file diff --git a/docs/_static/css/header.css b/docs/_static/css/header.css index ea0ef898ad..22866f9f1a 100644 --- a/docs/_static/css/header.css +++ b/docs/_static/css/header.css @@ -1,112 +1,125 @@ .bd-header { - width: 100%; - height: var(--header-height); - background: var(--flair-orange) !important; - box-shadow: none; - *::selection { - background: var(--white-blue); - color: var(--flair-orange); - } - .bd-header__inner { - margin: 0 10%; - padding: 0; - height: 100%; - width: 100%; - max-width: initial; - display: flex; - justify-content: space-between; - .navbar-header-items__start { - padding: 0; - height: 55%; - .navbar-item { - height: 100%; - .navbar-brand.logo { - height: 100%; - padding: 0; - display: block; - .logo__image { - height: 100%; - width: auto; - } - } - } - } - nav { - height: 100%; - ul.bd-navbar-elements { - display: flex; - height: 100%; - li.nav-item { - height: auto; - display: flex; - align-items: center; - margin-right: 0; - margin-left: 3rem; - - &.active a { - font-weight: inherit; - &::after { - width: 100%; - } - } - a { - position: relative; - font-size: 1.1rem; - padding: 0; - color: white; - &:hover { - color: white; - &::after { - width: 100%; - } - } - &::after { - transition: width 200ms cubic-bezier(0,.35,.08,.89); - content: ""; - width: 0; - height: 2px; - background-color: white; - position: absolute; - bottom: -.5rem; - left: 50%; - transform: translateX(-50%); - } - } - } - } - } - .navbar-header-items__end { - .navbar-item:not(:last-of-type) { - margin-right: 2rem; - } - svg { - height: 35%; - width: auto; - } - } - } -} - -.search-button__wrapper.show { - .search-button__overlay { - background: var(--flair-blue); - } - form.bd-search { - i { - display: none; - } - input { - border: 2px var(--flair-orange) solid; - color: var(--flair-blue); - box-shadow: none; - padding: .25em 5.5em .25em .75em; - border-radius: 1rem; - font-size: 1.2rem; - } - .search-button__kbd-shortcut { - align-items: center; - } - } + width: 100%; + height: var(--header-height); + background: var(--flair-orange) !important; + box-shadow: none; +} + +.bd-header *::selection { + background: var(--white-blue); + color: var(--flair-orange); +} + +.bd-header .bd-header__inner { + margin: 0 10%; + padding: 0; + height: 100%; + width: 100%; + max-width: initial; + display: flex; + justify-content: space-between; +} + +.bd-header .bd-header__inner .navbar-header-items__start { + padding: 0; + height: 55%; +} + +.bd-header .bd-header__inner .navbar-header-items__start .navbar-item { + height: 100%; +} + +.bd-header .bd-header__inner .navbar-header-items__start .navbar-item .navbar-brand.logo { + height: 100%; + padding: 0; + display: block; +} + +.bd-header .bd-header__inner .navbar-header-items__start .navbar-item .navbar-brand.logo .logo__image { + height: 100%; + width: auto; +} + +.bd-header .bd-header__inner nav { + height: 100%; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements { + display: flex; + height: 100%; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item { + height: auto; + display: flex; + align-items: center; + margin-right: 0; + margin-left: 3rem; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item.active a { + font-weight: inherit; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item.active a::after { + width: 100%; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item a { + position: relative; + font-size: 1.1rem; + padding: 0; + color: white; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item a:hover { + color: white; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item a:hover::after { + width: 100%; +} + +.bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item a::after { + transition: width 200ms cubic-bezier(0, 0.35, 0.08, 0.89); + content: ""; + width: 0; + height: 2px; + background-color: white; + position: absolute; + bottom: -.5rem; + left: 50%; + transform: translateX(-50%); +} + +.bd-header .bd-header__inner .navbar-header-items__end .navbar-item:not(:last-of-type) { + margin-right: 2rem; +} + +.bd-header .bd-header__inner .navbar-header-items__end svg { + height: 35%; + width: auto; +} + +.search-button__wrapper.show .search-button__overlay { + background: var(--flair-blue); +} + +.search-button__wrapper.show form.bd-search i { + display: none; +} + +.search-button__wrapper.show form.bd-search input { + border: 2px var(--flair-orange) solid; + color: var(--flair-blue); + box-shadow: none; + padding: .25em 5.5em .25em .75em; + border-radius: 1rem; + font-size: 1.2rem; +} + +.search-button__wrapper.show form.bd-search .search-button__kbd-shortcut { + align-items: center; } .fill-white-blue { @@ -125,98 +138,96 @@ } button.search-button { - background-size: contain; - background-repeat: no-repeat; - background-position: center; - border-radius: 0; - height: 1.85rem; - width: 1.85rem; - i { - visibility: hidden; - &:focus, &, &:active { - border: none; - outline: none; - } - } -} - -.sidebar-toggle { - span { - color: white; - font-size: 2rem; - margin-right: 1rem; - } + background-size: contain; + background-repeat: no-repeat; + background-position: center; + border-radius: 0; + height: 1.85rem; + width: 1.85rem; +} + +button.search-button i { + visibility: hidden; +} + +button.search-button i:focus, button.search-button i, button.search-button i:active { + border: none; + outline: none; +} + +.sidebar-toggle span { + color: white; + font-size: 2rem; + margin-right: 1rem; } .dark-mode-toggle svg { - cursor: pointer; - &:first-of-type { - :root .dark-mode & { - display: none; - } - } - &:nth-of-type(2) { - display: none; - :root .dark-mode & { - display: block; - } - } + cursor: pointer; } -@media screen and (max-width: 1250px) { - .bd-header { - .bd-header__inner { - nav { - ul.bd-navbar-elements { - li.nav-item { - margin-left: 2rem; - } - } - } - .navbar-header-items__end { - .navbar-item:not(:last-of-type) { - margin-right: 1rem; - } - } - } - } +:root .dark-mode .dark-mode-toggle svg:first-of-type { + display: none; } +.dark-mode-toggle svg:nth-of-type(2) { + display: none; +} -@media screen and (max-width: 959px) { - .bd-sidebar-primary.bd-sidebar { - width: 100vw !important; - background: var(--flair-orange) !important; - height: calc(100vh - var(--header-height)); - max-width: initial !important; - display: block; - overflow-y: auto !important; - overflow-x: hidden !important; - .sidebar-header-items { - flex-direction: row; - max-width: 100%; - justify-content: space-between; - align-items: flex-start; - .sidebar-header-items__title, .nav-item .nav-link { - color: white; - } - padding: 10vw 10vw 5vw 10vw !important; - } - .sidebar-primary-items__start { - padding: 5vw 10vw 10vw 10vw !important; - border-top: 2px solid white; - } - .sidebar-header-items__end { - gap: 0; - } - .current { - color: var(--flair-blue) !important; - } - a:hover { - color: var(--flair-blue) !important; - } - * { - color: white !important; - } - } +:root .dark-mode .dark-mode-toggle svg:nth-of-type(2) { + display: block; +} + +@media screen and (max-width: 1250px) { + .bd-header .bd-header__inner nav ul.bd-navbar-elements li.nav-item { + margin-left: 2rem; + } + + .bd-header .bd-header__inner .navbar-header-items__end .navbar-item:not(:last-of-type) { + margin-right: 1rem; + } } + +@media screen and (max-width: 959px) { + .bd-sidebar-primary.bd-sidebar { + width: 100vw !important; + background: var(--flair-orange) !important; + height: calc(100vh - var(--header-height)); + max-width: initial !important; + display: block; + overflow-y: auto !important; + overflow-x: hidden !important; + } + + .bd-sidebar-primary.bd-sidebar .sidebar-header-items { + flex-direction: row; + max-width: 100%; + justify-content: space-between; + align-items: flex-start; + padding: 10vw 10vw 5vw 10vw !important; + } + + .bd-sidebar-primary.bd-sidebar .sidebar-header-items .sidebar-header-items__title, .bd-sidebar-primary.bd-sidebar .sidebar-header-items .nav-item .nav-link { + color: white; + } + + .bd-sidebar-primary.bd-sidebar .sidebar-primary-items__start { + padding: 5vw 10vw 10vw 10vw !important; + border-top: 2px solid white; + } + + .bd-sidebar-primary.bd-sidebar .sidebar-header-items__end { + gap: 0; + } + + .bd-sidebar-primary.bd-sidebar .current { + color: var(--flair-blue) !important; + } + + .bd-sidebar-primary.bd-sidebar a:hover { + color: var(--flair-blue) !important; + } + + .bd-sidebar-primary.bd-sidebar * { + color: white !important; + } +} \ No newline at end of file diff --git a/docs/_static/css/legal-notice.css b/docs/_static/css/legal-notice.css index de85cd6252..7b65e532d4 100644 --- a/docs/_static/css/legal-notice.css +++ b/docs/_static/css/legal-notice.css @@ -1,6 +1,4 @@ -.legal-notice { - strong { - font-size: 1.75rem; - font-weight: 600; - } +.legal-notice strong { + font-size: 1.75rem; + font-weight: 600; } \ No newline at end of file diff --git a/docs/_static/css/main.css b/docs/_static/css/main.css index 12a8a7abcb..2170090315 100644 --- a/docs/_static/css/main.css +++ b/docs/_static/css/main.css @@ -1,12 +1,12 @@ -@import url('https://fonts.googleapis.com/css2?family=Afacad:ital,wght@0,400..700;1,400..700&family=Source+Code+Pro:ital,wght@0,200..900;1,200..900&display=swap'); +@import url("https://fonts.googleapis.com/css2?family=Afacad:ital,wght@0,400..700;1,400..700&family=Source+Code+Pro:ital,wght@0,200..900;1,200..900&display=swap"); :root { - --flair-orange: #f79910; + --flair-orange: #f79910; --flair-orange-light: #ffc063; - --flair-orange-transparent: rgba(247, 153, 16, .5); - --white-transparent: rgba(255, 255, 255, .5); - --flair-blue: #2f2e41; - --flair-dark-blue: #262635; + --flair-orange-transparent: rgba(247, 153, 16, 0.5); + --white-transparent: rgba(255, 255, 255, 0.5); + --flair-blue: #2f2e41; + --flair-dark-blue: #262635; --light-gray: #e6e6e6; --white-blue: white; --blue-white: var(--flair-blue); @@ -22,11 +22,11 @@ --track-color: #dddddd; --track-thumb: #bbbbbb; --track-thumb-hover: #999999; - font-family: "Afacad", sans-serif; - font-optical-sizing: auto; - font-weight: 400; + font-family: "Afacad", sans-serif; + font-optical-sizing: auto; + font-weight: 400; font-size: 21px; - font-style: normal; + font-style: normal; } :root .dark-mode { @@ -48,7 +48,6 @@ stroke: none; } - .fill-orange { fill: var(--flair-orange); stroke: none; @@ -114,14 +113,16 @@ table { a { color: var(--flair-orange); text-decoration: none; - transition: color 200ms cubic-bezier(0,.35,.08,.89); - &:visited { - color: var(--flair-orange); - } - &:hover { - color: var(--flair-orange-light); - text-decoration: none; - } + transition: color 200ms cubic-bezier(0, 0.35, 0.08, 0.89); +} + +a:visited { + color: var(--flair-orange); +} + +a:hover { + color: var(--flair-orange-light); + text-decoration: none; } ::selection { diff --git a/docs/_static/css/search.css b/docs/_static/css/search.css index 62fd54bb4a..addc16ffea 100644 --- a/docs/_static/css/search.css +++ b/docs/_static/css/search.css @@ -1,72 +1,78 @@ -.bd-search-container { - h1 { - color: var(--blue-white); - font-size: 3rem; - font-weight: 600; - } - h2 { - color: var(--blue-white); - font-weight: 600; - font-size: 1.5rem; - } - form.bd-search { - i { - display: none; - } - input { - border: 2px var(--flair-orange) solid; - color: var(--flair-blue); - box-shadow: none; - padding: .25em 5.5em .25em .75em; - border-radius: 1rem; - font-size: 1.2rem; - } - .search-button__kbd-shortcut { - align-items: center; - } - } - p.search-summary { - color: var(--blue-white) !important; - } - ul.search { - li { - border-top: 2px solid var(--blue-white); - margin: 0 0 .05em 0; - &:first-of-type { - margin-top: 1rem; - border: none; - } - a { - font-weight: 400; - } - span { - color: var(--blue-white); - } - } - } +.bd-search-container h1 { + color: var(--blue-white); + font-size: 3rem; + font-weight: 600; } + +.bd-search-container h2 { + color: var(--blue-white); + font-weight: 600; + font-size: 1.5rem; +} + +.bd-search-container form.bd-search i { + display: none; +} + +.bd-search-container form.bd-search input { + border: 2px var(--flair-orange) solid; + color: var(--flair-blue); + box-shadow: none; + padding: .25em 5.5em .25em .75em; + border-radius: 1rem; + font-size: 1.2rem; +} + +.bd-search-container form.bd-search .search-button__kbd-shortcut { + align-items: center; +} + +.bd-search-container p.search-summary { + color: var(--blue-white) !important; +} + +.bd-search-container ul.search li { + border-top: 2px solid var(--blue-white); + margin: 0 0 .05em 0; +} + +.bd-search-container ul.search li:first-of-type { + margin-top: 1rem; + border: none; +} + +.bd-search-container ul.search li a { + font-weight: 400; +} + +.bd-search-container ul.search li span { + color: var(--blue-white); +} + .highlighted { - background: var(--flair-orange) !important; - color: white !important; - border-radius: 5px !important; + background: var(--flair-orange) !important; + color: white !important; + border-radius: 5px !important; +} + +div#searchbox p.highlight-link { + box-shadow: none !important; + margin: 1rem 0; +} + +div#searchbox p.highlight-link a { + background: var(--white-blue) !important; + border: 2px var(--flair-orange) solid; + color: var(--flair-orange); + transition: all 200ms ease-in-out; +} + +div#searchbox p.highlight-link a:hover { + background: var(--flair-orange) !important; + color: white !important; + box-shadow: none !important; } -div#searchbox { - p.highlight-link { - box-shadow: none !important; - margin: 1rem 0; - a { - background: var(--white-blue) !important; - border: 2px var(--flair-orange) solid; - color: var(--flair-orange); - transition: all 200ms ease-in-out; - &:hover { - background: var(--flair-orange) !important; - color: white !important; - box-shadow: none !important; - } - &::before { - content: none; - } - } - } + +div#searchbox p.highlight-link a::before { + content: none; } \ No newline at end of file diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css index 598be603cd..459aedc3db 100644 --- a/docs/_static/css/sidebar.css +++ b/docs/_static/css/sidebar.css @@ -1,68 +1,83 @@ .bd-sidebar-primary.bd-sidebar { - top: 90px; - width: 350px; - padding: 0; - border: none; - overflow: initial; - background: var(--white-blue); - #rtd-footer-container, .sidebar-primary-items__end { - display: none; - } - &::after { - content: ""; - background: linear-gradient(90deg, rgba(0,0,0,.2) 0%, rgba(0,0,0,0) 100%); - height: 100%; - width: 7px; - position: absolute; - right: 0; - transform: translateX(100%); - top: 0; - } - h3 { - display: none; - } - * { - color: var(--blue-white); - } - a:hover { - color: var(--flair-orange); - * { - color: var(--flair-orange); - } - } - a * { - transition: color 200ms cubic-bezier(0,.35,.08,.89); - } - .sidebar-primary-items__start { - overflow-y: auto; - padding: 2rem; - } - .sidebar-primary-item { - padding: 0; - } - .toctree-l1 { - font-size: 1.5rem; - font-weight: 600; - margin-top: .8rem; - } - .toctree-l2 { - font-size: 1rem; - font-weight: 400; - margin-top: .8rem; - ul { - margin-left: 1rem; - } - } - .toctree-l3 { - margin-top: .8rem; - } - .current { - color: var(--flair-orange); - } - code { - padding: 0; - background: transparent; - border: none; - font-weight: 400; - } + top: 90px; + width: 350px; + padding: 0; + border: none; + overflow: initial; + background: var(--white-blue); +} + +.bd-sidebar-primary.bd-sidebar #rtd-footer-container, .bd-sidebar-primary.bd-sidebar .sidebar-primary-items__end { + display: none; +} + +.bd-sidebar-primary.bd-sidebar::after { + content: ""; + background: linear-gradient(90deg, rgba(0, 0, 0, 0.2) 0%, transparent 100%); + height: 100%; + width: 7px; + position: absolute; + right: 0; + transform: translateX(100%); + top: 0; +} + +.bd-sidebar-primary.bd-sidebar h3 { + display: none; +} + +.bd-sidebar-primary.bd-sidebar * { + color: var(--blue-white); +} + +.bd-sidebar-primary.bd-sidebar a:hover { + color: var(--flair-orange); +} + +.bd-sidebar-primary.bd-sidebar a:hover * { + color: var(--flair-orange); +} + +.bd-sidebar-primary.bd-sidebar a * { + transition: color 200ms cubic-bezier(0, 0.35, 0.08, 0.89); +} + +.bd-sidebar-primary.bd-sidebar .sidebar-primary-items__start { + overflow-y: auto; + padding: 2rem; +} + +.bd-sidebar-primary.bd-sidebar .sidebar-primary-item { + padding: 0; +} + +.bd-sidebar-primary.bd-sidebar .toctree-l1 { + font-size: 1.5rem; + font-weight: 600; + margin-top: .8rem; +} + +.bd-sidebar-primary.bd-sidebar .toctree-l2 { + font-size: 1rem; + font-weight: 400; + margin-top: .8rem; +} + +.bd-sidebar-primary.bd-sidebar .toctree-l2 ul { + margin-left: 1rem; +} + +.bd-sidebar-primary.bd-sidebar .toctree-l3 { + margin-top: .8rem; +} + +.bd-sidebar-primary.bd-sidebar .current { + color: var(--flair-orange); +} + +.bd-sidebar-primary.bd-sidebar code { + padding: 0; + background: transparent; + border: none; + font-weight: 400; } \ No newline at end of file diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 7b738a0378..530277d097 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -1,375 +1,416 @@ -.bd-main .bd-content .bd-article-container{ - max-width: initial; - padding: 4rem 30% 4rem 5rem; - box-sizing: border-box; +@charset "UTF-8"; + +.bd-main .bd-content .bd-article-container { + max-width: initial; + padding: 4rem 30% 4rem 5rem; + box-sizing: border-box; } .bd-main .bd-content .bd-article-container .bd-article { - padding: 0; - * { - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; - } - .headerlink { - display: none; - } - h1 { - font-size: 3rem; - color: var(--blue-white); - font-weight: 600; - margin-top: 0.5em; - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; - } - h2 { - font-size: 1.6rem; - color: var(--blue-white); - font-weight: 600; - margin-top: 2em; - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; - } - h3 { - font-size: 1.2rem; - color: var(--blue-white); - font-weight: 600; - margin-top: 2em; - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; - } - p { - color: var(--blue-white); - margin: 0.4em 0 0.9em 0; - } - ul { - list-style-type: "»"; - margin-left: .6em; - li { - padding-left: .3em; - } - ::marker { - color: var(--blue-white); - font-size: inherit; - } - } - .highlight-python { - .highlight { - background: transparent; - pre { - font-family: "Source Code Pro", monospace; - padding: 1.1em 1.5em; - border-radius: 1rem; - background: var(--gray-dark-blue); - .nn, .n, .o, .p, .kp { - color: var(--blue-white); - } - .kn, .k, .ow, .kc { - color: #CF8E6D; - } - .nb { - color: #8888C6; - } - .nf { - color: #56A8F5; - } - .c1 { - color: #7A7E85; - } - .s1, .s2, .sa, .si { - color : #6AAB73; - } - .mi, .mf { - color: #2AACB8; - } - } - } - } - [class^="highlight-"]:not(.highlight-python) { - * { - color: var(--blue-white); - } - .highlight { - background: transparent; - } - pre { - font-family: "Source Code Pro", monospace; - border-radius: 1rem; - padding: 1.1em 1.5em; - background: var(--gray-dark-blue); - color: var(--blue-white); - .go { - font-size: inherit; - padding-right: 1.1em; - } - } - } - .admonition, div.admonition, .admonition.note, div.admonition.note { - box-shadow: none !important; - background: linear-gradient(180deg, var(--flair-orange) 0, var(--flair-orange) 2rem, var(--white-blue) 2rem, var(--white-blue) 100%); - border: 2px var(--flair-orange) solid; - border-radius: 1rem; - padding: 0; - box-sizing: border-box; - p.admonition-title { - background: var(--flair-orange); - border-radius: 0 0 1rem 1rem; - margin: 0; - font-size: 1.5rem; - font-weight: 400; - padding: 0.5em 2.2em; - position: relative; - line-height: 1em; - &::before { - content: none; - } - &::after { - color: white; - margin-left: 0.4em; - font-size: 0.9em; - } - } - p:not(.admonition-title) { - padding: 0.5em 2.2em 0.25em 3.3em; - } - [class^="highlight-"] { - margin-bottom: 0.7rem; - } - } + padding: 0; +} + +.bd-main .bd-content .bd-article-container .bd-article * { + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; +} + +.bd-main .bd-content .bd-article-container .bd-article .headerlink { + display: none; +} + +.bd-main .bd-content .bd-article-container .bd-article h1 { + font-size: 3rem; + color: var(--blue-white); + font-weight: 600; + margin-top: 0.5em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; +} + +.bd-main .bd-content .bd-article-container .bd-article h2 { + font-size: 1.6rem; + color: var(--blue-white); + font-weight: 600; + margin-top: 2em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; +} + +.bd-main .bd-content .bd-article-container .bd-article h3 { + font-size: 1.2rem; + color: var(--blue-white); + font-weight: 600; + margin-top: 2em; + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; +} + +.bd-main .bd-content .bd-article-container .bd-article p { + color: var(--blue-white); + margin: 0.4em 0 0.9em 0; +} + +.bd-main .bd-content .bd-article-container .bd-article ul { + list-style-type: "»"; + margin-left: .6em; +} + +.bd-main .bd-content .bd-article-container .bd-article ul li { + padding-left: .3em; +} + +.bd-main .bd-content .bd-article-container .bd-article ul ::marker { + color: var(--blue-white); + font-size: inherit; +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight { + background: transparent; +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre { + font-family: "Source Code Pro", monospace; + padding: 1.1em 1.5em; + border-radius: 1rem; + background: var(--gray-dark-blue); +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .nn, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .n, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .o, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .p, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .kp { + color: var(--blue-white); +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .kn, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .k, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .ow, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .kc { + color: #CF8E6D; +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .nb { + color: #8888C6; +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .nf { + color: #56A8F5; +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .c1 { + color: #7A7E85; +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .s1, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .s2, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .sa, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .si { + color: #6AAB73; +} + +.bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .mi, .bd-main .bd-content .bd-article-container .bd-article .highlight-python .highlight pre .mf { + color: #2AACB8; +} + +.bd-main .bd-content .bd-article-container .bd-article [class^="highlight-"]:not(.highlight-python) * { + color: var(--blue-white); +} + +.bd-main .bd-content .bd-article-container .bd-article [class^="highlight-"]:not(.highlight-python) .highlight { + background: transparent; +} + +.bd-main .bd-content .bd-article-container .bd-article [class^="highlight-"]:not(.highlight-python) pre { + font-family: "Source Code Pro", monospace; + border-radius: 1rem; + padding: 1.1em 1.5em; + background: var(--gray-dark-blue); + color: var(--blue-white); +} + +.bd-main .bd-content .bd-article-container .bd-article [class^="highlight-"]:not(.highlight-python) pre .go { + font-size: inherit; + padding-right: 1.1em; +} + +.bd-main .bd-content .bd-article-container .bd-article .admonition, .bd-main .bd-content .bd-article-container .bd-article div.admonition, .bd-main .bd-content .bd-article-container .bd-article .admonition.note, .bd-main .bd-content .bd-article-container .bd-article div.admonition.note { + box-shadow: none !important; + background: linear-gradient(180deg, var(--flair-orange) 0, var(--flair-orange) 2rem, var(--white-blue) 2rem, var(--white-blue) 100%); + border: 2px var(--flair-orange) solid; + border-radius: 1rem; + padding: 0; + box-sizing: border-box; +} + +.bd-main .bd-content .bd-article-container .bd-article .admonition p.admonition-title, .bd-main .bd-content .bd-article-container .bd-article div.admonition p.admonition-title, .bd-main .bd-content .bd-article-container .bd-article .admonition.note p.admonition-title, .bd-main .bd-content .bd-article-container .bd-article div.admonition.note p.admonition-title { + background: var(--flair-orange); + border-radius: 0 0 1rem 1rem; + margin: 0; + font-size: 1.5rem; + font-weight: 400; + padding: 0.5em 2.2em; + position: relative; + line-height: 1em; +} + +.bd-main .bd-content .bd-article-container .bd-article .admonition p.admonition-title::before, .bd-main .bd-content .bd-article-container .bd-article div.admonition p.admonition-title::before, .bd-main .bd-content .bd-article-container .bd-article .admonition.note p.admonition-title::before, .bd-main .bd-content .bd-article-container .bd-article div.admonition.note p.admonition-title::before { + content: none; +} + +.bd-main .bd-content .bd-article-container .bd-article .admonition p.admonition-title::after, .bd-main .bd-content .bd-article-container .bd-article div.admonition p.admonition-title::after, .bd-main .bd-content .bd-article-container .bd-article .admonition.note p.admonition-title::after, .bd-main .bd-content .bd-article-container .bd-article div.admonition.note p.admonition-title::after { + color: white; + margin-left: 0.4em; + font-size: 0.9em; +} + +.bd-main .bd-content .bd-article-container .bd-article .admonition p:not(.admonition-title), .bd-main .bd-content .bd-article-container .bd-article div.admonition p:not(.admonition-title), .bd-main .bd-content .bd-article-container .bd-article .admonition.note p:not(.admonition-title), .bd-main .bd-content .bd-article-container .bd-article div.admonition.note p:not(.admonition-title) { + padding: 0.5em 2.2em 0.25em 3.3em; +} + +.bd-main .bd-content .bd-article-container .bd-article .admonition [class^="highlight-"], .bd-main .bd-content .bd-article-container .bd-article div.admonition [class^="highlight-"], .bd-main .bd-content .bd-article-container .bd-article .admonition.note [class^="highlight-"], .bd-main .bd-content .bd-article-container .bd-article div.admonition.note [class^="highlight-"] { + margin-bottom: 0.7rem; } .header-article-items.header-article__inner { - padding: 0; + padding: 0; } ul.bd-breadcrumbs { - word-break: break-word; - overflow-wrap: break-word; - white-space: normal; - li.breadcrumb-home .fa-home { - &::before { - content: "Home"; - font-family: "Afacad", sans-serif; - font-size: 1rem; - font-weight: 400; - color: var(--blue-white); - transition: color 200ms ease-in-out; - } - } - li.breadcrumb-home a:hover .fa-home::before { - color: var(--flair-orange); - } - li.breadcrumb-item:not(.breadcrumb-home) { - &::before { - content: ""; - width: 1px; - height: 1rem; - background: var(--blue-white); - display: inline-block; - padding: 0; - transform: rotate(30deg); - margin: 0 .6rem; - } - a { - color: var(--blue-white); - font-size: 1rem; - font-weight: 400; - &:hover { - color: var(--flair-orange); - } - } - &.active { - color: var(--blue-white); - font-size: 1rem; - font-weight: 400; - } - } -} - -table.table {border-collapse: collapse; - border-radius: 1rem; - box-shadow: 0 0 0 2px var(--blue-white); - tr { - &:not(:last-of-type), &:has(th) { - border-bottom: 2px var(--blue-white) solid; - } - th { - font-weight: 600; - } - th, td { - padding: .4em .2em 0 .2em; - &:first-of-type { - padding-left: .5em; - } - &:last-of-type { - padding-right: .5em; - } - } - } + word-break: break-word; + overflow-wrap: break-word; + white-space: normal; +} + +ul.bd-breadcrumbs li.breadcrumb-home .fa-home::before { + content: "Home"; + font-family: "Afacad", sans-serif; + font-size: 1rem; + font-weight: 400; + color: var(--blue-white); + transition: color 200ms ease-in-out; +} + +ul.bd-breadcrumbs li.breadcrumb-home a:hover .fa-home::before { + color: var(--flair-orange); +} + +ul.bd-breadcrumbs li.breadcrumb-item:not(.breadcrumb-home)::before { + content: ""; + width: 1px; + height: 1rem; + background: var(--blue-white); + display: inline-block; + padding: 0; + transform: rotate(30deg); + margin: 0 .6rem; +} + +ul.bd-breadcrumbs li.breadcrumb-item:not(.breadcrumb-home) a { + color: var(--blue-white); + font-size: 1rem; + font-weight: 400; +} + +ul.bd-breadcrumbs li.breadcrumb-item:not(.breadcrumb-home) a:hover { + color: var(--flair-orange); +} + +ul.bd-breadcrumbs li.breadcrumb-item:not(.breadcrumb-home).active { + color: var(--blue-white); + font-size: 1rem; + font-weight: 400; +} + +table.table { + border-collapse: collapse; + border-radius: 1rem; + box-shadow: 0 0 0 2px var(--blue-white); +} + +table.table tr:not(:last-of-type), table.table tr:has(th) { + border-bottom: 2px var(--blue-white) solid; +} + +table.table tr th { + font-weight: 600; +} + +table.table tr th, table.table tr td { + padding: .4em .2em 0 .2em; +} + +table.table tr th:first-of-type, table.table tr td:first-of-type { + padding-left: .5em; +} + +table.table tr th:last-of-type, table.table tr td:last-of-type { + padding-right: .5em; } .docutils, .docutils.literal { - border: none; - background: transparent; - border-radius: 0.5em; - font-weight: 400; - font-family: "Source Code Pro", monospace; - font-size: 0.8em; - color: var(--blue-white); - position: relative; - z-index: 1; - &::before { - content: ""; - width: 100%; - height: 90%; - top: 50%; - left: 0; - transform: translateY(-50%); - border-radius: 0.5em; - border: 2px var(--blue-white) solid; - position: absolute; - } -} -.reference { - .docutils, .docutils.literal { - transition: color 200ms ease-in-out; - &:hover { - &::before { - background: var(--flair-orange); - } - color: white; - } - &::before { - z-index: -1; - border-color: var(--flair-orange); - transition: background-color 200ms ease-in-out; - } - } -} - -#flair-tutorials, #api-docs, #contributing, [id^="tutorial"], [id^="flair"] { - li { - list-style-type: "»"; - padding-left: 0.25em; - margin-top: .5em; - ::marker { - color: var(--blue-white); - font-size: 1.2rem; - } - } - .toctree-wrapper li[class^="toctree-l"] { - margin-top: .5em; - &::marker { - font-size: 1.2rem; - } - & > a { - font-size: 1.2rem; - } - } - .docutils:not(.py) { - color: var(--flair-orange); - &:hover { - color: var(--flair-orange-light); - } - &::before { - content: none; - } - .pre { - font-family: "Afacad", sans-serif; - text-overflow: ellipsis; - font-size: 1.2rem; - } - } + border: none; + background: transparent; + border-radius: 0.5em; + font-weight: 400; + font-family: "Source Code Pro", monospace; + font-size: 0.8em; + color: var(--blue-white); + position: relative; + z-index: 1; +} + +.docutils::before, .docutils.literal::before { + content: ""; + width: 100%; + height: 90%; + top: 50%; + left: 0; + transform: translateY(-50%); + border-radius: 0.5em; + border: 2px var(--blue-white) solid; + position: absolute; +} + +.reference .docutils, .reference .docutils.literal { + transition: color 200ms ease-in-out; +} + +.reference .docutils:hover, .reference .docutils.literal:hover { + color: white; +} + +.reference .docutils:hover::before, .reference .docutils.literal:hover::before { + background: var(--flair-orange); +} + +.reference .docutils::before, .reference .docutils.literal::before { + z-index: -1; + border-color: var(--flair-orange); + transition: background-color 200ms ease-in-out; +} + +#flair-tutorials li, #api-docs li, #contributing li, [id^="tutorial"] li, [id^="flair"] li { + list-style-type: "»"; + padding-left: 0.25em; + margin-top: .5em; +} + +#flair-tutorials li ::marker, #api-docs li ::marker, #contributing li ::marker, [id^="tutorial"] li ::marker, [id^="flair"] li ::marker { + color: var(--blue-white); + font-size: 1.2rem; +} + +#flair-tutorials .toctree-wrapper li[class^="toctree-l"], #api-docs .toctree-wrapper li[class^="toctree-l"], #contributing .toctree-wrapper li[class^="toctree-l"], [id^="tutorial"] .toctree-wrapper li[class^="toctree-l"], [id^="flair"] .toctree-wrapper li[class^="toctree-l"] { + margin-top: .5em; +} + +#flair-tutorials .toctree-wrapper li[class^="toctree-l"]::marker, #api-docs .toctree-wrapper li[class^="toctree-l"]::marker, #contributing .toctree-wrapper li[class^="toctree-l"]::marker, [id^="tutorial"] .toctree-wrapper li[class^="toctree-l"]::marker, [id^="flair"] .toctree-wrapper li[class^="toctree-l"]::marker { + font-size: 1.2rem; +} + +#flair-tutorials .toctree-wrapper li[class^="toctree-l"] > a, #api-docs .toctree-wrapper li[class^="toctree-l"] > a, #contributing .toctree-wrapper li[class^="toctree-l"] > a, [id^="tutorial"] .toctree-wrapper li[class^="toctree-l"] > a, [id^="flair"] .toctree-wrapper li[class^="toctree-l"] > a { + font-size: 1.2rem; +} + +#flair-tutorials .docutils:not(.py), #api-docs .docutils:not(.py), #contributing .docutils:not(.py), [id^="tutorial"] .docutils:not(.py), [id^="flair"] .docutils:not(.py) { + color: var(--flair-orange); +} + +#flair-tutorials .docutils:not(.py):hover, #api-docs .docutils:not(.py):hover, #contributing .docutils:not(.py):hover, [id^="tutorial"] .docutils:not(.py):hover, [id^="flair"] .docutils:not(.py):hover { + color: var(--flair-orange-light); +} + +#flair-tutorials .docutils:not(.py)::before, #api-docs .docutils:not(.py)::before, #contributing .docutils:not(.py)::before, [id^="tutorial"] .docutils:not(.py)::before, [id^="flair"] .docutils:not(.py)::before { + content: none; +} + +#flair-tutorials .docutils:not(.py) .pre, #api-docs .docutils:not(.py) .pre, #contributing .docutils:not(.py) .pre, [id^="tutorial"] .docutils:not(.py) .pre, [id^="flair"] .docutils:not(.py) .pre { + font-family: "Afacad", sans-serif; + text-overflow: ellipsis; + font-size: 1.2rem; } @media screen and (max-width: 700px) { - .bd-article-container { - padding: 3rem !important; - } - .admonition, div.admonition, .admonition.note, div.admonition.note { - p { - padding-left: 1rem !important; - &.admonition-title { - padding-left: 1rem !important; - &::after { - position: relative; - left: -0.1rem; - } - } - } - } - .deprecated { - padding-left: 1rem !important; - .deprecated { - padding-left: 0 !important; - } - &::before { - position: relative !important; - } - } + .bd-article-container { + padding: 3rem !important; + } + + .admonition p, div.admonition p, .admonition.note p, div.admonition.note p { + padding-left: 1rem !important; + } + + .admonition p.admonition-title, div.admonition p.admonition-title, .admonition.note p.admonition-title, div.admonition.note p.admonition-title { + padding-left: 1rem !important; + } + + .admonition p.admonition-title::after, div.admonition p.admonition-title::after, .admonition.note p.admonition-title::after, div.admonition.note p.admonition-title::after { + position: relative; + left: -0.1rem; + } + + .deprecated { + padding-left: 1rem !important; + } + + .deprecated .deprecated { + padding-left: 0 !important; + } + + .deprecated::before { + position: relative !important; + } } @media screen and (max-width: 500px) { - #flair-tutorials, #api-docs, #contributing, [id^="tutorial"], [id^="flair"] { - & .toctree-wrapper li[class^="toctree-l"]{ - & > a, & > a span { - font-size: 1rem !important; - } - } - } - .toctree-wrapper li[class^="toctree-l"] > ul { - padding-inline-start: 0; - } - .bd-article-container { - padding: 2rem !important; - } + #flair-tutorials .toctree-wrapper li[class^="toctree-l"] > a, #flair-tutorials .toctree-wrapper li[class^="toctree-l"] > a span, #api-docs .toctree-wrapper li[class^="toctree-l"] > a, #api-docs .toctree-wrapper li[class^="toctree-l"] > a span, #contributing .toctree-wrapper li[class^="toctree-l"] > a, #contributing .toctree-wrapper li[class^="toctree-l"] > a span, [id^="tutorial"] .toctree-wrapper li[class^="toctree-l"] > a, [id^="tutorial"] .toctree-wrapper li[class^="toctree-l"] > a span, [id^="flair"] .toctree-wrapper li[class^="toctree-l"] > a, [id^="flair"] .toctree-wrapper li[class^="toctree-l"] > a span { + font-size: 1rem !important; + } + + .toctree-wrapper li[class^="toctree-l"] > ul { + padding-inline-start: 0; + } + + .bd-article-container { + padding: 2rem !important; + } } @media screen and (max-width: 1200px) { - .bd-main .bd-content .bd-article-container .bd-article { + .bd-main .bd-content .bd-article-container .bd-article h1 { + font-size: 2rem; + } - h1 { - font-size: 2rem; - } + .bd-main .bd-content .bd-article-container .bd-article h2 { + font-size: 1.2rem; + } - h2 { - font-size: 1.2rem; - } + .bd-main .bd-content .bd-article-container .bd-article h3 { + font-size: 1rem; + } - h3 { - font-size: 1rem; - } - } - .bd-article-container { - padding: 4rem; - } + .bd-article-container { + padding: 4rem; + } } @media screen and (max-width: 900px) { - .bd-main .bd-content .bd-article-container .bd-article { - - h1 { - font-size: 1.6rem; - } - - h2 { - font-size: 1rem; - } - - h3 { - font-size: 1rem; - } - } - .bd-article-container { - padding: 4rem; - } - .bd-header-article { - margin-bottom: 3rem; - } - pre, pre * { - font-size: 0.8rem; - } + .bd-main .bd-content .bd-article-container .bd-article h1 { + font-size: 1.6rem; + } + + .bd-main .bd-content .bd-article-container .bd-article h2 { + font-size: 1rem; + } + + .bd-main .bd-content .bd-article-container .bd-article h3 { + font-size: 1rem; + } + + .bd-article-container { + padding: 4rem; + } + + .bd-header-article { + margin-bottom: 3rem; + } + + pre, pre * { + font-size: 0.8rem; + } } \ No newline at end of file diff --git a/docs/_static/css/version-switcher.css b/docs/_static/css/version-switcher.css index 3d88ea7abd..5b54fb6270 100644 --- a/docs/_static/css/version-switcher.css +++ b/docs/_static/css/version-switcher.css @@ -1,68 +1,78 @@ .version-switcher__container { - position: relative; - z-index: 10; - #versionswitcherbutton { - background: var(--flair-orange); - border: 2px white solid; - color: white; - font-weight: 500; - cursor: pointer; - font-size: 1rem; - padding: 0.2em 1em; - border-radius: 0.66em; - font-family: inherit; - width: 7em; - &::after { - content: none; - } - } - #versionswitcherlist { - background: var(--flair-orange); - position: absolute; - top: calc(100% + 0.4rem); - left: 0; - display: none; - flex-direction: column; - border: 2px white solid; - border-radius: 0.66em; - font-size: 1rem; - min-width: 100%; - box-sizing: border-box; - overflow: hidden; - } - &.open #versionswitcherlist { - display: flex; - } - .version-switcher__menu a.list-group-item:not(:last-child) { - border-bottom: none; - } - a.list-group-item { - background: var(--flair-orange); - font-size: 1rem; - padding: 0.2em 1em 0.2em 1em; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - position: relative; - transition: all 200ms cubic-bezier(0,.35,.08,.89); - &:hover { - background: var(--white-transparent); - color: var(--white-blue); - } - span { - color: white; - } - &:not(:last-of-type) { - padding-bottom: calc(0.2em + 2px); - &::after { - content: ""; - height: 2px; - width: 100%; - background: white; - position: absolute; - bottom: 0; - left: 0; - } - } - } + position: relative; + z-index: 10; +} + +.version-switcher__container #versionswitcherbutton { + background: var(--flair-orange); + border: 2px white solid; + color: white; + font-weight: 500; + cursor: pointer; + font-size: 1rem; + padding: 0.2em 1em; + border-radius: 0.66em; + font-family: inherit; + width: 7em; +} + +.version-switcher__container #versionswitcherbutton::after { + content: none; +} + +.version-switcher__container #versionswitcherlist { + background: var(--flair-orange); + position: absolute; + top: calc(100% + 0.4rem); + left: 0; + display: none; + flex-direction: column; + border: 2px white solid; + border-radius: 0.66em; + font-size: 1rem; + min-width: 100%; + box-sizing: border-box; + overflow: hidden; +} + +.version-switcher__container.open #versionswitcherlist { + display: flex; +} + +.version-switcher__container .version-switcher__menu a.list-group-item:not(:last-child) { + border-bottom: none; +} + +.version-switcher__container a.list-group-item { + background: var(--flair-orange); + font-size: 1rem; + padding: 0.2em 1em 0.2em 1em; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + position: relative; + transition: all 200ms cubic-bezier(0, 0.35, 0.08, 0.89); +} + +.version-switcher__container a.list-group-item:hover { + background: var(--white-transparent); + color: var(--white-blue); +} + +.version-switcher__container a.list-group-item span { + color: white; +} + +.version-switcher__container a.list-group-item:not(:last-of-type) { + padding-bottom: calc(0.2em + 2px); +} + +.version-switcher__container a.list-group-item:not(:last-of-type)::after { + content: ""; + height: 2px; + width: 100%; + background: white; + position: absolute; + bottom: 0; + left: 0; } \ No newline at end of file From fbe37781d2b48f67e6d2a50502b46b75e7cc3f0e Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 27 Aug 2024 19:46:15 +0200 Subject: [PATCH 204/333] changes --- docs/_static/css/sidebar.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css index 459aedc3db..b685bb4e52 100644 --- a/docs/_static/css/sidebar.css +++ b/docs/_static/css/sidebar.css @@ -71,7 +71,7 @@ margin-top: .8rem; } -.bd-sidebar-primary.bd-sidebar .current { +.bd-sidebar-primary.bd-sidebar .current, .bd-sidebar-primary.bd-sidebar .current * { color: var(--flair-orange); } From 836d1ad7cc83ed5ea019b1b055099ab89a473cf7 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:28:43 +0200 Subject: [PATCH 205/333] changes --- docs/api/datasets/base.rst | 4 ---- docs/api/datasets/base/DataLoader.rst | 4 ++++ docs/api/datasets/base/FlairDatapointDataset.rst | 4 ++++ docs/api/datasets/base/SentenceDataset.rst | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 docs/api/datasets/base.rst create mode 100644 docs/api/datasets/base/DataLoader.rst create mode 100644 docs/api/datasets/base/FlairDatapointDataset.rst create mode 100644 docs/api/datasets/base/SentenceDataset.rst diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst deleted file mode 100644 index e42784deb0..0000000000 --- a/docs/api/datasets/base.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.base -=================== - -.. automodule:: flair.datasets.base \ No newline at end of file diff --git a/docs/api/datasets/base/DataLoader.rst b/docs/api/datasets/base/DataLoader.rst new file mode 100644 index 0000000000..49d60a251b --- /dev/null +++ b/docs/api/datasets/base/DataLoader.rst @@ -0,0 +1,4 @@ +flair.datasets.base.DataLoader +============================== + +.. autoclass:: flair.datasets.base.DataLoader \ No newline at end of file diff --git a/docs/api/datasets/base/FlairDatapointDataset.rst b/docs/api/datasets/base/FlairDatapointDataset.rst new file mode 100644 index 0000000000..e6f13c204e --- /dev/null +++ b/docs/api/datasets/base/FlairDatapointDataset.rst @@ -0,0 +1,4 @@ +flair.datasets.base.FlairDatapointDataset +========================================= + +.. autoclass:: flair.datasets.base.FlairDatapointDataset \ No newline at end of file diff --git a/docs/api/datasets/base/SentenceDataset.rst b/docs/api/datasets/base/SentenceDataset.rst new file mode 100644 index 0000000000..94e2e09537 --- /dev/null +++ b/docs/api/datasets/base/SentenceDataset.rst @@ -0,0 +1,4 @@ +flair.datasets.base.SentenceDataset +=================================== + +.. autoclass:: flair.datasets.base.SentenceDataset \ No newline at end of file From 1bcf63ed8330ceeab4ea2da9ac07be7d43d3af4f Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:30:22 +0200 Subject: [PATCH 206/333] changes --- docs/api/flair.datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index 9a883c3e61..e663696a69 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -3,6 +3,6 @@ flair.datasets .. toctree:: :glob: - :maxdepth: 2 + :maxdepth: 1 datasets/* From 3e47fadfaf597f23b6060c9546744b6add6653e9 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:32:36 +0200 Subject: [PATCH 207/333] changes --- docs/api/datasets/flair.datasets.base.rst | 8 ++++++++ docs/api/flair.datasets.rst | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 docs/api/datasets/flair.datasets.base.rst delete mode 100644 docs/api/flair.datasets.rst diff --git a/docs/api/datasets/flair.datasets.base.rst b/docs/api/datasets/flair.datasets.base.rst new file mode 100644 index 0000000000..52ae184577 --- /dev/null +++ b/docs/api/datasets/flair.datasets.base.rst @@ -0,0 +1,8 @@ +flair.datasets.base +=================== + +.. toctree:: + :glob: + :maxdepth: 1 + + base/* diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst deleted file mode 100644 index e663696a69..0000000000 --- a/docs/api/flair.datasets.rst +++ /dev/null @@ -1,8 +0,0 @@ -flair.datasets -============== - -.. toctree:: - :glob: - :maxdepth: 1 - - datasets/* From 1b81a7147d96f4803cb698bfbc439ad8b3b6eef2 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:33:45 +0200 Subject: [PATCH 208/333] changes --- docs/api/datasets/{flair.datasets.base.rst => base/index.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/api/datasets/{flair.datasets.base.rst => base/index.rst} (100%) diff --git a/docs/api/datasets/flair.datasets.base.rst b/docs/api/datasets/base/index.rst similarity index 100% rename from docs/api/datasets/flair.datasets.base.rst rename to docs/api/datasets/base/index.rst From 69cea83ec45e4239359350fc7610e367b4e945a3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:35:29 +0200 Subject: [PATCH 209/333] changes --- docs/api/{datasets/base/index.rst => flair.datasets.base.rst} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/api/{datasets/base/index.rst => flair.datasets.base.rst} (81%) diff --git a/docs/api/datasets/base/index.rst b/docs/api/flair.datasets.base.rst similarity index 81% rename from docs/api/datasets/base/index.rst rename to docs/api/flair.datasets.base.rst index 52ae184577..436932b1e6 100644 --- a/docs/api/datasets/base/index.rst +++ b/docs/api/flair.datasets.base.rst @@ -5,4 +5,4 @@ flair.datasets.base :glob: :maxdepth: 1 - base/* + datasets/base/* From 1fb12f65b963e4ec293a276bbd3d0a9704b4f15a Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:36:54 +0200 Subject: [PATCH 210/333] changes --- docs/api/{flair.datasets.base.rst => datasets/base.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/api/{flair.datasets.base.rst => datasets/base.rst} (100%) diff --git a/docs/api/flair.datasets.base.rst b/docs/api/datasets/base.rst similarity index 100% rename from docs/api/flair.datasets.base.rst rename to docs/api/datasets/base.rst From 44f690bcb292d3bd967ef52ac12b1fd0fb58ab32 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:37:57 +0200 Subject: [PATCH 211/333] changes --- docs/api/flair.datasets.rst | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/api/flair.datasets.rst diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst new file mode 100644 index 0000000000..e663696a69 --- /dev/null +++ b/docs/api/flair.datasets.rst @@ -0,0 +1,8 @@ +flair.datasets +============== + +.. toctree:: + :glob: + :maxdepth: 1 + + datasets/* From f12c1d25f098d7f31672e43a691426d0867ed419 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:38:58 +0200 Subject: [PATCH 212/333] changes --- docs/api/datasets/base.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst index 436932b1e6..52ae184577 100644 --- a/docs/api/datasets/base.rst +++ b/docs/api/datasets/base.rst @@ -5,4 +5,4 @@ flair.datasets.base :glob: :maxdepth: 1 - datasets/base/* + base/* From dedbec91c707858336176d26d8550ae269557274 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:40:15 +0200 Subject: [PATCH 213/333] changes --- docs/api/flair.datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index e663696a69..9a883c3e61 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -3,6 +3,6 @@ flair.datasets .. toctree:: :glob: - :maxdepth: 1 + :maxdepth: 2 datasets/* From 9ed57bdc6fbf76d8da289bb4243dfaed0ef1440c Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:41:11 +0200 Subject: [PATCH 214/333] changes --- docs/api/flair.datasets.rst | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index 9a883c3e61..e663696a69 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -3,6 +3,6 @@ flair.datasets .. toctree:: :glob: - :maxdepth: 2 + :maxdepth: 1 datasets/* diff --git a/docs/index.rst b/docs/index.rst index c39010a2c7..eaa6ca94b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ :file: _templates/landing-page-illustrations.html .. toctree:: - :maxdepth: 3 + :maxdepth: 4 :hidden: Tutorials From 80dfcfe7fdec39063cf78c3ce837b459d9367dec Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:46:36 +0200 Subject: [PATCH 215/333] changes --- docs/_static/css/sidebar.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/css/sidebar.css b/docs/_static/css/sidebar.css index b685bb4e52..7d67c22041 100644 --- a/docs/_static/css/sidebar.css +++ b/docs/_static/css/sidebar.css @@ -71,6 +71,10 @@ margin-top: .8rem; } +.bd-sidebar-primary.bd-sidebar .toctree-l4 { + margin-top: .8rem; +} + .bd-sidebar-primary.bd-sidebar .current, .bd-sidebar-primary.bd-sidebar .current * { color: var(--flair-orange); } From 80726a9f7597b4cc68c5d5e918c21a1c7dd9f466 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:58:03 +0200 Subject: [PATCH 216/333] changes --- docs/_static/css/tutorial.css | 4 ++-- docs/api/flair.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 530277d097..035ccf6d03 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -286,7 +286,7 @@ table.table tr th:last-of-type, table.table tr td:last-of-type { } #flair-tutorials li, #api-docs li, #contributing li, [id^="tutorial"] li, [id^="flair"] li { - list-style-type: "»"; + list-style-type: "»" !important; padding-left: 0.25em; margin-top: .5em; } @@ -301,7 +301,7 @@ table.table tr th:last-of-type, table.table tr td:last-of-type { } #flair-tutorials .toctree-wrapper li[class^="toctree-l"]::marker, #api-docs .toctree-wrapper li[class^="toctree-l"]::marker, #contributing .toctree-wrapper li[class^="toctree-l"]::marker, [id^="tutorial"] .toctree-wrapper li[class^="toctree-l"]::marker, [id^="flair"] .toctree-wrapper li[class^="toctree-l"]::marker { - font-size: 1.2rem; + font-size: 1.2rem !important; } #flair-tutorials .toctree-wrapper li[class^="toctree-l"] > a, #api-docs .toctree-wrapper li[class^="toctree-l"] > a, #contributing .toctree-wrapper li[class^="toctree-l"] > a, [id^="tutorial"] .toctree-wrapper li[class^="toctree-l"] > a, [id^="flair"] .toctree-wrapper li[class^="toctree-l"] > a { diff --git a/docs/api/flair.rst b/docs/api/flair.rst index f2e9f5a9ab..4edfcc1c13 100644 --- a/docs/api/flair.rst +++ b/docs/api/flair.rst @@ -3,6 +3,6 @@ flair .. toctree:: :glob: - :maxdepth: 1 + :maxdepth: 0 flair/* From e2a48560ece115029d8b386aa06a8a7b7b0c46b1 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 11:59:23 +0200 Subject: [PATCH 217/333] changes --- docs/api/flair.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.rst b/docs/api/flair.rst index 4edfcc1c13..f2e9f5a9ab 100644 --- a/docs/api/flair.rst +++ b/docs/api/flair.rst @@ -3,6 +3,6 @@ flair .. toctree:: :glob: - :maxdepth: 0 + :maxdepth: 1 flair/* From 71456c47eea179ac70af6db8137234ec870da4bf Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 12:18:48 +0200 Subject: [PATCH 218/333] changes --- docs/api/data/BoundingBox.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/data/BoundingBox.rst b/docs/api/data/BoundingBox.rst index 09a27814f9..2318e280c6 100644 --- a/docs/api/data/BoundingBox.rst +++ b/docs/api/data/BoundingBox.rst @@ -1,4 +1,4 @@ -flair.data.BoundingBox -========================== +BoundingBox +=========== .. autoclass:: flair.data.BoundingBox \ No newline at end of file From a6c6177fcc4e34c52ea92f3438d8c19d2bb97190 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 12:21:33 +0200 Subject: [PATCH 219/333] changes --- docs/api/data/DataPoint.rst | 4 ++-- docs/api/data/Dictionary.rst | 4 ++-- docs/api/data/EntityCandidate.rst | 4 ++-- docs/api/data/Label.rst | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/api/data/DataPoint.rst b/docs/api/data/DataPoint.rst index e57f7ad264..738afa1bd0 100644 --- a/docs/api/data/DataPoint.rst +++ b/docs/api/data/DataPoint.rst @@ -1,4 +1,4 @@ -flair.data.DataPoint -==================== +DataPoint +========= .. autoclass:: flair.data.DataPoint \ No newline at end of file diff --git a/docs/api/data/Dictionary.rst b/docs/api/data/Dictionary.rst index 4a5534b96a..d86bc580c3 100644 --- a/docs/api/data/Dictionary.rst +++ b/docs/api/data/Dictionary.rst @@ -1,4 +1,4 @@ -flair.data.Dictionary -===================== +Dictionary +========== .. autoclass:: flair.data.Dictionary \ No newline at end of file diff --git a/docs/api/data/EntityCandidate.rst b/docs/api/data/EntityCandidate.rst index 898f992aa2..ffeb2587b1 100644 --- a/docs/api/data/EntityCandidate.rst +++ b/docs/api/data/EntityCandidate.rst @@ -1,4 +1,4 @@ -flair.data.EntityCandidate -========================== +EntityCandidate +=============== .. autoclass:: flair.data.EntityCandidate \ No newline at end of file diff --git a/docs/api/data/Label.rst b/docs/api/data/Label.rst index 9d97babe40..2d0aac0a28 100644 --- a/docs/api/data/Label.rst +++ b/docs/api/data/Label.rst @@ -1,4 +1,4 @@ -flair.data.Label -================ +Label +===== .. autoclass:: flair.data.Label \ No newline at end of file From 492ce67c3405e6f7046b4f69ab04af8c51c101a0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 12:42:47 +0200 Subject: [PATCH 220/333] changes --- docs/api/flair.models.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/api/flair.models.rst b/docs/api/flair.models.rst index 8679b3fb7d..402d7fb90b 100644 --- a/docs/api/flair.models.rst +++ b/docs/api/flair.models.rst @@ -1,4 +1,12 @@ flair.models ============ -.. automodule:: flair.models \ No newline at end of file +.. automodule:: flair.models +.. currentmodule:: flair.models + +.. autosummary:: + :toctree: generated + :nosignatures: + + SpanClassifier + LanguageModel \ No newline at end of file From 076d5d8ee566b24d6cf353ca82542437310f7fe5 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 12:43:52 +0200 Subject: [PATCH 221/333] changes --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index eaa6ca94b1..c39010a2c7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ :file: _templates/landing-page-illustrations.html .. toctree:: - :maxdepth: 4 + :maxdepth: 3 :hidden: Tutorials From c26e59e231946aec5efac7731407f69e9a1ba6c3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 12:55:12 +0200 Subject: [PATCH 222/333] changes --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 752ee2f5b0..2a7e3d2115 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,6 +46,7 @@ def linkcode_resolve(*args): # ones. extensions = [ "sphinx.ext.autodoc", + 'sphinx.ext.autosummary', "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.ifconfig", From 155e5f20ac4a25cbb54d16773c3e2e2afb9063e0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 12:59:30 +0200 Subject: [PATCH 223/333] changes --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 2a7e3d2115..25eefe321a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ def linkcode_resolve(*args): "sphinx_multiversion", "sphinx_design", ] - +autosummary_generate = True # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From 5f08801cbbca1cda055af473d0fb0e77615cdef1 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:03:56 +0200 Subject: [PATCH 224/333] changes --- docs/api/flair.models.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/api/flair.models.rst b/docs/api/flair.models.rst index 402d7fb90b..7e522f3d05 100644 --- a/docs/api/flair.models.rst +++ b/docs/api/flair.models.rst @@ -1,9 +1,6 @@ flair.models ============ -.. automodule:: flair.models -.. currentmodule:: flair.models - .. autosummary:: :toctree: generated :nosignatures: From 864d14183f60cd45ffe74a1be0c9692ed62cb8f9 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:06:33 +0200 Subject: [PATCH 225/333] changes --- docs/api/flair.models.rst | 2 ++ docs/conf.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/api/flair.models.rst b/docs/api/flair.models.rst index 7e522f3d05..a41d6170ac 100644 --- a/docs/api/flair.models.rst +++ b/docs/api/flair.models.rst @@ -1,6 +1,8 @@ flair.models ============ +.. currentmodule:: flair.models + .. autosummary:: :toctree: generated :nosignatures: diff --git a/docs/conf.py b/docs/conf.py index 25eefe321a..01a2f9f6a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,7 +46,7 @@ def linkcode_resolve(*args): # ones. extensions = [ "sphinx.ext.autodoc", - 'sphinx.ext.autosummary', + "sphinx.ext.autosummary", "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.ifconfig", From f74cc5968a014e70c6f7c575d56464fef68266b0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:17:30 +0200 Subject: [PATCH 226/333] changes --- docs/_static/css/api.css | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 4666de16ea..4fe148fff6 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -148,6 +148,12 @@ blockquote p { margin: 0 !important; } +.rubric { + font-size: 2rem; + margin: 1em 0 .5em 0 !important; + border-bottom: none !important; +} + @media screen and (max-width: 1200px) { .class dd, .method dd, .function dd, .data dd { padding-left: 1.5rem !important; From 8fb215125197ec59830316ba047b3d61c0fa34dd Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:21:36 +0200 Subject: [PATCH 227/333] changes --- docs/api/flair.rst | 16 ++++++++++++++++ docs/api/flair/cache_root.rst | 4 ---- docs/api/flair/device.rst | 4 ---- docs/api/flair/logger.rst | 4 ---- docs/api/flair/set_proxies.rst | 4 ---- docs/api/flair/set_seed.rst | 4 ---- 6 files changed, 16 insertions(+), 20 deletions(-) delete mode 100644 docs/api/flair/cache_root.rst delete mode 100644 docs/api/flair/device.rst delete mode 100644 docs/api/flair/logger.rst delete mode 100644 docs/api/flair/set_proxies.rst delete mode 100644 docs/api/flair/set_seed.rst diff --git a/docs/api/flair.rst b/docs/api/flair.rst index f2e9f5a9ab..296c0852fa 100644 --- a/docs/api/flair.rst +++ b/docs/api/flair.rst @@ -6,3 +6,19 @@ flair :maxdepth: 1 flair/* + + +flair +===== + +.. currentmodule:: flair + +.. autosummary:: + :toctree: generated + :nosignatures: + + cache_root + device + logger + set_seed + set_proxies \ No newline at end of file diff --git a/docs/api/flair/cache_root.rst b/docs/api/flair/cache_root.rst deleted file mode 100644 index ee6b8a806d..0000000000 --- a/docs/api/flair/cache_root.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.cache_root -================ - -.. autodata:: flair.cache_root \ No newline at end of file diff --git a/docs/api/flair/device.rst b/docs/api/flair/device.rst deleted file mode 100644 index a172113a31..0000000000 --- a/docs/api/flair/device.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.device -============ - -.. autodata:: flair.device \ No newline at end of file diff --git a/docs/api/flair/logger.rst b/docs/api/flair/logger.rst deleted file mode 100644 index 8e14d4304f..0000000000 --- a/docs/api/flair/logger.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.logger -============ - -.. autodata:: flair.logger \ No newline at end of file diff --git a/docs/api/flair/set_proxies.rst b/docs/api/flair/set_proxies.rst deleted file mode 100644 index 56b883fbf3..0000000000 --- a/docs/api/flair/set_proxies.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.set_proxies -================= - -.. autofunction:: flair.set_proxies \ No newline at end of file diff --git a/docs/api/flair/set_seed.rst b/docs/api/flair/set_seed.rst deleted file mode 100644 index bbdac8bac6..0000000000 --- a/docs/api/flair/set_seed.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.set_seed -============== - -.. autofunction:: flair.set_seed \ No newline at end of file From d9145beb130e4ac14ee5e56d1027c503ce50999a Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:23:17 +0200 Subject: [PATCH 228/333] changes --- docs/api/flair.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/api/flair.rst b/docs/api/flair.rst index 296c0852fa..6c2b90e00f 100644 --- a/docs/api/flair.rst +++ b/docs/api/flair.rst @@ -1,16 +1,6 @@ flair ===== -.. toctree:: - :glob: - :maxdepth: 1 - - flair/* - - -flair -===== - .. currentmodule:: flair .. autosummary:: From bc69b4df64703d21f421bfbb572fc7ffd2ef3b53 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:26:07 +0200 Subject: [PATCH 229/333] changes --- docs/_static/css/api.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index 4fe148fff6..f0fc53d8a4 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -154,6 +154,10 @@ blockquote p { border-bottom: none !important; } +table.autosummary { + margin-top: 1.5em; +} + @media screen and (max-width: 1200px) { .class dd, .method dd, .function dd, .data dd { padding-left: 1.5rem !important; From f1cfc72f5055a6783496cfe23ba0238571b38082 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:26:50 +0200 Subject: [PATCH 230/333] changes --- docs/api/data/BoundingBox.rst | 4 ---- docs/api/data/DataPoint.rst | 4 ---- docs/api/data/Dictionary.rst | 4 ---- docs/api/data/EntityCandidate.rst | 4 ---- docs/api/data/Label.rst | 4 ---- docs/api/flair.data.rst | 27 +++++++++++++++++++++++---- 6 files changed, 23 insertions(+), 24 deletions(-) delete mode 100644 docs/api/data/BoundingBox.rst delete mode 100644 docs/api/data/DataPoint.rst delete mode 100644 docs/api/data/Dictionary.rst delete mode 100644 docs/api/data/EntityCandidate.rst delete mode 100644 docs/api/data/Label.rst diff --git a/docs/api/data/BoundingBox.rst b/docs/api/data/BoundingBox.rst deleted file mode 100644 index 2318e280c6..0000000000 --- a/docs/api/data/BoundingBox.rst +++ /dev/null @@ -1,4 +0,0 @@ -BoundingBox -=========== - -.. autoclass:: flair.data.BoundingBox \ No newline at end of file diff --git a/docs/api/data/DataPoint.rst b/docs/api/data/DataPoint.rst deleted file mode 100644 index 738afa1bd0..0000000000 --- a/docs/api/data/DataPoint.rst +++ /dev/null @@ -1,4 +0,0 @@ -DataPoint -========= - -.. autoclass:: flair.data.DataPoint \ No newline at end of file diff --git a/docs/api/data/Dictionary.rst b/docs/api/data/Dictionary.rst deleted file mode 100644 index d86bc580c3..0000000000 --- a/docs/api/data/Dictionary.rst +++ /dev/null @@ -1,4 +0,0 @@ -Dictionary -========== - -.. autoclass:: flair.data.Dictionary \ No newline at end of file diff --git a/docs/api/data/EntityCandidate.rst b/docs/api/data/EntityCandidate.rst deleted file mode 100644 index ffeb2587b1..0000000000 --- a/docs/api/data/EntityCandidate.rst +++ /dev/null @@ -1,4 +0,0 @@ -EntityCandidate -=============== - -.. autoclass:: flair.data.EntityCandidate \ No newline at end of file diff --git a/docs/api/data/Label.rst b/docs/api/data/Label.rst deleted file mode 100644 index 2d0aac0a28..0000000000 --- a/docs/api/data/Label.rst +++ /dev/null @@ -1,4 +0,0 @@ -Label -===== - -.. autoclass:: flair.data.Label \ No newline at end of file diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index 50bf8f0254..6395685fac 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -1,8 +1,27 @@ flair.data ========== -.. toctree:: - :glob: - :maxdepth: 1 +.. currentmodule:: flair.data - data/* +.. autosummary:: + :toctree: generated + :nosignatures: + + + BoundingBox + Dictionary + Label + DataPoint + Token + Span + Relation + Sentence + DataPair + Image + Corpus + MultiCorpus + FlairDataset + ConcatFlairDataset + iob2() + randomly_split_into_two_datasets() + get_spans_from_bio() From 7c2a2064bb4ab49a7a3c27c01e72939b46f68765 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:30:11 +0200 Subject: [PATCH 231/333] changes --- docs/api/datasets/base.rst | 8 ------- docs/api/datasets/base/DataLoader.rst | 4 ---- .../datasets/base/FlairDatapointDataset.rst | 4 ---- docs/api/datasets/base/SentenceDataset.rst | 4 ---- docs/api/datasets/biomedical.rst | 4 ---- docs/api/datasets/document_classification.rst | 4 ---- docs/api/datasets/entity_linking.rst | 4 ---- docs/api/datasets/ocr.rst | 4 ---- docs/api/datasets/relation_extraction.rst | 4 ---- docs/api/datasets/sequence_labeling.rst | 4 ---- docs/api/datasets/text_image.rst | 4 ---- docs/api/datasets/text_text.rst | 4 ---- docs/api/datasets/treebanks.rst | 4 ---- docs/api/flair.datasets.rst | 22 +++++++++++++++---- 14 files changed, 18 insertions(+), 60 deletions(-) delete mode 100644 docs/api/datasets/base.rst delete mode 100644 docs/api/datasets/base/DataLoader.rst delete mode 100644 docs/api/datasets/base/FlairDatapointDataset.rst delete mode 100644 docs/api/datasets/base/SentenceDataset.rst delete mode 100644 docs/api/datasets/biomedical.rst delete mode 100644 docs/api/datasets/document_classification.rst delete mode 100644 docs/api/datasets/entity_linking.rst delete mode 100644 docs/api/datasets/ocr.rst delete mode 100644 docs/api/datasets/relation_extraction.rst delete mode 100644 docs/api/datasets/sequence_labeling.rst delete mode 100644 docs/api/datasets/text_image.rst delete mode 100644 docs/api/datasets/text_text.rst delete mode 100644 docs/api/datasets/treebanks.rst diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst deleted file mode 100644 index 52ae184577..0000000000 --- a/docs/api/datasets/base.rst +++ /dev/null @@ -1,8 +0,0 @@ -flair.datasets.base -=================== - -.. toctree:: - :glob: - :maxdepth: 1 - - base/* diff --git a/docs/api/datasets/base/DataLoader.rst b/docs/api/datasets/base/DataLoader.rst deleted file mode 100644 index 49d60a251b..0000000000 --- a/docs/api/datasets/base/DataLoader.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.base.DataLoader -============================== - -.. autoclass:: flair.datasets.base.DataLoader \ No newline at end of file diff --git a/docs/api/datasets/base/FlairDatapointDataset.rst b/docs/api/datasets/base/FlairDatapointDataset.rst deleted file mode 100644 index e6f13c204e..0000000000 --- a/docs/api/datasets/base/FlairDatapointDataset.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.base.FlairDatapointDataset -========================================= - -.. autoclass:: flair.datasets.base.FlairDatapointDataset \ No newline at end of file diff --git a/docs/api/datasets/base/SentenceDataset.rst b/docs/api/datasets/base/SentenceDataset.rst deleted file mode 100644 index 94e2e09537..0000000000 --- a/docs/api/datasets/base/SentenceDataset.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.base.SentenceDataset -=================================== - -.. autoclass:: flair.datasets.base.SentenceDataset \ No newline at end of file diff --git a/docs/api/datasets/biomedical.rst b/docs/api/datasets/biomedical.rst deleted file mode 100644 index d59bd8c589..0000000000 --- a/docs/api/datasets/biomedical.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.biomedical -========================= - -.. automodule:: flair.datasets.biomedical \ No newline at end of file diff --git a/docs/api/datasets/document_classification.rst b/docs/api/datasets/document_classification.rst deleted file mode 100644 index d8303f3aeb..0000000000 --- a/docs/api/datasets/document_classification.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.document_classification -====================================== - -.. automodule:: flair.datasets.document_classification \ No newline at end of file diff --git a/docs/api/datasets/entity_linking.rst b/docs/api/datasets/entity_linking.rst deleted file mode 100644 index cdb2b32356..0000000000 --- a/docs/api/datasets/entity_linking.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.entity_linking -============================= - -.. automodule:: flair.datasets.entity_linking \ No newline at end of file diff --git a/docs/api/datasets/ocr.rst b/docs/api/datasets/ocr.rst deleted file mode 100644 index 3f85340440..0000000000 --- a/docs/api/datasets/ocr.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.ocr -================== - -.. automodule:: flair.datasets.ocr \ No newline at end of file diff --git a/docs/api/datasets/relation_extraction.rst b/docs/api/datasets/relation_extraction.rst deleted file mode 100644 index 62dcdd55d1..0000000000 --- a/docs/api/datasets/relation_extraction.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.relation_extraction -================================== - -.. automodule:: flair.datasets.relation_extraction \ No newline at end of file diff --git a/docs/api/datasets/sequence_labeling.rst b/docs/api/datasets/sequence_labeling.rst deleted file mode 100644 index 875d4831b1..0000000000 --- a/docs/api/datasets/sequence_labeling.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.sequence_labeling -================================ - -.. automodule:: flair.datasets.sequence_labeling \ No newline at end of file diff --git a/docs/api/datasets/text_image.rst b/docs/api/datasets/text_image.rst deleted file mode 100644 index f14e564916..0000000000 --- a/docs/api/datasets/text_image.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.text_image -========================= - -.. automodule:: flair.datasets.text_image \ No newline at end of file diff --git a/docs/api/datasets/text_text.rst b/docs/api/datasets/text_text.rst deleted file mode 100644 index f88dfd1aed..0000000000 --- a/docs/api/datasets/text_text.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.text_text -========================= - -.. automodule:: flair.datasets.text_text \ No newline at end of file diff --git a/docs/api/datasets/treebanks.rst b/docs/api/datasets/treebanks.rst deleted file mode 100644 index 0d6c14a281..0000000000 --- a/docs/api/datasets/treebanks.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.datasets.treebanks -======================== - -.. automodule:: flair.datasets.treebanks \ No newline at end of file diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index e663696a69..d68987c657 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -1,8 +1,22 @@ flair.datasets ============== -.. toctree:: - :glob: - :maxdepth: 1 +.. currentmodule:: flair.datasets + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + flair.datasets.base + flair.datasets.biomedical + flair.datasets.document_classification + flair.datasets.entity_linking + flair.datasets.ocr + flair.datasets.relation_extraction + flair.datasets.sequence_labeling + flair.datasets.text_image + flair.datasets.text_text + flair.datasets.treebanks - datasets/* From 6a094994054d48ee962fa6201526c1f9ef8b7686 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:31:49 +0200 Subject: [PATCH 232/333] changes --- docs/api/datasets/base.rst | 18 +++++++++++++++ docs/api/datasets/biomedical.rst | 4 ++++ docs/api/datasets/document_classification.rst | 4 ++++ docs/api/datasets/entity_linking.rst | 4 ++++ docs/api/datasets/ocr.rst | 4 ++++ docs/api/datasets/relation_extraction.rst | 4 ++++ docs/api/datasets/sequence_labeling.rst | 4 ++++ docs/api/datasets/text_image.rst | 4 ++++ docs/api/datasets/text_text.rst | 4 ++++ docs/api/datasets/treebanks.rst | 4 ++++ docs/api/embeddings/base.rst | 4 ---- docs/api/embeddings/document.rst | 4 ---- docs/api/embeddings/image.rst | 4 ---- docs/api/embeddings/legacy.rst | 8 ------- docs/api/embeddings/token.rst | 4 ---- docs/api/embeddings/transformer.rst | 4 ---- docs/api/flair.datasets.rst | 22 ++++--------------- 17 files changed, 58 insertions(+), 46 deletions(-) create mode 100644 docs/api/datasets/base.rst create mode 100644 docs/api/datasets/biomedical.rst create mode 100644 docs/api/datasets/document_classification.rst create mode 100644 docs/api/datasets/entity_linking.rst create mode 100644 docs/api/datasets/ocr.rst create mode 100644 docs/api/datasets/relation_extraction.rst create mode 100644 docs/api/datasets/sequence_labeling.rst create mode 100644 docs/api/datasets/text_image.rst create mode 100644 docs/api/datasets/text_text.rst create mode 100644 docs/api/datasets/treebanks.rst delete mode 100644 docs/api/embeddings/base.rst delete mode 100644 docs/api/embeddings/document.rst delete mode 100644 docs/api/embeddings/image.rst delete mode 100644 docs/api/embeddings/legacy.rst delete mode 100644 docs/api/embeddings/token.rst delete mode 100644 docs/api/embeddings/transformer.rst diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst new file mode 100644 index 0000000000..eeb5fb2d36 --- /dev/null +++ b/docs/api/datasets/base.rst @@ -0,0 +1,18 @@ +flair.datasets.base +=================== + +.. currentmodule:: flair.datasets.base + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + DataLoader + FlairDatapointDataset + SentenceDataset + StringDataset + MongoDataset + find_train_dev_test_files() + diff --git a/docs/api/datasets/biomedical.rst b/docs/api/datasets/biomedical.rst new file mode 100644 index 0000000000..d59bd8c589 --- /dev/null +++ b/docs/api/datasets/biomedical.rst @@ -0,0 +1,4 @@ +flair.datasets.biomedical +========================= + +.. automodule:: flair.datasets.biomedical \ No newline at end of file diff --git a/docs/api/datasets/document_classification.rst b/docs/api/datasets/document_classification.rst new file mode 100644 index 0000000000..d8303f3aeb --- /dev/null +++ b/docs/api/datasets/document_classification.rst @@ -0,0 +1,4 @@ +flair.datasets.document_classification +====================================== + +.. automodule:: flair.datasets.document_classification \ No newline at end of file diff --git a/docs/api/datasets/entity_linking.rst b/docs/api/datasets/entity_linking.rst new file mode 100644 index 0000000000..cdb2b32356 --- /dev/null +++ b/docs/api/datasets/entity_linking.rst @@ -0,0 +1,4 @@ +flair.datasets.entity_linking +============================= + +.. automodule:: flair.datasets.entity_linking \ No newline at end of file diff --git a/docs/api/datasets/ocr.rst b/docs/api/datasets/ocr.rst new file mode 100644 index 0000000000..3f85340440 --- /dev/null +++ b/docs/api/datasets/ocr.rst @@ -0,0 +1,4 @@ +flair.datasets.ocr +================== + +.. automodule:: flair.datasets.ocr \ No newline at end of file diff --git a/docs/api/datasets/relation_extraction.rst b/docs/api/datasets/relation_extraction.rst new file mode 100644 index 0000000000..62dcdd55d1 --- /dev/null +++ b/docs/api/datasets/relation_extraction.rst @@ -0,0 +1,4 @@ +flair.datasets.relation_extraction +================================== + +.. automodule:: flair.datasets.relation_extraction \ No newline at end of file diff --git a/docs/api/datasets/sequence_labeling.rst b/docs/api/datasets/sequence_labeling.rst new file mode 100644 index 0000000000..875d4831b1 --- /dev/null +++ b/docs/api/datasets/sequence_labeling.rst @@ -0,0 +1,4 @@ +flair.datasets.sequence_labeling +================================ + +.. automodule:: flair.datasets.sequence_labeling \ No newline at end of file diff --git a/docs/api/datasets/text_image.rst b/docs/api/datasets/text_image.rst new file mode 100644 index 0000000000..f14e564916 --- /dev/null +++ b/docs/api/datasets/text_image.rst @@ -0,0 +1,4 @@ +flair.datasets.text_image +========================= + +.. automodule:: flair.datasets.text_image \ No newline at end of file diff --git a/docs/api/datasets/text_text.rst b/docs/api/datasets/text_text.rst new file mode 100644 index 0000000000..f88dfd1aed --- /dev/null +++ b/docs/api/datasets/text_text.rst @@ -0,0 +1,4 @@ +flair.datasets.text_text +========================= + +.. automodule:: flair.datasets.text_text \ No newline at end of file diff --git a/docs/api/datasets/treebanks.rst b/docs/api/datasets/treebanks.rst new file mode 100644 index 0000000000..0d6c14a281 --- /dev/null +++ b/docs/api/datasets/treebanks.rst @@ -0,0 +1,4 @@ +flair.datasets.treebanks +======================== + +.. automodule:: flair.datasets.treebanks \ No newline at end of file diff --git a/docs/api/embeddings/base.rst b/docs/api/embeddings/base.rst deleted file mode 100644 index 1bf51ffa7a..0000000000 --- a/docs/api/embeddings/base.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.embeddings.base -===================== - -.. automodule:: flair.embeddings.base \ No newline at end of file diff --git a/docs/api/embeddings/document.rst b/docs/api/embeddings/document.rst deleted file mode 100644 index ca870fc8ea..0000000000 --- a/docs/api/embeddings/document.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.embeddings.document -========================= - -.. automodule:: flair.embeddings.document \ No newline at end of file diff --git a/docs/api/embeddings/image.rst b/docs/api/embeddings/image.rst deleted file mode 100644 index 2a701b9e0b..0000000000 --- a/docs/api/embeddings/image.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.embeddings.image -====================== - -.. automodule:: flair.embeddings.image \ No newline at end of file diff --git a/docs/api/embeddings/legacy.rst b/docs/api/embeddings/legacy.rst deleted file mode 100644 index 974a777eb9..0000000000 --- a/docs/api/embeddings/legacy.rst +++ /dev/null @@ -1,8 +0,0 @@ -flair.embeddings.legacy -============================ - -.. warning:: - All embeddings in `flair.embeddings.legacy` are considered deprecated. - there is no guarantee that they are still working and we recommend using different embeddings instead. - -.. automodule:: flair.embeddings.legacy \ No newline at end of file diff --git a/docs/api/embeddings/token.rst b/docs/api/embeddings/token.rst deleted file mode 100644 index 3705fedb1d..0000000000 --- a/docs/api/embeddings/token.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.embeddings.token -====================== - -.. automodule:: flair.embeddings.token \ No newline at end of file diff --git a/docs/api/embeddings/transformer.rst b/docs/api/embeddings/transformer.rst deleted file mode 100644 index 2bda02f771..0000000000 --- a/docs/api/embeddings/transformer.rst +++ /dev/null @@ -1,4 +0,0 @@ -flair.embeddings.transformer -============================ - -.. automodule:: flair.embeddings.transformer \ No newline at end of file diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index d68987c657..e663696a69 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -1,22 +1,8 @@ flair.datasets ============== -.. currentmodule:: flair.datasets - -.. autosummary:: - :toctree: generated - :nosignatures: - - - - flair.datasets.base - flair.datasets.biomedical - flair.datasets.document_classification - flair.datasets.entity_linking - flair.datasets.ocr - flair.datasets.relation_extraction - flair.datasets.sequence_labeling - flair.datasets.text_image - flair.datasets.text_text - flair.datasets.treebanks +.. toctree:: + :glob: + :maxdepth: 1 + datasets/* From ce0eeb257f6b3f2160ff8530bd16ca5d3d11f6e2 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:32:40 +0200 Subject: [PATCH 233/333] changes --- docs/api/datasets/base.rst | 2 +- docs/api/flair.data.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst index eeb5fb2d36..fbb80b0281 100644 --- a/docs/api/datasets/base.rst +++ b/docs/api/datasets/base.rst @@ -14,5 +14,5 @@ flair.datasets.base SentenceDataset StringDataset MongoDataset - find_train_dev_test_files() + find_train_dev_test_files diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index 6395685fac..5e20e3f51a 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -22,6 +22,6 @@ flair.data MultiCorpus FlairDataset ConcatFlairDataset - iob2() - randomly_split_into_two_datasets() - get_spans_from_bio() + iob2 + randomly_split_into_two_datasets + get_spans_from_bio From 849be88c188fcd30e4986969d3232e30e8bbf7ec Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:36:42 +0200 Subject: [PATCH 234/333] changes --- docs/api/flair.datasets.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index e663696a69..3e49610a9c 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -1,8 +1,10 @@ flair.datasets ============== -.. toctree:: - :glob: - :maxdepth: 1 +.. currentmodule:: flair.datasets + +.. autosummary:: + :toctree: generated + :nosignatures: datasets/* From 4f23340daded61f68259344bf5df93f4f49eca96 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:38:19 +0200 Subject: [PATCH 235/333] changes --- docs/api/flair.datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index 3e49610a9c..ae8d6d0124 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -7,4 +7,4 @@ flair.datasets :toctree: generated :nosignatures: - datasets/* + base From cd8ec0c899fd573fd136db7b0dc52f5b3c143847 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:40:42 +0200 Subject: [PATCH 236/333] changes --- docs/api/datasets/base.rst | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 docs/api/datasets/base.rst diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst deleted file mode 100644 index fbb80b0281..0000000000 --- a/docs/api/datasets/base.rst +++ /dev/null @@ -1,18 +0,0 @@ -flair.datasets.base -=================== - -.. currentmodule:: flair.datasets.base - -.. autosummary:: - :toctree: generated - :nosignatures: - - - - DataLoader - FlairDatapointDataset - SentenceDataset - StringDataset - MongoDataset - find_train_dev_test_files - From f826b0561d73b759ff0ba9ff868dd28217fafacd Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:41:25 +0200 Subject: [PATCH 237/333] changes --- docs/api/flair.datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index ae8d6d0124..5ac96ea0e6 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -5,6 +5,6 @@ flair.datasets .. autosummary:: :toctree: generated - :nosignatures: + :recursive: base From edad3749f46c379b7954db78a85a1c34ddcf96ce Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:42:41 +0200 Subject: [PATCH 238/333] changes --- docs/api/datasets/base.rst | 18 ++++++++++++++++++ docs/api/flair.datasets.rst | 10 ++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 docs/api/datasets/base.rst diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst new file mode 100644 index 0000000000..fbb80b0281 --- /dev/null +++ b/docs/api/datasets/base.rst @@ -0,0 +1,18 @@ +flair.datasets.base +=================== + +.. currentmodule:: flair.datasets.base + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + DataLoader + FlairDatapointDataset + SentenceDataset + StringDataset + MongoDataset + find_train_dev_test_files + diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index 5ac96ea0e6..e663696a69 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -1,10 +1,8 @@ flair.datasets ============== -.. currentmodule:: flair.datasets +.. toctree:: + :glob: + :maxdepth: 1 -.. autosummary:: - :toctree: generated - :recursive: - - base + datasets/* From 756b5d67bc9e73012cbb7ffe37b5959c202f0f9c Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:53:32 +0200 Subject: [PATCH 239/333] changes --- docs/api/flair.datasets.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index e663696a69..e27515a835 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -1,6 +1,8 @@ flair.datasets ============== +.. currentmodule:: flair.datasets + .. toctree:: :glob: :maxdepth: 1 From 8a12fcdd1a87136c840bf501eed88968743624ce Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:54:34 +0200 Subject: [PATCH 240/333] changes --- docs/api/flair.datasets.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index e27515a835..e663696a69 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -1,8 +1,6 @@ flair.datasets ============== -.. currentmodule:: flair.datasets - .. toctree:: :glob: :maxdepth: 1 From 17a921bb87714fbef2e8783b28c83bb0892a9c27 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:57:15 +0200 Subject: [PATCH 241/333] changes --- docs/_static/css/tutorial.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 035ccf6d03..d84fdf2abe 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -206,6 +206,10 @@ ul.bd-breadcrumbs li.breadcrumb-item:not(.breadcrumb-home) a { font-size: 1rem; font-weight: 400; } +ul.bd-breadcrumbs li.breadcrumb-item:not(.breadcrumb-home) i { + color: var(--blue-white); +} + ul.bd-breadcrumbs li.breadcrumb-item:not(.breadcrumb-home) a:hover { color: var(--flair-orange); From 4f8124dbd501def026c0e5055c0e911ba46ef33a Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:57:58 +0200 Subject: [PATCH 242/333] changes --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index c39010a2c7..eaa6ca94b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ :file: _templates/landing-page-illustrations.html .. toctree:: - :maxdepth: 3 + :maxdepth: 4 :hidden: Tutorials From 8162178019702509fed3a7240b383db82db3cd8b Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 13:58:47 +0200 Subject: [PATCH 243/333] changes --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index eaa6ca94b1..c39010a2c7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ :file: _templates/landing-page-illustrations.html .. toctree:: - :maxdepth: 4 + :maxdepth: 3 :hidden: Tutorials From 1bb8207e814a2ee4f33bf017b94cadd007584b79 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 14:00:18 +0200 Subject: [PATCH 244/333] changes --- docs/api/datasets/biomedical.rst | 145 ++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/docs/api/datasets/biomedical.rst b/docs/api/datasets/biomedical.rst index d59bd8c589..a58dc51a6c 100644 --- a/docs/api/datasets/biomedical.rst +++ b/docs/api/datasets/biomedical.rst @@ -1,4 +1,147 @@ flair.datasets.biomedical ========================= -.. automodule:: flair.datasets.biomedical \ No newline at end of file +.. currentmodule:: flair.datasets.biomedical + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + Entity + InternalBioNerDataset + DpEntry + merge_datasets + filter_and_map_entities + filter_nested_entities + bioc_to_internal + brat_to_internal + CoNLLWriter + HunerDataset + BIO_INFER + HUNER_GENE_BIO_INFER + JNLPBA + HunerJNLPBA + HUNER_GENE_JNLPBA + HUNER_CELL_LINE_JNLPBA + CELL_FINDER + HUNER_CELL_LINE_CELL_FINDER + HUNER_SPECIES_CELL_FINDER + HUNER_GENE_CELL_FINDER + MIRNA + HunerMiRNAHelper + HUNER_GENE_MIRNA + HUNER_SPECIES_MIRNA + HUNER_DISEASE_MIRNA + KaewphanCorpusHelper + CLL + HUNER_CELL_LINE_CLL + GELLUS + HUNER_CELL_LINE_GELLUS + LOCTEXT + HUNER_SPECIES_LOCTEXT + HUNER_GENE_LOCTEXT + CHEMDNER + HUNER_CHEMICAL_CHEMDNER + IEPA + HUNER_GENE_IEPA + LINNEAUS + HUNER_SPECIES_LINNEAUS + CDR + HUNER_DISEASE_CDR + HUNER_CHEMICAL_CDR + VARIOME + HUNER_GENE_VARIOME + HUNER_DISEASE_VARIOME + HUNER_SPECIES_VARIOME + NCBI_DISEASE + HUNER_DISEASE_NCBI + ScaiCorpus + SCAI_CHEMICALS + SCAI_DISEASE + HUNER_CHEMICAL_SCAI + HUNER_DISEASE_SCAI + OSIRIS + HUNER_GENE_OSIRIS + S800 + HUNER_SPECIES_S800 + GPRO + HUNER_GENE_GPRO + DECA + HUNER_GENE_DECA + FSU + HUNER_GENE_FSU + CRAFT + BIOSEMANTICS + BC2GM + HUNER_GENE_BC2GM + CEMP + HUNER_CHEMICAL_CEMP + CHEBI + HUNER_CHEMICAL_CHEBI + HUNER_GENE_CHEBI + HUNER_SPECIES_CHEBI + BioNLPCorpus + BIONLP2013_PC + BIONLP2013_CG + ANAT_EM + BioBertHelper + BIOBERT_CHEMICAL_BC4CHEMD + BIOBERT_GENE_BC2GM + BIOBERT_GENE_JNLPBA + BIOBERT_CHEMICAL_BC5CDR + BIOBERT_DISEASE_BC5CDR + BIOBERT_DISEASE_NCBI + BIOBERT_SPECIES_LINNAEUS + BIOBERT_SPECIES_S800 + CRAFT_V4 + HUNER_CHEMICAL_CRAFT_V4 + HUNER_GENE_CRAFT_V4 + HUNER_SPECIES_CRAFT_V4 + HUNER_CHEMICAL_BIONLP2013_CG + HUNER_DISEASE_BIONLP2013_CG + HUNER_GENE_BIONLP2013_CG + HUNER_SPECIES_BIONLP2013_CG + AZDZ + PDR + HUNER_DISEASE_PDR + HunerMultiCorpus + HUNER_CELL_LINE + HUNER_CHEMICAL + HUNER_DISEASE + HUNER_GENE + HUNER_SPECIES + BIGBIO_NER_CORPUS + HUNER_GENE_NLM_GENE + HUNER_GENE_DRUGPROT + HUNER_CHEMICAL_DRUGPROT + HUNER_GENE_BIORED + HUNER_CHEMICAL_BIORED + HUNER_DISEASE_BIORED + HUNER_SPECIES_BIORED + HUNER_CELL_LINE_BIORED + HUNER_GENE_CPI + HUNER_CHEMICAL_CPI + HUNER_GENE_BIONLP_ST_2013_PC + HUNER_CHEMICAL_BIONLP_ST_2013_PC + HUNER_GENE_BIONLP_ST_2013_GE + HUNER_GENE_BIONLP_ST_2011_GE + HUNER_GENE_BIONLP_ST_2011_ID + HUNER_CHEMICAL_BIONLP_ST_2011_ID + HUNER_SPECIES_BIONLP_ST_2011_ID + HUNER_GENE_BIONLP_ST_2011_REL + HUNER_GENE_BIONLP_ST_2011_EPI + HUNER_SPECIES_BIONLP_ST_2019_BB + HUNER_GENE_BIOID + HUNER_CHEMICAL_BIOID + HUNER_SPECIES_BIOID + HUNER_CELL_LINE_BIOID + HUNER_GENE_GNORMPLUS + HUNER_GENE_PROGENE + HUNER_CHEMICAL_NLM_CHEM + HUNER_GENE_SETH_CORPUS + HUNER_GENE_TMVAR_V3 + HUNER_SPECIES_TMVAR_V3 + HUNER_CELL_LINE_TMVAR_V3 + From acab1efea3c5203353dbc3e2c778b18bc51f9cb5 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 14:11:29 +0200 Subject: [PATCH 245/333] changes --- docs/_static/css/api.css | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index f0fc53d8a4..e2d1ba3e8c 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -158,6 +158,20 @@ table.autosummary { margin-top: 1.5em; } +table .field-odd, table .field-even { + color: var(--blue-white); + font-family: "Afacad", sans-serif; +} + +table .field-odd p, table .field-even p { + margin: 0 !important; +} + +table p:empty { + display: none; +} + + @media screen and (max-width: 1200px) { .class dd, .method dd, .function dd, .data dd { padding-left: 1.5rem !important; From a67782dcd01935c6b4e6387524a92c48050243c2 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 14:17:26 +0200 Subject: [PATCH 246/333] changes --- docs/_static/css/tutorial.css | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index d84fdf2abe..4afc7dcdba 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -257,6 +257,7 @@ table.table tr th:last-of-type, table.table tr td:last-of-type { color: var(--blue-white); position: relative; z-index: 1; + display: inline-block; } .docutils::before, .docutils.literal::before { From 64de6bbf9b2a071b1010ff0235eee84ca5ba84d7 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 14:26:19 +0200 Subject: [PATCH 247/333] changes --- docs/api/datasets/document_classification.rst | 36 ++++++++- docs/api/datasets/entity_linking.rst | 26 ++++++- docs/api/datasets/ocr.rst | 13 +++- docs/api/datasets/relation_extraction.rst | 15 +++- docs/api/datasets/sequence_labeling.rst | 74 ++++++++++++++++++- docs/api/datasets/text_image.rst | 12 ++- docs/api/datasets/text_text.rst | 29 +++++++- docs/api/datasets/treebanks.rst | 70 +++++++++++++++++- 8 files changed, 266 insertions(+), 9 deletions(-) diff --git a/docs/api/datasets/document_classification.rst b/docs/api/datasets/document_classification.rst index d8303f3aeb..8d775e1413 100644 --- a/docs/api/datasets/document_classification.rst +++ b/docs/api/datasets/document_classification.rst @@ -1,4 +1,38 @@ flair.datasets.document_classification ====================================== -.. automodule:: flair.datasets.document_classification \ No newline at end of file +.. currentmodule:: flair.datasets.document_classification + +.. autosummary:: + :toctree: generated + :nosignatures: + + + ClassificationCorpus + ClassificationDataset + CSVClassificationCorpus + CSVClassificationDataset + AMAZON_REVIEWS + IMDB + NEWSGROUPS + AGNEWS + STACKOVERFLOW + SENTIMENT_140 + SENTEVAL_CR + SENTEVAL_MR + SENTEVAL_SUBJ + SENTEVAL_MPQA + SENTEVAL_SST_BINARY + SENTEVAL_SST_GRANULAR + GLUE_COLA + GLUE_SST2 + GO_EMOTIONS + TREC_50 + TREC_6 + YAHOO_ANSWERS + GERMEVAL_2018_OFFENSIVE_LANGUAGE + COMMUNICATIVE_FUNCTIONS + WASSA_ANGER + WASSA_FEAR + WASSA_JOY + WASSA_SADNESS diff --git a/docs/api/datasets/entity_linking.rst b/docs/api/datasets/entity_linking.rst index cdb2b32356..7c616f3b85 100644 --- a/docs/api/datasets/entity_linking.rst +++ b/docs/api/datasets/entity_linking.rst @@ -1,4 +1,28 @@ flair.datasets.entity_linking ============================= -.. automodule:: flair.datasets.entity_linking \ No newline at end of file +.. currentmodule:: flair.datasets.entity_linking + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + ZELDA + NEL_ENGLISH_AQUAINT + NEL_GERMAN_HIPE + NEL_ENGLISH_AIDA + NEL_ENGLISH_IITB + NEL_ENGLISH_TWEEKI + NEL_ENGLISH_REDDIT + from_ufsac_to_tsv + determine_tsv_file + WSD_UFSAC + WSD_RAGANATO_ALL + WSD_SEMCOR + WSD_WORDNET_GLOSS_TAGGED + WSD_MASC + WSD_OMSTI + WSD_TRAINOMATIC + diff --git a/docs/api/datasets/ocr.rst b/docs/api/datasets/ocr.rst index 3f85340440..d9b005a0ea 100644 --- a/docs/api/datasets/ocr.rst +++ b/docs/api/datasets/ocr.rst @@ -1,4 +1,15 @@ flair.datasets.ocr ================== -.. automodule:: flair.datasets.ocr \ No newline at end of file +.. currentmodule:: flair.datasets.ocr + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + OcrJsonDataset + OcrCorpus + SROIE + diff --git a/docs/api/datasets/relation_extraction.rst b/docs/api/datasets/relation_extraction.rst index 62dcdd55d1..c840b00fd6 100644 --- a/docs/api/datasets/relation_extraction.rst +++ b/docs/api/datasets/relation_extraction.rst @@ -1,4 +1,17 @@ flair.datasets.relation_extraction ================================== -.. automodule:: flair.datasets.relation_extraction \ No newline at end of file +.. currentmodule:: flair.datasets.relation_extraction + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + convert_ptb_token + RE_ENGLISH_SEMEVAL2010 + RE_ENGLISH_TACRED + RE_ENGLISH_CONLL04 + RE_ENGLISH_DRUGPROT + diff --git a/docs/api/datasets/sequence_labeling.rst b/docs/api/datasets/sequence_labeling.rst index 875d4831b1..ae5b48669b 100644 --- a/docs/api/datasets/sequence_labeling.rst +++ b/docs/api/datasets/sequence_labeling.rst @@ -1,4 +1,76 @@ flair.datasets.sequence_labeling ================================ -.. automodule:: flair.datasets.sequence_labeling \ No newline at end of file +.. currentmodule:: flair.datasets.sequence_labeling + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + MultiFileJsonlCorpus + JsonlCorpus + JsonlDataset + MultiFileColumnCorpus + ColumnCorpus + ColumnDataset + ONTONOTES + CONLL_03 + CONLL_03_GERMAN + CONLL_03_DUTCH + CONLL_03_SPANISH + CONLL_2000 + WNUT_17 + FEWNERD + BIOSCOPE + NER_ARABIC_ANER + NER_ARABIC_AQMAR + NER_BASQUE + NER_CHINESE_WEIBO + NER_DANISH_DANE + NER_ENGLISH_MOVIE_SIMPLE + NER_ENGLISH_MOVIE_COMPLEX + NER_ENGLISH_SEC_FILLINGS + NER_ENGLISH_RESTAURANT + NER_ENGLISH_STACKOVERFLOW + NER_ENGLISH_TWITTER + NER_ENGLISH_PERSON + NER_ENGLISH_WEBPAGES + NER_ENGLISH_WNUT_2020 + NER_ENGLISH_WIKIGOLD + NER_FINNISH + NER_GERMAN_BIOFID + NER_GERMAN_EUROPARL + NER_GERMAN_LEGAL + NER_GERMAN_GERMEVAL + NER_GERMAN_POLITICS + NER_HUNGARIAN + NER_ICELANDIC + NER_JAPANESE + NER_MASAKHANE + NER_MULTI_CONER + NER_MULTI_CONER_V2 + NER_MULTI_WIKIANN + NER_MULTI_XTREME + NER_MULTI_WIKINER + NER_SWEDISH + NER_TURKU + NER_UKRAINIAN + KEYPHRASE_SEMEVAL2017 + KEYPHRASE_INSPEC + KEYPHRASE_SEMEVAL2010 + UP_CHINESE + UP_ENGLISH + UP_FRENCH + UP_FINNISH + UP_GERMAN + UP_ITALIAN + UP_SPANISH + UP_SPANISH_ANCORA + NER_HIPE_2022 + NER_ICDAR_EUROPEANA + NER_NERMUD + NER_GERMAN_MOBIE + MASAKHA_POS + diff --git a/docs/api/datasets/text_image.rst b/docs/api/datasets/text_image.rst index f14e564916..d6dd70df86 100644 --- a/docs/api/datasets/text_image.rst +++ b/docs/api/datasets/text_image.rst @@ -1,4 +1,14 @@ flair.datasets.text_image ========================= -.. automodule:: flair.datasets.text_image \ No newline at end of file +.. currentmodule:: flair.datasets.text_image + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + FeideggerCorpus + FeideggerDataset + diff --git a/docs/api/datasets/text_text.rst b/docs/api/datasets/text_text.rst index f88dfd1aed..94e007b440 100644 --- a/docs/api/datasets/text_text.rst +++ b/docs/api/datasets/text_text.rst @@ -1,4 +1,29 @@ flair.datasets.text_text -========================= +======================== + +.. currentmodule:: flair.datasets.text_text + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + + ParallelTextCorpus + OpusParallelCorpus + ParallelTextDataset + DataPairCorpus + DataPairDataset + GLUE_RTE + GLUE_MNLI + GLUE_MRPC + GLUE_QNLI + GLUE_QQP + GLUE_WNLI + GLUE_STSB + SUPERGLUE_RTE + rte_jsonl_to_tsv + + -.. automodule:: flair.datasets.text_text \ No newline at end of file diff --git a/docs/api/datasets/treebanks.rst b/docs/api/datasets/treebanks.rst index 0d6c14a281..df2bcdd673 100644 --- a/docs/api/datasets/treebanks.rst +++ b/docs/api/datasets/treebanks.rst @@ -1,4 +1,72 @@ flair.datasets.treebanks ======================== -.. automodule:: flair.datasets.treebanks \ No newline at end of file +.. currentmodule:: flair.datasets.treebanks + +.. autosummary:: + :toctree: generated + :nosignatures: + + + + UniversalDependenciesCorpus + UniversalDependenciesDataset + UD_ENGLISH + UD_GALICIAN + UD_ANCIENT_GREEK + UD_KAZAKH + UD_OLD_CHURCH_SLAVONIC + UD_ARMENIAN + UD_ESTONIAN + UD_GERMAN + UD_GERMAN_HDT + UD_DUTCH + UD_FAROESE + UD_FRENCH + UD_ITALIAN + UD_LATIN + UD_SPANISH + UD_PORTUGUESE + UD_ROMANIAN + UD_CATALAN + UD_POLISH + UD_CZECH + UD_SLOVAK + UD_SWEDISH + UD_DANISH + UD_NORWEGIAN + UD_FINNISH + UD_SLOVENIAN + UD_CROATIAN + UD_SERBIAN + UD_BULGARIAN + UD_ARABIC + UD_HEBREW + UD_TURKISH + UD_UKRAINIAN + UD_PERSIAN + UD_RUSSIAN + UD_HINDI + UD_INDONESIAN + UD_JAPANESE + UD_CHINESE + UD_KOREAN + UD_BASQUE + UD_CHINESE_KYOTO + UD_GREEK + UD_NAIJA + UD_LIVVI + UD_BURYAT + UD_NORTH_SAMI + UD_MARATHI + UD_MALTESE + UD_AFRIKAANS + UD_GOTHIC + UD_OLD_FRENCH + UD_WOLOF + UD_BELARUSIAN + UD_COPTIC + UD_IRISH + UD_LATVIAN + UD_LITHUANIAN + From 2c0f5c951d77af8161add91cb6cdf4b3777b548f Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 14:31:47 +0200 Subject: [PATCH 248/333] changes --- docs/api/embeddings/base.rst | 16 ++++++++++++++++ docs/api/embeddings/document.rst | 17 +++++++++++++++++ docs/api/embeddings/image.rst | 15 +++++++++++++++ docs/api/embeddings/legacy.rst | 14 ++++++++++++++ docs/api/embeddings/token.rst | 22 ++++++++++++++++++++++ docs/api/embeddings/transformer.rst | 27 +++++++++++++++++++++++++++ 6 files changed, 111 insertions(+) create mode 100644 docs/api/embeddings/base.rst create mode 100644 docs/api/embeddings/document.rst create mode 100644 docs/api/embeddings/image.rst create mode 100644 docs/api/embeddings/legacy.rst create mode 100644 docs/api/embeddings/token.rst create mode 100644 docs/api/embeddings/transformer.rst diff --git a/docs/api/embeddings/base.rst b/docs/api/embeddings/base.rst new file mode 100644 index 0000000000..ffa59ae86e --- /dev/null +++ b/docs/api/embeddings/base.rst @@ -0,0 +1,16 @@ +flair.embeddings.base +===================== + +.. currentmodule:: flair.embeddings.base + +.. autosummary:: + :toctree: generated + :nosignatures: + + + Embeddings + ScalarMix + DocumentEmbeddings + TokenEmbeddings + register_embeddings + load_embeddings diff --git a/docs/api/embeddings/document.rst b/docs/api/embeddings/document.rst new file mode 100644 index 0000000000..7c97c1ed4b --- /dev/null +++ b/docs/api/embeddings/document.rst @@ -0,0 +1,17 @@ +flair.embeddings.document +========================= + +.. currentmodule:: flair.embeddings.document + +.. autosummary:: + :toctree: generated + :nosignatures: + + + TransformerDocumentEmbeddings + DocumentPoolEmbeddings + DocumentTFIDFEmbeddings + DocumentRNNEmbeddings + DocumentLMEmbeddings + SentenceTransformerDocumentEmbeddings + DocumentCNNEmbeddings diff --git a/docs/api/embeddings/image.rst b/docs/api/embeddings/image.rst new file mode 100644 index 0000000000..4abb5a3cf7 --- /dev/null +++ b/docs/api/embeddings/image.rst @@ -0,0 +1,15 @@ +flair.embeddings.image +====================== + +.. currentmodule:: flair.embeddings.image + +.. autosummary:: + :toctree: generated + :nosignatures: + + + ImageEmbeddings + IdentityImageEmbeddings + PrecomputedImageEmbeddings + NetworkImageEmbeddings + ConvTransformNetworkImageEmbeddings diff --git a/docs/api/embeddings/legacy.rst b/docs/api/embeddings/legacy.rst new file mode 100644 index 0000000000..342200702e --- /dev/null +++ b/docs/api/embeddings/legacy.rst @@ -0,0 +1,14 @@ +flair.embeddings.legacy +======================= + +.. currentmodule:: flair.embeddings.legacy + +.. autosummary:: + :toctree: generated + :nosignatures: + + + ELMoEmbeddings + CharLMEmbeddings + DocumentMeanEmbeddings + DocumentLSTMEmbeddings diff --git a/docs/api/embeddings/token.rst b/docs/api/embeddings/token.rst new file mode 100644 index 0000000000..fef572a758 --- /dev/null +++ b/docs/api/embeddings/token.rst @@ -0,0 +1,22 @@ +flair.embeddings.token +====================== + +.. currentmodule:: flair.embeddings.token + +.. autosummary:: + :toctree: generated + :nosignatures: + + + TransformerWordEmbeddings + StackedEmbeddings + WordEmbeddings + CharacterEmbeddings + FlairEmbeddings + PooledFlairEmbeddings + FastTextEmbeddings + OneHotEmbeddings + HashEmbeddings + MuseCrosslingualEmbeddings + BytePairEmbeddings + NILCEmbeddings diff --git a/docs/api/embeddings/transformer.rst b/docs/api/embeddings/transformer.rst new file mode 100644 index 0000000000..c1c368d3b3 --- /dev/null +++ b/docs/api/embeddings/transformer.rst @@ -0,0 +1,27 @@ +flair.embeddings.transformer +============================ + +.. currentmodule:: flair.embeddings.transformer + +.. autosummary:: + :toctree: generated + :nosignatures: + + + pad_sequence_embeddings + truncate_hidden_states + combine_strided_tensors + fill_masked_elements + insert_missing_embeddings + fill_mean_token_embeddings + document_mean_pooling + document_max_pooling + remove_special_markup + TransformerBaseEmbeddings + TransformerOnnxEmbeddings + TransformerJitEmbeddings + TransformerJitWordEmbeddings + TransformerJitDocumentEmbeddings + TransformerOnnxWordEmbeddings + TransformerOnnxDocumentEmbeddings + TransformerEmbeddings From b2c6f0befd03110facd44b4075fa323c87277ab5 Mon Sep 17 00:00:00 2001 From: konstantin Date: Wed, 28 Aug 2024 14:34:21 +0200 Subject: [PATCH 249/333] changes --- docs/api/flair.nn.rst | 14 +++++++++++++- docs/api/flair.splitter.rst | 16 +++++++++++++++- docs/api/flair.tokenization.rst | 14 +++++++++++++- docs/api/flair.trainers.plugins.rst | 24 +++++++++++++++++++++++- docs/api/flair.trainers.rst | 12 +++++++++++- 5 files changed, 75 insertions(+), 5 deletions(-) diff --git a/docs/api/flair.nn.rst b/docs/api/flair.nn.rst index 4eb066d3ea..04a31517b9 100644 --- a/docs/api/flair.nn.rst +++ b/docs/api/flair.nn.rst @@ -1,4 +1,16 @@ flair.nn ======== -.. automodule:: flair.nn \ No newline at end of file +.. currentmodule:: flair.nn + +.. autosummary:: + :toctree: generated + :nosignatures: + + LockedDropout + WordDropout + Classifier + DefaultClassifier + Model + PrototypicalDecoder + LabelVerbalizerDecoder diff --git a/docs/api/flair.splitter.rst b/docs/api/flair.splitter.rst index 5863df5788..43c7771eec 100644 --- a/docs/api/flair.splitter.rst +++ b/docs/api/flair.splitter.rst @@ -1,4 +1,18 @@ flair.splitter ============== -.. automodule:: flair.splitter \ No newline at end of file +.. currentmodule:: flair.splitter + +.. autosummary:: + :toctree: generated + :nosignatures: + + + SentenceSplitter + SegtokSentenceSplitter + SpacySentenceSplitter + SciSpacySentenceSplitter + TagSentenceSplitter + NewlineSentenceSplitter + NoSentenceSplitter + diff --git a/docs/api/flair.tokenization.rst b/docs/api/flair.tokenization.rst index 00f2bc4bfd..5c67e72e20 100644 --- a/docs/api/flair.tokenization.rst +++ b/docs/api/flair.tokenization.rst @@ -1,4 +1,16 @@ flair.tokenization ================== -.. automodule:: flair.tokenization \ No newline at end of file +.. currentmodule:: flair.tokenization + +.. autosummary:: + :toctree: generated + :nosignatures: + + Tokenizer + SpacyTokenizer + SegtokTokenizer + SpaceTokenizer + JapaneseTokenizer + TokenizerWrapper + SciSpacyTokenizer \ No newline at end of file diff --git a/docs/api/flair.trainers.plugins.rst b/docs/api/flair.trainers.plugins.rst index 4bb766876b..7c0591cf28 100644 --- a/docs/api/flair.trainers.plugins.rst +++ b/docs/api/flair.trainers.plugins.rst @@ -1,4 +1,26 @@ flair.trainers.plugins ====================== -.. automodule:: flair.trainers.plugins \ No newline at end of file +.. currentmodule:: flair.trainers.plugins + +.. autosummary:: + :toctree: generated + :nosignatures: + + + AnnealingPlugin + CheckpointPlugin + ClearmlLoggerPlugin + LinearSchedulerPlugin + WeightExtractorPlugin + LogFilePlugin + LossFilePlugin + MetricHistoryPlugin + TensorboardLogger + BasePlugin + Pluggable + TrainerPlugin + TrainingInterrupt + ReduceTransformerVocabPlugin + MetricName + MetricRecord diff --git a/docs/api/flair.trainers.rst b/docs/api/flair.trainers.rst index db11b5029c..7cff5b1b0c 100644 --- a/docs/api/flair.trainers.rst +++ b/docs/api/flair.trainers.rst @@ -1,4 +1,14 @@ flair.trainers ============== -.. automodule:: flair.trainers \ No newline at end of file +.. currentmodule:: flair.trainers + +.. autosummary:: + :toctree: generated + :nosignatures: + + + ModelTrainer + LanguageModelTrainer + TextCorpus + From 74be758a0829be70311dceec850935be1e729a71 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 3 Sep 2024 12:49:17 +0200 Subject: [PATCH 250/333] changes --- docs/_static/css/api.css | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/_static/css/api.css b/docs/_static/css/api.css index e2d1ba3e8c..e0e35f3a96 100644 --- a/docs/_static/css/api.css +++ b/docs/_static/css/api.css @@ -158,6 +158,16 @@ table.autosummary { margin-top: 1.5em; } +table.autosummary .field-list { + margin-bottom: 0.5rem; + display: flex !important; + font-size: 1rem; +} + +table.autosummary .field-list *:not(code) { + font-size: inherit !important; +} + table .field-odd, table .field-even { color: var(--blue-white); font-family: "Afacad", sans-serif; From b25bf66fae4ac08cdf5ec98c48dd32eb5fbf71e3 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 3 Sep 2024 13:27:54 +0200 Subject: [PATCH 251/333] remove documentation branch from sphinx conf --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 01a2f9f6a2..7bd241ccdd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -126,7 +126,7 @@ def linkcode_resolve(*args): smv_tag_whitelist = r"^v\d+\.\d+\.\d+$" # Whitelist pattern for branches (set to None to ignore all branches) -smv_branch_whitelist = r"^master|documentation$" +smv_branch_whitelist = r"^master$" # Whitelist pattern for remotes (set to None to use local branches only) smv_remote_whitelist = r"^origin$" From 14fda779b89fadbbe5bbbdd827a07244b120e1eb Mon Sep 17 00:00:00 2001 From: Konstantin Lukas Date: Tue, 17 Sep 2024 15:23:30 +0200 Subject: [PATCH 252/333] fix missing line breaks in code examples --- docs/_static/css/tutorial.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/tutorial.css b/docs/_static/css/tutorial.css index 4afc7dcdba..9139e8984f 100644 --- a/docs/_static/css/tutorial.css +++ b/docs/_static/css/tutorial.css @@ -10,7 +10,7 @@ padding: 0; } -.bd-main .bd-content .bd-article-container .bd-article * { +.bd-main .bd-content .bd-article-container .bd-article *:not(pre) { word-break: break-word; overflow-wrap: break-word; white-space: normal; From 1f33ac4d1cddd80a57edc953d864ba627fe5451d Mon Sep 17 00:00:00 2001 From: Konstantin Lukas Date: Tue, 17 Sep 2024 15:58:27 +0200 Subject: [PATCH 253/333] add missing classes in models module --- docs/api/flair.models.rst | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/api/flair.models.rst b/docs/api/flair.models.rst index a41d6170ac..dd1e8fb442 100644 --- a/docs/api/flair.models.rst +++ b/docs/api/flair.models.rst @@ -7,5 +7,25 @@ flair.models :toctree: generated :nosignatures: + + EntityMentionLinker SpanClassifier - LanguageModel \ No newline at end of file + LanguageModel + Lemmatizer + TextPairClassifier + TextTripleClassifier + TextPairRegressor + RelationClassifier + RelationExtractor + RegexpTagger + SequenceTagger + PrefixedSequenceTagger + TokenClassifier + WordTagger + FewshotClassifier + TARSClassifier + TARSTagger + TextClassifier + TextRegressor + ClusteringModel + MultitaskModel From ede09aa4e2a0e0eb31dfa18582ad5c67d3632bab Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 17 Sep 2024 19:22:09 +0200 Subject: [PATCH 254/333] add currentmodule directive to datasets and embeddings --- docs/api/flair.datasets.rst | 2 ++ docs/api/flair.embeddings.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index e663696a69..e27515a835 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -1,6 +1,8 @@ flair.datasets ============== +.. currentmodule:: flair.datasets + .. toctree:: :glob: :maxdepth: 1 diff --git a/docs/api/flair.embeddings.rst b/docs/api/flair.embeddings.rst index 3f70e62bef..82905def03 100644 --- a/docs/api/flair.embeddings.rst +++ b/docs/api/flair.embeddings.rst @@ -1,6 +1,8 @@ flair.embeddings ================ +.. currentmodule:: flair.embeddings + .. toctree:: :glob: :maxdepth: 2 From d8df4df42d012a5ec45db5afae07a59bdc99453e Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 17 Sep 2024 19:30:07 +0200 Subject: [PATCH 255/333] fix syntax error in markdown --- docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md b/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md index ed29bea502..f5cd654ed7 100644 --- a/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md +++ b/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md @@ -168,7 +168,7 @@ from flair.data import MultiCorpus multi_corpus = MultiCorpus([english_corpus, german_corpus, dutch_corpus]) ``` -The [`MultiCorpus`](#flair.data.MultiCorpus) inherits from `[`Corpus`](#flair.data.Corpus), so you can use it like any other corpus to train your models. +The [`MultiCorpus`](#flair.data.MultiCorpus) inherits from [`Corpus`](#flair.data.Corpus), so you can use it like any other corpus to train your models. ## Datasets included in Flair From d44206c8fbf77444a5bf24cd3718643cb6edf0b0 Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 17 Sep 2024 19:35:46 +0200 Subject: [PATCH 256/333] change max depth --- docs/api/flair.datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flair.datasets.rst b/docs/api/flair.datasets.rst index e27515a835..d822186e67 100644 --- a/docs/api/flair.datasets.rst +++ b/docs/api/flair.datasets.rst @@ -5,6 +5,6 @@ flair.datasets .. toctree:: :glob: - :maxdepth: 1 + :maxdepth: 2 datasets/* From a911a01c6dc1146a7a8469205b0aa441aae4abad Mon Sep 17 00:00:00 2001 From: konstantin Date: Tue, 17 Sep 2024 22:00:45 +0200 Subject: [PATCH 257/333] remove reference to glossary --- .../tutorial/tutorial-training/how-to-train-span-classifier.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tutorial/tutorial-training/how-to-train-span-classifier.md b/docs/tutorial/tutorial-training/how-to-train-span-classifier.md index e5d32cb426..21c5ee7de7 100644 --- a/docs/tutorial/tutorial-training/how-to-train-span-classifier.md +++ b/docs/tutorial/tutorial-training/how-to-train-span-classifier.md @@ -1,8 +1,7 @@ # Train a span classifier Span Classification models are used to model problems such as entity linking, where you already have extracted some -relevant spans -within the {term}`Sentence` and want to predict some more fine-grained labels. +relevant spans within the `Sentence` and want to predict some more fine-grained labels. This tutorial section show you how to train models using the [Span Classifier](#flair.models.SpanClassifier) in Flair. From 935bb44399cfc61c95d47003a1e953cf4133892b Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 27 Sep 2024 19:12:18 +0200 Subject: [PATCH 258/333] try make flair.models use autocollection --- docs/api/datasets/biomedical.rst | 3 --- docs/api/flair.models.rst | 26 +++----------------------- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/docs/api/datasets/biomedical.rst b/docs/api/datasets/biomedical.rst index a58dc51a6c..48e092d504 100644 --- a/docs/api/datasets/biomedical.rst +++ b/docs/api/datasets/biomedical.rst @@ -141,7 +141,4 @@ flair.datasets.biomedical HUNER_GENE_PROGENE HUNER_CHEMICAL_NLM_CHEM HUNER_GENE_SETH_CORPUS - HUNER_GENE_TMVAR_V3 - HUNER_SPECIES_TMVAR_V3 - HUNER_CELL_LINE_TMVAR_V3 diff --git a/docs/api/flair.models.rst b/docs/api/flair.models.rst index dd1e8fb442..bcbc87b1ec 100644 --- a/docs/api/flair.models.rst +++ b/docs/api/flair.models.rst @@ -1,31 +1,11 @@ flair.models ============ -.. currentmodule:: flair.models +.. currentmodule:: models .. autosummary:: :toctree: generated :nosignatures: + :recursive: - - EntityMentionLinker - SpanClassifier - LanguageModel - Lemmatizer - TextPairClassifier - TextTripleClassifier - TextPairRegressor - RelationClassifier - RelationExtractor - RegexpTagger - SequenceTagger - PrefixedSequenceTagger - TokenClassifier - WordTagger - FewshotClassifier - TARSClassifier - TARSTagger - TextClassifier - TextRegressor - ClusteringModel - MultitaskModel + models From 88374ff229c0cdedea4c8c348cc1af51c040ccf0 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 27 Sep 2024 19:31:22 +0200 Subject: [PATCH 259/333] try make flair.datasets.base use autocollection --- docs/api/datasets/base.rst | 10 ++-------- docs/api/flair.models.rst | 26 +++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst index fbb80b0281..f8c0109916 100644 --- a/docs/api/datasets/base.rst +++ b/docs/api/datasets/base.rst @@ -6,13 +6,7 @@ flair.datasets.base .. autosummary:: :toctree: generated :nosignatures: + :recursive: - - - DataLoader - FlairDatapointDataset - SentenceDataset - StringDataset - MongoDataset - find_train_dev_test_files + ..base diff --git a/docs/api/flair.models.rst b/docs/api/flair.models.rst index bcbc87b1ec..dd1e8fb442 100644 --- a/docs/api/flair.models.rst +++ b/docs/api/flair.models.rst @@ -1,11 +1,31 @@ flair.models ============ -.. currentmodule:: models +.. currentmodule:: flair.models .. autosummary:: :toctree: generated :nosignatures: - :recursive: - models + + EntityMentionLinker + SpanClassifier + LanguageModel + Lemmatizer + TextPairClassifier + TextTripleClassifier + TextPairRegressor + RelationClassifier + RelationExtractor + RegexpTagger + SequenceTagger + PrefixedSequenceTagger + TokenClassifier + WordTagger + FewshotClassifier + TARSClassifier + TARSTagger + TextClassifier + TextRegressor + ClusteringModel + MultitaskModel From 676bee2ae835b009936172b6784c189017f8ea29 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 27 Sep 2024 23:50:36 +0200 Subject: [PATCH 260/333] fix legal notice --- docs/legal-notice/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/legal-notice/index.rst b/docs/legal-notice/index.rst index 5da0252fd4..585047fc11 100644 --- a/docs/legal-notice/index.rst +++ b/docs/legal-notice/index.rst @@ -10,6 +10,6 @@ Legal Notice :maxdepth: 3 :hidden: - Tutorials - API - Contributing \ No newline at end of file + Tutorials <../tutorial/index> + API <../api/index> + Contributing <../contributing/index> \ No newline at end of file From 6ff14b1646e3884887a4332a903fe2682f18d5a6 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 27 Sep 2024 23:50:47 +0200 Subject: [PATCH 261/333] cleanup datasets.base --- docs/api/datasets/base.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/api/datasets/base.rst b/docs/api/datasets/base.rst index f8c0109916..80c375eec8 100644 --- a/docs/api/datasets/base.rst +++ b/docs/api/datasets/base.rst @@ -6,7 +6,3 @@ flair.datasets.base .. autosummary:: :toctree: generated :nosignatures: - :recursive: - - ..base - From a1691677eb217ae592f8bb73b6174dd0046af068 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Sat, 28 Sep 2024 01:19:42 +0200 Subject: [PATCH 262/333] don't specify all elements directly --- docs/api/datasets/biomedical.rst | 136 ------------------ docs/api/datasets/document_classification.rst | 30 ---- docs/api/datasets/entity_linking.rst | 20 --- docs/api/datasets/ocr.rst | 7 - docs/api/datasets/relation_extraction.rst | 9 -- docs/api/datasets/sequence_labeling.rst | 68 --------- docs/api/datasets/text_image.rst | 6 - docs/api/datasets/text_text.rst | 19 --- docs/api/datasets/treebanks.rst | 64 --------- docs/api/embeddings/base.rst | 8 -- docs/api/embeddings/document.rst | 9 -- docs/api/embeddings/image.rst | 7 - docs/api/embeddings/legacy.rst | 6 - docs/api/embeddings/token.rst | 14 -- docs/api/embeddings/transformer.rst | 19 --- docs/api/flair.data.rst | 19 --- docs/api/flair.models.rst | 23 --- docs/api/flair.nn.rst | 8 -- docs/api/flair.rst | 6 - docs/api/flair.splitter.rst | 9 -- docs/api/flair.tokenization.rst | 8 -- docs/api/flair.trainers.plugins.rst | 18 --- docs/api/flair.trainers.rst | 6 - 23 files changed, 519 deletions(-) diff --git a/docs/api/datasets/biomedical.rst b/docs/api/datasets/biomedical.rst index 48e092d504..c1d2525ece 100644 --- a/docs/api/datasets/biomedical.rst +++ b/docs/api/datasets/biomedical.rst @@ -6,139 +6,3 @@ flair.datasets.biomedical .. autosummary:: :toctree: generated :nosignatures: - - - - Entity - InternalBioNerDataset - DpEntry - merge_datasets - filter_and_map_entities - filter_nested_entities - bioc_to_internal - brat_to_internal - CoNLLWriter - HunerDataset - BIO_INFER - HUNER_GENE_BIO_INFER - JNLPBA - HunerJNLPBA - HUNER_GENE_JNLPBA - HUNER_CELL_LINE_JNLPBA - CELL_FINDER - HUNER_CELL_LINE_CELL_FINDER - HUNER_SPECIES_CELL_FINDER - HUNER_GENE_CELL_FINDER - MIRNA - HunerMiRNAHelper - HUNER_GENE_MIRNA - HUNER_SPECIES_MIRNA - HUNER_DISEASE_MIRNA - KaewphanCorpusHelper - CLL - HUNER_CELL_LINE_CLL - GELLUS - HUNER_CELL_LINE_GELLUS - LOCTEXT - HUNER_SPECIES_LOCTEXT - HUNER_GENE_LOCTEXT - CHEMDNER - HUNER_CHEMICAL_CHEMDNER - IEPA - HUNER_GENE_IEPA - LINNEAUS - HUNER_SPECIES_LINNEAUS - CDR - HUNER_DISEASE_CDR - HUNER_CHEMICAL_CDR - VARIOME - HUNER_GENE_VARIOME - HUNER_DISEASE_VARIOME - HUNER_SPECIES_VARIOME - NCBI_DISEASE - HUNER_DISEASE_NCBI - ScaiCorpus - SCAI_CHEMICALS - SCAI_DISEASE - HUNER_CHEMICAL_SCAI - HUNER_DISEASE_SCAI - OSIRIS - HUNER_GENE_OSIRIS - S800 - HUNER_SPECIES_S800 - GPRO - HUNER_GENE_GPRO - DECA - HUNER_GENE_DECA - FSU - HUNER_GENE_FSU - CRAFT - BIOSEMANTICS - BC2GM - HUNER_GENE_BC2GM - CEMP - HUNER_CHEMICAL_CEMP - CHEBI - HUNER_CHEMICAL_CHEBI - HUNER_GENE_CHEBI - HUNER_SPECIES_CHEBI - BioNLPCorpus - BIONLP2013_PC - BIONLP2013_CG - ANAT_EM - BioBertHelper - BIOBERT_CHEMICAL_BC4CHEMD - BIOBERT_GENE_BC2GM - BIOBERT_GENE_JNLPBA - BIOBERT_CHEMICAL_BC5CDR - BIOBERT_DISEASE_BC5CDR - BIOBERT_DISEASE_NCBI - BIOBERT_SPECIES_LINNAEUS - BIOBERT_SPECIES_S800 - CRAFT_V4 - HUNER_CHEMICAL_CRAFT_V4 - HUNER_GENE_CRAFT_V4 - HUNER_SPECIES_CRAFT_V4 - HUNER_CHEMICAL_BIONLP2013_CG - HUNER_DISEASE_BIONLP2013_CG - HUNER_GENE_BIONLP2013_CG - HUNER_SPECIES_BIONLP2013_CG - AZDZ - PDR - HUNER_DISEASE_PDR - HunerMultiCorpus - HUNER_CELL_LINE - HUNER_CHEMICAL - HUNER_DISEASE - HUNER_GENE - HUNER_SPECIES - BIGBIO_NER_CORPUS - HUNER_GENE_NLM_GENE - HUNER_GENE_DRUGPROT - HUNER_CHEMICAL_DRUGPROT - HUNER_GENE_BIORED - HUNER_CHEMICAL_BIORED - HUNER_DISEASE_BIORED - HUNER_SPECIES_BIORED - HUNER_CELL_LINE_BIORED - HUNER_GENE_CPI - HUNER_CHEMICAL_CPI - HUNER_GENE_BIONLP_ST_2013_PC - HUNER_CHEMICAL_BIONLP_ST_2013_PC - HUNER_GENE_BIONLP_ST_2013_GE - HUNER_GENE_BIONLP_ST_2011_GE - HUNER_GENE_BIONLP_ST_2011_ID - HUNER_CHEMICAL_BIONLP_ST_2011_ID - HUNER_SPECIES_BIONLP_ST_2011_ID - HUNER_GENE_BIONLP_ST_2011_REL - HUNER_GENE_BIONLP_ST_2011_EPI - HUNER_SPECIES_BIONLP_ST_2019_BB - HUNER_GENE_BIOID - HUNER_CHEMICAL_BIOID - HUNER_SPECIES_BIOID - HUNER_CELL_LINE_BIOID - HUNER_GENE_GNORMPLUS - HUNER_GENE_PROGENE - HUNER_CHEMICAL_NLM_CHEM - HUNER_GENE_SETH_CORPUS - diff --git a/docs/api/datasets/document_classification.rst b/docs/api/datasets/document_classification.rst index 8d775e1413..0de14b6cdd 100644 --- a/docs/api/datasets/document_classification.rst +++ b/docs/api/datasets/document_classification.rst @@ -6,33 +6,3 @@ flair.datasets.document_classification .. autosummary:: :toctree: generated :nosignatures: - - - ClassificationCorpus - ClassificationDataset - CSVClassificationCorpus - CSVClassificationDataset - AMAZON_REVIEWS - IMDB - NEWSGROUPS - AGNEWS - STACKOVERFLOW - SENTIMENT_140 - SENTEVAL_CR - SENTEVAL_MR - SENTEVAL_SUBJ - SENTEVAL_MPQA - SENTEVAL_SST_BINARY - SENTEVAL_SST_GRANULAR - GLUE_COLA - GLUE_SST2 - GO_EMOTIONS - TREC_50 - TREC_6 - YAHOO_ANSWERS - GERMEVAL_2018_OFFENSIVE_LANGUAGE - COMMUNICATIVE_FUNCTIONS - WASSA_ANGER - WASSA_FEAR - WASSA_JOY - WASSA_SADNESS diff --git a/docs/api/datasets/entity_linking.rst b/docs/api/datasets/entity_linking.rst index 7c616f3b85..a88308e97d 100644 --- a/docs/api/datasets/entity_linking.rst +++ b/docs/api/datasets/entity_linking.rst @@ -6,23 +6,3 @@ flair.datasets.entity_linking .. autosummary:: :toctree: generated :nosignatures: - - - - ZELDA - NEL_ENGLISH_AQUAINT - NEL_GERMAN_HIPE - NEL_ENGLISH_AIDA - NEL_ENGLISH_IITB - NEL_ENGLISH_TWEEKI - NEL_ENGLISH_REDDIT - from_ufsac_to_tsv - determine_tsv_file - WSD_UFSAC - WSD_RAGANATO_ALL - WSD_SEMCOR - WSD_WORDNET_GLOSS_TAGGED - WSD_MASC - WSD_OMSTI - WSD_TRAINOMATIC - diff --git a/docs/api/datasets/ocr.rst b/docs/api/datasets/ocr.rst index d9b005a0ea..f5629a5631 100644 --- a/docs/api/datasets/ocr.rst +++ b/docs/api/datasets/ocr.rst @@ -6,10 +6,3 @@ flair.datasets.ocr .. autosummary:: :toctree: generated :nosignatures: - - - - OcrJsonDataset - OcrCorpus - SROIE - diff --git a/docs/api/datasets/relation_extraction.rst b/docs/api/datasets/relation_extraction.rst index c840b00fd6..fdbb690cfe 100644 --- a/docs/api/datasets/relation_extraction.rst +++ b/docs/api/datasets/relation_extraction.rst @@ -6,12 +6,3 @@ flair.datasets.relation_extraction .. autosummary:: :toctree: generated :nosignatures: - - - - convert_ptb_token - RE_ENGLISH_SEMEVAL2010 - RE_ENGLISH_TACRED - RE_ENGLISH_CONLL04 - RE_ENGLISH_DRUGPROT - diff --git a/docs/api/datasets/sequence_labeling.rst b/docs/api/datasets/sequence_labeling.rst index ae5b48669b..0c0abc520d 100644 --- a/docs/api/datasets/sequence_labeling.rst +++ b/docs/api/datasets/sequence_labeling.rst @@ -6,71 +6,3 @@ flair.datasets.sequence_labeling .. autosummary:: :toctree: generated :nosignatures: - - - - MultiFileJsonlCorpus - JsonlCorpus - JsonlDataset - MultiFileColumnCorpus - ColumnCorpus - ColumnDataset - ONTONOTES - CONLL_03 - CONLL_03_GERMAN - CONLL_03_DUTCH - CONLL_03_SPANISH - CONLL_2000 - WNUT_17 - FEWNERD - BIOSCOPE - NER_ARABIC_ANER - NER_ARABIC_AQMAR - NER_BASQUE - NER_CHINESE_WEIBO - NER_DANISH_DANE - NER_ENGLISH_MOVIE_SIMPLE - NER_ENGLISH_MOVIE_COMPLEX - NER_ENGLISH_SEC_FILLINGS - NER_ENGLISH_RESTAURANT - NER_ENGLISH_STACKOVERFLOW - NER_ENGLISH_TWITTER - NER_ENGLISH_PERSON - NER_ENGLISH_WEBPAGES - NER_ENGLISH_WNUT_2020 - NER_ENGLISH_WIKIGOLD - NER_FINNISH - NER_GERMAN_BIOFID - NER_GERMAN_EUROPARL - NER_GERMAN_LEGAL - NER_GERMAN_GERMEVAL - NER_GERMAN_POLITICS - NER_HUNGARIAN - NER_ICELANDIC - NER_JAPANESE - NER_MASAKHANE - NER_MULTI_CONER - NER_MULTI_CONER_V2 - NER_MULTI_WIKIANN - NER_MULTI_XTREME - NER_MULTI_WIKINER - NER_SWEDISH - NER_TURKU - NER_UKRAINIAN - KEYPHRASE_SEMEVAL2017 - KEYPHRASE_INSPEC - KEYPHRASE_SEMEVAL2010 - UP_CHINESE - UP_ENGLISH - UP_FRENCH - UP_FINNISH - UP_GERMAN - UP_ITALIAN - UP_SPANISH - UP_SPANISH_ANCORA - NER_HIPE_2022 - NER_ICDAR_EUROPEANA - NER_NERMUD - NER_GERMAN_MOBIE - MASAKHA_POS - diff --git a/docs/api/datasets/text_image.rst b/docs/api/datasets/text_image.rst index d6dd70df86..173928dfc1 100644 --- a/docs/api/datasets/text_image.rst +++ b/docs/api/datasets/text_image.rst @@ -6,9 +6,3 @@ flair.datasets.text_image .. autosummary:: :toctree: generated :nosignatures: - - - - FeideggerCorpus - FeideggerDataset - diff --git a/docs/api/datasets/text_text.rst b/docs/api/datasets/text_text.rst index 94e007b440..79a60ae609 100644 --- a/docs/api/datasets/text_text.rst +++ b/docs/api/datasets/text_text.rst @@ -8,22 +8,3 @@ flair.datasets.text_text :nosignatures: - - - ParallelTextCorpus - OpusParallelCorpus - ParallelTextDataset - DataPairCorpus - DataPairDataset - GLUE_RTE - GLUE_MNLI - GLUE_MRPC - GLUE_QNLI - GLUE_QQP - GLUE_WNLI - GLUE_STSB - SUPERGLUE_RTE - rte_jsonl_to_tsv - - - diff --git a/docs/api/datasets/treebanks.rst b/docs/api/datasets/treebanks.rst index df2bcdd673..82cba954f7 100644 --- a/docs/api/datasets/treebanks.rst +++ b/docs/api/datasets/treebanks.rst @@ -6,67 +6,3 @@ flair.datasets.treebanks .. autosummary:: :toctree: generated :nosignatures: - - - - UniversalDependenciesCorpus - UniversalDependenciesDataset - UD_ENGLISH - UD_GALICIAN - UD_ANCIENT_GREEK - UD_KAZAKH - UD_OLD_CHURCH_SLAVONIC - UD_ARMENIAN - UD_ESTONIAN - UD_GERMAN - UD_GERMAN_HDT - UD_DUTCH - UD_FAROESE - UD_FRENCH - UD_ITALIAN - UD_LATIN - UD_SPANISH - UD_PORTUGUESE - UD_ROMANIAN - UD_CATALAN - UD_POLISH - UD_CZECH - UD_SLOVAK - UD_SWEDISH - UD_DANISH - UD_NORWEGIAN - UD_FINNISH - UD_SLOVENIAN - UD_CROATIAN - UD_SERBIAN - UD_BULGARIAN - UD_ARABIC - UD_HEBREW - UD_TURKISH - UD_UKRAINIAN - UD_PERSIAN - UD_RUSSIAN - UD_HINDI - UD_INDONESIAN - UD_JAPANESE - UD_CHINESE - UD_KOREAN - UD_BASQUE - UD_CHINESE_KYOTO - UD_GREEK - UD_NAIJA - UD_LIVVI - UD_BURYAT - UD_NORTH_SAMI - UD_MARATHI - UD_MALTESE - UD_AFRIKAANS - UD_GOTHIC - UD_OLD_FRENCH - UD_WOLOF - UD_BELARUSIAN - UD_COPTIC - UD_IRISH - UD_LATVIAN - UD_LITHUANIAN - diff --git a/docs/api/embeddings/base.rst b/docs/api/embeddings/base.rst index ffa59ae86e..02a65a20b9 100644 --- a/docs/api/embeddings/base.rst +++ b/docs/api/embeddings/base.rst @@ -6,11 +6,3 @@ flair.embeddings.base .. autosummary:: :toctree: generated :nosignatures: - - - Embeddings - ScalarMix - DocumentEmbeddings - TokenEmbeddings - register_embeddings - load_embeddings diff --git a/docs/api/embeddings/document.rst b/docs/api/embeddings/document.rst index 7c97c1ed4b..8c5aea548d 100644 --- a/docs/api/embeddings/document.rst +++ b/docs/api/embeddings/document.rst @@ -6,12 +6,3 @@ flair.embeddings.document .. autosummary:: :toctree: generated :nosignatures: - - - TransformerDocumentEmbeddings - DocumentPoolEmbeddings - DocumentTFIDFEmbeddings - DocumentRNNEmbeddings - DocumentLMEmbeddings - SentenceTransformerDocumentEmbeddings - DocumentCNNEmbeddings diff --git a/docs/api/embeddings/image.rst b/docs/api/embeddings/image.rst index 4abb5a3cf7..6a115e705c 100644 --- a/docs/api/embeddings/image.rst +++ b/docs/api/embeddings/image.rst @@ -6,10 +6,3 @@ flair.embeddings.image .. autosummary:: :toctree: generated :nosignatures: - - - ImageEmbeddings - IdentityImageEmbeddings - PrecomputedImageEmbeddings - NetworkImageEmbeddings - ConvTransformNetworkImageEmbeddings diff --git a/docs/api/embeddings/legacy.rst b/docs/api/embeddings/legacy.rst index 342200702e..a177a1ffc2 100644 --- a/docs/api/embeddings/legacy.rst +++ b/docs/api/embeddings/legacy.rst @@ -6,9 +6,3 @@ flair.embeddings.legacy .. autosummary:: :toctree: generated :nosignatures: - - - ELMoEmbeddings - CharLMEmbeddings - DocumentMeanEmbeddings - DocumentLSTMEmbeddings diff --git a/docs/api/embeddings/token.rst b/docs/api/embeddings/token.rst index fef572a758..7fc6305bdb 100644 --- a/docs/api/embeddings/token.rst +++ b/docs/api/embeddings/token.rst @@ -6,17 +6,3 @@ flair.embeddings.token .. autosummary:: :toctree: generated :nosignatures: - - - TransformerWordEmbeddings - StackedEmbeddings - WordEmbeddings - CharacterEmbeddings - FlairEmbeddings - PooledFlairEmbeddings - FastTextEmbeddings - OneHotEmbeddings - HashEmbeddings - MuseCrosslingualEmbeddings - BytePairEmbeddings - NILCEmbeddings diff --git a/docs/api/embeddings/transformer.rst b/docs/api/embeddings/transformer.rst index c1c368d3b3..51e04a43c9 100644 --- a/docs/api/embeddings/transformer.rst +++ b/docs/api/embeddings/transformer.rst @@ -6,22 +6,3 @@ flair.embeddings.transformer .. autosummary:: :toctree: generated :nosignatures: - - - pad_sequence_embeddings - truncate_hidden_states - combine_strided_tensors - fill_masked_elements - insert_missing_embeddings - fill_mean_token_embeddings - document_mean_pooling - document_max_pooling - remove_special_markup - TransformerBaseEmbeddings - TransformerOnnxEmbeddings - TransformerJitEmbeddings - TransformerJitWordEmbeddings - TransformerJitDocumentEmbeddings - TransformerOnnxWordEmbeddings - TransformerOnnxDocumentEmbeddings - TransformerEmbeddings diff --git a/docs/api/flair.data.rst b/docs/api/flair.data.rst index 5e20e3f51a..dae99e093b 100644 --- a/docs/api/flair.data.rst +++ b/docs/api/flair.data.rst @@ -6,22 +6,3 @@ flair.data .. autosummary:: :toctree: generated :nosignatures: - - - BoundingBox - Dictionary - Label - DataPoint - Token - Span - Relation - Sentence - DataPair - Image - Corpus - MultiCorpus - FlairDataset - ConcatFlairDataset - iob2 - randomly_split_into_two_datasets - get_spans_from_bio diff --git a/docs/api/flair.models.rst b/docs/api/flair.models.rst index dd1e8fb442..0451fb30bd 100644 --- a/docs/api/flair.models.rst +++ b/docs/api/flair.models.rst @@ -6,26 +6,3 @@ flair.models .. autosummary:: :toctree: generated :nosignatures: - - - EntityMentionLinker - SpanClassifier - LanguageModel - Lemmatizer - TextPairClassifier - TextTripleClassifier - TextPairRegressor - RelationClassifier - RelationExtractor - RegexpTagger - SequenceTagger - PrefixedSequenceTagger - TokenClassifier - WordTagger - FewshotClassifier - TARSClassifier - TARSTagger - TextClassifier - TextRegressor - ClusteringModel - MultitaskModel diff --git a/docs/api/flair.nn.rst b/docs/api/flair.nn.rst index 04a31517b9..6a7247256a 100644 --- a/docs/api/flair.nn.rst +++ b/docs/api/flair.nn.rst @@ -6,11 +6,3 @@ flair.nn .. autosummary:: :toctree: generated :nosignatures: - - LockedDropout - WordDropout - Classifier - DefaultClassifier - Model - PrototypicalDecoder - LabelVerbalizerDecoder diff --git a/docs/api/flair.rst b/docs/api/flair.rst index 6c2b90e00f..946c68a6a4 100644 --- a/docs/api/flair.rst +++ b/docs/api/flair.rst @@ -6,9 +6,3 @@ flair .. autosummary:: :toctree: generated :nosignatures: - - cache_root - device - logger - set_seed - set_proxies \ No newline at end of file diff --git a/docs/api/flair.splitter.rst b/docs/api/flair.splitter.rst index 43c7771eec..f9b71de316 100644 --- a/docs/api/flair.splitter.rst +++ b/docs/api/flair.splitter.rst @@ -7,12 +7,3 @@ flair.splitter :toctree: generated :nosignatures: - - SentenceSplitter - SegtokSentenceSplitter - SpacySentenceSplitter - SciSpacySentenceSplitter - TagSentenceSplitter - NewlineSentenceSplitter - NoSentenceSplitter - diff --git a/docs/api/flair.tokenization.rst b/docs/api/flair.tokenization.rst index 5c67e72e20..ec5ca557d1 100644 --- a/docs/api/flair.tokenization.rst +++ b/docs/api/flair.tokenization.rst @@ -6,11 +6,3 @@ flair.tokenization .. autosummary:: :toctree: generated :nosignatures: - - Tokenizer - SpacyTokenizer - SegtokTokenizer - SpaceTokenizer - JapaneseTokenizer - TokenizerWrapper - SciSpacyTokenizer \ No newline at end of file diff --git a/docs/api/flair.trainers.plugins.rst b/docs/api/flair.trainers.plugins.rst index 7c0591cf28..acd8109cf4 100644 --- a/docs/api/flair.trainers.plugins.rst +++ b/docs/api/flair.trainers.plugins.rst @@ -6,21 +6,3 @@ flair.trainers.plugins .. autosummary:: :toctree: generated :nosignatures: - - - AnnealingPlugin - CheckpointPlugin - ClearmlLoggerPlugin - LinearSchedulerPlugin - WeightExtractorPlugin - LogFilePlugin - LossFilePlugin - MetricHistoryPlugin - TensorboardLogger - BasePlugin - Pluggable - TrainerPlugin - TrainingInterrupt - ReduceTransformerVocabPlugin - MetricName - MetricRecord diff --git a/docs/api/flair.trainers.rst b/docs/api/flair.trainers.rst index 7cff5b1b0c..ac59a85039 100644 --- a/docs/api/flair.trainers.rst +++ b/docs/api/flair.trainers.rst @@ -6,9 +6,3 @@ flair.trainers .. autosummary:: :toctree: generated :nosignatures: - - - ModelTrainer - LanguageModelTrainer - TextCorpus - From e141499f23e6cadb2d38d06bafb99e253ce5eac7 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Sat, 28 Sep 2024 20:48:10 +0200 Subject: [PATCH 263/333] use sphinx-autosummary-autocollect version --- docs/conf.py | 3 ++- docs/requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 7bd241ccdd..ffc74c4c4e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,12 +46,12 @@ def linkcode_resolve(*args): # ones. extensions = [ "sphinx.ext.autodoc", - "sphinx.ext.autosummary", "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.ifconfig", "sphinx.ext.napoleon", # to render Google format docstrings "sphinx.ext.githubpages", + "sphinx_autosummary_autocollect", "myst_parser", "sphinx_github_style", "sphinx_autodoc_typehints", @@ -59,6 +59,7 @@ def linkcode_resolve(*args): "sphinx_design", ] autosummary_generate = True +autosummary_ignore_module_all = False # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/requirements.txt b/docs/requirements.txt index 0e8c4f6141..210d0a424a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,6 +6,7 @@ importlib-metadata sphinx-multiversion pydata-sphinx-theme<0.14 sphinx_design +sphinx-autosummary-autocollect # previous dependencies that are required to build docs for later versions too. semver From 76c190b31d5e25fd1552d9d56e43f5caa4f7a92e Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 12:07:18 +0100 Subject: [PATCH 264/333] add test docs upload --- .github/workflows/test-docs.yml | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/test-docs.yml diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml new file mode 100644 index 0000000000..6dcff287ec --- /dev/null +++ b/.github/workflows/test-docs.yml @@ -0,0 +1,43 @@ +name: 'Build doc page' +on: + push: + branches: "*" + +jobs: + publish_docs: + name: Build the docs using Sphinx and push to gh-pages + runs-on: ubuntu-latest + env: + python-version: 3.9 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: setup python ${{ env.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.python-version }} + - name: Install Torch cpu + run: pip install torch --index-url https://download.pytorch.org/whl/cpu + - name: Install Flair dependencies + run: pip install -e . + - name: Install unittest dependencies + run: pip install -r requirements-dev.txt + - name: Install doc dependencies + run: pip install -r docs/requirements.txt + - name: Fetch git tags + run: git fetch --tags origin + - name: Build docs + run: | + sphinx-multiversion docs doc_build/ + - name: Add redirect to stable doc + run: | + cp assets/redirect.html doc_build/index.html + cp assets/redirect.html doc_build/404.html + cp assets/README.md doc_build/README.md + sed -i "s/\[VERSION\]/$(python -c 'import flair;print(flair.__version__)')/g" doc_build/index.html + sed -i "s/\[VERSION\]/$(python -c 'import flair;print(flair.__version__)')/g" doc_build/404.html + - name: Archive code coverage results + uses: actions/upload-artifact@v4 + with: + name: website + path: doc_build \ No newline at end of file From bfb05d96279e4c1dde9b1d4f94b91a360dc199e7 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 12:11:32 +0100 Subject: [PATCH 265/333] fix naming of ci-stage --- .github/workflows/test-docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml index 6dcff287ec..c534e13555 100644 --- a/.github/workflows/test-docs.yml +++ b/.github/workflows/test-docs.yml @@ -1,11 +1,11 @@ -name: 'Build doc page' +name: 'Build doc page for artifact' on: push: branches: "*" jobs: - publish_docs: - name: Build the docs using Sphinx and push to gh-pages + build_docs: + name: Build the docs using Sphinx and publish as artifact runs-on: ubuntu-latest env: python-version: 3.9 From 0aaf0e5292a0be2d3bd1d2cee24aa8e0084c06fe Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 12:50:47 +0100 Subject: [PATCH 266/333] only publish current branch as doc page --- .github/workflows/test-docs.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml index c534e13555..b45d2a10b9 100644 --- a/.github/workflows/test-docs.yml +++ b/.github/workflows/test-docs.yml @@ -26,16 +26,22 @@ jobs: run: pip install -r docs/requirements.txt - name: Fetch git tags run: git fetch --tags origin + - name: Change config to current branch + run: | + sed -i 's/^smv_branch_whitelist = r"^master$"/smv_branch_whitelist = r"^${{ env.BRANCH_NAME }}$"/' docs/conf.py + sed -i 's/^smv_released_pattern = r"^refs\/tags\/v\\d+\\.\\d+\\.\\d+$"/smv_released_pattern = ""/' docs/conf.py - name: Build docs run: | sphinx-multiversion docs doc_build/ - name: Add redirect to stable doc + env: + BRANCH_NAME: ${{ github.ref_name }} run: | cp assets/redirect.html doc_build/index.html cp assets/redirect.html doc_build/404.html cp assets/README.md doc_build/README.md - sed -i "s/\[VERSION\]/$(python -c 'import flair;print(flair.__version__)')/g" doc_build/index.html - sed -i "s/\[VERSION\]/$(python -c 'import flair;print(flair.__version__)')/g" doc_build/404.html + sed -i "s/\[VERSION\]/${{ env.BRANCH_NAME }}/g" doc_build/index.html + sed -i "s/\[VERSION\]/${{ env.BRANCH_NAME }}/g" doc_build/404.html - name: Archive code coverage results uses: actions/upload-artifact@v4 with: From 4f33a5f8cd227340ad738859bdea8ee1c0d384df Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 13:21:57 +0100 Subject: [PATCH 267/333] fix test docs --- .github/workflows/test-docs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml index b45d2a10b9..c8afcae24c 100644 --- a/.github/workflows/test-docs.yml +++ b/.github/workflows/test-docs.yml @@ -29,7 +29,8 @@ jobs: - name: Change config to current branch run: | sed -i 's/^smv_branch_whitelist = r"^master$"/smv_branch_whitelist = r"^${{ env.BRANCH_NAME }}$"/' docs/conf.py - sed -i 's/^smv_released_pattern = r"^refs\/tags\/v\\d+\\.\\d+\\.\\d+$"/smv_released_pattern = ""/' docs/conf.py + sed -i 's/^smv_tag_whitelist = r"^v\d+\.\d+\.\d+$"/smv_tag_whitelist = ""/' docs/conf.py + cat docs/conf.py - name: Build docs run: | sphinx-multiversion docs doc_build/ From 2e89d520ff618f4c039bc5ec1889d9834920c69e Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 13:46:35 +0100 Subject: [PATCH 268/333] fix gha docs test --- .github/workflows/test-docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml index c8afcae24c..adf22bdbc6 100644 --- a/.github/workflows/test-docs.yml +++ b/.github/workflows/test-docs.yml @@ -27,6 +27,8 @@ jobs: - name: Fetch git tags run: git fetch --tags origin - name: Change config to current branch + env: + BRANCH_NAME: ${{ github.ref_name }} run: | sed -i 's/^smv_branch_whitelist = r"^master$"/smv_branch_whitelist = r"^${{ env.BRANCH_NAME }}$"/' docs/conf.py sed -i 's/^smv_tag_whitelist = r"^v\d+\.\d+\.\d+$"/smv_tag_whitelist = ""/' docs/conf.py From 14ab7a25b30b5466216cd22c49080edf024623bc Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 15:13:55 +0100 Subject: [PATCH 269/333] pin sphinx version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 210d0a424a..6c6f0d4786 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ sphinx-github-style<=1.0.2 # 1.0.3 changes logic that breaks with sphinx-multiversion sphinx-autodoc-typehints myst-parser -sphinx +sphinx<7.4.0 importlib-metadata sphinx-multiversion pydata-sphinx-theme<0.14 From c45086a1cff1c1e08f5f8e1f8827f1170bb0ab86 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 16:00:50 +0100 Subject: [PATCH 270/333] pin sphinx version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 6c6f0d4786..b169c8af23 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ sphinx-github-style<=1.0.2 # 1.0.3 changes logic that breaks with sphinx-multiversion sphinx-autodoc-typehints myst-parser -sphinx<7.4.0 +sphinx<7.3.0 importlib-metadata sphinx-multiversion pydata-sphinx-theme<0.14 From f09e64abb5a434a8e7a1e99621aa3895ba5683ab Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 16:04:00 +0100 Subject: [PATCH 271/333] pin sphinx version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b169c8af23..dfc897c61b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ sphinx-github-style<=1.0.2 # 1.0.3 changes logic that breaks with sphinx-multiversion sphinx-autodoc-typehints myst-parser -sphinx<7.3.0 +sphinx<7.2.0 importlib-metadata sphinx-multiversion pydata-sphinx-theme<0.14 From 526f8b21e20eb9a2584979b69e786a25ae103fea Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 16:36:37 +0100 Subject: [PATCH 272/333] pin sphinx version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index dfc897c61b..ff23c9f129 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ sphinx-github-style<=1.0.2 # 1.0.3 changes logic that breaks with sphinx-multiversion sphinx-autodoc-typehints myst-parser -sphinx<7.2.0 +sphinx<8.0.0 importlib-metadata sphinx-multiversion pydata-sphinx-theme<0.14 From f6b011f97e4229259716e562a87f66b7f5a26c95 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 18:22:54 +0100 Subject: [PATCH 273/333] fix gh version resolve --- docs/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index ffc74c4c4e..c1be9c0079 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,5 @@ # noqa: INP001 +import inspect import importlib_metadata @@ -35,8 +36,10 @@ def linkcode_resolve(*args): + app = inspect.currentframe().f_back.f_locals.get("app") + current_version = app.config.smv_current_version # use smv_current_version as the git url - real_linkcode_url = linkcode_url + f"/blob/{smv_current_version}/" + "{filepath}#L{linestart}-L{linestop}" + real_linkcode_url = linkcode_url + f"/blob/{current_version}/" + "{filepath}#L{linestart}-L{linestop}" return get_linkcode_resolve(real_linkcode_url)(*args) From 1689453dbdd8741a8ef854259d58c18bf4bdc442 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 8 Nov 2024 18:37:17 +0100 Subject: [PATCH 274/333] ignore tags ad doc test --- .github/workflows/test-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml index adf22bdbc6..02219db4e1 100644 --- a/.github/workflows/test-docs.yml +++ b/.github/workflows/test-docs.yml @@ -31,7 +31,7 @@ jobs: BRANCH_NAME: ${{ github.ref_name }} run: | sed -i 's/^smv_branch_whitelist = r"^master$"/smv_branch_whitelist = r"^${{ env.BRANCH_NAME }}$"/' docs/conf.py - sed -i 's/^smv_tag_whitelist = r"^v\d+\.\d+\.\d+$"/smv_tag_whitelist = ""/' docs/conf.py + sed -i 's/^smv_tag_whitelist = r"^v\d+\.\d+\.\d+$"/smv_tag_whitelist = "^$"/' docs/conf.py cat docs/conf.py - name: Build docs run: | From 904052d76ff83c13bdf33f325d6612ee3225ed7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Karlo=20Do=C5=A1ilovi=C4=87?= Date: Fri, 29 Nov 2024 14:24:39 +0100 Subject: [PATCH 275/333] Address reviewer's comments. --- tests/embedding_test_utils.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/tests/embedding_test_utils.py b/tests/embedding_test_utils.py index 706a5cb587..c1a0b1a791 100644 --- a/tests/embedding_test_utils.py +++ b/tests/embedding_test_utils.py @@ -19,11 +19,11 @@ class BaseEmbeddingsTest: name_field: Optional[str] = None weired_texts: list[str] = [ "Hybrid mesons , qq ̄ states with an admixture", - "typical proportionalities of \u223c 1nmV \u2212 1 [ 3,4 ] .", + "typical proportionalities of \u223C 1nmV \u2212 1 [ 3,4 ] .", "🤟 🤟 🤟 hüllo", "🤟hallo 🤟 🤟 🤟 🤟", "🤟", - "\uf8f9", + "\uF8F9", ] def create_embedding_from_name(self, name: str): @@ -150,17 +150,9 @@ def test_embeddings_stay_the_same_after_saving_and_loading(self, args): if self.is_token_embedding: for token_old, token_new in zip(sentence_old, sentence_new): - assert torch.allclose( - token_old.get_embedding(names_old), - token_new.get_embedding(names_new), - atol=1e-6, - ) + assert (token_old.get_embedding(names_old) == token_new.get_embedding(names_new)).all() if self.is_document_embedding: - assert torch.allclose( - sentence_old.get_embedding(names_old), - sentence_new.get_embedding(names_new), - atol=1e-6, - ) + assert (sentence_old.get_embedding(names_old) == sentence_new.get_embedding(names_new)).all() def test_default_embeddings_stay_the_same_after_saving_and_loading(self): embeddings = self.create_embedding_with_args(self.default_args) @@ -184,17 +176,9 @@ def test_default_embeddings_stay_the_same_after_saving_and_loading(self): if self.is_token_embedding: for token_old, token_new in zip(sentence_old, sentence_new): - assert torch.allclose( - token_old.get_embedding(names_old), - token_new.get_embedding(names_new), - atol=1e-6, - ) + assert (token_old.get_embedding(names_old) == token_new.get_embedding(names_new)).all() if self.is_document_embedding: - assert torch.allclose( - sentence_old.get_embedding(names_old), - sentence_new.get_embedding(names_new), - atol=1e-6, - ) + assert (sentence_old.get_embedding(names_old) == sentence_new.get_embedding(names_new)).all() def test_embeddings_load_in_eval_mode(self): embeddings = self.create_embedding_with_args(self.default_args) From f301eb01945026839fae651df141d04c8289ddfb Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Mon, 2 Dec 2024 23:21:43 +0100 Subject: [PATCH 276/333] Test docstrings --- docs/conf.py | 26 ++++++++++++-------------- flair/data.py | 11 +++++++++++ flair/nn/model.py | 31 ++++++++++++++++--------------- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c1be9c0079..cca49a3f38 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,7 @@ "navbar_end": ["darkmode-toggle", "version-switcher", "navbar-icon-links"], "show_prev_next": False, "footer_end": ["footer-links/legal-notice.html", "footer-links/x.html", "footer-links/linkedin.html"], - "secondary_sidebar_items": [] + "secondary_sidebar_items": [], } @@ -82,18 +82,18 @@ def linkcode_resolve(*args): # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -html_title = 'Flair Documentation' +html_title = "Flair Documentation" html_css_files = [ - 'css/main.css', - 'css/header.css', - 'css/footer.css', - 'css/version-switcher.css', - 'css/sidebar.css', - 'css/tutorial.css', - 'css/api.css', - 'css/legal-notice.css', - 'css/search.css', + "css/main.css", + "css/header.css", + "css/footer.css", + "css/version-switcher.css", + "css/sidebar.css", + "css/tutorial.css", + "css/api.css", + "css/legal-notice.css", + "css/search.css", ] html_logo = "_static/flair_logo_white.svg" @@ -118,9 +118,7 @@ def linkcode_resolve(*args): } html_sidebars = { - "**": [ - "globaltoc.html" - ], + "**": ["globaltoc.html"], "index": [], } diff --git a/flair/data.py b/flair/data.py index 8a76a82c1a..071c48f705 100644 --- a/flair/data.py +++ b/flair/data.py @@ -347,6 +347,17 @@ def has_metadata(self, key: str) -> bool: return key in self._metadata def add_label(self, typename: str, value: str, score: float = 1.0, **metadata) -> "DataPoint": + """Adds a label to the DataPoint by internally creating a Label object. + + Args: + typename: A string that identifies the layer of annotation, such as "ner" for named entity labels or "sentiment" for sentiment labels + value: A string that sets the value of the label. + score: Optional value setting the confidence level of the label (between 0 and 1). If not set, a default confidence of 1 is used. + **metadata: Additional metadata information. + + Returns: + A pointer to itself (DataPoint object, now with an added label). + """ label = Label(self, value, score, **metadata) if typename not in self.annotation_layers: diff --git a/flair/nn/model.py b/flair/nn/model.py index 8f9d0aa09f..66d7574791 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -69,12 +69,12 @@ def evaluate( Args: data_points: The labeled data_points to evaluate. gold_label_type: The label type indicating the gold labels - out_path: Optional output path to store predictions + out_path: Optional output path to store predictions. embedding_storage_mode: One of 'none', 'cpu' or 'gpu'. 'none' means all embeddings are deleted and freshly recomputed, 'cpu' means all embeddings are stored on CPU, or 'gpu' means all embeddings are stored on GPU - mini_batch_size: The batch_size to use for predictions - main_evaluation_metric: Specify which metric to highlight as main_score - exclude_labels: Specify classes that won't be considered in evaluation + mini_batch_size: The batch_size to use for predictions. + main_evaluation_metric: Specify which metric to highlight as main_score. + exclude_labels: Specify classes that won't be considered in evaluation. gold_label_dictionary: Specify which classes should be considered, all other classes will be taken as . return_loss: Weather to additionally compute the loss on the data-points. **kwargs: Arguments that will be ignored. @@ -116,8 +116,8 @@ def save(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: """Saves the current model to the provided file. Args: - model_file: the model file - checkpoint: currently unused. + model_file: The model file. + checkpoint: This parameter is currently unused. """ model_state = self._get_state_dict() @@ -133,9 +133,10 @@ def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "Model": """Loads the model from the given file. Args: - model_path: the model file or the already loaded state dict + model_path: The model file or the already loaded state dict. - Returns: the loaded text classifier model + Returns: + The loaded model. """ # if this class is abstract, go through all inheriting classes and try to fetch and load the model if inspect.isabstract(cls): @@ -528,13 +529,13 @@ def predict( The labels are directly added to the sentences. Args: - sentences: list of sentences - mini_batch_size: mini batch size to use - return_probabilities_for_all_classes: return probabilities for all classes instead of only best predicted - verbose: set to True to display a progress bar - return_loss: set to True to return loss - label_name: set this to change the name of the label type that is predicted # noqa: E501 - embedding_storage_mode: default is 'none' which is always best. Only set to 'cpu' or 'gpu' if you wish to not only predict, but also keep the generated embeddings in CPU or GPU memory respectively. 'gpu' to store embeddings in GPU memory. # noqa: E501 + sentences: The sentences for which the model should predict labels. + mini_batch_size: The mini batch size to use. + return_probabilities_for_all_classes: If set to True, the model will store probabilities for all classes instead of only the predicted class. + verbose: If set to True, will display a progress bar while predicting. By default, this parameter is set to False. + return_loss: Set this to True to return loss (only possible if gold labels are set for the sentences). + label_name: Optional parameter that if set, changes the identifier of the label type that is predicted. # noqa: E501 + embedding_storage_mode: Default is 'none' which is always best. Only set to 'cpu' or 'gpu' if you wish to not only predict, but also keep the generated embeddings in CPU or GPU memory respectively. 'gpu' to store embeddings in GPU memory. # noqa: E501 """ raise NotImplementedError From b6f9206c30da73b22c41f4a84adff401814382df Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Mon, 2 Dec 2024 23:50:01 +0100 Subject: [PATCH 277/333] Test example --- flair/data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flair/data.py b/flair/data.py index 071c48f705..1569bec2d5 100644 --- a/flair/data.py +++ b/flair/data.py @@ -355,6 +355,12 @@ def add_label(self, typename: str, value: str, score: float = 1.0, **metadata) - score: Optional value setting the confidence level of the label (between 0 and 1). If not set, a default confidence of 1 is used. **metadata: Additional metadata information. + Examples: + Examples should be written in doctest format, and should illustrate how + to use the function. + >>> print([i for i in example_generator(4)]) + [0, 1, 2, 3] + Returns: A pointer to itself (DataPoint object, now with an added label). """ From 314d59d5367ac1b671acdea547af81fe8d680339 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Tue, 3 Dec 2024 00:05:46 +0100 Subject: [PATCH 278/333] Test example --- flair/data.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flair/data.py b/flair/data.py index 1569bec2d5..9fc811189e 100644 --- a/flair/data.py +++ b/flair/data.py @@ -347,7 +347,7 @@ def has_metadata(self, key: str) -> bool: return key in self._metadata def add_label(self, typename: str, value: str, score: float = 1.0, **metadata) -> "DataPoint": - """Adds a label to the DataPoint by internally creating a Label object. + """Adds a label to the DataPoint by internally creating a :class:`Label` object. Args: typename: A string that identifies the layer of annotation, such as "ner" for named entity labels or "sentiment" for sentiment labels @@ -356,10 +356,11 @@ def add_label(self, typename: str, value: str, score: float = 1.0, **metadata) - **metadata: Additional metadata information. Examples: - Examples should be written in doctest format, and should illustrate how - to use the function. - >>> print([i for i in example_generator(4)]) - [0, 1, 2, 3] + >>> add(2, 3) + 5 + + >>> add(-1, 1) + 0 Returns: A pointer to itself (DataPoint object, now with an added label). From f1ab536d7de10bf0f0214caaafac33ddfac2e923 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Tue, 3 Dec 2024 00:30:05 +0100 Subject: [PATCH 279/333] Test example --- flair/data.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/flair/data.py b/flair/data.py index 9fc811189e..3ed9239d72 100644 --- a/flair/data.py +++ b/flair/data.py @@ -347,7 +347,21 @@ def has_metadata(self, key: str) -> bool: return key in self._metadata def add_label(self, typename: str, value: str, score: float = 1.0, **metadata) -> "DataPoint": - """Adds a label to the DataPoint by internally creating a :class:`Label` object. + """Adds a label to the :class:`DataPoint` by internally creating a :class:`Label` object. + + Examples: + >>> sentence = Sentence("this is good") + >>> sentence.add_label("sentiment", "positive") + >>> print(sentence) + Sentence[3]: "this is good" → positive (1.0000) + + >>> # make a sentence and get the last token + >>> sentence = Sentence("this is good") + >>> token = sentence[2] + >>> # add a label to token + >>> token.add_label("part-of-speech", "adjective") + >>> print(token) + Token[2]: "good" → adjective (1.0000) Args: typename: A string that identifies the layer of annotation, such as "ner" for named entity labels or "sentiment" for sentiment labels @@ -355,13 +369,6 @@ def add_label(self, typename: str, value: str, score: float = 1.0, **metadata) - score: Optional value setting the confidence level of the label (between 0 and 1). If not set, a default confidence of 1 is used. **metadata: Additional metadata information. - Examples: - >>> add(2, 3) - 5 - - >>> add(-1, 1) - 0 - Returns: A pointer to itself (DataPoint object, now with an added label). """ From a90a7cb688a7e1b6d7507086bb7bffd5ce2f3455 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 06:38:02 +0100 Subject: [PATCH 280/333] Work through tutorial 1 to improve docstrings and tutorial text --- docs/tutorial/tutorial-basics/basic-types.md | 8 +++- .../tutorial-basics/entity-linking.md | 12 ++++++ .../tutorial-basics/how-predictions-work.md | 5 +++ docs/tutorial/tutorial-basics/other-models.md | 7 ++++ .../tutorial-basics/part-of-speech-tagging.md | 5 +++ .../tutorial-basics/tagging-entities.md | 7 ++++ .../tutorial-basics/tagging-sentiment.md | 4 ++ flair/data.py | 41 +++++++++---------- flair/nn/model.py | 20 +++++---- 9 files changed, 79 insertions(+), 30 deletions(-) diff --git a/docs/tutorial/tutorial-basics/basic-types.md b/docs/tutorial/tutorial-basics/basic-types.md index 703a5d7cd5..5ddf247166 100644 --- a/docs/tutorial/tutorial-basics/basic-types.md +++ b/docs/tutorial/tutorial-basics/basic-types.md @@ -242,7 +242,7 @@ for label in sentence.get_labels('ner'): ### Information for each label -Each label is of class `Label` which next to the value has a score indicating confidence. It also has a pointer back to the data point to which it attaches. +Each label is of class [`Label`](#flair.data.Label) which next to the value has a score indicating confidence. It also has a pointer back to the data point to which it attaches. This means that you can print the value, the confidence and the labeled text of each label: @@ -267,3 +267,9 @@ This should print: Our color tag has a score of 1.0 since we manually added it. If a tag is predicted by our sequence labeler, the score value will indicate classifier confidence. + +### Next + +Congrats, you now understand Flair's basic types. + +Next, learn how to use [Flair models to make predictions](how-predictions-work.md). \ No newline at end of file diff --git a/docs/tutorial/tutorial-basics/entity-linking.md b/docs/tutorial/tutorial-basics/entity-linking.md index 8137c2dc5f..808d5c91ad 100644 --- a/docs/tutorial/tutorial-basics/entity-linking.md +++ b/docs/tutorial/tutorial-basics/entity-linking.md @@ -83,3 +83,15 @@ As we can see, the linker can resolve that: - the first mention of "Barcelona" refers to the soccer club "[FC Barcelona](https://en.wikipedia.org/wiki/FC_Barcelona)" - the second mention of "Barcelona" refers to the city of "[Barcelona](https://en.wikipedia.org/wiki/Barcelona)" + +### Linking biomedical entities + +If you are working with biomedical data, we have a special entity linker capable of linking +biomedical entities to specific knowledge bases. In this case, check out this [advanced tutorial on +linking biomedical entities](entity-mention-linking.md). + +### Next + +Congrats, you learned how to link entities with Flair! + +Next, let's discuss how to [predict part-of-speech tags with Flair](part-of-speech-tagging.md). \ No newline at end of file diff --git a/docs/tutorial/tutorial-basics/how-predictions-work.md b/docs/tutorial/tutorial-basics/how-predictions-work.md index 9911f6efa5..a1dffa8913 100644 --- a/docs/tutorial/tutorial-basics/how-predictions-work.md +++ b/docs/tutorial/tutorial-basics/how-predictions-work.md @@ -76,3 +76,8 @@ the text of label.data_point is: "Washington" ``` +### Next + +Congrats, you've made your first predictions with Flair and accessed value and confidence scores of each prediction. + +Next, let's discuss specifically how to [predict named entities with Flair](tagging-entities.md). diff --git a/docs/tutorial/tutorial-basics/other-models.md b/docs/tutorial/tutorial-basics/other-models.md index dbab4f40d1..9bd02bda7e 100644 --- a/docs/tutorial/tutorial-basics/other-models.md +++ b/docs/tutorial/tutorial-basics/other-models.md @@ -150,3 +150,10 @@ We end this section with a list of all other models we currently ship with Flair | 'de-historic-reported' | historical reported speech | German | @redewiedergabe project | **87.94** (F1) | [redewiedergabe](https://github.com/redewiedergabe/tagger) | | | 'de-historic-free-indirect' | historical free-indirect speech | German | @redewiedergabe project | **87.94** (F1) | [redewiedergabe](https://github.com/redewiedergabe/tagger) | | + +### Next + +Congrats, you learned about some other models we have in Flair! + +So far, we only focused on predicting for single sentences. Next, let's discuss how +to create [predictions for a whole corpus of documents](how-to-tag-corpus.md). \ No newline at end of file diff --git a/docs/tutorial/tutorial-basics/part-of-speech-tagging.md b/docs/tutorial/tutorial-basics/part-of-speech-tagging.md index 3da1774bf1..9a9dc54f55 100644 --- a/docs/tutorial/tutorial-basics/part-of-speech-tagging.md +++ b/docs/tutorial/tutorial-basics/part-of-speech-tagging.md @@ -167,4 +167,9 @@ You choose which pre-trained model you load by passing the appropriate string to A full list of our current and community-contributed models can be browsed on the [__model hub__](https://huggingface.co/models?library=flair&sort=downloads). +### Next + +Congrats, you learned how to predict part-of-speech tags with Flair! + +Next, we'll present some [other models in Flair](other-models.md) you might find useful. diff --git a/docs/tutorial/tutorial-basics/tagging-entities.md b/docs/tutorial/tutorial-basics/tagging-entities.md index a3b41ff80c..39ccdd8e1a 100644 --- a/docs/tutorial/tutorial-basics/tagging-entities.md +++ b/docs/tutorial/tutorial-basics/tagging-entities.md @@ -200,3 +200,10 @@ You choose which pre-trained model you load by passing the appropriate string to A full list of our current and community-contributed models can be browsed on the [__model hub__](https://huggingface.co/models?library=flair&sort=downloads). + +### Next + +Congrats, you learned how to predict entities with Flair and got an overview of different models! + +Next, let's discuss how to [predict sentiment with Flair](tagging-sentiment.md). + diff --git a/docs/tutorial/tutorial-basics/tagging-sentiment.md b/docs/tutorial/tutorial-basics/tagging-sentiment.md index 0c6c9f5789..6bbebfb178 100644 --- a/docs/tutorial/tutorial-basics/tagging-sentiment.md +++ b/docs/tutorial/tutorial-basics/tagging-sentiment.md @@ -75,5 +75,9 @@ We end this section with a list of all models we currently ship with Flair: | 'de-offensive-language' | German | detecting offensive language | [GermEval 2018 Task 1](https://projects.fzai.h-da.de/iggsa/projekt/) | **75.71** (Macro F1) | +### Next +Congrats, you learned how to predict sentiment with Flair! + +Next, let's discuss how to [link entities to Wikipedia with Flair](entity-linking.md). diff --git a/flair/data.py b/flair/data.py index 3ed9239d72..ed22c0aa65 100644 --- a/flair/data.py +++ b/flair/data.py @@ -349,20 +349,6 @@ def has_metadata(self, key: str) -> bool: def add_label(self, typename: str, value: str, score: float = 1.0, **metadata) -> "DataPoint": """Adds a label to the :class:`DataPoint` by internally creating a :class:`Label` object. - Examples: - >>> sentence = Sentence("this is good") - >>> sentence.add_label("sentiment", "positive") - >>> print(sentence) - Sentence[3]: "this is good" → positive (1.0000) - - >>> # make a sentence and get the last token - >>> sentence = Sentence("this is good") - >>> token = sentence[2] - >>> # add a label to token - >>> token.add_label("part-of-speech", "adjective") - >>> print(token) - Token[2]: "good" → adjective (1.0000) - Args: typename: A string that identifies the layer of annotation, such as "ner" for named entity labels or "sentiment" for sentiment labels value: A string that sets the value of the label. @@ -395,6 +381,17 @@ def get_label(self, label_type: Optional[str] = None, zero_tag_value: str = "O") return self.get_labels(label_type)[0] def get_labels(self, typename: Optional[str] = None) -> list[Label]: + """Returns all labels of this datapoint belonging to a specific annotation layer. + + For instance, if a data point has been labeled with `"sentiment"`-labels, you can call this function as + `get_labels("sentiment")` to return a list of all sentiment labels. + + Args: + typename: The string identifier of the annotation layer, like "sentiment" or "ner". + + Returns: + A list of :class:`Label` objects belonging to this annotation layer for this data point. + """ if typename is None: return self.labels @@ -791,7 +788,11 @@ def to_dict(self, tag_type: Optional[str] = None): class Sentence(DataPoint): - """A Sentence is a list of tokens and is used to represent a sentence or text fragment.""" + """A Sentence is a central object in Flair that represents either a single sentence or a whole text. + + Internally, it consists of a list of Token objects that represent each word in the text. Additionally, + this object stores all metadata related to a text such as labels, language code, etc. + """ def __init__( self, @@ -800,14 +801,12 @@ def __init__( language_code: Optional[str] = None, start_position: int = 0, ) -> None: - """Class to hold all metadata related to a text. - - Metadata can be tokens, labels, predictions, language code, etc. + """Create a sentence object by passing either a text or a list of tokens. Args: - text: original string (sentence), or a pre tokenized list of tokens. - use_tokenizer: Specify a custom tokenizer to split the text into tokens. The Default is - :class:`flair.tokenization.SegTokTokenizer`. If `use_tokenizer` is set to False, + text: Either pass the text as a string, or provide an already tokenized text as either a list of strings or a list of :class:`Token` objects. + use_tokenizer: You can optionally specify a custom tokenizer to split the text into tokens. By default we use + :class:`flair.tokenization.SegtokTokenizer`. If `use_tokenizer` is set to False, :class:`flair.tokenization.SpaceTokenizer` will be used instead. The tokenizer will be ignored, if `text` refers to pretokenized tokens. language_code: Language of the sentence. If not provided, `langdetect `_ diff --git a/flair/nn/model.py b/flair/nn/model.py index 66d7574791..5cd6ab1fd7 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -130,13 +130,13 @@ def save(self, model_file: Union[str, Path], checkpoint: bool = False) -> None: @classmethod def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "Model": - """Loads the model from the given file. + """Loads a Flair model from the given file or state dictionary. Args: - model_path: The model file or the already loaded state dict. + model_path: Either the path to the model (as string or `Path` variable) or the already loaded state dict. Returns: - The loaded model. + The loaded Flair model. """ # if this class is abstract, go through all inheriting classes and try to fetch and load the model if inspect.isabstract(cls): @@ -524,14 +524,18 @@ def predict( return_loss: bool = False, embedding_storage_mode: EmbeddingStorageMode = "none", ): - """Predicts the class labels for the given sentences. + """Uses the model to predict labels for a given set of data points. - The labels are directly added to the sentences. + The method does not directly return the predicted labels. Rather, labels are added as :class:`flair.data.Label` objects to + the respective data points. You can then access these predictions by calling :func:`flair.data.DataPoint.get_labels` + on each data point that you passed through this method. Args: - sentences: The sentences for which the model should predict labels. - mini_batch_size: The mini batch size to use. - return_probabilities_for_all_classes: If set to True, the model will store probabilities for all classes instead of only the predicted class. + sentences: The data points for which the model should predict labels, most commonly Sentence objects. + mini_batch_size: The mini batch size to use. Setting this value higher typically makes predictions faster, + but also costs more memory. + return_probabilities_for_all_classes: If set to True, the model will store probabilities for all classes + instead of only the predicted class. verbose: If set to True, will display a progress bar while predicting. By default, this parameter is set to False. return_loss: Set this to True to return loss (only possible if gold labels are set for the sentences). label_name: Optional parameter that if set, changes the identifier of the label type that is predicted. # noqa: E501 From 7cd659fe91b379a06b156704415b5d2f636f24cd Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 09:49:57 +0100 Subject: [PATCH 281/333] Test modification in Classifier.predict docstrings --- flair/nn/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flair/nn/model.py b/flair/nn/model.py index 5cd6ab1fd7..90144d8d97 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -526,8 +526,8 @@ def predict( ): """Uses the model to predict labels for a given set of data points. - The method does not directly return the predicted labels. Rather, labels are added as :class:`flair.data.Label` objects to - the respective data points. You can then access these predictions by calling :func:`flair.data.DataPoint.get_labels` + The method does not directly return the predicted labels. Rather, labels are added as :class:`Label` objects to + the respective data points. You can then access these predictions by calling :func:`DataPoint.get_labels` on each data point that you passed through this method. Args: From fd49827af08411f0a69c6e67817955d1098e571c Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 09:59:12 +0100 Subject: [PATCH 282/333] Add docstrings to sentence splitters --- .../tutorial-basics/entity-mention-linking.md | 12 ++++++--- flair/splitter.py | 26 ++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/tutorial/tutorial-basics/entity-mention-linking.md b/docs/tutorial/tutorial-basics/entity-mention-linking.md index e9d442c9e1..56803f1b02 100644 --- a/docs/tutorial/tutorial-basics/entity-mention-linking.md +++ b/docs/tutorial/tutorial-basics/entity-mention-linking.md @@ -1,6 +1,6 @@ # Using and creating entity mention linker -As of Flair 0.14 we ship the [entity mention linker](#flair.models.EntityMentionLinker) - the core framework behind the [Hunflair BioNEN approach](https://huggingface.co/hunflair)]. +As of Flair 0.14 we ship the [entity mention linker](#flair.models.EntityMentionLinker) - the core framework behind the [Hunflair BioNEN approach](https://huggingface.co/hunflair). You can read more at the [Hunflair2 tutorials](project:../tutorial-hunflair2/overview.md) ## Example 1: Printing Entity linking outputs to console @@ -124,5 +124,11 @@ print(result_mentions) ```{note} If you need more than the extracted ids, you can use `nen_tagger.dictionary[span_data["nen_id"]]` - to look up the [`flair.data.EntityCandidate`](#flair.data.EntityCandidate) which contains further information. -``` \ No newline at end of file + to look up the [`EntityCandidate`](#flair.data.EntityCandidate) which contains further information. +``` + +### Next + +Congrats, you learned how to link biomedical entities with Flair! + +Next, let's discuss how to [predict part-of-speech tags with Flair](part-of-speech-tagging.md). \ No newline at end of file diff --git a/flair/splitter.py b/flair/splitter.py index 2b6c90cd7f..b9e70aa1cb 100644 --- a/flair/splitter.py +++ b/flair/splitter.py @@ -16,16 +16,34 @@ class SentenceSplitter(ABC): r"""An abstract class representing a :class:`SentenceSplitter`. Sentence splitters are used to represent algorithms and models to split plain text into - sentences and individual tokens / words. All subclasses should overwrite :meth:`splits`, - which splits the given plain text into a sequence of sentences (:class:`Sentence`). The - individual sentences are in turn subdivided into tokens / words. In most cases, this can - be controlled by passing custom implementation of :class:`Tokenizer`. + sentences and individual tokens / words. All subclasses should overwrite :func:`split`, + which splits the given plain text into a list of :class:`flair.data.Sentence` objects. The + individual sentences are in turn subdivided into tokens. In most cases, this can + be controlled by passing custom implementation of :class:`flair.tokenization.Tokenizer`. Moreover, subclasses may overwrite :meth:`name`, returning a unique identifier representing the sentence splitter's configuration. + + The most common class in Flair that implements this base class is :class:`SegtokSentenceSplitter`. """ def split(self, text: str, link_sentences: Optional[bool] = True) -> list[Sentence]: + """ + Takes as input a text as a plain string and outputs a list of :class:`flair.data.Sentence` objects. + + If link_sentences is set (by default, it is). The :class:`flair.data.Sentence` objects will include pointers + to the preceding and following sentences in the original text. This way, the original sequence information will + always be preserved. + + Args: + text: The plain text to split. + link_sentences: If set to True, :class:`flair.data.Sentence` objects will include pointers + to the preceding and following sentences in the original text. + + Returns: + A list of :class:`flair.data.Sentence` objects that each represent one sentence in the given text. + + """ sentences = self._perform_split(text) if not link_sentences: return sentences From 820bc2e3f8aab87ce03f2aad189c327f38b5df16 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 10:50:28 +0100 Subject: [PATCH 283/333] Test mod directives to shorten class references --- docs/conf.py | 4 ++++ flair/nn/model.py | 4 ++-- flair/splitter.py | 16 ++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index cca49a3f38..cd5d885611 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -143,3 +143,7 @@ def linkcode_resolve(*args): smv_prefer_remote_refs = False html_favicon = "_static/favicon.ico" + +# Define a mapping for shorter module names in the documentation +# (for example, `car` instead of `longpath.to.module.car`) +modindex_common_prefix = ["flair.", "flair.data"] diff --git a/flair/nn/model.py b/flair/nn/model.py index 90144d8d97..5cd6ab1fd7 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -526,8 +526,8 @@ def predict( ): """Uses the model to predict labels for a given set of data points. - The method does not directly return the predicted labels. Rather, labels are added as :class:`Label` objects to - the respective data points. You can then access these predictions by calling :func:`DataPoint.get_labels` + The method does not directly return the predicted labels. Rather, labels are added as :class:`flair.data.Label` objects to + the respective data points. You can then access these predictions by calling :func:`flair.data.DataPoint.get_labels` on each data point that you passed through this method. Args: diff --git a/flair/splitter.py b/flair/splitter.py index b9e70aa1cb..2a46b66277 100644 --- a/flair/splitter.py +++ b/flair/splitter.py @@ -17,7 +17,7 @@ class SentenceSplitter(ABC): Sentence splitters are used to represent algorithms and models to split plain text into sentences and individual tokens / words. All subclasses should overwrite :func:`split`, - which splits the given plain text into a list of :class:`flair.data.Sentence` objects. The + which splits the given plain text into a list of :class:`Sentence` objects. The individual sentences are in turn subdivided into tokens. In most cases, this can be controlled by passing custom implementation of :class:`flair.tokenization.Tokenizer`. @@ -27,21 +27,25 @@ class SentenceSplitter(ABC): The most common class in Flair that implements this base class is :class:`SegtokSentenceSplitter`. """ - def split(self, text: str, link_sentences: Optional[bool] = True) -> list[Sentence]: + def split(self, text: str, link_sentences: bool = True) -> list[Sentence]: """ - Takes as input a text as a plain string and outputs a list of :class:`flair.data.Sentence` objects. + Takes as input a text as a plain string and outputs a list of :class:`Sentence` objects. If link_sentences is set (by default, it is). The :class:`flair.data.Sentence` objects will include pointers to the preceding and following sentences in the original text. This way, the original sequence information will always be preserved. Args: - text: The plain text to split. - link_sentences: If set to True, :class:`flair.data.Sentence` objects will include pointers + text (str): The plain text to split. + link_sentences (bool): If set to True, :class:`Sentence` objects will include pointers to the preceding and following sentences in the original text. + Attributes: + name (str): A string identifier of the sentence splitter. + tokenizer: The :class:`tokenization.Tokenizer` class used to tokenize sentences after they are split. + Returns: - A list of :class:`flair.data.Sentence` objects that each represent one sentence in the given text. + A list of :class:`data.Sentence` objects that each represent one sentence in the given text. """ sentences = self._perform_split(text) From 8a397ad894b9d6e0df717ae80958291494e05f4e Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 11:17:58 +0100 Subject: [PATCH 284/333] More tests on shortening module references --- docs/conf.py | 4 +++- flair/splitter.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index cd5d885611..e48af89916 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,6 +112,8 @@ def linkcode_resolve(*args): "inherited": True, } +autodoc_add_module_names = False + source_suffix = { ".rst": "restructuredtext", ".md": "markdown", @@ -146,4 +148,4 @@ def linkcode_resolve(*args): # Define a mapping for shorter module names in the documentation # (for example, `car` instead of `longpath.to.module.car`) -modindex_common_prefix = ["flair.", "flair.data"] +modindex_common_prefix = ["flair.", "flair.data."] diff --git a/flair/splitter.py b/flair/splitter.py index 2a46b66277..3a033f7e4b 100644 --- a/flair/splitter.py +++ b/flair/splitter.py @@ -17,10 +17,17 @@ class SentenceSplitter(ABC): Sentence splitters are used to represent algorithms and models to split plain text into sentences and individual tokens / words. All subclasses should overwrite :func:`split`, - which splits the given plain text into a list of :class:`Sentence` objects. The + which splits the given plain text into a list of :class:`~Sentence` objects. The individual sentences are in turn subdivided into tokens. In most cases, this can be controlled by passing custom implementation of :class:`flair.tokenization.Tokenizer`. + Test: + - :class:`flair.data.Sentence` + - :class:`data.Sentence` + - :class:`Sentence` + - :class:`~Sentence` + - :class:`~data.Sentence` + Moreover, subclasses may overwrite :meth:`name`, returning a unique identifier representing the sentence splitter's configuration. From fd9ef768e163edad4e0a121d5a725b3c5651602c Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 11:43:22 +0100 Subject: [PATCH 285/333] Undo module paths and try out attributes --- docs/conf.py | 6 ------ flair/splitter.py | 23 ++++++++--------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e48af89916..cca49a3f38 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,8 +112,6 @@ def linkcode_resolve(*args): "inherited": True, } -autodoc_add_module_names = False - source_suffix = { ".rst": "restructuredtext", ".md": "markdown", @@ -145,7 +143,3 @@ def linkcode_resolve(*args): smv_prefer_remote_refs = False html_favicon = "_static/favicon.ico" - -# Define a mapping for shorter module names in the documentation -# (for example, `car` instead of `longpath.to.module.car`) -modindex_common_prefix = ["flair.", "flair.data."] diff --git a/flair/splitter.py b/flair/splitter.py index 3a033f7e4b..9cc7a2e912 100644 --- a/flair/splitter.py +++ b/flair/splitter.py @@ -17,26 +17,23 @@ class SentenceSplitter(ABC): Sentence splitters are used to represent algorithms and models to split plain text into sentences and individual tokens / words. All subclasses should overwrite :func:`split`, - which splits the given plain text into a list of :class:`~Sentence` objects. The + which splits the given plain text into a list of :class:`flair.data.Sentence` objects. The individual sentences are in turn subdivided into tokens. In most cases, this can be controlled by passing custom implementation of :class:`flair.tokenization.Tokenizer`. - Test: - - :class:`flair.data.Sentence` - - :class:`data.Sentence` - - :class:`Sentence` - - :class:`~Sentence` - - :class:`~data.Sentence` - Moreover, subclasses may overwrite :meth:`name`, returning a unique identifier representing the sentence splitter's configuration. The most common class in Flair that implements this base class is :class:`SegtokSentenceSplitter`. + + Attributes: + name (str): A string identifier of the sentence splitter. + tokenizer: The :class:`flair.tokenization.Tokenizer` class used to tokenize sentences after they are split. """ def split(self, text: str, link_sentences: bool = True) -> list[Sentence]: """ - Takes as input a text as a plain string and outputs a list of :class:`Sentence` objects. + Takes as input a text as a plain string and outputs a list of :class:`flair.data.Sentence` objects. If link_sentences is set (by default, it is). The :class:`flair.data.Sentence` objects will include pointers to the preceding and following sentences in the original text. This way, the original sequence information will @@ -44,15 +41,11 @@ def split(self, text: str, link_sentences: bool = True) -> list[Sentence]: Args: text (str): The plain text to split. - link_sentences (bool): If set to True, :class:`Sentence` objects will include pointers + link_sentences (bool): If set to True, :class:`flair.data.Sentence` objects will include pointers to the preceding and following sentences in the original text. - Attributes: - name (str): A string identifier of the sentence splitter. - tokenizer: The :class:`tokenization.Tokenizer` class used to tokenize sentences after they are split. - Returns: - A list of :class:`data.Sentence` objects that each represent one sentence in the given text. + A list of :class:`flair.data.Sentence` objects that each represent one sentence in the given text. """ sentences = self._perform_split(text) From 057ea2e37a4dce9ba279a78183f22f125f0d84b6 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 12:47:52 +0100 Subject: [PATCH 286/333] Handle attributes as normal class comments --- flair/splitter.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flair/splitter.py b/flair/splitter.py index 9cc7a2e912..6246969f28 100644 --- a/flair/splitter.py +++ b/flair/splitter.py @@ -25,10 +25,6 @@ class SentenceSplitter(ABC): the sentence splitter's configuration. The most common class in Flair that implements this base class is :class:`SegtokSentenceSplitter`. - - Attributes: - name (str): A string identifier of the sentence splitter. - tokenizer: The :class:`flair.tokenization.Tokenizer` class used to tokenize sentences after they are split. """ def split(self, text: str, link_sentences: bool = True) -> list[Sentence]: @@ -61,10 +57,12 @@ def _perform_split(self, text: str) -> list[Sentence]: @property def name(self) -> str: + """A string identifier of the sentence splitter.""" return self.__class__.__name__ @property def tokenizer(self) -> Tokenizer: + """The :class:`flair.tokenization.Tokenizer` class used to tokenize sentences after they are split.""" raise NotImplementedError @tokenizer.setter From dd3416143c4392388f2f6dda75cb2e1b15674dc1 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 12:56:37 +0100 Subject: [PATCH 287/333] More docstrings --- docs/tutorial/tutorial-basics/how-to-tag-corpus.md | 7 +++++++ flair/nn/model.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/tutorial/tutorial-basics/how-to-tag-corpus.md b/docs/tutorial/tutorial-basics/how-to-tag-corpus.md index 8aa75a4027..1dee865e34 100644 --- a/docs/tutorial/tutorial-basics/how-to-tag-corpus.md +++ b/docs/tutorial/tutorial-basics/how-to-tag-corpus.md @@ -30,3 +30,10 @@ for sentence in sentences: Using the `mini_batch_size` parameter of the [`Classifier.predict()`](#flair.nn.Classifier.predict) method, you can set the size of mini batches passed to the tagger. Depending on your resources, you might want to play around with this parameter to optimize speed. +### Next + +That's it - you completed tutorial 1! Congrats! + +You've learned how basic classes work and how to use Flair to make various predictions. + +Next, you can check out our tutorial on how to [train your own model](../tutorial-training/how-model-training-works.md). diff --git a/flair/nn/model.py b/flair/nn/model.py index 5cd6ab1fd7..03834afc76 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -208,6 +208,14 @@ def load(cls, model_path: Union[str, Path, dict[str, Any]]) -> "Model": return model def print_model_card(self): + """ + This method produces a log message that includes all recorded parameters the model was trained with. + + The model card includes information such as the Flair, PyTorch and Transformers versions used during training, + and the training parameters. + + Only available for models trained with with Flair >= 0.9.1. + """ if hasattr(self, "model_card"): param_out = "\n------------------------------------\n" param_out += "--------- Flair Model Card ---------\n" From e9e9679f2eac355f8aaf3908bf2789afa5abd3c0 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 13:02:38 +0100 Subject: [PATCH 288/333] Add docstring to downsample operation --- flair/data.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/flair/data.py b/flair/data.py index ed22c0aa65..a8d221697e 100644 --- a/flair/data.py +++ b/flair/data.py @@ -1434,7 +1434,23 @@ def downsample( downsample_test: bool = True, random_seed: Optional[int] = None, ) -> "Corpus": - """Reduce all datasets in corpus proportionally to the given percentage.""" + """Randomly downsample the corpus to the given percentage (by removing data points). + + This method is an in-place operation, meaning that the Corpus object itself is modified by removing + data points. It additionally returns a pointer to itself for use in method chaining. + + Args: + percentage (float): A float value between 0. and 1. that indicates to which percentage the corpus + should be downsampled. Default value is 0.1, meaning it gets downsampled to 10%. + downsample_train (bool): Whether or not to include the training split in downsampling. Default is True. + downsample_dev (bool): Whether or not to include the dev split in downsampling. Default is True. + downsample_test (bool): Whether or not to include the test split in downsampling. Default is True. + random_seed (int): An optional random seed to make downsampling reproducible. + + Returns: + A pointer to itself for optional use in method chaining. + """ + if downsample_train and self._train is not None: self._train = self._downsample_to_proportion(self._train, percentage, random_seed) From da27c961df7daf02b632a41b0df11331c39a28d5 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 14:22:12 +0100 Subject: [PATCH 289/333] Fix return statements --- flair/data.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/flair/data.py b/flair/data.py index a8d221697e..f5d902f0e6 100644 --- a/flair/data.py +++ b/flair/data.py @@ -74,7 +74,8 @@ def add_item(self, item: str) -> int: Args: item: a string for which to assign an id. - Returns: ID of string + Returns: + ID of string """ bytes_item = item.encode("utf-8") if bytes_item not in self.item2idx: @@ -88,7 +89,8 @@ def get_idx_for_item(self, item: str) -> int: Args: item: string for which ID is requested - Returns: ID of string, otherwise 0 + Returns: + ID of string, otherwise 0 """ item_encoded = item.encode("utf-8") if item_encoded in self.item2idx: @@ -108,7 +110,8 @@ def get_idx_for_items(self, items: list[str]) -> list[int]: Args: items: List of string for which IDs are requested - Returns: List of ID of strings + Returns: + List of ID of strings """ if not hasattr(self, "item2idx_not_encoded"): d = {key.decode("UTF-8"): value for key, value in self.item2idx.items()} @@ -1463,6 +1466,10 @@ def downsample( return self def filter_empty_sentences(self): + """A method that filters all sentences consisting of 0 tokens. + + This is an in-place operation that directly modifies the Corpus object itself by removing these sentences. + """ log.info("Filtering empty sentences") if self._train is not None: self._train = Corpus._filter_empty_sentences(self._train) @@ -1473,6 +1480,15 @@ def filter_empty_sentences(self): log.info(self) def filter_long_sentences(self, max_charlength: int): + """ + A method that filters all sentences for which the plain text is longer than a specified number of characters. + + This is an in-place operation that directly modifies the Corpus object itself by removing these sentences. + + Args: + max_charlength: The maximum permissible character length of a sentence. + + """ log.info("Filtering long sentences") if self._train is not None: self._train = Corpus._filter_long_sentences(self._train, max_charlength) @@ -1517,7 +1533,7 @@ def _filter_empty_sentences(dataset) -> Dataset: return subset def make_vocab_dictionary(self, max_tokens: int = -1, min_freq: int = 1) -> Dictionary: - """Creates a dictionary of all tokens contained in the corpus. + """Creates a :class:`Dictionary` of all tokens contained in the corpus. By defining `max_tokens` you can set the maximum number of tokens that should be contained in the dictionary. If there are more than `max_tokens` tokens in the corpus, the most frequent tokens are added first. @@ -1525,10 +1541,13 @@ def make_vocab_dictionary(self, max_tokens: int = -1, min_freq: int = 1) -> Dict to be added to the dictionary. Args: - max_tokens: the maximum number of tokens that should be added to the dictionary (-1 = take all tokens) - min_freq: a token needs to occur at least `min_freq` times to be added to the dictionary (-1 = there is no limitation) + max_tokens: The maximum number of tokens that should be added to the dictionary (providing a value of "-1" + means that there is no maximum in this regard). + min_freq: A token needs to occur at least `min_freq` times to be added to the dictionary (providing a value + of "-1" means that there is no limitation in this regard). - Returns: dictionary of tokens + Returns: + A :class:`Dictionary` of all unique tokens in the corpus. """ tokens = self._get_most_common_tokens(max_tokens, min_freq) @@ -1837,7 +1856,8 @@ def make_tag_dictionary(self, tag_type: str) -> Dictionary: Args: tag_type: the label type to gather the tag labels - Returns: A Dictionary containing the labeled tags, including "O" and "" and "" + Returns: + A Dictionary containing the labeled tags, including "O" and "" and "" """ tag_dictionary: Dictionary = Dictionary(add_unk=False) From 39d495254327b8f6c2538d590fe08404600af820 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 23:03:06 +0100 Subject: [PATCH 290/333] Try excluding private methods from docs --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index cca49a3f38..22ffbd0194 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -101,7 +101,7 @@ def linkcode_resolve(*args): # Napoleon settings napoleon_include_init_with_doc = True -napoleon_include_private_with_doc = True +napoleon_include_private_with_doc = False autodoc_default_options = { "member-order": "bysource", From 5fa90359f886a42c931384616f3898d29017733d Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 23:03:23 +0100 Subject: [PATCH 291/333] Add docstrings for ModelTrainer --- flair/trainers/plugins/base.py | 5 +++++ flair/trainers/trainer.py | 27 +++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/flair/trainers/plugins/base.py b/flair/trainers/plugins/base.py index dcf7240a83..33e787a063 100644 --- a/flair/trainers/plugins/base.py +++ b/flair/trainers/plugins/base.py @@ -59,6 +59,11 @@ def __init__(self, *, plugins: Sequence[PluginArgument] = []) -> None: @property def plugins(self): + """Returns all plugins attached to this instance as a list of :class:`BasePlugin`. + + Returns: + List of :class:`BasePlugin` instances attached to this `Pluggable`. + """ return self._plugins def append_plugin(self, plugin): diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index cf3bb80eb6..a5bf9dfd38 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -42,6 +42,22 @@ class ModelTrainer(Pluggable): + """Use this class to train a Flair model. + + The ModelTrainer is initialized using a :class:`flair.nn.Model` (the architecture you want to train) and a + :class:`flair.data.Corpus` (the labeled data you use to train and evaluate the model). It offers two main training + functions for the two main modes of training a model: (1) :func:`train`, which is used to train a model from scratch or + to fit a classification head on a frozen transformer language model. (2) :func:`fine_tune`, which is used if you + do not freeze the transformer language model and rather fine-tune it for a specific task. + + Additionally, there is also a `train_custom` method that allows you to fully customize the training run. + + ModelTrainer inherits from :class:`flair.trainers.plugins.base.Pluggable` and thus uses a plugin system to inject + specific functionality into the training process. You can add any number of plugins to the above-mentioned training + modes. For instance, if you want to use an annealing scheduler during training, you can add the + :class:`flair.trainers.plugins.functional.AnnealingPlugin` plugin to the train command. + """ + valid_events = { "after_setup", "before_training_epoch", @@ -59,11 +75,14 @@ class ModelTrainer(Pluggable): } def __init__(self, model: flair.nn.Model, corpus: Corpus) -> None: - """Initialize a model trainer. + """Initialize a model trainer by passing a :class:`flair.nn.Model` (the architecture you want to train) and a + :class:`flair.data.Corpus` (the labeled data you use to train and evaluate the model). Args: - model: The model that you want to train. The model should inherit from flair.nn.Model # noqa: E501 - corpus: The dataset used to train the model, should be of type Corpus + model: The model that you want to train. The model should inherit from :class:`flair.nn.Model`. So for + instance you should pass a :class:`flair.models.TextClassifier` if you want to train a text classifier, + or :class:`flair.models.SequenceLabeler` if you want to train an RNN-based sequence labeler. + corpus: The dataset (of type :class:`flair.data.Corpus`) used to train the model. """ super().__init__() self.model: flair.nn.Model = model @@ -346,7 +365,7 @@ def train_custom( plugins: Optional[list[TrainerPlugin]] = None, **kwargs, ) -> dict: - """Trains any class that implements the flair.nn.Model interface. + """Trains any class that implements the :class:`flair.nn.Model` interface. Args: base_path: Main path to which all output during training is logged and models are saved From 9fdc2df115c9dbf60aa5b2fc7649098bd8c5fb31 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 23:25:26 +0100 Subject: [PATCH 292/333] Add docstrings for ModelTrainer --- flair/trainers/trainer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index a5bf9dfd38..4e3c9ea0f5 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -55,7 +55,11 @@ class ModelTrainer(Pluggable): ModelTrainer inherits from :class:`flair.trainers.plugins.base.Pluggable` and thus uses a plugin system to inject specific functionality into the training process. You can add any number of plugins to the above-mentioned training modes. For instance, if you want to use an annealing scheduler during training, you can add the - :class:`flair.trainers.plugins.functional.AnnealingPlugin` plugin to the train command. + :class:`flair.trainers.plugins.functional.AnnealingPlugin` plugin to the train command. + + from flair import Sentence + sentence = Sentence("I love Berlin") + """ valid_events = { @@ -76,12 +80,12 @@ class ModelTrainer(Pluggable): def __init__(self, model: flair.nn.Model, corpus: Corpus) -> None: """Initialize a model trainer by passing a :class:`flair.nn.Model` (the architecture you want to train) and a - :class:`flair.data.Corpus` (the labeled data you use to train and evaluate the model). + :class:`flair.data.Corpus` (the labeled data you use to train and evaluate the model). Args: model: The model that you want to train. The model should inherit from :class:`flair.nn.Model`. So for instance you should pass a :class:`flair.models.TextClassifier` if you want to train a text classifier, - or :class:`flair.models.SequenceLabeler` if you want to train an RNN-based sequence labeler. + or :class:`flair.models.SequenceTagger` if you want to train an RNN-based sequence labeler. corpus: The dataset (of type :class:`flair.data.Corpus`) used to train the model. """ super().__init__() From d4a3b57fd1fba5a1cfe4a3e428e81b9a398b1d2d Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Wed, 4 Dec 2024 23:44:01 +0100 Subject: [PATCH 293/333] Remove code comment --- flair/trainers/trainer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 4e3c9ea0f5..03879a2b13 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -56,10 +56,6 @@ class ModelTrainer(Pluggable): specific functionality into the training process. You can add any number of plugins to the above-mentioned training modes. For instance, if you want to use an annealing scheduler during training, you can add the :class:`flair.trainers.plugins.functional.AnnealingPlugin` plugin to the train command. - - from flair import Sentence - sentence = Sentence("I love Berlin") - """ valid_events = { From 5cea71f080ba148a183f1cafbaf0b4dce52347e7 Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 6 Dec 2024 20:55:35 +0100 Subject: [PATCH 294/333] fix transformers special tokens --- flair/embeddings/transformer.py | 13 +++++++------ flair/models/tars_model.py | 10 ++++++---- requirements.txt | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py index a27b57ab96..fdb16eea28 100644 --- a/flair/embeddings/transformer.py +++ b/flair/embeddings/transformer.py @@ -278,11 +278,11 @@ def _reconstruct_word_ids_from_subtokens(embedding, tokens: list[str], subtokens special_tokens = [] # check if special tokens exist to circumvent error message - if embedding.tokenizer._bos_token: + if embedding.tokenizer.bos_token is not None: special_tokens.append(embedding.tokenizer.bos_token) - if embedding.tokenizer._cls_token: + if embedding.tokenizer.cls_token is not None: special_tokens.append(embedding.tokenizer.cls_token) - if embedding.tokenizer._sep_token: + if embedding.tokenizer.sep_token is not None: special_tokens.append(embedding.tokenizer.sep_token) # iterate over subtokens and reconstruct tokens @@ -1354,9 +1354,10 @@ def from_params(cls, params): def to_params(self): config_dict = self.model.config.to_dict() - # do not switch the attention implementation upon reload. - config_dict["attn_implementation"] = self.model.config._attn_implementation - config_dict.pop("_attn_implementation_autoset", None) + if hasattr(self.model.config, "_attn_implementation"): + # do not switch the attention implementation upon reload. + config_dict["attn_implementation"] = self.model.config._attn_implementation + config_dict.pop("_attn_implementation_autoset", None) super_params = super().to_params() diff --git a/flair/models/tars_model.py b/flair/models/tars_model.py index 4f5cb85731..f4171fdb27 100644 --- a/flair/models/tars_model.py +++ b/flair/models/tars_model.py @@ -383,7 +383,7 @@ def __init__( # transformer separator self.separator = str(self.tars_embeddings.tokenizer.sep_token) - if self.tars_embeddings.tokenizer._bos_token: + if self.tars_embeddings.tokenizer.bos_token is not None: self.separator += str(self.tars_embeddings.tokenizer.bos_token) self.prefix = prefix @@ -718,9 +718,11 @@ def __init__( ) # transformer separator - self.separator = str(self.tars_embeddings.tokenizer.sep_token) - if self.tars_embeddings.tokenizer._bos_token: - self.separator += str(self.tars_embeddings.tokenizer.bos_token) + self.separator = ( + self.tars_embeddings.tokenizer.sep_token if self.tars_embeddings.tokenizer.sep_token is not None else "" + ) + if self.tars_embeddings.tokenizer.bos_token is not None: + self.separator += self.tars_embeddings.tokenizer.bos_token self.prefix = prefix self.num_negative_labels_to_sample = num_negative_labels_to_sample diff --git a/requirements.txt b/requirements.txt index bb5ecafd45..2704114ace 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,6 @@ tabulate>=0.8.10 torch>=1.5.0,!=1.8 tqdm>=4.63.0 transformer-smaller-training-vocab>=0.2.3 -transformers[sentencepiece]>=4.18.0,<5.0.0 +transformers[sentencepiece]>=4.25.0,<5.0.0 wikipedia-api>=0.5.7 bioc<3.0.0,>=2.0.0 From c0bf2af03869f4e08599e5f584340e71caa36cfd Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Tue, 23 Jan 2024 13:23:13 +0100 Subject: [PATCH 295/333] add cleanconll dataset --- flair/datasets/__init__.py | 2 + flair/datasets/sequence_labeling.py | 64 +++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/flair/datasets/__init__.py b/flair/datasets/__init__.py index 2837e017c0..76c2c08388 100644 --- a/flair/datasets/__init__.py +++ b/flair/datasets/__init__.py @@ -175,6 +175,7 @@ CONLL_03_DUTCH, CONLL_03_GERMAN, CONLL_03_SPANISH, + CLEANCONLL, CONLL_2000, FEWNERD, KEYPHRASE_INSPEC, @@ -465,6 +466,7 @@ "CONLL_03_DUTCH", "CONLL_03_GERMAN", "CONLL_03_SPANISH", + "CLEANCONLL", "CONLL_2000", "FEWNERD", "KEYPHRASE_INSPEC", diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 55e50723d1..9cc0d3eae6 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -1421,6 +1421,70 @@ def __init__( ) +class CLEANCONLL(ColumnCorpus): + def __init__( + self, + base_path: Optional[Union[str, Path]] = None, + in_memory: bool = True, + **corpusargs, + ) -> None: + """Initialize the CleanCoNLL corpus. + + The first time you call this constructor it will automatically download the necessary files from the repository + (https://github.com/flairNLP/CleanCoNLL) and construct the corpus. + + Args: + base_path: Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this to point to a different folder but typically this should not be necessary. + in_memory: If True, keeps dataset in memory giving speedups in training. + """ + base_path = flair.cache_root / "datasets" if not base_path else Path(base_path) + + # column format + columns = {0: "text", + 1: "pos", + 2: "nel", + 3: "ner*", + 4: "ner"} + + # this dataset name + dataset_name = self.__class__.__name__.lower() + + data_folder = base_path / dataset_name + + # download data if necessary (checking for existence of train set) + train_set = data_folder / 'CleanCoNLL-main/data/cleanconll/cleanconll.train' + + if not train_set.exists(): + print("CleanCoNLL files not found, so downloading and creating them.") + + github_url = "https://github.com/flairNLP/CleanCoNLL/archive/master.zip" + zip_path = cached_path(f"{github_url}", Path("datasets") / dataset_name) + # unzip the downloaded repo + unpack_file(zip_path, data_folder, "zip", False) + + # set the root directory used in the shell script + script_root = data_folder / 'CleanCoNLL-main' + + # run shell script for creating corpus files + import subprocess + subprocess.call(['chmod', 'u+x', script_root / 'create_cleanconll_from_conll03.sh']) + subprocess.call(['bash', script_root / 'create_cleanconll_from_conll03.sh'], + env={'SCRIPT_ROOT': str(script_root)}) + print("CleanCoNLL files are placed here:", script_root / 'data/cleanconll/') + + else: + print("Found files for CleanCoNLL in:", data_folder / 'CleanCoNLL-main/data/cleanconll/') + + super().__init__( + data_folder / 'CleanCoNLL-main/data/cleanconll', + columns, + encoding="utf-8", + in_memory=in_memory, + document_separator_token="-DOCSTART-", + **corpusargs, + ) + + class CONLL_2000(ColumnCorpus): def __init__( self, From dce9b441d98e8fd0153f54108372bc004708f430 Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 31 Jul 2024 14:04:18 +0200 Subject: [PATCH 296/333] revise cleanconll file download and merge, now using tempfile --- flair/datasets/sequence_labeling.py | 76 ++++++++++++++++++----------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 9cc0d3eae6..81e7840d26 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -8,6 +8,8 @@ from collections import defaultdict from collections.abc import Iterable, Iterator from pathlib import Path +import tempfile +import shutil from typing import ( Any, Optional, @@ -1423,60 +1425,78 @@ def __init__( class CLEANCONLL(ColumnCorpus): def __init__( - self, - base_path: Optional[Union[str, Path]] = None, - in_memory: bool = True, - **corpusargs, + self, + base_path: Optional[Union[str, Path]] = None, + in_memory: bool = True, + **corpusargs, ) -> None: - """Initialize the CleanCoNLL corpus. - - The first time you call this constructor it will automatically download the necessary files from the repository - (https://github.com/flairNLP/CleanCoNLL) and construct the corpus. + """ + Initialize the CleanCoNLL corpus. Args: - base_path: Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this to point to a different folder but typically this should not be necessary. - in_memory: If True, keeps dataset in memory giving speedups in training. + base_path: Base directory for the dataset. If None, defaults to flair.cache_root / "datasets". + in_memory: If True, keeps dataset in memory for faster training. """ + # Set the base path for the dataset base_path = flair.cache_root / "datasets" if not base_path else Path(base_path) - # column format + # Define column format columns = {0: "text", 1: "pos", 2: "nel", 3: "ner*", 4: "ner"} - # this dataset name + # Define dataset name dataset_name = self.__class__.__name__.lower() + # Define data folder path data_folder = base_path / dataset_name - # download data if necessary (checking for existence of train set) - train_set = data_folder / 'CleanCoNLL-main/data/cleanconll/cleanconll.train' + # Check if the train data file exists, otherwise download and prepare the dataset + train_set = data_folder / 'cleanconll.train' if not train_set.exists(): print("CleanCoNLL files not found, so downloading and creating them.") - github_url = "https://github.com/flairNLP/CleanCoNLL/archive/master.zip" - zip_path = cached_path(f"{github_url}", Path("datasets") / dataset_name) - # unzip the downloaded repo - unpack_file(zip_path, data_folder, "zip", False) + github_url = "https://github.com/flairNLP/CleanCoNLL/archive/main.zip" + + # Create a temporary directory + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdir = Path(tmpdirname) + + zip_path = cached_path(github_url, tmpdir) + unpack_file(zip_path, tmpdir, "zip", False) + + script_root = tmpdir / 'CleanCoNLL-main' - # set the root directory used in the shell script - script_root = data_folder / 'CleanCoNLL-main' + # Verify the script path + create_script_path = script_root / 'create_cleanconll_from_conll03.sh' + print(f"Path to the create script: {create_script_path}") - # run shell script for creating corpus files - import subprocess - subprocess.call(['chmod', 'u+x', script_root / 'create_cleanconll_from_conll03.sh']) - subprocess.call(['bash', script_root / 'create_cleanconll_from_conll03.sh'], - env={'SCRIPT_ROOT': str(script_root)}) - print("CleanCoNLL files are placed here:", script_root / 'data/cleanconll/') + if not create_script_path.exists(): + raise FileNotFoundError(f"Script not found: {create_script_path}") + + # Run shell script for creating corpus files + import subprocess + subprocess.call(['chmod', 'u+x', create_script_path]) + subprocess.call(['bash', create_script_path], cwd=script_root) + + print("CleanCoNLL files created in temporary directory:", str(script_root / 'data/cleanconll')) + + # Ensure the final data folder exists + data_folder.mkdir(parents=True, exist_ok=True) + + # Copy the files to the final data folder + shutil.copytree(script_root / 'data/cleanconll', data_folder, dirs_exist_ok=True) + print("CleanCoNLL files are placed here:", data_folder) else: - print("Found files for CleanCoNLL in:", data_folder / 'CleanCoNLL-main/data/cleanconll/') + print("Found files for CleanCoNLL in:", data_folder) + # Initialize the parent class with the specified parameters super().__init__( - data_folder / 'CleanCoNLL-main/data/cleanconll', + data_folder, columns, encoding="utf-8", in_memory=in_memory, From 4fb78fb8f6333c104460f200caddf5f316216e9a Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 31 Jul 2024 16:08:36 +0200 Subject: [PATCH 297/333] moving functionality of bash script into corpus method --- flair/datasets/sequence_labeling.py | 213 +++++++++++++++++++++++----- 1 file changed, 181 insertions(+), 32 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 81e7840d26..79d0dedd85 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -3,6 +3,7 @@ import logging import os import re +import gzip import shutil import tarfile from collections import defaultdict @@ -10,6 +11,9 @@ from pathlib import Path import tempfile import shutil +import requests +import zipfile +import subprocess from typing import ( Any, Optional, @@ -1423,6 +1427,7 @@ def __init__( ) + class CLEANCONLL(ColumnCorpus): def __init__( self, @@ -1459,37 +1464,8 @@ def __init__( if not train_set.exists(): print("CleanCoNLL files not found, so downloading and creating them.") - github_url = "https://github.com/flairNLP/CleanCoNLL/archive/main.zip" - - # Create a temporary directory - with tempfile.TemporaryDirectory() as tmpdirname: - tmpdir = Path(tmpdirname) - - zip_path = cached_path(github_url, tmpdir) - unpack_file(zip_path, tmpdir, "zip", False) - - script_root = tmpdir / 'CleanCoNLL-main' - - # Verify the script path - create_script_path = script_root / 'create_cleanconll_from_conll03.sh' - print(f"Path to the create script: {create_script_path}") - - if not create_script_path.exists(): - raise FileNotFoundError(f"Script not found: {create_script_path}") - - # Run shell script for creating corpus files - import subprocess - subprocess.call(['chmod', 'u+x', create_script_path]) - subprocess.call(['bash', create_script_path], cwd=script_root) - - print("CleanCoNLL files created in temporary directory:", str(script_root / 'data/cleanconll')) - - # Ensure the final data folder exists - data_folder.mkdir(parents=True, exist_ok=True) - - # Copy the files to the final data folder - shutil.copytree(script_root / 'data/cleanconll', data_folder, dirs_exist_ok=True) - print("CleanCoNLL files are placed here:", data_folder) + # Download and prepare the dataset + self.download_and_prepare_data(data_folder, base_path) else: print("Found files for CleanCoNLL in:", data_folder) @@ -1504,6 +1480,180 @@ def __init__( **corpusargs, ) + @staticmethod + def download_and_prepare_data(data_folder: Path, base_path: Path): + # Create a temporary directory + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdir = Path(tmpdirname) + + github_url = "https://github.com/flairNLP/CleanCoNLL/archive/main.zip" + zip_path = cached_path(github_url, tmpdir) + unpack_file(zip_path, tmpdir, "zip", False) + cleanconll_data_root = tmpdir / 'CleanCoNLL-main' + + # Check the contents of the temporary directory + print(f"Contents of the temporary directory: {list(tmpdir.iterdir())}") + + # TODO These files are different + # CoNLL-03 Flair path + # conll03_path = base_path / 'conll_03' + # # Check if the CoNLL-03 files already exists in flair's dataset directory + # if conll03_path.exists() and conll03_path.is_dir() and "eng.testa" in [f.name for f in conll03_path.iterdir()]: + # print(f"Original CoNLL03 files detected here: {conll03_path}") + # print(f"Contents: {[f.name for f in conll03_path.iterdir()]}") + # conll03_train = conll03_path / "eng.train" + # conll03_dev = conll03_path / "eng.testa" + # conll03_test = conll03_path / "eng.testb" + # print(f"So not downloading from {conll_url}, using the following files: 'eng.train', 'eng.testa', 'eng.testb'") + + conll03_dir = data_folder / 'original_conll-03' + if conll03_dir.exists() and conll03_dir.is_dir() and "train.txt" in [f.name for f in conll03_dir.iterdir()]: + print(f"Original CoNLL03 files detected here: {conll03_dir}") + + else: + conll_url = "https://data.deepai.org/conll2003.zip" + + conll03_dir.mkdir(parents=True, exist_ok=True) + print(f"Downloading the original CoNLL-03 from {conll_url} into {conll03_dir} ...") + + zip_path = conll03_dir / "conll2003.zip" + response = requests.get(conll_url) + zip_path.write_bytes(response.content) + + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(conll03_dir) + + conll03_train = conll03_dir / "train.txt" + conll03_dev = conll03_dir / "valid.txt" + conll03_test = conll03_dir / "test.txt" + + # Apply the patch files + patch_dir = cleanconll_data_root / 'data' / 'patch_files' + tokens_dir = cleanconll_data_root / 'data' / 'tokens_updated' + tokens_dir.mkdir(parents=True, exist_ok=True) + + def apply_patch(file_path, patch_path, output_path): + subprocess.run(['patch', str(file_path), str(patch_path), '-o', output_path]) + + def extract_tokens(file_path: Path, output_path: Path): + with open(file_path, 'r') as f_in, open(output_path, 'w') as f_out: + for line in f_in: + # Strip whitespace to check if the line is empty + stripped_line = line.strip() + if stripped_line: + # Write the first token followed by a newline if the line is not empty + f_out.write(stripped_line.split()[0] + '\n') + else: + # Write an empty line if the line is empty + f_out.write('\n') + + extract_tokens(conll03_train, tokens_dir / 'train_tokens.txt') + extract_tokens(conll03_dev, tokens_dir / 'valid_tokens.txt') + extract_tokens(conll03_test, tokens_dir / 'test_tokens.txt') + + apply_patch(tokens_dir / 'train_tokens.txt', patch_dir / 'train_tokens.patch', tokens_dir / 'train_tokens_updated.txt') + apply_patch(tokens_dir / 'valid_tokens.txt', patch_dir / 'dev_tokens.patch', tokens_dir / 'dev_tokens_updated.txt') + apply_patch(tokens_dir / 'test_tokens.txt', patch_dir / 'test_tokens.patch', tokens_dir / 'test_tokens_updated.txt') + + # Merge the updated token files with the CleanCoNLL annotations + cleanconll_annotations_dir = cleanconll_data_root / 'data' / 'cleanconll_annotations' + data_folder.mkdir(parents=True, exist_ok=True) + + def merge_annotations(tokens_file, annotations_file, output_file): + with open(tokens_file, 'r') as tokens, open(annotations_file, 'r') as annotations, open(output_file, 'w') as output: + for token, annotation in zip(tokens, annotations): + # Strip the leading '[TOKEN]\t' from the annotation + stripped_annotation = '\t'.join(annotation.strip().split('\t')[1:]) + output.write(token.strip() + '\t' + stripped_annotation + '\n') + merge_annotations(tokens_dir / 'train_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.train', data_folder / 'cleanconll.train') + merge_annotations(tokens_dir / 'dev_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.dev', data_folder / 'cleanconll.dev') + merge_annotations(tokens_dir / 'test_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.test', data_folder / 'cleanconll.test') + + print("Done with creating. CleanCoNLL files are placed here:", data_folder) + + + +# class CLEANCONLL(ColumnCorpus): +# def __init__( +# self, +# base_path: Optional[Union[str, Path]] = None, +# in_memory: bool = True, +# **corpusargs, +# ) -> None: +# """ +# Initialize the CleanCoNLL corpus. +# +# Args: +# base_path: Base directory for the dataset. If None, defaults to flair.cache_root / "datasets". +# in_memory: If True, keeps dataset in memory for faster training. +# """ +# # Set the base path for the dataset +# base_path = flair.cache_root / "datasets" if not base_path else Path(base_path) +# +# # Define column format +# columns = {0: "text", +# 1: "pos", +# 2: "nel", +# 3: "ner*", +# 4: "ner"} +# +# # Define dataset name +# dataset_name = self.__class__.__name__.lower() +# +# # Define data folder path +# data_folder = base_path / dataset_name +# +# # Check if the train data file exists, otherwise download and prepare the dataset +# train_set = data_folder / 'cleanconll.train' +# +# if not train_set.exists(): +# print("CleanCoNLL files not found, so downloading and creating them.") +# +# github_url = "https://github.com/flairNLP/CleanCoNLL/archive/main.zip" +# +# # Create a temporary directory +# with tempfile.TemporaryDirectory() as tmpdirname: +# tmpdir = Path(tmpdirname) +# +# zip_path = cached_path(github_url, tmpdir) +# unpack_file(zip_path, tmpdir, "zip", False) +# +# script_root = tmpdir / 'CleanCoNLL-main' +# +# # Verify the script path +# create_script_path = script_root / 'create_cleanconll_from_conll03.sh' +# print(f"Path to the create script: {create_script_path}") +# +# if not create_script_path.exists(): +# raise FileNotFoundError(f"Script not found: {create_script_path}") +# +# # Run shell script for creating corpus files +# import subprocess +# subprocess.call(['chmod', 'u+x', create_script_path]) +# subprocess.call(['bash', create_script_path], cwd=script_root) +# +# print("CleanCoNLL files created in temporary directory:", str(script_root / 'data/cleanconll')) +# +# # Ensure the final data folder exists +# data_folder.mkdir(parents=True, exist_ok=True) +# +# # Copy the files to the final data folder +# shutil.copytree(script_root / 'data/cleanconll', data_folder, dirs_exist_ok=True) +# print("CleanCoNLL files are placed here:", data_folder) +# +# else: +# print("Found files for CleanCoNLL in:", data_folder) +# +# # Initialize the parent class with the specified parameters +# super().__init__( +# data_folder, +# columns, +# encoding="utf-8", +# in_memory=in_memory, +# document_separator_token="-DOCSTART-", +# **corpusargs, +# ) + class CONLL_2000(ColumnCorpus): def __init__( @@ -1535,7 +1685,6 @@ def __init__( if not data_file.is_file(): cached_path(f"{conll_2000_path}train.txt.gz", Path("datasets") / dataset_name) cached_path(f"{conll_2000_path}test.txt.gz", Path("datasets") / dataset_name) - import gzip with ( gzip.open(flair.cache_root / "datasets" / dataset_name / "train.txt.gz", "rb") as f_in, From df976f2720bcba4c938f91da7e02952225d09923 Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 31 Jul 2024 16:12:34 +0200 Subject: [PATCH 298/333] minor --- flair/datasets/sequence_labeling.py | 60 +++++++++++++---------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 79d0dedd85..d69303ae0c 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -1482,6 +1482,30 @@ def __init__( @staticmethod def download_and_prepare_data(data_folder: Path, base_path: Path): + + def apply_patch(file_path, patch_path, output_path): + subprocess.run(['patch', str(file_path), str(patch_path), '-o', output_path]) + + def extract_tokens(file_path: Path, output_path: Path): + with open(file_path, 'r') as f_in, open(output_path, 'w') as f_out: + for line in f_in: + # Strip whitespace to check if the line is empty + stripped_line = line.strip() + if stripped_line: + # Write the first token followed by a newline if the line is not empty + f_out.write(stripped_line.split()[0] + '\n') + else: + # Write an empty line if the line is empty + f_out.write('\n') + + def merge_annotations(tokens_file, annotations_file, output_file): + with open(tokens_file, 'r') as tokens, open(annotations_file, 'r') as annotations, open(output_file, + 'w') as output: + for token, annotation in zip(tokens, annotations): + # Strip the leading '[TOKEN]\t' from the annotation + stripped_annotation = '\t'.join(annotation.strip().split('\t')[1:]) + output.write(token.strip() + '\t' + stripped_annotation + '\n') + # Create a temporary directory with tempfile.TemporaryDirectory() as tmpdirname: tmpdir = Path(tmpdirname) @@ -1494,18 +1518,6 @@ def download_and_prepare_data(data_folder: Path, base_path: Path): # Check the contents of the temporary directory print(f"Contents of the temporary directory: {list(tmpdir.iterdir())}") - # TODO These files are different - # CoNLL-03 Flair path - # conll03_path = base_path / 'conll_03' - # # Check if the CoNLL-03 files already exists in flair's dataset directory - # if conll03_path.exists() and conll03_path.is_dir() and "eng.testa" in [f.name for f in conll03_path.iterdir()]: - # print(f"Original CoNLL03 files detected here: {conll03_path}") - # print(f"Contents: {[f.name for f in conll03_path.iterdir()]}") - # conll03_train = conll03_path / "eng.train" - # conll03_dev = conll03_path / "eng.testa" - # conll03_test = conll03_path / "eng.testb" - # print(f"So not downloading from {conll_url}, using the following files: 'eng.train', 'eng.testa', 'eng.testb'") - conll03_dir = data_folder / 'original_conll-03' if conll03_dir.exists() and conll03_dir.is_dir() and "train.txt" in [f.name for f in conll03_dir.iterdir()]: print(f"Original CoNLL03 files detected here: {conll03_dir}") @@ -1527,30 +1539,16 @@ def download_and_prepare_data(data_folder: Path, base_path: Path): conll03_dev = conll03_dir / "valid.txt" conll03_test = conll03_dir / "test.txt" - # Apply the patch files patch_dir = cleanconll_data_root / 'data' / 'patch_files' tokens_dir = cleanconll_data_root / 'data' / 'tokens_updated' tokens_dir.mkdir(parents=True, exist_ok=True) - def apply_patch(file_path, patch_path, output_path): - subprocess.run(['patch', str(file_path), str(patch_path), '-o', output_path]) - - def extract_tokens(file_path: Path, output_path: Path): - with open(file_path, 'r') as f_in, open(output_path, 'w') as f_out: - for line in f_in: - # Strip whitespace to check if the line is empty - stripped_line = line.strip() - if stripped_line: - # Write the first token followed by a newline if the line is not empty - f_out.write(stripped_line.split()[0] + '\n') - else: - # Write an empty line if the line is empty - f_out.write('\n') - + # Cut the tokens from the original CoNLL-03 files extract_tokens(conll03_train, tokens_dir / 'train_tokens.txt') extract_tokens(conll03_dev, tokens_dir / 'valid_tokens.txt') extract_tokens(conll03_test, tokens_dir / 'test_tokens.txt') + # Apply the downloaded patch files apply_patch(tokens_dir / 'train_tokens.txt', patch_dir / 'train_tokens.patch', tokens_dir / 'train_tokens_updated.txt') apply_patch(tokens_dir / 'valid_tokens.txt', patch_dir / 'dev_tokens.patch', tokens_dir / 'dev_tokens_updated.txt') apply_patch(tokens_dir / 'test_tokens.txt', patch_dir / 'test_tokens.patch', tokens_dir / 'test_tokens_updated.txt') @@ -1559,12 +1557,6 @@ def extract_tokens(file_path: Path, output_path: Path): cleanconll_annotations_dir = cleanconll_data_root / 'data' / 'cleanconll_annotations' data_folder.mkdir(parents=True, exist_ok=True) - def merge_annotations(tokens_file, annotations_file, output_file): - with open(tokens_file, 'r') as tokens, open(annotations_file, 'r') as annotations, open(output_file, 'w') as output: - for token, annotation in zip(tokens, annotations): - # Strip the leading '[TOKEN]\t' from the annotation - stripped_annotation = '\t'.join(annotation.strip().split('\t')[1:]) - output.write(token.strip() + '\t' + stripped_annotation + '\n') merge_annotations(tokens_dir / 'train_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.train', data_folder / 'cleanconll.train') merge_annotations(tokens_dir / 'dev_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.dev', data_folder / 'cleanconll.dev') merge_annotations(tokens_dir / 'test_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.test', data_folder / 'cleanconll.test') From 8230190cd166b9893c20ed0d1339aed95a38c917 Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 31 Jul 2024 16:17:48 +0200 Subject: [PATCH 299/333] typos --- flair/datasets/sequence_labeling.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index d69303ae0c..5b27c0815a 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -1526,7 +1526,7 @@ def merge_annotations(tokens_file, annotations_file, output_file): conll_url = "https://data.deepai.org/conll2003.zip" conll03_dir.mkdir(parents=True, exist_ok=True) - print(f"Downloading the original CoNLL-03 from {conll_url} into {conll03_dir} ...") + print(f"Downloading the original CoNLL03 from {conll_url} into {conll03_dir} ...") zip_path = conll03_dir / "conll2003.zip" response = requests.get(conll_url) @@ -1543,12 +1543,12 @@ def merge_annotations(tokens_file, annotations_file, output_file): tokens_dir = cleanconll_data_root / 'data' / 'tokens_updated' tokens_dir.mkdir(parents=True, exist_ok=True) - # Cut the tokens from the original CoNLL-03 files + # Extract only the tokens from the original CoNLL03 files extract_tokens(conll03_train, tokens_dir / 'train_tokens.txt') extract_tokens(conll03_dev, tokens_dir / 'valid_tokens.txt') extract_tokens(conll03_test, tokens_dir / 'test_tokens.txt') - # Apply the downloaded patch files + # Apply the downloaded patch files to apply our token modifications (e.g. line breaks) apply_patch(tokens_dir / 'train_tokens.txt', patch_dir / 'train_tokens.patch', tokens_dir / 'train_tokens_updated.txt') apply_patch(tokens_dir / 'valid_tokens.txt', patch_dir / 'dev_tokens.patch', tokens_dir / 'dev_tokens_updated.txt') apply_patch(tokens_dir / 'test_tokens.txt', patch_dir / 'test_tokens.patch', tokens_dir / 'test_tokens_updated.txt') From 3b6b9605d017bc84f52deec9ca49ce2f8520fcae Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Mon, 14 Oct 2024 14:59:40 +0200 Subject: [PATCH 300/333] replace external subprocess patch call with own methods --- flair/datasets/sequence_labeling.py | 183 +++++++++++++++------------- 1 file changed, 95 insertions(+), 88 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 5b27c0815a..3c0804d619 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -1481,10 +1481,96 @@ def __init__( ) @staticmethod - def download_and_prepare_data(data_folder: Path, base_path: Path): + def download_and_prepare_data(data_folder: Path): + + def parse_patch(patch_file_path): + """ + Parses a patch file and returns a structured representation of the changes. + """ + + changes = [] + current_change = None + + with open(patch_file_path, 'r') as patch_file: + for line in patch_file: + + # Check if the line is a change, delete or add command (like 17721c17703,17705 or 5728d5727) + if line and (line[0].isdigit() and ('c' in line or 'd' in line or 'a' in line)): + if current_change: + # Append the previous change block to the changes list + changes.append(current_change) + + # Start a new change block + current_change = {'command': line, 'original': [], 'new': []} + + # Capture original lines (those marked with "<") + elif line.startswith('<'): + if current_change: + current_change['original'].append(line[2:]) # Remove the "< " part + + # Capture new lines (those marked with ">") + elif line.startswith('>'): + if current_change: + current_change['new'].append(line[2:]) # Remove the "> " part + + # Append the last change block to the changes list + if current_change: + changes.append(current_change) + + return changes + + def parse_line_range(line_range_str): + """ + Utility function to parse a line range string like '17703,17705' or '5727' and returns a tuple (start, end) + """ + parts = line_range_str.split(',') + if len(parts) == 1: + start = int(parts[0]) - 1 + return (start, start+1) + else: + start = int(parts[0]) - 1 + end = int(parts[1]) + return (start, end) + + def apply_patch_to_file(original_file, changes, output_file_path): + """ + Applies the patch instructions to the content of the original file. + """ + with open(original_file, 'r') as f: + original_lines = f.readlines() + + modified_lines = original_lines[:] # Make a copy of original lines + + # Apply each change in reverse order (important to avoid index shift issues) + for change in reversed(changes): + command = change['command'] + + # Determine the type of the change: `c` for change, `d` for delete, `a` for add + if 'c' in command: + # Example command: 17721c17703,17705 + original_line_range, new_line_range = command.split('c') + original_line_range = parse_line_range(original_line_range) + modified_lines[original_line_range[0]:original_line_range[1]] = change['new'] + + elif 'd' in command: + # Example command: 5728d5727 + original_line_number = int(command.split('d')[0]) - 1 + del modified_lines[original_line_number] + + elif 'a' in command: + # Example command: 1000a1001,1002 + original_line_number = int(command.split('a')[0]) - 1 + insertion_point = original_line_number + 1 + for new_token in reversed(change['new']): + modified_lines.insert(insertion_point, new_token) + + # Write the modified content to the output file + with open(output_file_path, 'w') as output_file: + output_file.writelines(modified_lines) def apply_patch(file_path, patch_path, output_path): - subprocess.run(['patch', str(file_path), str(patch_path), '-o', output_path]) + patch_instructions = parse_patch(patch_path) + apply_patch_to_file(file_path, patch_instructions, output_path) def extract_tokens(file_path: Path, output_path: Path): with open(file_path, 'r') as f_in, open(output_path, 'w') as f_out: @@ -1499,12 +1585,16 @@ def extract_tokens(file_path: Path, output_path: Path): f_out.write('\n') def merge_annotations(tokens_file, annotations_file, output_file): - with open(tokens_file, 'r') as tokens, open(annotations_file, 'r') as annotations, open(output_file, - 'w') as output: + with open(tokens_file, 'r') as tokens_file, \ + open(annotations_file, 'r') as annotations_file, \ + open(output_file, 'w') as output_file: + tokens = tokens_file.readlines() + annotations = annotations_file.readlines() + for token, annotation in zip(tokens, annotations): # Strip the leading '[TOKEN]\t' from the annotation stripped_annotation = '\t'.join(annotation.strip().split('\t')[1:]) - output.write(token.strip() + '\t' + stripped_annotation + '\n') + output_file.write(token.strip() + '\t' + stripped_annotation + '\n') # Create a temporary directory with tempfile.TemporaryDirectory() as tmpdirname: @@ -1564,89 +1654,6 @@ def merge_annotations(tokens_file, annotations_file, output_file): print("Done with creating. CleanCoNLL files are placed here:", data_folder) - -# class CLEANCONLL(ColumnCorpus): -# def __init__( -# self, -# base_path: Optional[Union[str, Path]] = None, -# in_memory: bool = True, -# **corpusargs, -# ) -> None: -# """ -# Initialize the CleanCoNLL corpus. -# -# Args: -# base_path: Base directory for the dataset. If None, defaults to flair.cache_root / "datasets". -# in_memory: If True, keeps dataset in memory for faster training. -# """ -# # Set the base path for the dataset -# base_path = flair.cache_root / "datasets" if not base_path else Path(base_path) -# -# # Define column format -# columns = {0: "text", -# 1: "pos", -# 2: "nel", -# 3: "ner*", -# 4: "ner"} -# -# # Define dataset name -# dataset_name = self.__class__.__name__.lower() -# -# # Define data folder path -# data_folder = base_path / dataset_name -# -# # Check if the train data file exists, otherwise download and prepare the dataset -# train_set = data_folder / 'cleanconll.train' -# -# if not train_set.exists(): -# print("CleanCoNLL files not found, so downloading and creating them.") -# -# github_url = "https://github.com/flairNLP/CleanCoNLL/archive/main.zip" -# -# # Create a temporary directory -# with tempfile.TemporaryDirectory() as tmpdirname: -# tmpdir = Path(tmpdirname) -# -# zip_path = cached_path(github_url, tmpdir) -# unpack_file(zip_path, tmpdir, "zip", False) -# -# script_root = tmpdir / 'CleanCoNLL-main' -# -# # Verify the script path -# create_script_path = script_root / 'create_cleanconll_from_conll03.sh' -# print(f"Path to the create script: {create_script_path}") -# -# if not create_script_path.exists(): -# raise FileNotFoundError(f"Script not found: {create_script_path}") -# -# # Run shell script for creating corpus files -# import subprocess -# subprocess.call(['chmod', 'u+x', create_script_path]) -# subprocess.call(['bash', create_script_path], cwd=script_root) -# -# print("CleanCoNLL files created in temporary directory:", str(script_root / 'data/cleanconll')) -# -# # Ensure the final data folder exists -# data_folder.mkdir(parents=True, exist_ok=True) -# -# # Copy the files to the final data folder -# shutil.copytree(script_root / 'data/cleanconll', data_folder, dirs_exist_ok=True) -# print("CleanCoNLL files are placed here:", data_folder) -# -# else: -# print("Found files for CleanCoNLL in:", data_folder) -# -# # Initialize the parent class with the specified parameters -# super().__init__( -# data_folder, -# columns, -# encoding="utf-8", -# in_memory=in_memory, -# document_separator_token="-DOCSTART-", -# **corpusargs, -# ) - - class CONLL_2000(ColumnCorpus): def __init__( self, From e5ca9b58b327fbd943a99e57f3311d15e09acac8 Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 16 Oct 2024 11:32:37 +0200 Subject: [PATCH 301/333] formatting --- flair/datasets/sequence_labeling.py | 127 +++++++++++++++------------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 3c0804d619..1a669bb081 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -1427,13 +1427,12 @@ def __init__( ) - class CLEANCONLL(ColumnCorpus): def __init__( - self, - base_path: Optional[Union[str, Path]] = None, - in_memory: bool = True, - **corpusargs, + self, + base_path: Optional[Union[str, Path]] = None, + in_memory: bool = True, + **corpusargs, ) -> None: """ Initialize the CleanCoNLL corpus. @@ -1446,11 +1445,7 @@ def __init__( base_path = flair.cache_root / "datasets" if not base_path else Path(base_path) # Define column format - columns = {0: "text", - 1: "pos", - 2: "nel", - 3: "ner*", - 4: "ner"} + columns = {0: "text", 1: "pos", 2: "nel", 3: "ner*", 4: "ner"} # Define dataset name dataset_name = self.__class__.__name__.lower() @@ -1459,13 +1454,13 @@ def __init__( data_folder = base_path / dataset_name # Check if the train data file exists, otherwise download and prepare the dataset - train_set = data_folder / 'cleanconll.train' + train_set = data_folder / "cleanconll.train" if not train_set.exists(): print("CleanCoNLL files not found, so downloading and creating them.") # Download and prepare the dataset - self.download_and_prepare_data(data_folder, base_path) + self.download_and_prepare_data(data_folder) else: print("Found files for CleanCoNLL in:", data_folder) @@ -1482,7 +1477,6 @@ def __init__( @staticmethod def download_and_prepare_data(data_folder: Path): - def parse_patch(patch_file_path): """ Parses a patch file and returns a structured representation of the changes. @@ -1491,27 +1485,26 @@ def parse_patch(patch_file_path): changes = [] current_change = None - with open(patch_file_path, 'r') as patch_file: + with open(patch_file_path, "r", encoding="utf-8") as patch_file: for line in patch_file: - # Check if the line is a change, delete or add command (like 17721c17703,17705 or 5728d5727) - if line and (line[0].isdigit() and ('c' in line or 'd' in line or 'a' in line)): + if line and (line[0].isdigit() and ("c" in line or "d" in line or "a" in line)): if current_change: # Append the previous change block to the changes list changes.append(current_change) # Start a new change block - current_change = {'command': line, 'original': [], 'new': []} + current_change = {"command": line, "original": [], "new": []} # Capture original lines (those marked with "<") - elif line.startswith('<'): + elif line.startswith("<"): if current_change: - current_change['original'].append(line[2:]) # Remove the "< " part + current_change["original"].append(line[2:]) # Remove the "< " part # Capture new lines (those marked with ">") - elif line.startswith('>'): + elif line.startswith(">"): if current_change: - current_change['new'].append(line[2:]) # Remove the "> " part + current_change["new"].append(line[2:]) # Remove the "> " part # Append the last change block to the changes list if current_change: @@ -1523,10 +1516,10 @@ def parse_line_range(line_range_str): """ Utility function to parse a line range string like '17703,17705' or '5727' and returns a tuple (start, end) """ - parts = line_range_str.split(',') + parts = line_range_str.split(",") if len(parts) == 1: start = int(parts[0]) - 1 - return (start, start+1) + return (start, start + 1) else: start = int(parts[0]) - 1 end = int(parts[1]) @@ -1536,36 +1529,36 @@ def apply_patch_to_file(original_file, changes, output_file_path): """ Applies the patch instructions to the content of the original file. """ - with open(original_file, 'r') as f: + with open(original_file, "r", encoding="utf-8") as f: original_lines = f.readlines() modified_lines = original_lines[:] # Make a copy of original lines # Apply each change in reverse order (important to avoid index shift issues) for change in reversed(changes): - command = change['command'] + command = change["command"] # Determine the type of the change: `c` for change, `d` for delete, `a` for add - if 'c' in command: + if "c" in command: # Example command: 17721c17703,17705 - original_line_range, new_line_range = command.split('c') + original_line_range, new_line_range = command.split("c") original_line_range = parse_line_range(original_line_range) - modified_lines[original_line_range[0]:original_line_range[1]] = change['new'] + modified_lines[original_line_range[0] : original_line_range[1]] = change["new"] - elif 'd' in command: + elif "d" in command: # Example command: 5728d5727 - original_line_number = int(command.split('d')[0]) - 1 + original_line_number = int(command.split("d")[0]) - 1 del modified_lines[original_line_number] - elif 'a' in command: + elif "a" in command: # Example command: 1000a1001,1002 - original_line_number = int(command.split('a')[0]) - 1 + original_line_number = int(command.split("a")[0]) - 1 insertion_point = original_line_number + 1 - for new_token in reversed(change['new']): + for new_token in reversed(change["new"]): modified_lines.insert(insertion_point, new_token) # Write the modified content to the output file - with open(output_file_path, 'w') as output_file: + with open(output_file_path, "w", encoding="utf-8") as output_file: output_file.writelines(modified_lines) def apply_patch(file_path, patch_path, output_path): @@ -1573,28 +1566,28 @@ def apply_patch(file_path, patch_path, output_path): apply_patch_to_file(file_path, patch_instructions, output_path) def extract_tokens(file_path: Path, output_path: Path): - with open(file_path, 'r') as f_in, open(output_path, 'w') as f_out: + with open(file_path, "r", encoding="utf-8") as f_in, open(output_path, "w", encoding="utf-8") as f_out: for line in f_in: # Strip whitespace to check if the line is empty stripped_line = line.strip() if stripped_line: # Write the first token followed by a newline if the line is not empty - f_out.write(stripped_line.split()[0] + '\n') + f_out.write(stripped_line.split()[0] + "\n") else: # Write an empty line if the line is empty - f_out.write('\n') + f_out.write("\n") def merge_annotations(tokens_file, annotations_file, output_file): - with open(tokens_file, 'r') as tokens_file, \ - open(annotations_file, 'r') as annotations_file, \ - open(output_file, 'w') as output_file: + with open(tokens_file, "r", encoding="utf-8") as tokens_file, open( + annotations_file, "r", encoding="utf-8" + ) as annotations_file, open(output_file, "w", encoding="utf-8") as output_file: tokens = tokens_file.readlines() annotations = annotations_file.readlines() for token, annotation in zip(tokens, annotations): # Strip the leading '[TOKEN]\t' from the annotation - stripped_annotation = '\t'.join(annotation.strip().split('\t')[1:]) - output_file.write(token.strip() + '\t' + stripped_annotation + '\n') + stripped_annotation = "\t".join(annotation.strip().split("\t")[1:]) + output_file.write(token.strip() + "\t" + stripped_annotation + "\n") # Create a temporary directory with tempfile.TemporaryDirectory() as tmpdirname: @@ -1603,12 +1596,12 @@ def merge_annotations(tokens_file, annotations_file, output_file): github_url = "https://github.com/flairNLP/CleanCoNLL/archive/main.zip" zip_path = cached_path(github_url, tmpdir) unpack_file(zip_path, tmpdir, "zip", False) - cleanconll_data_root = tmpdir / 'CleanCoNLL-main' + cleanconll_data_root = tmpdir / "CleanCoNLL-main" # Check the contents of the temporary directory print(f"Contents of the temporary directory: {list(tmpdir.iterdir())}") - conll03_dir = data_folder / 'original_conll-03' + conll03_dir = data_folder / "original_conll-03" if conll03_dir.exists() and conll03_dir.is_dir() and "train.txt" in [f.name for f in conll03_dir.iterdir()]: print(f"Original CoNLL03 files detected here: {conll03_dir}") @@ -1622,34 +1615,54 @@ def merge_annotations(tokens_file, annotations_file, output_file): response = requests.get(conll_url) zip_path.write_bytes(response.content) - with zipfile.ZipFile(zip_path, 'r') as zip_ref: + with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(conll03_dir) conll03_train = conll03_dir / "train.txt" conll03_dev = conll03_dir / "valid.txt" conll03_test = conll03_dir / "test.txt" - patch_dir = cleanconll_data_root / 'data' / 'patch_files' - tokens_dir = cleanconll_data_root / 'data' / 'tokens_updated' + patch_dir = cleanconll_data_root / "data" / "patch_files" + tokens_dir = cleanconll_data_root / "data" / "tokens_updated" tokens_dir.mkdir(parents=True, exist_ok=True) # Extract only the tokens from the original CoNLL03 files - extract_tokens(conll03_train, tokens_dir / 'train_tokens.txt') - extract_tokens(conll03_dev, tokens_dir / 'valid_tokens.txt') - extract_tokens(conll03_test, tokens_dir / 'test_tokens.txt') + extract_tokens(conll03_train, tokens_dir / "train_tokens.txt") + extract_tokens(conll03_dev, tokens_dir / "valid_tokens.txt") + extract_tokens(conll03_test, tokens_dir / "test_tokens.txt") # Apply the downloaded patch files to apply our token modifications (e.g. line breaks) - apply_patch(tokens_dir / 'train_tokens.txt', patch_dir / 'train_tokens.patch', tokens_dir / 'train_tokens_updated.txt') - apply_patch(tokens_dir / 'valid_tokens.txt', patch_dir / 'dev_tokens.patch', tokens_dir / 'dev_tokens_updated.txt') - apply_patch(tokens_dir / 'test_tokens.txt', patch_dir / 'test_tokens.patch', tokens_dir / 'test_tokens_updated.txt') + apply_patch( + tokens_dir / "train_tokens.txt", + patch_dir / "train_tokens.patch", + tokens_dir / "train_tokens_updated.txt", + ) + apply_patch( + tokens_dir / "valid_tokens.txt", patch_dir / "dev_tokens.patch", tokens_dir / "dev_tokens_updated.txt" + ) + apply_patch( + tokens_dir / "test_tokens.txt", patch_dir / "test_tokens.patch", tokens_dir / "test_tokens_updated.txt" + ) # Merge the updated token files with the CleanCoNLL annotations - cleanconll_annotations_dir = cleanconll_data_root / 'data' / 'cleanconll_annotations' + cleanconll_annotations_dir = cleanconll_data_root / "data" / "cleanconll_annotations" data_folder.mkdir(parents=True, exist_ok=True) - merge_annotations(tokens_dir / 'train_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.train', data_folder / 'cleanconll.train') - merge_annotations(tokens_dir / 'dev_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.dev', data_folder / 'cleanconll.dev') - merge_annotations(tokens_dir / 'test_tokens_updated.txt', cleanconll_annotations_dir / 'cleanconll_annotations.test', data_folder / 'cleanconll.test') + merge_annotations( + tokens_dir / "train_tokens_updated.txt", + cleanconll_annotations_dir / "cleanconll_annotations.train", + data_folder / "cleanconll.train", + ) + merge_annotations( + tokens_dir / "dev_tokens_updated.txt", + cleanconll_annotations_dir / "cleanconll_annotations.dev", + data_folder / "cleanconll.dev", + ) + merge_annotations( + tokens_dir / "test_tokens_updated.txt", + cleanconll_annotations_dir / "cleanconll_annotations.test", + data_folder / "cleanconll.test", + ) print("Done with creating. CleanCoNLL files are placed here:", data_folder) From e572e99bfd2b970e893feca272ee5f0f8445ccba Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 16 Oct 2024 12:11:26 +0200 Subject: [PATCH 302/333] formatting --- flair/datasets/sequence_labeling.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 1a669bb081..954a3dd71b 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -13,7 +13,6 @@ import shutil import requests import zipfile -import subprocess from typing import ( Any, Optional, @@ -1434,8 +1433,7 @@ def __init__( in_memory: bool = True, **corpusargs, ) -> None: - """ - Initialize the CleanCoNLL corpus. + """Initialize the CleanCoNLL corpus. Args: base_path: Base directory for the dataset. If None, defaults to flair.cache_root / "datasets". @@ -1478,14 +1476,11 @@ def __init__( @staticmethod def download_and_prepare_data(data_folder: Path): def parse_patch(patch_file_path): - """ - Parses a patch file and returns a structured representation of the changes. - """ - + """Parses a patch file and returns a structured representation of the changes.""" changes = [] current_change = None - with open(patch_file_path, "r", encoding="utf-8") as patch_file: + with open(patch_file_path, encoding="utf-8") as patch_file: for line in patch_file: # Check if the line is a change, delete or add command (like 17721c17703,17705 or 5728d5727) if line and (line[0].isdigit() and ("c" in line or "d" in line or "a" in line)): @@ -1513,9 +1508,7 @@ def parse_patch(patch_file_path): return changes def parse_line_range(line_range_str): - """ - Utility function to parse a line range string like '17703,17705' or '5727' and returns a tuple (start, end) - """ + """Utility function to parse a line range string like '17703,17705' or '5727' and returns a tuple (start, end)""" parts = line_range_str.split(",") if len(parts) == 1: start = int(parts[0]) - 1 @@ -1529,7 +1522,7 @@ def apply_patch_to_file(original_file, changes, output_file_path): """ Applies the patch instructions to the content of the original file. """ - with open(original_file, "r", encoding="utf-8") as f: + with open(original_file, encoding="utf-8") as f: original_lines = f.readlines() modified_lines = original_lines[:] # Make a copy of original lines @@ -1566,7 +1559,7 @@ def apply_patch(file_path, patch_path, output_path): apply_patch_to_file(file_path, patch_instructions, output_path) def extract_tokens(file_path: Path, output_path: Path): - with open(file_path, "r", encoding="utf-8") as f_in, open(output_path, "w", encoding="utf-8") as f_out: + with open(file_path, encoding="utf-8") as f_in, open(output_path, "w", encoding="utf-8") as f_out: for line in f_in: # Strip whitespace to check if the line is empty stripped_line = line.strip() @@ -1578,8 +1571,8 @@ def extract_tokens(file_path: Path, output_path: Path): f_out.write("\n") def merge_annotations(tokens_file, annotations_file, output_file): - with open(tokens_file, "r", encoding="utf-8") as tokens_file, open( - annotations_file, "r", encoding="utf-8" + with open(tokens_file, encoding="utf-8") as tokens_file, open( + annotations_file, encoding="utf-8" ) as annotations_file, open(output_file, "w", encoding="utf-8") as output_file: tokens = tokens_file.readlines() annotations = annotations_file.readlines() From 1387ac2583c371ea96728e5b13e5ed67b65f7318 Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 16 Oct 2024 12:15:43 +0200 Subject: [PATCH 303/333] formatting --- flair/datasets/sequence_labeling.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 954a3dd71b..f13dc5c76b 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -9,10 +9,6 @@ from collections import defaultdict from collections.abc import Iterable, Iterator from pathlib import Path -import tempfile -import shutil -import requests -import zipfile from typing import ( Any, Optional, @@ -20,6 +16,7 @@ cast, ) +import requests from torch.utils.data import ConcatDataset, Dataset import flair @@ -1508,7 +1505,7 @@ def parse_patch(patch_file_path): return changes def parse_line_range(line_range_str): - """Utility function to parse a line range string like '17703,17705' or '5727' and returns a tuple (start, end)""" + """Utility function to parse a line range string like '17703,17705' or '5727' and returns a tuple (start, end).""" parts = line_range_str.split(",") if len(parts) == 1: start = int(parts[0]) - 1 @@ -1519,9 +1516,7 @@ def parse_line_range(line_range_str): return (start, end) def apply_patch_to_file(original_file, changes, output_file_path): - """ - Applies the patch instructions to the content of the original file. - """ + """Applies the patch instructions to the content of the original file.""" with open(original_file, encoding="utf-8") as f: original_lines = f.readlines() From 20895c47efb8c104c5252c040e4b04b54c8d6922 Mon Sep 17 00:00:00 2001 From: susannaruecker Date: Wed, 16 Oct 2024 13:08:58 +0200 Subject: [PATCH 304/333] correct order --- flair/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flair/datasets/__init__.py b/flair/datasets/__init__.py index 76c2c08388..d480c93552 100644 --- a/flair/datasets/__init__.py +++ b/flair/datasets/__init__.py @@ -171,11 +171,11 @@ # Expose all sequence labeling datasets from .sequence_labeling import ( BIOSCOPE, + CLEANCONLL, CONLL_03, CONLL_03_DUTCH, CONLL_03_GERMAN, CONLL_03_SPANISH, - CLEANCONLL, CONLL_2000, FEWNERD, KEYPHRASE_INSPEC, From 79287c3587085efb08e29a5e9f00e65567f2d64f Mon Sep 17 00:00:00 2001 From: Benedikt Fuchs Date: Fri, 6 Dec 2024 20:56:43 +0100 Subject: [PATCH 305/333] format code --- flair/datasets/sequence_labeling.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index f13dc5c76b..fa0962f2f4 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -6,6 +6,8 @@ import gzip import shutil import tarfile +import tempfile +import zipfile from collections import defaultdict from collections.abc import Iterable, Iterator from pathlib import Path @@ -1566,9 +1568,11 @@ def extract_tokens(file_path: Path, output_path: Path): f_out.write("\n") def merge_annotations(tokens_file, annotations_file, output_file): - with open(tokens_file, encoding="utf-8") as tokens_file, open( - annotations_file, encoding="utf-8" - ) as annotations_file, open(output_file, "w", encoding="utf-8") as output_file: + with ( + open(tokens_file, encoding="utf-8") as tokens_file, + open(annotations_file, encoding="utf-8") as annotations_file, + open(output_file, "w", encoding="utf-8") as output_file, + ): tokens = tokens_file.readlines() annotations = annotations_file.readlines() From 0795f63b71cc4c95155cb11c0d964df7d35e1e9f Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 14:51:29 +0100 Subject: [PATCH 306/333] Add docstrings to Corpus object --- flair/data.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/flair/data.py b/flair/data.py index f5d902f0e6..aa5d5ba2da 100644 --- a/flair/data.py +++ b/flair/data.py @@ -1372,6 +1372,14 @@ def unlabeled_identifier(self) -> str: class Corpus(typing.Generic[T_co]): + """The main object in Flair for holding a dataset used for training and testing. + + A corpus consists of three splits: A `train` split used for training, a `dev` split used for model selection + and/or early stopping and a `test` split used for testing. All three splits are optional, so it is possible + to create a corpus only using one or two splits. If the option `sample_missing_splits` is set to True, + missing splits will be randomly sampled from the training split. + """ + def __init__( self, train: Optional[Dataset[T_co]] = None, @@ -1381,6 +1389,26 @@ def __init__( sample_missing_splits: Union[bool, str] = True, random_seed: Optional[int] = None, ) -> None: + """ + Constructor method to initialize a :class:`Corpus`. You can define the train, dev and test split + by passing the corresponding Dataset object to the constructor. At least one split should be defined. + If the option `sample_missing_splits` is set to True, missing splits will be randomly sampled from the + train split. + + In most cases, you will not use the constructor yourself. Rather, you will create a corpus using one of our + helper methods that read common NLP filetypes. For instance, you can use + :class:`flair.datasets.sequence_labeling.ColumnCorpus` to read CoNLL-formatted files directly into + a :class:`Corpus`. + + Args: + train (torch.utils.data.Dataset): The split you use for model training. + dev (torch.utils.data.Dataset): A holdout split typically used for model selection or early stopping. + test (torch.utils.data.Dataset): The final test data to compute the score of the model. + name (str): A name that identifies the corpus. + sample_missing_splits (bool): If set to True, missing splits are sampled from train. If set to False, + missing splits are not sampled and left empty. Default: True. + random_seed (int): Set a random seed to control the sampling of missing splits. + """ # set name self.name: str = name @@ -1419,14 +1447,17 @@ def __init__( @property def train(self) -> Optional[Dataset[T_co]]: + """The training split as a :class:`torch.utils.data.Dataset` object.""" return self._train @property def dev(self) -> Optional[Dataset[T_co]]: + """The dev split as a :class:`torch.utils.data.Dataset` object.""" return self._dev @property def test(self) -> Optional[Dataset[T_co]]: + """The test split as a :class:`torch.utils.data.Dataset` object.""" return self._test def downsample( @@ -1833,6 +1864,13 @@ def add_label_noise( ) def get_label_distribution(self): + """Counts occurrences of each label in the corpus and returns them as a dictionary object. + + This allows you to get an idea of which label appears how often in the Corpus. + + Returns: + Dictionary with labels as keys and their occurrences as values. + """ class_to_count = defaultdict(lambda: 0) for sent in self.train: for label in sent.labels: @@ -1840,6 +1878,11 @@ def get_label_distribution(self): return class_to_count def get_all_sentences(self) -> ConcatDataset: + """Returns all sentences (spanning all three splits) in the :class:`Corpus`. + + Returns: + A :class:`torch.utils.data.Dataset` object that includes all sentences of this corpus. + """ parts = [] if self.train: parts.append(self.train) From 4f82732b14571a1504aee1faa39b24cbdc86dfbb Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 15:11:01 +0100 Subject: [PATCH 307/333] Add docstrings to Corpus object --- flair/data.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/flair/data.py b/flair/data.py index aa5d5ba2da..fe92a3cbd4 100644 --- a/flair/data.py +++ b/flair/data.py @@ -1397,8 +1397,8 @@ def __init__( In most cases, you will not use the constructor yourself. Rather, you will create a corpus using one of our helper methods that read common NLP filetypes. For instance, you can use - :class:`flair.datasets.sequence_labeling.ColumnCorpus` to read CoNLL-formatted files directly into - a :class:`Corpus`. + :class:`flair.datasets.sequence_labeling.ColumnCorpus` to read CoNLL-formatted files directly into + a :class:`Corpus`. Args: train (torch.utils.data.Dataset): The split you use for model training. @@ -1611,9 +1611,17 @@ def _downsample_to_proportion(dataset: Dataset, proportion: float, random_seed: return splits[0] def obtain_statistics(self, label_type: Optional[str] = None, pretty_print: bool = True) -> Union[dict, str]: - """Print statistics about the class distribution and sentence sizes. + """Print statistics about the corpus, including the length of the sentences and the labels in the corpus. - only labels of sentences are taken into account + Args: + label_type (str): Optionally set this value to obtain statistics only for one specific type of label (such + as "ner" or "pos"). If not set, statistics for all labels will be returned. + pretty_print (bool): If set to True, returns pretty json (indented for readabilty). If not, the json is + returned as a single line. Default: True. + + Returns: + A pretty print formatted string in json format if pretty_print is set to True. + A dictionary holding a json if pretty_print is set to False. """ json_data = { "TRAIN": self._obtain_statistics_for(self.train, "TRAIN", label_type), @@ -1685,7 +1693,21 @@ def make_label_dictionary( ) -> Dictionary: """Creates a dictionary of all labels assigned to the sentences in the corpus. - :return: dictionary of labels + Args: + label_type (str): The name of the label type for which the dictionary should be created. Some corpora have + multiple layers of annotation, such as "pos" and "ner". In this case, you should choose the label type + you are interested in. + min_count (int): Optionally set this to exclude rare labels from the dictionary (i.e., labels seen fewer + than the provided integer value). + add_unk (bool): Optionally set this to True to include a "UNK" value in the dictionary. In most cases, this + is not needed since the label dictionary is well-defined, but some use cases might have open classes + and require this. + add_dev_test (bool): Optionally set this to True to construct the label dictionary not only from the train + split, but also from dev and test. This is only necessary if some labels never appear in train but do + appear in one of the other splits. + + Returns: + A Dictionary of all unique labels in the corpus. """ if min_count > 0 and not add_unk: add_unk = True From a5aa07f5701c1b8197da16435873f9790cb77c7a Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 15:20:39 +0100 Subject: [PATCH 308/333] Add docstrings to SequenceTagger --- flair/models/sequence_tagger_model.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index 68e8e95b57..c375730d7c 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -22,6 +22,15 @@ class SequenceTagger(flair.nn.Classifier[Sentence]): + """The SequenceTagger is one of two main architectures in Flair used for sequence tagging. + + Sequence tagging means classifying words in a sentence, for instance for part-of-speech tagging or named entity + recognition. The SequenceTagger implements the "classic" model based on the LSTM-CRF architecture: words are first + embedded using one or multiple :class:`flair.embeddings.TokenEmbeddings`, these embeddings are then passed to the + LSTM. Its hidden states for each input word are used to make the final prediction with a softmax classifier. + For decoding, the SequenceTagger by default uses a CRF approach. + """ + def __init__( self, embeddings: TokenEmbeddings, @@ -44,7 +53,7 @@ def __init__( init_from_state_dict: bool = False, allow_unk_predictions: bool = False, ) -> None: - """Sequence Tagger class for predicting labels for single tokens. Can be parameterized by several attributes. + """Sequence tagger class for predicting labels for single tokens. Can be parameterized by several attributes. In case of multitask learning, pass shared embeddings or shared rnn into respective attributes. From ad028d0320f4d2a15870d116d01a1665db21ba36 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 19:58:31 +0100 Subject: [PATCH 309/333] Remove type hints from arguments in docstring --- flair/data.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/flair/data.py b/flair/data.py index fe92a3cbd4..7ee32f40b9 100644 --- a/flair/data.py +++ b/flair/data.py @@ -1401,13 +1401,13 @@ def __init__( a :class:`Corpus`. Args: - train (torch.utils.data.Dataset): The split you use for model training. - dev (torch.utils.data.Dataset): A holdout split typically used for model selection or early stopping. - test (torch.utils.data.Dataset): The final test data to compute the score of the model. - name (str): A name that identifies the corpus. - sample_missing_splits (bool): If set to True, missing splits are sampled from train. If set to False, + train: The split you use for model training. + dev: A holdout split typically used for model selection or early stopping. + test: The final test data to compute the score of the model. + name: A name that identifies the corpus. + sample_missing_splits: If set to True, missing splits are sampled from train. If set to False, missing splits are not sampled and left empty. Default: True. - random_seed (int): Set a random seed to control the sampling of missing splits. + random_seed: Set a random seed to control the sampling of missing splits. """ # set name self.name: str = name @@ -1474,12 +1474,12 @@ def downsample( data points. It additionally returns a pointer to itself for use in method chaining. Args: - percentage (float): A float value between 0. and 1. that indicates to which percentage the corpus + percentage: A float value between 0. and 1. that indicates to which percentage the corpus should be downsampled. Default value is 0.1, meaning it gets downsampled to 10%. - downsample_train (bool): Whether or not to include the training split in downsampling. Default is True. - downsample_dev (bool): Whether or not to include the dev split in downsampling. Default is True. - downsample_test (bool): Whether or not to include the test split in downsampling. Default is True. - random_seed (int): An optional random seed to make downsampling reproducible. + downsample_train: Whether or not to include the training split in downsampling. Default is True. + downsample_dev: Whether or not to include the dev split in downsampling. Default is True. + downsample_test: Whether or not to include the test split in downsampling. Default is True. + random_seed: An optional random seed to make downsampling reproducible. Returns: A pointer to itself for optional use in method chaining. @@ -1614,14 +1614,14 @@ def obtain_statistics(self, label_type: Optional[str] = None, pretty_print: bool """Print statistics about the corpus, including the length of the sentences and the labels in the corpus. Args: - label_type (str): Optionally set this value to obtain statistics only for one specific type of label (such + label_type: Optionally set this value to obtain statistics only for one specific type of label (such as "ner" or "pos"). If not set, statistics for all labels will be returned. - pretty_print (bool): If set to True, returns pretty json (indented for readabilty). If not, the json is + pretty_print: If set to True, returns pretty json (indented for readabilty). If not, the json is returned as a single line. Default: True. Returns: - A pretty print formatted string in json format if pretty_print is set to True. - A dictionary holding a json if pretty_print is set to False. + If pretty_print is True, returns a pretty print formatted string in json format. Otherwise, returns a + dictionary holding a json. """ json_data = { "TRAIN": self._obtain_statistics_for(self.train, "TRAIN", label_type), @@ -1694,15 +1694,15 @@ def make_label_dictionary( """Creates a dictionary of all labels assigned to the sentences in the corpus. Args: - label_type (str): The name of the label type for which the dictionary should be created. Some corpora have + label_type: The name of the label type for which the dictionary should be created. Some corpora have multiple layers of annotation, such as "pos" and "ner". In this case, you should choose the label type you are interested in. - min_count (int): Optionally set this to exclude rare labels from the dictionary (i.e., labels seen fewer + min_count: Optionally set this to exclude rare labels from the dictionary (i.e., labels seen fewer than the provided integer value). - add_unk (bool): Optionally set this to True to include a "UNK" value in the dictionary. In most cases, this + add_unk: Optionally set this to True to include a "UNK" value in the dictionary. In most cases, this is not needed since the label dictionary is well-defined, but some use cases might have open classes and require this. - add_dev_test (bool): Optionally set this to True to construct the label dictionary not only from the train + add_dev_test: Optionally set this to True to construct the label dictionary not only from the train split, but also from dev and test. This is only necessary if some labels never appear in train but do appear in one of the other splits. From a2f9d39f160c504f98e12c32b3e96ecd81d82c5b Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 19:58:45 +0100 Subject: [PATCH 310/333] Try excluding init from apidocs --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 22ffbd0194..6234c2c0fd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -100,7 +100,7 @@ def linkcode_resolve(*args): html_show_sphinx = False # Napoleon settings -napoleon_include_init_with_doc = True +napoleon_include_init_with_doc = False napoleon_include_private_with_doc = False autodoc_default_options = { From 9fe883fc054c4a6bdf6c3fff4d7959163d74e2b9 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 20:14:07 +0100 Subject: [PATCH 311/333] Try suppressing external inheritance --- docs/conf.py | 18 ++++++++++++++++++ flair/models/sequence_tagger_model.py | 2 ++ 2 files changed, 20 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 6234c2c0fd..cb4a13e794 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -122,6 +122,24 @@ def linkcode_resolve(*args): "index": [], } +import sphinx.ext.autodoc + + +def suppress_inherited_from_external(app, what, name, obj, options, signature, return_annotation): + # Filter out inherited members that come from external modules + # You can add conditions based on module names or other attributes + if hasattr(obj, "__module__"): + module_name = obj.__module__ + if "external_module_name" in module_name: # replace 'external_module_name' with the actual name + return None # Suppress this member from being documented + + return return_annotation + + +def setup(app): + app.connect("autodoc-process-docstring", suppress_inherited_from_external) + + smv_latest_version = importlib_metadata.version(project) # Whitelist pattern for tags (set to None to ignore all tags) diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index c375730d7c..c803d130d1 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -29,6 +29,8 @@ class SequenceTagger(flair.nn.Classifier[Sentence]): embedded using one or multiple :class:`flair.embeddings.TokenEmbeddings`, these embeddings are then passed to the LSTM. Its hidden states for each input word are used to make the final prediction with a softmax classifier. For decoding, the SequenceTagger by default uses a CRF approach. + + Alternatively, you can use the class :class:`flair.models.TokenClassifier` for sequence tagging without a LSTM-CRF. """ def __init__( From 9deabd5c36339dcb178fc662a92d4b9f56c8ccec Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 20:14:29 +0100 Subject: [PATCH 312/333] Try suppressing external inheritance --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index cb4a13e794..f86a519b98 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -130,7 +130,7 @@ def suppress_inherited_from_external(app, what, name, obj, options, signature, r # You can add conditions based on module names or other attributes if hasattr(obj, "__module__"): module_name = obj.__module__ - if "external_module_name" in module_name: # replace 'external_module_name' with the actual name + if "torch" in module_name: # replace 'external_module_name' with the actual name return None # Suppress this member from being documented return return_annotation From 16e5a419836636989d8200728b34f0048e8bbb1f Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 20:27:08 +0100 Subject: [PATCH 313/333] Try suppressing external inheritance --- docs/conf.py | 6 ++---- flair/models/sequence_tagger_model.py | 22 +++++++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f86a519b98..76ba11ee6f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,7 @@ # noqa: INP001 import inspect - import importlib_metadata +import sphinx.ext.autodoc # -- Project information ----------------------------------------------------- from sphinx_github_style import get_linkcode_resolve @@ -122,15 +122,13 @@ def linkcode_resolve(*args): "index": [], } -import sphinx.ext.autodoc - def suppress_inherited_from_external(app, what, name, obj, options, signature, return_annotation): # Filter out inherited members that come from external modules # You can add conditions based on module names or other attributes if hasattr(obj, "__module__"): module_name = obj.__module__ - if "torch" in module_name: # replace 'external_module_name' with the actual name + if "external_module_name" in module_name: # replace 'external_module_name' with the actual name return None # Suppress this member from being documented return return_annotation diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index c803d130d1..7bc9ec051b 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -55,9 +55,7 @@ def __init__( init_from_state_dict: bool = False, allow_unk_predictions: bool = False, ) -> None: - """Sequence tagger class for predicting labels for single tokens. Can be parameterized by several attributes. - - In case of multitask learning, pass shared embeddings or shared rnn into respective attributes. + """Constructor for this class. Args: embeddings: Embeddings to use during training and prediction @@ -279,6 +277,16 @@ def RNN( return RNN def forward_loss(self, sentences: list[Sentence]) -> tuple[torch.Tensor, int]: + """Conducts a forward pass through the SequenceTagger using labeled sentences and return the loss. + + Args: + sentences: A batch of labeled sentences. + + Returns: + A tuple consisting of the loss tensor and the number of tokens in the batch. + + """ + # if there are no sentences, there is no loss if len(sentences) == 0: return torch.tensor(0.0, dtype=torch.float, device=flair.device, requires_grad=True), 0 @@ -302,7 +310,7 @@ def _prepare_tensors(self, data_points: Union[list[Sentence], Sentence]) -> tupl return sentence_tensor, lengths def forward(self, sentence_tensor: torch.Tensor, lengths: torch.LongTensor): - """Forward propagation through network. + """Forward pass through the SequenceTagger. Args: sentence_tensor: A tensor representing the batch of sentences. @@ -450,7 +458,11 @@ def predict( embedding_storage_mode="none", force_token_predictions: bool = False, ): - """Predicts labels for current batch with CRF or Softmax. + """Call this method to predict labels for sentences. + + Predictions are directly added to the Sentence objects that are passed to this method. This means that + the predict() method does not return predictions. Rather, predictions are stored at each sentence and can + be retrieved by calling :func:`flair.data.Sentence.get_labels()` on each :class:`flair.data.Sentence`. Args: sentences: List of sentences in batch From 537511b56e8502aec67e57b4e3b4aff8f16785f0 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 20:31:51 +0100 Subject: [PATCH 314/333] Try suppressing external inheritance --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 76ba11ee6f..bd56154f70 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -123,7 +123,7 @@ def linkcode_resolve(*args): } -def suppress_inherited_from_external(app, what, name, obj, options, signature, return_annotation): +def suppress_inherited_from_external(app, what, name, obj, options, signature): # Filter out inherited members that come from external modules # You can add conditions based on module names or other attributes if hasattr(obj, "__module__"): From 6859b3b73ba72e2dee51af291d20412d897034c8 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 20:36:51 +0100 Subject: [PATCH 315/333] Try suppressing external inheritance --- docs/conf.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index bd56154f70..c2d60f0809 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -123,19 +123,16 @@ def linkcode_resolve(*args): } -def suppress_inherited_from_external(app, what, name, obj, options, signature): - # Filter out inherited members that come from external modules - # You can add conditions based on module names or other attributes - if hasattr(obj, "__module__"): - module_name = obj.__module__ - if "external_module_name" in module_name: # replace 'external_module_name' with the actual name - return None # Suppress this member from being documented - - return return_annotation +def skip_external_inherited_member(app, what, name, obj, skip, options): + if isinstance(obj, (type,)): + # Check if the object is inherited from an external module + if "external_module_name" in obj.__module__: # adjust this condition as needed + return True # Skip this member + return skip def setup(app): - app.connect("autodoc-process-docstring", suppress_inherited_from_external) + app.connect("autodoc-skip-member", skip_external_inherited_member) smv_latest_version = importlib_metadata.version(project) From 24f4a40ae675de54ba55f3d22400663fea9c7e84 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 20:47:24 +0100 Subject: [PATCH 316/333] Try suppressing external inheritance --- docs/conf.py | 52 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c2d60f0809..7aabb47cf8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,9 @@ # noqa: INP001 import inspect +import types + import importlib_metadata -import sphinx.ext.autodoc +import torch # -- Project information ----------------------------------------------------- from sphinx_github_style import get_linkcode_resolve @@ -122,19 +124,6 @@ def linkcode_resolve(*args): "index": [], } - -def skip_external_inherited_member(app, what, name, obj, skip, options): - if isinstance(obj, (type,)): - # Check if the object is inherited from an external module - if "external_module_name" in obj.__module__: # adjust this condition as needed - return True # Skip this member - return skip - - -def setup(app): - app.connect("autodoc-skip-member", skip_external_inherited_member) - - smv_latest_version = importlib_metadata.version(project) # Whitelist pattern for tags (set to None to ignore all tags) @@ -156,3 +145,38 @@ def setup(app): smv_prefer_remote_refs = False html_favicon = "_static/favicon.ico" + + +def skip_inherited_methods_from_base_class(app, what, name, obj, skip, options): + """ + This function is triggered by Sphinx to determine whether to include or + skip a specific member in the documentation. We use this to conditionally + skip inherited methods from certain base classes. + """ + # Check if the object is a method or function and if it's inherited + if what == "method" or what == "function": + # Check if the object is a method of a class + if isinstance(obj, (types.FunctionType, types.MethodType)): + # Get the class that owns the method + if hasattr(obj, "__self__"): + cls = obj.__self__.__class__ # The class of the method + else: + return skip # Skip non-methods + + # Check if the class is inheriting from BaseClassA (or other class to include) + # if issubclass(cls, BaseClassA): + # return skip # Keep the inherited methods for BaseClassA (do not skip) + + # If the class inherits from BaseClassB or another class to exclude, skip the inherited method + if issubclass(cls, torch.nn.Module): + return True # Skip methods inherited from BaseClassB + + return skip # No changes for other cases + + +def setup(app): + """ + Connect the skip function to the `autodoc-skip-member` event. + This will call the function each time Sphinx decides whether to include a member. + """ + app.connect("autodoc-skip-member", skip_inherited_methods_from_base_class) From e72cfc67feb13e7c9eb19a06163dc82a490f4542 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 20:55:39 +0100 Subject: [PATCH 317/333] Try suppressing external inheritance --- docs/classes.rst | 8 ++++++++ docs/conf.py | 37 ------------------------------------- 2 files changed, 8 insertions(+), 37 deletions(-) create mode 100644 docs/classes.rst diff --git a/docs/classes.rst b/docs/classes.rst new file mode 100644 index 0000000000..fe92183859 --- /dev/null +++ b/docs/classes.rst @@ -0,0 +1,8 @@ +Classes Documentation +===================== + +Here we document some of the classes in the project. + +.. autoclass:: flair.nn.Model + :members: + :inherited-members: torch.nn.Module diff --git a/docs/conf.py b/docs/conf.py index 7aabb47cf8..6234c2c0fd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,7 @@ # noqa: INP001 import inspect -import types import importlib_metadata -import torch # -- Project information ----------------------------------------------------- from sphinx_github_style import get_linkcode_resolve @@ -145,38 +143,3 @@ def linkcode_resolve(*args): smv_prefer_remote_refs = False html_favicon = "_static/favicon.ico" - - -def skip_inherited_methods_from_base_class(app, what, name, obj, skip, options): - """ - This function is triggered by Sphinx to determine whether to include or - skip a specific member in the documentation. We use this to conditionally - skip inherited methods from certain base classes. - """ - # Check if the object is a method or function and if it's inherited - if what == "method" or what == "function": - # Check if the object is a method of a class - if isinstance(obj, (types.FunctionType, types.MethodType)): - # Get the class that owns the method - if hasattr(obj, "__self__"): - cls = obj.__self__.__class__ # The class of the method - else: - return skip # Skip non-methods - - # Check if the class is inheriting from BaseClassA (or other class to include) - # if issubclass(cls, BaseClassA): - # return skip # Keep the inherited methods for BaseClassA (do not skip) - - # If the class inherits from BaseClassB or another class to exclude, skip the inherited method - if issubclass(cls, torch.nn.Module): - return True # Skip methods inherited from BaseClassB - - return skip # No changes for other cases - - -def setup(app): - """ - Connect the skip function to the `autodoc-skip-member` event. - This will call the function each time Sphinx decides whether to include a member. - """ - app.connect("autodoc-skip-member", skip_inherited_methods_from_base_class) From 6fa05d67827bbe33d6cf9291ef5d8582b14c8445 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 21:16:38 +0100 Subject: [PATCH 318/333] Try suppressing external inheritance --- docs/classes.rst | 2 +- .../how-model-training-works.md | 12 +++------ .../tutorial-training/train-vs-fine-tune.md | 27 ++++++++++++++++++- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/docs/classes.rst b/docs/classes.rst index fe92183859..284dcc460f 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -5,4 +5,4 @@ Here we document some of the classes in the project. .. autoclass:: flair.nn.Model :members: - :inherited-members: torch.nn.Module + :inherited-members: Module diff --git a/docs/tutorial/tutorial-training/how-model-training-works.md b/docs/tutorial/tutorial-training/how-model-training-works.md index 9241213c98..a4b2392a7c 100644 --- a/docs/tutorial/tutorial-training/how-model-training-works.md +++ b/docs/tutorial/tutorial-training/how-model-training-works.md @@ -279,16 +279,10 @@ print(sentence.to_tagged_string()) If the model works well, it will correctly tag 'love' as a verb in this example. -## Summary +## Next -This tutorial gave you a general overview of the main steps to train a model: +Congrats, you now have a general overview of the main steps to train a model in Flair! -- load a corpus -- choose a label type -- create a label dictionary -- choose embeddings -- initialize model -- initialize trainer -- train +Next, learn about the [two main training approaches in Flair](train-vs-fine-tune.md). diff --git a/docs/tutorial/tutorial-training/train-vs-fine-tune.md b/docs/tutorial/tutorial-training/train-vs-fine-tune.md index fd45e90ea0..252cd24e7c 100644 --- a/docs/tutorial/tutorial-training/train-vs-fine-tune.md +++ b/docs/tutorial/tutorial-training/train-vs-fine-tune.md @@ -1,11 +1,36 @@ # Training vs fine-tuning There are two broad ways you train a model: The "classic" approach and the fine-tuning approach. This section -explains the differences, and the things you need to do. +explains the differences. ## Fine-Tuning +Fine-tuning is the current state-of-the-art approach. The main idea is that you take a pre-trained language model that +consists of (hundreds of) millions of trained parameters. To this language model you add a simple prediction head with +randomly initialized weights. + +Since in this case, the vast majority of parameters in the model is already trained, you only need to "fine-tune" this +model. This means: Very small learning rate (LR) and just a few epochs. You are essentially just minimally modifying +the model to adapt it to the task you want to solve. + +Most models in Flair are trained using fine-tuning. So this is likely the approach you'll want to use. + ## Training +On the other hand, you should use the classic training approach if the majority of the trainable parameters in your +model is randomly initialized. This is essentially the "old way", before fine-tuning of transformers. + +Since the majority of parameters is randomly initialized, you need to fully train the model. This means: high learning +rate and many epochs. + +Another application of classic training is for linear probing of pre-trained language models. In this scenario, you +"freeze" the weights of the language model (meaning that they cannot be changed) and add a prediction head that is +trained from scratch. So, even though a language model is involved, its parameters are not trainable. This means that +all trainable parameters in this scenario are randomly initialized, therefore necessitating the use of the classic +training approach. + +## Paper + +Our paper \ No newline at end of file From 57c7109a956c063090d4fd29124641e71b2648f5 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 21:43:26 +0100 Subject: [PATCH 319/333] Try suppressing external inheritance --- docs/classes.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/classes.rst b/docs/classes.rst index 284dcc460f..ad09d52d12 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -3,6 +3,10 @@ Classes Documentation Here we document some of the classes in the project. -.. autoclass:: flair.nn.Model +.. autoclass:: flair.nn::Model :members: - :inherited-members: Module + :inherited-members: torch.nn::Module + +.. autoclass:: flair.models::TextClassifier + :members: + :inherited-members: torch.nn::Module \ No newline at end of file From fa07cb3b5e69a7ee515ebfa0818d6207877a6700 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 21:58:30 +0100 Subject: [PATCH 320/333] Try suppressing external inheritance --- docs/classes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/classes.rst b/docs/classes.rst index ad09d52d12..2014267f4c 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -5,8 +5,8 @@ Here we document some of the classes in the project. .. autoclass:: flair.nn::Model :members: - :inherited-members: torch.nn::Module + :inherited-members: Module .. autoclass:: flair.models::TextClassifier :members: - :inherited-members: torch.nn::Module \ No newline at end of file + :inherited-members: Module \ No newline at end of file From 22cc158af018f156b35d323ae5cf065caecdb775 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 21:58:57 +0100 Subject: [PATCH 321/333] Expand tutorial 2 --- .../how-to-load-custom-dataset.md | 3 +++ .../how-to-load-prepared-dataset.md | 4 ++++ .../how-to-train-sequence-tagger.md | 3 +++ .../how-to-train-text-classifier.md | 4 ++++ .../tutorial-training/train-vs-fine-tune.md | 22 +++++++++++++++---- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/tutorial/tutorial-training/how-to-load-custom-dataset.md b/docs/tutorial/tutorial-training/how-to-load-custom-dataset.md index 1e7fadb0f1..4bd71d4340 100644 --- a/docs/tutorial/tutorial-training/how-to-load-custom-dataset.md +++ b/docs/tutorial/tutorial-training/how-to-load-custom-dataset.md @@ -159,3 +159,6 @@ example we chose `label_type='topic'` to denote that we are loading a corpus wit +## Next + +Next, learn [how to train a sequence tagger](how-to-train-sequence-tagger.md). diff --git a/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md b/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md index f5cd654ed7..b53aeef917 100644 --- a/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md +++ b/docs/tutorial/tutorial-training/how-to-load-prepared-dataset.md @@ -193,3 +193,7 @@ The following datasets are supported: | Universal Dependency Treebanks | [flair.datasets.treebanks](#flair.datasets.treebanks) | | OCR-Layout-NER | [flair.datasets.ocr](#flair.datasets.ocr) | + +## Next + +Next, learn how to load a [custom dataset](how-to-load-custom-dataset.md). \ No newline at end of file diff --git a/docs/tutorial/tutorial-training/how-to-train-sequence-tagger.md b/docs/tutorial/tutorial-training/how-to-train-sequence-tagger.md index fc9bc492b1..0f79022b70 100644 --- a/docs/tutorial/tutorial-training/how-to-train-sequence-tagger.md +++ b/docs/tutorial/tutorial-training/how-to-train-sequence-tagger.md @@ -223,3 +223,6 @@ trainer.train('resources/taggers/example-universal-pos', This gives you a multilingual model. Try experimenting with more languages! +## Next + +Next, learn [how to train a text classifier](how-to-train-text-classifier.md). diff --git a/docs/tutorial/tutorial-training/how-to-train-text-classifier.md b/docs/tutorial/tutorial-training/how-to-train-text-classifier.md index 265689c21f..b88084ff00 100644 --- a/docs/tutorial/tutorial-training/how-to-train-text-classifier.md +++ b/docs/tutorial/tutorial-training/how-to-train-text-classifier.md @@ -58,3 +58,7 @@ classifier.predict(sentence) print(sentence.labels) ``` + +## Next + +Next, learn [how to train an entity linker](how-to-train-span-classifier.md). \ No newline at end of file diff --git a/docs/tutorial/tutorial-training/train-vs-fine-tune.md b/docs/tutorial/tutorial-training/train-vs-fine-tune.md index 252cd24e7c..657a3a0aa8 100644 --- a/docs/tutorial/tutorial-training/train-vs-fine-tune.md +++ b/docs/tutorial/tutorial-training/train-vs-fine-tune.md @@ -14,23 +14,37 @@ Since in this case, the vast majority of parameters in the model is already trai model. This means: Very small learning rate (LR) and just a few epochs. You are essentially just minimally modifying the model to adapt it to the task you want to solve. -Most models in Flair are trained using fine-tuning. So this is likely the approach you'll want to use. +Use this method by calling [`ModelTrainer.fine_tune()`](#flair.trainers.ModelTrainer.fine_tune). +Since most models in Flair were trained this way, this is likely the approach you'll want to use. ## Training On the other hand, you should use the classic training approach if the majority of the trainable parameters in your -model is randomly initialized. This is essentially the "old way", before fine-tuning of transformers. - +model is randomly initialized. This can happen for instance if you freeze the model weights of the pre-trained language +model, leaving only the randomly initialited prediction head as trainable parameters. This training approach is also +referred to as "feature-based" or "probing" in some papers. + Since the majority of parameters is randomly initialized, you need to fully train the model. This means: high learning rate and many epochs. +Use this method by calling [`ModelTrainer.train()`](#flair.trainers.ModelTrainer.train) . + +```{note} Another application of classic training is for linear probing of pre-trained language models. In this scenario, you "freeze" the weights of the language model (meaning that they cannot be changed) and add a prediction head that is trained from scratch. So, even though a language model is involved, its parameters are not trainable. This means that all trainable parameters in this scenario are randomly initialized, therefore necessitating the use of the classic training approach. +``` + ## Paper -Our paper \ No newline at end of file +If you are interested in an experimental comparison of the two above-mentioned approach, check out [our paper](https://arxiv.org/pdf/2011.06993) +that compares fine-tuning to the feature-based approach. + + +## Next + +Next, learn how to load a [training dataset](how-to-load-prepared-dataset.md). \ No newline at end of file From 955587934665057c4e4e7c8b50bd0c3a77dc87ba Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 22:14:51 +0100 Subject: [PATCH 322/333] Try suppressing external inheritance --- docs/classes.rst | 12 ------------ docs/conf.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 docs/classes.rst diff --git a/docs/classes.rst b/docs/classes.rst deleted file mode 100644 index 2014267f4c..0000000000 --- a/docs/classes.rst +++ /dev/null @@ -1,12 +0,0 @@ -Classes Documentation -===================== - -Here we document some of the classes in the project. - -.. autoclass:: flair.nn::Model - :members: - :inherited-members: Module - -.. autoclass:: flair.models::TextClassifier - :members: - :inherited-members: Module \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 6234c2c0fd..4d23224f28 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -109,7 +109,7 @@ def linkcode_resolve(*args): "members": True, "show-inheritance": True, "private-members": False, - "inherited": True, + "inherited": False, } source_suffix = { From 3b2152db9d5c1896d2f88a21783c478e17568a21 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 22:39:52 +0100 Subject: [PATCH 323/333] Try suppressing external inheritance --- docs/conf.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 4d23224f28..52fda8d60a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,6 +5,7 @@ # -- Project information ----------------------------------------------------- from sphinx_github_style import get_linkcode_resolve +from torch.nn import Module version = "0.14.0" release = "0.14.0" @@ -143,3 +144,36 @@ def linkcode_resolve(*args): smv_prefer_remote_refs = False html_favicon = "_static/favicon.ico" + +import types + + +def skip_inherited_methods_from_base_class(app, what, name, obj, skip, options): + """ + This function is triggered by Sphinx to determine whether to include or + skip a specific member in the documentation. We use this to suppress inherited + methods from a particular base class. + """ + # Check if the object is a method (or function) + if what == "method" or what == "function": + # Check if the object is a method of a class + if isinstance(obj, (types.FunctionType, types.MethodType)): + # Get the class that owns the method + if hasattr(obj, "__self__"): + cls = obj.__self__.__class__ # The class of the method + else: + return skip # Skip non-methods + + # Check if the class is inheriting from BaseClassA (the base class to exclude) + if issubclass(cls, Module): + return True # Skip methods inherited from BaseClassA + + return skip # No changes for other cases + + +def setup(app): + """ + Connect the skip function to the `autodoc-skip-member` event. + This will call the function each time Sphinx decides whether to include a member. + """ + app.connect("autodoc-skip-member", skip_inherited_methods_from_base_class) From 35db2c8861aae4722e393c32aecbe0bab6f4b487 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sat, 7 Dec 2024 22:47:13 +0100 Subject: [PATCH 324/333] Try suppressing external inheritance --- docs/conf.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 52fda8d60a..2e4b431ed8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -144,36 +144,3 @@ def linkcode_resolve(*args): smv_prefer_remote_refs = False html_favicon = "_static/favicon.ico" - -import types - - -def skip_inherited_methods_from_base_class(app, what, name, obj, skip, options): - """ - This function is triggered by Sphinx to determine whether to include or - skip a specific member in the documentation. We use this to suppress inherited - methods from a particular base class. - """ - # Check if the object is a method (or function) - if what == "method" or what == "function": - # Check if the object is a method of a class - if isinstance(obj, (types.FunctionType, types.MethodType)): - # Get the class that owns the method - if hasattr(obj, "__self__"): - cls = obj.__self__.__class__ # The class of the method - else: - return skip # Skip non-methods - - # Check if the class is inheriting from BaseClassA (the base class to exclude) - if issubclass(cls, Module): - return True # Skip methods inherited from BaseClassA - - return skip # No changes for other cases - - -def setup(app): - """ - Connect the skip function to the `autodoc-skip-member` event. - This will call the function each time Sphinx decides whether to include a member. - """ - app.connect("autodoc-skip-member", skip_inherited_methods_from_base_class) From 8bf84b6c0b45f27c161d8dc14446ceb3f293bef1 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Sun, 8 Dec 2024 11:22:01 +0100 Subject: [PATCH 325/333] Try suppressing external inheritance --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 2e4b431ed8..3f5d95cb04 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -110,7 +110,7 @@ def linkcode_resolve(*args): "members": True, "show-inheritance": True, "private-members": False, - "inherited": False, + "inherited": True, } source_suffix = { From 7903d2d8fd0d2c985563dd50e571b0b008c60d75 Mon Sep 17 00:00:00 2001 From: elenamer Date: Fri, 13 Dec 2024 12:57:00 +0100 Subject: [PATCH 326/333] change to use CLEANCONLL corpus --- flair/datasets/sequence_labeling.py | 41 +++++------------------------ 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 455d3c4104..5abd0b56a3 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -5263,46 +5263,19 @@ def __init__( self.base_path / "clean.test", ] files_exist = [path.exists() for path in file_paths] - self.cleanconll_base_path = flair.cache_root / "datasets" / "cleanconll" if not all(files_exist): cached_path(f"{self.label_url}/{filename}.traindev", self.base_path / "annotations_only") cached_path(f"{self.label_url}/index.txt", self.base_path / "annotations_only") + + cleanconll_corpus = CLEANCONLL() - cleanconll_files_exist = [ - Path(f"{self.cleanconll_base_path}/cleanconll.{split}").exists() for split in ["train", "dev", "test"] - ] - if not all(cleanconll_files_exist): - # download cleanconll - - clone = f"git clone https://github.com/flairNLP/CleanCoNLL.git {self.cleanconll_base_path}/CleanCoNLL" - os.system(clone) # Cloning - cwd = os.getcwd() - - os.chdir(f"{self.cleanconll_base_path}/CleanCoNLL") - chmod = "chmod u+x create_cleanconll_from_conll03.sh" - os.system(chmod) - create = "bash create_cleanconll_from_conll03.sh" - - os.system(create) - os.chdir(cwd) - - shutil.move( - f"{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.train", - self.cleanconll_base_path, - ) - shutil.move( - f"{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.dev", self.cleanconll_base_path - ) - shutil.move( - f"{self.cleanconll_base_path}/CleanCoNLL/data/cleanconll/cleanconll.test", self.cleanconll_base_path - ) - - shutil.rmtree(self.cleanconll_base_path / "CleanCoNLL") + self.cleanconll_base_path = flair.cache_root / "datasets" / cleanconll_corpus.__class__.__name__.lower() # create dataset files from index and train/test splits self.generate_data_files( filename, + cleanconll_corpus.__class__.__name__.lower() ) super().__init__( @@ -5403,12 +5376,12 @@ def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): self.save_to_column_file(os.path.join(self.base_path, f"{corpus}.traindev"), noisy_labels) return noisy_labels - def generate_data_files(self, filename): + def generate_data_files(self, filename, origin_dataset_name): with open(os.path.join(self.base_path, "annotations_only", "index.txt")) as index_file: token_indices = index_file.readlines() - all_clean_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, "cleanconll.train")) + all_clean_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, f"{origin_dataset_name}.train")) # os.makedirs(os.path.join('data','noisebench'), exist_ok=True) @@ -5418,7 +5391,7 @@ def generate_data_files(self, filename): ) # copy test set - all_clean_test_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, "cleanconll.test")) + all_clean_test_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, f"{origin_dataset_name}.test")) test_sentences = [] for s in all_clean_test_sentences: From 1893d65c067857d375b99eeba0d2ce134f5c9b1b Mon Sep 17 00:00:00 2001 From: elenamer Date: Fri, 13 Dec 2024 15:02:11 +0100 Subject: [PATCH 327/333] raise ValueError and list supported noise types in the message --- flair/datasets/sequence_labeling.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 5abd0b56a3..f33c38a4a7 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -5251,9 +5251,12 @@ def __init__( in_memory (bool): If True the dataset is kept in memory achieving speedups in training. **corpusargs: The arguments propagated to :meth:'flair.datasets.ColumnCorpus.__init__'. """ - if noise not in ["clean", "crowd", "crowdbest", "expert", "distant", "weak", "llm"]: - raise Exception("Please choose a valid version") + VALUE_NOISE_VALUES = ["clean", "crowd", "crowdbest", "expert", "distant", "weak", "llm"] + + if noise not in VALUE_NOISE_VALUES: + raise ValueError(f"Unsupported value for noise type argument. Got {noise}, expected one of {VALUE_NOISE_VALUES}!") + self._set_path(base_path) filename = "clean" if noise == "clean" else f"noise_{noise}" From 24a28291d3f7a63fdf001aa927823d0f1a56a946 Mon Sep 17 00:00:00 2001 From: elenamer Date: Fri, 13 Dec 2024 15:58:03 +0100 Subject: [PATCH 328/333] add file encoding --- flair/datasets/sequence_labeling.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index f33c38a4a7..862ca2bf44 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -5298,7 +5298,7 @@ def _set_path(self, base_path): @staticmethod def read_column_file(filename): - with open(filename, errors="replace") as file: + with open(filename, "r", errors="replace", encoding="utf-8") as file: lines = file.readlines() all_x = [] point = [] @@ -5320,7 +5320,7 @@ def read_column_file(filename): @staticmethod def save_to_column_file(filename, list): - with open(filename, "w") as f: + with open(filename, "w", encoding="utf-8") as f: for sentence in list: for token in sentence: f.write("\t".join(token)) @@ -5381,7 +5381,7 @@ def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): def generate_data_files(self, filename, origin_dataset_name): - with open(os.path.join(self.base_path, "annotations_only", "index.txt")) as index_file: + with open(os.path.join(self.base_path, "annotations_only", "index.txt"), "r", encoding="utf-8") as index_file: token_indices = index_file.readlines() all_clean_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, f"{origin_dataset_name}.train")) From ae10a61fc4a7068f9c34a5ca2be347134eec4d17 Mon Sep 17 00:00:00 2001 From: elenamer Date: Fri, 13 Dec 2024 16:01:01 +0100 Subject: [PATCH 329/333] simplify paths --- flair/datasets/sequence_labeling.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 862ca2bf44..1be58b9a37 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -5257,7 +5257,7 @@ def __init__( if noise not in VALUE_NOISE_VALUES: raise ValueError(f"Unsupported value for noise type argument. Got {noise}, expected one of {VALUE_NOISE_VALUES}!") - self._set_path(base_path) + self.base_path = flair.cache_root / "datasets" / "noisebench" if not base_path else Path(base_path) filename = "clean" if noise == "clean" else f"noise_{noise}" file_paths = [ @@ -5293,9 +5293,6 @@ def __init__( **corpusargs, ) - def _set_path(self, base_path): - self.base_path = flair.cache_root / "datasets" / "noisebench" if not base_path else Path(base_path) - @staticmethod def read_column_file(filename): with open(filename, "r", errors="replace", encoding="utf-8") as file: @@ -5349,20 +5346,18 @@ def _create_train_dev_splits(self, filename, all_sentences=None, datestring="199 train_sentences.append(s) self.save_to_column_file( - os.sep.join(filename.split(os.sep)[:-1]) + os.sep + filename.split(os.sep)[-1].split(".")[0] + ".dev", + filename.parent / f"{filename.stem}.dev", dev_sentences, ) self.save_to_column_file( - os.sep.join(filename.split(os.sep)[:-1]) + os.sep + filename.split(os.sep)[-1].split(".")[0] + ".train", + filename.parent / f"{filename.stem}.train", train_sentences, ) def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): # generate NoiseBench dataset variants, given CleanCoNLL, noisy label files and index file - noisy_labels = self.read_column_file(os.path.join(self.base_path, "annotations_only", f"{corpus}.traindev")) - # print(noisy_labels) - # print(token_indices) + noisy_labels = self.read_column_file(self.base_path / "annotations_only" / f"{corpus}.traindev") for index, sentence in zip(token_indices, noisy_labels): if index.strip() == "docstart": @@ -5376,25 +5371,24 @@ def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): for token, label in zip(clean_sentence, sentence): label[0] = token[0] # token[0] -> text, token[1] -> BIO label if self.SAVE_TRAINDEV_FILE: - self.save_to_column_file(os.path.join(self.base_path, f"{corpus}.traindev"), noisy_labels) + self.save_to_column_file(self.base_path / f"{corpus}.traindev", noisy_labels) return noisy_labels def generate_data_files(self, filename, origin_dataset_name): - with open(os.path.join(self.base_path, "annotations_only", "index.txt"), "r", encoding="utf-8") as index_file: + with open(self.base_path / "annotations_only" / "index.txt", "r", encoding="utf-8") as index_file: token_indices = index_file.readlines() - - all_clean_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, f"{origin_dataset_name}.train")) + all_clean_sentences = self.read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.train") # os.makedirs(os.path.join('data','noisebench'), exist_ok=True) noisy_sentences = self._merge_tokens_labels(filename, all_clean_sentences, token_indices) self._create_train_dev_splits( - all_sentences=noisy_sentences, filename=os.path.join(self.base_path, f"{filename}.traindev") + all_sentences=noisy_sentences, filename=self.base_path / f"{filename}.traindev" ) # copy test set - all_clean_test_sentences = self.read_column_file(os.path.join(self.cleanconll_base_path, f"{origin_dataset_name}.test")) + all_clean_test_sentences = self.read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.test") test_sentences = [] for s in all_clean_test_sentences: @@ -5403,7 +5397,7 @@ def generate_data_files(self, filename, origin_dataset_name): new_s.append([token[0], token[4]]) test_sentences.append(new_s) - self.save_to_column_file(os.path.join(self.base_path, "clean.test"), test_sentences) + self.save_to_column_file(self.base_path / "clean.test", test_sentences) class MASAKHA_POS(MultiCorpus): From be83c9eafc997580e2dc04170562d4df5ec415af Mon Sep 17 00:00:00 2001 From: elenamer Date: Fri, 13 Dec 2024 16:18:56 +0100 Subject: [PATCH 330/333] make functions private and add type annotations --- flair/datasets/sequence_labeling.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 1be58b9a37..12da50bd49 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -5276,7 +5276,7 @@ def __init__( self.cleanconll_base_path = flair.cache_root / "datasets" / cleanconll_corpus.__class__.__name__.lower() # create dataset files from index and train/test splits - self.generate_data_files( + self._generate_data_files( filename, cleanconll_corpus.__class__.__name__.lower() ) @@ -5294,7 +5294,7 @@ def __init__( ) @staticmethod - def read_column_file(filename): + def _read_column_file(filename: Union[str, Path]) -> list[list[str]]: with open(filename, "r", errors="replace", encoding="utf-8") as file: lines = file.readlines() all_x = [] @@ -5316,7 +5316,7 @@ def read_column_file(filename): return all_x @staticmethod - def save_to_column_file(filename, list): + def _save_to_column_file(filename: Union[str, Path], list: list[list[str]]) -> None: with open(filename, "w", encoding="utf-8") as f: for sentence in list: for token in sentence: @@ -5324,9 +5324,9 @@ def save_to_column_file(filename, list): f.write("\n") f.write("\n") - def _create_train_dev_splits(self, filename, all_sentences=None, datestring="1996-08-24"): + def _create_train_dev_splits(self, filename: Path, all_sentences: list = None, datestring: str ="1996-08-24") -> None: if not all_sentences: - all_sentences = self.read_column_file(filename) + all_sentences = self._read_column_file(filename) train_sentences = [] dev_sentences = [] @@ -5345,19 +5345,19 @@ def _create_train_dev_splits(self, filename, all_sentences=None, datestring="199 else: train_sentences.append(s) - self.save_to_column_file( + self._save_to_column_file( filename.parent / f"{filename.stem}.dev", dev_sentences, ) - self.save_to_column_file( + self._save_to_column_file( filename.parent / f"{filename.stem}.train", train_sentences, ) - def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): + def _merge_tokens_labels(self, corpus: str, all_clean_sentences: list, token_indices: list) -> list[list[str]]: # generate NoiseBench dataset variants, given CleanCoNLL, noisy label files and index file - noisy_labels = self.read_column_file(self.base_path / "annotations_only" / f"{corpus}.traindev") + noisy_labels = self._read_column_file(self.base_path / "annotations_only" / f"{corpus}.traindev") for index, sentence in zip(token_indices, noisy_labels): if index.strip() == "docstart": @@ -5371,14 +5371,14 @@ def _merge_tokens_labels(self, corpus, all_clean_sentences, token_indices): for token, label in zip(clean_sentence, sentence): label[0] = token[0] # token[0] -> text, token[1] -> BIO label if self.SAVE_TRAINDEV_FILE: - self.save_to_column_file(self.base_path / f"{corpus}.traindev", noisy_labels) + self._save_to_column_file(self.base_path / f"{corpus}.traindev", noisy_labels) return noisy_labels - def generate_data_files(self, filename, origin_dataset_name): + def _generate_data_files(self, filename: Union[str, Path], origin_dataset_name: str) -> None: with open(self.base_path / "annotations_only" / "index.txt", "r", encoding="utf-8") as index_file: token_indices = index_file.readlines() - all_clean_sentences = self.read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.train") + all_clean_sentences = self._read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.train") # os.makedirs(os.path.join('data','noisebench'), exist_ok=True) @@ -5388,7 +5388,7 @@ def generate_data_files(self, filename, origin_dataset_name): ) # copy test set - all_clean_test_sentences = self.read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.test") + all_clean_test_sentences = self._read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.test") test_sentences = [] for s in all_clean_test_sentences: @@ -5397,7 +5397,7 @@ def generate_data_files(self, filename, origin_dataset_name): new_s.append([token[0], token[4]]) test_sentences.append(new_s) - self.save_to_column_file(self.base_path / "clean.test", test_sentences) + self._save_to_column_file(self.base_path / "clean.test", test_sentences) class MASAKHA_POS(MultiCorpus): From c7f4a6cd0448386252a55170719ab39fbaf46443 Mon Sep 17 00:00:00 2001 From: elenamer Date: Fri, 13 Dec 2024 16:20:42 +0100 Subject: [PATCH 331/333] rename some variables and small refactor --- flair/datasets/sequence_labeling.py | 37 +++++++++++++++-------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 12da50bd49..76c842a68e 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -5297,28 +5297,30 @@ def __init__( def _read_column_file(filename: Union[str, Path]) -> list[list[str]]: with open(filename, "r", errors="replace", encoding="utf-8") as file: lines = file.readlines() - all_x = [] - point = [] + all_sentences = [] + sentence = [] for line in lines: if "\t" in line.strip(): - stripped_line = line.strip().split("\t") if "\t" in line.strip() else line.strip().split(" ") + stripped_line = line.strip().split("\t") + else: + stripped_line = line.strip().split(" ") - point.append(stripped_line) + sentence.append(stripped_line) if line.strip() == "": - if len(point[:-1]) > 0: - all_x.append(point[:-1]) - point = [] + if len(sentence[:-1]) > 0: + all_sentences.append(sentence[:-1]) + sentence = [] - if len(point) > 0: - all_x.append(point) + if len(sentence) > 0: + all_sentences.append(sentence) - all_x = all_x - return all_x + all_sentences = all_sentences + return all_sentences @staticmethod - def _save_to_column_file(filename: Union[str, Path], list: list[list[str]]) -> None: + def _save_to_column_file(filename: Union[str, Path], sentences: list[list[str]]) -> None: with open(filename, "w", encoding="utf-8") as f: - for sentence in list: + for sentence in sentences: for token in sentence: f.write("\t".join(token)) f.write("\n") @@ -5391,11 +5393,10 @@ def _generate_data_files(self, filename: Union[str, Path], origin_dataset_name: all_clean_test_sentences = self._read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.test") test_sentences = [] - for s in all_clean_test_sentences: - new_s = [] - for token in s: - new_s.append([token[0], token[4]]) - test_sentences.append(new_s) + + for sentence in all_clean_test_sentences: + new_sentence = [[tokens[0], tokens[4]] for tokens in sentence] + test_sentences.append(new_sentence) self._save_to_column_file(self.base_path / "clean.test", test_sentences) From 4e159925f961d2475f0694cefaf1840c52f383fd Mon Sep 17 00:00:00 2001 From: elenamer Date: Fri, 13 Dec 2024 16:35:16 +0100 Subject: [PATCH 332/333] formatting and fix some typing --- flair/datasets/sequence_labeling.py | 41 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py index 76c842a68e..b2ab2f45dd 100644 --- a/flair/datasets/sequence_labeling.py +++ b/flair/datasets/sequence_labeling.py @@ -1,9 +1,9 @@ import copy +import gzip import json import logging import os import re -import gzip import shutil import tarfile import tempfile @@ -5251,12 +5251,13 @@ def __init__( in_memory (bool): If True the dataset is kept in memory achieving speedups in training. **corpusargs: The arguments propagated to :meth:'flair.datasets.ColumnCorpus.__init__'. """ - VALUE_NOISE_VALUES = ["clean", "crowd", "crowdbest", "expert", "distant", "weak", "llm"] - + if noise not in VALUE_NOISE_VALUES: - raise ValueError(f"Unsupported value for noise type argument. Got {noise}, expected one of {VALUE_NOISE_VALUES}!") - + raise ValueError( + f"Unsupported value for noise type argument. Got {noise}, expected one of {VALUE_NOISE_VALUES}!" + ) + self.base_path = flair.cache_root / "datasets" / "noisebench" if not base_path else Path(base_path) filename = "clean" if noise == "clean" else f"noise_{noise}" @@ -5270,16 +5271,13 @@ def __init__( if not all(files_exist): cached_path(f"{self.label_url}/{filename}.traindev", self.base_path / "annotations_only") cached_path(f"{self.label_url}/index.txt", self.base_path / "annotations_only") - + cleanconll_corpus = CLEANCONLL() self.cleanconll_base_path = flair.cache_root / "datasets" / cleanconll_corpus.__class__.__name__.lower() # create dataset files from index and train/test splits - self._generate_data_files( - filename, - cleanconll_corpus.__class__.__name__.lower() - ) + self._generate_data_files(filename, cleanconll_corpus.__class__.__name__.lower()) super().__init__( data_folder=self.base_path, @@ -5294,16 +5292,13 @@ def __init__( ) @staticmethod - def _read_column_file(filename: Union[str, Path]) -> list[list[str]]: - with open(filename, "r", errors="replace", encoding="utf-8") as file: + def _read_column_file(filename: Union[str, Path]) -> list[list[list[str]]]: + with open(filename, errors="replace", encoding="utf-8") as file: lines = file.readlines() all_sentences = [] sentence = [] for line in lines: - if "\t" in line.strip(): - stripped_line = line.strip().split("\t") - else: - stripped_line = line.strip().split(" ") + stripped_line = line.strip().split("\t") if "\t" in line.strip() else line.strip().split(" ") sentence.append(stripped_line) if line.strip() == "": @@ -5318,7 +5313,7 @@ def _read_column_file(filename: Union[str, Path]) -> list[list[str]]: return all_sentences @staticmethod - def _save_to_column_file(filename: Union[str, Path], sentences: list[list[str]]) -> None: + def _save_to_column_file(filename: Union[str, Path], sentences: list[list[list[str]]]) -> None: with open(filename, "w", encoding="utf-8") as f: for sentence in sentences: for token in sentence: @@ -5326,7 +5321,9 @@ def _save_to_column_file(filename: Union[str, Path], sentences: list[list[str]]) f.write("\n") f.write("\n") - def _create_train_dev_splits(self, filename: Path, all_sentences: list = None, datestring: str ="1996-08-24") -> None: + def _create_train_dev_splits( + self, filename: Path, all_sentences: Optional[list] = None, datestring: str = "1996-08-24" + ) -> None: if not all_sentences: all_sentences = self._read_column_file(filename) @@ -5356,7 +5353,9 @@ def _create_train_dev_splits(self, filename: Path, all_sentences: list = None, d train_sentences, ) - def _merge_tokens_labels(self, corpus: str, all_clean_sentences: list, token_indices: list) -> list[list[str]]: + def _merge_tokens_labels( + self, corpus: str, all_clean_sentences: list, token_indices: list + ) -> list[list[list[str]]]: # generate NoiseBench dataset variants, given CleanCoNLL, noisy label files and index file noisy_labels = self._read_column_file(self.base_path / "annotations_only" / f"{corpus}.traindev") @@ -5376,9 +5375,9 @@ def _merge_tokens_labels(self, corpus: str, all_clean_sentences: list, token_ind self._save_to_column_file(self.base_path / f"{corpus}.traindev", noisy_labels) return noisy_labels - def _generate_data_files(self, filename: Union[str, Path], origin_dataset_name: str) -> None: + def _generate_data_files(self, filename: str, origin_dataset_name: str) -> None: - with open(self.base_path / "annotations_only" / "index.txt", "r", encoding="utf-8") as index_file: + with open(self.base_path / "annotations_only" / "index.txt", encoding="utf-8") as index_file: token_indices = index_file.readlines() all_clean_sentences = self._read_column_file(self.cleanconll_base_path / f"{origin_dataset_name}.train") From 93592321c61d3736ef89043242e28499973cbec5 Mon Sep 17 00:00:00 2001 From: Alan Akbik Date: Thu, 19 Dec 2024 13:23:43 +0100 Subject: [PATCH 333/333] GH-3585: Fix DefaultClassifier deserialization --- flair/nn/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flair/nn/model.py b/flair/nn/model.py index 03834afc76..f28b5e993f 100644 --- a/flair/nn/model.py +++ b/flair/nn/model.py @@ -993,7 +993,7 @@ def _get_state_dict(self): state["locked_dropout"] = self.locked_dropout.dropout_rate state["multi_label"] = self.multi_label state["multi_label_threshold"] = self.multi_label_threshold - state["loss_weights"] = self.loss_weights + state["loss_weights"] = self.weight_dict state["train_on_gold_pairs_only"] = self.train_on_gold_pairs_only state["inverse_model"] = self.inverse_model if self._custom_decoder: