[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

ac_dc/anonymization.py

@@ -30,7 +30,7 @@ def apply_regex_anonymization(
         tag_type=tag_type,
     )
     if anonymize_condition:
-        for (ent, start, end, tag) in ner:
+        for ent, start, end, tag in ner:
             # we need to actually walk through and replace by start, end span.
             sentence = sentence.replace(ent, f" <{tag}> ")
     return sentence, ner

ac_dc/deduplicate/self_deduplicate.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # @Date       : 2022-01-08 22:39:29
 # @Author     : Chenghao Mou ([email protected])
 # @Description: Self-deduplication with `datasets`
@@ -27,8 +26,7 @@
 
 
 def main(conf: str) -> None:
-
-    with open(conf, "r") as f:
+    with open(conf) as f:
         conf = yaml.safe_load(f.read())
 
     if conf["load_from_disk"]["path"]:
@@ -201,5 +199,4 @@ def main(conf: str) -> None:
 
 
 if __name__ == "__main__":
-
     typer.run(main)

ac_dc/visualization/get_data_for_visualization.py

@@ -21,7 +21,6 @@ def __init__(
         path_kenlm_model,
         path_save_stats,
     ):
-
         self.ds = dataset
         self.num_iter = num_iter
 
@@ -166,7 +165,6 @@ def compute_stats(self):
 
 
 if __name__ == "__main__":
-
     lang_dataset_id = "en"
 
     dataset_name = "oscar"  # "TurkuNLP/register_oscar"

ac_dc/visualization/visualization.py

@@ -625,7 +625,6 @@ def filtering_of_words(self):
                 )
 
                 if display_discarded_words_by_filter:
-
                     if "len_word" in columns:
                         cond_filter = np.invert(conds_words["len_word"])
                         Visualization_for_lang.display_dataset(
@@ -698,7 +697,6 @@ def is_doc_discarded(key, score):
                     return score < key[1]
 
             if personal_doc:
-
                 st.markdown("Statistics of the document:")
 
                 for key in self.keys:

bertin/evaluation/run_glue.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding=utf-8
 # Copyright 2020 The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

bertin/evaluation/run_ner.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding=utf-8
 # Copyright 2020 The HuggingFace Team All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

bertin/mc4/mc4.py

@@ -404,7 +404,7 @@ def _generate_examples(self, filepaths):
         for filepath in filepaths:
             logger.info("generating examples from = %s", filepath)
             if filepath.endswith("jsonl"):
-                with open(filepath, "r", encoding="utf-8") as f:
+                with open(filepath, encoding="utf-8") as f:
                     for line in f:
                         if line:
                             example = json.loads(line)

bertin/run_mlm_flax.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding=utf-8
 # Copyright 2021 The HuggingFace Team All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

bertin/run_mlm_flax_stream.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding=utf-8
 # Copyright 2021 The HuggingFace Team All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -446,7 +445,7 @@ def restore_checkpoint(save_dir, state):
     args = joblib.load(os.path.join(save_dir, "training_args.joblib"))
     data_collator = joblib.load(os.path.join(save_dir, "data_collator.joblib"))
 
-    with open(os.path.join(save_dir, "training_state.json"), "r") as f:
+    with open(os.path.join(save_dir, "training_state.json")) as f:
         training_state = json.load(f)
     step = training_state["step"]
 

bertin/utils/dataset_perplexity.py

@@ -17,7 +17,7 @@ def get_perplexity(doc):
 
 
 with open("mc4-es-train-50M-stats.csv", "w") as csv:
-    with open("mc4-es-train-50M-steps.jsonl", "r") as data:
+    with open("mc4-es-train-50M-steps.jsonl") as data:
         for line in tqdm(data):
             text = json.loads(line)["text"]
             csv.write(f"{len(text.split())},{get_perplexity(text)}\n")

cc_pseudo_crawl/python_scripts/download_warc.py

@@ -143,9 +143,9 @@ def get_warcs(batch):
         existing_compressed_warcs,
     )
 
-    batch["compressed_warc"], batch["download_exception"] = [
+    batch["compressed_warc"], batch["download_exception"] = (
         list(l) for l in zip(*warcs_or_exceptions)
-    ]
+    )
     return batch
 
 

cc_pseudo_crawl/python_scripts/extract_text/extract_text_and_html_metadata.py

@@ -431,7 +431,6 @@ def main(args: PreprocessingConfig) -> None:  # Setup logging
         ]
 
     def process_file(file_name: str):
-
         logger.info(config.HF_DATASETS_CACHE)
         processing_name = (
             "-".join(args.metadata_to_include)

cc_pseudo_crawl/python_scripts/load_all_seed_ids.py

@@ -21,7 +21,7 @@ def main():
 
     seed_ids = []
     for seed_path in args.seed_paths:
-        with open(seed_path, "r") as fi:
+        with open(seed_path) as fi:
             data = csv.reader(fi)
             # First line is all the headers that we remove.
             seed_ids += [row[0] for row_id, row in enumerate(data) if row_id > 0]

cc_pseudo_crawl/python_scripts/pseudo_crawl_seed_to_lm_dset_v2.py

@@ -126,7 +126,7 @@ def process_batch(batch, skip_set):
 # looks at up to the first 10K pages for a seed and
 # records lines that appear in at least 1% of the unique pages
 def get_lines_to_skip(dset, n_records, pourcentage_threshold, min_repetition_threshold):
-    line_counts = defaultdict(lambda: 0)
+    line_counts = defaultdict(int)
     seen_pages = set()
 
     seed = SeedSequence(42)

kenlm_training/cc_net/execution.py

@@ -42,7 +42,6 @@ def get_executor(
     task_parallelism: int = -1,
     options: dict = {},
 ) -> Executor:
-
     execution_mode = execution.split(",")[0]
     options.update(
         {kv.split("=", 1)[0]: kv.split("=", 1)[1] for kv in execution.split(",")[1:]}

kenlm_training/cc_net/jsonql.py

@@ -880,8 +880,7 @@ def describe(source, columns=None, weights=None, **kwargs):
             continue
         if "." in k or k == ALL_DOCUMENTS:
             continue
-        for line in display_stats(stats, k, weights=weights, **kwargs):
-            yield line
+        yield from display_stats(stats, k, weights=weights, **kwargs)
 
 
 def shard(lines):
@@ -961,7 +960,7 @@ def open_read(filename: ReadableFileLike) -> Iterable[str]:
     if filename.suffix == ".gz":
         file: TextIO = gzip.open(filename, "rt")  # type: ignore
     else:
-        file = open(filename, "rt")
+        file = open(filename)
 
     return _close_when_exhausted(file)
 
@@ -1015,7 +1014,7 @@ def open_write(
     if filename.suffix == ".gz":
         return BlockedGzipWriter(Path(filename), mode, block_size="64M")
 
-    return open(filename, "wt")
+    return open(filename, "w")
 
 
 def parse_size(size):

kenlm_training/tests/test_jsonql.py

@@ -262,7 +262,7 @@ def do(self, x):
     def acc(values):
         print("acc: started")
         res = 0
-        for (x, _) in values:
+        for x, _ in values:
             res += int(x)
         print("acc: done")
         yield f"acc: result={res}"

pii-manager/setup.py

@@ -27,15 +27,15 @@
 
 def requirements(filename="requirements.txt"):
     """Read the requirements file"""
-    with io.open(filename, "r") as f:
+    with open(filename) as f:
         return [line.strip() for line in f if line and line[0] != "#"]
 
 
 def long_description():
     """
     Take the README and remove markdown hyperlinks
     """
-    with open("README.md", "rt", encoding="utf-8") as f:
+    with open("README.md", encoding="utf-8") as f:
         desc = f.read()
         desc = re.sub(r"^\[ ([^\]]+) \]: \s+ \S.*\n", r"", desc, flags=re.X | re.M)
         return re.sub(r"\[ ([^\]]+) \]", r"\1", desc, flags=re.X)

pii-manager/src/pii_manager/api/manager.py

@@ -31,22 +31,19 @@ def fetch_all_tasks(
     """
     taskdict = get_taskdict(debug=debug)
     # Language-independent
-    for task in taskdict[LANG_ANY].values():
-        yield task
+    yield from taskdict[LANG_ANY].values()
 
     langdict = taskdict.get(lang, {})
     # Country-independent
-    for task in langdict.get(COUNTRY_ANY, {}).values():
-        yield task
+    yield from langdict.get(COUNTRY_ANY, {}).values()
     # Country-specific
     if country:
         if country[0] in (COUNTRY_ANY, "all"):
             country = country_list(lang)
         for c in country:
             if c == COUNTRY_ANY:  # already included above
                 continue
-            for task in langdict.get(c, {}).values():
-                yield task
+            yield from langdict.get(c, {}).values()
 
 
 def fetch_task(

pii-manager/test/unit/api/test_file.py

@@ -12,7 +12,7 @@ def datafile(name: str) -> str:
 
 
 def readfile(name: str) -> str:
-    with open(name, "rt", encoding="utf-8") as f:
+    with open(name, encoding="utf-8") as f:
         return f.read().strip()
 
 

pii-manager/test/unit/api/test_file_taskfile.py

@@ -14,7 +14,7 @@ def datafile(name: str) -> str:
 
 
 def readfile(name: str) -> str:
-    with open(name, "rt", encoding="utf-8") as f:
+    with open(name, encoding="utf-8") as f:
         return f.read().strip()
 
 

pii-manager/test/unit/api/test_manager.py

@@ -21,7 +21,10 @@ def test20_info():
     info = obj.task_info()
 
     exp = {
-        (PiiEnum.CREDIT_CARD, None,): [
+        (
+            PiiEnum.CREDIT_CARD,
+            None,
+        ): [
             (
                 "credit card",
                 "Credit card numbers for most international credit cards (detect & validate)",

pii-manager/test/unit/api/test_manager_add.py

@@ -47,7 +47,7 @@ def test110_call():
     obj = PiiManager("en", None, PiiEnum.EMAIL_ADDRESS)
     obj.add_tasks([DUMMY_REGEX])
 
-    for (doc, exp) in TEST_REGEX:
+    for doc, exp in TEST_REGEX:
         got = obj(doc)
         assert got == exp
 
@@ -86,6 +86,6 @@ def test200_call():
     obj = PiiManager("en")
     obj.add_tasks([DUMMY_CLASS])
 
-    for (doc, exp) in TEST_CLASS:
+    for doc, exp in TEST_CLASS:
         got = obj(doc)
         assert got == exp

pii-manager/test/unit/api/test_manager_ctx.py

@@ -38,7 +38,7 @@ def test10_context_regex():
     """
     obj = PiiManager("en", mode="extract")
     obj.add_tasks([DUMMY_REGEX])
-    for (text, exp) in TEST:
+    for text, exp in TEST:
         got = obj(text)
         assert list(got) == exp
 
@@ -64,6 +64,6 @@ def test20_context_class():
     """
     obj = PiiManager("en", mode="extract")
     obj.add_tasks([DUMMY_CLASS])
-    for (text, exp) in TEST:
+    for text, exp in TEST:
         got = obj(text)
         assert list(got) == exp

pii-manager/test/unit/helper/test_context.py

@@ -74,7 +74,7 @@ def test10_context_true():
     """
     Check valid contexts
     """
-    for (text, context) in TEST_TRUE:
+    for text, context in TEST_TRUE:
         spec = mod.context_spec(context)
         assert mod.context_check(text, spec, 20) is True
 
@@ -83,7 +83,7 @@ def test20_context_false():
     """
     Check invalid contexts
     """
-    for (text, context) in TEST_FALSE:
+    for text, context in TEST_FALSE:
         spec = mod.context_spec(context)
         assert mod.context_check(text, spec, 20) is False
 

pii-manager/test/unit/helper/test_norm.py

@@ -8,5 +8,5 @@ def test10_normalizer():
     """
     Create base object
     """
-    for (text, exp) in TEST:
+    for text, exp in TEST:
         assert mod.normalize(text, "en", whitespace=True, lowercase=True) == exp

tokenizer/python_script/dedup_lines.py

@@ -28,6 +28,7 @@
 
 META_COLUMNS = ["meta"]
 
+
 # filter text to remove certain lines (e.g. menu items, copyright notice)
 def filter_lines(article, skip_set, used_lines):
     # TODO discuss the strip
@@ -86,8 +87,8 @@ def process_batch(batch, skip_set, used_lines, args):
 # looks at up to the first 10K pages for a seed and
 # records lines that appear in at least 1% of the unique pages
 def get_lines_to_skip(dset, n_records, pourcentage_threshold, min_repetition_threshold):
-    line_counts = defaultdict(lambda: 0)
-    seen_pages = defaultdict(lambda: 0)
+    line_counts = defaultdict(int)
+    seen_pages = defaultdict(int)
 
     seed = SeedSequence(42)
     rng = default_rng(seed)

tokenizer/python_script/ram_dedup_lines.py

@@ -78,7 +78,7 @@ def main():
         f"Filtered out {number_of_samples_before - number_of_samples_after_filtering_none} / {number_of_samples_before}"
     )
 
-    seen = defaultdict(lambda: 0)
+    seen = defaultdict(int)
 
     def remove_duplicate_lines(examples):
         new_exemples = {"text": [], "meta": []}