Add Russian tasks (RU-MTEB)

mteb/tasks/Classification/__init__.py

@@ -105,6 +105,13 @@
 from .ron.Moroco import *
 from .ron.RomanianReviewsSentiment import *
 from .ron.RomanianSentimentClassification import *
+from .rus.GeoreviewClassification import *
+from .rus.HeadlineClassification import *
+from .rus.InappropriatenessClassification import *
+from .rus.KinopoiskClassification import *
+from .rus.RuReviewsClassification import *
+from .rus.RuSciBenchGRNTIClassification import *
+from .rus.RuSciBenchOECDClassification import *
 from .san.SanskritShlokasClassification import *
 from .sin.SinhalaNewsClassification import *
 from .sin.SinhalaNewsSourceClassification import *

mteb/tasks/Classification/rus/GeoreviewClassification.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks import AbsTaskClassification
+
+
+class GeoreviewClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="GeoreviewClassification",
+        dataset={
+            "path": "ai-forever/georeview-classification",
+            "revision": "3765c0d1de6b7d264bc459433c45e5a75513839c",
+        },
+        description="Review classification (5-point scale) based on Yandex Georeview dataset",
+        reference="https://github.com/yandex/geo-reviews-dataset-2023",
+        type="Classification",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["rus-Cyrl"],
+        main_score="accuracy",
+        date=("2023-01-01", "2023-08-01"),
+        form=["written"],
+        domains=["Reviews"],
+        task_subtypes=["Sentiment/Hate speech"],
+        license="mit",
+        socioeconomic_status="mixed",
+        annotations_creators="derived",
+        dialect=[],
+        text_creation="found",
+        bibtex_citation="",
+        n_samples={"test": 2048},
+        avg_character_length={"test": 409.0},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.stratified_subsampling(
+            self.dataset, seed=self.seed, n_samples=2048, splits=["test"]
+        )

mteb/tasks/Classification/rus/HeadlineClassification.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks import AbsTaskClassification
+
+
+class HeadlineClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="HeadlineClassification",
+        dataset={
+            "path": "ai-forever/headline-classification",
+            "revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb",
+        },
+        description="Headline rubric classification based on the paraphraser plus dataset.",
+        reference="https://aclanthology.org/2020.ngt-1.6/",
+        type="Classification",
+        category="s2s",
+        eval_splits=["test"],
+        eval_langs=["rus-Cyrl"],
+        main_score="accuracy",
+        date=("2009-01-01", "2020-01-01"),
+        form=["written"],
+        domains=["News"],
+        task_subtypes=["Topic classification"],
+        license="mit",
+        socioeconomic_status="mixed",
+        annotations_creators="derived",
+        dialect=[],
+        text_creation="found",
+        bibtex_citation="""@inproceedings{gudkov-etal-2020-automatically,
+        title = "Automatically Ranked {R}ussian Paraphrase Corpus for Text Generation",
+        author = "Gudkov, Vadim  and
+        Mitrofanova, Olga  and
+        Filippskikh, Elizaveta",
+        editor = "Birch, Alexandra  and
+        Finch, Andrew  and
+        Hayashi, Hiroaki  and
+        Heafield, Kenneth  and
+        Junczys-Dowmunt, Marcin  and
+        Konstas, Ioannis  and
+        Li, Xian  and
+        Neubig, Graham  and
+        Oda, Yusuke",
+        booktitle = "Proceedings of the Fourth Workshop on Neural Generation and Translation",
+        month = jul,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.ngt-1.6",
+        doi = "10.18653/v1/2020.ngt-1.6",
+        pages = "54--59",
+        abstract = "The article is focused on automatic development and ranking of a large corpus for Russian paraphrase generation which proves to be the first corpus of such type in Russian computational linguistics. Existing manually annotated paraphrase datasets for Russian are limited to small-sized ParaPhraser corpus and ParaPlag which are suitable for a set of NLP tasks, such as paraphrase and plagiarism detection, sentence similarity and relatedness estimation, etc. Due to size restrictions, these datasets can hardly be applied in end-to-end text generation solutions. Meanwhile, paraphrase generation requires a large amount of training data. In our study we propose a solution to the problem: we collect, rank and evaluate a new publicly available headline paraphrase corpus (ParaPhraser Plus), and then perform text generation experiments with manual evaluation on automatically ranked corpora using the Universal Transformer architecture.",
+        }""",
+        n_samples={"test": 2048},
+        avg_character_length={"test": 61.6},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.stratified_subsampling(
+            self.dataset, seed=self.seed, n_samples=2048, splits=["test"]
+        )

mteb/tasks/Classification/rus/InappropriatenessClassification.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks import AbsTaskClassification
+
+
+class InappropriatenessClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="InappropriatenessClassification",
+        dataset={
+            "path": "ai-forever/inappropriateness-classification",
+            "revision": "601651fdc45ef243751676e62dd7a19f491c0285",
+        },
+        description="Inappropriateness identification in the form of binary classification",
+        reference="https://aclanthology.org/2021.bsnlp-1.4",
+        type="Classification",
+        category="s2s",
+        eval_splits=["test"],
+        eval_langs=["rus-Cyrl"],
+        main_score="accuracy",
+        date=("2006-01-01", "2021-04-01"),
+        form=["written"],
+        domains=["Web", "Social"],
+        task_subtypes=["Sentiment/Hate speech"],
+        license="cc-by-nc-sa-4.0",
+        socioeconomic_status="mixed",
+        annotations_creators="human-annotated",
+        dialect=[],
+        text_creation="found",
+        bibtex_citation="""@inproceedings{babakov-etal-2021-detecting,
+        title = "Detecting Inappropriate Messages on Sensitive Topics that Could Harm a Company{'}s Reputation",
+        author = "Babakov, Nikolay  and
+        Logacheva, Varvara  and
+        Kozlova, Olga  and
+        Semenov, Nikita  and
+        Panchenko, Alexander",
+        editor = "Babych, Bogdan  and
+        Kanishcheva, Olga  and
+        Nakov, Preslav  and
+        Piskorski, Jakub  and
+        Pivovarova, Lidia  and
+        Starko, Vasyl  and
+        Steinberger, Josef  and
+        Yangarber, Roman  and
+        Marci{\'n}czuk, Micha{\l}  and
+        Pollak, Senja  and
+        P{\v{r}}ib{\'a}{\v{n}}, Pavel  and
+        Robnik-{\v{S}}ikonja, Marko",
+        booktitle = "Proceedings of the 8th Workshop on Balto-Slavic Natural Language Processing",
+        month = apr,
+        year = "2021",
+        address = "Kiyv, Ukraine",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2021.bsnlp-1.4",
+        pages = "26--36",
+        abstract = "Not all topics are equally {``}flammable{''} in terms of toxicity: a calm discussion of turtles or fishing less often fuels inappropriate toxic dialogues than a discussion of politics or sexual minorities. We define a set of sensitive topics that can yield inappropriate and toxic messages and describe the methodology of collecting and labelling a dataset for appropriateness. While toxicity in user-generated data is well-studied, we aim at defining a more fine-grained notion of inappropriateness. The core of inappropriateness is that it can harm the reputation of a speaker. This is different from toxicity in two respects: (i) inappropriateness is topic-related, and (ii) inappropriate message is not toxic but still unacceptable. We collect and release two datasets for Russian: a topic-labelled dataset and an appropriateness-labelled dataset. We also release pre-trained classification models trained on this data.",
+        }""",
+        n_samples={"test": 2048},
+        avg_character_length={"test": 97.7},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.stratified_subsampling(
+            self.dataset, seed=self.seed, n_samples=2048, splits=["test"]
+        )

mteb/tasks/Classification/rus/KinopoiskClassification.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks import AbsTaskClassification
+
+
+class KinopoiskClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="KinopoiskClassification",
+        dataset={
+            "path": "ai-forever/kinopoisk-sentiment-classification",
+            "revision": "5911f26666ac11af46cb9c6849d0dc80a378af24",
+        },
+        description="Kinopoisk review sentiment classification",
+        reference="https://www.dialog-21.ru/media/1226/blinovpd.pdf",
+        type="Classification",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["rus-Cyrl"],
+        main_score="accuracy",
+        date=("2004-07-01", "2012-12-01"),
+        form=["written"],
+        domains=["Reviews"],
+        task_subtypes=["Sentiment/Hate speech"],
+        license="Not specified",
+        socioeconomic_status="mixed",
+        annotations_creators="derived",
+        dialect=[],
+        text_creation="found",
+        bibtex_citation="""@article{blinov2013research,
+        title={Research of lexical approach and machine learning methods for sentiment analysis},
+        author={Blinov, PD and Klekovkina, Maria and Kotelnikov, Eugeny and Pestov, Oleg},
+        journal={Computational Linguistics and Intellectual Technologies},
+        volume={2},
+        number={12},
+        pages={48--58},
+        year={2013}
+        }""",
+        n_samples={"test": 1500},
+        avg_character_length={"test": 1897.3},
+    )

mteb/tasks/Classification/rus/RuReviewsClassification.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks import AbsTaskClassification
+
+
+class RuReviewsClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="RuReviewsClassification",
+        dataset={
+            "path": "ai-forever/ru-reviews-classification",
+            "revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a",
+        },
+        description="Product review classification (3-point scale) based on RuRevies dataset",
+        reference="https://github.com/sismetanin/rureviews",
+        type="Classification",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["rus-Cyrl"],
+        main_score="accuracy",
+        date=("2000-01-01", "2020-01-01"),
+        form=["written"],
+        domains=["Reviews"],
+        task_subtypes=["Sentiment/Hate speech"],
+        license="apache-2.0",
+        socioeconomic_status="mixed",
+        annotations_creators="derived",
+        dialect=[],
+        text_creation="found",
+        bibtex_citation="""@INPROCEEDINGS{Smetanin-SA-2019,
+        author={Sergey Smetanin and Michail Komarov},
+        booktitle={2019 IEEE 21st Conference on Business Informatics (CBI)},
+        title={Sentiment Analysis of Product Reviews in Russian using Convolutional Neural Networks},
+        year={2019},
+        volume={01},
+        number={},
+        pages={482-486},
+        doi={10.1109/CBI.2019.00062},
+        ISSN={2378-1963},
+        month={July}
+        }""",
+        n_samples={"test": 2048},
+        avg_character_length={"test": 133.2},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.stratified_subsampling(
+            self.dataset, seed=self.seed, n_samples=2048, splits=["test"]
+        )

mteb/tasks/Classification/rus/RuSciBenchGRNTIClassification.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks import AbsTaskClassification
+
+
+class RuSciBenchGRNTIClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="RuSciBenchGRNTIClassification",
+        dataset={
+            "path": "ai-forever/ru-scibench-grnti-classification",
+            "revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1",
+        },
+        description="Classification of scientific papers (title+abstract) by rubric",
+        reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench/",
+        type="Classification",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["rus-Cyrl"],
+        main_score="accuracy",
+        date=("1999-01-01", "2024-01-01"),
+        form=["written"],
+        domains=["Academic"],
+        task_subtypes=["Topic classification"],
+        license="Not specified",
+        socioeconomic_status="mixed",
+        annotations_creators="derived",
+        dialect=[],
+        text_creation="found",
+        bibtex_citation="",
+        n_samples={"test": 2048},
+        avg_character_length={"test": 890.1},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.stratified_subsampling(
+            self.dataset, seed=self.seed, n_samples=2048, splits=["test"]
+        )

mteb/tasks/Classification/rus/RuSciBenchOECDClassification.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks import AbsTaskClassification
+
+
+class RuSciBenchOECDClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="RuSciBenchOECDClassification",
+        dataset={
+            "path": "ai-forever/ru-scibench-oecd-classification",
+            "revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471",
+        },
+        description="Classification of scientific papers (title+abstract) by rubric",
+        reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench/",
+        type="Classification",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["rus-Cyrl"],
+        main_score="accuracy",
+        date=("1999-01-01", "2024-01-01"),
+        form=["written"],
+        domains=["Academic"],
+        task_subtypes=["Topic classification"],
+        license="Not specified",
+        socioeconomic_status="mixed",
+        annotations_creators="derived",
+        dialect=[],
+        text_creation="found",
+        bibtex_citation="",
+        n_samples={"test": 2048},
+        avg_character_length={"test": 838.9},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.stratified_subsampling(
+            self.dataset, seed=self.seed, n_samples=2048, splits=["test"]
+        )

mteb/tasks/Clustering/__init__.py

@@ -41,6 +41,9 @@
 from .nob.VGHierarchicalClustering import *
 from .pol.PolishClustering import *
 from .rom.RomaniBibleClustering import *
+from .rus.GeoreviewClusteringP2P import *
+from .rus.RuSciBenchGRNTIClusteringP2P import *
+from .rus.RuSciBenchOECDClusteringP2P import *
 from .spa.SpanishNewsClusteringP2P import *
 from .swe.swedn_clustering import *
 from .swe.SwednClustering import *