autorag -> external_Rag

Signed-off-by: lilacheden <[email protected]>

prepare/metrics/llm_as_judge/rag_judge.py

@@ -110,7 +110,7 @@ def get_prediction_field(metric_type):
     realization_sufffix = metric_type_to_realization[metric_type]
     template = f"templates.rag_eval.{metric_type}.{template_name}{realization_sufffix}"
     for inf_label, inference_model in inference_models_v2.items():
-        for rag_unitxt_task in ["autorag", "response_generation", "end_to_end"]:
+        for rag_unitxt_task in ["external_rag", "response_generation", "end_to_end"]:
             if (
                 rag_unitxt_task == "response_generation"
                 and metric_type == "context_relevance"
@@ -119,7 +119,7 @@ def get_prediction_field(metric_type):
 
             judge_to_generator_fields_mapping = (
                 {}
-                if rag_unitxt_task == "autorag"
+                if rag_unitxt_task == "external_rag"
                 else {"ground_truths": "reference_answers"}
             )
 

prepare/metrics/rag_answer_correctness.py

@@ -3,7 +3,7 @@
 from unitxt.operators import Copy, Rename
 from unitxt.test_utils.metrics import test_metric
 
-task_names = ["autorag", "response_generation", "end_to_end"]
+task_names = ["external_rag", "response_generation", "end_to_end"]
 base = "metrics.rag"
 default = "token_recall"
 dimension = "answer_correctness"
@@ -29,7 +29,7 @@ def add_scores_prefix_to_target(target, metric_catalog_name, dim_name):
 
 
 def get_test_pipeline_task_preprocess_steps(task):
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Rename(field_to_field={"task_data/ground_truths": "ground_truths"}),
             Rename(field_to_field={"task_data/answer": "answer"}),
@@ -83,7 +83,7 @@ def test_answer_correctness(
 
 
 def get_preprocess_steps(task):
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Copy(
                 field_to_field={
@@ -137,7 +137,7 @@ def get_preprocess_steps(task):
             overwrite=True,
         )
 
-        if new_catalog_name == default and task == "autorag":
+        if new_catalog_name == default and task == "external_rag":
             metric = MetricPipeline(
                 main_score=main_score,
                 preprocess_steps=preprocess_steps.copy(),

prepare/metrics/rag_answer_relevance.py

@@ -4,7 +4,7 @@
 )
 from unitxt.operators import Copy, ListFieldValues
 
-task_names = ["autorag", "response_generation", "end_to_end"]
+task_names = ["external_rag", "response_generation", "end_to_end"]
 base = "metrics.rag"
 
 
@@ -13,7 +13,7 @@ def get_preprocess_steps(task):
     # We have to wrap the question by a list (otherwise it will be a string),
     # because references are expected to be lists
     last_step = ListFieldValues(fields=["references"], to_field="references")
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Copy(
                 field_to_field={
@@ -55,7 +55,7 @@ def get_preprocess_steps(task):
     add_to_catalog(
         answer_reward, f"{base}.{task}.answer_relevance.answer_reward", overwrite=True
     )
-    if task == "autorag":
+    if task == "external_rag":
         add_to_catalog(answer_reward, f"{base}.{task}.answer_reward", overwrite=True)
 
     answer_token_overlap = MetricPipeline(

prepare/metrics/rag_context_correctness.py

@@ -6,7 +6,7 @@
 
 default = "mrr"
 base = "metrics.rag"
-tasks = ["autorag", "end_to_end"]
+tasks = ["external_rag", "end_to_end"]
 dimension = "context_correctness"
 
 
@@ -28,7 +28,7 @@ def add_scores_prefix_to_target(target, metric_catalog_name, dim_name):
 
 
 def get_preprocess_steps(task):
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Copy(field="context_ids", to_field="prediction"),
             Wrap(
@@ -48,7 +48,7 @@ def get_preprocess_steps(task):
 
 
 def get_test_pipeline_task_preprocess_steps(task):
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Rename(field_to_field={"task_data/context_ids": "context_ids"}),
             Rename(
@@ -85,7 +85,7 @@ def get_test_pipeline_task_preprocess_steps(task):
             metric, f"{base}.{task}.{dimension}.{new_catalog_name}", overwrite=True
         )
 
-        if new_catalog_name == default and task == "autorag":
+        if new_catalog_name == default and task == "external_rag":
             add_to_catalog(metric, f"{base}.{task}.{dimension}", overwrite=True)
 
 

prepare/metrics/rag_context_relevance.py

@@ -5,13 +5,13 @@
 from unitxt.operators import Copy
 
 base = "metrics.rag"
-tasks = ["autorag", "end_to_end"]
+tasks = ["external_rag", "end_to_end"]
 default = "perplexity_flan_t5_small"
 dimension = "context_relevance"
 
 
 def get_preprocess_steps(task):
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Copy(field="contexts", to_field="references"),
             Copy(field="question", to_field="prediction"),
@@ -45,7 +45,7 @@ def get_preprocess_steps(task):
             metric, f"{base}.{task}.{dimension}.{new_catalog_name}", overwrite=True
         )
 
-        if new_catalog_name == default and task == "autorag":
+        if new_catalog_name == default and task == "external_rag":
             metric = MetricPipeline(
                 main_score=main_score,
                 preprocess_steps=get_preprocess_steps(task).copy(),

prepare/metrics/rag_faithfulness.py

@@ -8,7 +8,7 @@
 base = "metrics.rag"
 default = "token_k_precision"
 dimension = "faithfulness"
-task_names = ["autorag", "response_generation", "end_to_end"]
+task_names = ["external_rag", "response_generation", "end_to_end"]
 
 
 def get_scores_prefix(metric_catalog_name, dim_name):
@@ -31,7 +31,7 @@ def add_scores_prefix_to_target(target, metric_catalog_name, dim_name):
 
 
 def get_preprocess_steps(task):
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Copy(
                 field_to_field={
@@ -61,7 +61,7 @@ def get_preprocess_steps(task):
 
 
 def get_test_pipeline_task_preprocess_steps(task):
-    if task == "autorag":
+    if task == "external_rag":
         return [
             Rename(field_to_field={"task_data/contexts": "contexts"}),
             Rename(field_to_field={"task_data/answer": "answer"}),
@@ -105,7 +105,7 @@ def get_test_pipeline_task_preprocess_steps(task):
             metric, f"{base}.{task}.{dimension}.{new_catalog_name}", overwrite=True
         )
 
-        if new_catalog_name == default and task == "autorag":
+        if new_catalog_name == default and task == "external_rag":
             metric = MetricPipeline(
                 main_score=main_score,
                 preprocess_steps=get_preprocess_steps(task),

prepare/metrics/rag_metrics_deprecated.py

@@ -4,7 +4,7 @@
 from unitxt.operators import Copy, ListFieldValues
 
 base = "metrics.rag"
-new_base = "metrics.rag.autorag"
+new_base = "metrics.rag.external_rag"
 
 
 def add_metric_pipeline_to_catalog(
@@ -187,7 +187,7 @@ def get_replacing_metric(depr_metric):
     postprocess_steps=[
         Copy(field="score/instance/reference_scores", to_field="score/instance/score")
     ],
-    __deprecated_msg__="This metric is deprecated. Use metrics.rag.autorag.context_relevance instead.",
+    __deprecated_msg__="This metric is deprecated. Use metrics.rag.external_rag.context_relevance instead.",
 )
 add_to_catalog(context_perplexity, "metrics.rag.context_perplexity", overwrite=True)
 

prepare/metrics/rag_recommended_metric_lists.py

@@ -63,7 +63,7 @@ def get_recommended_metrics(resources_string, rag_unitxt_task):
 
 def register_recommended_metric_lists():
     for resource_str in recommended_metrics.keys():
-        for rag_unitxt_task in ["response_generation", "end_to_end", "autorag"]:
+        for rag_unitxt_task in ["response_generation", "end_to_end", "external_rag"]:
             metrics = MetricsList(
                 get_recommended_metrics(resource_str, rag_unitxt_task)
             )

src/unitxt/catalog/metrics/rag/answer_correctness.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.token_overlap",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_correctness"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_correctness"
 }

src/unitxt/catalog/metrics/rag/answer_correctness/bert_score_recall.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.bert_score.deberta_large_mnli",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_correctness.bert_score_recall"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_correctness.bert_score_recall"
 }

src/unitxt/catalog/metrics/rag/answer_correctness/bert_score_recall_ml.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.bert_score.deberta_v3_base_mnli_xnli_ml",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_correctness.bert_score_recall_ml"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_correctness.bert_score_recall_ml"
 }

src/unitxt/catalog/metrics/rag/answer_correctness/sentence_bert_bge.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.sentence_bert.bge_large_en_1_5",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_correctness.sentence_bert_bge"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_correctness.sentence_bert_bge"
 }

src/unitxt/catalog/metrics/rag/answer_correctness/sentence_bert_mini_lm.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.sentence_bert.minilm_l12_v2",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_correctness.sentence_bert_mini_lm"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_correctness.sentence_bert_mini_lm"
 }

src/unitxt/catalog/metrics/rag/answer_correctness/token_recall.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.token_overlap",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_correctness.token_recall"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_correctness.token_recall"
 }

src/unitxt/catalog/metrics/rag/answer_relevance/token_recall.json

@@ -26,5 +26,5 @@
         }
     ],
     "metric": "metrics.token_overlap",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_relevance.token_recall"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_relevance.token_recall"
 }

src/unitxt/catalog/metrics/rag/answer_reward.json

@@ -26,5 +26,5 @@
         }
     ],
     "metric": "metrics.reward.deberta_v3_large_v2",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.answer_reward"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.answer_reward"
 }

src/unitxt/catalog/metrics/rag/context_correctness.json

@@ -15,5 +15,5 @@
         }
     ],
     "metric": "metrics.mrr",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_correctness"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_correctness"
 }

src/unitxt/catalog/metrics/rag/context_correctness/map.json

@@ -15,5 +15,5 @@
         }
     ],
     "metric": "metrics.map",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_correctness.map"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_correctness.map"
 }

src/unitxt/catalog/metrics/rag/context_correctness/mrr.json

@@ -15,5 +15,5 @@
         }
     ],
     "metric": "metrics.mrr",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_correctness.mrr"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_correctness.mrr"
 }

src/unitxt/catalog/metrics/rag/context_correctness/retrieval_at_k.json

@@ -15,5 +15,5 @@
         }
     ],
     "metric": "metrics.retrieval_at_k",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_correctness.retrieval_at_k"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_correctness.retrieval_at_k"
 }

src/unitxt/catalog/metrics/rag/context_perplexity.json

@@ -26,5 +26,5 @@
             "to_field": "score/instance/score"
         }
     ],
-    "__deprecated_msg__": "This metric is deprecated. Use metrics.rag.autorag.context_relevance instead."
+    "__deprecated_msg__": "This metric is deprecated. Use metrics.rag.external_rag.context_relevance instead."
 }

src/unitxt/catalog/metrics/rag/context_relevance.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.perplexity_q.flan_t5_small",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_relevance"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_relevance"
 }

src/unitxt/catalog/metrics/rag/context_relevance/perplexity_flan_t5_small.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.perplexity_q.flan_t5_small",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_relevance.perplexity_flan_t5_small"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_relevance.perplexity_flan_t5_small"
 }

src/unitxt/catalog/metrics/rag/context_relevance/sentence_bert_bge.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.sentence_bert.bge_large_en_1_5",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_relevance.sentence_bert_bge"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_relevance.sentence_bert_bge"
 }

src/unitxt/catalog/metrics/rag/context_relevance/sentence_bert_mini_lm.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.sentence_bert.minilm_l12_v2",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_relevance.sentence_bert_mini_lm"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_relevance.sentence_bert_mini_lm"
 }

src/unitxt/catalog/metrics/rag/context_relevance/token_precision.json

@@ -19,5 +19,5 @@
         }
     ],
     "metric": "metrics.token_overlap",
-    "__deprecated_msg__": "This metric should be replaced with metrics.rag.autorag.context_relevance.token_precision"
+    "__deprecated_msg__": "This metric should be replaced with metrics.rag.external_rag.context_relevance.token_precision"
 }

src/unitxt/catalog/metrics/rag/external_rag/recommended/cpu_only/all.json

@@ -0,0 +1,10 @@
+{
+    "__type__": "metrics_list",
+    "items": [
+        "metrics.rag.external_rag.answer_correctness.token_recall",
+        "metrics.rag.external_rag.faithfulness.token_k_precision",
+        "metrics.rag.external_rag.answer_relevance.token_recall",
+        "metrics.rag.external_rag.context_relevance.token_precision",
+        "metrics.rag.external_rag.context_correctness.mrr"
+    ]
+}