From 9c462e2a20ae1a2c2ffc6d52e33d6067b153ee6e Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Thu, 16 Nov 2023 19:45:47 +0100 Subject: [PATCH 01/31] inital hallucination evaluation model --- .../vectara_hallucination_evaluation.py | 60 +++++++++++++++++++ tests/test_hallucination_evaluation.py | 18 ++++++ 2 files changed, 78 insertions(+) create mode 100644 deepeval/metrics/vectara_hallucination_evaluation.py create mode 100644 tests/test_hallucination_evaluation.py diff --git a/deepeval/metrics/vectara_hallucination_evaluation.py b/deepeval/metrics/vectara_hallucination_evaluation.py new file mode 100644 index 000000000..0b0ee0101 --- /dev/null +++ b/deepeval/metrics/vectara_hallucination_evaluation.py @@ -0,0 +1,60 @@ + +import os +from deepeval.singleton import Singleton +from deepeval.test_case import LLMTestCase +from deepeval.utils import chunk_text, softmax +from deepeval.metrics.base_metric import BaseMetric +from deepeval.evaluator import assert_test +from deepeval.progress_context import progress_context +from sentence_transformers import CrossEncoder + +class VectaraHallucinationEvaluationModel(metaclass=Singleton): + def __init__(self, model_name: str = "vectara/hallucination_evaluation_model"): + # We use a smple cross encoder model + os.environ["TOKENIZERS_PARALLELISM"] = "false" + self.model = CrossEncoder(model_name) + + def predict(self, text_a: str, text_b: str): + scores = self.model.predict([[text_a, text_b]]) + # https://huggingface.co/vectara/hallucination_evaluation_model + + return scores[0] + +class HallucinationEvaluationMetric(BaseMetric, metaclass=Singleton): + def __init__(self, + minimum_score: float = 0.5, + model_name: str = "vectara/hallucination_evaluation_model", + ): + # For Crossencoder model, move to singleton to avoid re-instantiating + + with progress_context( + "Downloading HallucinationEvaluationModel (may take up to 2 minutes if running for the first time)..." + ): + self.model = VectaraHallucinationEvaluationModel(model_name) + self.minimum_score = minimum_score + + def measure(self, test_case: LLMTestCase): + if test_case.actual_output is None or test_case.context is None: + raise ValueError("Output or context cannot be None") + + context_list = [] + if isinstance(test_case.context, str): + context_list.extend(chunk_text(test_case.context)) + elif isinstance(test_case.context, list): + for context in test_case.context: + context_list.extend(chunk_text(context)) + else: + raise ValueError("Context must be a string or a list of strings") + + max_score = 0 + for c in context_list: + score = self.model.predict(c, test_case.actual_output) + if score > max_score: + max_score = score + + self.success = max_score > self.minimum_score + self.score = max_score + return max_score + + def is_successful(self) -> bool: + return self.success \ No newline at end of file diff --git a/tests/test_hallucination_evaluation.py b/tests/test_hallucination_evaluation.py new file mode 100644 index 000000000..776ea9130 --- /dev/null +++ b/tests/test_hallucination_evaluation.py @@ -0,0 +1,18 @@ +import pytest +from deepeval.test_case import LLMTestCase +from deepeval.metrics.vectara_hallucination_evaluation import ( + HallucinationEvaluationMetric, +) +from deepeval.evaluator import assert_test + + +def test_hallucination_evaluation_metric(): + metric = HallucinationEvaluationMetric(minimum_score=0.5) + test_case = LLMTestCase( + input="placeholder", + actual_output="A person on a horse jumps over a broken down airplane.", + context=[ + "A person is at a diner, ordering an omelette." + ], + ) + assert_test(test_case, [metric]) \ No newline at end of file From f259ee431d0b13b70e9db6571ef3284910c34c56 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Thu, 16 Nov 2023 20:01:15 +0100 Subject: [PATCH 02/31] fixed linting --- .../metrics/vectara_hallucination_evaluation.py | 16 ++++++++++------ tests/test_hallucination_evaluation.py | 6 ++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/deepeval/metrics/vectara_hallucination_evaluation.py b/deepeval/metrics/vectara_hallucination_evaluation.py index 0b0ee0101..aaf78e5e1 100644 --- a/deepeval/metrics/vectara_hallucination_evaluation.py +++ b/deepeval/metrics/vectara_hallucination_evaluation.py @@ -1,4 +1,3 @@ - import os from deepeval.singleton import Singleton from deepeval.test_case import LLMTestCase @@ -8,8 +7,11 @@ from deepeval.progress_context import progress_context from sentence_transformers import CrossEncoder + class VectaraHallucinationEvaluationModel(metaclass=Singleton): - def __init__(self, model_name: str = "vectara/hallucination_evaluation_model"): + def __init__( + self, model_name: str = "vectara/hallucination_evaluation_model" + ): # We use a smple cross encoder model os.environ["TOKENIZERS_PARALLELISM"] = "false" self.model = CrossEncoder(model_name) @@ -17,11 +19,13 @@ def __init__(self, model_name: str = "vectara/hallucination_evaluation_model"): def predict(self, text_a: str, text_b: str): scores = self.model.predict([[text_a, text_b]]) # https://huggingface.co/vectara/hallucination_evaluation_model - + return scores[0] + class HallucinationEvaluationMetric(BaseMetric, metaclass=Singleton): - def __init__(self, + def __init__( + self, minimum_score: float = 0.5, model_name: str = "vectara/hallucination_evaluation_model", ): @@ -55,6 +59,6 @@ def measure(self, test_case: LLMTestCase): self.success = max_score > self.minimum_score self.score = max_score return max_score - + def is_successful(self) -> bool: - return self.success \ No newline at end of file + return self.success diff --git a/tests/test_hallucination_evaluation.py b/tests/test_hallucination_evaluation.py index 776ea9130..e8c337641 100644 --- a/tests/test_hallucination_evaluation.py +++ b/tests/test_hallucination_evaluation.py @@ -11,8 +11,6 @@ def test_hallucination_evaluation_metric(): test_case = LLMTestCase( input="placeholder", actual_output="A person on a horse jumps over a broken down airplane.", - context=[ - "A person is at a diner, ordering an omelette." - ], + context=["A person is at a diner, ordering an omelette."], ) - assert_test(test_case, [metric]) \ No newline at end of file + assert_test(test_case, [metric]) From 364bd04ae981c95dacc27b29a694572728175374 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Thu, 16 Nov 2023 20:23:30 +0100 Subject: [PATCH 03/31] test case changed to pass the test --- tests/test_hallucination_evaluation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_hallucination_evaluation.py b/tests/test_hallucination_evaluation.py index e8c337641..6d25a21e5 100644 --- a/tests/test_hallucination_evaluation.py +++ b/tests/test_hallucination_evaluation.py @@ -10,7 +10,7 @@ def test_hallucination_evaluation_metric(): metric = HallucinationEvaluationMetric(minimum_score=0.5) test_case = LLMTestCase( input="placeholder", - actual_output="A person on a horse jumps over a broken down airplane.", - context=["A person is at a diner, ordering an omelette."], + actual_output="A blond drinking water in public.", + context=["A man with blond-hair, and a brown shirt drinking out of a public water fountain."], ) assert_test(test_case, [metric]) From f5daf56394f9ffe1d8acc57ca82d7d07e36749c9 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Fri, 17 Nov 2023 10:19:54 +0100 Subject: [PATCH 04/31] fixed the linting issues --- tests/test_hallucination_evaluation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_hallucination_evaluation.py b/tests/test_hallucination_evaluation.py index 6d25a21e5..1b1a578a3 100644 --- a/tests/test_hallucination_evaluation.py +++ b/tests/test_hallucination_evaluation.py @@ -11,6 +11,8 @@ def test_hallucination_evaluation_metric(): test_case = LLMTestCase( input="placeholder", actual_output="A blond drinking water in public.", - context=["A man with blond-hair, and a brown shirt drinking out of a public water fountain."], + context=[ + "A man with blond-hair, and a brown shirt drinking out of a public water fountain." + ], ) assert_test(test_case, [metric]) From 90ac2382565dd385cfe4371ee171801c579b368b Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Sat, 18 Nov 2023 14:07:13 +0100 Subject: [PATCH 05/31] Hallucination score and some refactoring of scoring --- ...luation.py => hallucination_evaluation.py} | 4 +- deepeval/metrics/scoring.py | 28 +++++++- deepeval/models/__init__.py | 0 deepeval/models/hallucination_model.py | 19 ++++++ deepeval/models/model_map.py | 65 +++++++++++++++++++ .../summac_model.py} | 63 +----------------- tests/test_scoring.py | 6 ++ 7 files changed, 120 insertions(+), 65 deletions(-) rename deepeval/metrics/{vectara_hallucination_evaluation.py => hallucination_evaluation.py} (94%) create mode 100644 deepeval/models/__init__.py create mode 100644 deepeval/models/hallucination_model.py create mode 100644 deepeval/models/model_map.py rename deepeval/{metrics/_summac_model.py => models/summac_model.py} (91%) diff --git a/deepeval/metrics/vectara_hallucination_evaluation.py b/deepeval/metrics/hallucination_evaluation.py similarity index 94% rename from deepeval/metrics/vectara_hallucination_evaluation.py rename to deepeval/metrics/hallucination_evaluation.py index aaf78e5e1..bd706294b 100644 --- a/deepeval/metrics/vectara_hallucination_evaluation.py +++ b/deepeval/metrics/hallucination_evaluation.py @@ -8,7 +8,7 @@ from sentence_transformers import CrossEncoder -class VectaraHallucinationEvaluationModel(metaclass=Singleton): +class HallucinationEvaluationModel(metaclass=Singleton): def __init__( self, model_name: str = "vectara/hallucination_evaluation_model" ): @@ -23,7 +23,7 @@ def predict(self, text_a: str, text_b: str): return scores[0] -class HallucinationEvaluationMetric(BaseMetric, metaclass=Singleton): +class HallucinationMetric(BaseMetric, metaclass=Singleton): def __init__( self, minimum_score: float = 0.5, diff --git a/deepeval/metrics/scoring.py b/deepeval/metrics/scoring.py index 65f83fac6..0d0696626 100644 --- a/deepeval/metrics/scoring.py +++ b/deepeval/metrics/scoring.py @@ -4,7 +4,7 @@ from nltk.translate.bleu_score import sentence_bleu from typing import Union, List, Optional, Any from deepeval.utils import normalize_text -from deepeval.metrics._summac_model import SummaCZS +from deepeval.models.summac_model import SummaCZS class Scorer: @@ -200,7 +200,33 @@ def faithfulness_score( device=device, ) return scorer.score_one(target, prediction)["score"] + + @classmethod + def hallucination_score(cls, source: str, prediction: str, model: Optional[str] = None) -> float: + """Calculate the hallucination score of a prediction compared to a source text. + + This method computes a hallucination score, which measures the extent to which a generated prediction contains hallucinations. + The score is based on the Vectara Hallucination Evaluation Model. + Args: + source (str): The source document where the information is summarized from. + prediction (str): The generated summary that is validated against the source summary. + + Returns: + float: The computed hallucination score. Lower values indicate greater hallucination. + """ + try: + from deepeval.models.hallucination_model import ( + HallucinationModel, + ) + except ImportError as e: + print(e) + model = "vectara-hallucination" if model is None else model + + scorer = HallucinationModel(model_name=model) + + return scorer.score(source, prediction) + @classmethod def PII_score( cls, target: str, prediction: str, model: Optional[Any] = None diff --git a/deepeval/models/__init__.py b/deepeval/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deepeval/models/hallucination_model.py b/deepeval/models/hallucination_model.py new file mode 100644 index 000000000..3c7f53cfc --- /dev/null +++ b/deepeval/models/hallucination_model.py @@ -0,0 +1,19 @@ +import os +from deepeval.singleton import Singleton +from sentence_transformers import CrossEncoder +from deepeval.models.model_map import model_map, name_to_card + +class HallucinationModel(metaclass=Singleton): + def __init__( + self, model_name: str = "vectara-hallucination" + ): + # We use a smple cross encoder model + os.environ["TOKENIZERS_PARALLELISM"] = "false" + model_name = name_to_card(model_name) + self.model = CrossEncoder(model_name) + + def score(self, source: str, prediction: str): + scores = self.model.predict([[source, prediction]]) + # https://huggingface.co/vectara/hallucination_evaluation_model + + return scores[0] \ No newline at end of file diff --git a/deepeval/models/model_map.py b/deepeval/models/model_map.py new file mode 100644 index 000000000..9828dd5c1 --- /dev/null +++ b/deepeval/models/model_map.py @@ -0,0 +1,65 @@ +model_map = { + "snli-base": { + "model_card": "boychaboy/SNLI_roberta-base", + "entailment_idx": 0, + "contradiction_idx": 2, + }, + "snli-large": { + "model_card": "boychaboy/SNLI_roberta-large", + "entailment_idx": 0, + "contradiction_idx": 2, + }, + "mnli-base": { + "model_card": "microsoft/deberta-base-mnli", + "entailment_idx": 2, + "contradiction_idx": 0, + }, + "mnli": { + "model_card": "roberta-large-mnli", + "entailment_idx": 2, + "contradiction_idx": 0, + }, + "anli": { + "model_card": "ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli", + "entailment_idx": 0, + "contradiction_idx": 2, + }, + "vitc-base": { + "model_card": "tals/albert-base-vitaminc-mnli", + "entailment_idx": 0, + "contradiction_idx": 1, + }, + "vitc": { + "model_card": "tals/albert-xlarge-vitaminc-mnli", + "entailment_idx": 0, + "contradiction_idx": 1, + }, + "vitc-only": { + "model_card": "tals/albert-xlarge-vitaminc", + "entailment_idx": 0, + "contradiction_idx": 1, + }, + # "decomp": 0, + "vectara-hallucination": { + "model_card": "vectara/hallucination_evaluation_model", + "entailment_idx": None, + "contradiction_idx": None, + } +} + + +def card_to_name(card): + card2name = {v["model_card"]: k for k, v in model_map.items()} + if card in card2name: + return card2name[card] + return card + + +def name_to_card(name): + if name in model_map: + return model_map[name]["model_card"] + return name + + +def get_neutral_idx(ent_idx, con_idx): + return list(set([0, 1, 2]) - set([ent_idx, con_idx]))[0] \ No newline at end of file diff --git a/deepeval/metrics/_summac_model.py b/deepeval/models/summac_model.py similarity index 91% rename from deepeval/metrics/_summac_model.py rename to deepeval/models/summac_model.py index 1f58fc962..f3b67560f 100644 --- a/deepeval/metrics/_summac_model.py +++ b/deepeval/models/summac_model.py @@ -10,68 +10,7 @@ import os import json from deepeval import utils as utils_misc - -model_map = { - "snli-base": { - "model_card": "boychaboy/SNLI_roberta-base", - "entailment_idx": 0, - "contradiction_idx": 2, - }, - "snli-large": { - "model_card": "boychaboy/SNLI_roberta-large", - "entailment_idx": 0, - "contradiction_idx": 2, - }, - "mnli-base": { - "model_card": "microsoft/deberta-base-mnli", - "entailment_idx": 2, - "contradiction_idx": 0, - }, - "mnli": { - "model_card": "roberta-large-mnli", - "entailment_idx": 2, - "contradiction_idx": 0, - }, - "anli": { - "model_card": "ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli", - "entailment_idx": 0, - "contradiction_idx": 2, - }, - "vitc-base": { - "model_card": "tals/albert-base-vitaminc-mnli", - "entailment_idx": 0, - "contradiction_idx": 1, - }, - "vitc": { - "model_card": "tals/albert-xlarge-vitaminc-mnli", - "entailment_idx": 0, - "contradiction_idx": 1, - }, - "vitc-only": { - "model_card": "tals/albert-xlarge-vitaminc", - "entailment_idx": 0, - "contradiction_idx": 1, - }, - # "decomp": 0, -} - - -def card_to_name(card): - card2name = {v["model_card"]: k for k, v in model_map.items()} - if card in card2name: - return card2name[card] - return card - - -def name_to_card(name): - if name in model_map: - return model_map[name]["model_card"] - return name - - -def get_neutral_idx(ent_idx, con_idx): - return list(set([0, 1, 2]) - set([ent_idx, con_idx]))[0] - +from deepeval.models.model_map import name_to_card, get_neutral_idx, model_map class SummaCImager: def __init__( diff --git a/tests/test_scoring.py b/tests/test_scoring.py index a381eb90b..f1bc6f332 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -215,3 +215,9 @@ def test_neural_toxic_score_invalid_model(self): prediction = "This is a non-toxic text." with self.assertRaises(AssertionError): Scorer.neural_toxic_score(prediction, model="invalid_model") + + def test_hallucination_score(self): + prediction="A blond drinking water in public." + source = "A man with blond-hair, and a brown shirt drinking out of a public water fountain." + score = Scorer.hallucination_score(source, prediction) + self.assertTrue(0 <= score <= 1) \ No newline at end of file From c29fc2ad9b0684ee04ab6dbb927218210bc9c668 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Sat, 18 Nov 2023 14:09:00 +0100 Subject: [PATCH 06/31] linting fixed --- deepeval/metrics/scoring.py | 12 +++++++----- deepeval/models/hallucination_model.py | 7 +++---- deepeval/models/model_map.py | 4 ++-- deepeval/models/summac_model.py | 1 + tests/test_scoring.py | 8 ++++---- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/deepeval/metrics/scoring.py b/deepeval/metrics/scoring.py index 0d0696626..0ac1678e0 100644 --- a/deepeval/metrics/scoring.py +++ b/deepeval/metrics/scoring.py @@ -200,9 +200,11 @@ def faithfulness_score( device=device, ) return scorer.score_one(target, prediction)["score"] - + @classmethod - def hallucination_score(cls, source: str, prediction: str, model: Optional[str] = None) -> float: + def hallucination_score( + cls, source: str, prediction: str, model: Optional[str] = None + ) -> float: """Calculate the hallucination score of a prediction compared to a source text. This method computes a hallucination score, which measures the extent to which a generated prediction contains hallucinations. @@ -222,11 +224,11 @@ def hallucination_score(cls, source: str, prediction: str, model: Optional[str] except ImportError as e: print(e) model = "vectara-hallucination" if model is None else model - + scorer = HallucinationModel(model_name=model) - + return scorer.score(source, prediction) - + @classmethod def PII_score( cls, target: str, prediction: str, model: Optional[Any] = None diff --git a/deepeval/models/hallucination_model.py b/deepeval/models/hallucination_model.py index 3c7f53cfc..cd87e4c3c 100644 --- a/deepeval/models/hallucination_model.py +++ b/deepeval/models/hallucination_model.py @@ -3,10 +3,9 @@ from sentence_transformers import CrossEncoder from deepeval.models.model_map import model_map, name_to_card + class HallucinationModel(metaclass=Singleton): - def __init__( - self, model_name: str = "vectara-hallucination" - ): + def __init__(self, model_name: str = "vectara-hallucination"): # We use a smple cross encoder model os.environ["TOKENIZERS_PARALLELISM"] = "false" model_name = name_to_card(model_name) @@ -16,4 +15,4 @@ def score(self, source: str, prediction: str): scores = self.model.predict([[source, prediction]]) # https://huggingface.co/vectara/hallucination_evaluation_model - return scores[0] \ No newline at end of file + return scores[0] diff --git a/deepeval/models/model_map.py b/deepeval/models/model_map.py index 9828dd5c1..1b4058530 100644 --- a/deepeval/models/model_map.py +++ b/deepeval/models/model_map.py @@ -44,7 +44,7 @@ "model_card": "vectara/hallucination_evaluation_model", "entailment_idx": None, "contradiction_idx": None, - } + }, } @@ -62,4 +62,4 @@ def name_to_card(name): def get_neutral_idx(ent_idx, con_idx): - return list(set([0, 1, 2]) - set([ent_idx, con_idx]))[0] \ No newline at end of file + return list(set([0, 1, 2]) - set([ent_idx, con_idx]))[0] diff --git a/deepeval/models/summac_model.py b/deepeval/models/summac_model.py index f3b67560f..b5e887873 100644 --- a/deepeval/models/summac_model.py +++ b/deepeval/models/summac_model.py @@ -12,6 +12,7 @@ from deepeval import utils as utils_misc from deepeval.models.model_map import name_to_card, get_neutral_idx, model_map + class SummaCImager: def __init__( self, diff --git a/tests/test_scoring.py b/tests/test_scoring.py index f1bc6f332..160e0761b 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -215,9 +215,9 @@ def test_neural_toxic_score_invalid_model(self): prediction = "This is a non-toxic text." with self.assertRaises(AssertionError): Scorer.neural_toxic_score(prediction, model="invalid_model") - - def test_hallucination_score(self): - prediction="A blond drinking water in public." + + def test_hallucination_score(self): + prediction = "A blond drinking water in public." source = "A man with blond-hair, and a brown shirt drinking out of a public water fountain." score = Scorer.hallucination_score(source, prediction) - self.assertTrue(0 <= score <= 1) \ No newline at end of file + self.assertTrue(0 <= score <= 1) From e11af999b48f21dd50115b7bd789f660b44930b0 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Sat, 18 Nov 2023 14:15:25 +0100 Subject: [PATCH 07/31] removed old test file for hallucination score --- tests/test_hallucination_evaluation.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tests/test_hallucination_evaluation.py diff --git a/tests/test_hallucination_evaluation.py b/tests/test_hallucination_evaluation.py deleted file mode 100644 index 1b1a578a3..000000000 --- a/tests/test_hallucination_evaluation.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest -from deepeval.test_case import LLMTestCase -from deepeval.metrics.vectara_hallucination_evaluation import ( - HallucinationEvaluationMetric, -) -from deepeval.evaluator import assert_test - - -def test_hallucination_evaluation_metric(): - metric = HallucinationEvaluationMetric(minimum_score=0.5) - test_case = LLMTestCase( - input="placeholder", - actual_output="A blond drinking water in public.", - context=[ - "A man with blond-hair, and a brown shirt drinking out of a public water fountain." - ], - ) - assert_test(test_case, [metric]) From 03bce98a92086476442b41f07489c85d03e75189 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Sun, 19 Nov 2023 01:06:15 +0100 Subject: [PATCH 08/31] The hallucination metric refactored to use the Scorer removed chunking from the context and test added again --- ..._evaluation.py => hallucination_metric.py} | 29 +++---------------- deepeval/metrics/scoring.py | 5 ++++ tests/test_hallucination_metric.py | 18 ++++++++++++ 3 files changed, 27 insertions(+), 25 deletions(-) rename deepeval/metrics/{hallucination_evaluation.py => hallucination_metric.py} (54%) create mode 100644 tests/test_hallucination_metric.py diff --git a/deepeval/metrics/hallucination_evaluation.py b/deepeval/metrics/hallucination_metric.py similarity index 54% rename from deepeval/metrics/hallucination_evaluation.py rename to deepeval/metrics/hallucination_metric.py index bd706294b..5a117baa6 100644 --- a/deepeval/metrics/hallucination_evaluation.py +++ b/deepeval/metrics/hallucination_metric.py @@ -6,35 +6,14 @@ from deepeval.evaluator import assert_test from deepeval.progress_context import progress_context from sentence_transformers import CrossEncoder - - -class HallucinationEvaluationModel(metaclass=Singleton): - def __init__( - self, model_name: str = "vectara/hallucination_evaluation_model" - ): - # We use a smple cross encoder model - os.environ["TOKENIZERS_PARALLELISM"] = "false" - self.model = CrossEncoder(model_name) - - def predict(self, text_a: str, text_b: str): - scores = self.model.predict([[text_a, text_b]]) - # https://huggingface.co/vectara/hallucination_evaluation_model - - return scores[0] +from deepeval.metrics.scoring import Scorer class HallucinationMetric(BaseMetric, metaclass=Singleton): def __init__( self, minimum_score: float = 0.5, - model_name: str = "vectara/hallucination_evaluation_model", ): - # For Crossencoder model, move to singleton to avoid re-instantiating - - with progress_context( - "Downloading HallucinationEvaluationModel (may take up to 2 minutes if running for the first time)..." - ): - self.model = VectaraHallucinationEvaluationModel(model_name) self.minimum_score = minimum_score def measure(self, test_case: LLMTestCase): @@ -43,16 +22,16 @@ def measure(self, test_case: LLMTestCase): context_list = [] if isinstance(test_case.context, str): - context_list.extend(chunk_text(test_case.context)) + context_list.extend(test_case.context) elif isinstance(test_case.context, list): for context in test_case.context: - context_list.extend(chunk_text(context)) + context_list.extend(context) else: raise ValueError("Context must be a string or a list of strings") max_score = 0 for c in context_list: - score = self.model.predict(c, test_case.actual_output) + score = Scorer.hallucination_score(c, test_case.actual_output) if score > max_score: max_score = score diff --git a/deepeval/metrics/scoring.py b/deepeval/metrics/scoring.py index 0ac1678e0..b72552a88 100644 --- a/deepeval/metrics/scoring.py +++ b/deepeval/metrics/scoring.py @@ -225,6 +225,11 @@ def hallucination_score( print(e) model = "vectara-hallucination" if model is None else model + # TODO: add this progress context in the correct place + # with progress_context( + # "Downloading HallucinationEvaluationModel (may take up to 2 minutes if running for the first time)..." + # ): + scorer = HallucinationModel(model_name=model) return scorer.score(source, prediction) diff --git a/tests/test_hallucination_metric.py b/tests/test_hallucination_metric.py new file mode 100644 index 000000000..cd2d12704 --- /dev/null +++ b/tests/test_hallucination_metric.py @@ -0,0 +1,18 @@ +import pytest +from deepeval.test_case import LLMTestCase +from deepeval.metrics.hallucination_metric import ( + HallucinationMetric, +) +from deepeval.evaluator import assert_test + + +def test_hallucination_metric(): + metric = HallucinationMetric(minimum_score=0.5) + test_case = LLMTestCase( + input="placeholder", + actual_output="A blond drinking water in public.", + context=[ + "A man with blond-hair, and a brown shirt drinking out of a public water fountain." + ], + ) + assert_test(test_case, [metric]) From ca8fbd2a96d34604a42419ecbcc300250bc4724a Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Sun, 19 Nov 2023 01:54:12 +0100 Subject: [PATCH 09/31] fixed for the side effects of removing chunking --- deepeval/metrics/hallucination_metric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepeval/metrics/hallucination_metric.py b/deepeval/metrics/hallucination_metric.py index 5a117baa6..d81908441 100644 --- a/deepeval/metrics/hallucination_metric.py +++ b/deepeval/metrics/hallucination_metric.py @@ -22,10 +22,10 @@ def measure(self, test_case: LLMTestCase): context_list = [] if isinstance(test_case.context, str): - context_list.extend(test_case.context) + context_list.append(test_case.context) elif isinstance(test_case.context, list): for context in test_case.context: - context_list.extend(context) + context_list.append(context) else: raise ValueError("Context must be a string or a list of strings") From 1e41766a685d9969a70c3cff9b2476dcde002b79 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Sun, 19 Nov 2023 20:18:59 +0100 Subject: [PATCH 10/31] removed scoring logic from the model and moved it to scoring class --- deepeval/metrics/scoring.py | 2 +- deepeval/models/hallucination_model.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/deepeval/metrics/scoring.py b/deepeval/metrics/scoring.py index b72552a88..de9c39db3 100644 --- a/deepeval/metrics/scoring.py +++ b/deepeval/metrics/scoring.py @@ -232,7 +232,7 @@ def hallucination_score( scorer = HallucinationModel(model_name=model) - return scorer.score(source, prediction) + return scorer.model.predict([source, prediction]) @classmethod def PII_score( diff --git a/deepeval/models/hallucination_model.py b/deepeval/models/hallucination_model.py index cd87e4c3c..d2e6ad4b2 100644 --- a/deepeval/models/hallucination_model.py +++ b/deepeval/models/hallucination_model.py @@ -10,9 +10,3 @@ def __init__(self, model_name: str = "vectara-hallucination"): os.environ["TOKENIZERS_PARALLELISM"] = "false" model_name = name_to_card(model_name) self.model = CrossEncoder(model_name) - - def score(self, source: str, prediction: str): - scores = self.model.predict([[source, prediction]]) - # https://huggingface.co/vectara/hallucination_evaluation_model - - return scores[0] From 2d748a03fbec35dea8947bb5bd83b1e2921e89ef Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Mon, 20 Nov 2023 06:23:02 +0100 Subject: [PATCH 11/31] removed corssEnconder import --- deepeval/metrics/hallucination_metric.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepeval/metrics/hallucination_metric.py b/deepeval/metrics/hallucination_metric.py index d81908441..229abcfb8 100644 --- a/deepeval/metrics/hallucination_metric.py +++ b/deepeval/metrics/hallucination_metric.py @@ -5,7 +5,6 @@ from deepeval.metrics.base_metric import BaseMetric from deepeval.evaluator import assert_test from deepeval.progress_context import progress_context -from sentence_transformers import CrossEncoder from deepeval.metrics.scoring import Scorer From 7fe252d98c08f6f161b85eded583a2f2de43c8a4 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Mon, 20 Nov 2023 10:10:34 +0100 Subject: [PATCH 12/31] PR comments resolved --- deepeval/metrics/scoring.py | 5 ----- deepeval/models/hallucination_model.py | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deepeval/metrics/scoring.py b/deepeval/metrics/scoring.py index de9c39db3..63f94d5af 100644 --- a/deepeval/metrics/scoring.py +++ b/deepeval/metrics/scoring.py @@ -225,11 +225,6 @@ def hallucination_score( print(e) model = "vectara-hallucination" if model is None else model - # TODO: add this progress context in the correct place - # with progress_context( - # "Downloading HallucinationEvaluationModel (may take up to 2 minutes if running for the first time)..." - # ): - scorer = HallucinationModel(model_name=model) return scorer.model.predict([source, prediction]) diff --git a/deepeval/models/hallucination_model.py b/deepeval/models/hallucination_model.py index d2e6ad4b2..6de38543c 100644 --- a/deepeval/models/hallucination_model.py +++ b/deepeval/models/hallucination_model.py @@ -8,5 +8,11 @@ class HallucinationModel(metaclass=Singleton): def __init__(self, model_name: str = "vectara-hallucination"): # We use a smple cross encoder model os.environ["TOKENIZERS_PARALLELISM"] = "false" + + # TODO: add this progress context in the correct place + # with progress_context( + # "Downloading HallucinationEvaluationModel (may take up to 2 minutes if running for the first time)..." + # ): + model_name = name_to_card(model_name) self.model = CrossEncoder(model_name) From 8f72b8aa3d4ff3821b3d4693c80f28046ccbbfb9 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Mon, 20 Nov 2023 14:25:09 +0100 Subject: [PATCH 13/31] docs updated --- docs/docs/evaluation-metrics.mdx | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/docs/evaluation-metrics.mdx b/docs/docs/evaluation-metrics.mdx index b427d84d6..321aec70e 100644 --- a/docs/docs/evaluation-metrics.mdx +++ b/docs/docs/evaluation-metrics.mdx @@ -141,6 +141,31 @@ length_metric = LengthMetric(max_length=20) You must implement `measure`, `is_successful`, and `name` yourself, as these are abstract methods and properties inherited from `Metric`. ::: +## Hallucination Metric +Hallucination metric measure the factual correctness between the `actual_output (output of LLM/typically summary)` of your LLM application to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate hallucination metric. + +```python +import pytest +from deepeval.metrics.hallucination_metric import HallucinationMetric +from deepeval.test_case import LLMTestCase +from deepeval.evaluator import run_test + +# Replace this with the actual documents that you are passing as input to your LLM. +context=["A man with blond-hair, and a brown shirt drinking out of a public water fountain."] + +# Replace this with the actual output from your LLM application +actual_output="A blond drinking water in public.", + +test_case = LLMTestCase(input="placeholder", actual_output=actual_output, context=context) +metric = HallucinationMetric(minimum_score=0.5) +run_test(test_case, [metric]) +``` + +:::note +This metric uses vectara's hallucination evaluation model, which was specifically designed for summarization task. However, if you are just looking for factual consistency between input and output +where the application is not designed for QA task, this metric should work. +::: + ## Factual Consistency Factual consistency measures how factually correct the `actual_output` of your LLM application is compared to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate factual consistency. From d061fb7c7b51a683e93341550b4e064a27651c90 Mon Sep 17 00:00:00 2001 From: Aman Gokrani Date: Mon, 20 Nov 2023 17:50:06 +0100 Subject: [PATCH 14/31] added progress context check again and removed unnecessary checks --- deepeval/metrics/hallucination_metric.py | 9 ++------- deepeval/models/hallucination_model.py | 12 ++++++------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/deepeval/metrics/hallucination_metric.py b/deepeval/metrics/hallucination_metric.py index 229abcfb8..ab3d82288 100644 --- a/deepeval/metrics/hallucination_metric.py +++ b/deepeval/metrics/hallucination_metric.py @@ -20,13 +20,8 @@ def measure(self, test_case: LLMTestCase): raise ValueError("Output or context cannot be None") context_list = [] - if isinstance(test_case.context, str): - context_list.append(test_case.context) - elif isinstance(test_case.context, list): - for context in test_case.context: - context_list.append(context) - else: - raise ValueError("Context must be a string or a list of strings") + for context in test_case.context: + context_list.append(context) max_score = 0 for c in context_list: diff --git a/deepeval/models/hallucination_model.py b/deepeval/models/hallucination_model.py index 6de38543c..5e3f48464 100644 --- a/deepeval/models/hallucination_model.py +++ b/deepeval/models/hallucination_model.py @@ -1,6 +1,7 @@ import os from deepeval.singleton import Singleton from sentence_transformers import CrossEncoder +from deepeval.progress_context import progress_context from deepeval.models.model_map import model_map, name_to_card @@ -10,9 +11,8 @@ def __init__(self, model_name: str = "vectara-hallucination"): os.environ["TOKENIZERS_PARALLELISM"] = "false" # TODO: add this progress context in the correct place - # with progress_context( - # "Downloading HallucinationEvaluationModel (may take up to 2 minutes if running for the first time)..." - # ): - - model_name = name_to_card(model_name) - self.model = CrossEncoder(model_name) + with progress_context( + "Downloading HallucinationEvaluationModel (may take up to 2 minutes if running for the first time)..." + ): + model_name = name_to_card(model_name) + self.model = CrossEncoder(model_name) From ae75a63001f74f23fc119da6a53d72cf0578ce0f Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Tue, 21 Nov 2023 20:26:53 -0800 Subject: [PATCH 15/31] Refactor and updated docs --- deepeval/cli/examples.py | 200 ------------------ deepeval/metrics/__init__.py | 4 +- deepeval/metrics/answer_relevancy.py | 2 +- deepeval/metrics/conceptual_similarity.py | 49 ----- deepeval/metrics/hallucination_metric.py | 8 +- deepeval/metrics/judgemental_gpt.py | 2 +- deepeval/metrics/llm_eval_metric.py | 5 +- ...oxic_classifier.py => non_toxic_metric.py} | 0 deepeval/metrics/ragas_metric.py | 47 ++-- .../{bias_classifier.py => unbias_metric.py} | 9 +- deepeval/scorer/__init__.py | 1 + .../{metrics/scoring.py => scorer/scorer.py} | 0 deepeval/test_quickstart.py | 141 ------------ docs/docs/evaluation-datasets.mdx | 4 +- docs/docs/evaluation-metrics.mdx | 101 +++------ docs/docs/evaluation-test-cases.mdx | 6 +- docs/docs/evaluation-tracing.mdx | 2 +- docs/docs/getting-started.mdx | 10 +- docs/docs/integrations-llamaindex.mdx | 2 +- examples/getting_started/test_example.py | 4 +- examples/tracing/test_chatbot.py | 2 +- tests/test_bias.py | 2 +- tests/test_hallucination_metric.py | 4 +- tests/test_llm_metric.py | 62 +++--- tests/test_scoring.py | 2 +- tests/test_toxic.py | 2 +- 26 files changed, 112 insertions(+), 559 deletions(-) delete mode 100644 deepeval/cli/examples.py delete mode 100644 deepeval/metrics/conceptual_similarity.py rename deepeval/metrics/{toxic_classifier.py => non_toxic_metric.py} (100%) rename deepeval/metrics/{bias_classifier.py => unbias_metric.py} (93%) create mode 100644 deepeval/scorer/__init__.py rename deepeval/{metrics/scoring.py => scorer/scorer.py} (100%) delete mode 100644 deepeval/test_quickstart.py diff --git a/deepeval/cli/examples.py b/deepeval/cli/examples.py deleted file mode 100644 index 434372cc1..000000000 --- a/deepeval/cli/examples.py +++ /dev/null @@ -1,200 +0,0 @@ -CUSTOMER_EXAMPLE = """import pytest -from deepeval.test_case import LLMTestCase -from deepeval.evaluator import assert_test -from deepeval.metrics.factual_consistency import FactualConsistencyMetric -from deepeval.metrics.answer_relevancy import AnswerRelevancyMetric - -# Let's start with a simple test case to understand the basic structure -# We will use a single query, output, and context for this example -# The query is the question asked, the output is the expected answer, and the context is the actual answer - - -# Define the test case -def test_customer_chatbot_simple(): - input = "What are your operating hours?" - output = "Our operating hours are from 9 AM to 5 PM, Monday to Friday." - context = "Our company operates from 10 AM to 6 PM, Monday to Friday." - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.3) - answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) - test_case = LLMTestCase(input=input, actual_output=output, context=context) - assert_test( - test_case, [factual_consistency_metric, answer_relevancy_metric] - ) - - -# Now, let's move on to a more complex example using pytest fixtures -# Fixtures allow us to define a set of test data that can be used across multiple test cases -# In this case, we define a fixture that returns a list of tuples, each containing a query, output, and context - - -# Example bulk case -CHATBOT_TEST_CASES = [ - { - "query": "What are your operating hours?", - "output": "Our operating hours are from 9 AM to 5 PM, Monday to Friday.", - # Context mocks search results returned in a retrieval-augmented generation pipeline - "context": [ - "Our company operates from 10 AM to 6 PM, Monday to Friday.", - "We are closed on weekends and public holidays.", - "Our customer service is available 24/7.", - ], - }, - { - "query": "What are your operating hours?", - "output": "We are open from 10 AM to 6 PM, Monday to Friday.", - "context": [ - "Our company operates from 10 AM to 6 PM, Monday to Friday.", - "We are closed on weekends and public holidays.", - "Our customer service is available 24/7.", - ], - }, - { - "query": "Do you offer free shipping?", - "output": "Yes, we offer free shipping on orders over $50.", - "context": [ - "Our company offers free shipping on orders over $100.", - "Free shipping applies to all products.", - "The free shipping offer is valid only within the country.", - ], - }, - { - "query": "Do you offer free shipping?", - "output": "Absolutely, free shipping is available for orders over $50.", - "context": [ - "Our company offers free shipping on orders over $100.", - "Free shipping applies to all products.", - "The free shipping offer is valid only within the country.", - ], - }, - { - "query": "What is your return policy?", - "output": "We have a 30-day return policy for unused and unopened items.", - "context": [ - "Our company has a 14-day return policy for unused and unopened items.", - "Items must be returned in their original packaging.", - "Customers are responsible for return shipping costs.", - ], - }, - { - "query": "What is your return policy?", - "output": "Unused and unopened items can be returned within 30 days.", - "context": [ - "Our company has a 14-day return policy for unused and unopened items.", - "Items must be returned in their original packaging.", - "Customers are responsible for return shipping costs.", - ], - }, - { - "query": "Do you have a physical store?", - "output": "No, we are an online-only store.", - "context": [ - "Our company has physical stores in several locations.", - "We have stores in major cities across the country.", - "You can find the nearest store using our store locator.", - ], - }, - { - "query": "Do you have a physical store?", - "output": "We operate solely online, we do not have a physical store.", - "context": [ - "Our company operates solely online.", - "We do not have any physical stores.", - "All our products are available on our website.", - ], - }, - { - "query": "How can I track my order?", - "output": "You can track your order through the link provided in your confirmation email.", - "context": [ - "Customers can track their orders by calling our customer service.", - "You can also track your order using the tracking number provided in the confirmation email.", - "For any issues with tracking, please contact our customer support.", - ], - }, - { - "query": "How can I track my order?", - "output": "Order tracking is available via the link in your confirmation email.", - "context": [ - "Customers can track their orders by calling our customer service.", - "You can also track your order using the tracking number provided in the confirmation email.", - "For any issues with tracking, please contact our customer support.", - ], - }, -] - - -# We then use this constant in our test case -# The test case is run once for each tuple in the constant -@pytest.mark.parametrize( - "test_case", - CHATBOT_TEST_CASES, -) -def test_customer_chatbot(test_case: dict): - input = test_case["input"] - output = test_case["output"] - context = test_case["context"] - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.3) - answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) - test_case = LLMTestCase(input=input, actual_output=output, context=context) - assert_test( - test_case, [factual_consistency_metric, answer_relevancy_metric] - ) -""" - - -EXAMPLE_BROCHURE = """[Front Cover] - -Customer Support Guide -Providing You with Exceptional Service - -[Inside Cover] - -At TrendyTrends, we value your satisfaction and are committed to delivering top-notch customer support. This brochure is designed to assist you with any inquiries you may have regarding shipping and delivery times. We're here to make your experience as seamless as possible. - -[Page 1] - -Shipping & Delivery Times - -At TrendyTrends, we understand that timely delivery is crucial to your satisfaction. Our dedicated team works tirelessly to ensure your orders reach you promptly. Here's what you need to know about our shipping and delivery times: - -1. Standard Shipping: - -Delivery Time: 3-5 business days -Cost: Free for orders over $50; $5 for orders under $50 -Coverage: Nationwide -2. Express Shipping: - -Delivery Time: 1-2 business days -Cost: $15 -Coverage: Nationwide -[Page 2] - -Order Tracking - -We offer convenient order tracking so you can monitor the progress of your package. Simply visit our website or use our mobile app to enter your order number and get real-time updates on the status of your shipment. We believe in transparency and keeping you informed every step of the way. - -[Page 3] - -Our Commitment to You - -At TrendyTrends, our commitment to exceptional customer support goes beyond just shipping and delivery times. We are dedicated to: - -Providing friendly and knowledgeable customer service representatives to assist you. -Resolving any issues or concerns promptly and efficiently. -Ensuring the safe and secure delivery of your orders. -[Page 4] - -Contact Us - -Should you have any questions, concerns, or need assistance with your order, our customer support team is here to help: - -Customer Support Hotline: 1-800-123-4567 -Email: support@trendytrends.com -Live Chat: Available on our website during business hours -[Back Cover] - -Thank you for choosing TrendyTrends. Your satisfaction is our top priority, and we look forward to serving you. For the latest updates, promotions, and more, follow us on social media or visit our website at www.trendytrends.com. - -[Disclaimer] - -Shipping and delivery times are estimates and may vary due to factors beyond our control. For the most accurate delivery information, please refer to your order tracking or contact our customer support team.""" diff --git a/deepeval/metrics/__init__.py b/deepeval/metrics/__init__.py index bb4a2301c..38117dfb6 100644 --- a/deepeval/metrics/__init__.py +++ b/deepeval/metrics/__init__.py @@ -1,8 +1,10 @@ from .base_metric import BaseMetric from .answer_relevancy import AnswerRelevancyMetric from .base_metric import BaseMetric -from .conceptual_similarity import ConceptualSimilarityMetric from .factual_consistency import FactualConsistencyMetric from .judgemental_gpt import JudgementalGPT from .llm_eval_metric import LLMEvalMetric from .ragas_metric import RagasMetric +from .unbias_metric import UnBiasedMetric +from .non_toxic_metric import NonToxicMetric +from .hallucination_metric import HallucinationMetric \ No newline at end of file diff --git a/deepeval/metrics/answer_relevancy.py b/deepeval/metrics/answer_relevancy.py index 9c4219109..43bc2c9a6 100644 --- a/deepeval/metrics/answer_relevancy.py +++ b/deepeval/metrics/answer_relevancy.py @@ -1,6 +1,6 @@ from deepeval.singleton import Singleton from deepeval.test_case import LLMTestCase -from deepeval.metrics.base_metric import BaseMetric +from deepeval.metrics import BaseMetric import numpy as np diff --git a/deepeval/metrics/conceptual_similarity.py b/deepeval/metrics/conceptual_similarity.py deleted file mode 100644 index 49131c6b7..000000000 --- a/deepeval/metrics/conceptual_similarity.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Asserting conceptual similarity -""" -from typing import Optional - -from deepeval.singleton import Singleton -from deepeval.test_case import LLMTestCase -from deepeval.utils import cosine_similarity -from deepeval.progress_context import progress_context -from deepeval.metrics.base_metric import BaseMetric - - -class ConceptualSimilarityMetric(BaseMetric, metaclass=Singleton): - """basic implementation of ConceptualSimilarityMetric""" - - def __init__( - self, - model_name: Optional[str] = "sentence-transformers/all-mpnet-base-v2", - minimum_score: float = 0.7, - ): - from sentence_transformers import SentenceTransformer - - self.model_name = model_name - with progress_context( - "Downloading Conceptual Similarity model (may take up to 2 minutes if running for the first time)..." - ): - self.model = SentenceTransformer(self.model_name).eval() - self.minimum_score = minimum_score - - def _vectorize(self, text_a: str, text_b: str): - vectors = self.model.encode([text_a, text_b]) - return vectors - - def measure(self, test_case: LLMTestCase) -> float: - if test_case.actual_output is None or test_case.expected_output is None: - raise ValueError("Output or expected_output cannot be None") - - vectors = self._vectorize( - test_case.actual_output, test_case.expected_output - ) - self.score = cosine_similarity(vectors[0], vectors[1]) - self.success = self.score >= self.minimum_score - return float(self.score) - - def is_successful(self) -> bool: - return self.success - - @property - def __name__(self): - return "Conceptual Similarity" diff --git a/deepeval/metrics/hallucination_metric.py b/deepeval/metrics/hallucination_metric.py index ab3d82288..9beb07b2c 100644 --- a/deepeval/metrics/hallucination_metric.py +++ b/deepeval/metrics/hallucination_metric.py @@ -1,11 +1,7 @@ -import os from deepeval.singleton import Singleton from deepeval.test_case import LLMTestCase -from deepeval.utils import chunk_text, softmax -from deepeval.metrics.base_metric import BaseMetric -from deepeval.evaluator import assert_test -from deepeval.progress_context import progress_context -from deepeval.metrics.scoring import Scorer +from deepeval.metrics import BaseMetric +from deepeval.scorer import Scorer class HallucinationMetric(BaseMetric, metaclass=Singleton): diff --git a/deepeval/metrics/judgemental_gpt.py b/deepeval/metrics/judgemental_gpt.py index b735993be..3cd2257eb 100644 --- a/deepeval/metrics/judgemental_gpt.py +++ b/deepeval/metrics/judgemental_gpt.py @@ -1,4 +1,4 @@ -from deepeval.metrics.base_metric import BaseMetric +from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCaseParams, LLMTestCase from typing import List from pydantic import BaseModel diff --git a/deepeval/metrics/llm_eval_metric.py b/deepeval/metrics/llm_eval_metric.py index b7c8a0d73..fead9b40b 100644 --- a/deepeval/metrics/llm_eval_metric.py +++ b/deepeval/metrics/llm_eval_metric.py @@ -1,5 +1,5 @@ from typing import Optional, List -from deepeval.metrics.base_metric import BaseMetric +from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCase, LLMTestCaseParams from deepeval.templates import ( evaluation_steps_template, @@ -7,10 +7,7 @@ ) from deepeval.chat_completion.retry import call_openai_with_retry from pydantic import BaseModel -import openai from langchain.chat_models import ChatOpenAI -from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler - class LLMEvalMetricResponse(BaseModel): score: float diff --git a/deepeval/metrics/toxic_classifier.py b/deepeval/metrics/non_toxic_metric.py similarity index 100% rename from deepeval/metrics/toxic_classifier.py rename to deepeval/metrics/non_toxic_metric.py diff --git a/deepeval/metrics/ragas_metric.py b/deepeval/metrics/ragas_metric.py index 989011bbe..89adb222a 100644 --- a/deepeval/metrics/ragas_metric.py +++ b/deepeval/metrics/ragas_metric.py @@ -1,11 +1,11 @@ """An implementation of the Ragas metric """ -from deepeval.metrics.base_metric import BaseMetric +from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCase from typing import List -class RagasContextualRelevancyMetric(BaseMetric): +class ContextualRelevancyMetric(BaseMetric): """This metric checks the contextual relevancy using Ragas""" def __init__( @@ -62,10 +62,10 @@ def is_successful(self): @property def __name__(self): - return "Contextual Relevancy (RAGAS)" + return "Contextual Relevancy" -class RagasAnswerRelevancyMetric(BaseMetric): +class AnswerRelevancyMetric(BaseMetric): """This metric checks the answer relevancy using Ragas""" def __init__( @@ -115,10 +115,10 @@ def is_successful(self): @property def __name__(self): - return "Answer Relevancy (RAGAS)" + return "Answer Relevancy" -class RagasFaithfulnessMetric(BaseMetric): +class FaithfulnessMetric(BaseMetric): def __init__( self, minimum_score: float = 0.3, @@ -166,10 +166,10 @@ def is_successful(self): @property def __name__(self): - return "Faithfulness (RAGAS)" + return "Faithfulness" -class RagasContextRecallMetric(BaseMetric): +class ContextRecallMetric(BaseMetric): """This metric checks the context recall using Ragas""" def __init__( @@ -219,10 +219,10 @@ def is_successful(self): @property def __name__(self): - return "Context Recall (RAGAS)" + return "Context Recall" -class RagasHarmfulnessMetric(BaseMetric): +class HarmfulnessMetric(BaseMetric): """This metric checks the harmfulness using Ragas""" def __init__( @@ -272,10 +272,10 @@ def is_successful(self): @property def __name__(self): - return "Harmfulness (RAGAS)" + return "Harmfulness" -class RagasCoherenceMetric(BaseMetric): +class CoherenceMetric(BaseMetric): """This metric checks the coherence using Ragas""" def __init__( @@ -324,10 +324,10 @@ def is_successful(self): @property def __name__(self): - return "Coherence (RAGAS)" + return "Coherence" -class RagasMaliciousnessMetric(BaseMetric): +class MaliciousnessMetric(BaseMetric): """This metric checks the maliciousness using Ragas""" def __init__( @@ -376,10 +376,10 @@ def is_successful(self): @property def __name__(self): - return "Maliciousness (RAGAS)" + return "Maliciousness" -class RagasCorrectnessMetric(BaseMetric): +class CorrectnessMetric(BaseMetric): """This metric checks the correctness using Ragas""" def __init__( @@ -428,10 +428,10 @@ def is_successful(self): @property def __name__(self): - return "Correctness (RAGAS)" + return "Correctness" -class RagasConcisenessMetric(BaseMetric): +class ConcisenessMetric(BaseMetric): """This metric checks the conciseness using Ragas""" def __init__( @@ -480,7 +480,7 @@ def is_successful(self): @property def __name__(self): - return "Conciseness (RAGAS)" + return "Conciseness" class RagasMetric(BaseMetric): @@ -494,11 +494,10 @@ def __init__( self.minimum_score = minimum_score if metrics is None: self.metrics = [ - RagasContextualRelevancyMetric, - RagasHarmfulnessMetric, - RagasContextRecallMetric, - RagasFaithfulnessMetric, - RagasAnswerRelevancyMetric, + ContextualRelevancyMetric, + ContextRecallMetric, + FaithfulnessMetric, + AnswerRelevancyMetric, ] else: self.metrics = metrics diff --git a/deepeval/metrics/bias_classifier.py b/deepeval/metrics/unbias_metric.py similarity index 93% rename from deepeval/metrics/bias_classifier.py rename to deepeval/metrics/unbias_metric.py index 45d510a6a..a01177d6b 100644 --- a/deepeval/metrics/bias_classifier.py +++ b/deepeval/metrics/unbias_metric.py @@ -7,7 +7,7 @@ import warnings from typing import Optional, List -from deepeval.metrics.base_metric import BaseMetric +from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCase, LLMTestCaseParams @@ -76,10 +76,7 @@ def measure(self, test_case: LLMTestCase, return_all_scores: bool = False): def is_successful(self): return self.success - - def assert_successful(self): - assert self.success, f"Text is biased - score: {self.score}" - + @property def __name__(self): - return "Bias Score" + return "Unbiased Metric" diff --git a/deepeval/scorer/__init__.py b/deepeval/scorer/__init__.py new file mode 100644 index 000000000..5bf277a68 --- /dev/null +++ b/deepeval/scorer/__init__.py @@ -0,0 +1 @@ +from .scorer import Scorer \ No newline at end of file diff --git a/deepeval/metrics/scoring.py b/deepeval/scorer/scorer.py similarity index 100% rename from deepeval/metrics/scoring.py rename to deepeval/scorer/scorer.py diff --git a/deepeval/test_quickstart.py b/deepeval/test_quickstart.py deleted file mode 100644 index 63b1b87e1..000000000 --- a/deepeval/test_quickstart.py +++ /dev/null @@ -1,141 +0,0 @@ -import pytest -from deepeval.test_case import LLMTestCase -from deepeval.evaluator import assert_test -from deepeval.metrics.factual_consistency import FactualConsistencyMetric -from deepeval.metrics.answer_relevancy import AnswerRelevancyMetric - -# This tutorial will guide you on how to use pytest for testing chatbot responses. -# pytest is a testing framework that allows you to easily create small, simple tests, yet scales to support complex functional testing. -# We will use the deepeval library which provides tools for evaluating language models. -# The library includes test cases, metrics, and a function to assert the test results. - - -# First, we will define a simple test case. The test case includes a query, expected output, and context. -# The query is the question asked to the chatbot, the output is the expected answer, and the context is the actual answer. -# We then use the assert_test function from deepeval to check if the chatbot's response is as expected. -def test_customer_chatbot_simple(): - input = "What are your operating hours?" - output = "Our operating hours are from 9 AM to 5 PM, Monday to Friday." - context = ["Our company operates from 10 AM to 6 PM, Monday to Friday."] - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.3) - answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) - test_case = LLMTestCase(input=input, actual_output=output, context=context) - assert_test( - test_case, [factual_consistency_metric, answer_relevancy_metric] - ) - - -# We can also define multiple test cases at once in a list of dictionaries. -# Each dictionary represents a test case and can be directly used in our tests. -# The keys of the dictionary are the column names in the DataFrame (query, output, context). -# The values are the corresponding values for each test case. -CHATBOT_TEST_CASES = [ - { - "input": "What are your operating hours?", - "output": "Our operating hours are from 9 AM to 5 PM, Monday to Friday.", - "context": [ - "Our company operates from 10 AM to 6 PM, Monday to Friday.", - "We are closed on weekends and public holidays.", - "Our customer service is available 24/7.", - ], - }, - { - "input": "Do you offer free shipping?", - "output": "Yes, we offer free shipping on orders over $50.", - "context": [ - "Our company offers free shipping on orders over $100.", - "Free shipping applies to all products.", - "The free shipping offer is valid only within the country.", - ], - }, - { - "input": "What is your return policy?", - "output": "We accept returns within 30 days of purchase.", - "context": [ - "Our company accepts returns within 60 days of purchase.", - "The product must be in its original condition.", - "The return shipping cost will be covered by the customer.", - ], - }, - { - "input": "Do you have a physical store?", - "output": "Yes, we have a physical store in New York.", - "context": [ - "Our company has physical stores in several locations across the country.", - "The New York store is our flagship store.", - "Our stores offer a wide range of products.", - ], - }, - { - "input": "Do you offer international shipping?", - "output": "Yes, we offer international shipping to selected countries.", - "context": [ - "Our company offers international shipping to all countries.", - "International shipping rates apply.", - "Delivery times vary depending on the destination.", - ], - }, - { - "input": "Do you have a customer loyalty program?", - "output": "Yes, we have a loyalty program called 'Rewards Club'.", - "context": [ - "Our company has a loyalty program that offers exclusive benefits to members.", - "The 'Rewards Club' program offers discounts, early access to sales, and more.", - "Customers can join the 'Rewards Club' by signing up on our website.", - ], - }, -] - - -# pytest provides a decorator called 'parametrize' that allows you to run a test function multiple times with different arguments. -# Here, we use it to run the test function for each test case in CHATBOT_TEST_CASES. -# The test function takes a test case as an argument, extracts the query, output, and context, and then runs the test. -@pytest.mark.parametrize( - "test_case", - CHATBOT_TEST_CASES, -) -def test_customer_chatbot(test_case: dict): - input = test_case["input"] - output = test_case["output"] - context = test_case["context"] - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.3) - answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) - test_case = LLMTestCase(input=input, actual_output=output, context=context) - assert_test( - test_case, [factual_consistency_metric, answer_relevancy_metric] - ) - - -# # You can also load this in a CSV - -import pandas as pd - -# Here we are defining a CSV formatted string that contains our test cases. -# Each line represents a test case with a query, expected output, and context. -# The context is a string of sentences separated by '|'. -CSV_DATA = """query,output,context -"What are your operating hours?","Our operating hours are from 9 AM to 5 PM, Monday to Friday.","Our company operates from 10 AM to 6 PM, Monday to Friday.|We are closed on weekends and public holidays.|Our customer service is available 24/7." -"What are your operating hours?","We are open from 10 AM to 6 PM, Monday to Friday.","Our company operates from 10 AM to 6 PM, Monday to Friday.|We are closed on weekends and public holidays.|Our customer service is available 24/7." -"Do you offer free shipping?","Yes, we offer free shipping on orders over $50.","Our company offers free shipping on orders over $100.|Free shipping applies to all products.|The free shipping offer is valid only within the country." -""" - -# We read the CSV_DATA into a pandas DataFrame. -# This allows us to easily manipulate and process the data. -import io - -# Create a temporary file with CSV_DATA -temp_file = io.StringIO(CSV_DATA) - -# Read the temporary file as a CSV -df = pd.read_csv(temp_file) - -# We then split the context column into a list of sentences. -# This is done by splitting the string on each '|'. -# The result is a list of context sentences for each test case. -df["context"] = df["context"].apply(lambda x: x.split("|")) - -# Finally, we convert the DataFrame to a list of dictionaries. -# Each dictionary represents a test case and can be directly used in our tests. -# The keys of the dictionary are the column names in the DataFrame (query, output, context). -# The values are the corresponding values for each test case. -CHATBOT_TEST_CASES = df.to_dict("records") diff --git a/docs/docs/evaluation-datasets.mdx b/docs/docs/evaluation-datasets.mdx index e8089c25d..9102134bb 100644 --- a/docs/docs/evaluation-datasets.mdx +++ b/docs/docs/evaluation-datasets.mdx @@ -70,7 +70,7 @@ deepeval login ```python title="test_bulk.py" from deepeval.test_case import LLMTestCase -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.metric.answer_relevancy import AnswerRelevancyMetric from deepeval.evaluate import assert_test @@ -98,7 +98,7 @@ Alternately, you can use deepeval's `evaluate` function to evaluate datasets. Th ```python from deepeval.evaluator import evaluate -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.metric.answer_relevancy import AnswerRelevancyMetric dataset = [...] diff --git a/docs/docs/evaluation-metrics.mdx b/docs/docs/evaluation-metrics.mdx index 682e83362..90a3b5653 100644 --- a/docs/docs/evaluation-metrics.mdx +++ b/docs/docs/evaluation-metrics.mdx @@ -8,9 +8,8 @@ sidebar_label: Metrics In `deepeval`, a metric serves as a standard of measurement for evaluating the performance of an LLM output based on a specific criteria of interest. Essentially, while the metric acts as the ruler, the test case represents what you're assessing. `deepeval` offers a range of default metrics for you to quickly get started with, which includes: -- Factual Consistency +- Hallucination - Answer Relevancy -- Conceptual Similarity - RAGAS - Toxicity - Bias @@ -37,7 +36,7 @@ Our suggestion is to begin with custom LLM evaluated metrics (which frequently s A custom LLM evalated metric, is a custom metric whose evaluation is powered by LLMs. To create a custom metric that uses LLMs for evaluation, simply instantiate an `LLMEvalMetric` class: ```python -from deepeval.metrics.llm_eval_metric import LLMEvalMetric +from deepeval.metrics import LLMEvalMetric from deepeval.test_case import LLMTestCase, LLMTestCaseParams summarization_metric = LLMEvalMetric( @@ -72,15 +71,17 @@ summarization_metric = LLMEvalMetric( criteria="Summarization - determine if the actual output is an accurate and concise summarization of the input.", evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], minimum_score=0.5, - deployment_id="your-deployment-id" + deployment_id="your-deployment-id" ) ``` :::note -Don't forget to set your Azure OpenAI key +Don't forget to set your Azure OpenAI key + ```python openai.api_key = "your-Azure-OpenAI-API-key" -``` +``` + ::: ## JudgementalGPT @@ -100,7 +101,7 @@ deepeval login Then paste in the following code to define a metric powered by `JudgementalGPT`: ```python -from deepeval.metrics.judgemental_gpt import JudgementalGPT +from deepeval.metrics import JudgementalGPT from deepeval.test_case import LLMTestCase, LLMTestCaseParams code_correctness_metric = JudgementalGPT( @@ -123,7 +124,7 @@ Under the hood, `JudgementalGPT(...)` sends a request to Confident AI's servers You can implement your own evaluator (for example, your own GPT evaluator) by creating a custom metric. All custom metrics are automatically integrated with Confident AI. ```python -from deepeval.metrics.base_metric import BaseMetric +from deepeval.metrics import BaseMetric # Inherit BaseMetric class LengthMetric(BaseMetric): @@ -161,15 +162,16 @@ You must implement `measure`, `is_successful`, and `name` yourself, as these are ::: ## Hallucination Metric -Hallucination metric measure the factual correctness between the `actual_output (output of LLM/typically summary)` of your LLM application to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate hallucination metric. -```python +Hallucination metric measures whether your LLM application is saying things that are untrue by comparing the `actual_output' to the provided `context`. You'll have to supply `context`when creating an`LLMTestCase` to evaluate hallucination. + +```python import pytest -from deepeval.metrics.hallucination_metric import HallucinationMetric +from deepeval.metrics import HallucinationMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import run_test -# Replace this with the actual documents that you are passing as input to your LLM. +# Replace this with the actual documents that you are passing as input to your LLM. context=["A man with blond-hair, and a brown shirt drinking out of a public water fountain."] # Replace this with the actual output from your LLM application @@ -177,43 +179,21 @@ actual_output="A blond drinking water in public.", test_case = LLMTestCase(input="placeholder", actual_output=actual_output, context=context) metric = HallucinationMetric(minimum_score=0.5) -run_test(test_case, [metric]) +run_test(test_case, [metric]) ``` :::note -This metric uses vectara's hallucination evaluation model, which was specifically designed for summarization task. However, if you are just looking for factual consistency between input and output -where the application is not designed for QA task, this metric should work. +This metric uses vectara's hallucination evaluation model, which was specifically designed for summarization task. However, if you are just looking for factual consistency between input and output +where the application is not designed for QA task, this metric should work. ::: -## Factual Consistency - -Factual consistency measures how factually correct the `actual_output` of your LLM application is compared to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate factual consistency. - -```python -import pytest -from deepeval.metrics.factual_consistency import FactualConsistencyMetric -from deepeval.test_case import LLMTestCase -from deepeval.evaluator import run_test - -input = "What if these shoes don't fit?" -context = ["All customers are eligible for a 30 day full refund at no extra cost."] - -# Replace this with the actual output from your LLM application -actual_output = "We offer a 30-day full refund at no extra cost." - -factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.7) -test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) - -run_test(test_case, [metric]) -``` - ## Answer Relevancy Answer Relevancy measures how relevant the `actual_output` of your LLM application is compared to the provided `input`. You don't have to supply `context` or `expected_output` when creating an `LLMTestCase` if you're just evaluating answer relevancy. ```python import pytest -from deepeval.metrics.answer_relevancy import AnswerRelevancyMetric +from deepeval.metrics import AnswerRelevancyMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import run_test @@ -229,29 +209,6 @@ test_case = LLMTestCase(input=input, actual_output=actual_output) run_test(test_case, [answer_relevancy_metric]) ``` -## Conceptual Similarity - -Conceptual Similarity measures how conceptually similar the `actual_output` of your LLM application is compared to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate conceptual similarity. - -```python -import pytest -from deepeval.metrics.conceptual_similarity import ConceptualSimilarityMetric -from deepeval.test_case import LLMTestCase -from deepeval.evaluator import run_test - - -input = "What if these shoes don't fit?" -context = ["All customers are eligible for a 30 day full refund at no extra cost."] - -# Replace this with the actual output from your LLM application -actual_output = "We offer a 30-day full refund at no extra cost." - -conceptual_similarity_metric = ConceptualSimilarityMetric(minimum_score=0.7) -test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) - -run_test(test_case, [conceptual_similarity_metric]) -``` - ## RAGAS The RAGAS metric is useful for evaluating RAG pipelines (ie. LLM applications built with RAG). The RAGAS score is calculated by taking an unweighted harmonic mean of five distinct metrics. @@ -277,7 +234,7 @@ pip install ragas Create an `LLMTestCase` and supply all parameters to calculate the RAGAS score: ```python -from deepeval.metrics.ragas_metric import RagasMetric +from deepeval.metrics import RagasMetric from deepeval.test_case import LLMTestCase input = "What if these shoes don't fit?" @@ -301,14 +258,14 @@ run_test(test_case, [ragas_metric]) As mentioned earlier, the RAGAS score is the harmonic mean of five different metrics. You can however import these metrics individually and utilize them in exactly the same way as all other metrics offered by `deepeval`. ```python -from deepeval.metrics.ragas_metric import RagasContextualRelevancyMetric -from deepeval.metrics.ragas_metric import RagasAnswerRelevancyMetric -from deepeval.metrics.ragas_metric import RagasFaithfulnessMetric -from deepeval.metrics.ragas_metric import RagasContextRecallMetric -from deepeval.metrics.ragas_metric import RagasConcisenessMetric -from deepeval.metrics.ragas_metric import RagasCorrectnessMetric -from deepeval.metrics.ragas_metric import RagasCoherenceMetric -from deepeval.metrics.ragas_metric import RagasMaliciousnessMetric +from deepeval.metrics import ContextualRelevancyMetric +from deepeval.metrics import AnswerRelevancyMetric +from deepeval.metrics import FaithfulnessMetric +from deepeval.metrics import ContextRecallMetric +from deepeval.metrics import ConcisenessMetric +from deepeval.metrics import CorrectnessMetric +from deepeval.metrics import CoherenceMetric +from deepeval.metrics import MaliciousnessMetric ``` ## Toxicity @@ -322,7 +279,7 @@ pip install detoxify Being a referenceless metric means `NonToxicMetric` requires an extra parameter named `evaluation_params`. This parameter is an array, containing elements of the type `LLMTestCaseParams`, and specifies the parameter(s) of a given `LLMTestCase` that will be assessed for toxicity. The `NonToxicMetric` will then compute a score based on the average toxicity levels of each individual component being evaluated. ```python -from deepeval.metrics.toxic_classifier import NonToxicMetric +from deepeval.metrics import NonToxicMetric from deepeval.evaluator import run_test from deepeval.test_case import LLMTestCase, LLMTestCaseParams @@ -363,7 +320,7 @@ pip install Dbias `UnBiasedMetric` is similar to `NonToxicMetric` because it is also a referenceless metric. ```python -from deepeval.metrics.bias_classifier import UnBiasedMetric +from deepeval.metrics import UnBiasedMetric from deepeval.test_case import LLMTestCase, LLMTestCaseParams from deepeval.evaluator import run_test diff --git a/docs/docs/evaluation-test-cases.mdx b/docs/docs/evaluation-test-cases.mdx index f98eb6425..275c555b5 100644 --- a/docs/docs/evaluation-test-cases.mdx +++ b/docs/docs/evaluation-test-cases.mdx @@ -181,7 +181,7 @@ Remember, `context` is the ideal retrieval results for a given input and typical ```python # A hypothetical LLM application example import chatbot -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.evaluator import run_test from deepeval.test_case import LLMTestCase @@ -224,7 +224,7 @@ A test case passes only if all metrics meet their respective evaluation criterio ```python title="test_assert_example.py" # A hypothetical LLM application example import chatbot -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.evaluator import assert_test from deepeval.test_case import LLMTestCase @@ -276,7 +276,7 @@ Lastly, `deepeval` offers an `evaluate` function to evaluate multiple test cases ```python # A hypothetical LLM application example import chatbot -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.evaluator import evaluate from deepeval.test_case import LLMTestCase diff --git a/docs/docs/evaluation-tracing.mdx b/docs/docs/evaluation-tracing.mdx index ad95219a0..afdef1130 100644 --- a/docs/docs/evaluation-tracing.mdx +++ b/docs/docs/evaluation-tracing.mdx @@ -133,7 +133,7 @@ Continuning from the previous code snippet where you've defined your `Chatbot` c import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.evaluator import assert_test chatbot = Chatbot() diff --git a/docs/docs/getting-started.mdx b/docs/docs/getting-started.mdx index 018056de3..dc3b59cf9 100644 --- a/docs/docs/getting-started.mdx +++ b/docs/docs/getting-started.mdx @@ -50,7 +50,7 @@ Run `touch test_example.py` to create a test file in your root directory. Open ` ```python title="test_example.py" import pytest -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import assert_test @@ -96,7 +96,7 @@ export DEEPEVAL_RESULTS_FOLDER="./data" An LLM evaluated metric, is one where evaluation is carried out by an LLM. Here's how you can create a custom, LLM evaluated metric. ```python title="test_example.py" -from deepeval.metrics.llm_eval_metric import LLMEvalMetric +from deepeval.metrics import LLMEvalMetric from deepeval.evaluator import assert_test from deepeval.test_case import LLMTestCase, LLMTestCaseParams @@ -123,11 +123,11 @@ def test_summarization(): A classic metric is a metric where evluation is not carried out by another LLM. You can define a custom classic metric by defining the `measure` and `is_successful` methods upon inheriting the base `Metric` class. Paste in the following: ```python title="test_example.py" -from deepeval.metrics.metric import Metric +from deepeval.metrics import BaseMetric ... -class LengthMetric(Metric): +class LengthMetric(BaseMetric): # This metric checks if the output length is greater than 10 characters def __init__(self, max_length: int=10): self.minimum_score = max_length @@ -221,7 +221,7 @@ Utilize the `@pytest.mark.parametrize` decorator to loop through and evaluate yo ```python title="test_bulk.py" import pytest -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import assert_test diff --git a/docs/docs/integrations-llamaindex.mdx b/docs/docs/integrations-llamaindex.mdx index d82d62dc1..c442b35d0 100644 --- a/docs/docs/integrations-llamaindex.mdx +++ b/docs/docs/integrations-llamaindex.mdx @@ -12,7 +12,7 @@ DeepEval integrates nicely with LlamaIndex's `ResponseEvaluator` class. Below is from llama_index.response.schema import Response from typing import List from llama_index.schema import Document -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from llama_index import ( TreeIndex, VectorStoreIndex, diff --git a/examples/getting_started/test_example.py b/examples/getting_started/test_example.py index 642520829..c0810ee92 100644 --- a/examples/getting_started/test_example.py +++ b/examples/getting_started/test_example.py @@ -1,9 +1,7 @@ import pytest -from deepeval.metrics.factual_consistency import FactualConsistencyMetric from deepeval.test_case import LLMTestCase, LLMTestCaseParams from deepeval.evaluator import assert_test -from deepeval.metrics.llm_eval_metric import LLMEvalMetric -from deepeval.metrics.base_metric import BaseMetric +from deepeval.metrics import BaseMetric, LLMEvalMetric, FactualConsistencyMetric import deepeval # To run this file: deepeval test run .py diff --git a/examples/tracing/test_chatbot.py b/examples/tracing/test_chatbot.py index 5079b2d15..bad61ba88 100644 --- a/examples/tracing/test_chatbot.py +++ b/examples/tracing/test_chatbot.py @@ -63,7 +63,7 @@ def query(self, user_input=input): import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import FactualConsistencyMetric from deepeval.evaluator import assert_test chatbot = Chatbot() diff --git a/tests/test_bias.py b/tests/test_bias.py index 861ce4ebf..6f6642c85 100644 --- a/tests/test_bias.py +++ b/tests/test_bias.py @@ -4,7 +4,7 @@ import pytest from deepeval.test_case import LLMTestCase, LLMTestCaseParams -from deepeval.metrics.bias_classifier import UnBiasedMetric +from deepeval.metrics import UnBiasedMetric from deepeval.evaluator import assert_test output = "Winners of the FIFA world cup were the French national football team" diff --git a/tests/test_hallucination_metric.py b/tests/test_hallucination_metric.py index cd2d12704..4a9af4346 100644 --- a/tests/test_hallucination_metric.py +++ b/tests/test_hallucination_metric.py @@ -1,8 +1,6 @@ import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics.hallucination_metric import ( - HallucinationMetric, -) +from deepeval.metrics import HallucinationMetric from deepeval.evaluator import assert_test diff --git a/tests/test_llm_metric.py b/tests/test_llm_metric.py index 12973d488..47122f263 100644 --- a/tests/test_llm_metric.py +++ b/tests/test_llm_metric.py @@ -4,17 +4,6 @@ from deepeval.metrics import LLMEvalMetric from deepeval.evaluator import assert_test -# set openai api type -openai.api_type = "azure" - -# The azure openai version you want to use -openai.api_version = "2023-03-15" - -# The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource. -openai.api_base = "https://your-resource-name.openai.azure.com/" -openai.api_key = "" - - def test_chat_completion(): """Test Chat Completion""" metric = LLMEvalMetric( @@ -35,24 +24,33 @@ def test_chat_completion(): assert_test(test_case, [metric]) - -def test_azure_openai_chat_completion(): - """Test Chat Completion""" - metric = LLMEvalMetric( - name="Validity", - criteria="The response is a valid response to the prompt.", - minimum_score=0.5, - evaluation_params=[ - LLMTestCaseParams.INPUT, - LLMTestCaseParams.ACTUAL_OUTPUT, - ], - deployment_id="your-deployment-id", - ) - test_case = LLMTestCase( - input="What is the capital of France?", - actual_output="Paris", - expected_output="Paris", - context=["Geography"], - ) - - assert_test(test_case, [metric]) +# # set openai api type +# openai.api_type = "azure" + +# # The azure openai version you want to use +# openai.api_version = "2023-03-15" + +# # The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource. +# openai.api_base = "https://your-resource-name.openai.azure.com/" +# openai.api_key = "" + +# def test_azure_openai_chat_completion(): +# """Test Chat Completion""" +# metric = LLMEvalMetric( +# name="Validity", +# criteria="The response is a valid response to the prompt.", +# minimum_score=0.5, +# evaluation_params=[ +# LLMTestCaseParams.INPUT, +# LLMTestCaseParams.ACTUAL_OUTPUT, +# ], +# deployment_id="your-deployment-id", +# ) +# test_case = LLMTestCase( +# input="What is the capital of France?", +# actual_output="Paris", +# expected_output="Paris", +# context=["Geography"], +# ) + +# assert_test(test_case, [metric]) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 160e0761b..dfb93db3b 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -2,7 +2,7 @@ """ import unittest -from deepeval.metrics.scoring import Scorer +from deepeval.scorer import Scorer class TestScorer(unittest.TestCase): diff --git a/tests/test_toxic.py b/tests/test_toxic.py index 5735f3c2b..8e0f46868 100644 --- a/tests/test_toxic.py +++ b/tests/test_toxic.py @@ -4,7 +4,7 @@ import pytest from deepeval.test_case import LLMTestCase, LLMTestCaseParams -from deepeval.metrics.toxic_classifier import NonToxicMetric +from deepeval.metrics import NonToxicMetric from deepeval.evaluator import assert_test output = "Winners of the FIFA world cup were the French national football team" From 375886b037ea24d9b4a7b5fe7d19456d8c7bf9eb Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Tue, 21 Nov 2023 20:28:04 -0800 Subject: [PATCH 16/31] Reformat --- deepeval/metrics/__init__.py | 2 +- deepeval/metrics/llm_eval_metric.py | 1 + deepeval/metrics/unbias_metric.py | 2 +- deepeval/scorer/__init__.py | 2 +- tests/test_llm_metric.py | 2 ++ 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deepeval/metrics/__init__.py b/deepeval/metrics/__init__.py index 38117dfb6..ba121bdbb 100644 --- a/deepeval/metrics/__init__.py +++ b/deepeval/metrics/__init__.py @@ -7,4 +7,4 @@ from .ragas_metric import RagasMetric from .unbias_metric import UnBiasedMetric from .non_toxic_metric import NonToxicMetric -from .hallucination_metric import HallucinationMetric \ No newline at end of file +from .hallucination_metric import HallucinationMetric diff --git a/deepeval/metrics/llm_eval_metric.py b/deepeval/metrics/llm_eval_metric.py index fead9b40b..e689c73c0 100644 --- a/deepeval/metrics/llm_eval_metric.py +++ b/deepeval/metrics/llm_eval_metric.py @@ -9,6 +9,7 @@ from pydantic import BaseModel from langchain.chat_models import ChatOpenAI + class LLMEvalMetricResponse(BaseModel): score: float diff --git a/deepeval/metrics/unbias_metric.py b/deepeval/metrics/unbias_metric.py index a01177d6b..2945a05e8 100644 --- a/deepeval/metrics/unbias_metric.py +++ b/deepeval/metrics/unbias_metric.py @@ -76,7 +76,7 @@ def measure(self, test_case: LLMTestCase, return_all_scores: bool = False): def is_successful(self): return self.success - + @property def __name__(self): return "Unbiased Metric" diff --git a/deepeval/scorer/__init__.py b/deepeval/scorer/__init__.py index 5bf277a68..9e55c60ab 100644 --- a/deepeval/scorer/__init__.py +++ b/deepeval/scorer/__init__.py @@ -1 +1 @@ -from .scorer import Scorer \ No newline at end of file +from .scorer import Scorer diff --git a/tests/test_llm_metric.py b/tests/test_llm_metric.py index 47122f263..ffaef5532 100644 --- a/tests/test_llm_metric.py +++ b/tests/test_llm_metric.py @@ -4,6 +4,7 @@ from deepeval.metrics import LLMEvalMetric from deepeval.evaluator import assert_test + def test_chat_completion(): """Test Chat Completion""" metric = LLMEvalMetric( @@ -24,6 +25,7 @@ def test_chat_completion(): assert_test(test_case, [metric]) + # # set openai api type # openai.api_type = "azure" From d94a854a3bf3464cb4f16367995784908925bd5c Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Tue, 21 Nov 2023 22:14:40 -0800 Subject: [PATCH 17/31] Sync dataset with Confident --- deepeval/api.py | 11 +++++++- deepeval/dataset/__init__.py | 1 + deepeval/dataset/api.py | 18 ++++++++++++ deepeval/{ => dataset}/dataset.py | 46 +++++++++++++++++++++++++++++-- deepeval/dataset/utils.py | 19 +++++++++++++ deepeval/plugins/plugin.py | 7 ++++- deepeval/test_case.py | 3 -- deepeval/test_run.py | 5 ++-- docs/docs/evaluation-metrics.mdx | 7 ++--- tests/data/dataset.csv | 6 ++++ tests/data/dataset.json | 29 +++++++++++++++++++ tests/test_dataset.py | 46 ++++++++++++++++--------------- 12 files changed, 161 insertions(+), 37 deletions(-) create mode 100644 deepeval/dataset/__init__.py create mode 100644 deepeval/dataset/api.py rename deepeval/{ => dataset}/dataset.py (86%) create mode 100644 deepeval/dataset/utils.py create mode 100644 tests/data/dataset.csv create mode 100644 tests/data/dataset.json diff --git a/deepeval/api.py b/deepeval/api.py index f80ea925d..236f613b9 100644 --- a/deepeval/api.py +++ b/deepeval/api.py @@ -6,6 +6,7 @@ from requests.adapters import HTTPAdapter, Response, Retry from deepeval.constants import API_KEY_ENV from deepeval.key_handler import KEY_FILE_HANDLER +from enum import Enum API_BASE_URL = "https://app.confident-ai.com/api" @@ -16,6 +17,11 @@ HTTP_RETRY_ALLOWED_METHODS = frozenset({"GET", "POST", "DELETE"}) +class Endpoints(Enum): + CREATE_DATASET_ENDPOINT = "/v1/dataset" + CREATE_TEST_RUN_ENDPOINT = "/v1/test-run" + + class Api: """Internal Api reference for handling http operations""" @@ -141,7 +147,10 @@ def _api_request( proxies=self.proxies, cert=self.cert, ) - + print(url) + print(method) + print(res.status_code) + print(res.reason) json = None if res.status_code == 200: try: diff --git a/deepeval/dataset/__init__.py b/deepeval/dataset/__init__.py new file mode 100644 index 000000000..ec2f47787 --- /dev/null +++ b/deepeval/dataset/__init__.py @@ -0,0 +1 @@ +from .dataset import EvaluationDataset diff --git a/deepeval/dataset/api.py b/deepeval/dataset/api.py new file mode 100644 index 000000000..4fcbaeb37 --- /dev/null +++ b/deepeval/dataset/api.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel, Field +from typing import Optional, List + + +class Golden(BaseModel): + input: str + actual_output: Optional[str] = Field(None, alias="actualOutput") + expected_output: Optional[str] = Field(None, alias="expectedOutput") + context: Optional[list] = Field(None) + + +class APIDataset(BaseModel): + alias: str + goldens: Optional[List[Golden]] = Field(default=None) + + +class CreateDatasetHttpResponse(BaseModel): + link: str diff --git a/deepeval/dataset.py b/deepeval/dataset/dataset.py similarity index 86% rename from deepeval/dataset.py rename to deepeval/dataset/dataset.py index d8a9fb1a4..25e9413b0 100644 --- a/deepeval/dataset.py +++ b/deepeval/dataset/dataset.py @@ -1,11 +1,17 @@ from typing import List, Optional from dataclasses import dataclass import pandas as pd +from rich.console import Console import json +import webbrowser +import os from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import evaluate +from deepeval.api import Api, Endpoints +from deepeval.dataset.utils import convert_test_cases_to_goldens +from deepeval.dataset.api import APIDataset, CreateDatasetHttpResponse @dataclass @@ -24,7 +30,7 @@ def __iter__(self): def evaluate(self, metrics: List[BaseMetric]): return evaluate(self.test_cases, metrics) - def load_csv( + def add_test_cases_from_csv_file( self, file_path: str, input_col_name: str, @@ -90,7 +96,7 @@ def _get_column_data(self, df: pd.DataFrame, col_name: str, default=None): else [default] * len(df) ) - def load_json_list( + def add_test_cases_from_json_file( self, file_path: str, input_key_name: str, @@ -152,7 +158,7 @@ def load_json_list( ) ) - def load_hf_dataset( + def add_test_cases_from_hf_dataset( self, dataset_name: str, input_field_name: str, @@ -218,3 +224,37 @@ def load_hf_dataset( context=context, ) ) + + def push(self, alias: str): + if len(self.test_cases) == 0: + raise ValueError( + "Unable to push empty dataset to Confident AI, there must be at least one test case in dataset" + ) + if os.path.exists(".deepeval"): + goldens = convert_test_cases_to_goldens(self.test_cases) + body = APIDataset(alias=alias, goldens=goldens).model_dump( + by_alias=True, exclude_none=True + ) + api = Api() + result = api.post_request( + endpoint=Endpoints.CREATE_DATASET_ENDPOINT.value, + body=body, + ) + response = CreateDatasetHttpResponse( + link=result["link"], + ) + link = response.link + console = Console() + console.print( + "✅ Dataset pushed to Confidnet AI! View on " + f"[link={link}]{link}[/link]" + ) + # webbrowser.open(link) + else: + raise Exception( + "To push dataset to Confident AI, run `deepeval login`" + ) + + # TODO + def pull(self, alias: str): + pass diff --git a/deepeval/dataset/utils.py b/deepeval/dataset/utils.py new file mode 100644 index 000000000..9f59e3103 --- /dev/null +++ b/deepeval/dataset/utils.py @@ -0,0 +1,19 @@ +from typing import List +from deepeval.dataset.api import Golden +from deepeval.test_case import LLMTestCase +from dataclasses import asdict + + +def convert_test_cases_to_goldens( + test_cases: List[LLMTestCase], +) -> List[Golden]: + goldens = [] + for test_case in test_cases: + golden = { + "input": test_case.input, + "actualOutput": test_case.actual_output, + "expectedOutput": test_case.expected_output, + "context": test_case.context, + } + goldens.append(Golden(**golden)) + return goldens diff --git a/deepeval/plugins/plugin.py b/deepeval/plugins/plugin.py index 5d034ebd5..48aae2614 100644 --- a/deepeval/plugins/plugin.py +++ b/deepeval/plugins/plugin.py @@ -8,7 +8,12 @@ def pytest_sessionstart(session: pytest.Session): test_run_manager.save_to_disk = True - test_run_manager.create_test_run(session.config.getoption("file_or_dir")[0]) + try: + test_run_manager.create_test_run( + session.config.getoption("file_or_dir")[0] + ) + except: + test_run_manager.create_test_run() @pytest.hookimpl(tryfirst=True) diff --git a/deepeval/test_case.py b/deepeval/test_case.py index fa107671a..54b66a67e 100644 --- a/deepeval/test_case.py +++ b/deepeval/test_case.py @@ -11,7 +11,6 @@ class LLMTestCaseParams(Enum): ACTUAL_OUTPUT = "actual_output" EXPECTED_OUTPUT = "expected_output" CONTEXT = "context" - RETRIEVAL_CONTEXT = "retrieval_context" @dataclass @@ -32,14 +31,12 @@ def __init__( actual_output: str, expected_output: Optional[str] = None, context: Optional[List[str]] = None, - retrieval_context: Optional[List[str]] = None, id: Optional[str] = None, ): super().__init__(id) self.input = input self.actual_output = actual_output self.expected_output = expected_output - self.retrieval_context = retrieval_context self.context = context def __post_init__(self): diff --git a/deepeval/test_run.py b/deepeval/test_run.py index 9bd532872..29d68e3a3 100644 --- a/deepeval/test_run.py +++ b/deepeval/test_run.py @@ -8,7 +8,7 @@ from deepeval.tracing import get_trace_stack from deepeval.constants import PYTEST_RUN_TEST_NAME from deepeval.decorators.hyperparameters import get_hyperparameters -from deepeval.api import Api +from deepeval.api import Api, Endpoints import shutil import webbrowser from deepeval.utils import delete_file_if_exists @@ -314,14 +314,13 @@ def post_test_run(self, test_run: TestRun): # TODO: change this, very hacky way to check if api key exists if os.path.exists(".deepeval"): try: - # make sure to exclude none for `context` to ensure it is handled properly body = test_run.model_dump(by_alias=True, exclude_none=True) except AttributeError: # Pydantic version below 2.0 body = test_run.dict(by_alias=True, exclude_none=True) api = Api() result = api.post_request( - endpoint="/v1/test-run", + endpoint=Endpoints.CREATE_TEST_RUN_ENDPOINT, body=body, ) response = TestRunHttpResponse( diff --git a/docs/docs/evaluation-metrics.mdx b/docs/docs/evaluation-metrics.mdx index 90a3b5653..891219d7d 100644 --- a/docs/docs/evaluation-metrics.mdx +++ b/docs/docs/evaluation-metrics.mdx @@ -163,7 +163,7 @@ You must implement `measure`, `is_successful`, and `name` yourself, as these are ## Hallucination Metric -Hallucination metric measures whether your LLM application is saying things that are untrue by comparing the `actual_output' to the provided `context`. You'll have to supply `context`when creating an`LLMTestCase` to evaluate hallucination. +The hallucination metric measures whether your LLM application outputs factually correct information by comparing the `actual_output` to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate hallucination. ```python import pytest @@ -182,9 +182,8 @@ metric = HallucinationMetric(minimum_score=0.5) run_test(test_case, [metric]) ``` -:::note -This metric uses vectara's hallucination evaluation model, which was specifically designed for summarization task. However, if you are just looking for factual consistency between input and output -where the application is not designed for QA task, this metric should work. +:::info +This metric uses vectara's hallucination evaluation model. ::: ## Answer Relevancy diff --git a/tests/data/dataset.csv b/tests/data/dataset.csv new file mode 100644 index 000000000..061740384 --- /dev/null +++ b/tests/data/dataset.csv @@ -0,0 +1,6 @@ +query,expected_output,context,actual_output +1,"Hello, world!","This is a greeting.","Context for 1; Additional info","Actual output for 1" +2,"OpenAI GPT-3","A powerful language model.","Context for 2; Additional info","Actual output for 2" +3,"CSV Example","Working with CSV data.","Context for 3; Additional info","Actual output for 3" +4,"Python Programming","Coding in Python.","Context for 4; Additional info","Actual output for 4" +5,"Data Science","Analyzing data.","Context for 5; Additional info","Actual output for 5" diff --git a/tests/data/dataset.json b/tests/data/dataset.json new file mode 100644 index 000000000..26b745ab3 --- /dev/null +++ b/tests/data/dataset.json @@ -0,0 +1,29 @@ +[ + { + "query": "1", + "expected_output": "Hello, world!", + "actual_output": "Actual output for 1" + }, + { + "query": "2", + "context": ["A powerful language model.", "Context for 2", "Additional info"], + "actual_output": "Actual output for 2" + }, + { + "query": "3", + "expected_output": "CSV Example", + "context": ["Working with CSV data.", "Context for 3", "Additional info"], + "actual_output": "Actual output for 3" + }, + { + "query": "4", + "expected_output": "Python Programming", + "actual_output": "Actual output for 4" + }, + { + "query": "5", + "expected_output": "Data Science", + "context": ["Analyzing data.", "Context for 5", "Additional info"], + "actual_output": "Actual output for 5" + } +] diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 8c300664d..5e83db2e7 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -1,28 +1,30 @@ -# import os +import os -# import pytest +import pytest +from deepeval.dataset import EvaluationDataset +dataset = EvaluationDataset() -# def test_evaluation_dataset(): -# from deepeval.dataset import EvaluationDataset -# csv_filename = "sample.csv" +def test_create_dataset(): + module_b_dir = os.path.dirname(os.path.realpath(__file__)) -# csv_file = """id,query,expected_output -# 1,"Hello, world!","This is a greeting." -# 2,"OpenAI GPT-3","A powerful language model." -# 3,"CSV Example","Working with CSV data." -# 4,"Python Programming","Coding in Python." -# 5,"Data Science","Analyzing data." -# """ + file_path = os.path.join(module_b_dir, "data", "dataset.csv") -# with open(csv_filename, "w") as file: -# file.write(csv_file) - -# dataset = EvaluationDataset.from_csv( -# csv_filename, -# query_column="query", -# expected_output_column="expected_output", -# id_column="id", -# ) -# assert len(dataset) == 5 + dataset.add_test_cases_from_csv_file( + file_path, + input_col_name="query", + actual_output_col_name="actual_output", + expected_output_col_name="expected_output", + context_col_name="context", + ) + assert len(dataset.test_cases) == 5, "Test Cases not loaded from CSV" + file_path = os.path.join(module_b_dir, "data", "dataset.json") + dataset.add_test_cases_from_json_file( + file_path, + input_key_name="query", + actual_output_key_name="actual_output", + expected_output_key_name="expected_output", + context_key_name="context", + ) + assert len(dataset.test_cases) == 10, "Test Cases not loaded from JSON" From 309f42eac01c8dd9bb40f24a3df0ca8bbaebe071 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Tue, 21 Nov 2023 22:15:32 -0800 Subject: [PATCH 18/31] removed prints --- deepeval/api.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deepeval/api.py b/deepeval/api.py index 236f613b9..b9c703192 100644 --- a/deepeval/api.py +++ b/deepeval/api.py @@ -147,10 +147,6 @@ def _api_request( proxies=self.proxies, cert=self.cert, ) - print(url) - print(method) - print(res.status_code) - print(res.reason) json = None if res.status_code == 200: try: From bde7c9ae1f757ebce13cf1259fa5234ef1daf704 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 03:49:03 -0800 Subject: [PATCH 19/31] Deprecate Factual Consistency --- deepeval/chat_completion/retry.py | 10 +- deepeval/metrics/__init__.py | 12 +- deepeval/metrics/hallucination_metric.py | 4 + deepeval/metrics/llm_eval_metric.py | 8 +- deepeval/metrics/ragas_metric.py | 10 +- deepeval/test_run.py | 2 +- docs/docs/evaluation-datasets.mdx | 12 +- docs/docs/evaluation-metrics.mdx | 233 ++-- docs/docs/evaluation-test-cases.mdx | 14 +- docs/docs/evaluation-tracing.mdx | 6 +- docs/docs/getting-started.mdx | 26 +- docs/docs/integrations-llamaindex.mdx | 9 +- examples/getting_started/test_example.py | 12 +- examples/tracing/test_chatbot.py | 6 +- poetry.lock | 1480 +++++++++++----------- pyproject.toml | 2 +- tests/test_bias.py | 23 +- tests/test_chatbot_example.py | 35 - tests/test_factual_consistency.py | 39 - tests/test_hallucination_metric.py | 22 + tests/test_quickstart.py | 8 +- tests/test_ragas.py | 45 +- 22 files changed, 993 insertions(+), 1025 deletions(-) delete mode 100644 tests/test_chatbot_example.py delete mode 100644 tests/test_factual_consistency.py diff --git a/deepeval/chat_completion/retry.py b/deepeval/chat_completion/retry.py index e53ae8320..501bd5aa0 100644 --- a/deepeval/chat_completion/retry.py +++ b/deepeval/chat_completion/retry.py @@ -1,23 +1,15 @@ from typing import Callable, Any -import openai import time -import os -import sys def call_openai_with_retry( callable: Callable[[], Any], max_retries: int = 2 ) -> Any: - if not openai.api_key: - raise ValueError( - "OpenAI API key is not set. Please ensure it's set in your environment variables or passed explicitly." - ) - for _ in range(max_retries): try: response = callable() return response - except openai.error.OpenAIError as e: + except Exception as e: print(f"An error occurred: {e}. Retrying...") time.sleep(2) continue diff --git a/deepeval/metrics/__init__.py b/deepeval/metrics/__init__.py index ba121bdbb..54eb2f56b 100644 --- a/deepeval/metrics/__init__.py +++ b/deepeval/metrics/__init__.py @@ -1,10 +1,18 @@ from .base_metric import BaseMetric from .answer_relevancy import AnswerRelevancyMetric from .base_metric import BaseMetric -from .factual_consistency import FactualConsistencyMetric from .judgemental_gpt import JudgementalGPT from .llm_eval_metric import LLMEvalMetric -from .ragas_metric import RagasMetric +from .ragas_metric import ( + RagasMetric, + ContextualRelevancyMetric, + FaithfulnessMetric, + ContextRecallMetric, + ConcisenessMetric, + CorrectnessMetric, + CoherenceMetric, + MaliciousnessMetric, +) from .unbias_metric import UnBiasedMetric from .non_toxic_metric import NonToxicMetric from .hallucination_metric import HallucinationMetric diff --git a/deepeval/metrics/hallucination_metric.py b/deepeval/metrics/hallucination_metric.py index 9beb07b2c..3f63c6910 100644 --- a/deepeval/metrics/hallucination_metric.py +++ b/deepeval/metrics/hallucination_metric.py @@ -31,3 +31,7 @@ def measure(self, test_case: LLMTestCase): def is_successful(self) -> bool: return self.success + + @property + def __name__(self): + return "Hallucination" diff --git a/deepeval/metrics/llm_eval_metric.py b/deepeval/metrics/llm_eval_metric.py index e689c73c0..30f53f6a9 100644 --- a/deepeval/metrics/llm_eval_metric.py +++ b/deepeval/metrics/llm_eval_metric.py @@ -35,10 +35,6 @@ def __init__( if "deployment_id" in kwargs: self.deployment_id = kwargs["deployment_id"] - @property - def __name__(self): - return self.name - def measure(self, test_case: LLMTestCase): """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf""" @@ -119,3 +115,7 @@ def evaluate(self, test_case: LLMTestCase): pass return total_scores / count + + @property + def __name__(self): + return self.name diff --git a/deepeval/metrics/ragas_metric.py b/deepeval/metrics/ragas_metric.py index 89adb222a..a4eabee67 100644 --- a/deepeval/metrics/ragas_metric.py +++ b/deepeval/metrics/ragas_metric.py @@ -48,6 +48,7 @@ def measure(self, test_case: LLMTestCase): } dataset = Dataset.from_dict(data) + print("!!!!!!!!!!!!!!!!") # Evaluate the dataset using Ragas scores = evaluate(dataset, metrics=self.metrics) @@ -155,6 +156,7 @@ def measure(self, test_case: LLMTestCase): "id": [[test_case.id]], } dataset = Dataset.from_dict(data) + print("!!!!!!!!!!!!!!!!") scores = evaluate(dataset, metrics=self.metrics) faithfulness_score = scores["faithfulness"] self.success = faithfulness_score >= self.minimum_score @@ -307,7 +309,7 @@ def measure(self, test_case: LLMTestCase): data = { "ground_truths": [[test_case.expected_output]], - "contexts": [[test_case.context]], + "contexts": [test_case.context], "question": [test_case.input], "answer": [test_case.actual_output], "id": [[test_case.id]], @@ -359,7 +361,7 @@ def measure(self, test_case: LLMTestCase): data = { "ground_truths": [[test_case.expected_output]], - "contexts": [[test_case.context]], + "contexts": [test_case.context], "question": [test_case.input], "answer": [test_case.actual_output], "id": [[test_case.id]], @@ -411,7 +413,7 @@ def measure(self, test_case: LLMTestCase): data = { "ground_truths": [[test_case.expected_output]], - "contexts": [[test_case.context]], + "contexts": [test_case.context], "question": [test_case.input], "answer": [test_case.actual_output], "id": [[test_case.id]], @@ -463,7 +465,7 @@ def measure(self, test_case: LLMTestCase): data = { "ground_truths": [[test_case.expected_output]], - "contexts": [[test_case.context]], + "contexts": [test_case.context], "question": [test_case.input], "answer": [test_case.actual_output], "id": [[test_case.id]], diff --git a/deepeval/test_run.py b/deepeval/test_run.py index 29d68e3a3..a8ac6ebd1 100644 --- a/deepeval/test_run.py +++ b/deepeval/test_run.py @@ -320,7 +320,7 @@ def post_test_run(self, test_run: TestRun): body = test_run.dict(by_alias=True, exclude_none=True) api = Api() result = api.post_request( - endpoint=Endpoints.CREATE_TEST_RUN_ENDPOINT, + endpoint=Endpoints.CREATE_TEST_RUN_ENDPOINT.value, body=body, ) response = TestRunHttpResponse( diff --git a/docs/docs/evaluation-datasets.mdx b/docs/docs/evaluation-datasets.mdx index 9102134bb..9a093ef78 100644 --- a/docs/docs/evaluation-datasets.mdx +++ b/docs/docs/evaluation-datasets.mdx @@ -70,7 +70,7 @@ deepeval login ```python title="test_bulk.py" from deepeval.test_case import LLMTestCase -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.metric.answer_relevancy import AnswerRelevancyMetric from deepeval.evaluate import assert_test @@ -81,9 +81,9 @@ dataset = [...] dataset, ) def test_customer_chatbot(test_case: LLMTestCase): - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.3) + hallucination_metric = HallucinationMetric(minimum_score=0.3) answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) - assert_test(test_case, [factual_consistency_metric, answer_relevancy_metric]) + assert_test(test_case, [hallucination_metric, answer_relevancy_metric]) ``` To run several tests cases at once in parallel, use the optional `-n` flag followed by a number (that determines the number of processes that will be used) when executing `deepeval test run`: @@ -98,12 +98,12 @@ Alternately, you can use deepeval's `evaluate` function to evaluate datasets. Th ```python from deepeval.evaluator import evaluate -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.metric.answer_relevancy import AnswerRelevancyMetric dataset = [...] -factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.3) +hallucination_metric = HallucinationMetric(minimum_score=0.3) answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) -evaluate(dataset, [factual_consistency_metric, answer_relevancy_metric]) +evaluate(dataset, [hallucination_metric, answer_relevancy_metric]) ``` diff --git a/docs/docs/evaluation-metrics.mdx b/docs/docs/evaluation-metrics.mdx index 891219d7d..b4555ffda 100644 --- a/docs/docs/evaluation-metrics.mdx +++ b/docs/docs/evaluation-metrics.mdx @@ -31,159 +31,61 @@ All of `deepeval`'s default metrics output a score between 0-1, and require a `m Our suggestion is to begin with custom LLM evaluated metrics (which frequently surpass and offer more versatility than leading NLP models), and gradually transition to `deepeval`'s default metrics when feasible. We recommend using default metrics as an optimization to your evaluation workflow because they are more cost-effective. ::: -## Custom LLM Evaluated Metrics +## Hallucination -A custom LLM evalated metric, is a custom metric whose evaluation is powered by LLMs. To create a custom metric that uses LLMs for evaluation, simply instantiate an `LLMEvalMetric` class: +Hallucination determines whether your LLM application outputs factually correct information by comparing the `actual_output` to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate hallucination. ```python -from deepeval.metrics import LLMEvalMetric -from deepeval.test_case import LLMTestCase, LLMTestCaseParams - -summarization_metric = LLMEvalMetric( - name="Summarization", - criteria="Summarization - determine if the actual output is an accurate and concise summarization of the input.", - evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], - minimum_score=0.5, - model="gpt-4" -) -``` - -There are three mandatory and two optional parameters required when instantiating an `LLMEvalMetric` class: - -- `name`: name of metric -- `criteria`: a description outlining the specific evaluation aspects for each test case. -- `evaluation_params`: a list of type `LLMTestCaseParams`. Include only the parameters that are relevant for evaluation. -- [Optional] `minimum_score`: the passing threshold, defaulted to 0.5. -- [Optional] `model`: the model name. This is defaulted to 'gpt-4-1106-preview' and we currently only support models from OpenAI. - -All instances of `LLMEvalMetric` returns a score ranging from 0 - 1. A metric is only successful if the evaluation score is equal to or greater than `minimum_score`. - -:::danger -For accurate and valid results, only the parameters that are mentioned in `criteria` should be included as a member of `evaluation_params`. -::: - -By default, `LLMEvalMetric` is evaluated using `GPT-4` from OpenAI. Azure OpenAI endpoints are also supported via LLMEvalMetric, the `deployment_id` parameter should be passed -during instantiation of `LLMEvalMetric` class +import pytest +from deepeval.metrics import HallucinationMetric +from deepeval.test_case import LLMTestCase +from deepeval.evaluator import run_test -```python -summarization_metric = LLMEvalMetric( - name="Summarization", - criteria="Summarization - determine if the actual output is an accurate and concise summarization of the input.", - evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], - minimum_score=0.5, - deployment_id="your-deployment-id" -) -``` +# Replace this with the actual documents that you are passing as input to your LLM. +context=["A man with blond-hair, and a brown shirt drinking out of a public water fountain."] -:::note -Don't forget to set your Azure OpenAI key +# Replace this with the actual output from your LLM application +actual_output="A blond drinking water in public.", -```python -openai.api_key = "your-Azure-OpenAI-API-key" +test_case = LLMTestCase(input="placeholder", actual_output=actual_output, context=context) +metric = HallucinationMetric(minimum_score=0.5) +run_test(test_case, [metric]) ``` +:::info +This metric uses vectara's hallucination evaluation model. ::: -## JudgementalGPT +## LLM Evaluated Metrics -`JudgementalGPT` is an LLM agent developed in-house by [Confident AI](https://confident-ai.com) that's dedicated to evaluation and is superior to `LLMEvalMetric`. While it operates similarly to `LLMEvalMetric` by utilizing LLMs for scoring, it: - -- offers enhanced accuracy and reliability. -- is capable of generating justifications for its scores -- has the ability to conditionally execute code that helps detect logical fallacies during evaluations - -To use `JudgementalGPT`, start by logging into Confident AI: - -```console -deepeval login -``` - -Then paste in the following code to define a metric powered by `JudgementalGPT`: +A LLM evalated metric, is a custom metric whose score is calculated by LLMs. To create a custom metric that uses LLMs for evaluation, simply instantiate an `LLMEvalMetric` class and define an evaluation criteria in natural language: ```python -from deepeval.metrics import JudgementalGPT +from deepeval.metrics import LLMEvalMetric from deepeval.test_case import LLMTestCase, LLMTestCaseParams -code_correctness_metric = JudgementalGPT( - name="Code Correctness", - criteria="Code Correctness - determine whether the python code in the 'actual output' produces a valid JSON.", - evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT], +summarization_metric = LLMEvalMetric( + name="Summarization", + criteria="Summarization - determine if the actual output is an accurate and concise summarization of the input.", + evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], minimum_score=0.5, + model="gpt-4" ) ``` -Under the hood, `JudgementalGPT(...)` sends a request to Confident AI's servers that hosts `JudgementalGPT`. `JudgementalGPT` accepts four arguments: +There are three mandatory and two optional parameters required when instantiating an `LLMEvalMetric` class: - `name`: name of metric - `criteria`: a description outlining the specific evaluation aspects for each test case. - `evaluation_params`: a list of type `LLMTestCaseParams`. Include only the parameters that are relevant for evaluation. - [Optional] `minimum_score`: the passing threshold, defaulted to 0.5. +- [Optional] `model`: the model name. This is defaulted to 'gpt-4-1106-preview' and we currently only support models from (Azure) OpenAI. +- [Optional] `deployment_id`: the deployment name you chose when you deployed Azure OpenAI. Only required if you're using Azure OpenAI. -## Custom Metrics - -You can implement your own evaluator (for example, your own GPT evaluator) by creating a custom metric. All custom metrics are automatically integrated with Confident AI. - -```python -from deepeval.metrics import BaseMetric - -# Inherit BaseMetric -class LengthMetric(BaseMetric): - # This metric checks if the output length is greater than 10 characters - def __init__(self, max_length: int=10): - self.minimum_score = max_length - - def measure(self, test_case: LLMTestCase): - # Set self.success and self.score in the "measure" method - self.success = len(test_case.actual_output) > self.minimum_score - if self.success: - self.score = 1 - else: - self.score = 0 - return self.score - - def is_successful(self): - return self.success - - @property - def name(self): - return "Length" -``` - -Noticed that we accessed `test_case.actual_output` in `measure`. you will have to supply the optional `context` or `expected_output` arguments in the `LLMTestCase` depending on your `measure` implementation. - -In this example, you would instantiate `LengthMetric` as follows: - -```python -length_metric = LengthMetric(max_length=20) -``` - -:::info -You must implement `measure`, `is_successful`, and `name` yourself, as these are abstract methods and properties inherited from `Metric`. -::: - -## Hallucination Metric - -The hallucination metric measures whether your LLM application outputs factually correct information by comparing the `actual_output` to the provided `context`. You'll have to supply `context` when creating an `LLMTestCase` to evaluate hallucination. - -```python -import pytest -from deepeval.metrics import HallucinationMetric -from deepeval.test_case import LLMTestCase -from deepeval.evaluator import run_test - -# Replace this with the actual documents that you are passing as input to your LLM. -context=["A man with blond-hair, and a brown shirt drinking out of a public water fountain."] - -# Replace this with the actual output from your LLM application -actual_output="A blond drinking water in public.", - -test_case = LLMTestCase(input="placeholder", actual_output=actual_output, context=context) -metric = HallucinationMetric(minimum_score=0.5) -run_test(test_case, [metric]) -``` +All instances of `LLMEvalMetric` returns a score ranging from 0 - 1. A metric is only successful if the evaluation score is equal to or greater than `minimum_score`. -:::info -This metric uses vectara's hallucination evaluation model. +:::danger +For accurate and valid results, only the parameters that are mentioned in `criteria` should be included as a member of `evaluation_params`. ::: ## Answer Relevancy @@ -341,3 +243,80 @@ test_case = LLMTestCase( run_test(test_case, [unbias_metric]) ``` + +## Custom Metrics + +You can implement your own evaluator (for example, your own GPT evaluator) by creating a custom metric. All custom metrics are automatically integrated with Confident AI. + +```python +from deepeval.metrics import BaseMetric + +# Inherit BaseMetric +class LengthMetric(BaseMetric): + # This metric checks if the output length is greater than 10 characters + def __init__(self, max_length: int=10): + self.minimum_score = max_length + + def measure(self, test_case: LLMTestCase): + # Set self.success and self.score in the "measure" method + self.success = len(test_case.actual_output) > self.minimum_score + if self.success: + self.score = 1 + else: + self.score = 0 + return self.score + + def is_successful(self): + return self.success + + @property + def name(self): + return "Length" +``` + +Noticed that we accessed `test_case.actual_output` in `measure`. you will have to supply the optional `context` or `expected_output` arguments in the `LLMTestCase` depending on your `measure` implementation. + +In this example, you would instantiate `LengthMetric` as follows: + +```python +length_metric = LengthMetric(max_length=20) +``` + +:::info +You must implement `measure`, `is_successful`, and `name` yourself, as these are abstract methods and properties inherited from `Metric`. +::: + +## JudgementalGPT + +`JudgementalGPT` is an LLM agent developed in-house by [Confident AI](https://confident-ai.com) that's dedicated to evaluation and is superior to `LLMEvalMetric`. While it operates similarly to `LLMEvalMetric` by utilizing LLMs for scoring, it: + +- offers enhanced accuracy and reliability. +- is capable of generating justifications for its scores +- has the ability to conditionally execute code that helps detect logical fallacies during evaluations + +To use `JudgementalGPT`, start by logging into Confident AI: + +```console +deepeval login +``` + +Then paste in the following code to define a metric powered by `JudgementalGPT`: + +```python +from deepeval.metrics import JudgementalGPT +from deepeval.test_case import LLMTestCase, LLMTestCaseParams + +code_correctness_metric = JudgementalGPT( + name="Code Correctness", + criteria="Code Correctness - determine whether the python code in the 'actual output' produces a valid JSON.", + evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT], + minimum_score=0.5, +) +``` + +Under the hood, `JudgementalGPT(...)` sends a request to Confident AI's servers that hosts `JudgementalGPT`. `JudgementalGPT` accepts four arguments: + +- `name`: name of metric +- `criteria`: a description outlining the specific evaluation aspects for each test case. +- `evaluation_params`: a list of type `LLMTestCaseParams`. Include only the parameters that are relevant for evaluation. +- [Optional] `minimum_score`: the passing threshold, defaulted to 0.5. diff --git a/docs/docs/evaluation-test-cases.mdx b/docs/docs/evaluation-test-cases.mdx index 275c555b5..e2b1d4c6b 100644 --- a/docs/docs/evaluation-test-cases.mdx +++ b/docs/docs/evaluation-test-cases.mdx @@ -181,7 +181,7 @@ Remember, `context` is the ideal retrieval results for a given input and typical ```python # A hypothetical LLM application example import chatbot -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.evaluator import run_test from deepeval.test_case import LLMTestCase @@ -202,7 +202,7 @@ test_case = LLMTestCase( context=context ) -metric = FactualConsistencyMetric(minimum_score=0.7) +metric = HallucinationMetric(minimum_score=0.7) run_test(test_case, [metric]) ``` @@ -224,7 +224,7 @@ A test case passes only if all metrics meet their respective evaluation criterio ```python title="test_assert_example.py" # A hypothetical LLM application example import chatbot -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.evaluator import assert_test from deepeval.test_case import LLMTestCase @@ -245,7 +245,7 @@ def test_case_1(): expected_output="Me, ruff!", context=context ) - metric = FactualConsistencyMetric(minimum_score=0.7) + metric = HallucinationMetric(minimum_score=0.7) assert_test(test_case, metrics=[metric]) def test_case_2(): @@ -259,7 +259,7 @@ def test_case_2(): expected_output="Me, ruff!", context=context ) - metric = FactualConsistencyMetric(minimum_score=0.7) + metric = HallucinationMetric(minimum_score=0.7) assert_test(test_case, metrics=[metric]) ``` @@ -276,7 +276,7 @@ Lastly, `deepeval` offers an `evaluate` function to evaluate multiple test cases ```python # A hypothetical LLM application example import chatbot -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.evaluator import evaluate from deepeval.test_case import LLMTestCase @@ -307,7 +307,7 @@ second_test_case = LLMTestCase( dataset = [first_test_case, second_test_case] -metric = FactualConsistencyMetric(minimum_score=0.7) +metric = HallucinationMetric(minimum_score=0.7) evaluate(dataset, [metric]) ``` diff --git a/docs/docs/evaluation-tracing.mdx b/docs/docs/evaluation-tracing.mdx index afdef1130..c41839602 100644 --- a/docs/docs/evaluation-tracing.mdx +++ b/docs/docs/evaluation-tracing.mdx @@ -133,12 +133,12 @@ Continuning from the previous code snippet where you've defined your `Chatbot` c import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.evaluator import assert_test chatbot = Chatbot() -def test_factual_consistency(): +def test_hallucination(): context = [ "Be a natural-born citizen of the United States.", "Be at least 35 years old.", @@ -146,7 +146,7 @@ def test_factual_consistency(): ] input = "What are the requimrents to be president?" - metric = FactualConsistencyMetric(minimum_score=0.8) + metric = HallucinationMetric(minimum_score=0.8) test_case = LLMTestCase( input=input, actual_output=chatbot.query(user_input=input), diff --git a/docs/docs/getting-started.mdx b/docs/docs/getting-started.mdx index dc3b59cf9..d380a3d6b 100644 --- a/docs/docs/getting-started.mdx +++ b/docs/docs/getting-started.mdx @@ -50,19 +50,19 @@ Run `touch test_example.py` to create a test file in your root directory. Open ` ```python title="test_example.py" import pytest -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import assert_test -def test_factual_consistency(): +def test_hallucination(): input = "What if these shoes don't fit?" context = ["All customers are eligible for a 30 day full refund at no extra cost."] # Replace this with the actual output of your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(minimum_score=0.7) test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) - assert_test(test_case, [factual_consistency_metric]) + assert_test(test_case, [hallucination_metric]) ``` Run `deepeval test run` from the root directory of your project: @@ -74,7 +74,7 @@ deepeval test run test_example.py **Congratulations! Your test case should have passed ✅** Let's breakdown what happened. - The variable `input` mimics a user input, and `actual_output` is a placeholder for what your application's supposed to output based on this input. -- The variable `context` contains the relevant information from your knowledge base, and `FactualConsistencyMetric(minimum_score=0.7)` is an default metric provided by DeepEval for you to evaluate how factually correct your application's output is based on the provided context. +- The variable `context` contains the relevant information from your knowledge base, and `HallucinationMetric(minimum_score=0.7)` is an default metric provided by DeepEval for you to evaluate how factually correct your application's output is based on the provided context. - All metric scores range from 0 - 1, which the `minimum_score=0.7` threshold ultimately determines if your test have passed or not. :::note @@ -164,7 +164,7 @@ Run `deepeval test run` from the root directory of your project again: deepeval test run test_example.py ``` -You should see both `test_factual_consistency` and `test_length` passing. +You should see both `test_hallucination` and `test_length` passing. **Two things to note:** @@ -173,7 +173,7 @@ You should see both `test_factual_consistency` and `test_length` passing. ## Combine Your Metrics -You might've noticed we have duplicated test cases for both `test_factual_consistency` and `test_length` (ie. they have the same input and expected output). To avoid this redundancy, `deepeval` offers an easy way to apply as many metrics as you wish on a single test case. +You might've noticed we have duplicated test cases for both `test_hallucination` and `test_length` (ie. they have the same input and expected output). To avoid this redundancy, `deepeval` offers an easy way to apply as many metrics as you wish on a single test case. ```python title="test_example.py" ... @@ -184,7 +184,7 @@ def test_everything(): # Replace this with the actual output of your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(minimum_score=0.7) length_metric = LengthMetric(max_length=10) summarization_metric = LLMEvalMetric( name="Summarization", @@ -194,7 +194,7 @@ def test_everything(): ) test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) - assert_test(test_case, [factual_consistency_metric, length_metric, summarization_metric]) + assert_test(test_case, [hallucination_metric, length_metric, summarization_metric]) ``` In this scenario, `test_everything` only passes if all metrics are passing. Run `deepeval test run` again to see the results: @@ -221,7 +221,7 @@ Utilize the `@pytest.mark.parametrize` decorator to loop through and evaluate yo ```python title="test_bulk.py" import pytest -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import assert_test @@ -254,7 +254,7 @@ dataset = [ "test_case", dataset, ) -def test_factual_consistency(test_case: dict): +def test_hallucination(test_case: dict): input = test_case.get("input") expected_output = test_case.get("expected_output") context = test_case.get("context") @@ -262,14 +262,14 @@ def test_factual_consistency(test_case: dict): # Replace this with the actual output of your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(minimum_score=0.7) test_case = LLMTestCase( input=input, actual_output=actual_output, expected_output=expected_output, context=context ) - assert_test(test_case, [factual_consistency_metric]) + assert_test(test_case, [hallucination_metric]) ``` To run test cases at once in parallel, use the optional `-n` flag followed by a number (that determines the number of processes that will be used) when executing `deepeval test run`: diff --git a/docs/docs/integrations-llamaindex.mdx b/docs/docs/integrations-llamaindex.mdx index c442b35d0..b3908cb2b 100644 --- a/docs/docs/integrations-llamaindex.mdx +++ b/docs/docs/integrations-llamaindex.mdx @@ -12,7 +12,7 @@ DeepEval integrates nicely with LlamaIndex's `ResponseEvaluator` class. Below is from llama_index.response.schema import Response from typing import List from llama_index.schema import Document -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from llama_index import ( TreeIndex, VectorStoreIndex, @@ -56,8 +56,9 @@ In this example, we show you how to write a factual consistency check. ```python from deepeval.test_case import LLMTestCase +from deepeval.metrics import HallucinationMetric -class FactualConsistencyResponseEvaluator: +class HallucinationResponseEvaluator: def get_context(self, response: Response) -> List[Document]: """Get context information from given Response object using source nodes. @@ -78,7 +79,7 @@ class FactualConsistencyResponseEvaluator: # Evaluate factual consistency metrics answer = str(response) - metric = FactualConsistencyMetric() + metric = HallucinationMetric() context = self.get_context(response) context = " ".join([d.text for d in context]) test_case = LLMTestCase(input="This is an example input", context=context, actual_output=answer) @@ -88,7 +89,7 @@ class FactualConsistencyResponseEvaluator: else: return "NO" -evaluator = FactualConsistencyResponseEvaluator() +evaluator = HallucinationResponseEvaluator() ``` You can then evaluate as such: diff --git a/examples/getting_started/test_example.py b/examples/getting_started/test_example.py index c0810ee92..addc99ed8 100644 --- a/examples/getting_started/test_example.py +++ b/examples/getting_started/test_example.py @@ -1,13 +1,13 @@ import pytest from deepeval.test_case import LLMTestCase, LLMTestCaseParams from deepeval.evaluator import assert_test -from deepeval.metrics import BaseMetric, LLMEvalMetric, FactualConsistencyMetric +from deepeval.metrics import BaseMetric, LLMEvalMetric, HallucinationMetric import deepeval # To run this file: deepeval test run .py -def test_factual_consistency(): +def test_hallucination(): input = "What if these shoes don't fit?" context = [ "All customers are eligible for a 30 day full refund at no extra cost." @@ -15,11 +15,11 @@ def test_factual_consistency(): # Replace this with the actual output from your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(minimum_score=0.7) test_case = LLMTestCase( input=input, actual_output=actual_output, context=context ) - assert_test(test_case, [factual_consistency_metric]) + assert_test(test_case, [hallucination_metric]) def test_summarization(): @@ -80,7 +80,7 @@ def test_everything(): # Replace this with the actual output from your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(minimum_score=0.7) length_metric = LengthMetric(max_length=10) summarization_metric = LLMEvalMetric( name="Summarization", @@ -97,7 +97,7 @@ def test_everything(): ) assert_test( test_case, - [factual_consistency_metric, length_metric, summarization_metric], + [hallucination_metric, length_metric, summarization_metric], ) diff --git a/examples/tracing/test_chatbot.py b/examples/tracing/test_chatbot.py index bad61ba88..98861909b 100644 --- a/examples/tracing/test_chatbot.py +++ b/examples/tracing/test_chatbot.py @@ -63,13 +63,13 @@ def query(self, user_input=input): import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.evaluator import assert_test chatbot = Chatbot() -def test_factual_consistency(): +def test_hallucination(): context = [ "Be a natural-born citizen of the United States.", "Be at least 35 years old.", @@ -77,7 +77,7 @@ def test_factual_consistency(): ] input = "What are the requimrents to be president?" - metric = FactualConsistencyMetric(minimum_score=0.8) + metric = HallucinationMetric(minimum_score=0.8) test_case = LLMTestCase( input=input, actual_output=chatbot.query(user_input=input), diff --git a/poetry.lock b/poetry.lock index df3a612c4..73da65e40 100644 --- a/poetry.lock +++ b/poetry.lock @@ -13,111 +13,99 @@ files = [ [[package]] name = "aiohttp" -version = "3.8.6" +version = "3.9.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41d55fc043954cddbbd82503d9cc3f4814a40bcef30b3569bc7b5e34130718c1"}, - {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1d84166673694841d8953f0a8d0c90e1087739d24632fe86b1a08819168b4566"}, - {file = "aiohttp-3.8.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:253bf92b744b3170eb4c4ca2fa58f9c4b87aeb1df42f71d4e78815e6e8b73c9e"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fd194939b1f764d6bb05490987bfe104287bbf51b8d862261ccf66f48fb4096"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c5f938d199a6fdbdc10bbb9447496561c3a9a565b43be564648d81e1102ac22"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2817b2f66ca82ee699acd90e05c95e79bbf1dc986abb62b61ec8aaf851e81c93"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fa375b3d34e71ccccf172cab401cd94a72de7a8cc01847a7b3386204093bb47"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9de50a199b7710fa2904be5a4a9b51af587ab24c8e540a7243ab737b45844543"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e1d8cb0b56b3587c5c01de3bf2f600f186da7e7b5f7353d1bf26a8ddca57f965"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8e31e9db1bee8b4f407b77fd2507337a0a80665ad7b6c749d08df595d88f1cf5"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7bc88fc494b1f0311d67f29fee6fd636606f4697e8cc793a2d912ac5b19aa38d"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ec00c3305788e04bf6d29d42e504560e159ccaf0be30c09203b468a6c1ccd3b2"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad1407db8f2f49329729564f71685557157bfa42b48f4b93e53721a16eb813ed"}, - {file = "aiohttp-3.8.6-cp310-cp310-win32.whl", hash = "sha256:ccc360e87341ad47c777f5723f68adbb52b37ab450c8bc3ca9ca1f3e849e5fe2"}, - {file = "aiohttp-3.8.6-cp310-cp310-win_amd64.whl", hash = "sha256:93c15c8e48e5e7b89d5cb4613479d144fda8344e2d886cf694fd36db4cc86865"}, - {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e2f9cc8e5328f829f6e1fb74a0a3a939b14e67e80832975e01929e320386b34"}, - {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e6a00ffcc173e765e200ceefb06399ba09c06db97f401f920513a10c803604ca"}, - {file = "aiohttp-3.8.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:41bdc2ba359032e36c0e9de5a3bd00d6fb7ea558a6ce6b70acedf0da86458321"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14cd52ccf40006c7a6cd34a0f8663734e5363fd981807173faf3a017e202fec9"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2d5b785c792802e7b275c420d84f3397668e9d49ab1cb52bd916b3b3ffcf09ad"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1bed815f3dc3d915c5c1e556c397c8667826fbc1b935d95b0ad680787896a358"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96603a562b546632441926cd1293cfcb5b69f0b4159e6077f7c7dbdfb686af4d"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d76e8b13161a202d14c9584590c4df4d068c9567c99506497bdd67eaedf36403"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e3f1e3f1a1751bb62b4a1b7f4e435afcdade6c17a4fd9b9d43607cebd242924a"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:76b36b3124f0223903609944a3c8bf28a599b2cc0ce0be60b45211c8e9be97f8"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:a2ece4af1f3c967a4390c284797ab595a9f1bc1130ef8b01828915a05a6ae684"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:16d330b3b9db87c3883e565340d292638a878236418b23cc8b9b11a054aaa887"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:42c89579f82e49db436b69c938ab3e1559e5a4409eb8639eb4143989bc390f2f"}, - {file = "aiohttp-3.8.6-cp311-cp311-win32.whl", hash = "sha256:efd2fcf7e7b9d7ab16e6b7d54205beded0a9c8566cb30f09c1abe42b4e22bdcb"}, - {file = "aiohttp-3.8.6-cp311-cp311-win_amd64.whl", hash = "sha256:3b2ab182fc28e7a81f6c70bfbd829045d9480063f5ab06f6e601a3eddbbd49a0"}, - {file = "aiohttp-3.8.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fdee8405931b0615220e5ddf8cd7edd8592c606a8e4ca2a00704883c396e4479"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d25036d161c4fe2225d1abff2bd52c34ed0b1099f02c208cd34d8c05729882f0"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d791245a894be071d5ab04bbb4850534261a7d4fd363b094a7b9963e8cdbd31"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0cccd1de239afa866e4ce5c789b3032442f19c261c7d8a01183fd956b1935349"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f13f60d78224f0dace220d8ab4ef1dbc37115eeeab8c06804fec11bec2bbd07"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a9b5a0606faca4f6cc0d338359d6fa137104c337f489cd135bb7fbdbccb1e39"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:13da35c9ceb847732bf5c6c5781dcf4780e14392e5d3b3c689f6d22f8e15ae31"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:4d4cbe4ffa9d05f46a28252efc5941e0462792930caa370a6efaf491f412bc66"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:229852e147f44da0241954fc6cb910ba074e597f06789c867cb7fb0621e0ba7a"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:713103a8bdde61d13490adf47171a1039fd880113981e55401a0f7b42c37d071"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:45ad816b2c8e3b60b510f30dbd37fe74fd4a772248a52bb021f6fd65dff809b6"}, - {file = "aiohttp-3.8.6-cp36-cp36m-win32.whl", hash = "sha256:2b8d4e166e600dcfbff51919c7a3789ff6ca8b3ecce16e1d9c96d95dd569eb4c"}, - {file = "aiohttp-3.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:0912ed87fee967940aacc5306d3aa8ba3a459fcd12add0b407081fbefc931e53"}, - {file = "aiohttp-3.8.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e2a988a0c673c2e12084f5e6ba3392d76c75ddb8ebc6c7e9ead68248101cd446"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf3fd9f141700b510d4b190094db0ce37ac6361a6806c153c161dc6c041ccda"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3161ce82ab85acd267c8f4b14aa226047a6bee1e4e6adb74b798bd42c6ae1f80"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95fc1bf33a9a81469aa760617b5971331cdd74370d1214f0b3109272c0e1e3c"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c43ecfef7deaf0617cee936836518e7424ee12cb709883f2c9a1adda63cc460"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca80e1b90a05a4f476547f904992ae81eda5c2c85c66ee4195bb8f9c5fb47f28"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:90c72ebb7cb3a08a7f40061079817133f502a160561d0675b0a6adf231382c92"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bb54c54510e47a8c7c8e63454a6acc817519337b2b78606c4e840871a3e15349"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:de6a1c9f6803b90e20869e6b99c2c18cef5cc691363954c93cb9adeb26d9f3ae"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:a3628b6c7b880b181a3ae0a0683698513874df63783fd89de99b7b7539e3e8a8"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:fc37e9aef10a696a5a4474802930079ccfc14d9f9c10b4662169671ff034b7df"}, - {file = "aiohttp-3.8.6-cp37-cp37m-win32.whl", hash = "sha256:f8ef51e459eb2ad8e7a66c1d6440c808485840ad55ecc3cafefadea47d1b1ba2"}, - {file = "aiohttp-3.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:b2fe42e523be344124c6c8ef32a011444e869dc5f883c591ed87f84339de5976"}, - {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9e2ee0ac5a1f5c7dd3197de309adfb99ac4617ff02b0603fd1e65b07dc772e4b"}, - {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01770d8c04bd8db568abb636c1fdd4f7140b284b8b3e0b4584f070180c1e5c62"}, - {file = "aiohttp-3.8.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c68330a59506254b556b99a91857428cab98b2f84061260a67865f7f52899f5"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89341b2c19fb5eac30c341133ae2cc3544d40d9b1892749cdd25892bbc6ac951"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71783b0b6455ac8f34b5ec99d83e686892c50498d5d00b8e56d47f41b38fbe04"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f628dbf3c91e12f4d6c8b3f092069567d8eb17814aebba3d7d60c149391aee3a"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b04691bc6601ef47c88f0255043df6f570ada1a9ebef99c34bd0b72866c217ae"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee912f7e78287516df155f69da575a0ba33b02dd7c1d6614dbc9463f43066e3"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9c19b26acdd08dd239e0d3669a3dddafd600902e37881f13fbd8a53943079dbc"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:99c5ac4ad492b4a19fc132306cd57075c28446ec2ed970973bbf036bcda1bcc6"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f0f03211fd14a6a0aed2997d4b1c013d49fb7b50eeb9ffdf5e51f23cfe2c77fa"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:8d399dade330c53b4106160f75f55407e9ae7505263ea86f2ccca6bfcbdb4921"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ec4fd86658c6a8964d75426517dc01cbf840bbf32d055ce64a9e63a40fd7b771"}, - {file = "aiohttp-3.8.6-cp38-cp38-win32.whl", hash = "sha256:33164093be11fcef3ce2571a0dccd9041c9a93fa3bde86569d7b03120d276c6f"}, - {file = "aiohttp-3.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:bdf70bfe5a1414ba9afb9d49f0c912dc524cf60141102f3a11143ba3d291870f"}, - {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d52d5dc7c6682b720280f9d9db41d36ebe4791622c842e258c9206232251ab2b"}, - {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ac39027011414dbd3d87f7edb31680e1f430834c8cef029f11c66dad0670aa5"}, - {file = "aiohttp-3.8.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3f5c7ce535a1d2429a634310e308fb7d718905487257060e5d4598e29dc17f0b"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b30e963f9e0d52c28f284d554a9469af073030030cef8693106d918b2ca92f54"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:918810ef188f84152af6b938254911055a72e0f935b5fbc4c1a4ed0b0584aed1"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:002f23e6ea8d3dd8d149e569fd580c999232b5fbc601c48d55398fbc2e582e8c"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fcf3eabd3fd1a5e6092d1242295fa37d0354b2eb2077e6eb670accad78e40e1"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:255ba9d6d5ff1a382bb9a578cd563605aa69bec845680e21c44afc2670607a95"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d67f8baed00870aa390ea2590798766256f31dc5ed3ecc737debb6e97e2ede78"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:86f20cee0f0a317c76573b627b954c412ea766d6ada1a9fcf1b805763ae7feeb"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:39a312d0e991690ccc1a61f1e9e42daa519dcc34ad03eb6f826d94c1190190dd"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e827d48cf802de06d9c935088c2924e3c7e7533377d66b6f31ed175c1620e05e"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bd111d7fc5591ddf377a408ed9067045259ff2770f37e2d94e6478d0f3fc0c17"}, - {file = "aiohttp-3.8.6-cp39-cp39-win32.whl", hash = "sha256:caf486ac1e689dda3502567eb89ffe02876546599bbf915ec94b1fa424eeffd4"}, - {file = "aiohttp-3.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:3f0e27e5b733803333bb2371249f41cf42bae8884863e8e8965ec69bebe53132"}, - {file = "aiohttp-3.8.6.tar.gz", hash = "sha256:b0cf2a4501bff9330a8a5248b4ce951851e415bdcce9dc158e76cfd55e15085c"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6896b8416be9ada4d22cd359d7cb98955576ce863eadad5596b7cdfbf3e17c6c"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1736d87dad8ef46a8ec9cddd349fa9f7bd3a064c47dd6469c0d6763d3d49a4fc"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c9e5f4d7208cda1a2bb600e29069eecf857e6980d0ccc922ccf9d1372c16f4b"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8488519aa05e636c5997719fe543c8daf19f538f4fa044f3ce94bee608817cff"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ab16c254e2312efeb799bc3c06897f65a133b38b69682bf75d1f1ee1a9c43a9"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a94bde005a8f926d0fa38b88092a03dea4b4875a61fbcd9ac6f4351df1b57cd"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b777c9286b6c6a94f50ddb3a6e730deec327e9e2256cb08b5530db0f7d40fd8"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:571760ad7736b34d05597a1fd38cbc7d47f7b65deb722cb8e86fd827404d1f6b"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:deac0a32aec29608eb25d730f4bc5a261a65b6c48ded1ed861d2a1852577c932"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:4ee1b4152bc3190cc40ddd6a14715e3004944263ea208229ab4c297712aa3075"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:3607375053df58ed6f23903aa10cf3112b1240e8c799d243bbad0f7be0666986"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:65b0a70a25456d329a5e1426702dde67be0fb7a4ead718005ba2ca582d023a94"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a2eb5311a37fe105aa35f62f75a078537e1a9e4e1d78c86ec9893a3c97d7a30"}, + {file = "aiohttp-3.9.0-cp310-cp310-win32.whl", hash = "sha256:2cbc14a13fb6b42d344e4f27746a4b03a2cb0c1c3c5b932b0d6ad8881aa390e3"}, + {file = "aiohttp-3.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ac9669990e2016d644ba8ae4758688534aabde8dbbc81f9af129c3f5f01ca9cd"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f8e05f5163528962ce1d1806fce763ab893b1c5b7ace0a3538cd81a90622f844"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4afa8f71dba3a5a2e1e1282a51cba7341ae76585345c43d8f0e624882b622218"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f929f4c9b9a00f3e6cc0587abb95ab9c05681f8b14e0fe1daecfa83ea90f8318"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28185e36a78d247c55e9fbea2332d16aefa14c5276a582ce7a896231c6b1c208"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a486ddf57ab98b6d19ad36458b9f09e6022de0381674fe00228ca7b741aacb2f"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70e851f596c00f40a2f00a46126c95c2e04e146015af05a9da3e4867cfc55911"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5b7bf8fe4d39886adc34311a233a2e01bc10eb4e842220235ed1de57541a896"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c67a51ea415192c2e53e4e048c78bab82d21955b4281d297f517707dc836bf3d"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:694df243f394629bcae2d8ed94c589a181e8ba8604159e6e45e7b22e58291113"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3dd8119752dd30dd7bca7d4bc2a92a59be6a003e4e5c2cf7e248b89751b8f4b7"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:eb6dfd52063186ac97b4caa25764cdbcdb4b10d97f5c5f66b0fa95052e744eb7"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d97c3e286d0ac9af6223bc132dc4bad6540b37c8d6c0a15fe1e70fb34f9ec411"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:816f4db40555026e4cdda604a1088577c1fb957d02f3f1292e0221353403f192"}, + {file = "aiohttp-3.9.0-cp311-cp311-win32.whl", hash = "sha256:3abf0551874fecf95f93b58f25ef4fc9a250669a2257753f38f8f592db85ddea"}, + {file = "aiohttp-3.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:e18d92c3e9e22553a73e33784fcb0ed484c9874e9a3e96c16a8d6a1e74a0217b"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:99ae01fb13a618b9942376df77a1f50c20a281390dad3c56a6ec2942e266220d"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:05857848da443c8c12110d99285d499b4e84d59918a21132e45c3f0804876994"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:317719d7f824eba55857fe0729363af58e27c066c731bc62cd97bc9c3d9c7ea4"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1e3b3c107ccb0e537f309f719994a55621acd2c8fdf6d5ce5152aed788fb940"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45820ddbb276113ead8d4907a7802adb77548087ff5465d5c554f9aa3928ae7d"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:05a183f1978802588711aed0dea31e697d760ce9055292db9dc1604daa9a8ded"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a4cd44788ea0b5e6bb8fa704597af3a30be75503a7ed1098bc5b8ffdf6c982"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:673343fbc0c1ac44d0d2640addc56e97a052504beacd7ade0dc5e76d3a4c16e8"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7e8a3b79b6d186a9c99761fd4a5e8dd575a48d96021f220ac5b5fa856e5dd029"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6777a390e41e78e7c45dab43a4a0196c55c3b8c30eebe017b152939372a83253"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7ae5f99a32c53731c93ac3075abd3e1e5cfbe72fc3eaac4c27c9dd64ba3b19fe"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:f1e4f254e9c35d8965d377e065c4a8a55d396fe87c8e7e8429bcfdeeb229bfb3"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11ca808f9a6b63485059f5f6e164ef7ec826483c1212a44f268b3653c91237d8"}, + {file = "aiohttp-3.9.0-cp312-cp312-win32.whl", hash = "sha256:de3cc86f4ea8b4c34a6e43a7306c40c1275e52bfa9748d869c6b7d54aa6dad80"}, + {file = "aiohttp-3.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca4fddf84ac7d8a7d0866664936f93318ff01ee33e32381a115b19fb5a4d1202"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f09960b5bb1017d16c0f9e9f7fc42160a5a49fa1e87a175fd4a2b1a1833ea0af"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8303531e2c17b1a494ffaeba48f2da655fe932c4e9a2626c8718403c83e5dd2b"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4790e44f46a4aa07b64504089def5744d3b6780468c4ec3a1a36eb7f2cae9814"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1d7edf74a36de0e5ca50787e83a77cf352f5504eb0ffa3f07000a911ba353fb"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94697c7293199c2a2551e3e3e18438b4cba293e79c6bc2319f5fd652fccb7456"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a1b66dbb8a7d5f50e9e2ea3804b01e766308331d0cac76eb30c563ac89c95985"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9623cfd9e85b76b83ef88519d98326d4731f8d71869867e47a0b979ffec61c73"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f32c86dc967ab8c719fd229ce71917caad13cc1e8356ee997bf02c5b368799bf"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f50b4663c3e0262c3a361faf440761fbef60ccdde5fe8545689a4b3a3c149fb4"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dcf71c55ec853826cd70eadb2b6ac62ec577416442ca1e0a97ad875a1b3a0305"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:42fe4fd9f0dfcc7be4248c162d8056f1d51a04c60e53366b0098d1267c4c9da8"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:76a86a9989ebf82ee61e06e2bab408aec4ea367dc6da35145c3352b60a112d11"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f9e09a1c83521d770d170b3801eea19b89f41ccaa61d53026ed111cb6f088887"}, + {file = "aiohttp-3.9.0-cp38-cp38-win32.whl", hash = "sha256:a00ce44c21612d185c5275c5cba4bab8d7c1590f248638b667ed8a782fa8cd6f"}, + {file = "aiohttp-3.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:d5b9345ab92ebe6003ae11d8092ce822a0242146e6fa270889b9ba965457ca40"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98d21092bf2637c5fa724a428a69e8f5955f2182bff61f8036827cf6ce1157bf"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35a68cd63ca6aaef5707888f17a70c36efe62b099a4e853d33dc2e9872125be8"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d7f6235c7475658acfc1769d968e07ab585c79f6ca438ddfecaa9a08006aee2"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db04d1de548f7a62d1dd7e7cdf7c22893ee168e22701895067a28a8ed51b3735"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:536b01513d67d10baf6f71c72decdf492fb7433c5f2f133e9a9087379d4b6f31"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c8b0a6487e8109427ccf638580865b54e2e3db4a6e0e11c02639231b41fc0f"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7276fe0017664414fdc3618fca411630405f1aaf0cc3be69def650eb50441787"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23170247ef89ffa842a02bbfdc425028574d9e010611659abeb24d890bc53bb8"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b1a2ea8252cacc7fd51df5a56d7a2bb1986ed39be9397b51a08015727dfb69bd"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d71abc15ff7047412ef26bf812dfc8d0d1020d664617f4913df2df469f26b76"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:2d820162c8c2bdbe97d328cd4f417c955ca370027dce593345e437b2e9ffdc4d"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:2779f5e7c70f7b421915fd47db332c81de365678180a9f3ab404088f87ba5ff9"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:366bc870d7ac61726f32a489fbe3d1d8876e87506870be66b01aeb84389e967e"}, + {file = "aiohttp-3.9.0-cp39-cp39-win32.whl", hash = "sha256:1df43596b826022b14998f0460926ce261544fedefe0d2f653e1b20f49e96454"}, + {file = "aiohttp-3.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:9c196b30f1b1aa3363a69dd69079ae9bec96c2965c4707eaa6914ba099fb7d4f"}, + {file = "aiohttp-3.9.0.tar.gz", hash = "sha256:09f23292d29135025e19e8ff4f0a68df078fe4ee013bca0105b2e803989de92d"}, ] [package.dependencies] aiosignal = ">=1.1.2" -async-timeout = ">=4.0.0a3,<5.0" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<4.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["Brotli", "aiodns", "brotlicffi"] [[package]] name = "aiosignal" @@ -342,13 +330,13 @@ files = [ [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] @@ -484,35 +472,6 @@ azure = ["azure-storage-blob (>=12)"] gs = ["google-cloud-storage"] s3 = ["boto3"] -[[package]] -name = "cmake" -version = "3.27.7" -description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" -optional = false -python-versions = "*" -files = [ - {file = "cmake-3.27.7-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:d582ef3e9ff0bd113581c1a32e881d1c2f9a34d2de76c93324a28593a76433db"}, - {file = "cmake-3.27.7-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:8056c99e371ff57229df2068364d7c32fea716cb53b4675f639edfb62663decf"}, - {file = "cmake-3.27.7-py2.py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:68983b09de633fc1ce6ab6bce9a25bfa181e41598e7c6bc0a6c0108773ee01cb"}, - {file = "cmake-3.27.7-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8bd1e1fa4fc8de7605c663d9408dceb649112f855aab05cca31fdb72e4d78364"}, - {file = "cmake-3.27.7-py2.py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:c981aafcca2cd7210bd210ec75710c0f34e1fde1998cdcab812e4133e3ab615d"}, - {file = "cmake-3.27.7-py2.py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1b9067ce0251cba3d4c018f2e1577ba9078e9c1eff6ad607ad5ce867843d4571"}, - {file = "cmake-3.27.7-py2.py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b8a2fcb619b89d1cce7b52828316de9a1f27f0c90c2e39d1eae886428c8ee8c6"}, - {file = "cmake-3.27.7-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:499b38c77d52fb1964dbb38d0228fed246263a181939a8e753fde8ca227c8e1e"}, - {file = "cmake-3.27.7-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:2fb48c780f1a6a3d19e785ebbb754be79d369e25a1cb81043fab049e709564da"}, - {file = "cmake-3.27.7-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:7bf96237ba11ce2437dc5e071d96b510120a1be4708c631a64b2f38fb46bbd77"}, - {file = "cmake-3.27.7-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:009058bdf4f488709f38eaa5dd0ef0f89c6b9c6b6edd9d5b475a308ef75f80bb"}, - {file = "cmake-3.27.7-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:591f6b056527aefec009bc61a388776b2fc62444deb0038112a471031f61aeca"}, - {file = "cmake-3.27.7-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:bd40d46dbad3555d5b3ce054bef24b85f256b19139493773751ab6f2b71c1219"}, - {file = "cmake-3.27.7-py2.py3-none-win32.whl", hash = "sha256:bdbf0256f554f68c7b1d9740f5d059daf875b685c81a479cbe69038e84eb2fb9"}, - {file = "cmake-3.27.7-py2.py3-none-win_amd64.whl", hash = "sha256:810e592b606d05a3080a9c19ea839b13226f62cae447a22485b2365782f6b926"}, - {file = "cmake-3.27.7-py2.py3-none-win_arm64.whl", hash = "sha256:72289361866314f73be2ae63ddee224ff70223dcef9feb66d0072bf17e245564"}, - {file = "cmake-3.27.7.tar.gz", hash = "sha256:9f4a7c7be2a25de5901f045618f41b833ea6c0f647115201d38e4fdf7e2815bc"}, -] - -[package.extras] -test = ["coverage (>=4.2)", "flake8 (>=3.0.4)", "path.py (>=11.5.0)", "pytest (>=3.0.3)", "pytest-cov (>=2.4.0)", "pytest-runner (>=2.9)", "pytest-virtualenv (>=1.7.0)", "scikit-build (>=0.10.0)", "setuptools (>=28.0.0)", "virtualenv (>=15.0.3)", "wheel"] - [[package]] name = "colorama" version = "0.4.6" @@ -740,13 +699,13 @@ typing-inspect = ">=0.4.0,<1" [[package]] name = "datasets" -version = "2.14.6" +version = "2.14.7" description = "HuggingFace community-driven open-source library of datasets" optional = false python-versions = ">=3.8.0" files = [ - {file = "datasets-2.14.6-py3-none-any.whl", hash = "sha256:4de857ffce21cfc847236745c69f102e33cd1f0fa8398e7be9964525fd4cd5db"}, - {file = "datasets-2.14.6.tar.gz", hash = "sha256:97ebbace8ec7af11434a87d1215379927f8fee2beab2c4a674003756ecfe920c"}, + {file = "datasets-2.14.7-py3-none-any.whl", hash = "sha256:1a64041a7da4f4130f736fc371c1f528b8ddd208cebe156400f65719bdbba79d"}, + {file = "datasets-2.14.7.tar.gz", hash = "sha256:394cf9b4ec0694b25945977b16ad5d18d5c15fb0e94141713eb8ead7452caf9e"}, ] [package.dependencies] @@ -759,6 +718,7 @@ numpy = ">=1.17" packaging = "*" pandas = "*" pyarrow = ">=8.0.0" +pyarrow-hotfix = "*" pyyaml = ">=5.1" requests = ">=2.19.0" tqdm = ">=4.62.1" @@ -833,15 +793,26 @@ files = [ [package.extras] graph = ["objgraph (>=1.7.2)"] +[[package]] +name = "distro" +version = "1.8.0" +description = "Distro - an OS platform information API" +optional = false +python-versions = ">=3.6" +files = [ + {file = "distro-1.8.0-py3-none-any.whl", hash = "sha256:99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff"}, + {file = "distro-1.8.0.tar.gz", hash = "sha256:02e111d1dc6a50abb8eed6bf31c3e48ed8b0830d1ea2a1b78c61765c2513fdd8"}, +] + [[package]] name = "exceptiongroup" -version = "1.1.3" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, - {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -890,53 +861,53 @@ files = [ [[package]] name = "fonttools" -version = "4.44.0" +version = "4.45.0" description = "Tools to manipulate font files" optional = false python-versions = ">=3.8" files = [ - {file = "fonttools-4.44.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1cd1c6bb097e774d68402499ff66185190baaa2629ae2f18515a2c50b93db0c"}, - {file = "fonttools-4.44.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b9eab7f9837fdaa2a10a524fbcc2ec24bf60637c044b6e4a59c3f835b90f0fae"}, - {file = "fonttools-4.44.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f412954275e594f7a51c16f3b3edd850acb0d842fefc33856b63a17e18499a5"}, - {file = "fonttools-4.44.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50d25893885e80a5955186791eed5579f1e75921751539cc1dc3ffd1160b48cf"}, - {file = "fonttools-4.44.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:22ea8aa7b3712450b42b044702bd3a64fd118006bad09a6f94bd1b227088492e"}, - {file = "fonttools-4.44.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df40daa6c03b98652ffe8110ae014fe695437f6e1cb5a07e16ea37f40e73ac86"}, - {file = "fonttools-4.44.0-cp310-cp310-win32.whl", hash = "sha256:bca49da868e8bde569ef36f0cc1b6de21d56bf9c3be185c503b629c19a185287"}, - {file = "fonttools-4.44.0-cp310-cp310-win_amd64.whl", hash = "sha256:dbac86d83d96099890e731cc2af97976ff2c98f4ba432fccde657c5653a32f1c"}, - {file = "fonttools-4.44.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e8ff7d19a6804bfd561cfcec9b4200dd1788e28f7de4be70189801530c47c1b3"}, - {file = "fonttools-4.44.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8a1fa9a718de0bc026979c93e1e9b55c5efde60d76f91561fd713387573817d"}, - {file = "fonttools-4.44.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05064f95aacdfc06f21e55096c964b2228d942b8675fa26995a2551f6329d2d"}, - {file = "fonttools-4.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31b38528f25bc662401e6ffae14b3eb7f1e820892fd80369a37155e3b636a2f4"}, - {file = "fonttools-4.44.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:05d7c4d2c95b9490e669f3cb83918799bf1c838619ac6d3bad9ea017cfc63f2e"}, - {file = "fonttools-4.44.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6999e80a125b0cd8e068d0210b63323f17338038c2ecd2e11b9209ec430fe7f2"}, - {file = "fonttools-4.44.0-cp311-cp311-win32.whl", hash = "sha256:a7aec7f5d14dfcd71fb3ebc299b3f000c21fdc4043079101777ed2042ba5b7c5"}, - {file = "fonttools-4.44.0-cp311-cp311-win_amd64.whl", hash = "sha256:518a945dbfe337744bfff31423c1430303b8813c5275dffb0f2577f0734a1189"}, - {file = "fonttools-4.44.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:59b6ad83cce067d10f4790c037a5904424f45bebb5e7be2eb2db90402f288267"}, - {file = "fonttools-4.44.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c2de1fb18198acd400c45ffe2aef5420c8d55fde903e91cba705596099550f3b"}, - {file = "fonttools-4.44.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84f308b7a8d28208d54315d11d35f9888d6d607673dd4d42d60b463682ee0400"}, - {file = "fonttools-4.44.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66bc6efd829382f7a7e6cf33c2fb32b13edc8a239eb15f32acbf197dce7a0165"}, - {file = "fonttools-4.44.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a8b99713d3a0d0e876b6aecfaada5e7dc9fe979fcd90ef9fa0ba1d9b9aed03f2"}, - {file = "fonttools-4.44.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b63da598d9cbc52e2381f922da0e94d60c0429f92207bd3fb04d112fc82ea7cb"}, - {file = "fonttools-4.44.0-cp312-cp312-win32.whl", hash = "sha256:f611c97678604e302b725f71626edea113a5745a7fb557c958b39edb6add87d5"}, - {file = "fonttools-4.44.0-cp312-cp312-win_amd64.whl", hash = "sha256:58af428746fa73a2edcbf26aff33ac4ef3c11c8d75bb200eaea2f7e888d2de4e"}, - {file = "fonttools-4.44.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9ee8692e23028564c13d924004495f284df8ac016a19f17a87251210e1f1f928"}, - {file = "fonttools-4.44.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dab3d00d27b1a79ae4d4a240e8ceea8af0ff049fd45f05adb4f860d93744110d"}, - {file = "fonttools-4.44.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f53526668beccdb3409c6055a4ffe50987a7f05af6436fa55d61f5e7bd450219"}, - {file = "fonttools-4.44.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3da036b016c975c2d8c69005bdc4d5d16266f948a7fab950244e0f58301996a"}, - {file = "fonttools-4.44.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b99fe8ef4093f672d00841569d2d05691e50334d79f4d9c15c1265d76d5580d2"}, - {file = "fonttools-4.44.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d16d9634ff1e5cea2cf4a8cbda9026f766e4b5f30b48f8180f0e99133d3abfc"}, - {file = "fonttools-4.44.0-cp38-cp38-win32.whl", hash = "sha256:3d29509f6e05e8d725db59c2d8c076223d793e4e35773040be6632a0349f2f97"}, - {file = "fonttools-4.44.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4fa4f4bc8fd86579b8cdbe5e948f35d82c0eda0091c399d009b2a5a6b61c040"}, - {file = "fonttools-4.44.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c794de4086f06ae609b71ac944ec7deb09f34ecf73316fddc041087dd24bba39"}, - {file = "fonttools-4.44.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2db63941fee3122e31a21dd0f5b2138ce9906b661a85b63622421d3654a74ae2"}, - {file = "fonttools-4.44.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb01c49c8aa035d5346f46630209923d4927ed15c2493db38d31da9f811eb70d"}, - {file = "fonttools-4.44.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46c79af80a835410874683b5779b6c1ec1d5a285e11c45b5193e79dd691eb111"}, - {file = "fonttools-4.44.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b6e6aa2d066f8dafd06d8d0799b4944b5d5a1f015dd52ac01bdf2895ebe169a0"}, - {file = "fonttools-4.44.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:63a3112f753baef8c6ac2f5f574bb9ac8001b86c8c0c0380039db47a7f512d20"}, - {file = "fonttools-4.44.0-cp39-cp39-win32.whl", hash = "sha256:54efed22b2799a85475e6840e907c402ba49892c614565dc770aa97a53621b2b"}, - {file = "fonttools-4.44.0-cp39-cp39-win_amd64.whl", hash = "sha256:2e91e19b583961979e2e5a701269d3cfc07418963bee717f8160b0a24332826b"}, - {file = "fonttools-4.44.0-py3-none-any.whl", hash = "sha256:b9beb0fa6ff3ea808ad4a6962d68ac0f140ddab080957b20d9e268e4d67fb335"}, - {file = "fonttools-4.44.0.tar.gz", hash = "sha256:4e90dd81b6e0d97ebfe52c0d12a17a9ef7f305d6bfbb93081265057d6092f252"}, + {file = "fonttools-4.45.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:18138744540413eb2ebeff6ce8b9d617926f1ed08da5d1676f99f1966988264e"}, + {file = "fonttools-4.45.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0b3d240933045b9dbbe6e8c1e28ffe89be72c9be927b6e572e55be5e2b2604f7"}, + {file = "fonttools-4.45.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5853263505f407b69c0d1cbf3ed1c30f985b9505523989b20aa18a5231d4a08a"}, + {file = "fonttools-4.45.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c21f306f1e71146cf7587916d6de5e9c4bf26057aad602a6c7fad4b6e05bf1f"}, + {file = "fonttools-4.45.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1bb4f01018b9f4e2d7b07c2bf79e2ef498acb6f99321b72b5c44b1333481f569"}, + {file = "fonttools-4.45.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d0e6603c3b00604574d84fabbcf9dee81efa7c89d38ed4dcbf4e6c654e1ebb99"}, + {file = "fonttools-4.45.0-cp310-cp310-win32.whl", hash = "sha256:c3e676e50a0713c9a1e46185b806967d3c012643d1936ca814eb9ab62027c090"}, + {file = "fonttools-4.45.0-cp310-cp310-win_amd64.whl", hash = "sha256:e819f14d315024750b1ad2842da605051444b944cd983ea987359ace5213bcb9"}, + {file = "fonttools-4.45.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a6d68b97b967a3361e0ddf14425e4fe030c9f19462b445ce0b190c4a6834eb46"}, + {file = "fonttools-4.45.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:619227c7b9088add4d4e5959bf0fa3c29a71c191baa8b989bf532645876b2168"}, + {file = "fonttools-4.45.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cafe001811ad1ac2a5d357fc99c490357d758569f69511d14da0311c02814e15"}, + {file = "fonttools-4.45.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:756c6f4324fd4bb4d7517462d7237ff65e45da7a002f9e6e08a48c25a11bf073"}, + {file = "fonttools-4.45.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ecc97628be1bf7fa93463e1e791915da66de51df8e655a5a6c846fd9b8ceaa98"}, + {file = "fonttools-4.45.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:636177ffc4479bc824a356c00a3c9a74a2ce591fa6057109321e0a0ffd126e40"}, + {file = "fonttools-4.45.0-cp311-cp311-win32.whl", hash = "sha256:cac462dbd9058778c89bc608ac29ba93ab3fbc37f305d260aa2d771cfb0fa694"}, + {file = "fonttools-4.45.0-cp311-cp311-win_amd64.whl", hash = "sha256:2bd3f33a5d5630cc20cf3f8631074cac6eafdb2aa3ac8745966c3b4bf93656b4"}, + {file = "fonttools-4.45.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:5b3da7791a58c97763d1704c2b76a9d654b8f2ef233e64248960bd2c6e669fe4"}, + {file = "fonttools-4.45.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4fbc3d8acb578ba0932fcabc01a962f23b0dd254ab103dd0606128fff0175095"}, + {file = "fonttools-4.45.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f69e07ebfd89d96485349dc10bd81a027fc0e927f988fa31bd9cd06359e06ed"}, + {file = "fonttools-4.45.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b122fe802839bfc8f9603233e5fcbdc98b5e27876f7945b426adfea983918a7b"}, + {file = "fonttools-4.45.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8f8736e885700ae22970519b8f5c7f4c2f29c6e9459d05c649c4e99012c20b23"}, + {file = "fonttools-4.45.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:70f047ba37c6aac3d331b71bb784373e251bd86678da72526edc8585e79418e1"}, + {file = "fonttools-4.45.0-cp312-cp312-win32.whl", hash = "sha256:fdb43f68bce545f494fed1bfb60d2c32b53f410758919112923c2d600cb9a24c"}, + {file = "fonttools-4.45.0-cp312-cp312-win_amd64.whl", hash = "sha256:102a7ca8700a078265c002c29d2ae498edeb14b636375ceec2425b561ce08037"}, + {file = "fonttools-4.45.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4b154cbf93515e4eb477f5cf99de79b46c17229781f321907940bdbabbd64708"}, + {file = "fonttools-4.45.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f546a1b676622638a12c721d89cfb513ad7189548eadac885cdd555e35021557"}, + {file = "fonttools-4.45.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45c5a0612049e0d06b467c3a0837d9efe37934acab64ba922f00e1d07c1555a7"}, + {file = "fonttools-4.45.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f27166d00e0cd3ea49461b053f55e75676f1109e5483170a14d70c397d082a4c"}, + {file = "fonttools-4.45.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:980ba4e673439db22a00501fac087957ce0731351b042816f6c02df81cadc612"}, + {file = "fonttools-4.45.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:83b70b99f3f55b046cb51ca20fc15702567710233b2cd757a79e1916c25a25f8"}, + {file = "fonttools-4.45.0-cp38-cp38-win32.whl", hash = "sha256:fe8ad943f62bf16273154ebcdf855c44a3b46eac36abea338c209209439b4eb6"}, + {file = "fonttools-4.45.0-cp38-cp38-win_amd64.whl", hash = "sha256:6fb1fdcee2b36e012283805ef0380e4508dbb504950b1c94d0343f8dbbad7d7e"}, + {file = "fonttools-4.45.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5bbcb75ffcea64543ab8203e132e2019b226f59a4a6958637e78c21f9ca560ff"}, + {file = "fonttools-4.45.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ade07953b35ebf66c445a5e02f28ecd038ea588dc7673c555afe319b6e3c5168"}, + {file = "fonttools-4.45.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54ac8be3f693062fc08550213edd40db8f4fe1dd095a1246ed18e887fc254d76"}, + {file = "fonttools-4.45.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc991712aaea9d545b13ec480aaf2ebd12ccdea180fce864dd9863f5134f5a06"}, + {file = "fonttools-4.45.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:420139743e21d832de230757fb7b0c285d8024d602af8064d9506fa055bb62ae"}, + {file = "fonttools-4.45.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:58da830a52c05f18a7cc8a279a8bdddf2e45cdc685b360699653fb3405557459"}, + {file = "fonttools-4.45.0-cp39-cp39-win32.whl", hash = "sha256:715e18f67f0587a16476c7f62b8ff9a165ddceb8c2a262fb08df9f71c7790f0e"}, + {file = "fonttools-4.45.0-cp39-cp39-win_amd64.whl", hash = "sha256:dd26fda8c879558216458a801c1dba52f35dca0e96640fd9c75e86b6574cf1c3"}, + {file = "fonttools-4.45.0-py3-none-any.whl", hash = "sha256:835cf5d0e1b37bbed1d64c286611cc4da9ff19df952400f191ba9142b3cb97f6"}, + {file = "fonttools-4.45.0.tar.gz", hash = "sha256:c1c79d7d4093396892575115c214b24f09e68997cb5c0ab2d99bfdaff74c64b6"}, ] [package.extras] @@ -1098,13 +1069,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-auth-oauthlib" -version = "1.0.0" +version = "1.1.0" description = "Google Authentication Library" optional = false python-versions = ">=3.6" files = [ - {file = "google-auth-oauthlib-1.0.0.tar.gz", hash = "sha256:e375064964820b47221a7e1b7ee1fd77051b6323c3f9e3e19785f78ab67ecfc5"}, - {file = "google_auth_oauthlib-1.0.0-py2.py3-none-any.whl", hash = "sha256:95880ca704928c300f48194d1770cf5b1462835b6e49db61445a520f793fd5fb"}, + {file = "google-auth-oauthlib-1.1.0.tar.gz", hash = "sha256:83ea8c3b0881e453790baff4448e8a6112ac8778d1de9da0b68010b843937afb"}, + {file = "google_auth_oauthlib-1.1.0-py2.py3-none-any.whl", hash = "sha256:089c6e587d36f4803ac7e0720c045c6a8b1fd1790088b8424975b90d0ee61c12"}, ] [package.dependencies] @@ -1201,69 +1172,80 @@ test = ["objgraph", "psutil"] [[package]] name = "grpcio" -version = "1.59.2" +version = "1.59.3" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.59.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:d2fa68a96a30dd240be80bbad838a0ac81a61770611ff7952b889485970c4c71"}, - {file = "grpcio-1.59.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:cf0dead5a2c5a3347af2cfec7131d4f2a2e03c934af28989c9078f8241a491fa"}, - {file = "grpcio-1.59.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:e420ced29b5904cdf9ee5545e23f9406189d8acb6750916c2db4793dada065c6"}, - {file = "grpcio-1.59.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b230028a008ae1d0f430acb227d323ff8a619017415cf334c38b457f814119f"}, - {file = "grpcio-1.59.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a4a3833c0e067f3558538727235cd8a49709bff1003200bbdefa2f09334e4b1"}, - {file = "grpcio-1.59.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6b25ed37c27e652db01be341af93fbcea03d296c024d8a0e680017a268eb85dd"}, - {file = "grpcio-1.59.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73abb8584b0cf74d37f5ef61c10722adc7275502ab71789a8fe3cb7ef04cf6e2"}, - {file = "grpcio-1.59.2-cp310-cp310-win32.whl", hash = "sha256:d6f70406695e3220f09cd7a2f879333279d91aa4a8a1d34303b56d61a8180137"}, - {file = "grpcio-1.59.2-cp310-cp310-win_amd64.whl", hash = "sha256:3c61d641d4f409c5ae46bfdd89ea42ce5ea233dcf69e74ce9ba32b503c727e29"}, - {file = "grpcio-1.59.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:3059668df17627f0e0fa680e9ef8c995c946c792612e9518f5cc1503be14e90b"}, - {file = "grpcio-1.59.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:72ca2399097c0b758198f2ff30f7178d680de8a5cfcf3d9b73a63cf87455532e"}, - {file = "grpcio-1.59.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c978f864b35f2261e0819f5cd88b9830b04dc51bcf055aac3c601e525a10d2ba"}, - {file = "grpcio-1.59.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9411e24328a2302e279e70cae6e479f1fddde79629fcb14e03e6d94b3956eabf"}, - {file = "grpcio-1.59.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb7e0fe6ad73b7f06d7e2b689c19a71cf5cc48f0c2bf8608469e51ffe0bd2867"}, - {file = "grpcio-1.59.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c2504eed520958a5b77cc99458297cb7906308cb92327f35fb7fbbad4e9b2188"}, - {file = "grpcio-1.59.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2171c39f355ba5b551c5d5928d65aa6c69807fae195b86ef4a7d125bcdb860a9"}, - {file = "grpcio-1.59.2-cp311-cp311-win32.whl", hash = "sha256:d2794f0e68b3085d99b4f6ff9c089f6fdd02b32b9d3efdfbb55beac1bf22d516"}, - {file = "grpcio-1.59.2-cp311-cp311-win_amd64.whl", hash = "sha256:2067274c88bc6de89c278a672a652b4247d088811ece781a4858b09bdf8448e3"}, - {file = "grpcio-1.59.2-cp312-cp312-linux_armv7l.whl", hash = "sha256:535561990e075fa6bd4b16c4c3c1096b9581b7bb35d96fac4650f1181e428268"}, - {file = "grpcio-1.59.2-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:a213acfbf186b9f35803b52e4ca9addb153fc0b67f82a48f961be7000ecf6721"}, - {file = "grpcio-1.59.2-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:6959fb07e8351e20501ffb8cc4074c39a0b7ef123e1c850a7f8f3afdc3a3da01"}, - {file = "grpcio-1.59.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e82c5cf1495244adf5252f925ac5932e5fd288b3e5ab6b70bec5593074b7236c"}, - {file = "grpcio-1.59.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023088764012411affe7db183d1ada3ad9daf2e23ddc719ff46d7061de661340"}, - {file = "grpcio-1.59.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:da2d94c15f88cd40d7e67f7919d4f60110d2b9d5b1e08cf354c2be773ab13479"}, - {file = "grpcio-1.59.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:6009386a2df66159f64ac9f20425ae25229b29b9dd0e1d3dd60043f037e2ad7e"}, - {file = "grpcio-1.59.2-cp312-cp312-win32.whl", hash = "sha256:75c6ecb70e809cf1504465174343113f51f24bc61e22a80ae1c859f3f7034c6d"}, - {file = "grpcio-1.59.2-cp312-cp312-win_amd64.whl", hash = "sha256:cbe946b3e6e60a7b4618f091e62a029cb082b109a9d6b53962dd305087c6e4fd"}, - {file = "grpcio-1.59.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:f8753a6c88d1d0ba64302309eecf20f70d2770f65ca02d83c2452279085bfcd3"}, - {file = "grpcio-1.59.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:f1ef0d39bc1feb420caf549b3c657c871cad4ebbcf0580c4d03816b0590de0cf"}, - {file = "grpcio-1.59.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:4c93f4abbb54321ee6471e04a00139c80c754eda51064187963ddf98f5cf36a4"}, - {file = "grpcio-1.59.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:08d77e682f2bf730a4961eea330e56d2f423c6a9b91ca222e5b1eb24a357b19f"}, - {file = "grpcio-1.59.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ff16d68bf453275466a9a46739061a63584d92f18a0f5b33d19fc97eb69867c"}, - {file = "grpcio-1.59.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4abb717e320e74959517dc8e84a9f48fbe90e9abe19c248541e9418b1ce60acd"}, - {file = "grpcio-1.59.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:36f53c2b3449c015880e7d55a89c992c357f176327b0d2873cdaaf9628a37c69"}, - {file = "grpcio-1.59.2-cp37-cp37m-win_amd64.whl", hash = "sha256:cc3e4cd087f07758b16bef8f31d88dbb1b5da5671d2f03685ab52dece3d7a16e"}, - {file = "grpcio-1.59.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:27f879ae604a7fcf371e59fba6f3ff4635a4c2a64768bd83ff0cac503142fef4"}, - {file = "grpcio-1.59.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:7cf05053242f61ba94014dd3a986e11a083400a32664058f80bf4cf817c0b3a1"}, - {file = "grpcio-1.59.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:e1727c1c0e394096bb9af185c6923e8ea55a5095b8af44f06903bcc0e06800a2"}, - {file = "grpcio-1.59.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d573e70a6fe77555fb6143c12d3a7d3fa306632a3034b4e7c59ca09721546f8"}, - {file = "grpcio-1.59.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31176aa88f36020055ace9adff2405a33c8bdbfa72a9c4980e25d91b2f196873"}, - {file = "grpcio-1.59.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:11168ef43e4a43ff1b1a65859f3e0ef1a173e277349e7fb16923ff108160a8cd"}, - {file = "grpcio-1.59.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:53c9aa5ddd6857c0a1cd0287225a2a25873a8e09727c2e95c4aebb1be83a766a"}, - {file = "grpcio-1.59.2-cp38-cp38-win32.whl", hash = "sha256:3b4368b33908f683a363f376dfb747d40af3463a6e5044afee07cf9436addf96"}, - {file = "grpcio-1.59.2-cp38-cp38-win_amd64.whl", hash = "sha256:0a754aff9e3af63bdc4c75c234b86b9d14e14a28a30c4e324aed1a9b873d755f"}, - {file = "grpcio-1.59.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:1f9524d1d701e399462d2c90ba7c193e49d1711cf429c0d3d97c966856e03d00"}, - {file = "grpcio-1.59.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:f93dbf58f03146164048be5426ffde298b237a5e059144847e4940f5b80172c3"}, - {file = "grpcio-1.59.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:6da6dea3a1bacf99b3c2187e296db9a83029ed9c38fd4c52b7c9b7326d13c828"}, - {file = "grpcio-1.59.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5f09cffa619adfb44799fa4a81c2a1ad77c887187613fb0a8f201ab38d89ba1"}, - {file = "grpcio-1.59.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c35aa9657f5d5116d23b934568e0956bd50c615127810fffe3ac356a914c176a"}, - {file = "grpcio-1.59.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:74100fecaec8a535e380cf5f2fb556ff84957d481c13e54051c52e5baac70541"}, - {file = "grpcio-1.59.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:128e20f57c5f27cb0157e73756d1586b83c1b513ebecc83ea0ac37e4b0e4e758"}, - {file = "grpcio-1.59.2-cp39-cp39-win32.whl", hash = "sha256:686e975a5d16602dc0982c7c703948d17184bd1397e16c8ee03511ecb8c4cdda"}, - {file = "grpcio-1.59.2-cp39-cp39-win_amd64.whl", hash = "sha256:242adc47725b9a499ee77c6a2e36688fa6c96484611f33b1be4c57ab075a92dd"}, - {file = "grpcio-1.59.2.tar.gz", hash = "sha256:d8f9cd4ad1be90b0cf350a2f04a38a36e44a026cac1e036ac593dc48efe91d52"}, + {file = "grpcio-1.59.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:aca028a6c7806e5b61e5f9f4232432c52856f7fcb98e330b20b6bc95d657bdcc"}, + {file = "grpcio-1.59.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:19ad26a7967f7999c8960d2b9fe382dae74c55b0c508c613a6c2ba21cddf2354"}, + {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:72b71dad2a3d1650e69ad42a5c4edbc59ee017f08c32c95694172bc501def23c"}, + {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c0f0a11d82d0253656cc42e04b6a149521e02e755fe2e4edd21123de610fd1d4"}, + {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cddafb70f9a2c81ba251b53b4007e07cca7389e704f86266e22c4bffd8bf1d"}, + {file = "grpcio-1.59.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6c75a1fa0e677c1d2b6d4196ad395a5c381dfb8385f07ed034ef667cdcdbcc25"}, + {file = "grpcio-1.59.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1d8e01438d5964a11167eec1edb5f85ed8e475648f36c834ed5db4ffba24ac8"}, + {file = "grpcio-1.59.3-cp310-cp310-win32.whl", hash = "sha256:c4b0076f0bf29ee62335b055a9599f52000b7941f577daa001c7ef961a1fbeab"}, + {file = "grpcio-1.59.3-cp310-cp310-win_amd64.whl", hash = "sha256:b1f00a3e6e0c3dccccffb5579fc76ebfe4eb40405ba308505b41ef92f747746a"}, + {file = "grpcio-1.59.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:3996aaa21231451161dc29df6a43fcaa8b332042b6150482c119a678d007dd86"}, + {file = "grpcio-1.59.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:cb4e9cbd9b7388fcb06412da9f188c7803742d06d6f626304eb838d1707ec7e3"}, + {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8022ca303d6c694a0d7acfb2b472add920217618d3a99eb4b14edc7c6a7e8fcf"}, + {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b36683fad5664283755a7f4e2e804e243633634e93cd798a46247b8e54e3cb0d"}, + {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8239b853226e4824e769517e1b5232e7c4dda3815b200534500338960fcc6118"}, + {file = "grpcio-1.59.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0511af8653fbda489ff11d542a08505d56023e63cafbda60e6e00d4e0bae86ea"}, + {file = "grpcio-1.59.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e78dc982bda74cef2ddfce1c91d29b96864c4c680c634e279ed204d51e227473"}, + {file = "grpcio-1.59.3-cp311-cp311-win32.whl", hash = "sha256:6a5c3a96405966c023e139c3bcccb2c7c776a6f256ac6d70f8558c9041bdccc3"}, + {file = "grpcio-1.59.3-cp311-cp311-win_amd64.whl", hash = "sha256:ed26826ee423b11477297b187371cdf4fa1eca874eb1156422ef3c9a60590dd9"}, + {file = "grpcio-1.59.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:45dddc5cb5227d30fa43652d8872dc87f086d81ab4b500be99413bad0ae198d7"}, + {file = "grpcio-1.59.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:1736496d74682e53dd0907fd515f2694d8e6a96c9a359b4080b2504bf2b2d91b"}, + {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:ddbd1a16138e52e66229047624de364f88a948a4d92ba20e4e25ad7d22eef025"}, + {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcfa56f8d031ffda902c258c84c4b88707f3a4be4827b4e3ab8ec7c24676320d"}, + {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2eb8f0c7c0c62f7a547ad7a91ba627a5aa32a5ae8d930783f7ee61680d7eb8d"}, + {file = "grpcio-1.59.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8d993399cc65e3a34f8fd48dd9ad7a376734564b822e0160dd18b3d00c1a33f9"}, + {file = "grpcio-1.59.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0bd141f4f41907eb90bda74d969c3cb21c1c62779419782a5b3f5e4b5835718"}, + {file = "grpcio-1.59.3-cp312-cp312-win32.whl", hash = "sha256:33b8fd65d4e97efa62baec6171ce51f9cf68f3a8ba9f866f4abc9d62b5c97b79"}, + {file = "grpcio-1.59.3-cp312-cp312-win_amd64.whl", hash = "sha256:0e735ed002f50d4f3cb9ecfe8ac82403f5d842d274c92d99db64cfc998515e07"}, + {file = "grpcio-1.59.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:ea40ce4404e7cca0724c91a7404da410f0144148fdd58402a5942971e3469b94"}, + {file = "grpcio-1.59.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:83113bcc393477b6f7342b9f48e8a054330c895205517edc66789ceea0796b53"}, + {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:73afbac602b8f1212a50088193601f869b5073efa9855b3e51aaaec97848fc8a"}, + {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:575d61de1950b0b0699917b686b1ca108690702fcc2df127b8c9c9320f93e069"}, + {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd76057b5c9a4d68814610ef9226925f94c1231bbe533fdf96f6181f7d2ff9e"}, + {file = "grpcio-1.59.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:95d6fd804c81efe4879e38bfd84d2b26e339a0a9b797e7615e884ef4686eb47b"}, + {file = "grpcio-1.59.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0d42048b8a3286ea4134faddf1f9a59cf98192b94aaa10d910a25613c5eb5bfb"}, + {file = "grpcio-1.59.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4619fea15c64bcdd9d447cdbdde40e3d5f1da3a2e8ae84103d94a9c1df210d7e"}, + {file = "grpcio-1.59.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:95b5506e70284ac03b2005dd9ffcb6708c9ae660669376f0192a710687a22556"}, + {file = "grpcio-1.59.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:9e17660947660ccfce56c7869032910c179a5328a77b73b37305cd1ee9301c2e"}, + {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:00912ce19914d038851be5cd380d94a03f9d195643c28e3ad03d355cc02ce7e8"}, + {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e58b3cadaa3c90f1efca26ba33e0d408b35b497307027d3d707e4bcd8de862a6"}, + {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d787ecadea865bdf78f6679f6f5bf4b984f18f659257ba612979df97a298b3c3"}, + {file = "grpcio-1.59.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0814942ba1bba269db4e760a34388640c601dece525c6a01f3b4ff030cc0db69"}, + {file = "grpcio-1.59.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fb111aa99d3180c361a35b5ae1e2c63750220c584a1344229abc139d5c891881"}, + {file = "grpcio-1.59.3-cp38-cp38-win32.whl", hash = "sha256:eb8ba504c726befe40a356ecbe63c6c3c64c9a439b3164f5a718ec53c9874da0"}, + {file = "grpcio-1.59.3-cp38-cp38-win_amd64.whl", hash = "sha256:cdbc6b32fadab9bebc6f49d3e7ec4c70983c71e965497adab7f87de218e84391"}, + {file = "grpcio-1.59.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:c82ca1e4be24a98a253d6dbaa216542e4163f33f38163fc77964b0f0d255b552"}, + {file = "grpcio-1.59.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:36636babfda14f9e9687f28d5b66d349cf88c1301154dc71c6513de2b6c88c59"}, + {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5f9b2e591da751ac7fdd316cc25afafb7a626dededa9b414f90faad7f3ccebdb"}, + {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a93a82876a4926bf451db82ceb725bd87f42292bacc94586045261f501a86994"}, + {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce31fa0bfdd1f2bb15b657c16105c8652186eab304eb512e6ae3b99b2fdd7d13"}, + {file = "grpcio-1.59.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:16da0e40573962dab6cba16bec31f25a4f468e6d05b658e589090fe103b03e3d"}, + {file = "grpcio-1.59.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d1a17372fd425addd5812049fa7374008ffe689585f27f802d0935522cf4b7"}, + {file = "grpcio-1.59.3-cp39-cp39-win32.whl", hash = "sha256:52cc38a7241b5f7b4a91aaf9000fdd38e26bb00d5e8a71665ce40cfcee716281"}, + {file = "grpcio-1.59.3-cp39-cp39-win_amd64.whl", hash = "sha256:b491e5bbcad3020a96842040421e508780cade35baba30f402df9d321d1c423e"}, + {file = "grpcio-1.59.3.tar.gz", hash = "sha256:7800f99568a74a06ebdccd419dd1b6e639b477dcaf6da77ea702f8fb14ce5f80"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.59.2)"] +protobuf = ["grpcio-tools (>=1.59.3)"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] [[package]] name = "h5py" @@ -1302,6 +1284,51 @@ files = [ [package.dependencies] numpy = ">=1.17.3" +[[package]] +name = "httpcore" +version = "1.0.2" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] + +[[package]] +name = "httpx" +version = "0.25.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.25.1-py3-none-any.whl", hash = "sha256:fec7d6cc5c27c578a391f7e87b9aa7d3d8fbcd034f6399f9f79b45bcc12a866a"}, + {file = "httpx-0.25.1.tar.gz", hash = "sha256:ffd96d5cf901e63863d9f1b4b6807861dbea4d301613415d9e6e57ead15fc5d0"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + [[package]] name = "huggingface-hub" version = "0.17.3" @@ -1412,13 +1439,13 @@ files = [ [[package]] name = "keras" -version = "2.14.0" +version = "2.15.0" description = "Deep learning for humans." optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" files = [ - {file = "keras-2.14.0-py3-none-any.whl", hash = "sha256:d7429d1d2131cc7eb1f2ea2ec330227c7d9d38dab3dfdf2e78defee4ecc43fcd"}, - {file = "keras-2.14.0.tar.gz", hash = "sha256:22788bdbc86d9988794fe9703bb5205141da797c4faeeb59497c58c3d94d34ed"}, + {file = "keras-2.15.0-py3-none-any.whl", hash = "sha256:2dcc6d2e30cf9c951064b63c1f4c404b966c59caf09e01f3549138ec8ee0dd1f"}, + {file = "keras-2.15.0.tar.gz", hash = "sha256:81871d298c064dc4ac6b58440fdae67bfcf47c8d7ad28580fab401834c06a575"}, ] [[package]] @@ -1536,13 +1563,13 @@ files = [ [[package]] name = "langchain" -version = "0.0.335" +version = "0.0.339" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain-0.0.335-py3-none-any.whl", hash = "sha256:f74c98366070a46953c071c69f6c01671a9437569c08406cace256ccaabdfcaf"}, - {file = "langchain-0.0.335.tar.gz", hash = "sha256:93136fe6cc9ac06a80ccf7cf581e58af5cfcc31fef1083b30165df9a9bc53f5d"}, + {file = "langchain-0.0.339-py3-none-any.whl", hash = "sha256:fec250074a6fbb3711a51423d830006d69f34aedb67604df39c642be80852cbb"}, + {file = "langchain-0.0.339.tar.gz", hash = "sha256:34eb4d7987d979663e361da435479c6f0648a170dae3eb1e9f0f7417f033a2c1"}, ] [package.dependencies] @@ -1560,17 +1587,17 @@ SQLAlchemy = ">=1.4,<3" tenacity = ">=8.1.0,<9.0.0" [package.extras] -all = ["O365 (>=2.0.26,<3.0.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "amadeus (>=8.1.0)", "arxiv (>=1.4,<2.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "awadb (>=0.3.9,<0.4.0)", "azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "beautifulsoup4 (>=4,<5)", "clarifai (>=9.1.0)", "clickhouse-connect (>=0.5.14,<0.6.0)", "cohere (>=4,<5)", "deeplake (>=3.8.3,<4.0.0)", "docarray[hnswlib] (>=0.32.0,<0.33.0)", "duckduckgo-search (>=3.8.3,<4.0.0)", "elasticsearch (>=8,<9)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "google-api-python-client (==2.70.0)", "google-auth (>=2.18.1,<3.0.0)", "google-search-results (>=2,<3)", "gptcache (>=0.1.7)", "html2text (>=2020.1.16,<2021.0.0)", "huggingface_hub (>=0,<1)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lancedb (>=0.1,<0.2)", "langkit (>=0.0.6,<0.1.0)", "lark (>=1.1.5,<2.0.0)", "librosa (>=0.10.0.post2,<0.11.0)", "lxml (>=4.9.2,<5.0.0)", "manifest-ml (>=0.0.1,<0.0.2)", "marqo (>=1.2.4,<2.0.0)", "momento (>=1.10.1,<2.0.0)", "nebula3-python (>=3.4.0,<4.0.0)", "neo4j (>=5.8.1,<6.0.0)", "networkx (>=2.6.3,<4)", "nlpcloud (>=1,<2)", "nltk (>=3,<4)", "nomic (>=1.0.43,<2.0.0)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "opensearch-py (>=2.0.0,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pexpect (>=4.8.0,<5.0.0)", "pgvector (>=0.1.6,<0.2.0)", "pinecone-client (>=2,<3)", "pinecone-text (>=0.4.2,<0.5.0)", "psycopg2-binary (>=2.9.5,<3.0.0)", "pymongo (>=4.3.3,<5.0.0)", "pyowm (>=3.3.0,<4.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pytesseract (>=0.3.10,<0.4.0)", "python-arango (>=7.5.9,<8.0.0)", "pyvespa (>=0.33.0,<0.34.0)", "qdrant-client (>=1.3.1,<2.0.0)", "rdflib (>=6.3.2,<7.0.0)", "redis (>=4,<5)", "requests-toolbelt (>=1.0.0,<2.0.0)", "sentence-transformers (>=2,<3)", "singlestoredb (>=0.7.1,<0.8.0)", "tensorflow-text (>=2.11.0,<3.0.0)", "tigrisdb (>=1.0.0b6,<2.0.0)", "tiktoken (>=0.3.2,<0.6.0)", "torch (>=1,<3)", "transformers (>=4,<5)", "weaviate-client (>=3,<4)", "wikipedia (>=1,<2)", "wolframalpha (==5.0.0)"] -azure = ["azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-core (>=1.26.4,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "azure-search-documents (==11.4.0b8)", "openai (>=0,<1)"] +all = ["O365 (>=2.0.26,<3.0.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "amadeus (>=8.1.0)", "arxiv (>=1.4,<2.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "awadb (>=0.3.9,<0.4.0)", "azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-textanalytics (>=5.3.0,<6.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "beautifulsoup4 (>=4,<5)", "clarifai (>=9.1.0)", "clickhouse-connect (>=0.5.14,<0.6.0)", "cohere (>=4,<5)", "deeplake (>=3.8.3,<4.0.0)", "docarray[hnswlib] (>=0.32.0,<0.33.0)", "duckduckgo-search (>=3.8.3,<4.0.0)", "elasticsearch (>=8,<9)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "google-api-python-client (==2.70.0)", "google-auth (>=2.18.1,<3.0.0)", "google-search-results (>=2,<3)", "gptcache (>=0.1.7)", "html2text (>=2020.1.16,<2021.0.0)", "huggingface_hub (>=0,<1)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lancedb (>=0.1,<0.2)", "langkit (>=0.0.6,<0.1.0)", "lark (>=1.1.5,<2.0.0)", "librosa (>=0.10.0.post2,<0.11.0)", "lxml (>=4.9.2,<5.0.0)", "manifest-ml (>=0.0.1,<0.0.2)", "marqo (>=1.2.4,<2.0.0)", "momento (>=1.13.0,<2.0.0)", "nebula3-python (>=3.4.0,<4.0.0)", "neo4j (>=5.8.1,<6.0.0)", "networkx (>=2.6.3,<4)", "nlpcloud (>=1,<2)", "nltk (>=3,<4)", "nomic (>=1.0.43,<2.0.0)", "openai (<2)", "openlm (>=0.0.5,<0.0.6)", "opensearch-py (>=2.0.0,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pexpect (>=4.8.0,<5.0.0)", "pgvector (>=0.1.6,<0.2.0)", "pinecone-client (>=2,<3)", "pinecone-text (>=0.4.2,<0.5.0)", "psycopg2-binary (>=2.9.5,<3.0.0)", "pymongo (>=4.3.3,<5.0.0)", "pyowm (>=3.3.0,<4.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pytesseract (>=0.3.10,<0.4.0)", "python-arango (>=7.5.9,<8.0.0)", "pyvespa (>=0.33.0,<0.34.0)", "qdrant-client (>=1.3.1,<2.0.0)", "rdflib (>=6.3.2,<7.0.0)", "redis (>=4,<5)", "requests-toolbelt (>=1.0.0,<2.0.0)", "sentence-transformers (>=2,<3)", "singlestoredb (>=0.7.1,<0.8.0)", "tensorflow-text (>=2.11.0,<3.0.0)", "tigrisdb (>=1.0.0b6,<2.0.0)", "tiktoken (>=0.3.2,<0.6.0)", "torch (>=1,<3)", "transformers (>=4,<5)", "weaviate-client (>=3,<4)", "wikipedia (>=1,<2)", "wolframalpha (==5.0.0)"] +azure = ["azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-textanalytics (>=5.3.0,<6.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-core (>=1.26.4,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "azure-search-documents (==11.4.0b8)", "openai (<2)"] clarifai = ["clarifai (>=9.1.0)"] cli = ["typer (>=0.9.0,<0.10.0)"] cohere = ["cohere (>=4,<5)"] docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"] embeddings = ["sentence-transformers (>=2,<3)"] -extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.6.0,<0.7.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (>=0,<1)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] +extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.6.0,<0.7.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] javascript = ["esprima (>=4.0.1,<5.0.0)"] -llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"] -openai = ["openai (>=0,<1)", "tiktoken (>=0.3.2,<0.6.0)"] +llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (<2)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"] +openai = ["openai (<2)", "tiktoken (>=0.3.2,<0.6.0)"] qdrant = ["qdrant-client (>=1.3.1,<2.0.0)"] text-helpers = ["chardet (>=5.1.0,<6.0.0)"] @@ -1590,13 +1617,13 @@ data = ["language-data (>=1.1,<2.0)"] [[package]] name = "langsmith" -version = "0.0.63" +version = "0.0.66" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langsmith-0.0.63-py3-none-any.whl", hash = "sha256:43a521dd10d8405ac21a0b959e3de33e2270e4abe6c73cc4036232a6990a0793"}, - {file = "langsmith-0.0.63.tar.gz", hash = "sha256:ddb2dfadfad3e05151ed8ba1643d1c516024b80fbd0c6263024400ced06a3768"}, + {file = "langsmith-0.0.66-py3-none-any.whl", hash = "sha256:e5e6d2deff19de827ac04db106b900091c75b6a3c1a1c047a8aa78caf72a63ea"}, + {file = "langsmith-0.0.66.tar.gz", hash = "sha256:33d011c9db9236c06789b17dba97acc023275bafd0c2bf097283730d6608dea7"}, ] [package.dependencies] @@ -1623,16 +1650,6 @@ files = [ {file = "libclang-16.0.6.tar.gz", hash = "sha256:4acdde39dfe410c877b4ccc0d4b57eb952100e4ee26bbdf6cfdb88e2033a7d31"}, ] -[[package]] -name = "lit" -version = "17.0.4" -description = "A Software Testing Tool" -optional = false -python-versions = "*" -files = [ - {file = "lit-17.0.4.tar.gz", hash = "sha256:ee2e180128e770abc6aed3a02de2daf09d81b7d30225e315205d3599c311d304"}, -] - [[package]] name = "markdown" version = "3.5.1" @@ -1763,39 +1780,39 @@ tests = ["pytest", "pytz", "simplejson"] [[package]] name = "matplotlib" -version = "3.8.1" +version = "3.8.2" description = "Python plotting package" optional = false python-versions = ">=3.9" files = [ - {file = "matplotlib-3.8.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:e11ab864323fa73ac1b7849688d9671c47a2665242e899785b4db1a375b547e1"}, - {file = "matplotlib-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:43a9d40feb63c9e31a0b8b069dcbd74a912f59bdc0095d187126694cd26977e4"}, - {file = "matplotlib-3.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:608ea2951838d391e45dec2e644888db6899c752d3c29e157af9dcefb3d7d8d5"}, - {file = "matplotlib-3.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82ec95b02e894561c21e066bd0c716e4b410df141ce9441aa5af6cd937e4ade2"}, - {file = "matplotlib-3.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e3ad1759ad4a5245172c6d32b8ada603a6020d03211524c39d78d25c9a7dc0d2"}, - {file = "matplotlib-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:20a0fdfd3ee836179047f3782be060057b878ad37f5abe29edf006a1ff3ecd73"}, - {file = "matplotlib-3.8.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7658b7073c1d6a2922ecc0ed41602410fae88586cb8a54f7a2063d537b6beaf7"}, - {file = "matplotlib-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf6889643d4560fcc56f9f0941f078e4df0d72a6c3e4ca548841fc13c5642664"}, - {file = "matplotlib-3.8.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff842e27bc6a80de08c40e0bfdce460bd08080e8a94af131162b6a1b8948f2cc"}, - {file = "matplotlib-3.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f99d07c0e753717775be7be39ab383453b4d8b629c9fa174596b970c6555890"}, - {file = "matplotlib-3.8.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f34b46dbb1db1f09bfa937cd5853e5f2af232caeeff509c3ab6e43fd33780eae"}, - {file = "matplotlib-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:1fcb49b6baf0375281979cbf26695ec10bd1cada1e311893e89533b3b70143e7"}, - {file = "matplotlib-3.8.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e17674ee127f78f26fea237e7f4d5cf910a8be82beb6260fedf358b88075b823"}, - {file = "matplotlib-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d921c0270647ab11c3ef283efaaa3d46fd005ba233bfb3aea75231cdf3656de8"}, - {file = "matplotlib-3.8.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2afe7d2f8c9e35e94fbcfcfd9b28f29cb32f0a9068cba469cf907428379c8db9"}, - {file = "matplotlib-3.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5a504ff40f81d6233603475a45497a6dca37a873393fa20ae6f7dd6596ef72b"}, - {file = "matplotlib-3.8.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cd54bbf089953140905768ed4626d7223e1ad1d7e2a138410a9c4d3b865ccd80"}, - {file = "matplotlib-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:27502d2452208ae784c19504644f09f83742809143bbeae147617640930aa344"}, - {file = "matplotlib-3.8.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:f55fb5ff02d999a100be28bf6ffe826e1867a54c7b465409685332c9dd48ffa5"}, - {file = "matplotlib-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:afb72822ae410d62aa1a2920c6563cb5680de9078358f0e9474396c6c3e06be2"}, - {file = "matplotlib-3.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43cf368a4a1d8cbc426944806e5e183cead746647a64d2cdb786441546235967"}, - {file = "matplotlib-3.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c54c55457c7f5ea4dfdba0020004fc7667f5c10c8d9b8010d735345acc06c9b8"}, - {file = "matplotlib-3.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e3bb809b743653b5aab5d72ee45c8c937c28e147b0846b0826a54bece898608c"}, - {file = "matplotlib-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:c1b0ecaa0d1f4fe1e30f625a2347f0034a89a7d17c39efbb502e554d92ee2f61"}, - {file = "matplotlib-3.8.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ca84deaa38cb64b7dd160ca2046b45f7b5dbff2b0179642e1339fadc337446c9"}, - {file = "matplotlib-3.8.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed3b29f54f6bbf3eaca4cbd23bc260155153ace63b7f597c474fa6fc6f386530"}, - {file = "matplotlib-3.8.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0d24c47a1bb47e392fbcd26fe322e4ff3431653ac1e8718e4e147d450ae97a44"}, - {file = "matplotlib-3.8.1.tar.gz", hash = "sha256:044df81c1f6f3a8e52d70c4cfcb44e77ea9632a10929932870dfaa90de94365d"}, + {file = "matplotlib-3.8.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:09796f89fb71a0c0e1e2f4bdaf63fb2cefc84446bb963ecdeb40dfee7dfa98c7"}, + {file = "matplotlib-3.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f9c6976748a25e8b9be51ea028df49b8e561eed7809146da7a47dbecebab367"}, + {file = "matplotlib-3.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b78e4f2cedf303869b782071b55fdde5987fda3038e9d09e58c91cc261b5ad18"}, + {file = "matplotlib-3.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e208f46cf6576a7624195aa047cb344a7f802e113bb1a06cfd4bee431de5e31"}, + {file = "matplotlib-3.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:46a569130ff53798ea5f50afce7406e91fdc471ca1e0e26ba976a8c734c9427a"}, + {file = "matplotlib-3.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:830f00640c965c5b7f6bc32f0d4ce0c36dfe0379f7dd65b07a00c801713ec40a"}, + {file = "matplotlib-3.8.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d86593ccf546223eb75a39b44c32788e6f6440d13cfc4750c1c15d0fcb850b63"}, + {file = "matplotlib-3.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9a5430836811b7652991939012f43d2808a2db9b64ee240387e8c43e2e5578c8"}, + {file = "matplotlib-3.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9576723858a78751d5aacd2497b8aef29ffea6d1c95981505877f7ac28215c6"}, + {file = "matplotlib-3.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ba9cbd8ac6cf422f3102622b20f8552d601bf8837e49a3afed188d560152788"}, + {file = "matplotlib-3.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:03f9d160a29e0b65c0790bb07f4f45d6a181b1ac33eb1bb0dd225986450148f0"}, + {file = "matplotlib-3.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:3773002da767f0a9323ba1a9b9b5d00d6257dbd2a93107233167cfb581f64717"}, + {file = "matplotlib-3.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:4c318c1e95e2f5926fba326f68177dee364aa791d6df022ceb91b8221bd0a627"}, + {file = "matplotlib-3.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:091275d18d942cf1ee9609c830a1bc36610607d8223b1b981c37d5c9fc3e46a4"}, + {file = "matplotlib-3.8.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b0f3b8ea0e99e233a4bcc44590f01604840d833c280ebb8fe5554fd3e6cfe8d"}, + {file = "matplotlib-3.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7b1704a530395aaf73912be741c04d181f82ca78084fbd80bc737be04848331"}, + {file = "matplotlib-3.8.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:533b0e3b0c6768eef8cbe4b583731ce25a91ab54a22f830db2b031e83cca9213"}, + {file = "matplotlib-3.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:0f4fc5d72b75e2c18e55eb32292659cf731d9d5b312a6eb036506304f4675630"}, + {file = "matplotlib-3.8.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:deaed9ad4da0b1aea77fe0aa0cebb9ef611c70b3177be936a95e5d01fa05094f"}, + {file = "matplotlib-3.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:172f4d0fbac3383d39164c6caafd3255ce6fa58f08fc392513a0b1d3b89c4f89"}, + {file = "matplotlib-3.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7d36c2209d9136cd8e02fab1c0ddc185ce79bc914c45054a9f514e44c787917"}, + {file = "matplotlib-3.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5864bdd7da445e4e5e011b199bb67168cdad10b501750367c496420f2ad00843"}, + {file = "matplotlib-3.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ef8345b48e95cee45ff25192ed1f4857273117917a4dcd48e3905619bcd9c9b8"}, + {file = "matplotlib-3.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:7c48d9e221b637c017232e3760ed30b4e8d5dfd081daf327e829bf2a72c731b4"}, + {file = "matplotlib-3.8.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:aa11b3c6928a1e496c1a79917d51d4cd5d04f8a2e75f21df4949eeefdf697f4b"}, + {file = "matplotlib-3.8.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1095fecf99eeb7384dabad4bf44b965f929a5f6079654b681193edf7169ec20"}, + {file = "matplotlib-3.8.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:bddfb1db89bfaa855912261c805bd0e10218923cc262b9159a49c29a7a1c1afa"}, + {file = "matplotlib-3.8.2.tar.gz", hash = "sha256:01a978b871b881ee76017152f1f1a0cbf6bd5f7b8ff8c96df0df1bd57d8755a1"}, ] [package.dependencies] @@ -2125,162 +2142,145 @@ files = [ ] [[package]] -name = "nvidia-cublas-cu11" -version = "11.10.3.66" +name = "nvidia-cublas-cu12" +version = "12.1.3.1" description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, + {file = "nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:ee53ccca76a6fc08fb9701aa95b6ceb242cdaab118c3bb152af4e579af792728"}, + {file = "nvidia_cublas_cu12-12.1.3.1-py3-none-win_amd64.whl", hash = "sha256:2b964d60e8cf11b5e1073d179d85fa340c120e99b3067558f3cf98dd69d02906"}, ] -[package.dependencies] -setuptools = "*" -wheel = "*" - [[package]] -name = "nvidia-cuda-cupti-cu11" -version = "11.7.101" +name = "nvidia-cuda-cupti-cu12" +version = "12.1.105" description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" files = [ - {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl", hash = "sha256:e0cfd9854e1f2edaa36ca20d21cd0bdd5dcfca4e3b9e130a082e05b33b6c5895"}, - {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-win_amd64.whl", hash = "sha256:7cc5b8f91ae5e1389c3c0ad8866b3b016a175e827ea8f162a672990a402ab2b0"}, + {file = "nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:e54fde3983165c624cb79254ae9818a456eb6e87a7fd4d56a2352c24ee542d7e"}, + {file = "nvidia_cuda_cupti_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:bea8236d13a0ac7190bd2919c3e8e6ce1e402104276e6f9694479e48bb0eb2a4"}, ] -[package.dependencies] -setuptools = "*" -wheel = "*" - [[package]] -name = "nvidia-cuda-nvrtc-cu11" -version = "11.7.99" +name = "nvidia-cuda-nvrtc-cu12" +version = "12.1.105" description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3"}, + {file = "nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:339b385f50c309763ca65456ec75e17bbefcbbf2893f462cb8b90584cd27a1c2"}, + {file = "nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:0a98a522d9ff138b96c010a65e145dc1b4850e9ecb75a0172371793752fd46ed"}, ] -[package.dependencies] -setuptools = "*" -wheel = "*" - [[package]] -name = "nvidia-cuda-runtime-cu11" -version = "11.7.99" +name = "nvidia-cuda-runtime-cu12" +version = "12.1.105" description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, + {file = "nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:6e258468ddf5796e25f1dc591a31029fa317d97a0a94ed93468fc86301d61e40"}, + {file = "nvidia_cuda_runtime_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:dfb46ef84d73fababab44cf03e3b83f80700d27ca300e537f85f636fac474344"}, ] -[package.dependencies] -setuptools = "*" -wheel = "*" - [[package]] -name = "nvidia-cudnn-cu11" -version = "8.5.0.96" +name = "nvidia-cudnn-cu12" +version = "8.9.2.26" description = "cuDNN runtime libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, - {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, + {file = "nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl", hash = "sha256:5ccb288774fdfb07a7e7025ffec286971c06d8d7b4fb162525334616d7629ff9"}, ] [package.dependencies] -setuptools = "*" -wheel = "*" +nvidia-cublas-cu12 = "*" [[package]] -name = "nvidia-cufft-cu11" -version = "10.9.0.58" +name = "nvidia-cufft-cu12" +version = "11.0.2.54" description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl", hash = "sha256:222f9da70c80384632fd6035e4c3f16762d64ea7a843829cb278f98b3cb7dd81"}, - {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-win_amd64.whl", hash = "sha256:c4d316f17c745ec9c728e30409612eaf77a8404c3733cdf6c9c1569634d1ca03"}, + {file = "nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl", hash = "sha256:794e3948a1aa71fd817c3775866943936774d1c14e7628c74f6f7417224cdf56"}, + {file = "nvidia_cufft_cu12-11.0.2.54-py3-none-win_amd64.whl", hash = "sha256:d9ac353f78ff89951da4af698f80870b1534ed69993f10a4cf1d96f21357e253"}, ] [[package]] -name = "nvidia-curand-cu11" -version = "10.2.10.91" +name = "nvidia-curand-cu12" +version = "10.3.2.106" description = "CURAND native runtime libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:eecb269c970fa599a2660c9232fa46aaccbf90d9170b96c462e13bcb4d129e2c"}, - {file = "nvidia_curand_cu11-10.2.10.91-py3-none-win_amd64.whl", hash = "sha256:f742052af0e1e75523bde18895a9ed016ecf1e5aa0ecddfcc3658fd11a1ff417"}, + {file = "nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:9d264c5036dde4e64f1de8c50ae753237c12e0b1348738169cd0f8a536c0e1e0"}, + {file = "nvidia_curand_cu12-10.3.2.106-py3-none-win_amd64.whl", hash = "sha256:75b6b0c574c0037839121317e17fd01f8a69fd2ef8e25853d826fec30bdba74a"}, ] -[package.dependencies] -setuptools = "*" -wheel = "*" - [[package]] -name = "nvidia-cusolver-cu11" -version = "11.4.0.1" +name = "nvidia-cusolver-cu12" +version = "11.4.5.107" description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:72fa7261d755ed55c0074960df5904b65e2326f7adce364cbe4945063c1be412"}, - {file = "nvidia_cusolver_cu11-11.4.0.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:700b781bfefd57d161443aff9ace1878584b93e0b2cfef3d6e9296d96febbf99"}, - {file = "nvidia_cusolver_cu11-11.4.0.1-py3-none-win_amd64.whl", hash = "sha256:00f70b256add65f8c1eb3b6a65308795a93e7740f6df9e273eccbba770d370c4"}, + {file = "nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl", hash = "sha256:8a7ec542f0412294b15072fa7dab71d31334014a69f953004ea7a118206fe0dd"}, + {file = "nvidia_cusolver_cu12-11.4.5.107-py3-none-win_amd64.whl", hash = "sha256:74e0c3a24c78612192a74fcd90dd117f1cf21dea4822e66d89e8ea80e3cd2da5"}, ] [package.dependencies] -setuptools = "*" -wheel = "*" +nvidia-cublas-cu12 = "*" +nvidia-cusparse-cu12 = "*" +nvidia-nvjitlink-cu12 = "*" [[package]] -name = "nvidia-cusparse-cu11" -version = "11.7.4.91" +name = "nvidia-cusparse-cu12" +version = "12.1.0.106" description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" files = [ - {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:a3389de714db63321aa11fbec3919271f415ef19fda58aed7f2ede488c32733d"}, - {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-win_amd64.whl", hash = "sha256:304a01599534f5186a8ed1c3756879282c72c118bc77dd890dc1ff868cad25b9"}, + {file = "nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:f3b50f42cf363f86ab21f720998517a659a48131e8d538dc02f8768237bd884c"}, + {file = "nvidia_cusparse_cu12-12.1.0.106-py3-none-win_amd64.whl", hash = "sha256:b798237e81b9719373e8fae8d4f091b70a0cf09d9d85c95a557e11df2d8e9a5a"}, ] [package.dependencies] -setuptools = "*" -wheel = "*" +nvidia-nvjitlink-cu12 = "*" [[package]] -name = "nvidia-nccl-cu11" -version = "2.14.3" +name = "nvidia-nccl-cu12" +version = "2.18.1" description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" files = [ - {file = "nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl", hash = "sha256:5e5534257d1284b8e825bc3a182c6f06acd6eb405e9f89d49340e98cd8f136eb"}, + {file = "nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:1a6c4acefcbebfa6de320f412bf7866de856e786e0462326ba1bac40de0b5e71"}, ] [[package]] -name = "nvidia-nvtx-cu11" -version = "11.7.91" -description = "NVIDIA Tools Extension" +name = "nvidia-nvjitlink-cu12" +version = "12.3.101" +description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" files = [ - {file = "nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:b22c64eee426a62fc00952b507d6d29cf62b4c9df7a480fcc417e540e05fd5ac"}, - {file = "nvidia_nvtx_cu11-11.7.91-py3-none-win_amd64.whl", hash = "sha256:dfd7fcb2a91742513027d63a26b757f38dd8b07fecac282c4d132a9d373ff064"}, + {file = "nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl", hash = "sha256:64335a8088e2b9d196ae8665430bc6a2b7e6ef2eb877a9c735c804bd4ff6467c"}, + {file = "nvidia_nvjitlink_cu12-12.3.101-py3-none-win_amd64.whl", hash = "sha256:1b2e317e437433753530792f13eece58f0aec21a2b05903be7bffe58a606cbd1"}, ] -[package.dependencies] -setuptools = "*" -wheel = "*" +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.1.105" +description = "NVIDIA Tools Extension" +optional = false +python-versions = ">=3" +files = [ + {file = "nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:dc21cf308ca5691e7c04d962e213f8a4aa9bbfa23d95412f452254c2caeb09e5"}, + {file = "nvidia_nvtx_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:65f4d98982b31b60026e0e6de73fbdfc09d08a96f4656dd3665ca616a11e1e82"}, +] [[package]] name = "oauthlib" @@ -2300,25 +2300,25 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openai" -version = "0.28.0" -description = "Python client library for the OpenAI API" +version = "1.3.4" +description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-0.28.0-py3-none-any.whl", hash = "sha256:d207ece78469be5648eb87b825753282225155a29d0eec6e02013ddbf8c31c0c"}, - {file = "openai-0.28.0.tar.gz", hash = "sha256:417b78c4c2864ba696aedaf1ccff77be1f04a581ab1739f0a56e0aae19e5a794"}, + {file = "openai-1.3.4-py3-none-any.whl", hash = "sha256:a8daed9553bbfc9e6d6a74615a2d48e4ff293d74d51f75b45ff18a9edbe61ce3"}, + {file = "openai-1.3.4.tar.gz", hash = "sha256:9fd7a8d9efbff4c35b4026266432e49372ca0da653a5d12dac3e4330f7e29b88"}, ] [package.dependencies] -aiohttp = "*" -requests = ">=2.20" -tqdm = "*" +anyio = ">=3.5.0,<4" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +pydantic = ">=1.9.0,<3" +tqdm = ">4" +typing-extensions = ">=4.5,<5" [package.extras] -datalib = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] -dev = ["black (>=21.6b0,<22.0)", "pytest (==6.*)", "pytest-asyncio", "pytest-mock"] -embeddings = ["matplotlib", "numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "plotly", "scikit-learn (>=1.0.2)", "scipy", "tenacity (>=8.0.1)"] -wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "wandb"] +datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] [[package]] name = "opt-einsum" @@ -2606,22 +2606,24 @@ murmurhash = ">=0.28.0,<1.1.0" [[package]] name = "protobuf" -version = "4.25.0" +version = "4.23.4" description = "" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "protobuf-4.25.0-cp310-abi3-win32.whl", hash = "sha256:5c1203ac9f50e4853b0a0bfffd32c67118ef552a33942982eeab543f5c634395"}, - {file = "protobuf-4.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:c40ff8f00aa737938c5378d461637d15c442a12275a81019cc2fef06d81c9419"}, - {file = "protobuf-4.25.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:cf21faba64cd2c9a3ed92b7a67f226296b10159dbb8fbc5e854fc90657d908e4"}, - {file = "protobuf-4.25.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:32ac2100b0e23412413d948c03060184d34a7c50b3e5d7524ee96ac2b10acf51"}, - {file = "protobuf-4.25.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:683dc44c61f2620b32ce4927de2108f3ebe8ccf2fd716e1e684e5a50da154054"}, - {file = "protobuf-4.25.0-cp38-cp38-win32.whl", hash = "sha256:1a3ba712877e6d37013cdc3476040ea1e313a6c2e1580836a94f76b3c176d575"}, - {file = "protobuf-4.25.0-cp38-cp38-win_amd64.whl", hash = "sha256:b2cf8b5d381f9378afe84618288b239e75665fe58d0f3fd5db400959274296e9"}, - {file = "protobuf-4.25.0-cp39-cp39-win32.whl", hash = "sha256:63714e79b761a37048c9701a37438aa29945cd2417a97076048232c1df07b701"}, - {file = "protobuf-4.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:d94a33db8b7ddbd0af7c467475fb9fde0c705fb315a8433c0e2020942b863a1f"}, - {file = "protobuf-4.25.0-py3-none-any.whl", hash = "sha256:1a53d6f64b00eecf53b65ff4a8c23dc95df1fa1e97bb06b8122e5a64f49fc90a"}, - {file = "protobuf-4.25.0.tar.gz", hash = "sha256:68f7caf0d4f012fd194a301420cf6aa258366144d814f358c5b32558228afa7c"}, + {file = "protobuf-4.23.4-cp310-abi3-win32.whl", hash = "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b"}, + {file = "protobuf-4.23.4-cp310-abi3-win_amd64.whl", hash = "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12"}, + {file = "protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd"}, + {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a"}, + {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597"}, + {file = "protobuf-4.23.4-cp37-cp37m-win32.whl", hash = "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e"}, + {file = "protobuf-4.23.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0"}, + {file = "protobuf-4.23.4-cp38-cp38-win32.whl", hash = "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70"}, + {file = "protobuf-4.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2"}, + {file = "protobuf-4.23.4-cp39-cp39-win32.whl", hash = "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720"}, + {file = "protobuf-4.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474"}, + {file = "protobuf-4.23.4-py3-none-any.whl", hash = "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff"}, + {file = "protobuf-4.23.4.tar.gz", hash = "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9"}, ] [[package]] @@ -2672,15 +2674,26 @@ files = [ [package.dependencies] numpy = ">=1.16.6" +[[package]] +name = "pyarrow-hotfix" +version = "0.6" +description = "" +optional = false +python-versions = ">=3.5" +files = [ + {file = "pyarrow_hotfix-0.6-py3-none-any.whl", hash = "sha256:dcc9ae2d220dff0083be6a9aa8e0cdee5182ad358d4931fce825c545e5c89178"}, + {file = "pyarrow_hotfix-0.6.tar.gz", hash = "sha256:79d3e030f7ff890d408a100ac16d6f00b14d44a502d7897cd9fc3e3a534e9945"}, +] + [[package]] name = "pyasn1" -version = "0.5.0" +version = "0.5.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, - {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, ] [[package]] @@ -2699,18 +2712,18 @@ pyasn1 = ">=0.4.6,<0.6.0" [[package]] name = "pydantic" -version = "2.4.2" +version = "2.5.1" description = "Data validation using Python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-2.4.2-py3-none-any.whl", hash = "sha256:bc3ddf669d234f4220e6e1c4d96b061abe0998185a8d7855c0126782b7abc8c1"}, - {file = "pydantic-2.4.2.tar.gz", hash = "sha256:94f336138093a5d7f426aac732dcfe7ab4eb4da243c88f891d65deb4a2556ee7"}, + {file = "pydantic-2.5.1-py3-none-any.whl", hash = "sha256:dc5244a8939e0d9a68f1f1b5f550b2e1c879912033b1becbedb315accc75441b"}, + {file = "pydantic-2.5.1.tar.gz", hash = "sha256:0b8be5413c06aadfbe56f6dc1d45c9ed25fd43264414c571135c97dd77c2bedb"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.10.1" +pydantic-core = "2.14.3" typing-extensions = ">=4.6.1" [package.extras] @@ -2718,117 +2731,116 @@ email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.10.1" +version = "2.14.3" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic_core-2.10.1-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:d64728ee14e667ba27c66314b7d880b8eeb050e58ffc5fec3b7a109f8cddbd63"}, - {file = "pydantic_core-2.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:48525933fea744a3e7464c19bfede85df4aba79ce90c60b94d8b6e1eddd67096"}, - {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef337945bbd76cce390d1b2496ccf9f90b1c1242a3a7bc242ca4a9fc5993427a"}, - {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1392e0638af203cee360495fd2cfdd6054711f2db5175b6e9c3c461b76f5175"}, - {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0675ba5d22de54d07bccde38997e780044dcfa9a71aac9fd7d4d7a1d2e3e65f7"}, - {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:128552af70a64660f21cb0eb4876cbdadf1a1f9d5de820fed6421fa8de07c893"}, - {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f6e6aed5818c264412ac0598b581a002a9f050cb2637a84979859e70197aa9e"}, - {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ecaac27da855b8d73f92123e5f03612b04c5632fd0a476e469dfc47cd37d6b2e"}, - {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3c01c2fb081fced3bbb3da78510693dc7121bb893a1f0f5f4b48013201f362e"}, - {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:92f675fefa977625105708492850bcbc1182bfc3e997f8eecb866d1927c98ae6"}, - {file = "pydantic_core-2.10.1-cp310-none-win32.whl", hash = "sha256:420a692b547736a8d8703c39ea935ab5d8f0d2573f8f123b0a294e49a73f214b"}, - {file = "pydantic_core-2.10.1-cp310-none-win_amd64.whl", hash = "sha256:0880e239827b4b5b3e2ce05e6b766a7414e5f5aedc4523be6b68cfbc7f61c5d0"}, - {file = "pydantic_core-2.10.1-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:073d4a470b195d2b2245d0343569aac7e979d3a0dcce6c7d2af6d8a920ad0bea"}, - {file = "pydantic_core-2.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:600d04a7b342363058b9190d4e929a8e2e715c5682a70cc37d5ded1e0dd370b4"}, - {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39215d809470f4c8d1881758575b2abfb80174a9e8daf8f33b1d4379357e417c"}, - {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eeb3d3d6b399ffe55f9a04e09e635554012f1980696d6b0aca3e6cf42a17a03b"}, - {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a7902bf75779bc12ccfc508bfb7a4c47063f748ea3de87135d433a4cca7a2f"}, - {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3625578b6010c65964d177626fde80cf60d7f2e297d56b925cb5cdeda6e9925a"}, - {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:caa48fc31fc7243e50188197b5f0c4228956f97b954f76da157aae7f67269ae8"}, - {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:07ec6d7d929ae9c68f716195ce15e745b3e8fa122fc67698ac6498d802ed0fa4"}, - {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e6f31a17acede6a8cd1ae2d123ce04d8cca74056c9d456075f4f6f85de055607"}, - {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d8f1ebca515a03e5654f88411420fea6380fc841d1bea08effb28184e3d4899f"}, - {file = "pydantic_core-2.10.1-cp311-none-win32.whl", hash = "sha256:6db2eb9654a85ada248afa5a6db5ff1cf0f7b16043a6b070adc4a5be68c716d6"}, - {file = "pydantic_core-2.10.1-cp311-none-win_amd64.whl", hash = "sha256:4a5be350f922430997f240d25f8219f93b0c81e15f7b30b868b2fddfc2d05f27"}, - {file = "pydantic_core-2.10.1-cp311-none-win_arm64.whl", hash = "sha256:5fdb39f67c779b183b0c853cd6b45f7db84b84e0571b3ef1c89cdb1dfc367325"}, - {file = "pydantic_core-2.10.1-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1f22a9ab44de5f082216270552aa54259db20189e68fc12484873d926426921"}, - {file = "pydantic_core-2.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8572cadbf4cfa95fb4187775b5ade2eaa93511f07947b38f4cd67cf10783b118"}, - {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db9a28c063c7c00844ae42a80203eb6d2d6bbb97070cfa00194dff40e6f545ab"}, - {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2a35baa428181cb2270a15864ec6286822d3576f2ed0f4cd7f0c1708472aff"}, - {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05560ab976012bf40f25d5225a58bfa649bb897b87192a36c6fef1ab132540d7"}, - {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d6495008733c7521a89422d7a68efa0a0122c99a5861f06020ef5b1f51f9ba7c"}, - {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ac492c686defc8e6133e3a2d9eaf5261b3df26b8ae97450c1647286750b901"}, - {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8282bab177a9a3081fd3d0a0175a07a1e2bfb7fcbbd949519ea0980f8a07144d"}, - {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:aafdb89fdeb5fe165043896817eccd6434aee124d5ee9b354f92cd574ba5e78f"}, - {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f6defd966ca3b187ec6c366604e9296f585021d922e666b99c47e78738b5666c"}, - {file = "pydantic_core-2.10.1-cp312-none-win32.whl", hash = "sha256:7c4d1894fe112b0864c1fa75dffa045720a194b227bed12f4be7f6045b25209f"}, - {file = "pydantic_core-2.10.1-cp312-none-win_amd64.whl", hash = "sha256:5994985da903d0b8a08e4935c46ed8daf5be1cf217489e673910951dc533d430"}, - {file = "pydantic_core-2.10.1-cp312-none-win_arm64.whl", hash = "sha256:0d8a8adef23d86d8eceed3e32e9cca8879c7481c183f84ed1a8edc7df073af94"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:9badf8d45171d92387410b04639d73811b785b5161ecadabf056ea14d62d4ede"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:ebedb45b9feb7258fac0a268a3f6bec0a2ea4d9558f3d6f813f02ff3a6dc6698"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfe1090245c078720d250d19cb05d67e21a9cd7c257698ef139bc41cf6c27b4f"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e357571bb0efd65fd55f18db0a2fb0ed89d0bb1d41d906b138f088933ae618bb"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3dcd587b69bbf54fc04ca157c2323b8911033e827fffaecf0cafa5a892a0904"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c120c9ce3b163b985a3b966bb701114beb1da4b0468b9b236fc754783d85aa3"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15d6bca84ffc966cc9976b09a18cf9543ed4d4ecbd97e7086f9ce9327ea48891"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cabb9710f09d5d2e9e2748c3e3e20d991a4c5f96ed8f1132518f54ab2967221"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:82f55187a5bebae7d81d35b1e9aaea5e169d44819789837cdd4720d768c55d15"}, - {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1d40f55222b233e98e3921df7811c27567f0e1a4411b93d4c5c0f4ce131bc42f"}, - {file = "pydantic_core-2.10.1-cp37-none-win32.whl", hash = "sha256:14e09ff0b8fe6e46b93d36a878f6e4a3a98ba5303c76bb8e716f4878a3bee92c"}, - {file = "pydantic_core-2.10.1-cp37-none-win_amd64.whl", hash = "sha256:1396e81b83516b9d5c9e26a924fa69164156c148c717131f54f586485ac3c15e"}, - {file = "pydantic_core-2.10.1-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6835451b57c1b467b95ffb03a38bb75b52fb4dc2762bb1d9dbed8de31ea7d0fc"}, - {file = "pydantic_core-2.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b00bc4619f60c853556b35f83731bd817f989cba3e97dc792bb8c97941b8053a"}, - {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa467fd300a6f046bdb248d40cd015b21b7576c168a6bb20aa22e595c8ffcdd"}, - {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d99277877daf2efe074eae6338453a4ed54a2d93fb4678ddfe1209a0c93a2468"}, - {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa7db7558607afeccb33c0e4bf1c9a9a835e26599e76af6fe2fcea45904083a6"}, - {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aad7bd686363d1ce4ee930ad39f14e1673248373f4a9d74d2b9554f06199fb58"}, - {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:443fed67d33aa85357464f297e3d26e570267d1af6fef1c21ca50921d2976302"}, - {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:042462d8d6ba707fd3ce9649e7bf268633a41018d6a998fb5fbacb7e928a183e"}, - {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ecdbde46235f3d560b18be0cb706c8e8ad1b965e5c13bbba7450c86064e96561"}, - {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ed550ed05540c03f0e69e6d74ad58d026de61b9eaebebbaaf8873e585cbb18de"}, - {file = "pydantic_core-2.10.1-cp38-none-win32.whl", hash = "sha256:8cdbbd92154db2fec4ec973d45c565e767ddc20aa6dbaf50142676484cbff8ee"}, - {file = "pydantic_core-2.10.1-cp38-none-win_amd64.whl", hash = "sha256:9f6f3e2598604956480f6c8aa24a3384dbf6509fe995d97f6ca6103bb8c2534e"}, - {file = "pydantic_core-2.10.1-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:655f8f4c8d6a5963c9a0687793da37b9b681d9ad06f29438a3b2326d4e6b7970"}, - {file = "pydantic_core-2.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e570ffeb2170e116a5b17e83f19911020ac79d19c96f320cbfa1fa96b470185b"}, - {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64322bfa13e44c6c30c518729ef08fda6026b96d5c0be724b3c4ae4da939f875"}, - {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:485a91abe3a07c3a8d1e082ba29254eea3e2bb13cbbd4351ea4e5a21912cc9b0"}, - {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7c2b8eb9fc872e68b46eeaf835e86bccc3a58ba57d0eedc109cbb14177be531"}, - {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5cb87bdc2e5f620693148b5f8f842d293cae46c5f15a1b1bf7ceeed324a740c"}, - {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25bd966103890ccfa028841a8f30cebcf5875eeac8c4bde4fe221364c92f0c9a"}, - {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f323306d0556351735b54acbf82904fe30a27b6a7147153cbe6e19aaaa2aa429"}, - {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0c27f38dc4fbf07b358b2bc90edf35e82d1703e22ff2efa4af4ad5de1b3833e7"}, - {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f1365e032a477c1430cfe0cf2856679529a2331426f8081172c4a74186f1d595"}, - {file = "pydantic_core-2.10.1-cp39-none-win32.whl", hash = "sha256:a1c311fd06ab3b10805abb72109f01a134019739bd3286b8ae1bc2fc4e50c07a"}, - {file = "pydantic_core-2.10.1-cp39-none-win_amd64.whl", hash = "sha256:ae8a8843b11dc0b03b57b52793e391f0122e740de3df1474814c700d2622950a"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d43002441932f9a9ea5d6f9efaa2e21458221a3a4b417a14027a1d530201ef1b"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fcb83175cc4936a5425dde3356f079ae03c0802bbdf8ff82c035f8a54b333521"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:962ed72424bf1f72334e2f1e61b68f16c0e596f024ca7ac5daf229f7c26e4208"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cf5bb4dd67f20f3bbc1209ef572a259027c49e5ff694fa56bed62959b41e1f9"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e544246b859f17373bed915182ab841b80849ed9cf23f1f07b73b7c58baee5fb"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c0877239307b7e69d025b73774e88e86ce82f6ba6adf98f41069d5b0b78bd1bf"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:53df009d1e1ba40f696f8995683e067e3967101d4bb4ea6f667931b7d4a01357"}, - {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a1254357f7e4c82e77c348dabf2d55f1d14d19d91ff025004775e70a6ef40ada"}, - {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:524ff0ca3baea164d6d93a32c58ac79eca9f6cf713586fdc0adb66a8cdeab96a"}, - {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f0ac9fb8608dbc6eaf17956bf623c9119b4db7dbb511650910a82e261e6600f"}, - {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:320f14bd4542a04ab23747ff2c8a778bde727158b606e2661349557f0770711e"}, - {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:63974d168b6233b4ed6a0046296803cb13c56637a7b8106564ab575926572a55"}, - {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:417243bf599ba1f1fef2bb8c543ceb918676954734e2dcb82bf162ae9d7bd514"}, - {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dda81e5ec82485155a19d9624cfcca9be88a405e2857354e5b089c2a982144b2"}, - {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:14cfbb00959259e15d684505263d5a21732b31248a5dd4941f73a3be233865b9"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:631cb7415225954fdcc2a024119101946793e5923f6c4d73a5914d27eb3d3a05"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:bec7dd208a4182e99c5b6c501ce0b1f49de2802448d4056091f8e630b28e9a52"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:149b8a07712f45b332faee1a2258d8ef1fb4a36f88c0c17cb687f205c5dc6e7d"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d966c47f9dd73c2d32a809d2be529112d509321c5310ebf54076812e6ecd884"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7eb037106f5c6b3b0b864ad226b0b7ab58157124161d48e4b30c4a43fef8bc4b"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:154ea7c52e32dce13065dbb20a4a6f0cc012b4f667ac90d648d36b12007fa9f7"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e562617a45b5a9da5be4abe72b971d4f00bf8555eb29bb91ec2ef2be348cd132"}, - {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f23b55eb5464468f9e0e9a9935ce3ed2a870608d5f534025cd5536bca25b1402"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:e9121b4009339b0f751955baf4543a0bfd6bc3f8188f8056b1a25a2d45099934"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:0523aeb76e03f753b58be33b26540880bac5aa54422e4462404c432230543f33"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e0e2959ef5d5b8dc9ef21e1a305a21a36e254e6a34432d00c72a92fdc5ecda5"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da01bec0a26befab4898ed83b362993c844b9a607a86add78604186297eb047e"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f2e9072d71c1f6cfc79a36d4484c82823c560e6f5599c43c1ca6b5cdbd54f881"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f36a3489d9e28fe4b67be9992a23029c3cec0babc3bd9afb39f49844a8c721c5"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f64f82cc3443149292b32387086d02a6c7fb39b8781563e0ca7b8d7d9cf72bd7"}, - {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b4a6db486ac8e99ae696e09efc8b2b9fea67b63c8f88ba7a1a16c24a057a0776"}, - {file = "pydantic_core-2.10.1.tar.gz", hash = "sha256:0f8682dbdd2f67f8e1edddcbffcc29f60a6182b4901c367fc8c1c40d30bb0a82"}, + {file = "pydantic_core-2.14.3-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:ba44fad1d114539d6a1509966b20b74d2dec9a5b0ee12dd7fd0a1bb7b8785e5f"}, + {file = "pydantic_core-2.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4a70d23eedd88a6484aa79a732a90e36701048a1509078d1b59578ef0ea2cdf5"}, + {file = "pydantic_core-2.14.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cc24728a1a9cef497697e53b3d085fb4d3bc0ef1ef4d9b424d9cf808f52c146"}, + {file = "pydantic_core-2.14.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ab4a2381005769a4af2ffddae74d769e8a4aae42e970596208ec6d615c6fb080"}, + {file = "pydantic_core-2.14.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a12bf088d6fa20e094f9a477bf84bd823651d8b8384f59bcd50eaa92e6a52"}, + {file = "pydantic_core-2.14.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:38aed5a1bbc3025859f56d6a32f6e53ca173283cb95348e03480f333b1091e7d"}, + {file = "pydantic_core-2.14.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1767bd3f6370458e60c1d3d7b1d9c2751cc1ad743434e8ec84625a610c8b9195"}, + {file = "pydantic_core-2.14.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7cb0c397f29688a5bd2c0dbd44451bc44ebb9b22babc90f97db5ec3e5bb69977"}, + {file = "pydantic_core-2.14.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9ff737f24b34ed26de62d481ef522f233d3c5927279f6b7229de9b0deb3f76b5"}, + {file = "pydantic_core-2.14.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a1a39fecb5f0b19faee9a8a8176c805ed78ce45d760259a4ff3d21a7daa4dfc1"}, + {file = "pydantic_core-2.14.3-cp310-none-win32.whl", hash = "sha256:ccbf355b7276593c68fa824030e68cb29f630c50e20cb11ebb0ee450ae6b3d08"}, + {file = "pydantic_core-2.14.3-cp310-none-win_amd64.whl", hash = "sha256:536e1f58419e1ec35f6d1310c88496f0d60e4f182cacb773d38076f66a60b149"}, + {file = "pydantic_core-2.14.3-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:f1f46700402312bdc31912f6fc17f5ecaaaa3bafe5487c48f07c800052736289"}, + {file = "pydantic_core-2.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:88ec906eb2d92420f5b074f59cf9e50b3bb44f3cb70e6512099fdd4d88c2f87c"}, + {file = "pydantic_core-2.14.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:056ea7cc3c92a7d2a14b5bc9c9fa14efa794d9f05b9794206d089d06d3433dc7"}, + {file = "pydantic_core-2.14.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:076edc972b68a66870cec41a4efdd72a6b655c4098a232314b02d2bfa3bfa157"}, + {file = "pydantic_core-2.14.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e71f666c3bf019f2490a47dddb44c3ccea2e69ac882f7495c68dc14d4065eac2"}, + {file = "pydantic_core-2.14.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f518eac285c9632be337323eef9824a856f2680f943a9b68ac41d5f5bad7df7c"}, + {file = "pydantic_core-2.14.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dbab442a8d9ca918b4ed99db8d89d11b1f067a7dadb642476ad0889560dac79"}, + {file = "pydantic_core-2.14.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0653fb9fc2fa6787f2fa08631314ab7fc8070307bd344bf9471d1b7207c24623"}, + {file = "pydantic_core-2.14.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c54af5069da58ea643ad34ff32fd6bc4eebb8ae0fef9821cd8919063e0aeeaab"}, + {file = "pydantic_core-2.14.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc956f78651778ec1ab105196e90e0e5f5275884793ab67c60938c75bcca3989"}, + {file = "pydantic_core-2.14.3-cp311-none-win32.whl", hash = "sha256:5b73441a1159f1fb37353aaefb9e801ab35a07dd93cb8177504b25a317f4215a"}, + {file = "pydantic_core-2.14.3-cp311-none-win_amd64.whl", hash = "sha256:7349f99f1ef8b940b309179733f2cad2e6037a29560f1b03fdc6aa6be0a8d03c"}, + {file = "pydantic_core-2.14.3-cp311-none-win_arm64.whl", hash = "sha256:ec79dbe23702795944d2ae4c6925e35a075b88acd0d20acde7c77a817ebbce94"}, + {file = "pydantic_core-2.14.3-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:8f5624f0f67f2b9ecaa812e1dfd2e35b256487566585160c6c19268bf2ffeccc"}, + {file = "pydantic_core-2.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6c2d118d1b6c9e2d577e215567eedbe11804c3aafa76d39ec1f8bc74e918fd07"}, + {file = "pydantic_core-2.14.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe863491664c6720d65ae438d4efaa5eca766565a53adb53bf14bc3246c72fe0"}, + {file = "pydantic_core-2.14.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:136bc7247e97a921a020abbd6ef3169af97569869cd6eff41b6a15a73c44ea9b"}, + {file = "pydantic_core-2.14.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aeafc7f5bbddc46213707266cadc94439bfa87ecf699444de8be044d6d6eb26f"}, + {file = "pydantic_core-2.14.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e16aaf788f1de5a85c8f8fcc9c1ca1dd7dd52b8ad30a7889ca31c7c7606615b8"}, + {file = "pydantic_core-2.14.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8fc652c354d3362e2932a79d5ac4bbd7170757a41a62c4fe0f057d29f10bebb"}, + {file = "pydantic_core-2.14.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f1b92e72babfd56585c75caf44f0b15258c58e6be23bc33f90885cebffde3400"}, + {file = "pydantic_core-2.14.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:75f3f534f33651b73f4d3a16d0254de096f43737d51e981478d580f4b006b427"}, + {file = "pydantic_core-2.14.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c9ffd823c46e05ef3eb28b821aa7bc501efa95ba8880b4a1380068e32c5bed47"}, + {file = "pydantic_core-2.14.3-cp312-none-win32.whl", hash = "sha256:12e05a76b223577a4696c76d7a6b36a0ccc491ffb3c6a8cf92d8001d93ddfd63"}, + {file = "pydantic_core-2.14.3-cp312-none-win_amd64.whl", hash = "sha256:1582f01eaf0537a696c846bea92082082b6bfc1103a88e777e983ea9fbdc2a0f"}, + {file = "pydantic_core-2.14.3-cp312-none-win_arm64.whl", hash = "sha256:96fb679c7ca12a512d36d01c174a4fbfd912b5535cc722eb2c010c7b44eceb8e"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:71ed769b58d44e0bc2701aa59eb199b6665c16e8a5b8b4a84db01f71580ec448"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:5402ee0f61e7798ea93a01b0489520f2abfd9b57b76b82c93714c4318c66ca06"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaab9dc009e22726c62fe3b850b797e7f0e7ba76d245284d1064081f512c7226"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92486a04d54987054f8b4405a9af9d482e5100d6fe6374fc3303015983fc8bda"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cf08b43d1d5d1678f295f0431a4a7e1707d4652576e1d0f8914b5e0213bfeee5"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8ca13480ce16daad0504be6ce893b0ee8ec34cd43b993b754198a89e2787f7e"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44afa3c18d45053fe8d8228950ee4c8eaf3b5a7f3b64963fdeac19b8342c987f"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56814b41486e2d712a8bc02a7b1f17b87fa30999d2323bbd13cf0e52296813a1"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c3dc2920cc96f9aa40c6dc54256e436cc95c0a15562eb7bd579e1811593c377e"}, + {file = "pydantic_core-2.14.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e483b8b913fcd3b48badec54185c150cb7ab0e6487914b84dc7cde2365e0c892"}, + {file = "pydantic_core-2.14.3-cp37-none-win32.whl", hash = "sha256:364dba61494e48f01ef50ae430e392f67ee1ee27e048daeda0e9d21c3ab2d609"}, + {file = "pydantic_core-2.14.3-cp37-none-win_amd64.whl", hash = "sha256:a402ae1066be594701ac45661278dc4a466fb684258d1a2c434de54971b006ca"}, + {file = "pydantic_core-2.14.3-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:10904368261e4509c091cbcc067e5a88b070ed9a10f7ad78f3029c175487490f"}, + {file = "pydantic_core-2.14.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:260692420028319e201b8649b13ac0988974eeafaaef95d0dfbf7120c38dc000"}, + {file = "pydantic_core-2.14.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c1bf1a7b05a65d3b37a9adea98e195e0081be6b17ca03a86f92aeb8b110f468"}, + {file = "pydantic_core-2.14.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d7abd17a838a52140e3aeca271054e321226f52df7e0a9f0da8f91ea123afe98"}, + {file = "pydantic_core-2.14.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5c51460ede609fbb4fa883a8fe16e749964ddb459966d0518991ec02eb8dfb9"}, + {file = "pydantic_core-2.14.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d06c78074646111fb01836585f1198367b17d57c9f427e07aaa9ff499003e58d"}, + {file = "pydantic_core-2.14.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af452e69446fadf247f18ac5d153b1f7e61ef708f23ce85d8c52833748c58075"}, + {file = "pydantic_core-2.14.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e3ad4968711fb379a67c8c755beb4dae8b721a83737737b7bcee27c05400b047"}, + {file = "pydantic_core-2.14.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c5ea0153482e5b4d601c25465771c7267c99fddf5d3f3bdc238ef930e6d051cf"}, + {file = "pydantic_core-2.14.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:96eb10ef8920990e703da348bb25fedb8b8653b5966e4e078e5be382b430f9e0"}, + {file = "pydantic_core-2.14.3-cp38-none-win32.whl", hash = "sha256:ea1498ce4491236d1cffa0eee9ad0968b6ecb0c1cd711699c5677fc689905f00"}, + {file = "pydantic_core-2.14.3-cp38-none-win_amd64.whl", hash = "sha256:2bc736725f9bd18a60eec0ed6ef9b06b9785454c8d0105f2be16e4d6274e63d0"}, + {file = "pydantic_core-2.14.3-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:1ea992659c03c3ea811d55fc0a997bec9dde863a617cc7b25cfde69ef32e55af"}, + {file = "pydantic_core-2.14.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d2b53e1f851a2b406bbb5ac58e16c4a5496038eddd856cc900278fa0da97f3fc"}, + {file = "pydantic_core-2.14.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c7f8e8a7cf8e81ca7d44bea4f181783630959d41b4b51d2f74bc50f348a090f"}, + {file = "pydantic_core-2.14.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8d3b9c91eeb372a64ec6686c1402afd40cc20f61a0866850f7d989b6bf39a41a"}, + {file = "pydantic_core-2.14.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ef3e2e407e4cad2df3c89488a761ed1f1c33f3b826a2ea9a411b0a7d1cccf1b"}, + {file = "pydantic_core-2.14.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f86f20a9d5bee1a6ede0f2757b917bac6908cde0f5ad9fcb3606db1e2968bcf5"}, + {file = "pydantic_core-2.14.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61beaa79d392d44dc19d6f11ccd824d3cccb865c4372157c40b92533f8d76dd0"}, + {file = "pydantic_core-2.14.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d41df8e10b094640a6b234851b624b76a41552f637b9fb34dc720b9fe4ef3be4"}, + {file = "pydantic_core-2.14.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c08ac60c3caa31f825b5dbac47e4875bd4954d8f559650ad9e0b225eaf8ed0c"}, + {file = "pydantic_core-2.14.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d8b3932f1a369364606417ded5412c4ffb15bedbcf797c31317e55bd5d920e"}, + {file = "pydantic_core-2.14.3-cp39-none-win32.whl", hash = "sha256:caa94726791e316f0f63049ee00dff3b34a629b0d099f3b594770f7d0d8f1f56"}, + {file = "pydantic_core-2.14.3-cp39-none-win_amd64.whl", hash = "sha256:2494d20e4c22beac30150b4be3b8339bf2a02ab5580fa6553ca274bc08681a65"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:fe272a72c7ed29f84c42fedd2d06c2f9858dc0c00dae3b34ba15d6d8ae0fbaaf"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7e63a56eb7fdee1587d62f753ccd6d5fa24fbeea57a40d9d8beaef679a24bdd6"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7692f539a26265cece1e27e366df5b976a6db6b1f825a9e0466395b314ee48b"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af46f0b7a1342b49f208fed31f5a83b8495bb14b652f621e0a6787d2f10f24ee"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6e2f9d76c00e805d47f19c7a96a14e4135238a7551a18bfd89bb757993fd0933"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:de52ddfa6e10e892d00f747bf7135d7007302ad82e243cf16d89dd77b03b649d"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:38113856c7fad8c19be7ddd57df0c3e77b1b2336459cb03ee3903ce9d5e236ce"}, + {file = "pydantic_core-2.14.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:354db020b1f8f11207b35360b92d95725621eb92656725c849a61e4b550f4acc"}, + {file = "pydantic_core-2.14.3-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:76fc18653a5c95e5301a52d1b5afb27c9adc77175bf00f73e94f501caf0e05ad"}, + {file = "pydantic_core-2.14.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2646f8270f932d79ba61102a15ea19a50ae0d43b314e22b3f8f4b5fabbfa6e38"}, + {file = "pydantic_core-2.14.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37dad73a2f82975ed563d6a277fd9b50e5d9c79910c4aec787e2d63547202315"}, + {file = "pydantic_core-2.14.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:113752a55a8eaece2e4ac96bc8817f134c2c23477e477d085ba89e3aa0f4dc44"}, + {file = "pydantic_core-2.14.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:8488e973547e8fb1b4193fd9faf5236cf1b7cd5e9e6dc7ff6b4d9afdc4c720cb"}, + {file = "pydantic_core-2.14.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3d1dde10bd9962b1434053239b1d5490fc31a2b02d8950a5f731bc584c7a5a0f"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:2c83892c7bf92b91d30faca53bb8ea21f9d7e39f0ae4008ef2c2f91116d0464a"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:849cff945284c577c5f621d2df76ca7b60f803cc8663ff01b778ad0af0e39bb9"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa89919fbd8a553cd7d03bf23d5bc5deee622e1b5db572121287f0e64979476"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf15145b1f8056d12c67255cd3ce5d317cd4450d5ee747760d8d088d85d12a2d"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4cc6bb11f4e8e5ed91d78b9880774fbc0856cb226151b0a93b549c2b26a00c19"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:832d16f248ca0cc96929139734ec32d21c67669dcf8a9f3f733c85054429c012"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b02b5e1f54c3396c48b665050464803c23c685716eb5d82a1d81bf81b5230da4"}, + {file = "pydantic_core-2.14.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:1f2d4516c32255782153e858f9a900ca6deadfb217fd3fb21bb2b60b4e04d04d"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0a3e51c2be472b7867eb0c5d025b91400c2b73a0823b89d4303a9097e2ec6655"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:df33902464410a1f1a0411a235f0a34e7e129f12cb6340daca0f9d1390f5fe10"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27828f0227b54804aac6fb077b6bb48e640b5435fdd7fbf0c274093a7b78b69c"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e2979dc80246e18e348de51246d4c9b410186ffa3c50e77924bec436b1e36cb"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b28996872b48baf829ee75fa06998b607c66a4847ac838e6fd7473a6b2ab68e7"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ca55c9671bb637ce13d18ef352fd32ae7aba21b4402f300a63f1fb1fd18e0364"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:aecd5ed096b0e5d93fb0367fd8f417cef38ea30b786f2501f6c34eabd9062c38"}, + {file = "pydantic_core-2.14.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:44aaf1a07ad0824e407dafc637a852e9a44d94664293bbe7d8ee549c356c8882"}, + {file = "pydantic_core-2.14.3.tar.gz", hash = "sha256:3ad083df8fe342d4d8d00cc1d3c1a23f0dc84fce416eb301e69f1ddbbe124d3f"}, ] [package.dependencies] @@ -2836,17 +2848,18 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pygments" -version = "2.16.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, - {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyparsing" @@ -2929,13 +2942,13 @@ pytest = ">=6.2" [[package]] name = "pytest-xdist" -version = "3.4.0" +version = "3.5.0" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-xdist-3.4.0.tar.gz", hash = "sha256:3a94a931dd9e268e0b871a877d09fe2efb6175c2c23d60d56a6001359002b832"}, - {file = "pytest_xdist-3.4.0-py3-none-any.whl", hash = "sha256:e513118bf787677a427e025606f55e95937565e06dfaac8d87f55301e57ae607"}, + {file = "pytest-xdist-3.5.0.tar.gz", hash = "sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a"}, + {file = "pytest_xdist-3.5.0-py3-none-any.whl", hash = "sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24"}, ] [package.dependencies] @@ -3213,13 +3226,13 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] [[package]] name = "rich" -version = "13.6.0" +version = "13.7.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.6.0-py3-none-any.whl", hash = "sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245"}, - {file = "rich-13.6.0.tar.gz", hash = "sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef"}, + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, ] [package.dependencies] @@ -3427,36 +3440,36 @@ tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc ( [[package]] name = "scipy" -version = "1.11.3" +version = "1.11.4" description = "Fundamental algorithms for scientific computing in Python" optional = false -python-versions = "<3.13,>=3.9" -files = [ - {file = "scipy-1.11.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:370f569c57e1d888304052c18e58f4a927338eafdaef78613c685ca2ea0d1fa0"}, - {file = "scipy-1.11.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9885e3e4f13b2bd44aaf2a1a6390a11add9f48d5295f7a592393ceb8991577a3"}, - {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e04aa19acc324a1a076abb4035dabe9b64badb19f76ad9c798bde39d41025cdc"}, - {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e1a8a4657673bfae1e05e1e1d6e94b0cabe5ed0c7c144c8aa7b7dbb774ce5c1"}, - {file = "scipy-1.11.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7abda0e62ef00cde826d441485e2e32fe737bdddee3324e35c0e01dee65e2a88"}, - {file = "scipy-1.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:033c3fd95d55012dd1148b201b72ae854d5086d25e7c316ec9850de4fe776929"}, - {file = "scipy-1.11.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:925c6f09d0053b1c0f90b2d92d03b261e889b20d1c9b08a3a51f61afc5f58165"}, - {file = "scipy-1.11.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5664e364f90be8219283eeb844323ff8cd79d7acbd64e15eb9c46b9bc7f6a42a"}, - {file = "scipy-1.11.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f325434b6424952fbb636506f0567898dca7b0f7654d48f1c382ea338ce9a3"}, - {file = "scipy-1.11.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f290cf561a4b4edfe8d1001ee4be6da60c1c4ea712985b58bf6bc62badee221"}, - {file = "scipy-1.11.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:91770cb3b1e81ae19463b3c235bf1e0e330767dca9eb4cd73ba3ded6c4151e4d"}, - {file = "scipy-1.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1f97cd89c0fe1a0685f8f89d85fa305deb3067d0668151571ba50913e445820"}, - {file = "scipy-1.11.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dfcc1552add7cb7c13fb70efcb2389d0624d571aaf2c80b04117e2755a0c5d15"}, - {file = "scipy-1.11.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0d3a136ae1ff0883fffbb1b05b0b2fea251cb1046a5077d0b435a1839b3e52b7"}, - {file = "scipy-1.11.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bae66a2d7d5768eaa33008fa5a974389f167183c87bf39160d3fefe6664f8ddc"}, - {file = "scipy-1.11.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2f6dee6cbb0e263b8142ed587bc93e3ed5e777f1f75448d24fb923d9fd4dce6"}, - {file = "scipy-1.11.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:74e89dc5e00201e71dd94f5f382ab1c6a9f3ff806c7d24e4e90928bb1aafb280"}, - {file = "scipy-1.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:90271dbde4be191522b3903fc97334e3956d7cfb9cce3f0718d0ab4fd7d8bfd6"}, - {file = "scipy-1.11.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a63d1ec9cadecce838467ce0631c17c15c7197ae61e49429434ba01d618caa83"}, - {file = "scipy-1.11.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:5305792c7110e32ff155aed0df46aa60a60fc6e52cd4ee02cdeb67eaccd5356e"}, - {file = "scipy-1.11.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea7f579182d83d00fed0e5c11a4aa5ffe01460444219dedc448a36adf0c3917"}, - {file = "scipy-1.11.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c77da50c9a91e23beb63c2a711ef9e9ca9a2060442757dffee34ea41847d8156"}, - {file = "scipy-1.11.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:15f237e890c24aef6891c7d008f9ff7e758c6ef39a2b5df264650eb7900403c0"}, - {file = "scipy-1.11.3-cp39-cp39-win_amd64.whl", hash = "sha256:4b4bb134c7aa457e26cc6ea482b016fef45db71417d55cc6d8f43d799cdf9ef2"}, - {file = "scipy-1.11.3.tar.gz", hash = "sha256:bba4d955f54edd61899776bad459bf7326e14b9fa1c552181f0479cc60a568cd"}, +python-versions = ">=3.9" +files = [ + {file = "scipy-1.11.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710"}, + {file = "scipy-1.11.4-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41"}, + {file = "scipy-1.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4"}, + {file = "scipy-1.11.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56"}, + {file = "scipy-1.11.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446"}, + {file = "scipy-1.11.4-cp310-cp310-win_amd64.whl", hash = "sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3"}, + {file = "scipy-1.11.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be"}, + {file = "scipy-1.11.4-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8"}, + {file = "scipy-1.11.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c"}, + {file = "scipy-1.11.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff"}, + {file = "scipy-1.11.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993"}, + {file = "scipy-1.11.4-cp311-cp311-win_amd64.whl", hash = "sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd"}, + {file = "scipy-1.11.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6"}, + {file = "scipy-1.11.4-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d"}, + {file = "scipy-1.11.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4"}, + {file = "scipy-1.11.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79"}, + {file = "scipy-1.11.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660"}, + {file = "scipy-1.11.4-cp312-cp312-win_amd64.whl", hash = "sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97"}, + {file = "scipy-1.11.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7"}, + {file = "scipy-1.11.4-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec"}, + {file = "scipy-1.11.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea"}, + {file = "scipy-1.11.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937"}, + {file = "scipy-1.11.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd"}, + {file = "scipy-1.11.4-cp39-cp39-win_amd64.whl", hash = "sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65"}, + {file = "scipy-1.11.4.tar.gz", hash = "sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa"}, ] [package.dependencies] @@ -3545,13 +3558,13 @@ files = [ [[package]] name = "sentry-sdk" -version = "1.34.0" +version = "1.36.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.34.0.tar.gz", hash = "sha256:e5d0d2b25931d88fa10986da59d941ac6037f742ab6ff2fce4143a27981d60c3"}, - {file = "sentry_sdk-1.34.0-py2.py3-none-any.whl", hash = "sha256:76dd087f38062ac6c1e30ed6feb533ee0037ff9e709974802db7b5dbf2e5db21"}, + {file = "sentry-sdk-1.36.0.tar.gz", hash = "sha256:f32dd16547f2f45e1c71a96fd4a48925e629541f7ddfe3d5d25ef7d5e94eb3c8"}, + {file = "sentry_sdk-1.36.0-py2.py3-none-any.whl", hash = "sha256:25d574f94fdf72199e331c2401fdac60d01b5be8f32822174c51c3ff0fc2f8cb"}, ] [package.dependencies] @@ -3590,17 +3603,17 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "68.2.2" +version = "69.0.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, - {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, + {file = "setuptools-69.0.2-py3-none-any.whl", hash = "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2"}, + {file = "setuptools-69.0.2.tar.gz", hash = "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] @@ -3934,22 +3947,22 @@ doc = ["reno", "sphinx", "tornado (>=4.5)"] [[package]] name = "tensorboard" -version = "2.14.1" +version = "2.15.1" description = "TensorBoard lets you watch Tensors Flow" optional = false python-versions = ">=3.9" files = [ - {file = "tensorboard-2.14.1-py3-none-any.whl", hash = "sha256:3db108fb58f023b6439880e177743c5f1e703e9eeb5fb7d597871f949f85fd58"}, + {file = "tensorboard-2.15.1-py3-none-any.whl", hash = "sha256:c46c1d1cf13a458c429868a78b2531d8ff5f682058d69ec0840b0bc7a38f1c0f"}, ] [package.dependencies] absl-py = ">=0.4" google-auth = ">=1.6.3,<3" -google-auth-oauthlib = ">=0.5,<1.1" +google-auth-oauthlib = ">=0.5,<2" grpcio = ">=1.48.2" markdown = ">=2.6.8" numpy = ">=1.12.0" -protobuf = ">=3.19.6" +protobuf = ">=3.19.6,<4.24" requests = ">=2.21.0,<3" setuptools = ">=41.0.0" six = ">1.9" @@ -3970,26 +3983,26 @@ files = [ [[package]] name = "tensorflow" -version = "2.14.0" +version = "2.15.0" description = "TensorFlow is an open source machine learning framework for everyone." optional = false python-versions = ">=3.9" files = [ - {file = "tensorflow-2.14.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:318b21b18312df6d11f511d0f205d55809d9ad0f46d5f9c13d8325ce4fe3b159"}, - {file = "tensorflow-2.14.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:927868c9bd4b3d2026ac77ec65352226a9f25e2d24ec3c7d088c68cff7583c9b"}, - {file = "tensorflow-2.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3870063433aebbd1b8da65ed4dcb09495f9239397f8cb5a8822025b6bb65e04"}, - {file = "tensorflow-2.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c9c1101269efcdb63492b45c8e83df0fc30c4454260a252d507dfeaebdf77ff"}, - {file = "tensorflow-2.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:0b7eaab5e034f1695dc968f7be52ce7ccae4621182d1e2bf6d5b3fab583be98c"}, - {file = "tensorflow-2.14.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:00c42e7d8280c660b10cf5d0b3164fdc5e38fd0bf16b3f9963b7cd0e546346d8"}, - {file = "tensorflow-2.14.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c92f5526c2029d31a036be06eb229c71f1c1821472876d34d0184d19908e318c"}, - {file = "tensorflow-2.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c224c076160ef9f60284e88f59df2bed347d55e64a0ca157f30f9ca57e8495b0"}, - {file = "tensorflow-2.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a80cabe6ab5f44280c05533e5b4a08e5b128f0d68d112564cffa3b96638e28aa"}, - {file = "tensorflow-2.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:0587ece626c4f7c4fcb2132525ea6c77ad2f2f5659a9b0f4451b1000be1b5e16"}, - {file = "tensorflow-2.14.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:6d65b54f6928490e2b6ff51836b97f88f5d5b29b5943fe81d8ac5d8c809ccca4"}, - {file = "tensorflow-2.14.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:e2840b549686080bfb824cc1578b5a15d5ec416badacc0c327d93f8762ee6b56"}, - {file = "tensorflow-2.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fb16641092b04a37ec2916c30412f986ca6adf969e6062057839efb788985f8"}, - {file = "tensorflow-2.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ba2ee1f9fe7f453bcd27d39a36928142de75a427ac2097dee2db1516387c9d5"}, - {file = "tensorflow-2.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:6531e76276b1421f43e008280107ba215256d4570cc56fd54856db7ff45e58f7"}, + {file = "tensorflow-2.15.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:9b248e0f4316b3a3c54cd1f83edfb7a761d473060c1972a8ea31a90d5de3aa72"}, + {file = "tensorflow-2.15.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:eaf420d8b8ec1d4bd75859be7d7545d8e7052726eed8456fdbba63718e7e07ea"}, + {file = "tensorflow-2.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e98aab454fc73ff1900314821e5bafbf20840ada2004c8caccf4d92e0e12a628"}, + {file = "tensorflow-2.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed601b43df9b7d9bed0203b34bcb9356efd4f671eaaac1046b7166a2afee0cf8"}, + {file = "tensorflow-2.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:2d88f8b71f4a8d9ab9dc7c8e42b14ca0f53d1daab0f989b8f2918907c2891f41"}, + {file = "tensorflow-2.15.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1e0716622ed7af867d8b1997b00a2940f1a1587dee923ff53efa2ee506992f32"}, + {file = "tensorflow-2.15.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:124930e7d4f5d74c61a5c80d642a26c22fe0c42fdd383fe9ee5803c3ac9ed4ce"}, + {file = "tensorflow-2.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:852efeb4d18beedac0120c4f2d4f4dccf4c090bb6740c5199d395ff609e85e98"}, + {file = "tensorflow-2.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dee8ec2b2c6c942ae65d25746e53cdc475e82d5fcbbb3009ce47f5963d69ebfc"}, + {file = "tensorflow-2.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:e05a48006930e4e9e68468e7affed3bbce8a1c7fe6df86500496ad1558804a78"}, + {file = "tensorflow-2.15.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:2cfcdde1ff3c01be617e99ce9783c49cb11da5796ce32a31855412bd092c0bcf"}, + {file = "tensorflow-2.15.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:896bda03f722700a9918d144aee5152a75f1be5e6c5045fd0683b8318a3fc9d9"}, + {file = "tensorflow-2.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7697b005ce48fec8b2ee8cf25bcbd138f16b5e17f99f7c01a6ea3f2429f86c6"}, + {file = "tensorflow-2.15.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fa865956d96b7614f247c36e4c22b1543ba5ce656fbe8e4f6266ae7a4917132"}, + {file = "tensorflow-2.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:01108746e1bbfcd48dfabf7f51ddca7693b91ea6821f6f62a27b5a5ebf0817c5"}, ] [package.dependencies] @@ -4000,33 +4013,33 @@ gast = ">=0.2.1,<0.5.0 || >0.5.0,<0.5.1 || >0.5.1,<0.5.2 || >0.5.2" google-pasta = ">=0.1.1" grpcio = ">=1.24.3,<2.0" h5py = ">=2.9.0" -keras = ">=2.14.0,<2.15" +keras = ">=2.15.0,<2.16" libclang = ">=13.0.0" -ml-dtypes = "0.2.0" -numpy = ">=1.23.5" +ml-dtypes = ">=0.2.0,<0.3.0" +numpy = ">=1.23.5,<2.0.0" opt-einsum = ">=2.3.2" packaging = "*" protobuf = ">=3.20.3,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" setuptools = "*" six = ">=1.12.0" -tensorboard = ">=2.14,<2.15" -tensorflow-estimator = ">=2.14.0,<2.15" +tensorboard = ">=2.15,<2.16" +tensorflow-estimator = ">=2.15.0,<2.16" tensorflow-io-gcs-filesystem = ">=0.23.1" termcolor = ">=1.1.0" typing-extensions = ">=3.6.6" wrapt = ">=1.11.0,<1.15" [package.extras] -and-cuda = ["nvidia-cublas-cu11 (==11.11.3.6)", "nvidia-cuda-cupti-cu11 (==11.8.87)", "nvidia-cuda-nvcc-cu11 (==11.8.89)", "nvidia-cuda-runtime-cu11 (==11.8.89)", "nvidia-cudnn-cu11 (==8.7.0.84)", "nvidia-cufft-cu11 (==10.9.0.58)", "nvidia-curand-cu11 (==10.3.0.86)", "nvidia-cusolver-cu11 (==11.4.1.48)", "nvidia-cusparse-cu11 (==11.7.5.86)", "nvidia-nccl-cu11 (==2.16.5)", "tensorrt (==8.5.3.1)"] +and-cuda = ["nvidia-cublas-cu12 (==12.2.5.6)", "nvidia-cuda-cupti-cu12 (==12.2.142)", "nvidia-cuda-nvcc-cu12 (==12.2.140)", "nvidia-cuda-nvrtc-cu12 (==12.2.140)", "nvidia-cuda-runtime-cu12 (==12.2.140)", "nvidia-cudnn-cu12 (==8.9.4.25)", "nvidia-cufft-cu12 (==11.0.8.103)", "nvidia-curand-cu12 (==10.3.3.141)", "nvidia-cusolver-cu12 (==11.5.2.141)", "nvidia-cusparse-cu12 (==12.1.2.141)", "nvidia-nccl-cu12 (==2.16.5)", "nvidia-nvjitlink-cu12 (==12.2.140)", "tensorrt (==8.6.1.post1)", "tensorrt-bindings (==8.6.1)", "tensorrt-libs (==8.6.1)"] [[package]] name = "tensorflow-estimator" -version = "2.14.0" +version = "2.15.0" description = "TensorFlow Estimator." optional = false python-versions = ">=3.7" files = [ - {file = "tensorflow_estimator-2.14.0-py2.py3-none-any.whl", hash = "sha256:820bf57c24aa631abb1bbe4371739ed77edb11361d61381fd8e790115ac0fd57"}, + {file = "tensorflow_estimator-2.15.0-py2.py3-none-any.whl", hash = "sha256:aedf21eec7fb2dc91150fc91a1ce12bc44dbb72278a08b58e79ff87c9e28f153"}, ] [[package]] @@ -4343,93 +4356,91 @@ files = [ [[package]] name = "torch" -version = "2.0.0" +version = "2.1.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false python-versions = ">=3.8.0" files = [ - {file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"}, - {file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"}, - {file = "torch-2.0.0-1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9"}, - {file = "torch-2.0.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203"}, - {file = "torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca"}, - {file = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4"}, - {file = "torch-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a"}, - {file = "torch-2.0.0-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:ce9b5a49bd513dff7950a5a07d6e26594dd51989cee05ba388b03e8e366fd5d5"}, - {file = "torch-2.0.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:53e1c33c6896583cdb9a583693e22e99266444c4a43392dddc562640d39e542b"}, - {file = "torch-2.0.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:09651bff72e439d004c991f15add0c397c66f98ab36fe60d5514b44e4da722e8"}, - {file = "torch-2.0.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d439aec349c98f12819e8564b8c54008e4613dd4428582af0e6e14c24ca85870"}, - {file = "torch-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2802f84f021907deee7e9470ed10c0e78af7457ac9a08a6cd7d55adef835fede"}, - {file = "torch-2.0.0-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:01858620f25f25e7a9ec4b547ff38e5e27c92d38ec4ccba9cfbfb31d7071ed9c"}, - {file = "torch-2.0.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:9a2e53b5783ef5896a6af338b36d782f28e83c8ddfc2ac44b67b066d9d76f498"}, - {file = "torch-2.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:ec5fff2447663e369682838ff0f82187b4d846057ef4d119a8dea7772a0b17dd"}, - {file = "torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:11b0384fe3c18c01b8fc5992e70fc519cde65e44c51cc87be1838c1803daf42f"}, - {file = "torch-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:e54846aa63855298cfb1195487f032e413e7ac9cbfa978fda32354cc39551475"}, - {file = "torch-2.0.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:cc788cbbbbc6eb4c90e52c550efd067586c2693092cf367c135b34893a64ae78"}, - {file = "torch-2.0.0-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:d292640f0fd72b7a31b2a6e3b635eb5065fcbedd4478f9cad1a1e7a9ec861d35"}, - {file = "torch-2.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6befaad784004b7af357e3d87fa0863c1f642866291f12a4c2af2de435e8ac5c"}, - {file = "torch-2.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a83b26bd6ae36fbf5fee3d56973d9816e2002e8a3b7d9205531167c28aaa38a7"}, - {file = "torch-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:c7e67195e1c3e33da53954b026e89a8e1ff3bc1aeb9eb32b677172d4a9b5dcbf"}, - {file = "torch-2.0.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6e0b97beb037a165669c312591f242382e9109a240e20054d5a5782d9236cad0"}, - {file = "torch-2.0.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:297a4919aff1c0f98a58ebe969200f71350a1d4d4f986dbfd60c02ffce780e99"}, + {file = "torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:5ebc43f5355a9b7be813392b3fb0133991f0380f6f0fcc8218d5468dc45d1071"}, + {file = "torch-2.1.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:84fefd63356416c0cd20578637ccdbb82164993400ed17b57c951dd6376dcee8"}, + {file = "torch-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:0a7a9da0c324409bcb5a7bdad1b4e94e936d21c2590aaa7ac2f63968da8c62f7"}, + {file = "torch-2.1.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:1e1e5faddd43a8f2c0e0e22beacd1e235a2e447794d807483c94a9e31b54a758"}, + {file = "torch-2.1.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:e76bf3c5c354874f1da465c852a2fb60ee6cbce306e935337885760f080f9baa"}, + {file = "torch-2.1.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:98fea993639b0bb432dfceb7b538f07c0f1c33386d63f635219f49254968c80f"}, + {file = "torch-2.1.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:61b51b33c61737c287058b0c3061e6a9d3c363863e4a094f804bc486888a188a"}, + {file = "torch-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:1d70920da827e2276bf07f7ec46958621cad18d228c97da8f9c19638474dbd52"}, + {file = "torch-2.1.1-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:a70593806f1d7e6b53657d96810518da0f88ef2608c98a402955765b8c79d52c"}, + {file = "torch-2.1.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:e312f7e82e49565f7667b0bbf9559ab0c597063d93044740781c02acd5a87978"}, + {file = "torch-2.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:1e3cbecfa5a7314d828f4a37b0c286714dc9aa2e69beb7a22f7aca76567ed9f4"}, + {file = "torch-2.1.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:9ca0fcbf3d5ba644d6a8572c83a9abbdf5f7ff575bc38529ef6c185a3a71bde9"}, + {file = "torch-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:2dc9f312fc1fa0d61a565a0292ad73119d4b74c9f8b5031b55f8b4722abca079"}, + {file = "torch-2.1.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:d56b032176458e2af4709627bbd2c20fe2917eff8cd087a7fe313acccf5ce2f1"}, + {file = "torch-2.1.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:29e3b90a8c281f6660804a939d1f4218604c80162e521e1e6d8c8557325902a0"}, + {file = "torch-2.1.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bd95cee8511584b67ddc0ba465c3f1edeb5708d833ee02af1206b4486f1d9096"}, + {file = "torch-2.1.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b31230bd058424e56dba7f899280dbc6ac8b9948e43902e0c84a44666b1ec151"}, + {file = "torch-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:403f1095e665e4f35971b43797a920725b8b205723aa68254a4050c6beca29b6"}, + {file = "torch-2.1.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:715b50d8c1de5da5524a68287eb000f73e026e74d5f6b12bc450ef6995fcf5f9"}, + {file = "torch-2.1.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:db67e8725c76f4c7f4f02e7551bb16e81ba1a1912867bc35d7bb96d2be8c78b4"}, ] [package.dependencies] filelock = "*" +fsspec = "*" jinja2 = "*" networkx = "*" -nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-cupti-cu11 = {version = "11.7.101", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cufft-cu11 = {version = "10.9.0.58", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-curand-cu11 = {version = "10.2.10.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusolver-cu11 = {version = "11.4.0.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusparse-cu11 = {version = "11.7.4.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nccl-cu11 = {version = "2.14.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nvtx-cu11 = {version = "11.7.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cublas-cu12 = {version = "12.1.3.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu12 = {version = "8.9.2.26", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu12 = {version = "11.0.2.54", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu12 = {version = "10.3.2.106", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu12 = {version = "11.4.5.107", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu12 = {version = "12.1.0.106", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu12 = {version = "2.18.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} sympy = "*" -triton = {version = "2.0.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +triton = {version = "2.1.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} typing-extensions = "*" [package.extras] +dynamo = ["jinja2"] opt-einsum = ["opt-einsum (>=3.3)"] [[package]] name = "torchvision" -version = "0.15.1" +version = "0.16.1" description = "image and video datasets and models for torch deep learning" optional = false python-versions = ">=3.8" files = [ - {file = "torchvision-0.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc10d48e9a60d006d0c1b48dea87f1ec9b63d856737d592f7c5c44cd87f3f4b7"}, - {file = "torchvision-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3708d3410fdcaf6280e358cda9de2a4ab06cc0b4c0fd9aeeac550ec2563a887e"}, - {file = "torchvision-0.15.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:d4de10c837f1493c1c54344388e300a06c96914c6cc55fcb2527c21f2f010bbd"}, - {file = "torchvision-0.15.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:b82fcc5abc9b5c96495c76596a1573025cc1e09d97d2d6fda717c44b9ca45881"}, - {file = "torchvision-0.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:c84e97d8cc4fe167d87adad0a2a6424cff90544365545b20669bc50e6ea46875"}, - {file = "torchvision-0.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:97b90eb3b7333a31d049c4ccfd1064361e8491874959d38f466af64d67418cef"}, - {file = "torchvision-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6b60e1c839ae2a071befbba69b17468d67feafdf576e90ff9645bfbee998de17"}, - {file = "torchvision-0.15.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:13f71a3372d9168b01481a754ebaa171207f3dc455bf2fd86906c69222443738"}, - {file = "torchvision-0.15.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b2e8394726009090b40f6cc3a95cc878cc011dfac3d8e7a6060c79213d360880"}, - {file = "torchvision-0.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:2852f501189483187ce9eb0ccd01b3f4f0918d29057e4a18b3cce8dad9a8a964"}, - {file = "torchvision-0.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e5861baaeea87d19b6fd7d131e11a4a6bd17be14234c490a259bb360775e9520"}, - {file = "torchvision-0.15.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e714f362b9d8217cf4d68509b679ebc9ddf128cfe80f6c1def8e3f8a18466e75"}, - {file = "torchvision-0.15.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:43624accad1e47f16824be4db37ad678dd89326ad90b69c9c6363eeb22b9467e"}, - {file = "torchvision-0.15.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7fe9b0cd3311b0db9e6d45ffab594ced06418fa4e2aa15eb2e60d55e5c51135c"}, - {file = "torchvision-0.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:b45324ea4911a23a4b00b5a15cdbe36d47f93137206dab9f8c606d81b69dd3a7"}, - {file = "torchvision-0.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1dfdec7c7df967330bba3341a781e0c047d4e0163e67164a9918500362bf7d91"}, - {file = "torchvision-0.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c153710186cec0338d4fff411459a57ddbc8504436123ca73b3f0bdc26ff918c"}, - {file = "torchvision-0.15.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:ff4e650aa601f32ab97bce06704868dd2baad69ca4d454fa1f0012a51199f2bc"}, - {file = "torchvision-0.15.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:e9b4bb2a15849391df0415d2f76dd36e6528e4253f7b69322b7a0d682535544b"}, - {file = "torchvision-0.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:21e6beb69e77ef6575c4fdd0ab332b96e8a7f144eee0d333acff469c827a4b5e"}, + {file = "torchvision-0.16.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:987132795e5c037cb74e7be35a693999fdb2f603152266ee15b80206e83a5b0c"}, + {file = "torchvision-0.16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:25da6a7b22ea0348f62c45ec0daf157731096babcae65d222404081af96e085c"}, + {file = "torchvision-0.16.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:c82e291e674a18b67f92ddb476ae18498fb46d7032ae914f3fda90c955e7d86f"}, + {file = "torchvision-0.16.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:632887b22e67ce32a3ede806b868bba4057601e46d680de14b32a391eac1b483"}, + {file = "torchvision-0.16.1-cp310-cp310-win_amd64.whl", hash = "sha256:92c76a5092b4033efdb183b11fa4854a7630e23c46f4a1c3ffd70c30cb5be4fc"}, + {file = "torchvision-0.16.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:4aea5cf491c6c21b1cbdbb1bf2a3838a59d4db93ad5f49019a6564d3ca7127c7"}, + {file = "torchvision-0.16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3391757167637ace3ef33a67c9d5ef86b1f8cbd93eaa5bad45eebcf266ea6089"}, + {file = "torchvision-0.16.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:4f9d5b192b336982e6dbe32c070b05606f0b53e87d722ae332a02909fbf988ed"}, + {file = "torchvision-0.16.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:3d34601614958c4e30f53ec0eb7bf3f282ee72bb747734be2d75422831a43384"}, + {file = "torchvision-0.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:e11af530585574eb5ca837b8f151bcdd57c10e35c3af56c76a10f3281d2a2f2c"}, + {file = "torchvision-0.16.1-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:4f2cad621fb96cf10e29af93e16c98b3226bdd53ae712b57e873c3deaf061617"}, + {file = "torchvision-0.16.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1d614b3c9e2de9cd75cc0e4e1923fcfbbcd9fdb9f08a0bbbbf7e135e4a0a1cfa"}, + {file = "torchvision-0.16.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:73271e930501a008fe24ba38945b2a75b25a6098f4c2f4402e39a9d0dd305ca6"}, + {file = "torchvision-0.16.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fab67ddc4809fcc2a04610b13cac5193b9d3be2896b77538bfdff401b13022e5"}, + {file = "torchvision-0.16.1-cp38-cp38-win_amd64.whl", hash = "sha256:13782d574033efec6646d1a2f5d85f4c59fcf3f403367bb407b15df07adc87e0"}, + {file = "torchvision-0.16.1-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:f14d201c37176dc4106eec76b229d6585a1505266b8cea99d3366fd38897b7c0"}, + {file = "torchvision-0.16.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a15e88a93a7501cc75b761a2dcd07aaedaaf9cbfaf48c8affa8c98989ecbb19d"}, + {file = "torchvision-0.16.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:72fde5fdb462e66ebe25ae42d2ee11434cbc395f74cad0d3b22cf60524345cc5"}, + {file = "torchvision-0.16.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:153f753f14eba58969cdc86360893a57f8bf63f8136c7d1cd4388108560b5446"}, + {file = "torchvision-0.16.1-cp39-cp39-win_amd64.whl", hash = "sha256:75e33b198b1265f61d822aa66d646ec3df67a712470ffec1e0c37ff46d4103c1"}, ] [package.dependencies] numpy = "*" pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" requests = "*" -torch = "2.0.0" +torch = "2.1.1" [package.extras] scipy = ["scipy"] @@ -4525,38 +4536,26 @@ vision = ["Pillow (<10.0.0)"] [[package]] name = "triton" -version = "2.0.0" +version = "2.1.0" description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "*" files = [ - {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, - {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, - {file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"}, - {file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"}, - {file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"}, - {file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"}, - {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"}, - {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"}, - {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"}, - {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"}, - {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"}, - {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"}, - {file = "triton-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be"}, - {file = "triton-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c"}, - {file = "triton-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42"}, - {file = "triton-2.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae"}, - {file = "triton-2.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb"}, - {file = "triton-2.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f"}, + {file = "triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:66439923a30d5d48399b08a9eae10370f6c261a5ec864a64983bae63152d39d7"}, + {file = "triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:919b06453f0033ea52c13eaf7833de0e57db3178d23d4e04f9fc71c4f2c32bf8"}, + {file = "triton-2.1.0-0-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ae4bb8a91de790e1866405211c4d618379781188f40d5c4c399766914e84cd94"}, + {file = "triton-2.1.0-0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39f6fb6bdccb3e98f3152e3fbea724f1aeae7d749412bbb1fa9c441d474eba26"}, + {file = "triton-2.1.0-0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21544e522c02005a626c8ad63d39bdff2f31d41069592919ef281e964ed26446"}, + {file = "triton-2.1.0-0-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:143582ca31dd89cd982bd3bf53666bab1c7527d41e185f9e3d8a3051ce1b663b"}, + {file = "triton-2.1.0-0-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:82fc5aeeedf6e36be4e4530cbdcba81a09d65c18e02f52dc298696d45721f3bd"}, + {file = "triton-2.1.0-0-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81a96d110a738ff63339fc892ded095b31bd0d205e3aace262af8400d40b6fa8"}, ] [package.dependencies] -cmake = "*" filelock = "*" -lit = "*" -torch = "*" [package.extras] +build = ["cmake (>=3.18)", "lit"] tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)"] tutorials = ["matplotlib", "pandas", "tabulate"] @@ -4620,18 +4619,17 @@ files = [ [[package]] name = "urllib3" -version = "2.0.7" +version = "2.1.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"}, - {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"}, + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, ] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -4904,85 +4902,101 @@ files = [ [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.3" description = "Yet another URL library" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:32435d134414e01d937cd9d6cc56e8413a8d4741dea36af5840c7750f04d16ab"}, + {file = "yarl-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9a5211de242754b5e612557bca701f39f8b1a9408dff73c6db623f22d20f470e"}, + {file = "yarl-1.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:525cd69eff44833b01f8ef39aa33a9cc53a99ff7f9d76a6ef6a9fb758f54d0ff"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc94441bcf9cb8c59f51f23193316afefbf3ff858460cb47b5758bf66a14d130"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e36021db54b8a0475805acc1d6c4bca5d9f52c3825ad29ae2d398a9d530ddb88"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0f17d1df951336a02afc8270c03c0c6e60d1f9996fcbd43a4ce6be81de0bd9d"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5f3faeb8100a43adf3e7925d556801d14b5816a0ac9e75e22948e787feec642"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aed37db837ecb5962469fad448aaae0f0ee94ffce2062cf2eb9aed13328b5196"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:721ee3fc292f0d069a04016ef2c3a25595d48c5b8ddc6029be46f6158d129c92"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b8bc5b87a65a4e64bc83385c05145ea901b613d0d3a434d434b55511b6ab0067"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:dd952b9c64f3b21aedd09b8fe958e4931864dba69926d8a90c90d36ac4e28c9a"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:c405d482c320a88ab53dcbd98d6d6f32ada074f2d965d6e9bf2d823158fa97de"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9df9a0d4c5624790a0dea2e02e3b1b3c69aed14bcb8650e19606d9df3719e87d"}, + {file = "yarl-1.9.3-cp310-cp310-win32.whl", hash = "sha256:d34c4f80956227f2686ddea5b3585e109c2733e2d4ef12eb1b8b4e84f09a2ab6"}, + {file = "yarl-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:cf7a4e8de7f1092829caef66fd90eaf3710bc5efd322a816d5677b7664893c93"}, + {file = "yarl-1.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d61a0ca95503867d4d627517bcfdc28a8468c3f1b0b06c626f30dd759d3999fd"}, + {file = "yarl-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73cc83f918b69110813a7d95024266072d987b903a623ecae673d1e71579d566"}, + {file = "yarl-1.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d81657b23e0edb84b37167e98aefb04ae16cbc5352770057893bd222cdc6e45f"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a1a8443091c7fbc17b84a0d9f38de34b8423b459fb853e6c8cdfab0eacf613"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe34befb8c765b8ce562f0200afda3578f8abb159c76de3ab354c80b72244c41"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c757f64afe53a422e45e3e399e1e3cf82b7a2f244796ce80d8ca53e16a49b9f"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72a57b41a0920b9a220125081c1e191b88a4cdec13bf9d0649e382a822705c65"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:632c7aeb99df718765adf58eacb9acb9cbc555e075da849c1378ef4d18bf536a"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b0b8c06afcf2bac5a50b37f64efbde978b7f9dc88842ce9729c020dc71fae4ce"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1d93461e2cf76c4796355494f15ffcb50a3c198cc2d601ad8d6a96219a10c363"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:4003f380dac50328c85e85416aca6985536812c082387255c35292cb4b41707e"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4d6d74a97e898c1c2df80339aa423234ad9ea2052f66366cef1e80448798c13d"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b61e64b06c3640feab73fa4ff9cb64bd8182de52e5dc13038e01cfe674ebc321"}, + {file = "yarl-1.9.3-cp311-cp311-win32.whl", hash = "sha256:29beac86f33d6c7ab1d79bd0213aa7aed2d2f555386856bb3056d5fdd9dab279"}, + {file = "yarl-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:f7271d6bd8838c49ba8ae647fc06469137e1c161a7ef97d778b72904d9b68696"}, + {file = "yarl-1.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:dd318e6b75ca80bff0b22b302f83a8ee41c62b8ac662ddb49f67ec97e799885d"}, + {file = "yarl-1.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c4b1efb11a8acd13246ffb0bee888dd0e8eb057f8bf30112e3e21e421eb82d4a"}, + {file = "yarl-1.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6f034386e5550b5dc8ded90b5e2ff7db21f0f5c7de37b6efc5dac046eb19c10"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd49a908cb6d387fc26acee8b7d9fcc9bbf8e1aca890c0b2fdfd706057546080"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa4643635f26052401750bd54db911b6342eb1a9ac3e74f0f8b58a25d61dfe41"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e741bd48e6a417bdfbae02e088f60018286d6c141639359fb8df017a3b69415a"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c86d0d0919952d05df880a1889a4f0aeb6868e98961c090e335671dea5c0361"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d5434b34100b504aabae75f0622ebb85defffe7b64ad8f52b8b30ec6ef6e4b9"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79e1df60f7c2b148722fb6cafebffe1acd95fd8b5fd77795f56247edaf326752"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:44e91a669c43f03964f672c5a234ae0d7a4d49c9b85d1baa93dec28afa28ffbd"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3cfa4dbe17b2e6fca1414e9c3bcc216f6930cb18ea7646e7d0d52792ac196808"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:88d2c3cc4b2f46d1ba73d81c51ec0e486f59cc51165ea4f789677f91a303a9a7"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cccdc02e46d2bd7cb5f38f8cc3d9db0d24951abd082b2f242c9e9f59c0ab2af3"}, + {file = "yarl-1.9.3-cp312-cp312-win32.whl", hash = "sha256:96758e56dceb8a70f8a5cff1e452daaeff07d1cc9f11e9b0c951330f0a2396a7"}, + {file = "yarl-1.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:c4472fe53ebf541113e533971bd8c32728debc4c6d8cc177f2bff31d011ec17e"}, + {file = "yarl-1.9.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:126638ab961633f0940a06e1c9d59919003ef212a15869708dcb7305f91a6732"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c99ddaddb2fbe04953b84d1651149a0d85214780e4d0ee824e610ab549d98d92"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dab30b21bd6fb17c3f4684868c7e6a9e8468078db00f599fb1c14e324b10fca"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:828235a2a169160ee73a2fcfb8a000709edf09d7511fccf203465c3d5acc59e4"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc391e3941045fd0987c77484b2799adffd08e4b6735c4ee5f054366a2e1551d"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51382c72dd5377861b573bd55dcf680df54cea84147c8648b15ac507fbef984d"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:28a108cb92ce6cf867690a962372996ca332d8cda0210c5ad487fe996e76b8bb"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8f18a7832ff85dfcd77871fe677b169b1bc60c021978c90c3bb14f727596e0ae"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:7eaf13af79950142ab2bbb8362f8d8d935be9aaf8df1df89c86c3231e4ff238a"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:66a6dbf6ca7d2db03cc61cafe1ee6be838ce0fbc97781881a22a58a7c5efef42"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1a0a4f3aaa18580038cfa52a7183c8ffbbe7d727fe581300817efc1e96d1b0e9"}, + {file = "yarl-1.9.3-cp37-cp37m-win32.whl", hash = "sha256:946db4511b2d815979d733ac6a961f47e20a29c297be0d55b6d4b77ee4b298f6"}, + {file = "yarl-1.9.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2dad8166d41ebd1f76ce107cf6a31e39801aee3844a54a90af23278b072f1ccf"}, + {file = "yarl-1.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bb72d2a94481e7dc7a0c522673db288f31849800d6ce2435317376a345728225"}, + {file = "yarl-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9a172c3d5447b7da1680a1a2d6ecdf6f87a319d21d52729f45ec938a7006d5d8"}, + {file = "yarl-1.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2dc72e891672343b99db6d497024bf8b985537ad6c393359dc5227ef653b2f17"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8d51817cf4b8d545963ec65ff06c1b92e5765aa98831678d0e2240b6e9fd281"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53ec65f7eee8655bebb1f6f1607760d123c3c115a324b443df4f916383482a67"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cfd77e8e5cafba3fb584e0f4b935a59216f352b73d4987be3af51f43a862c403"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e73db54c967eb75037c178a54445c5a4e7461b5203b27c45ef656a81787c0c1b"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09c19e5f4404574fcfb736efecf75844ffe8610606f3fccc35a1515b8b6712c4"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6280353940f7e5e2efaaabd686193e61351e966cc02f401761c4d87f48c89ea4"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c25ec06e4241e162f5d1f57c370f4078797ade95c9208bd0c60f484834f09c96"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:7217234b10c64b52cc39a8d82550342ae2e45be34f5bff02b890b8c452eb48d7"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4ce77d289f8d40905c054b63f29851ecbfd026ef4ba5c371a158cfe6f623663e"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5f74b015c99a5eac5ae589de27a1201418a5d9d460e89ccb3366015c6153e60a"}, + {file = "yarl-1.9.3-cp38-cp38-win32.whl", hash = "sha256:8a2538806be846ea25e90c28786136932ec385c7ff3bc1148e45125984783dc6"}, + {file = "yarl-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:6465d36381af057d0fab4e0f24ef0e80ba61f03fe43e6eeccbe0056e74aadc70"}, + {file = "yarl-1.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2f3c8822bc8fb4a347a192dd6a28a25d7f0ea3262e826d7d4ef9cc99cd06d07e"}, + {file = "yarl-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7831566595fe88ba17ea80e4b61c0eb599f84c85acaa14bf04dd90319a45b90"}, + {file = "yarl-1.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ff34cb09a332832d1cf38acd0f604c068665192c6107a439a92abfd8acf90fe2"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe8080b4f25dfc44a86bedd14bc4f9d469dfc6456e6f3c5d9077e81a5fedfba7"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8535e111a064f3bdd94c0ed443105934d6f005adad68dd13ce50a488a0ad1bf3"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d155a092bf0ebf4a9f6f3b7a650dc5d9a5bbb585ef83a52ed36ba46f55cc39d"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:778df71c8d0c8c9f1b378624b26431ca80041660d7be7c3f724b2c7a6e65d0d6"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f9cafaf031c34d95c1528c16b2fa07b710e6056b3c4e2e34e9317072da5d1a"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ca6b66f69e30f6e180d52f14d91ac854b8119553b524e0e28d5291a724f0f423"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0e7e83f31e23c5d00ff618045ddc5e916f9e613d33c5a5823bc0b0a0feb522f"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:af52725c7c39b0ee655befbbab5b9a1b209e01bb39128dce0db226a10014aacc"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0ab5baaea8450f4a3e241ef17e3d129b2143e38a685036b075976b9c415ea3eb"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6d350388ba1129bc867c6af1cd17da2b197dff0d2801036d2d7d83c2d771a682"}, + {file = "yarl-1.9.3-cp39-cp39-win32.whl", hash = "sha256:e2a16ef5fa2382af83bef4a18c1b3bcb4284c4732906aa69422cf09df9c59f1f"}, + {file = "yarl-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:d92d897cb4b4bf915fbeb5e604c7911021a8456f0964f3b8ebbe7f9188b9eabb"}, + {file = "yarl-1.9.3-py3-none-any.whl", hash = "sha256:271d63396460b6607b588555ea27a1a02b717ca2e3f2cf53bdde4013d7790929"}, + {file = "yarl-1.9.3.tar.gz", hash = "sha256:4a14907b597ec55740f63e52d7fee0e9ee09d5b9d57a4f399a7423268e457b57"}, ] [package.dependencies] @@ -4992,4 +5006,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "890b1c71e3345a75fefe7f38f92f96ab6e49fc6b677826afc0d184b294c4ad64" +content-hash = "d80378a687acc347c1b8de189705c207b880d665fe22c66eafa2b17f0d4e0df8" diff --git a/pyproject.toml b/pyproject.toml index 5bcbcc436..b63a471cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ ragas = "^0.0.19" coverage = "*" black = "*" portalocker = "*" -openai = "0.28" +openai = "*" [tool.black] line-length = 80 diff --git a/tests/test_bias.py b/tests/test_bias.py index 6f6642c85..2e61210a4 100644 --- a/tests/test_bias.py +++ b/tests/test_bias.py @@ -12,28 +12,7 @@ metric = UnBiasedMetric(evaluation_params=[LLMTestCaseParams.INPUT]) -def test_non_toxic(): - assert_test( - test_case=LLMTestCase( - input="placeholder", - actual_output=output, - ), - metrics=[metric], - ) - - -def test_non_toxic_metric(): +def test_non_bias_metric(): metric = UnBiasedMetric(evaluation_params=[LLMTestCaseParams.INPUT]) test_case = LLMTestCase(input="placeholder", actual_output=output) assert_test(test_case, [metric]) - - -def test_bias_score(): - metric = UnBiasedMetric( - evaluation_params=[ - LLMTestCaseParams.INPUT, - LLMTestCaseParams.ACTUAL_OUTPUT, - ] - ) - test_case = LLMTestCase(input="placeholder", actual_output=output) - assert_test(test_case, [metric]) diff --git a/tests/test_chatbot_example.py b/tests/test_chatbot_example.py deleted file mode 100644 index ee9d50d0c..000000000 --- a/tests/test_chatbot_example.py +++ /dev/null @@ -1,35 +0,0 @@ -import pytest -from deepeval.metrics import FactualConsistencyMetric -from deepeval.test_case import LLMTestCase -from deepeval.evaluator import assert_test - - -def test_1(): - input = "What does your company do?" - output = ( - "Specializes in cloud computing, data analytics, and machine learning." - ) - context = [ - "Our company specializes in cloud computing, data analytics, and machine learning. We offer a range of services including cloud storage solutions, data analytics platforms, and custom machine learning models." - ] - factual_consistency_metric = FactualConsistencyMetric(minimum_score=1.0) - test_case = LLMTestCase(input=input, actual_output=output, context=context) - with pytest.raises(AssertionError): - assert_test(test_case, [factual_consistency_metric]) - - -def test_2(): - input = "What does your company do?" - output = ( - "Specializes in cloud computing, data analytics, and machine learning." - ) - context = [ - "Our company specializes in cloud computing, data analytics, and machine learning. We offer a range of services including cloud storage solutions, data analytics platforms, and custom machine learning models." - ] - factual_consistency_metric_half = FactualConsistencyMetric( - minimum_score=0.5 - ) - if factual_consistency_metric_half.minimum_score != 0.5: - raise ValueError("Minimum score should be 0.5") - test_case = LLMTestCase(input=input, actual_output=output, context=context) - assert_test(test_case, [factual_consistency_metric_half]) diff --git a/tests/test_factual_consistency.py b/tests/test_factual_consistency.py deleted file mode 100644 index 6654ef6ed..000000000 --- a/tests/test_factual_consistency.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest -from deepeval.test_case import LLMTestCase -from deepeval.metrics import FactualConsistencyMetric - -from deepeval.evaluator import assert_test - - -def test_factual_consistency_metric(): - metric = FactualConsistencyMetric(minimum_score=0.8) - test_case = LLMTestCase( - input="placeholder", - actual_output="Python is a programming language.", - context=[ - "Python is a high-level, versatile, and interpreted programming language known for its simplicity and readability." - ], - ) - assert_test(test_case, [metric]) - - -def test_factual_consistency_metric_2(): - metric = FactualConsistencyMetric(minimum_score=0.6) - test_case = LLMTestCase( - input="placeholder", - actual_output="Python is a programming language.", - context=["Python is NOT a programming language."], - ) - with pytest.raises(AssertionError): - assert_test(test_case, [metric]) - - -def test_factual_consistency_metric_3(): - metric = FactualConsistencyMetric(minimum_score=0.6) - test_case = LLMTestCase( - input="placeholder", - actual_output="Python is a programming language.", - context=["Python is a snake."], - ) - with pytest.raises(AssertionError): - assert_test(test_case, [metric]) diff --git a/tests/test_hallucination_metric.py b/tests/test_hallucination_metric.py index 4a9af4346..c7e3ad6be 100644 --- a/tests/test_hallucination_metric.py +++ b/tests/test_hallucination_metric.py @@ -14,3 +14,25 @@ def test_hallucination_metric(): ], ) assert_test(test_case, [metric]) + + +def test_hallucination_metric_2(): + metric = HallucinationMetric(minimum_score=0.6) + test_case = LLMTestCase( + input="placeholder", + actual_output="Python is a programming language.", + context=["Python is NOT a programming language."], + ) + with pytest.raises(AssertionError): + assert_test(test_case, [metric]) + + +def test_hallucination_metric_3(): + metric = HallucinationMetric(minimum_score=0.6) + test_case = LLMTestCase( + input="placeholder", + actual_output="Python is a programming language.", + context=["Python is a snake."], + ) + with pytest.raises(AssertionError): + assert_test(test_case, [metric]) diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py index 91ed763f8..c08ebd93d 100644 --- a/tests/test_quickstart.py +++ b/tests/test_quickstart.py @@ -2,7 +2,7 @@ """ import pytest -from deepeval.metrics import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import assert_test @@ -19,7 +19,7 @@ def test_llm_output(): test_case = LLMTestCase( input=input, actual_output=generate_llm_output(input), context=context ) - assert_test(test_case, [FactualConsistencyMetric(minimum_score=0.5)]) + assert_test(test_case, [HallucinationMetric(minimum_score=0.5)]) def test_llm_output_custom(): @@ -29,7 +29,7 @@ def test_llm_output_custom(): input="Placerholder", actual_output=actual_output, context=context ) with pytest.raises(AssertionError): - assert_test(test_case, [FactualConsistencyMetric(minimum_score=0.5)]) + assert_test(test_case, [HallucinationMetric(minimum_score=0.5)]) def test_0(): @@ -44,6 +44,6 @@ def test_0(): expected_output=expected_output, context=context, ) - metric = FactualConsistencyMetric() + metric = HallucinationMetric() with pytest.raises(AssertionError): assert_test(test_case, metrics=[metric]) diff --git a/tests/test_ragas.py b/tests/test_ragas.py index 1260af34e..f0cbea2f8 100644 --- a/tests/test_ragas.py +++ b/tests/test_ragas.py @@ -1,6 +1,16 @@ import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics import RagasMetric +from deepeval.metrics import ( + RagasMetric, + ContextualRelevancyMetric, + FaithfulnessMetric, + ContextRecallMetric, + ConcisenessMetric, + CorrectnessMetric, + CoherenceMetric, + MaliciousnessMetric, +) +from deepeval.metrics.ragas_metric import AnswerRelevancyMetric from deepeval.evaluator import assert_test query = "Who won the FIFA World Cup in 2018?" @@ -23,6 +33,37 @@ def test_ragas_score(): with pytest.raises(AssertionError): assert_test( - test_cases=[test_case], + test_case=[test_case], metrics=[metric], ) + + +@pytest.mark.skip(reason="openai is expensive") +def test_everything(): + test_case = LLMTestCase( + input=query, + actual_output=output, + expected_output=expected_output, + context=context, + ) + metric1 = ContextualRelevancyMetric() + metric2 = FaithfulnessMetric() + metric3 = ContextRecallMetric() + metric4 = ConcisenessMetric() + metric5 = CorrectnessMetric() + metric6 = CoherenceMetric() + metric7 = MaliciousnessMetric() + metric8 = AnswerRelevancyMetric() + assert_test( + test_case, + [ + metric1, + metric2, + metric3, + metric4, + metric5, + metric6, + metric7, + metric8, + ], + ) From 7d775443245c944bfe67ff7efbf54e1ac3802827 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 03:50:00 -0800 Subject: [PATCH 20/31] remove print --- deepeval/metrics/ragas_metric.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/deepeval/metrics/ragas_metric.py b/deepeval/metrics/ragas_metric.py index a4eabee67..25a658a14 100644 --- a/deepeval/metrics/ragas_metric.py +++ b/deepeval/metrics/ragas_metric.py @@ -48,7 +48,6 @@ def measure(self, test_case: LLMTestCase): } dataset = Dataset.from_dict(data) - print("!!!!!!!!!!!!!!!!") # Evaluate the dataset using Ragas scores = evaluate(dataset, metrics=self.metrics) @@ -156,7 +155,6 @@ def measure(self, test_case: LLMTestCase): "id": [[test_case.id]], } dataset = Dataset.from_dict(data) - print("!!!!!!!!!!!!!!!!") scores = evaluate(dataset, metrics=self.metrics) faithfulness_score = scores["faithfulness"] self.success = faithfulness_score >= self.minimum_score From 5242d15cdecb1ca59d19c418f08d997bf03b209e Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 04:30:03 -0800 Subject: [PATCH 21/31] remove generate --- deepeval/cli/main.py | 5 ++--- deepeval/cli/test.py | 14 -------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/deepeval/cli/main.py b/deepeval/cli/main.py index 0b50ee3b7..f50ba676c 100644 --- a/deepeval/cli/main.py +++ b/deepeval/cli/main.py @@ -40,11 +40,10 @@ def login( print("API Key cannot be empty. Please try again.\n") KEY_FILE_HANDLER.write_api_key(api_key) client = Api(api_key=api_key) - print("Success! :raising_hands:") + print("Congratulations! Login successful :raising_hands: ") print( - "If you are new to DeepEval, try generate a sample test: [bold]deepeval test generate --output-file test_sample.py[/bold]" + "If you are new to DeepEval, follow our quickstart tutorial here: [bold][link=https://docs.confident-ai.com/docs/getting-started]https://docs.confident-ai.com/docs/getting-started[/link][/bold]" ) - print("Run a sample test: [bold]deepeval test run test_sample.py[/bold]") if __name__ == "__main__": diff --git a/deepeval/cli/test.py b/deepeval/cli/test.py index 239327c7f..6e8d54d79 100644 --- a/deepeval/cli/test.py +++ b/deepeval/cli/test.py @@ -81,17 +81,3 @@ def run( test_run_manager.wrap_up_test_run() return retcode - - -@app.command() -def generate(output_file: str = "test_sample.py"): - with open( - os.path.join(os.path.dirname(__file__), "../test_quickstart.py"), - "r", - ) as f_in: - with open(output_file, "w") as f_out: - f_out.write(f_in.read()) - print(f"✨ Done! Now run: [bold]deepeval test run {output_file}[/bold]") - print( - "You can generate more tests in the future in our documentation at https://docs.confident-ai.com/docs" - ) From 9fce468b9250286dad93d6d26025936b98270e48 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip <143328635+penguine-ip@users.noreply.github.com> Date: Wed, 22 Nov 2023 04:33:42 -0800 Subject: [PATCH 22/31] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d07f3f891..b7810b1b1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

- + Read The Docs    ·    @@ -67,7 +67,7 @@ Open `test_chatbot.py` and write your first test case using DeepEval: ```python import pytest -from deepeval.metrics.factual_consistency import FactualConsistencyMetric +from deepeval.metrics import HallucinationMetric from deepeval.test_case import LLMTestCase from deepeval.evaluator import assert_test @@ -77,9 +77,9 @@ def test_case(): # Replace this with the actual output from your LLM application actual_output = "We offer a 30-day full refund at no extra costs." - factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(minimum_score=0.7) test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) - assert_test(test_case, [factual_consistency_metric]) + assert_test(test_case, [hallucination_metric]) ``` Run `test_chatbot.py` in the CLI: @@ -91,10 +91,10 @@ deepeval test run test_chatbot.py **Your test should have passed ✅** Let's breakdown what happened. - The variable `input` mimics user input, and `actual_output` is a placeholder for your chatbot's intended output based on this query. -- The variable `context` contains the relevant information from your knowledge base, and `FactualConsistencyMetric(minimum_score=0.7)` is an out-of-the-box metric provided by DeepEval. It helps you evaluate the factual accuracy of your chatbot's output based on the provided context. +- The variable `context` contains the relevant information from your knowledge base, and `HallucinationMetric(minimum_score=0.7)` is an out-of-the-box metric provided by DeepEval. It helps you evaluate the factual accuracy of your chatbot's output based on the provided context. - The metric score ranges from 0 - 1. The `minimum_score=0.7` threshold ultimately determines whether your test has passed or not. -[Read our documentation](https://docs.confident-ai.com) for more information on how to use additional metrics, create your own custom metrics, and tutorials on how to integrate with other tools like LangChain and LlamaIndex. +[Read our documentation](https://docs.confident-ai.com/docs/getting-started) for more information on how to use additional metrics, create your own custom metrics, and tutorials on how to integrate with other tools like LangChain and LlamaIndex.
From 6df168ef080fdab0e366bde72296a49defdcf210 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 04:41:18 -0800 Subject: [PATCH 23/31] prepare for release --- deepeval/_version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepeval/_version.py b/deepeval/_version.py index 9d0d5a8eb..82c135c66 100644 --- a/deepeval/_version.py +++ b/deepeval/_version.py @@ -1 +1 @@ -__version__: str = "0.20.19" +__version__: str = "0.20.20" diff --git a/pyproject.toml b/pyproject.toml index b63a471cd..ffd9f81f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "deepeval" -version = "0.20.19" +version = "0.20.20" description = "The Evaluation Framework for LLMs" authors = ["Jeffrey Ip "] license = "Apache-2.0" From 3f5f04c0e2a81da7f0c7aaf650e88a72b11ab676 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 05:07:05 -0800 Subject: [PATCH 24/31] Emergency fix for langchain --- poetry.lock | 2 +- pyproject.toml | 1 + setup.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 73da65e40..99e13002c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5006,4 +5006,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "d80378a687acc347c1b8de189705c207b880d665fe22c66eafa2b17f0d4e0df8" +content-hash = "0f3d90a8bc9d21588b9ad7b1165a9ad000f2886a55841c50a37ccee29454622f" diff --git a/pyproject.toml b/pyproject.toml index ffd9f81f8..67a246e07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ coverage = "*" black = "*" portalocker = "*" openai = "*" +langchain = "*" [tool.black] line-length = 80 diff --git a/setup.py b/setup.py index df15c5459..05eb2727b 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ "sentry-sdk", "pytest-xdist", "portalocker", + "langchain", ], extras_require={ "bias": [ From 07d9b7ca505413e326d2740c31830b13bbec20ca Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 05:14:23 -0800 Subject: [PATCH 25/31] fix release --- deepeval/_version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepeval/_version.py b/deepeval/_version.py index 82c135c66..5923bebb1 100644 --- a/deepeval/_version.py +++ b/deepeval/_version.py @@ -1 +1 @@ -__version__: str = "0.20.20" +__version__: str = "0.20.21" diff --git a/pyproject.toml b/pyproject.toml index 67a246e07..032461404 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "deepeval" -version = "0.20.20" +version = "0.20.21" description = "The Evaluation Framework for LLMs" authors = ["Jeffrey Ip "] license = "Apache-2.0" From 9a9766fe43cb624fc71fd478b73a952d7e86876d Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 05:20:14 -0800 Subject: [PATCH 26/31] added setup dependency --- deepeval/_version.py | 2 +- pyproject.toml | 2 +- setup.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/deepeval/_version.py b/deepeval/_version.py index 5923bebb1..fef506f06 100644 --- a/deepeval/_version.py +++ b/deepeval/_version.py @@ -1 +1 @@ -__version__: str = "0.20.21" +__version__: str = "0.20.22" diff --git a/pyproject.toml b/pyproject.toml index 032461404..5857c0a15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "deepeval" -version = "0.20.21" +version = "0.20.22" description = "The Evaluation Framework for LLMs" authors = ["Jeffrey Ip "] license = "Apache-2.0" diff --git a/setup.py b/setup.py index 05eb2727b..27bb69876 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,8 @@ "pytest-xdist", "portalocker", "langchain", + "rouge_score==0.1.2", + "nltk==3.8.1" ], extras_require={ "bias": [ From 2e513a4b27612566c7ea563ccabd5cb04aa409d6 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 05:21:36 -0800 Subject: [PATCH 27/31] reformat --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 27bb69876..a817e5554 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "portalocker", "langchain", "rouge_score==0.1.2", - "nltk==3.8.1" + "nltk==3.8.1", ], extras_require={ "bias": [ From 29853d211cec6381f77eb3ec0f97da5bf5a5bfc1 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 05:30:29 -0800 Subject: [PATCH 28/31] updated docs --- docs/docs/evaluation-datasets.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/docs/evaluation-datasets.mdx b/docs/docs/evaluation-datasets.mdx index 9a093ef78..1a19ed023 100644 --- a/docs/docs/evaluation-datasets.mdx +++ b/docs/docs/evaluation-datasets.mdx @@ -70,11 +70,11 @@ deepeval login ```python title="test_bulk.py" from deepeval.test_case import LLMTestCase -from deepeval.metrics import HallucinationMetric -from deepeval.metric.answer_relevancy import AnswerRelevancyMetric +from deepeval.metrics import HallucinationMetric, AnswerRelevancyMetric from deepeval.evaluate import assert_test +from deepeval.dataset import EvaluationDataset -dataset = [...] +dataset = EvaluationDataset(test_cases=[...]) @pytest.mark.parametrize( "test_case", @@ -98,10 +98,10 @@ Alternately, you can use deepeval's `evaluate` function to evaluate datasets. Th ```python from deepeval.evaluator import evaluate -from deepeval.metrics import HallucinationMetric -from deepeval.metric.answer_relevancy import AnswerRelevancyMetric +from deepeval.metrics import HallucinationMetric, AnswerRelevancyMetric +from deepeval.dataset import EvaluationDataset -dataset = [...] +dataset = EvaluationDataset(test_cases=[...]) hallucination_metric = HallucinationMetric(minimum_score=0.3) answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) From 88e6c734eef70ca72680ce108f374b41c98f4399 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 05:35:57 -0800 Subject: [PATCH 29/31] Fix docs --- docs/docs/evaluation-datasets.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/docs/evaluation-datasets.mdx b/docs/docs/evaluation-datasets.mdx index 1a19ed023..39201249c 100644 --- a/docs/docs/evaluation-datasets.mdx +++ b/docs/docs/evaluation-datasets.mdx @@ -69,9 +69,9 @@ deepeval login `deepeval` utilizes the `@pytest.mark.parametrize` decorator to loop through entire datasets. ```python title="test_bulk.py" +from deepeval.evaluator import assert_test from deepeval.test_case import LLMTestCase from deepeval.metrics import HallucinationMetric, AnswerRelevancyMetric -from deepeval.evaluate import assert_test from deepeval.dataset import EvaluationDataset dataset = EvaluationDataset(test_cases=[...]) @@ -105,5 +105,9 @@ dataset = EvaluationDataset(test_cases=[...]) hallucination_metric = HallucinationMetric(minimum_score=0.3) answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) + +dataset.evaluate([hallucination_metric, answer_relevancy_metric]) + +# You can also call the evaluate() function directly evaluate(dataset, [hallucination_metric, answer_relevancy_metric]) ``` From 60abb5ae56e98b359c4d546dde597cdcdc3e9b70 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 06:28:10 -0800 Subject: [PATCH 30/31] Update dataset docs --- docs/docs/evaluation-datasets.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/evaluation-datasets.mdx b/docs/docs/evaluation-datasets.mdx index 39201249c..f49e84847 100644 --- a/docs/docs/evaluation-datasets.mdx +++ b/docs/docs/evaluation-datasets.mdx @@ -42,7 +42,7 @@ original_dataset = [ }, ] -dataset = [] +dataset = EvaluationDataset() for datapoint in original_dataset: input = datapoint.get("input", None) actual_output = datapoint.get("actual_output", None) @@ -55,7 +55,7 @@ for datapoint in original_dataset: expected_output=expected_output, context=context ) - dataset.append(test_case) + dataset.add_test_case(test_case) ``` ## Evaluate Your Dataset With Pytest From 1ca8043721a74eb1f0054fffd3f4127ce0c112c2 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 22 Nov 2023 06:28:38 -0800 Subject: [PATCH 31/31] Update dataset docs --- docs/docs/evaluation-datasets.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/evaluation-datasets.mdx b/docs/docs/evaluation-datasets.mdx index f49e84847..07eae2dfa 100644 --- a/docs/docs/evaluation-datasets.mdx +++ b/docs/docs/evaluation-datasets.mdx @@ -17,6 +17,7 @@ Your original dataset can be in any format, such as CSV or JSON. Let's take JSON ```python from deepeval.test_case import LLMTestCase +from deepeval.dataset import EvaluationDataset original_dataset = [ {