diff --git a/.DS_Store b/.DS_Store index 03d385f22..b4ec85712 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/README.md b/README.md index caaab076b..62fcfc68a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@

-**DeepEval** is a simple-to-use, open-source evaluation framework for LLM applications. It is similar to Pytest but specialized for unit testing LLM applications. DeepEval evaluates performance based on metrics such as hallucination, answer relevancy, RAGAS, etc., using LLMs and various other NLP models **locally on your machine**. +**DeepEval** is a simple-to-use, open-source LLM evaluation framework for LLM applications. It is similar to Pytest but specialized for unit testing LLM applications. DeepEval evaluates performance based on metrics such as hallucination, answer relevancy, RAGAS, etc., using LLMs and various other NLP models that runs **locally on your machine**. Whether your application is implemented via RAG or fine-tuning, LangChain or LlamaIndex, DeepEval has you covered. With it, you can easily determine the optimal hyperparameters to improve your RAG pipeline, prevent prompt drifting, or even transition from OpenAI to hosting your own Llama2 with confidence. @@ -26,16 +26,20 @@ Whether your application is implemented via RAG or fine-tuning, LangChain or Lla # Features -- Large variety of ready-to-use evaluation metrics powered by LLMs, statistical methods, or NLP models that runs **locally on your machine**: +- Large variety of ready-to-use LLM evaluation metrics powered by LLMs (all with explanations), statistical methods, or NLP models that runs **locally on your machine**: - Hallucination + - Summarization - Answer Relevancy + - Faithfulness + - Contextual Recall + - Contextual Precision - RAGAS - G-Eval - Toxicity - Bias - etc. +- Evaluate your entire dataset in bulk in under 20 lines of Python code **in parallel**. Do this via the CLI in a Pytest-like manner, or through our `evaluate()` function. - Easily create your own custom metrics that are automatically integrated with DeepEval's ecosystem by inheriting DeepEval's base metric class. -- Evaluate your entire dataset in bulk using fewer than 20 lines of Python code **in parallel**. - [Automatically integrated with Confident AI](https://app.confident-ai.com) for continous evaluation throughout the lifetime of your LLM (app): - log evaluation results and analyze metrics pass / fails - compare and pick the optimal hyperparameters (eg. prompt templates, chunk size, models used, etc.) based on evaluation results @@ -90,7 +94,7 @@ def test_case(): # Replace this with the actual output from your LLM application actual_output = "We offer a 30-day full refund at no extra costs." - hallucination_metric = HallucinationMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(threshold=0.7) test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) assert_test(test_case, [hallucination_metric]) ``` @@ -104,13 +108,36 @@ deepeval test run test_chatbot.py **Your test should have passed ✅** Let's breakdown what happened. - The variable `input` mimics user input, and `actual_output` is a placeholder for your chatbot's intended output based on this query. -- The variable `context` contains the relevant information from your knowledge base, and `HallucinationMetric(minimum_score=0.7)` is an out-of-the-box metric provided by DeepEval. It helps you evaluate the factual accuracy of your chatbot's output based on the provided context. -- The metric score ranges from 0 - 1. The `minimum_score=0.7` threshold ultimately determines whether your test has passed or not. +- The variable `context` contains the relevant information from your knowledge base, and `HallucinationMetric(threshold=0.7)` is an out-of-the-box metric provided by DeepEval. It helps you evaluate the factual accuracy of your chatbot's output based on the provided context. +- The metric score ranges from 0 - 1. The `threshold=0.7` threshold ultimately determines whether your test has passed or not. [Read our documentation](https://docs.confident-ai.com/docs/getting-started) for more information on how to use additional metrics, create your own custom metrics, and tutorials on how to integrate with other tools like LangChain and LlamaIndex.
+## Evaluating Without Pytest Integration + +Alternatively, you can evaluate without Pytest, which is more suited for a notebook environment. + +```python +from deepeval import evaluate +from deepeval.metrics import HallucinationMetric +from deepeval.test_case import LLMTestCase + +input = "What if these shoes don't fit?" +context = ["All customers are eligible for a 30 day full refund at no extra costs."] +# Replace this with the actual output from your LLM application +actual_output = "We offer a 30-day full refund at no extra costs." + +hallucination_metric = HallucinationMetric(threshold=0.7) +test_case = LLMTestCase( + input=input, + actual_output=actual_output, + context=context +) +evaluate([test_case], [hallucination_metric]) +``` + ## Evaluting a Dataset / Test Cases in Bulk In DeepEval, a dataset is simply a collection of test cases. Here is how you can evaluate things in bulk: @@ -132,8 +159,8 @@ dataset = EvaluationDataset(test_cases=[first_test_case, second_test_case]) dataset, ) def test_customer_chatbot(test_case: LLMTestCase): - hallucination_metric = HallucinationMetric(minimum_score=0.3) - answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) + hallucination_metric = HallucinationMetric(threshold=0.3) + answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.5) assert_test(test_case, [hallucination_metric, answer_relevancy_metric]) ``` @@ -144,7 +171,7 @@ deepeval test run test_.py -n 4
-Alternatively, although we recommend using `deepeval test run`, you can evaluate a dataset/test cases without using pytest: +Alternatively, although we recommend using `deepeval test run`, you can evaluate a dataset/test cases without using our Pytest integration: ```python from deepeval import evaluate @@ -164,6 +191,7 @@ We offer a [free web platform](https://app.confident-ai.com) for you to: 3. Compare and pick the optimal hyperparameteres (prompt templates, models, chunk size, etc.). 4. Create, manage, and centralize your evaluation datasets. 5. Track events in production and augment your evaluation dataset for continous evaluation in production. +6. Track events in production and view live evaluation results over time. Everything on Confident AI, including how to use Confident is available [here](https://docs.confident-ai.com/docs/confident-ai-introduction). diff --git a/deepeval/__init__.py b/deepeval/__init__.py index 099bbdfb8..6d6c39dc6 100644 --- a/deepeval/__init__.py +++ b/deepeval/__init__.py @@ -2,13 +2,13 @@ import re # Optionally add telemtry -from .telemetry import * from ._version import __version__ from .decorators.hyperparameters import set_hyperparameters from deepeval.event import track from deepeval.evaluate import evaluate, run_test, assert_test from deepeval.test_run import on_test_run_end +from deepeval.telemetry import * __all__ = [ "set_hyperparameters", diff --git a/deepeval/_version.py b/deepeval/_version.py index 448d0ed8b..2c7d1d32f 100644 --- a/deepeval/_version.py +++ b/deepeval/_version.py @@ -1 +1 @@ -__version__: str = "0.20.42" +__version__: str = "0.20.46" diff --git a/deepeval/api.py b/deepeval/api.py index 719516786..734a5ee5f 100644 --- a/deepeval/api.py +++ b/deepeval/api.py @@ -4,9 +4,13 @@ import requests import warnings from requests.adapters import HTTPAdapter, Response, Retry +import aiohttp +from aiohttp import ClientSession +from requests.adapters import HTTPAdapter +from enum import Enum + from deepeval.constants import API_KEY_ENV from deepeval.key_handler import KEY_FILE_HANDLER, KeyValues -from enum import Enum API_BASE_URL = "https://app.confident-ai.com/api" @@ -259,3 +263,77 @@ def quote_string(text: str) -> str: str: Quoted text in return """ return urllib.parse.quote(text, safe="") + + async def _api_request_async( + self, + method, + endpoint, + headers=None, + auth=None, + params=None, + body=None, + files=None, + data=None, + ): + """Generic asynchronous HTTP request method with error handling.""" + url = f"{self.base_api_url}/{endpoint}" + async with ClientSession() as session: + try: + # Preparing the request body for file uploads if files are present + if files: + data = aiohttp.FormData() + for file_name, file_content in files.items(): + data.add_field( + file_name, file_content, filename=file_name + ) + + # Sending the request + res = await session.request( + method=method, + url=url, + headers=headers, + params=params, + json=body, + ) + + # Check response status + if res.status == 200: + try: + json = await res.json() + return json + except ValueError: + return await res.text() + else: + # Determine how to process the response based on Content-Type + content_type = res.headers.get("Content-Type", "") + if "application/json" in content_type: + error_response = await res.json() + else: + error_response = await res.text() + + # Specifically handle status code 400 + if res.status == 400: + print(f"Error 400: Bad Request - {error_response}") + + print(f"Error {res.status}: {error_response}") + return None + + except Exception as err: + raise Exception(f"HTTP request failed: {err}") from err + + async def post_request_async( + self, endpoint, body=None, files=None, data=None + ): + """Generic asynchronous POST Request Wrapper""" + print("hi") + return await self._api_request_async( + "POST", + endpoint, + headers=self._headers + if files is None + else self._headers_multipart_form_data, + auth=self._auth, + body=body, + files=files, + data=data, + ) diff --git a/deepeval/chat_completion/retry.py b/deepeval/chat_completion/retry.py index 501bd5aa0..1df7b2be0 100644 --- a/deepeval/chat_completion/retry.py +++ b/deepeval/chat_completion/retry.py @@ -1,19 +1,47 @@ -from typing import Callable, Any +import random import time +import openai -def call_openai_with_retry( - callable: Callable[[], Any], max_retries: int = 2 -) -> Any: - for _ in range(max_retries): - try: - response = callable() - return response - except Exception as e: - print(f"An error occurred: {e}. Retrying...") - time.sleep(2) - continue - - raise Exception( - "Max retries reached. Unable to make a successful API call to OpenAI." - ) +def retry_with_exponential_backoff( + func, + initial_delay: float = 1, + exponential_base: float = 2, + jitter: bool = True, + max_retries: int = 10, + errors: tuple = (openai.RateLimitError,), +): + """Retry a function with exponential backoff.""" + + def wrapper(*args, **kwargs): + # Initialize variables + num_retries = 0 + delay = initial_delay + + # Loop until a successful response or max_retries is hit or an exception is raised + while True: + try: + return func(*args, **kwargs) + + # Retry on specified errors + except errors as e: + # Increment retries + num_retries += 1 + + # Check if max retries has been reached + if num_retries > max_retries: + raise Exception( + f"Maximum number of retries ({max_retries}) exceeded." + ) + + # Increment the delay + delay *= exponential_base * (1 + jitter * random.random()) + + # Sleep for the delay + time.sleep(delay) + + # Raise exceptions for any errors not specified + except Exception as e: + raise e + + return wrapper diff --git a/deepeval/cli/azure_openai.py b/deepeval/cli/azure_openai.py deleted file mode 100644 index 70b20c8e8..000000000 --- a/deepeval/cli/azure_openai.py +++ /dev/null @@ -1,66 +0,0 @@ -import os -import typer -from typing import Optional - -try: - from rich import print -except Exception as e: - pass -from deepeval.key_handler import KEY_FILE_HANDLER, KeyValues - -app = typer.Typer(name="azure-openai") - - -@app.command(name="set") -def set_azure_openai_env( - azure_openai_api_key: str = typer.Option( - ..., "--openai-api-key", help="Azure OpenAI API key" - ), - azure_openai_endpoint: str = typer.Option( - ..., "--openai-endpoint", help="Azure OpenAI endpoint" - ), - openai_api_version: str = typer.Option( - ..., "--openai-api-version", help="OpenAI API version" - ), - azure_deployment_name: str = typer.Option( - ..., "--deployment-name", help="Azure deployment name" - ), - azure_model_version: Optional[str] = typer.Option( - None, "--model-version", help="Azure model version (optional)" - ), -): - KEY_FILE_HANDLER.write_key( - KeyValues.AZURE_OPENAI_API_KEY, azure_openai_api_key - ) - KEY_FILE_HANDLER.write_key( - KeyValues.AZURE_OPENAI_ENDPOINT, azure_openai_endpoint - ) - KEY_FILE_HANDLER.write_key(KeyValues.OPENAI_API_VERSION, openai_api_version) - KEY_FILE_HANDLER.write_key( - KeyValues.AZURE_DEPLOYMENT_NAME, azure_deployment_name - ) - - if azure_model_version is not None: - KEY_FILE_HANDLER.write_key( - KeyValues.AZURE_MODEL_VERSION, azure_model_version - ) - - KEY_FILE_HANDLER.write_key(KeyValues.USE_AZURE_OPENAI, "YES") - - print( - ":raising_hands: Congratulations! You're now using Azure OpenAI for all evals that require an LLM." - ) - - -@app.command(name="unset") -def unset_azure_openai_env(): - KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_OPENAI_API_KEY) - KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_OPENAI_ENDPOINT) - KEY_FILE_HANDLER.remove_key(KeyValues.OPENAI_API_VERSION) - KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_DEPLOYMENT_NAME) - KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_MODEL_VERSION) - KEY_FILE_HANDLER.remove_key(KeyValues.USE_AZURE_OPENAI) - - print( - ":raising_hands: Congratulations! You're now using regular OpenAI for all evals that require an LLM." - ) diff --git a/deepeval/cli/main.py b/deepeval/cli/main.py index b68f8af07..78596596b 100644 --- a/deepeval/cli/main.py +++ b/deepeval/cli/main.py @@ -8,14 +8,12 @@ pass from deepeval.key_handler import KEY_FILE_HANDLER, KeyValues from deepeval.cli.test import app as test_app -from deepeval.cli.azure_openai import app as azure_openai_app from typing import Optional import webbrowser app = typer.Typer(name="deepeval") app.add_typer(test_app, name="test") -app.add_typer(azure_openai_app, name="azure-openai") @app.command() @@ -58,5 +56,60 @@ def login( ) +@app.command(name="set-azure-openai") +def set_azure_openai_env( + azure_openai_api_key: str = typer.Option( + ..., "--openai-api-key", help="Azure OpenAI API key" + ), + azure_openai_endpoint: str = typer.Option( + ..., "--openai-endpoint", help="Azure OpenAI endpoint" + ), + openai_api_version: str = typer.Option( + ..., "--openai-api-version", help="OpenAI API version" + ), + azure_deployment_name: str = typer.Option( + ..., "--deployment-name", help="Azure deployment name" + ), + azure_model_version: Optional[str] = typer.Option( + None, "--model-version", help="Azure model version (optional)" + ), +): + KEY_FILE_HANDLER.write_key( + KeyValues.AZURE_OPENAI_API_KEY, azure_openai_api_key + ) + KEY_FILE_HANDLER.write_key( + KeyValues.AZURE_OPENAI_ENDPOINT, azure_openai_endpoint + ) + KEY_FILE_HANDLER.write_key(KeyValues.OPENAI_API_VERSION, openai_api_version) + KEY_FILE_HANDLER.write_key( + KeyValues.AZURE_DEPLOYMENT_NAME, azure_deployment_name + ) + + if azure_model_version is not None: + KEY_FILE_HANDLER.write_key( + KeyValues.AZURE_MODEL_VERSION, azure_model_version + ) + + KEY_FILE_HANDLER.write_key(KeyValues.USE_AZURE_OPENAI, "YES") + + print( + ":raising_hands: Congratulations! You're now using Azure OpenAI for all evals that require an LLM." + ) + + +@app.command(name="unset-azure-openai") +def unset_azure_openai_env(): + KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_OPENAI_API_KEY) + KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_OPENAI_ENDPOINT) + KEY_FILE_HANDLER.remove_key(KeyValues.OPENAI_API_VERSION) + KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_DEPLOYMENT_NAME) + KEY_FILE_HANDLER.remove_key(KeyValues.AZURE_MODEL_VERSION) + KEY_FILE_HANDLER.remove_key(KeyValues.USE_AZURE_OPENAI) + + print( + ":raising_hands: Congratulations! You're now using regular OpenAI for all evals that require an LLM." + ) + + if __name__ == "__main__": app() diff --git a/deepeval/cli/test.py b/deepeval/cli/test.py index 0187476e8..7ea86e1bb 100644 --- a/deepeval/cli/test.py +++ b/deepeval/cli/test.py @@ -6,6 +6,7 @@ from deepeval.test_run import test_run_manager, TEMP_FILE_NAME from deepeval.utils import delete_file_if_exists from deepeval.test_run import invoke_test_run_end_hook +from deepeval.telemetry import capture_evaluation_count app = typer.Typer(name="test") @@ -74,6 +75,7 @@ def run( pytest_args.extend(["-p", "plugins"]) retcode = pytest.main(pytest_args) + capture_evaluation_count() test_run_manager.wrap_up_test_run() invoke_test_run_end_hook() diff --git a/deepeval/evaluate.py b/deepeval/evaluate.py index 53ddf5a36..ae36a49d3 100644 --- a/deepeval/evaluate.py +++ b/deepeval/evaluate.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import copy +from deepeval.telemetry import capture_evaluation_count from deepeval.progress_context import progress_context from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCase @@ -90,6 +91,7 @@ def run_test( test_run_manager.reset() with progress_context("Executing run_test()..."): test_result = execute_test([test_case], metrics, False)[0] + capture_evaluation_count() print_test_result(test_result) print("") print("-" * 70) @@ -109,7 +111,7 @@ def assert_test(test_case: LLMTestCase, metrics: List[BaseMetric]): ] failed_metrics_str = ", ".join( [ - f"{metric.__name__} (score: {metric.score}, minimum_score: {metric.minimum_score})" + f"{metric.__name__} (score: {metric.score}, threshold: {metric.threshold})" for metric in failed_metrics ] ) @@ -120,6 +122,7 @@ def evaluate(test_cases: List[LLMTestCase], metrics: List[BaseMetric]): test_run_manager.reset() with progress_context("Evaluating testcases..."): test_results = execute_test(test_cases, metrics, True) + capture_evaluation_count() for test_result in test_results: print_test_result(test_result) print("") @@ -135,11 +138,11 @@ def print_test_result(test_result: TestResult): for metric in test_result.metrics: if not metric.is_successful(): print( - f" - ❌ {metric.__name__} (score: {metric.score}, minimum_score: {metric.minimum_score}, reason: {metric.reason})" + f" - ❌ {metric.__name__} (score: {metric.score}, threshold: {metric.threshold}, reason: {metric.reason})" ) else: print( - f" - ✅ {metric.__name__} (score: {metric.score}, minimum_score: {metric.minimum_score}, reason: {metric.reason})" + f" - ✅ {metric.__name__} (score: {metric.score}, threshold: {metric.threshold}, reason: {metric.reason})" ) if metric.score_metadata: for metric_name, score in metric.score_metadata.items(): diff --git a/deepeval/event.py b/deepeval/event.py index 709f03c09..73b6d6b1e 100644 --- a/deepeval/event.py +++ b/deepeval/event.py @@ -1,5 +1,7 @@ from typing import Optional, List, Dict from deepeval.api import Api, Endpoints +import threading +import asyncio from pydantic import BaseModel, Field @@ -32,7 +34,18 @@ def track( conversation_id: Optional[str] = None, additional_data: Optional[Dict] = None, fail_silently: Optional[bool] = True, + run_on_background_thread: Optional[bool] = True, ): + def track_event(event: APIEvent, api: Api, fail_silently: bool): + try: + _ = api.post_request( + endpoint=Endpoints.EVENT_ENDPOINT.value, + body=event.dict(by_alias=True, exclude_none=True), + ) + except Exception as e: + if not fail_silently: + raise e + event = APIEvent( name=event_name, model=model, @@ -47,11 +60,10 @@ def track( additionalData=additional_data, ) api = Api() - try: - _ = api.post_request( - endpoint=Endpoints.EVENT_ENDPOINT.value, - body=event.dict(by_alias=True, exclude_none=True), + if run_on_background_thread: + thread = threading.Thread( + target=track_event, args=(event, api, fail_silently), daemon=True ) - except Exception as e: - if not fail_silently: - raise (e) + thread.start() + else: + track_event(event, api, fail_silently) diff --git a/deepeval/metrics/__init__.py b/deepeval/metrics/__init__.py index 51a7dd2b5..07ec2edf5 100644 --- a/deepeval/metrics/__init__.py +++ b/deepeval/metrics/__init__.py @@ -11,10 +11,12 @@ from .contextual_precision import ContextualPrecisionMetric from .ragas_metric import ( RagasMetric, + RAGASAnswerRelevancyMetric, RAGASFaithfulnessMetric, RAGASContextualRecallMetric, RAGASContextualRelevancyMetric, RAGASContextualPrecisionMetric, + RAGASAnswerRelevancyMetric, RAGASConcisenessMetric as ConcisenessMetric, RAGASCorrectnessMetric as CorrectnessMetric, RAGASCoherenceMetric as CoherenceMetric, diff --git a/deepeval/metrics/answer_relevancy.py b/deepeval/metrics/answer_relevancy.py index 23154ca8d..d60a6091f 100644 --- a/deepeval/metrics/answer_relevancy.py +++ b/deepeval/metrics/answer_relevancy.py @@ -7,6 +7,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import AnswerRelevancyTemplate +from deepeval.progress_context import metrics_progress_context class AnswerRelvancyVerdict(BaseModel): @@ -17,11 +18,13 @@ class AnswerRelvancyVerdict(BaseModel): class AnswerRelevancyMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.5, + threshold: float = 0.5, model: Optional[str] = None, + include_reason: bool = True, ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model + self.include_reason = include_reason self.n = 5 def measure(self, test_case: LLMTestCase) -> float: @@ -33,24 +36,22 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's newest Answer Relevancy Metric! This may take a minute." - ) - self.key_points: List[str] = self._generate_key_points( - test_case.actual_output, "\n".join(test_case.retrieval_context) - ) - self.verdicts: List[AnswerRelvancyVerdict] = self._generate_verdicts( - test_case.input - ) + with metrics_progress_context(self.__name__): + self.key_points: List[str] = self._generate_key_points( + test_case.actual_output, "\n".join(test_case.retrieval_context) + ) + self.verdicts: List[ + AnswerRelvancyVerdict + ] = self._generate_verdicts(test_case.input) - answer_relevancy_score = self._generate_score() + answer_relevancy_score = self._generate_score() - self.reason = self._generate_reason( - test_case.input, test_case.actual_output, answer_relevancy_score - ) - self.success = answer_relevancy_score >= self.minimum_score - self.score = answer_relevancy_score - return self.score + self.reason = self._generate_reason( + test_case.input, test_case.actual_output, answer_relevancy_score + ) + self.success = answer_relevancy_score >= self.threshold + self.score = answer_relevancy_score + return self.score def _generate_score(self): relevant_count = 0 @@ -63,6 +64,9 @@ def _generate_score(self): def _generate_reason( self, original_question: str, answer: str, score: float ) -> str: + if self.include_reason is False: + return None + irrelevant_points = [] for verdict in self.verdicts: if verdict.verdict.strip().lower() == "no": @@ -111,6 +115,7 @@ def _generate_key_points( return data["key_points"] def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/base_metric.py b/deepeval/metrics/base_metric.py index ce2cdb866..f73a60143 100644 --- a/deepeval/metrics/base_metric.py +++ b/deepeval/metrics/base_metric.py @@ -5,34 +5,22 @@ class BaseMetric: - # set an arbitrary minimum score that will get over-ridden later score: float = 0 score_metadata: Dict = None reason: Optional[str] = None @property - def minimum_score(self) -> float: - return self._minimum_score + def threshold(self) -> float: + return self._threshold - @minimum_score.setter - def minimum_score(self, value: float): - self._minimum_score = value + @threshold.setter + def threshold(self, value: float): + self._threshold = value - # Measure function signature is subject to be different - not sure - # how applicable this is - might need a better abstraction @abstractmethod def measure(self, test_case: LLMTestCase, *args, **kwargs) -> float: raise NotImplementedError - def _get_init_values(self): - # We use this method for sending useful metadata - init_values = { - param: getattr(self, param) - for param in vars(self) - if isinstance(getattr(self, param), (str, int, float)) - } - return init_values - @abstractmethod def is_successful(self) -> bool: raise NotImplementedError diff --git a/deepeval/metrics/contextual_precision.py b/deepeval/metrics/contextual_precision.py index 8a658b38e..5591ef50d 100644 --- a/deepeval/metrics/contextual_precision.py +++ b/deepeval/metrics/contextual_precision.py @@ -7,6 +7,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import ContextualPrecisionTemplate +from deepeval.progress_context import metrics_progress_context class ContextualPrecisionVerdict(BaseModel): @@ -18,10 +19,12 @@ class ContextualPrecisionVerdict(BaseModel): class ContextualPrecisionMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.5, + threshold: float = 0.5, model: Optional[str] = None, + include_reason: bool = True, ): - self.minimum_score = minimum_score + self.threshold = threshold + self.include_reason = include_reason self.model = model def measure(self, test_case: LLMTestCase) -> float: @@ -34,27 +37,29 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, expected output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's newest Contextual Precision Metric! This may take a minute." - ) - self.verdicts: List[ - ContextualPrecisionVerdict - ] = self._generate_verdicts( - test_case.input, - test_case.expected_output, - test_case.retrieval_context, - ) - contextual_precision_score = self._generate_score() - self.reason = self._generate_reason( - test_case.input, contextual_precision_score - ) + with metrics_progress_context(self.__name__): + self.verdicts: List[ + ContextualPrecisionVerdict + ] = self._generate_verdicts( + test_case.input, + test_case.expected_output, + test_case.retrieval_context, + ) + contextual_precision_score = self._generate_score() - self.success = contextual_precision_score >= self.minimum_score - self.score = contextual_precision_score - return self.score + self.reason = self._generate_reason( + test_case.input, contextual_precision_score + ) + + self.success = contextual_precision_score >= self.threshold + self.score = contextual_precision_score + return self.score def _generate_reason(self, input: str, score: float): + if self.include_reason is False: + return None + retrieval_contexts_verdicts = [ { "verdict": verdict.verdict, @@ -131,6 +136,7 @@ def _generate_verdicts( return verdicts def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/contextual_recall.py b/deepeval/metrics/contextual_recall.py index 3067a9513..6cf2aeec5 100644 --- a/deepeval/metrics/contextual_recall.py +++ b/deepeval/metrics/contextual_recall.py @@ -7,6 +7,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import ContextualRecallTemplate +from deepeval.progress_context import metrics_progress_context class ContextualRecallVerdict(BaseModel): @@ -17,11 +18,13 @@ class ContextualRecallVerdict(BaseModel): class ContextualRecallMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.5, + threshold: float = 0.5, model: Optional[str] = None, + include_reason: bool = True, ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model + self.include_reason = include_reason self.n = 5 def measure(self, test_case: LLMTestCase) -> float: @@ -34,24 +37,27 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, expected output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's newest Contextual Recall Metric! This may take a minute." - ) - self.verdicts: List[ContextualRecallVerdict] = self._generate_verdicts( - test_case.expected_output, test_case.retrieval_context - ) + with metrics_progress_context(self.__name__): + self.verdicts: List[ + ContextualRecallVerdict + ] = self._generate_verdicts( + test_case.expected_output, test_case.retrieval_context + ) - contextual_recall_score = self._generate_score() + contextual_recall_score = self._generate_score() - self.reason = self._generate_reason( - test_case.expected_output, contextual_recall_score - ) + self.reason = self._generate_reason( + test_case.expected_output, contextual_recall_score + ) - self.success = contextual_recall_score >= self.minimum_score - self.score = contextual_recall_score - return self.score + self.success = contextual_recall_score >= self.threshold + self.score = contextual_recall_score + return self.score def _generate_reason(self, expected_output: str, score: float): + if self.include_reason is False: + return None + supportive_reasons = [] unsupportive_reasons = [] for verdict in self.verdicts: @@ -96,6 +102,7 @@ def _generate_verdicts( return verdicts def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/contextual_relevancy.py b/deepeval/metrics/contextual_relevancy.py index f6806cfe8..aa717fea4 100644 --- a/deepeval/metrics/contextual_relevancy.py +++ b/deepeval/metrics/contextual_relevancy.py @@ -8,6 +8,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import ContextualRelevancyTemplate +from deepeval.progress_context import metrics_progress_context class ContextualRelevancyVerdict(BaseModel): @@ -18,11 +19,13 @@ class ContextualRelevancyVerdict(BaseModel): class ContextualRelevancyMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.5, + threshold: float = 0.5, model: Optional[str] = "gpt-4", + include_reason: bool = True, ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model + self.include_reason = include_reason def measure(self, test_case: LLMTestCase) -> float: if ( @@ -33,26 +36,27 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's newest Contextual Relevancy Metric! This may take a minute." - ) - self.verdicts_list: List[ - List[ContextualRelevancyVerdict] - ] = self._generate_verdicts_list( - test_case.input, test_case.retrieval_context - ) - contextual_recall_score = self._generate_score() + with metrics_progress_context(self.__name__): + self.verdicts_list: List[ + List[ContextualRelevancyVerdict] + ] = self._generate_verdicts_list( + test_case.input, test_case.retrieval_context + ) + contextual_recall_score = self._generate_score() - self.reason = self._generate_reason( - test_case.input, contextual_recall_score - ) + self.reason = self._generate_reason( + test_case.input, contextual_recall_score + ) - self.success = contextual_recall_score >= self.minimum_score - self.score = contextual_recall_score + self.success = contextual_recall_score >= self.threshold + self.score = contextual_recall_score - return self.score + return self.score def _generate_reason(self, input: str, score: float): + if self.include_reason is False: + return None + irrelevant_sentences = [] for index, verdicts in enumerate(self.verdicts_list): for verdict in verdicts: @@ -127,6 +131,7 @@ def _generate_verdicts_list( return verdicts_list def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/faithfulness.py b/deepeval/metrics/faithfulness.py index 7f54a7458..85409fd5f 100644 --- a/deepeval/metrics/faithfulness.py +++ b/deepeval/metrics/faithfulness.py @@ -8,25 +8,26 @@ from deepeval.utils import trimToJson from deepeval.models import GPTModel from deepeval.templates import FaithfulnessTemplate +from deepeval.progress_context import metrics_progress_context class FaithfulnessVerdict(BaseModel): verdict: str - reason: str + reason: str = Field(default=None) truth: str = Field(default=None) class FaithfulnessMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.5, + threshold: float = 0.5, model: Optional[str] = None, + include_reason: bool = True, ): - self.minimum_score = minimum_score + self.threshold = threshold # Don't set self.chat_model when using threading self.model = model - self.truths_list = None - self.verdicts_list = None + self.include_reason = include_reason def measure(self, test_case: LLMTestCase): if ( @@ -37,22 +38,20 @@ def measure(self, test_case: LLMTestCase): raise ValueError( "Input, actual output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's newest Faithfulness Metric! This may take a minute." - ) - self.truths_list: List[List[str]] = self._generate_truths_list( - test_case.retrieval_context - ) - self.verdicts_list: List[ - List[FaithfulnessVerdict] - ] = self._generate_verdicts_list( - self.truths_list, test_case.actual_output - ) - faithfulness_score = self._generate_score() - self.reason = self._generate_reason(faithfulness_score) - self.success = faithfulness_score >= self.minimum_score - self.score = faithfulness_score - return self.score + with metrics_progress_context(self.__name__): + self.truths_list: List[List[str]] = self._generate_truths_list( + test_case.retrieval_context + ) + self.verdicts_list: List[ + List[FaithfulnessVerdict] + ] = self._generate_verdicts_list( + self.truths_list, test_case.actual_output + ) + faithfulness_score = self._generate_score() + self.reason = self._generate_reason(faithfulness_score) + self.success = faithfulness_score >= self.threshold + self.score = faithfulness_score + return self.score def _generate_score(self): total_verdicts = 0 @@ -66,14 +65,18 @@ def _generate_score(self): return faithful_count / total_verdicts def _generate_reason(self, score: float): - contradiction_reasons = [] - for verdicts in self.verdicts_list: + if self.include_reason is False: + return None + + contradictions = [] + for index, verdicts in enumerate(self.verdicts_list): for verdict in verdicts: if verdict.verdict.strip().lower() == "no": - contradiction_reasons.append(verdict.reason) + data = {"contradiction": verdict.reason, "rank": index + 1} + contradictions.append(data) prompt: dict = FaithfulnessTemplate.generate_reason( - contradiction_reasons=contradiction_reasons, + contradictions=contradictions, score=format(score, ".2f"), ) @@ -169,6 +172,7 @@ def _generate_verdicts_list( return verdicts_list def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/hallucination_metric.py b/deepeval/metrics/hallucination_metric.py index 3f63c6910..b9eb3346c 100644 --- a/deepeval/metrics/hallucination_metric.py +++ b/deepeval/metrics/hallucination_metric.py @@ -7,9 +7,9 @@ class HallucinationMetric(BaseMetric, metaclass=Singleton): def __init__( self, - minimum_score: float = 0.5, + threshold: float = 0.5, ): - self.minimum_score = minimum_score + self.threshold = threshold def measure(self, test_case: LLMTestCase): if test_case.actual_output is None or test_case.context is None: @@ -25,11 +25,12 @@ def measure(self, test_case: LLMTestCase): if score > max_score: max_score = score - self.success = max_score > self.minimum_score + self.success = max_score >= self.threshold self.score = max_score return max_score def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/judgemental_gpt.py b/deepeval/metrics/judgemental_gpt.py index 30f44836f..d707adade 100644 --- a/deepeval/metrics/judgemental_gpt.py +++ b/deepeval/metrics/judgemental_gpt.py @@ -24,7 +24,7 @@ def __init__( criteria: str, evaluation_params: List[LLMTestCaseParams], language: Languages = Languages.ENGLISH, - minimum_score: float = 0.5, + threshold: float = 0.5, ): if not isinstance(language, Languages): raise TypeError("'language' must be an instance of Languages.") @@ -33,7 +33,7 @@ def __init__( self.name = name self.evaluation_params = evaluation_params self.language = language.value - self.minimum_score = minimum_score + self.threshold = threshold self.success = None self.reason = None @@ -70,9 +70,10 @@ def measure(self, test_case: LLMTestCase): ) self.reason = response.reason self.score = response.score / 10 - self.success = self.score >= self.minimum_score + self.success = self.score >= self.threshold return self.score - def is_successful(self): + def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success diff --git a/deepeval/metrics/llm_eval_metric.py b/deepeval/metrics/llm_eval_metric.py index 4d08674f5..8726f9a79 100644 --- a/deepeval/metrics/llm_eval_metric.py +++ b/deepeval/metrics/llm_eval_metric.py @@ -26,7 +26,7 @@ def __init__( criteria: Optional[str] = None, evaluation_steps: Optional[List[str]] = None, model: Optional[str] = None, - minimum_score: float = 0.5, + threshold: float = 0.5, ): self.name = name self.evaluation_params = evaluation_params @@ -50,7 +50,7 @@ def __init__( self.criteria = criteria self.model = model self.evaluation_steps = evaluation_steps - self.minimum_score = minimum_score + self.threshold = threshold def measure(self, test_case: LLMTestCase): """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf""" @@ -73,10 +73,11 @@ def measure(self, test_case: LLMTestCase): score, reason = self.evaluate(test_case) self.reason = reason self.score = float(score) / 10 - self.success = score >= self.minimum_score + self.success = score >= self.threshold return self.score - def is_successful(self): + def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success def generate_evaluation_steps(self): diff --git a/deepeval/metrics/non_toxic_metric.py b/deepeval/metrics/non_toxic_metric.py index 96318bef2..20c9f78f2 100644 --- a/deepeval/metrics/non_toxic_metric.py +++ b/deepeval/metrics/non_toxic_metric.py @@ -13,13 +13,13 @@ def __init__( self, evaluation_params: List[LLMTestCaseParams], model_name: str = "original", - minimum_score: float = 0.5, + threshold: float = 0.5, ): if not evaluation_params: raise ValueError("evaluation_params cannot be empty or None") self.evaluation_params = evaluation_params - self.minimum_score, self.model_name = minimum_score, model_name + self.threshold, self.model_name = threshold, model_name def __call__(self, test_case: LLMTestCase): score = self.measure(test_case.actual_output) @@ -58,11 +58,12 @@ def measure(self, test_case: LLMTestCase): average_score = sum(total_scores) / len(total_scores) # Check if the average score meets the minimum requirement - self.success = average_score >= self.minimum_score + self.success = average_score >= self.threshold self.score = average_score return self.score - def is_successful(self): + def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/ragas_metric.py b/deepeval/metrics/ragas_metric.py index cf27ce146..a1e1e4309 100644 --- a/deepeval/metrics/ragas_metric.py +++ b/deepeval/metrics/ragas_metric.py @@ -17,10 +17,10 @@ class RAGASContextualPrecisionMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -57,7 +57,7 @@ def measure(self, test_case: LLMTestCase): # Ragas only does dataset-level comparisons context_precision_score = scores["context_precision"] - self.success = context_precision_score >= self.minimum_score + self.success = context_precision_score >= self.threshold self.score = context_precision_score return self.score @@ -74,10 +74,10 @@ class RAGASContextualRelevancyMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -113,7 +113,7 @@ def measure(self, test_case: LLMTestCase): # Ragas only does dataset-level comparisons context_relevancy_score = scores["context_relevancy"] - self.success = context_relevancy_score >= self.minimum_score + self.success = context_relevancy_score >= self.threshold self.score = context_relevancy_score return self.score @@ -130,10 +130,10 @@ class RAGASAnswerRelevancyMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -165,7 +165,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, metrics=[answer_relevancy]) answer_relevancy_score = scores["answer_relevancy"] - self.success = answer_relevancy_score >= self.minimum_score + self.success = answer_relevancy_score >= self.threshold self.score = answer_relevancy_score return self.score @@ -180,10 +180,10 @@ def __name__(self): class RAGASFaithfulnessMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -215,7 +215,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, metrics=[faithfulness]) faithfulness_score = scores["faithfulness"] - self.success = faithfulness_score >= self.minimum_score + self.success = faithfulness_score >= self.threshold self.score = faithfulness_score return self.score @@ -232,10 +232,10 @@ class RAGASContextualRecallMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -267,7 +267,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, [context_recall]) context_recall_score = scores["context_recall"] - self.success = context_recall_score >= self.minimum_score + self.success = context_recall_score >= self.threshold self.score = context_recall_score return self.score @@ -284,10 +284,10 @@ class RAGASHarmfulnessMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -320,7 +320,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, [harmfulness]) harmfulness_score = scores["harmfulness"] - self.success = harmfulness_score >= self.minimum_score + self.success = harmfulness_score >= self.threshold self.score = harmfulness_score return self.score @@ -337,10 +337,10 @@ class RAGASCoherenceMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -371,7 +371,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, [coherence]) coherence_score = scores["coherence"] - self.success = coherence_score >= self.minimum_score + self.success = coherence_score >= self.threshold self.score = coherence_score return self.score @@ -388,10 +388,10 @@ class RAGASMaliciousnessMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -423,7 +423,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, [maliciousness]) maliciousness_score = scores["maliciousness"] - self.success = maliciousness_score >= self.minimum_score + self.success = maliciousness_score >= self.threshold self.score = maliciousness_score return self.score @@ -440,10 +440,10 @@ class RAGASCorrectnessMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -475,7 +475,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, metrics=[correctness]) correctness_score = scores["correctness"] - self.success = correctness_score >= self.minimum_score + self.success = correctness_score >= self.threshold self.score = correctness_score return self.score @@ -492,10 +492,10 @@ class RAGASConcisenessMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -526,7 +526,7 @@ def measure(self, test_case: LLMTestCase): dataset = Dataset.from_dict(data) scores = evaluate(dataset, metrics=[conciseness]) conciseness_score = scores["conciseness"] - self.success = conciseness_score >= self.minimum_score + self.success = conciseness_score >= self.threshold self.score = conciseness_score return self.score @@ -543,10 +543,10 @@ class RagasMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.3, + threshold: float = 0.3, model: Optional[str] = "gpt-3.5-turbo", ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model def measure(self, test_case: LLMTestCase): @@ -594,7 +594,7 @@ def measure(self, test_case: LLMTestCase): 1.0 / score for score in score_metadata.values() ) - self.success = ragas_score >= self.minimum_score + self.success = ragas_score >= self.threshold self.score = ragas_score self.score_metadata = score_metadata return self.score diff --git a/deepeval/metrics/summarization.py b/deepeval/metrics/summarization.py index 9a005e6fd..c5c09ed1d 100644 --- a/deepeval/metrics/summarization.py +++ b/deepeval/metrics/summarization.py @@ -21,12 +21,12 @@ class ScoreType(Enum): class SummarizationMetric(BaseMetric): def __init__( self, - minimum_score: float = 0.5, + threshold: float = 0.5, model: Optional[str] = None, n: Optional[int] = 5, assessment_questions: Optional[List[str]] = None, ): - self.minimum_score = minimum_score + self.threshold = threshold self.model = model self.assessment_questions = assessment_questions self.n = n @@ -54,7 +54,7 @@ def measure(self, test_case: LLMTestCase): summarization_score = min(alignment_score, inclusion_score) - self.success = summarization_score >= self.minimum_score + self.success = summarization_score >= self.threshold self.score_metadata = { "Alignment": alignment_score, "Inclusion": inclusion_score, @@ -134,6 +134,7 @@ def get_answer(self, question: str, text: str) -> str: return res.content def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/metrics/unbias_metric.py b/deepeval/metrics/unbias_metric.py index 945d07c67..fb137caab 100644 --- a/deepeval/metrics/unbias_metric.py +++ b/deepeval/metrics/unbias_metric.py @@ -15,18 +15,18 @@ def __init__( self, evaluation_params: List[LLMTestCaseParams], model_name: str = "original", - minimum_score: float = 0.5, + threshold: float = 0.5, ): # see paper for rationale https://arxiv.org/pdf/2208.05777.pdf if not evaluation_params: raise ValueError("evaluation_params cannot be empty or None") self.evaluation_params = evaluation_params self.model_name = model_name - self.minimum_score = minimum_score + self.threshold = threshold def __call__(self, output, expected_output, query: Optional[str] = "-"): score = self.measure(output, expected_output) - success = score >= self.minimum_score + success = score >= self.threshold return score def measure(self, test_case: LLMTestCase, return_all_scores: bool = False): @@ -61,7 +61,7 @@ def measure(self, test_case: LLMTestCase, return_all_scores: bool = False): # Calculate the average score average_score = total_score / len(self.evaluation_params) - self.success = average_score > self.minimum_score + self.success = average_score > self.threshold self.score = average_score if return_all_scores: @@ -69,7 +69,8 @@ def measure(self, test_case: LLMTestCase, return_all_scores: bool = False): return average_score - def is_successful(self): + def is_successful(self) -> bool: + self.success = self.score >= self.threshold return self.success @property diff --git a/deepeval/models/_summac_model.py b/deepeval/models/_summac_model.py index 6541f103f..b7321e250 100644 --- a/deepeval/models/_summac_model.py +++ b/deepeval/models/_summac_model.py @@ -3,7 +3,6 @@ # Source: https://github.com/tingofurro/summac ############################################### -from transformers import AutoTokenizer, AutoModelForSequenceClassification import nltk import numpy as np import torch @@ -131,6 +130,15 @@ def load_nli(self): ) else: + try: + from transformers import ( + AutoTokenizer, + AutoModelForSequenceClassification, + ) + except ModuleNotFoundError: + print( + "transformers library is not installed. Run 'pip install transformers'" + ) self.tokenizer = AutoTokenizer.from_pretrained(self.model_card) self.model = AutoModelForSequenceClassification.from_pretrained( self.model_card diff --git a/deepeval/models/gpt_model.py b/deepeval/models/gpt_model.py index 1a8bdbda0..49637c8b7 100644 --- a/deepeval/models/gpt_model.py +++ b/deepeval/models/gpt_model.py @@ -1,10 +1,10 @@ import os from typing import Dict, Optional -from langchain.chat_models import ChatOpenAI, AzureChatOpenAI +from langchain_openai import ChatOpenAI, AzureChatOpenAI from deepeval.key_handler import KeyValues, KEY_FILE_HANDLER from deepeval.models.base import DeepEvalBaseModel -from deepeval.chat_completion.retry import call_openai_with_retry +from deepeval.chat_completion.retry import retry_with_exponential_backoff valid_gpt_models = [ "gpt-4-1106-preview", @@ -43,13 +43,6 @@ def __init__( def load_model(self): if self.should_use_azure_openai(): - model_version = KEY_FILE_HANDLER.fetch_data( - KeyValues.AZURE_MODEL_VERSION - ) - model_kwargs = {} - if model_version is not None: - model_kwargs["model_version"] = model_version - openai_api_key = KEY_FILE_HANDLER.fetch_data( KeyValues.AZURE_OPENAI_API_KEY ) @@ -63,20 +56,30 @@ def load_model(self): azure_endpoint = KEY_FILE_HANDLER.fetch_data( KeyValues.AZURE_OPENAI_ENDPOINT ) + + model_version = KEY_FILE_HANDLER.fetch_data( + KeyValues.AZURE_MODEL_VERSION + ) + + if model_version is None: + model_version = "" + return AzureChatOpenAI( openai_api_version=openai_api_version, azure_deployment=azure_deployment, azure_endpoint=azure_endpoint, openai_api_key=openai_api_key, - model_kwargs=model_kwargs, + model_version=model_version, ) + return ChatOpenAI( model_name=self.model_name, model_kwargs=self.model_kwargs ) + @retry_with_exponential_backoff def _call(self, prompt: str): chat_model = self.load_model() - return call_openai_with_retry(lambda: chat_model.invoke(prompt)) + return chat_model.invoke(prompt) def should_use_azure_openai(self): value = KEY_FILE_HANDLER.fetch_data(KeyValues.USE_AZURE_OPENAI) diff --git a/deepeval/models/summac_model.py b/deepeval/models/summac_model.py index 7df978794..daa6ab258 100644 --- a/deepeval/models/summac_model.py +++ b/deepeval/models/summac_model.py @@ -1,5 +1,5 @@ import torch -from typing import Union, List +from typing import Union, List, Optional from typing import List, Union, get_origin from deepeval.models.base import DeepEvalBaseModel from deepeval.models._summac_model import _SummaCZS @@ -8,9 +8,9 @@ class SummaCModels(DeepEvalBaseModel): def __init__( self, - model_name: str | None = None, - granularity: str | None = None, - device: str | None = None, + model_name: Optional[str] = None, + granularity: Optional[str] = None, + device: Optional[str] = None, *args, **kwargs ): @@ -27,11 +27,11 @@ def __init__( def load_model( self, - op1: str | None = "max", - op2: str | None = "mean", - use_ent: bool | None = True, - use_con: bool | None = True, - image_load_cache: bool | None = True, + op1: Optional[str] = "max", + op2: Optional[str] = "mean", + use_ent: Optional[bool] = True, + use_con: Optional[bool] = True, + image_load_cache: Optional[bool] = True, **kwargs ): return _SummaCZS( diff --git a/deepeval/progress_context.py b/deepeval/progress_context.py index ed1b3c523..f7986a602 100644 --- a/deepeval/progress_context.py +++ b/deepeval/progress_context.py @@ -17,3 +17,19 @@ def progress_context( ) as progress: progress.add_task(description=description, total=total) yield + + +@contextmanager +def metrics_progress_context( + metric_name: str, total: int = 9999, transient: bool = True +): + description = f"✨ 🍰 ✨ You're using DeepEval's latest {metric_name} Metric! This may take a minute..." + console = Console(file=sys.stderr) # Direct output to standard error + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, # Use the custom console + transient=transient, + ) as progress: + progress.add_task(description=description, total=total) + yield diff --git a/deepeval/telemetry.py b/deepeval/telemetry.py index c0c2e5364..d6915bf11 100644 --- a/deepeval/telemetry.py +++ b/deepeval/telemetry.py @@ -1,9 +1,10 @@ import os import socket import sys +import sentry_sdk -def check_firewall(): +def blocked_by_firewall(): try: socket.create_connection(("www.google.com", 80)) return False @@ -11,29 +12,24 @@ def check_firewall(): return True -if os.getenv("ERROR_REPORTING") == "YES" and not check_firewall(): - try: - import sentry_sdk - - sentry_sdk.init( - dsn="https://5ef587d58109ee45d6544f3657efdd1f@o4506098477236224.ingest.sentry.io/4506098479136768", - # Set traces_sample_rate to 1.0 to capture 100% - # of transactions for performance monitoring. - traces_sample_rate=1.0, - # Set profiles_sample_rate to 1.0 to profile 100% - # of sampled transactions. - # We recommend adjusting this value in production. - profiles_sample_rate=1.0, - ) - - # Add a global error handler - def handle_exception(exc_type, exc_value, exc_traceback): - print({"exc_type": exc_type, "exc_value": exc_value}) - sentry_sdk.capture_exception(exc_value) - sys.__excepthook__(exc_type, exc_value, exc_traceback) - - sys.excepthook = handle_exception - - except ModuleNotFoundError: - # sentry_sdk not installed - pass +def capture_evaluation_count(): + sentry_sdk.capture_message("evaluation ran!") + + +sentry_sdk.init( + dsn="https://5ef587d58109ee45d6544f3657efdd1f@o4506098477236224.ingest.sentry.io/4506098479136768", + profiles_sample_rate=1.0, + traces_sample_rate=1.0, # For performance monitoring + send_default_pii=False, # Don't send personally identifiable information + attach_stacktrace=False, # Don't attach stack traces to messages + default_integrations=False, # Disable Sentry's default integrations +) + +if os.getenv("ERROR_REPORTING") == "YES" and not blocked_by_firewall(): + + def handle_exception(exc_type, exc_value, exc_traceback): + print({"exc_type": exc_type, "exc_value": exc_value}) + sentry_sdk.capture_exception(exc_value) + sys.__excepthook__(exc_type, exc_value, exc_traceback) + + sys.excepthook = handle_exception diff --git a/deepeval/templates.py b/deepeval/templates.py index d7e53ff78..f32f4b10a 100644 --- a/deepeval/templates.py +++ b/deepeval/templates.py @@ -73,7 +73,8 @@ def generate_truths(text): def generate_verdicts(truths, text): return f"""Based on a list of strings, called contexts, please generate a list of JSON objects to indicate whether the given 'actual output' agrees with EACH context. The JSON will have 2 fields: 'verdict' and 'reason'. The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', and states whether the given text agrees with the context. -The 'reason' is the reason for the verdict. When the answer is 'no' or 'idk', try to provide a correction in the reason. +The 'reason' is the reason for the verdict. When the answer is 'no' or 'idk', try to provide a correction in the reason. +You DON'T have to provide a reason if the answer is 'yes'. ** IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects. @@ -85,19 +86,21 @@ def generate_verdicts(truths, text): "verdicts": [ {{ "verdict": "yes", - "reason": "The context states that Einstein won the Nobel Prize for his discovery of the photoelectric effect." + "reason": "The node in the retrieval context states that Einstein won the Nobel Prize for his discovery of the photoelectric effect." }}, {{ "verdict": "no", - "reason": "The context states that Einstein won the Nobel Prize in 1968, not 1969." + "reason": "The node in the retrieval context states that Einstein won the Nobel Prize in 1968, not 1969." }} ] }} You should NOT incorporate any prior knowledge you have and take each context at face value. Since you are going to generate a verdict for each context, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to that of contexts. +You DON'T have to provide a reason if the answer is 'yes'. +You should ONLY provide a 'no' answer if IT IS A CONTRADICTION. ** -Contexts: +Retrieval Contexts: {truths} Actual Output: @@ -107,19 +110,28 @@ def generate_verdicts(truths, text): """ @staticmethod - def generate_reason(score, contradiction_reasons): - return f"""Below is a list of Contradictions. It explains why the 'actual output' does not align with the 'retrieval context'. -Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is the context (higher the better), concisely summarize the contradictions to justify the score. If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying). + def generate_reason(score, contradictions): + return f"""Below is a list of Contradictions. It is a list of JSON with the `contradiction` and `rank` key. +The `contradiction` explains why the 'actual output' does not align with a certain node in the 'retrieval context'. Contradictions happen in the 'actual output', NOT the 'retrieval context'. +The `rank` tells you which node in the 'retrieval context' the actual output contradicted with. +Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), CONCISELY summarize the contradictions to justify the score. Faithfulness Score: {score} Contradictions: -{contradiction_reasons} +{contradictions} Example: The score is because . +** +IMPORTANT: +If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying). +Your reason MUST use information in `contradiction` and the node RANK (eg., first node of the retrieval context) in your reason. +Be sure in your reason, as if you know what the actual output is from the contradictions. +** + Reason: """ @@ -408,6 +420,7 @@ def generate_reason(input, verdicts, score): ** IMPORTANT: DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' are just here for you to understand the broader scope of things. +Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field. In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts. When addressing nodes, make it explicit that it is nodes in retrieval context. If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it otherwise it gets annoying). diff --git a/deepeval/test_case.py b/deepeval/test_case.py index ca3b2757d..48e17647c 100644 --- a/deepeval/test_case.py +++ b/deepeval/test_case.py @@ -20,6 +20,8 @@ def __init__( expected_output: Optional[str] = None, context: Optional[List[str]] = None, retrieval_context: Optional[List[str]] = None, + execution_time: Optional[float] = None, + cost: Optional[float] = None, id: Optional[str] = None, ): self.id = id @@ -28,3 +30,5 @@ def __init__( self.expected_output = expected_output self.context = context self.retrieval_context = retrieval_context + self.execution_time = execution_time + self.cost = cost diff --git a/deepeval/test_run/api.py b/deepeval/test_run/api.py index 3db64aec3..a80e55cce 100644 --- a/deepeval/test_run/api.py +++ b/deepeval/test_run/api.py @@ -5,7 +5,8 @@ class MetricsMetadata(BaseModel): metric: str score: float - minimum_score: float = Field(None, alias="minimumScore") + threshold: float + success: bool reason: Optional[str] = None diff --git a/deepeval/test_run/test_run.py b/deepeval/test_run/test_run.py index f28f61a23..dd82c9523 100644 --- a/deepeval/test_run/test_run.py +++ b/deepeval/test_run/test_run.py @@ -84,26 +84,25 @@ def add_llm_test_case( ): # Check if test case with the same ID already exists test_case_id = id(test_case) - existing_test_case: LLMTestCase = self.dict_test_cases.get( + existing_test_case: APITestCase = self.dict_test_cases.get( test_case_id, None ) - metrics_metadata = MetricsMetadata( + metric_metadata = MetricsMetadata( metric=metric.__name__, score=metric.score, - minimumScore=0.5, + threshold=metric.threshold, reason=metric.reason, + success=metric.is_successful(), ) if existing_test_case: # If it exists, append the metrics to the existing test case - existing_test_case.metrics_metadata.append(metrics_metadata) - success = all( - [ - metric.score >= metric.minimum_score - for metric in existing_test_case.metrics_metadata - ] - ) + existing_test_case.metrics_metadata.append(metric_metadata) + if metric.is_successful() and existing_test_case.success == True: + success = True + else: + success = False # Update the success status existing_test_case.success = success else: @@ -113,7 +112,7 @@ def add_llm_test_case( actualOutput=test_case.actual_output, expectedOutput=test_case.expected_output, success=metric.is_successful(), - metricsMetadata=[metrics_metadata], + metricsMetadata=[metric_metadata], runDuration=run_duration, context=test_case.context, retrievalContext=test_case.retrieval_context, @@ -201,11 +200,11 @@ def clear_test_run(self): def display_results_table(self, test_run: TestRun): table = Table(title="Test Results") - table.add_column("Test case", justify="right") - table.add_column("Metric", justify="right") - table.add_column("Score", justify="right") - table.add_column("Status", justify="right") - table.add_column("Overall Success Rate", justify="right") + table.add_column("Test case", justify="left") + table.add_column("Metric", justify="left") + table.add_column("Score", justify="left") + table.add_column("Status", justify="left") + table.add_column("Overall Success Rate", justify="left") for index, test_case in enumerate(test_run.test_cases): pass_count = 0 @@ -215,7 +214,7 @@ def display_results_table(self, test_run: TestRun): test_case_name += f" ({test_case.id})" for metric_metadata in test_case.metrics_metadata: - if metric_metadata.score >= metric_metadata.minimum_score: + if metric_metadata.success: pass_count += 1 else: fail_count += 1 @@ -229,7 +228,7 @@ def display_results_table(self, test_run: TestRun): ) for metric_metadata in test_case.metrics_metadata: - if metric_metadata.score >= metric_metadata.minimum_score: + if metric_metadata.success: status = "[green]PASSED[/green]" else: status = "[red]FAILED[/red]" @@ -237,7 +236,7 @@ def display_results_table(self, test_run: TestRun): table.add_row( "", str(metric_metadata.metric), - f"{round(metric_metadata.score,2)} (threshold={metric_metadata.minimum_score})", + f"{round(metric_metadata.score,2)} (threshold={metric_metadata.threshold}, reason={metric_metadata.reason})", status, "", ) diff --git a/deepeval/tracing/__init__.py b/deepeval/tracing/__init__.py index 76ea3893c..ce4c4642a 100644 --- a/deepeval/tracing/__init__.py +++ b/deepeval/tracing/__init__.py @@ -1 +1,12 @@ -from .tracing import trace, TraceType, get_trace_stack +from .tracing import ( + trace, + trace_manager, + get_trace_stack, + TraceType, + TraceStatus, + LlmTrace, + EmbeddingTrace, + GenericTrace, + LlmMetadata, + EmbeddingMetadata, +) diff --git a/deepeval/tracing/integrations/llama_index.py b/deepeval/tracing/integrations/llama_index.py new file mode 100644 index 000000000..0cd2d1736 --- /dev/null +++ b/deepeval/tracing/integrations/llama_index.py @@ -0,0 +1,152 @@ +from typing import Any, Dict, List, Optional, Union +from time import perf_counter + +from llama_index.bridge.pydantic import BaseModel +from llama_index.callbacks.base_handler import BaseCallbackHandler +from llama_index.callbacks.schema import CBEventType, EventPayload +from llama_index.llms import ChatMessage + +from deepeval.tracing import ( + trace_manager, + get_trace_stack, + LlmTrace, + GenericTrace, + EmbeddingTrace, + TraceStatus, + LlmMetadata, + EmbeddingMetadata, + TraceType, +) +from deepeval.utils import dataclass_to_dict + +events_to_ignore = [ + CBEventType.CHUNKING, + CBEventType.NODE_PARSING, + CBEventType.EMBEDDING, + CBEventType.TREE, + CBEventType.SUB_QUESTION, + CBEventType.FUNCTION_CALL, + CBEventType.EXCEPTION, + CBEventType.AGENT_STEP, +] + + +class LlamaIndexCallbackHandler(BaseCallbackHandler): + def __init__(self) -> None: + self.event_map = {} + super().__init__( + event_starts_to_ignore=events_to_ignore, + event_ends_to_ignore=events_to_ignore, + ) + + def start_trace(self, trace_id: Optional[str] = None) -> None: + self.event_map = {} + trace_manager.clear_trace_stack() + return + + def end_trace( + self, + trace_id: Optional[str] = None, + trace_map: Optional[Dict[str, List[str]]] = None, + ) -> None: + return + + def on_event_start( + self, + event_type: CBEventType, + payload: Optional[Dict[str, Any]] = None, + event_id: str = "", + parent_id: str = "", + **kwargs: Any, + ) -> str: + trace_instance = self.create_trace_instance(event_type) + self.event_map[event_id] = trace_instance + trace_manager.append_to_trace_stack(trace_instance) + return + + def on_event_end( + self, + event_type: CBEventType, + payload: Optional[Dict[str, Any]] = None, + event_id: str = "", + **kwargs: Any, + ) -> None: + trace_instance = self.event_map[event_id] + trace_instance.executionTime = ( + perf_counter() - trace_instance.executionTime + ) + input_kwargs = {} + if payload is not None: + for event in EventPayload: + value = payload.get(event.value) + if value is not None: + input_kwargs[event.value] = value + + current_trace_stack = trace_manager.get_trace_stack() + if len(current_trace_stack) > 1: + parent_trace = current_trace_stack[-2] + parent_trace.traces.append(trace_instance) + + if len(current_trace_stack) == 1: + dict_representation = dataclass_to_dict(current_trace_stack[0]) + trace_manager.set_dict_trace_stack(dict_representation) + trace_manager.clear_trace_stack() + else: + trace_manager.pop_trace_stack() + + return + + def create_trace_instance( + self, event_type: CBEventType + ) -> Union[EmbeddingTrace, LlmMetadata, GenericTrace]: + current_time = perf_counter() + type = self.convert_event_type_to_deepeval_trace_type(event_type) + name = event_type.capitalize() + trace_instance_input = {"args": None, "kwargs": None} + if event_type == CBEventType.LLM: + trace_instance = LlmTrace( + type=type, + executionTime=current_time, + name=name, + input=trace_instance_input, + output=None, + status=TraceStatus.SUCCESS, + traces=[], + llmMetadata=LlmMetadata(model="None"), + ) + elif event_type == CBEventType.EMBEDDING: + trace_instance = EmbeddingTrace( + type=type, + executionTime=current_time, + name=name, + input=trace_instance_input, + output=None, + status=TraceStatus.SUCCESS, + traces=[], + embeddingMetadata=EmbeddingMetadata(model="None"), + ) + else: + trace_instance = GenericTrace( + type=type, + executionTime=current_time, + name=name, + input=trace_instance_input, + output=None, + status=TraceStatus.SUCCESS, + traces=[], + ) + + return trace_instance + + def convert_event_type_to_deepeval_trace_type( + self, event_type: CBEventType + ): + # TODO: add more types + if event_type == CBEventType.LLM: + return TraceType.LLM + elif event_type == CBEventType.RETRIEVE: + return TraceType.RETRIEVER + elif event_type == CBEventType.EMBEDDING: + return TraceType.EMBEDDING + + return event_type.value.capitalize() diff --git a/docs/docs/confident-ai-debug-evaluations.mdx b/docs/docs/confident-ai-debug-evaluations.mdx index c4aa15364..13867035c 100644 --- a/docs/docs/confident-ai-debug-evaluations.mdx +++ b/docs/docs/confident-ai-debug-evaluations.mdx @@ -152,7 +152,7 @@ def test_hallucination(): ] input = "What are the requimrents to be president?" - metric = HallucinationMetric(minimum_score=0.8) + metric = HallucinationMetric(threshold=0.8) test_case = LLMTestCase( input=input, actual_output=chatbot.query(user_input=input), diff --git a/docs/docs/confident-ai-evals-in-production.mdx b/docs/docs/confident-ai-evals-in-production.mdx index 7a1fc87b9..98db35258 100644 --- a/docs/docs/confident-ai-evals-in-production.mdx +++ b/docs/docs/confident-ai-evals-in-production.mdx @@ -23,7 +23,12 @@ Simply add `deepeval.track(...)` in your application to start tracking events. T - [Optional] `token_usage`: type `float` - [Optional] `token_cost`: type `float` - [Optional] `additional_data`: type `dict` -- [Optional] `fail_silently`: type `bool`, defaults to True +- [Optional] `fail_silently`: type `bool`, defaults to True. You should try setting this to `False` if your events are not logging properly. +- [Optional] `run_on_background_thread`: type `bool`, defaults to True. You should try setting this to `False` if your events are not logging properly. + +:::note +Please do **NOT** provide placeholder values for optional parameters. Leave it blank instead. +::: ```python import deepeval @@ -44,6 +49,7 @@ deepeval.track( token_cost=0.23, additional_data={"example": "example"}, fail_silently=True + run_on_background_thread=True ) ``` diff --git a/docs/docs/confident-ai-evaluate-datasets.mdx b/docs/docs/confident-ai-evaluate-datasets.mdx index b5b5b6b7a..712b6ffdf 100644 --- a/docs/docs/confident-ai-evaluate-datasets.mdx +++ b/docs/docs/confident-ai-evaluate-datasets.mdx @@ -111,7 +111,7 @@ dataset.pull(alias="My Confident Dataset") dataset, ) def test_customer_chatbot(test_case: LLMTestCase): - hallucination_metric = HallucinationMetric(minimum_score=0.3) + hallucination_metric = HallucinationMetric(threshold=0.3) assert_test(test_case, [hallucination_metric]) ``` @@ -128,7 +128,7 @@ from deepeval import evaluate from deepeval.metrics import HallucinationMetric from deepeval.dataset import EvaluationDataset -hallucination_metric = HallucinationMetric(minimum_score=0.3) +hallucination_metric = HallucinationMetric(threshold=0.3) # Initialize empty dataset object and pull from Confident dataset = EvaluationDataset() diff --git a/docs/docs/confident-ai-introduction.mdx b/docs/docs/confident-ai-introduction.mdx index 7cc3a0bcb..bd6f69221 100644 --- a/docs/docs/confident-ai-introduction.mdx +++ b/docs/docs/confident-ai-introduction.mdx @@ -32,7 +32,7 @@ Continuous evaluation refers to the process of evaluating LLM applications in no /> -Everything in `deepeval` is already automatically integrated with Confident AI, including `deepeval`'s [custom metrics](evaluation-metrics#custom-metrics). To start using Confident AI with `deepeval`, simply login in the CLI: +Everything in `deepeval` is already automatically integrated with Confident AI, including `deepeval`'s [custom metrics](metrics-custom). To start using Confident AI with `deepeval`, simply login in the CLI: ``` deepeval login diff --git a/docs/docs/evaluation-datasets.mdx b/docs/docs/evaluation-datasets.mdx index b34503ea5..09fc2bfac 100644 --- a/docs/docs/evaluation-datasets.mdx +++ b/docs/docs/evaluation-datasets.mdx @@ -157,8 +157,8 @@ dataset = EvaluationDataset(test_cases=[...]) dataset, ) def test_customer_chatbot(test_case: LLMTestCase): - hallucination_metric = HallucinationMetric(minimum_score=0.3) - answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) + hallucination_metric = HallucinationMetric(threshold=0.3) + answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.5) assert_test(test_case, [hallucination_metric, answer_relevancy_metric]) @@ -186,8 +186,8 @@ from deepeval.dataset import EvaluationDataset dataset = EvaluationDataset(test_cases=[...]) -hallucination_metric = HallucinationMetric(minimum_score=0.3) -answer_relevancy_metric = AnswerRelevancyMetric(minimum_score=0.5) +hallucination_metric = HallucinationMetric(threshold=0.3) +answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.5) dataset.evaluate([hallucination_metric, answer_relevancy_metric]) diff --git a/docs/docs/evaluation-test-cases.mdx b/docs/docs/evaluation-test-cases.mdx index 0c544bd34..753690541 100644 --- a/docs/docs/evaluation-test-cases.mdx +++ b/docs/docs/evaluation-test-cases.mdx @@ -13,22 +13,25 @@ A test case is a blueprint provided by `deepeval` to unit test LLM outputs based - [Optional] `expected_output` - [Optional] `context` - [Optional] `retrieval_context` +- [Optional] `execution_time` (`float`) +- [Optional] `cost` (`float`) Except for `actual_output`, all parameters should originate from your evaluation dataset (if you have one). Here's an example implementation of a test case: ```python test_case = LLMTestCase( input="What if these shoes don't fit?", - expected_output = "You're eligible for a 30 day refund at no extra cost.", - actual_output = "We offer a 30-day full refund at no extra cost.", - context = ["All customers are eligible for a 30 day full refund at no extra cost."] - retrieval_context = ["Only shoes can be refunded."] + expected_output="You're eligible for a 30 day refund at no extra cost.", + actual_output="We offer a 30-day full refund at no extra cost.", + context=["All customers are eligible for a 30 day full refund at no extra cost."], + retrieval_context=["Only shoes can be refunded."], + execution_time=10.0 ) ``` **Note that only `input` and `actual_output` is mandatory.** -However, depending on the specific metric you're evaluating your test cases on, you may or may not require a `retrieval_context`, `expected_output` and/or `context` as additional parameters. For example, you won't need `expected_output` and `context` if you're just measuring answer relevancy, but if you're evaluating factual consistency you'll have to provide `context` in order for `deepeval` to know what the **ground truth** is. +However, depending on the specific metric you're evaluating your test cases on, you may or may not require a `retrieval_context`, `expected_output` and/or `context` as additional parameters. For example, you won't need `expected_output` and `context` if you're just measuring answer relevancy, but if you're evaluating hallucination you'll have to provide `context` in order for `deepeval` to know what the **ground truth** is. Let's go through the purpsoe of each parameter. @@ -82,7 +85,7 @@ test_case = LLMTestCase( An expected output is literally what you would want the ideal output to be. Note that this parameter is **optional** depending on the metric you want to evaluate. -The expected output doesn't have to exactly match the actual output in order for your test case to pass since `deepeval` uses a variety of methods to evaluate non-deterministic LLM outputs. We'll go into more details [in the metrics section.](evaluation-metrics) +The expected output doesn't have to exactly match the actual output in order for your test case to pass since `deepeval` uses a variety of methods to evaluate non-deterministic LLM outputs. We'll go into more details [in the metrics section.](metrics-introduction) ```python # A hypothetical LLM application example @@ -155,8 +158,6 @@ prompt_template = """ prompt = prompt_template.format(text="Who's a good boy?") -context = ["Rocky is a good boy."] - # Replace this with the actual retrieved context from your RAG pipeline retrieval_context = ["Rocky is a good cat."] @@ -165,7 +166,6 @@ test_case = LLMTestCase( # Replace this with your actual LLM application actual_output=chatbot.run(prompt), expected_output="Me, ruff!", - context=context, retrieval_context=retrieval_context ) ``` @@ -174,6 +174,40 @@ test_case = LLMTestCase( Remember, `context` is the ideal retrieval results for a given input and typically come from your evaluation dataset, whereas `retrieval_context` is your LLM application's actual retrieval results. ::: +## Execution Time + +The `execution_time` is an **optional** parameter that represents how long it took your LLM application to finish executing. However, if you're trying to measure something else, like the execution time for only the retrieval part of your RAG pipeline, feel free to supply that number instead of the overall execution time. + +```python +test_case = LLMTestCase( + input="...", + actual_output="...", + # Replace this with the actual execution time of your LLM application + execution_time=10.4 +) +``` + +:::note +`deepeval` does not offer metrics that evaluate on latency and cost, so feel free to supply the `execution_time` in either seconds, miliseconds, or even nanoseconds. That being said, [here is a full working example](metrics-custom#implementation) of how you can build your own `LatencyMetric` using the `execution_time` parameter. +::: + +## Cost + +The `cost` is an **optional** parameter that represents the token cost for a given execution of your LLM application. However, similar to `execution_time`, the `cost` parameter does not strictly have to be the total completion cost (eg., it could be the embedding cost), nor does it have to be in any set currency. + +```python +test_case = LLMTestCase( + input="...", + actual_output="...", + # Replace this with the actual execution time of your LLM application + cost=0.78 +) +``` + +:::info +`deepeval` does not offer cost and latency metrics because it is difficult to account for all different units and currencies available. We highly encourage you to look at the [custom metrics section](metrics-custom#implementation) for a full example on how to create your own metric if you are looking to evaluate cost and latency. +::: + ## Run A Test Case `deepeval` offers an option to quickly run a test case without going through the CLI. @@ -202,7 +236,7 @@ test_case = LLMTestCase( context=context ) -metric = HallucinationMetric(minimum_score=0.7) +metric = HallucinationMetric(threshold=0.7) run_test(test_case, [metric]) ``` @@ -219,7 +253,7 @@ Similar to Pytest, `deepeval` allows you to assert any test case you create by c - `test_case`: an `LLMTestCase` - `metrics`: a list of metrics -A test case passes only if all metrics meet their respective evaluation criterion. Depending on the metric, a combination of `input`, `actual_output`, `expected_output`, and `context` is used to ascertain whether their criterion have been met. +A test case passes only if all metrics meet their respective evaluation criterion. Depending on the metric, a combination of `input`, `actual_output`, `expected_output`, `context`, and `retrieval_context` is used to ascertain whether their criterion have been met. ```python title="test_assert_example.py" # A hypothetical LLM application example @@ -245,7 +279,7 @@ def test_case_1(): expected_output="Me, ruff!", context=context ) - metric = HallucinationMetric(minimum_score=0.7) + metric = HallucinationMetric(threshold=0.7) assert_test(test_case, metrics=[metric]) def test_case_2(): @@ -259,7 +293,7 @@ def test_case_2(): expected_output="Me, ruff!", context=context ) - metric = HallucinationMetric(minimum_score=0.7) + metric = HallucinationMetric(threshold=0.7) assert_test(test_case, metrics=[metric]) ``` @@ -271,7 +305,7 @@ deepeval test run test_assert_example.py -n 4 ## Evaluate Test Cases in Bulk -Lastly, `deepeval` offers an `evaluate` function to evaluate multiple test cases at once, which similar to assert_test but without needing pytest or the CLI. +Lastly, `deepeval` offers an `evaluate` function to evaluate multiple test cases at once, which similar to `assert_test` but without the need for Pytest or the CLI. ```python # A hypothetical LLM application example @@ -307,7 +341,7 @@ second_test_case = LLMTestCase( test_cases = [first_test_case, second_test_case] -metric = HallucinationMetric(minimum_score=0.7) +metric = HallucinationMetric(threshold=0.7) evaluate(test_cases, [metric]) ``` diff --git a/docs/docs/getting-started.mdx b/docs/docs/getting-started.mdx index 811802a6a..93fb66a4e 100644 --- a/docs/docs/getting-started.mdx +++ b/docs/docs/getting-started.mdx @@ -34,7 +34,7 @@ In your newly created virtual environement, run: pip install -U deepeval ``` -You can also keep track of all evaluation results by logging into our [in all one evaluation platform](https://confident-ai.com), and use Confident AI's [proprietary LLM evaluation agent](evaluation-metrics#judgementalgpt) for evaluation: +You can also keep track of all evaluation results by logging into our [in all one evaluation platform](https://confident-ai.com), and use Confident AI's [proprietary LLM evaluation agent](metrics-judgemental) for evaluation: ```console deepeval login @@ -60,7 +60,7 @@ def test_hallucination(): # Replace this with the actual output of your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - hallucination_metric = HallucinationMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(threshold=0.7) test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) assert_test(test_case, [hallucination_metric]) ``` @@ -74,8 +74,8 @@ deepeval test run test_example.py **Congratulations! Your test case should have passed ✅** Let's breakdown what happened. - The variable `input` mimics a user input, and `actual_output` is a placeholder for what your application's supposed to output based on this input. -- The variable `context` contains the relevant information from your knowledge base, and `HallucinationMetric(minimum_score=0.7)` is an default metric provided by DeepEval for you to evaluate how factually correct your application's output is based on the provided context. -- All metric scores range from 0 - 1, which the `minimum_score=0.7` threshold ultimately determines if your test have passed or not. +- The variable `context` contains the relevant information from your knowledge base, and `HallucinationMetric(threshold=0.7)` is an default metric provided by DeepEval for you to evaluate how factually correct your application's output is based on the provided context. +- All metric scores range from 0 - 1, which the `threshold=0.7` threshold ultimately determines if your test have passed or not. :::note `deepeval`'s default metrics are not evaluated using LLMs. Keep reading this tutorial to learn how to create an LLM based evaluation metric. @@ -112,7 +112,7 @@ def test_summarization(): name="Summarization", criteria="Summarization - determine if the actual output is an accurate and concise summarization of the input.", evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], - minimum_score=0.5 + threshold=0.5 ) test_case = LLMTestCase(input=input, actual_output=actual_output) assert_test(test_case, [summarization_metric]) @@ -130,10 +130,10 @@ from deepeval.metrics import BaseMetric class LengthMetric(BaseMetric): # This metric checks if the output length is greater than 10 characters def __init__(self, max_length: int=10): - self.minimum_score = max_length + self.threshold = max_length def measure(self, test_case: LLMTestCase): - self.success = len(test_case.actual_output) > self.minimum_score + self.success = len(test_case.actual_output) > self.threshold if self.success: score = 1 else: @@ -184,13 +184,13 @@ def test_everything(): # Replace this with the actual output of your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - hallucination_metric = HallucinationMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(threshold=0.7) length_metric = LengthMetric(max_length=10) summarization_metric = LLMEvalMetric( name="Summarization", criteria="Summarization - determine if the actual output is an accurate and concise summarization of the input.", evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], - minimum_score=0.5 + threshold=0.5 ) test_case = LLMTestCase(input=input, actual_output=actual_output, context=context) @@ -262,7 +262,7 @@ def test_hallucination(test_case: dict): # Replace this with the actual output of your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - hallucination_metric = HallucinationMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(threshold=0.7) test_case = LLMTestCase( input=input, actual_output=actual_output, diff --git a/docs/docs/metrics-answer-relevancy.mdx b/docs/docs/metrics-answer-relevancy.mdx index 7be738321..2dc996da7 100644 --- a/docs/docs/metrics-answer-relevancy.mdx +++ b/docs/docs/metrics-answer-relevancy.mdx @@ -4,7 +4,7 @@ title: Answer Relevancy sidebar_label: Answer Relevancy --- -Answer Relevancy measures how relevant the `actual_output` of your LLM application is compared to the provided `input`. You don't have to supply `context` or `expected_output` when creating an `LLMTestCase` if you're just evaluating answer relevancy. +The answer relevancy metric measures the quality of your RAG pipeline's generator by evaluating how relevant the `actual_output` of your LLM application is compared to the provided `input`. `deepeval`'s answer relevancy metric is a self-explaining LLM-Eval, meaning it outputs a reason for its metric score. ## Required Parameters @@ -27,7 +27,11 @@ actual_output = "We offer a 30-day full refund at no extra cost." # Replace this with the actual retrieved context from your RAG pipeline retrieval_context = ["All customers are eligible for a 30 day full refund at no extra cost."] -metric = AnswerRelevancyMetric(minimum_score=0.7) +metric = AnswerRelevancyMetric( + threshold=0.7, + model="gpt-4", + include_reason=True +) test_case = LLMTestCase( input="What if these shoes don't fit?", actual_output=actual_output, @@ -36,7 +40,14 @@ test_case = LLMTestCase( metric.measure(test_case) print(metric.score) +print(metric.reason) # or evaluate test cases in bulk evaluate([test_case], [metric]) ``` + +You can also choose to fallback to Ragas' answer relevancy metric (which has a similar implemention). This however is not capable of generating a reason. + +```python +from deepeval.metrics import RAGASAnswerRelevancyMetric +``` diff --git a/docs/docs/metrics-bias.mdx b/docs/docs/metrics-bias.mdx index 2c766ad6d..4cab96246 100644 --- a/docs/docs/metrics-bias.mdx +++ b/docs/docs/metrics-bias.mdx @@ -39,7 +39,7 @@ actual_output = "We offer a 30-day full refund at no extra cost." metric = UnBiasedMetric( evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT], - minimum_score=0.5 + threshold=0.5 ) test_case = LLMTestCase( input="What if these shoes don't fit?", diff --git a/docs/docs/metrics-contextual-precision.mdx b/docs/docs/metrics-contextual-precision.mdx index 78e1716b4..76ed83c42 100644 --- a/docs/docs/metrics-contextual-precision.mdx +++ b/docs/docs/metrics-contextual-precision.mdx @@ -4,7 +4,7 @@ title: Contextual Precision sidebar_label: Contextual Precision --- -The contextual precision metric determines whether more relevant retrieved contexts in your RAG pipeline are ranked higher than less relevant ones. It assesses your RAG pipeline's retriever and is calculated using `input` and `retrieval_context`. +The contextual precision metric measures your RAG pipeline's retriever by evaluating whether nodes in your `retrieval_context` that are relevant to the given `input` are ranked higher than irrelevant ones. `deepeval`'s contextual precision metric is a self-explaining LLM-Eval, meaning it outputs a reason for its metric score. ## Required Parameters @@ -27,7 +27,11 @@ actual_output = "We offer a 30-day full refund at no extra cost." # Replace this with the actual retrieved context from your RAG pipeline retrieval_context = ["All customers are eligible for a 30 day full refund at no extra cost."] -metric = ContextualPrecisionMetric(minimum_score=0.7, model="gpt-3.5-turbo") +metric = ContextualPrecisionMetric( + threshold=0.7, + model="gpt-4", + include_reason=True +) test_case = LLMTestCase( input="What if these shoes don't fit?", actual_output=actual_output, @@ -36,7 +40,14 @@ test_case = LLMTestCase( metric.measure(test_case) print(metric.score) +print(metric.reason) # or evaluate test cases in bulk evaluate([test_case], [metric]) ``` + +You can also choose to fallback to Ragas' contextual precision metric (which has a similar implemention). This however is not capable of generating a reason. + +```python +from deepeval.metrics import RAGASContextualPrecisionMetric +``` diff --git a/docs/docs/metrics-contextual-recall.mdx b/docs/docs/metrics-contextual-recall.mdx index e94ba792a..5be7ab128 100644 --- a/docs/docs/metrics-contextual-recall.mdx +++ b/docs/docs/metrics-contextual-recall.mdx @@ -4,7 +4,7 @@ title: Contextual Recall sidebar_label: Contextual Recall --- -The contextual recall metric determines the recall of the retrieved context using the annotated answer as a reference to evaluate the performance of your RAG pipeline's retriever. Calculated using `expected_output` and `retrieval_context`. +The contextual recall metric measures the quality of your RAG pipeline's retriever by evaluating the extent of which the `retrieval_context` aligns with the `expected_output`. `deepeval`'s contextual recall metric is a self-explaining LLM-Eval, meaning it outputs a reason for its metric score. ## Required Parameters @@ -31,7 +31,11 @@ expected_output = "You are eligible for a 30 day full refund at no extra cost." # Replace this with the actual retrieved context from your RAG pipeline retrieval_context = ["All customers are eligible for a 30 day full refund at no extra cost."] -metric = ContextualRecallMetric(minimum_score=0.7, model="gpt-3.5-turbo") +metric = ContextualRecallMetric( + threshold=0.7, + model="gpt-4", + include_reason=True +) test_case = LLMTestCase( input="What if these shoes don't fit?", actual_output=actual_output, @@ -41,7 +45,14 @@ test_case = LLMTestCase( metric.measure(test_case) print(metric.score) +print(metric.reason) # or evaluate test cases in bulk evaluate([test_case], [metric]) ``` + +You can also choose to fallback to Ragas' contextual recall metric (which has a similar implemention). This however is not capable of generating a reason. + +```python +from deepeval.metrics import RAGASContextualRecallMetric +``` diff --git a/docs/docs/metrics-contextual-relevancy.mdx b/docs/docs/metrics-contextual-relevancy.mdx index 0582a4cb1..4fa3604ba 100644 --- a/docs/docs/metrics-contextual-relevancy.mdx +++ b/docs/docs/metrics-contextual-relevancy.mdx @@ -4,7 +4,7 @@ title: Contextual Relevancy sidebar_label: Contextual Relevancy --- -The contextual relevancy metric assesses the relevance of the retrieved contexts to input, and penalizes redundant information. It evaluates the performance of your RAG pipeline's retriever and is calculated using `input` and `retrieval_context`. +The contextual relevancy metric measures the quality of your RAG pipeline's retriever by evaluating the overall relevance of the information presented in your `retrieval_context` for a given `input`. `deepeval`'s contextual relevancy metric is a self-explaining LLM-Eval, meaning it outputs a reason for its metric score. ## Required Parameters @@ -31,7 +31,11 @@ actual_output = "We offer a 30-day full refund at no extra cost." # Replace this with the actual retrieved context from your RAG pipeline retrieval_context = ["All customers are eligible for a 30 day full refund at no extra cost."] -metric = ContextualRelevancyMetric(minimum_score=0.7, model="gpt-3.5-turbo") +metric = ContextualRelevancyMetric( + threshold=0.7, + model="gpt-4", + include_reason=True +) test_case = LLMTestCase( input="What if these shoes don't fit?", actual_output=actual_output, @@ -40,7 +44,14 @@ test_case = LLMTestCase( metric.measure(test_case) print(metric.score) +print(metric.reason) # or evaluate test cases in bulk evaluate([test_case], [metric]) ``` + +You can also choose to fallback to Ragas' contextual relevancy metric (which has a similar implemention). This however is not capable of generating a reason. + +```python +from deepeval.metrics import RAGASContextualRelevancyMetric +``` diff --git a/docs/docs/metrics-custom.mdx b/docs/docs/metrics-custom.mdx index 7e24304c9..1621deed6 100644 --- a/docs/docs/metrics-custom.mdx +++ b/docs/docs/metrics-custom.mdx @@ -24,40 +24,53 @@ from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCase # Inherit BaseMetric -class LengthMetric(BaseMetric): - # This metric checks if the output length is greater than 10 characters - def __init__(self, max_length: int=10): - self.minimum_score = max_length +class LatencyMetric(BaseMetric): + # This metric by default checks if the latency is greater than 10 seconds + def __init__(self, max_seconds: int=10): + self.threshold = max_seconds def measure(self, test_case: LLMTestCase): # Set self.success and self.score in the "measure" method - self.success = len(test_case.actual_output) > self.minimum_score + self.success = test_case.execution_time <= self.threshold if self.success: self.score = 1 else: self.score = 0 - # You can also set a reason for the score returned. + # You can also optionally set a reason for the score returned. # This is particularly useful for a score computed using LLMs - self.reason = "..." + self.reason = "Too slow!" return self.score def is_successful(self): return self.success @property - def name(self): - return "Length" + def __name__(self): + return "Latency" ``` Notice that a few things has happened: -- `self.minimum_score` was set in `__init__()` +- `self.threshold` was set in `__init__()`, and this can be either a minimum or maximum threshold - `self.success`, `self.score`, and `self.reason` was set in `measure()` - `measure()` takes in an `LLMTestCase` -- `self.is_successful()` simply returns the success status -- `name()` simply returns a string representing the metric name +- `is_successful()` simply returns the success status +- `__name()__` simply returns a string representing the metric name -To create a custom metric without unexpected errors, we recommend you set the appropriate class variables in the appropriate methods as outlined above. +To create a custom metric without unexpected errors, we recommend you set the appropriate class variables in the appropriate methods as outlined above. You should also note that `self.reason` is **optional**. `self.reason` should be a string representing the rationale behind an LLM computed score. This is only applicable if you're using LLMs as an evaluator in the `measure()` method, and has implemented a way to generate a score reasoning. -You should also note that `self.reason` is **optional**. `self.reason` should be a string representing the rationale behind an LLM computed score. This is only applicable if you're using LLMs as an evaluator in the `measure()` method, and has implemented a way to generate a score reasoning. +After creating a custom metric, you can use it in the same way as all of `deepeval`'s metrics: + +```python +from deepeval import evaluate +from deepeval.test_case import LLMTestCase + +... + + +# Note that we pass in execution time since the measure method requires it for evaluation +latency_metric = LatencyMetric(max_seconds=10.0) +test_case = LLMTestCase(input="...", actual_output="...", execution_time=8.3) +evaluate([test_case], [latency_metric]) +``` diff --git a/docs/docs/metrics-faithfulness.mdx b/docs/docs/metrics-faithfulness.mdx index 45cbb4ca0..efed3eb5f 100644 --- a/docs/docs/metrics-faithfulness.mdx +++ b/docs/docs/metrics-faithfulness.mdx @@ -4,7 +4,7 @@ title: Faithfulness sidebar_label: Faithfulness --- -The faithfulness metric measures hallucination in a RAG pipeline to ensure output aligns with the retrieved context. It evaluates the quality of your RAG pipeline's generator and is calculated using `actual_output` and `retrieval_context`. +The faithfulness metric measures the quality of your RAG pipeline's generator by evaluating whether the `actual_output` factually aligns with the contents of your `retrieval_context`. `deepeval`'s faithfulness metric is a self-explaining LLM-Eval, meaning it outputs a reason for its metric score. :::info Although similar to the `HallucinationMetric`, the faithfulness metric in `deepeval` is more concerned with hallucination in RAG pipelines, rather than the actual LLM itself. @@ -31,7 +31,11 @@ actual_output = "We offer a 30-day full refund at no extra cost." # Replace this with the actual retrieved context from your RAG pipeline retrieval_context = ["All customers are eligible for a 30 day full refund at no extra cost."] -metric = FaithfulnessMetric(minimum_score=0.5, model="gpt-3.5-turbo") +metric = FaithfulnessMetric( + threshold=0.7, + model="gpt-4", + include_reason=True +) test_case = LLMTestCase( input="What if these shoes don't fit?", actual_output=actual_output, @@ -40,7 +44,14 @@ test_case = LLMTestCase( metric.measure(test_case) print(metric.score) +print(metric.reason) # or evaluate test cases in bulk evaluate([test_case], [metric]) ``` + +You can also choose to fallback to Ragas' faithfulness metric (which has a similar implemention). This however is not capable of generating a reason. + +```python +from deepeval.metrics import RAGASFaithfulnessMetric +``` diff --git a/docs/docs/metrics-hallucination.mdx b/docs/docs/metrics-hallucination.mdx index 531ec6126..8ed90e46a 100644 --- a/docs/docs/metrics-hallucination.mdx +++ b/docs/docs/metrics-hallucination.mdx @@ -6,6 +6,10 @@ sidebar_label: Hallucination The hallucination metric determines whether your LLM generates factually correct information by comparing the `actual_output` to the provided `context`. +:::info +If you're looking to evaluate hallucination for a RAG system, please refer to the [faithfulness metric](metrics-faithfulness) instead. +::: + ## Required Parameters To use the `HallucinationMetric`, you'll have to provide the following parameters when creating an `LLMTestCase`: @@ -36,7 +40,7 @@ test_case = LLMTestCase( actual_output=actual_output, context=context ) -metric = HallucinationMetric(minimum_score=0.5) +metric = HallucinationMetric(threshold=0.5) metric.measure(test_case) print(metric.score) diff --git a/docs/docs/metrics-introduction.mdx b/docs/docs/metrics-introduction.mdx index 562a6c27e..40eaf9115 100644 --- a/docs/docs/metrics-introduction.mdx +++ b/docs/docs/metrics-introduction.mdx @@ -6,31 +6,74 @@ sidebar_label: Introduction ## Quick Summary -In `deepeval`, a metric serves as a standard of measurement for evaluating the performance of an LLM output based on a specific criteria of interest. Essentially, while the metric acts as the ruler, a test case represents the thing you're trying to measure. `deepeval` offers a range of default metrics for you to quickly get started with, which includes: +In `deepeval`, a metric serves as a standard of measurement for evaluating the performance of an LLM output based on a specific criteria of interest. Essentially, while the metric acts as the ruler, a test case represents the thing you're trying to measure. `deepeval` offers a range of default metrics for you to quickly get started with, such as: - Hallucination +- Summarization +- Faithfulness - Answer Relevancy +- Contextual Relevancy +- Contextual Precision +- Contextual Recall - Ragas - Toxicity - Bias -`deepeval` also offers you a straightforward way to develop your own custom LLM-based evaluation metrics. This is noteworthy because all default metrics in `deepeval` are derived from traditional NLP models, not LLMs. All metrics are measured on a test case. Visit the [test cases section](evaluation-test-cases) to learn how to apply any metric on test cases for evaluation. +`deepeval` also offers you a straightforward way to develop your own custom evaluation metrics. All metrics are measured on a test case. Visit the [test cases section](evaluation-test-cases) to learn how to apply any metric on test cases for evaluation. ## Types of Metrics -A **_custom_** metric is a type of metric you can easily create by implementing abstract methods and properties of base classes provided by `deepeval`. They are extremely versitle and seamlessly integrate with Confident AI without requiring any additional setup. As you'll see later, a custom metric can either be an **_LLM evaluated_** or **_classic_** metric. A classic metric is a type of metric whose criteria isn't evaluated using an LLM. +A **_custom_** metric is a type of metric you can easily create by implementing abstract methods and properties of base classes provided by `deepeval`. They are extremely versitle and seamlessly integrate with Confident AI without requiring any additional setup. As you'll see later, a custom metric can either be an **_LLM-Eval_** (LLM evaluated) or **_classic_** metric. A classic metric is a type of metric whose criteria isn't evaluated using an LLM. -`deepeval` also offer **_default_** metrics. All default metrics offered by `deepeval` are classic metrics. This means all default metrics in `deepeval` does not use LLMs for evaluation. This is delibrate for two main reasons: +`deepeval` also offer **_default_** metrics. Most default metrics offered by `deepeval` are LLM-Evals, which means they are evaluated using LLMs. This is delibrate because LLM-Evals are versitle in nature and better aligns with human expectations when compared to traditional model based approaches. -- LLM evaluated metrics are versitle in nature and it's better if you create one using `deepeval`'s build-ins -- Classic metrics are much harder to compute and requires extensive research +`deepeval`'s LLM-Evals are a step up to other implementations because they: -All of `deepeval`'s default metrics output a score between 0-1, and require a `minimum_score` argument to instantiate. A default metric is only successful if the evaluation score is equal to or greater than `minimum_score`. +- are extra reliable as LLMs are only used for extremely specific tasks during evaluation to greatly reduce stochasticity and flakiness in scores. +- provide a comprehensive reason for the scores computed. + +All of `deepeval`'s default metrics output a score between 0-1, and require a `threshold` argument to instantiate. A default metric is only successful if the evaluation score is equal to or greater than `threshold`. :::info -All GPT models from OpenAI are available for metrics that use LLMs for evaluation. You can switch between models by providing a string corresponding to OpenAI's model names via the `model` argument. +All GPT models from OpenAI are available for LLM-Evals (metrics that use LLMs for evaluation). You can switch between models by providing a string corresponding to OpenAI's model names via the optional `model` argument when instantiating an LLM-Eval. +::: + +## Using OpenAI + +To use OpenAI for `deepeval`'s LLM-Evals (metrics evaluated using an LLM), supply your `OPENAI_API_KEY` in the CLI: + +```console +export OPENAI_API_KEY= +``` + +Alternatively, if you're working in a notebook enviornment (Jupyter or Colab), set your `OPENAI_API_KEY` in a cell: + +```console + %env OPENAI_API_KEY= +``` + +:::note +Please **do not include** quotation marks when setting your `OPENAI_API_KEY` if you're working in a notebook enviornment. ::: +## Using Azure OpenAI + +`deepeval` also allows you to use Azure OpenAI for metrics that are evaluated using an LLM. Run the following command in the CLI to configure your `deepeval` enviornment to use Azure OpenAI for **all** LLM-based metrics. + +```console +deepeval set-azure-openai --openai-endpoint= \ + --openai-api-key= \ + --deployment-name= \ + --openai-api-version= \ + --model-version= +``` + +Note that the `model-version` is **optional**. If you ever wish to stop using Azure OpenAI and move back to regular OpenAI, simply run: + +```console +deepeval unset-azure-openai +``` + ## Measuring a Metric All metrics in `deepeval`, including [custom metrics that you create](metrics-custom): @@ -39,19 +82,25 @@ All metrics in `deepeval`, including [custom metrics that you create](metrics-cu - can have its score accessed via `metric.score` - can have its status accessed via `metric.is_successful()` - can be used to evaluate test cases or entire datasets, with or without Pytest. -- has a `minimum_score` that acts as the threshold for success. `metric.is_successful()` is only true if `metric.score` >= `minimum_score`. +- has a `threshold` that acts as the threshold for success. `metric.is_successful()` is only true if `metric.score` >= `threshold`. + +In additional, most LLM-Evals in `deepeval` offers a reason for its score, which can be accessed via `metric.reason`. Here's a quick example. +```console +export OPENAI_API_KEY= +``` + ```python -from deepeval.metrics import HallucinationMetric +from deepeval.metrics import AnswerRelevancyMetric from deepeval.test_case import LLMTestCase # Initialize a test case test_case = LLMTestCase(input="...", actual_output="...") -# Initialize metric with minimum_score -metric = HallucinationMetric(minimum_score=0.5) +# Initialize metric with threshold +metric = AnswerRelevancyMetric(threshold=0.5) ``` Using this metric, you can either evaluate a test case using `deepeval test run`: @@ -60,7 +109,7 @@ Using this metric, you can either evaluate a test case using `deepeval test run` from deepeval import evaluate ... -def test_hallucination(): +def test_answer_relevancy(): assert_test(test_case, metric) ``` @@ -68,7 +117,7 @@ def test_hallucination(): deepeval test run test_file.py ``` -The `evaluate` function: +Or using the `evaluate` function: ```python from deepeval import assert_test @@ -84,24 +133,7 @@ Or execute the metric directly and get its score: metric.measure(test_case) print(metric.score) +print(metric.reason) ``` For more details on how a metric evaluates a test case, refer to the [test cases section.](evaluation-test-cases#assert-test-cases) - -## Using Azure OpenAI - -`deepeval` also allows you to use Azure OpenAI for metrics that are evaluated using an LLM. Run the following command in the CLI to configure your `deepeval` enviornment to use Azure OpenAI for **all** LLM-based metrics. - -```console -deepeval azure-openai set --openai-endpoint= \ - --openai-api-key= \ - --deployment-name= \ - --openai-api-version= \ - --model-version= -``` - -Note that the `model-version` is **optional**. If you ever wish to stop using Azure OpenAI and move back to regular OpenAI, simply run: - -```console -deepeval azure-openai unset -``` diff --git a/docs/docs/metrics-judgemental.mdx b/docs/docs/metrics-judgemental.mdx index 2a1ea074c..cee908375 100644 --- a/docs/docs/metrics-judgemental.mdx +++ b/docs/docs/metrics-judgemental.mdx @@ -1,7 +1,7 @@ --- id: metrics-judgemental -title: Judgemental GPT -sidebar_label: Judgemental GPT +title: JudgementalGPT +sidebar_label: JudgementalGPT --- `JudgementalGPT` is an LLM agent developed in-house by [Confident AI](https://confident-ai.com) that's dedicated to evaluation and is superior to `LLMEvalMetric`. While it operates similarly to `LLMEvalMetric` by utilizing LLMs for scoring, it: @@ -39,7 +39,7 @@ code_correctness_metric = JudgementalGPT( criteria="Code Correctness - determine whether the code in the 'actual output' produces a valid JSON.", evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT], language=Languages.SPANISH, - minimum_score=0.5, + threshold=0.5, ) ``` @@ -49,7 +49,7 @@ Under the hood, `JudgementalGPT` sends a request to Confident AI's servers that - `criteria`: a description outlining the specific evaluation aspects for each test case. - `evaluation_params`: a list of type `LLMTestCaseParams`. Include only the parameters that are relevant for evaluation. - [Optional] `language`: type `Language`, specifies what language to return the reasoning in. -- [Optional] `minimum_score`: the passing threshold, defaulted to 0.5. +- [Optional] `threshold`: the passing threshold, defaulted to 0.5. Similar to `LLMEvalMetric`, you can access the judgemental `score` and `reason` for `JudgementalGPT`: diff --git a/docs/docs/metrics-llm-evals.mdx b/docs/docs/metrics-llm-evals.mdx index 814f035eb..a7cdaff7c 100644 --- a/docs/docs/metrics-llm-evals.mdx +++ b/docs/docs/metrics-llm-evals.mdx @@ -38,14 +38,14 @@ There are three mandatory and two optional parameters required when instantiatin - `criteria`: a description outlining the specific evaluation aspects for each test case. - `evaluation_params`: a list of type `LLMTestCaseParams`. Include only the parameters that are relevant for evaluation. - [Optional] `evaluation_steps`: a list of strings outlining the exact steps the LLM should take for evaluation. You can only provide either `evaluation_steps` or `criteria`, and not both. -- [Optional] `minimum_score`: the passing threshold, defaulted to 0.5. +- [Optional] `threshold`: the passing threshold, defaulted to 0.5. - [Optional] `model`: the model name. This is defaulted to 'gpt-4-1106-preview' and we currently only support models from (Azure) OpenAI. :::danger For accurate and valid results, only the parameters that are mentioned in `criteria` should be included as a member of `evaluation_params`. ::: -As mentioned in the [metrics introduction section](metrics-introduction), all of `deepeval`'s metrics return a score ranging from 0 - 1, and a metric is only successful if the evaluation score is equal to or greater than `minimum_score`. An `LLMEvalMetric` is no exception. You can access the `score` and `reason` for each individual `LLMEvalMetric`: +As mentioned in the [metrics introduction section](metrics-introduction), all of `deepeval`'s metrics return a score ranging from 0 - 1, and a metric is only successful if the evaluation score is equal to or greater than `threshold`. An `LLMEvalMetric` is no exception. You can access the `score` and `reason` for each individual `LLMEvalMetric`: ```python from deepeval.test_case import LLMTestCase diff --git a/docs/docs/metrics-ragas.mdx b/docs/docs/metrics-ragas.mdx index 70cd0a11a..1fad689d7 100644 --- a/docs/docs/metrics-ragas.mdx +++ b/docs/docs/metrics-ragas.mdx @@ -6,10 +6,10 @@ sidebar_label: RAGAS The RAGAS metric is the harmonic mean of four distinct metrics: -- `AnswerRelevancyMetric` -- `FaithfulnessMetric` -- `ContextualPrecisionMetric` -- `ContextualRecallMetric` +- `RAGASAnswerRelevancyMetric` +- `RAGASFaithfulnessMetric` +- `RAGASContextualPrecisionMetric` +- `RAGASContextualRecallMetric` It provides a score to holistically evaluate of your RAG pipeline's generator and retriever. @@ -42,7 +42,7 @@ expected_output = "You are eligible for a 30 day full refund at no extra cost." # Replace this with the actual retrieved context from your RAG pipeline retrieval_context = ["All customers are eligible for a 30 day full refund at no extra cost."] -metric = RagasMetric(minimum_score=0.5, model="gpt-3.5-turbo") +metric = RagasMetric(threshold=0.5, model="gpt-3.5-turbo") test_case = LLMTestCase( input="What if these shoes don't fit?", actual_output=actual_output, diff --git a/docs/docs/metrics-summarization.mdx b/docs/docs/metrics-summarization.mdx index 98b0de7e6..9b06deca9 100644 --- a/docs/docs/metrics-summarization.mdx +++ b/docs/docs/metrics-summarization.mdx @@ -46,7 +46,7 @@ from deepeval.test_case import LLMTestCase test_case = LLMTestCase(input=input, actual_output=actual_output) metric = SummarizationMetric( - minimum_score=0.5, + threshold=0.5, model="gpt-4", assessment_questions=[ "Is the inclusion score based on a percentage of 'yes' answers?", @@ -64,7 +64,7 @@ evaluate([test_case], [metric]) There are five optional parameters when instantiating an `SummarizationMetric` class: -- [Optional] `minimum_score`: the passing threshold, defaulted to 0.5. +- [Optional] `threshold`: the passing threshold, defaulted to 0.5. - [Optional] `model`: the model name. This is defaulted to 'gpt-4-1106-preview' and we currently only support models from (Azure) OpenAI. - [Optional] `assessment_questions`: a list of **close-ended questions that can be answered with either a 'yes' or a 'no'**. These are questions you want your summary to be able to ideally answer, and is especially helpful if you already know what a good summary for your use case looks like. If `assessment_questions` is not provided, we will generate a set of `assessment_questions` for you at evaluation time. The `assessment_questions` are used to calculate the `inclusion_score`. - [Optional] `n`: the number of questions to generate when calculating the `alignment_score` and `inclusion_score`, defaulted to 5. @@ -76,7 +76,7 @@ In `deepeval`, we judge summarization by taking the minimum of the two distinct - `alignment_score`: determines whether the summary contains hallucinated or contradictory information to the original text. - `inclusion_score`: determines whether the summary contains the neccessary information from the original text. -These scores are calculated by generating `n` closed-ended questions that can only be answered with either a 'yes or a 'no', and calculating the ratio of which the original text and summary yields the same answer. +These scores are calculated by generating `n` closed-ended questions that can only be answered with either a 'yes or a 'no', and calculating the ratio of which the original text and summary yields the same answer. [Here is a great article](https://www.confident-ai.com/blog/a-step-by-step-guide-to-evaluating-an-llm-text-summarization-task) on how `deepeval`'s summarization metric was build. You can access the `alignment_score` and `inclusion_score` from a `SummarizationMetric` as follows: diff --git a/docs/docs/metrics-toxicity.mdx b/docs/docs/metrics-toxicity.mdx index 1604d30e7..cc44893ce 100644 --- a/docs/docs/metrics-toxicity.mdx +++ b/docs/docs/metrics-toxicity.mdx @@ -6,6 +6,14 @@ sidebar_label: Toxicity The toxicity metric is another **referenceless** metric that evaluates toxicness in your LLM's outputs. This is particularly useful for a fine-tuning use case. +## Installation + +Toxicity in `deepeval` requires an additional installation: + +``` +pip install detoxify +``` + ## Required Parameters To use the `NonToxicMetric`, you'll have to provide the following parameters when creating an `LLMTestCase`: @@ -25,9 +33,9 @@ from deepeval.test_case import LLMTestCase, LLMTestCaseParams # Replace this with the actual output from your LLM application actual_output = "We offer a 30-day full refund at no extra cost." -metric = UnBiasedMetric( +metric = NonToxicMetric( evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], - minimum_score=0.5 + threshold=0.5 ) test_case = LLMTestCase( input="What if these shoes don't fit?", diff --git a/examples/getting_started/test_example.py b/examples/getting_started/test_example.py index d910a2526..ff66655f5 100644 --- a/examples/getting_started/test_example.py +++ b/examples/getting_started/test_example.py @@ -15,7 +15,7 @@ def test_hallucination(): # Replace this with the actual output from your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - hallucination_metric = HallucinationMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(threshold=0.7) test_case = LLMTestCase( input=input, actual_output=actual_output, context=context ) @@ -35,7 +35,7 @@ def test_summarization(): LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT, ], - minimum_score=0.5, + threshold=0.5, ) test_case = LLMTestCase(input=input, actual_output=actual_output) assert_test(test_case, [summarization_metric]) @@ -44,10 +44,10 @@ def test_summarization(): class LengthMetric(BaseMetric): # This metric checks if the output length is greater than 10 characters def __init__(self, max_length: int = 10): - self.minimum_score = max_length + self.threshold = max_length def measure(self, test_case: LLMTestCase): - self.success = len(test_case.actual_output) > self.minimum_score + self.success = len(test_case.actual_output) > self.threshold if self.success: self.score = 1 else: @@ -80,7 +80,7 @@ def test_everything(): # Replace this with the actual output from your LLM application actual_output = "We offer a 30-day full refund at no extra cost." - hallucination_metric = HallucinationMetric(minimum_score=0.7) + hallucination_metric = HallucinationMetric(threshold=0.7) length_metric = LengthMetric(max_length=10) summarization_metric = LLMEvalMetric( name="Summarization", @@ -89,7 +89,7 @@ def test_everything(): LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT, ], - minimum_score=0.5, + threshold=0.5, ) test_case = LLMTestCase( diff --git a/examples/tracing/test_chatbot.py b/examples/tracing/test_chatbot.py index 62177cf5d..c8a55ee8b 100644 --- a/examples/tracing/test_chatbot.py +++ b/examples/tracing/test_chatbot.py @@ -83,7 +83,7 @@ def test_hallucination(): ] input = "What are the requimrents to be president?" - metric = HallucinationMetric(minimum_score=0.8) + metric = HallucinationMetric(threshold=0.8) test_case = LLMTestCase( input=input, actual_output=chatbot.query(user_input=input), diff --git a/llama_test/chatbot.py b/llama_test/chatbot.py new file mode 100644 index 000000000..a07724b5e --- /dev/null +++ b/llama_test/chatbot.py @@ -0,0 +1,14 @@ +from llama_index import VectorStoreIndex, SimpleDirectoryReader +from llama_index import ServiceContext +import llama_index + +llama_index.set_global_handler("deepeval") + +service_context = ServiceContext.from_defaults(chunk_size=1000) +documents = SimpleDirectoryReader("data").load_data() +index = VectorStoreIndex.from_documents(documents) +query_engine = index.as_query_engine(similarity_top_k=5) + + +def query(user_input): + return query_engine.query(user_input).response diff --git a/llama_test/data/data.txt b/llama_test/data/data.txt new file mode 100644 index 000000000..c2bd37022 --- /dev/null +++ b/llama_test/data/data.txt @@ -0,0 +1,25 @@ +About MadeUpCompany +MadeUpCompany is a pioneering technology firm founded in 2010, specializing in cloud computing, data analytics, and machine learning. Our headquarters is based in San Francisco, California, with satellite offices spread across New York, London, and Tokyo. We are committed to offering state-of-the-art solutions that help businesses and individuals achieve their full potential. With a diverse team of experts from various industries, we strive to redefine the boundaries of innovation and efficiency. + +Products and Services +We offer a suite of services ranging from cloud storage solutions, data analytics platforms, to custom machine learning models tailored for specific business needs. Our most popular product is CloudMate, a cloud storage solution designed for businesses of all sizes. It offers seamless data migration, top-tier security protocols, and an easy-to-use interface. Our data analytics service, DataWiz, helps companies turn raw data into actionable insights using advanced algorithms. + +Pricing +We have a variety of pricing options tailored to different needs. Our basic cloud storage package starts at $9.99 per month, with premium plans offering more storage and functionalities. We also provide enterprise solutions on a case-by-case basis, so it’s best to consult with our sales team for customized pricing. + +Technical Support +Our customer support team is available 24/7 to assist with any technical issues. We offer multiple channels for support including live chat, email, and a toll-free number. Most issues are typically resolved within 24 hours. We also have an extensive FAQ section on our website and a community forum for peer support. + +Security and Compliance +MadeUpCompany places the utmost importance on security and compliance. All our products are GDPR compliant and adhere to the highest security standards, including end-to-end encryption and multi-factor authentication. + +Account Management +Customers can easily manage their accounts through our online portal, which allows you to upgrade your service, view billing history, and manage users in your organization. If you encounter any issues or have questions about your account, our account management team is available weekdays from 9 AM to 6 PM. + +Refund and Cancellation Policy +We offer a 30-day money-back guarantee on all our products. If you're not satisfied for any reason, you can request a full refund within the first 30 days of your purchase. After that, you can still cancel your service at any time, but a prorated refund will be issued based on the remaining term of your subscription. + +Upcoming Features +We’re constantly working to improve our services and offer new features. Keep an eye out for updates on machine learning functionalities in DataWiz and more collaborative tools in CloudMate in the upcoming quarters. + +Your customer support staff can use these paragraphs to build their responses to customer inquiries, providing both detailed and precise information to address various questions. \ No newline at end of file diff --git a/llama_test/main.py b/llama_test/main.py new file mode 100644 index 000000000..9ddcfa104 --- /dev/null +++ b/llama_test/main.py @@ -0,0 +1,6 @@ +from chatbot import query + +while True: + user_input = input("Enter your question: ") + response = query(user_input) + print("Bot response:", response) diff --git a/poetry.lock b/poetry.lock index badd5ac49..ac4bcea85 100644 --- a/poetry.lock +++ b/poetry.lock @@ -167,21 +167,40 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "beautifulsoup4" +version = "4.12.2" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] [[package]] name = "bert-score" @@ -450,63 +469,63 @@ test-no-images = ["pytest", "pytest-cov", "pytest-xdist", "wurlitzer"] [[package]] name = "coverage" -version = "7.3.4" +version = "7.4.0" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:aff2bd3d585969cc4486bfc69655e862028b689404563e6b549e6a8244f226df"}, - {file = "coverage-7.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4353923f38d752ecfbd3f1f20bf7a3546993ae5ecd7c07fd2f25d40b4e54571"}, - {file = "coverage-7.3.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea473c37872f0159294f7073f3fa72f68b03a129799f3533b2bb44d5e9fa4f82"}, - {file = "coverage-7.3.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5214362abf26e254d749fc0c18af4c57b532a4bfde1a057565616dd3b8d7cc94"}, - {file = "coverage-7.3.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f99b7d3f7a7adfa3d11e3a48d1a91bb65739555dd6a0d3fa68aa5852d962e5b1"}, - {file = "coverage-7.3.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:74397a1263275bea9d736572d4cf338efaade2de9ff759f9c26bcdceb383bb49"}, - {file = "coverage-7.3.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f154bd866318185ef5865ace5be3ac047b6d1cc0aeecf53bf83fe846f4384d5d"}, - {file = "coverage-7.3.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e0d84099ea7cba9ff467f9c6f747e3fc3906e2aadac1ce7b41add72e8d0a3712"}, - {file = "coverage-7.3.4-cp310-cp310-win32.whl", hash = "sha256:3f477fb8a56e0c603587b8278d9dbd32e54bcc2922d62405f65574bd76eba78a"}, - {file = "coverage-7.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:c75738ce13d257efbb6633a049fb2ed8e87e2e6c2e906c52d1093a4d08d67c6b"}, - {file = "coverage-7.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:997aa14b3e014339d8101b9886063c5d06238848905d9ad6c6eabe533440a9a7"}, - {file = "coverage-7.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8a9c5bc5db3eb4cd55ecb8397d8e9b70247904f8eca718cc53c12dcc98e59fc8"}, - {file = "coverage-7.3.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27ee94f088397d1feea3cb524e4313ff0410ead7d968029ecc4bc5a7e1d34fbf"}, - {file = "coverage-7.3.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ce03e25e18dd9bf44723e83bc202114817f3367789052dc9e5b5c79f40cf59d"}, - {file = "coverage-7.3.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85072e99474d894e5df582faec04abe137b28972d5e466999bc64fc37f564a03"}, - {file = "coverage-7.3.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a877810ef918d0d345b783fc569608804f3ed2507bf32f14f652e4eaf5d8f8d0"}, - {file = "coverage-7.3.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9ac17b94ab4ca66cf803f2b22d47e392f0977f9da838bf71d1f0db6c32893cb9"}, - {file = "coverage-7.3.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:36d75ef2acab74dc948d0b537ef021306796da551e8ac8b467810911000af66a"}, - {file = "coverage-7.3.4-cp311-cp311-win32.whl", hash = "sha256:47ee56c2cd445ea35a8cc3ad5c8134cb9bece3a5cb50bb8265514208d0a65928"}, - {file = "coverage-7.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:11ab62d0ce5d9324915726f611f511a761efcca970bd49d876cf831b4de65be5"}, - {file = "coverage-7.3.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:33e63c578f4acce1b6cd292a66bc30164495010f1091d4b7529d014845cd9bee"}, - {file = "coverage-7.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:782693b817218169bfeb9b9ba7f4a9f242764e180ac9589b45112571f32a0ba6"}, - {file = "coverage-7.3.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c4277ddaad9293454da19121c59f2d850f16bcb27f71f89a5c4836906eb35ef"}, - {file = "coverage-7.3.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d892a19ae24b9801771a5a989fb3e850bd1ad2e2b6e83e949c65e8f37bc67a1"}, - {file = "coverage-7.3.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3024ec1b3a221bd10b5d87337d0373c2bcaf7afd86d42081afe39b3e1820323b"}, - {file = "coverage-7.3.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a1c3e9d2bbd6f3f79cfecd6f20854f4dc0c6e0ec317df2b265266d0dc06535f1"}, - {file = "coverage-7.3.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e91029d7f151d8bf5ab7d8bfe2c3dbefd239759d642b211a677bc0709c9fdb96"}, - {file = "coverage-7.3.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:6879fe41c60080aa4bb59703a526c54e0412b77e649a0d06a61782ecf0853ee1"}, - {file = "coverage-7.3.4-cp312-cp312-win32.whl", hash = "sha256:fd2f8a641f8f193968afdc8fd1697e602e199931012b574194052d132a79be13"}, - {file = "coverage-7.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:d1d0ce6c6947a3a4aa5479bebceff2c807b9f3b529b637e2b33dea4468d75fc7"}, - {file = "coverage-7.3.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:36797b3625d1da885b369bdaaa3b0d9fb8865caed3c2b8230afaa6005434aa2f"}, - {file = "coverage-7.3.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bfed0ec4b419fbc807dec417c401499ea869436910e1ca524cfb4f81cf3f60e7"}, - {file = "coverage-7.3.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f97ff5a9fc2ca47f3383482858dd2cb8ddbf7514427eecf5aa5f7992d0571429"}, - {file = "coverage-7.3.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:607b6c6b35aa49defaebf4526729bd5238bc36fe3ef1a417d9839e1d96ee1e4c"}, - {file = "coverage-7.3.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8e258dcc335055ab59fe79f1dec217d9fb0cdace103d6b5c6df6b75915e7959"}, - {file = "coverage-7.3.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a02ac7c51819702b384fea5ee033a7c202f732a2a2f1fe6c41e3d4019828c8d3"}, - {file = "coverage-7.3.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b710869a15b8caf02e31d16487a931dbe78335462a122c8603bb9bd401ff6fb2"}, - {file = "coverage-7.3.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c6a23ae9348a7a92e7f750f9b7e828448e428e99c24616dec93a0720342f241d"}, - {file = "coverage-7.3.4-cp38-cp38-win32.whl", hash = "sha256:758ebaf74578b73f727acc4e8ab4b16ab6f22a5ffd7dd254e5946aba42a4ce76"}, - {file = "coverage-7.3.4-cp38-cp38-win_amd64.whl", hash = "sha256:309ed6a559bc942b7cc721f2976326efbfe81fc2b8f601c722bff927328507dc"}, - {file = "coverage-7.3.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aefbb29dc56317a4fcb2f3857d5bce9b881038ed7e5aa5d3bcab25bd23f57328"}, - {file = "coverage-7.3.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:183c16173a70caf92e2dfcfe7c7a576de6fa9edc4119b8e13f91db7ca33a7923"}, - {file = "coverage-7.3.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a4184dcbe4f98d86470273e758f1d24191ca095412e4335ff27b417291f5964"}, - {file = "coverage-7.3.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93698ac0995516ccdca55342599a1463ed2e2d8942316da31686d4d614597ef9"}, - {file = "coverage-7.3.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb220b3596358a86361139edce40d97da7458412d412e1e10c8e1970ee8c09ab"}, - {file = "coverage-7.3.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5b14abde6f8d969e6b9dd8c7a013d9a2b52af1235fe7bebef25ad5c8f47fa18"}, - {file = "coverage-7.3.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:610afaf929dc0e09a5eef6981edb6a57a46b7eceff151947b836d869d6d567c1"}, - {file = "coverage-7.3.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d6ed790728fb71e6b8247bd28e77e99d0c276dff952389b5388169b8ca7b1c28"}, - {file = "coverage-7.3.4-cp39-cp39-win32.whl", hash = "sha256:c15fdfb141fcf6a900e68bfa35689e1256a670db32b96e7a931cab4a0e1600e5"}, - {file = "coverage-7.3.4-cp39-cp39-win_amd64.whl", hash = "sha256:38d0b307c4d99a7aca4e00cad4311b7c51b7ac38fb7dea2abe0d182dd4008e05"}, - {file = "coverage-7.3.4-pp38.pp39.pp310-none-any.whl", hash = "sha256:b1e0f25ae99cf247abfb3f0fac7ae25739e4cd96bf1afa3537827c576b4847e5"}, - {file = "coverage-7.3.4.tar.gz", hash = "sha256:020d56d2da5bc22a0e00a5b0d54597ee91ad72446fa4cf1b97c35022f6b6dbf0"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, + {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, + {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, + {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, + {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, + {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, + {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, + {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, + {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, + {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, + {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, + {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, + {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, ] [package.extras] @@ -544,25 +563,27 @@ typing-inspect = ">=0.4.0,<1" [[package]] name = "datasets" -version = "2.14.4" +version = "2.16.1" description = "HuggingFace community-driven open-source library of datasets" optional = false python-versions = ">=3.8.0" files = [ - {file = "datasets-2.14.4-py3-none-any.whl", hash = "sha256:29336bd316a7d827ccd4da2236596279b20ca2ac78f64c04c9483da7cbc2459b"}, - {file = "datasets-2.14.4.tar.gz", hash = "sha256:ef29c2b5841de488cd343cfc26ab979bff77efa4d2285af51f1ad7db5c46a83b"}, + {file = "datasets-2.16.1-py3-none-any.whl", hash = "sha256:fafa300c78ff92d521473a3d47d60c2d3e0d6046212cc03ceb6caf6550737257"}, + {file = "datasets-2.16.1.tar.gz", hash = "sha256:ad3215e9b1984d1de4fda2123bc7319ccbdf1e17d0c3d5590d13debff308a080"}, ] [package.dependencies] aiohttp = "*" dill = ">=0.3.0,<0.3.8" -fsspec = {version = ">=2021.11.1", extras = ["http"]} -huggingface-hub = ">=0.14.0,<1.0.0" +filelock = "*" +fsspec = {version = ">=2023.1.0,<=2023.10.0", extras = ["http"]} +huggingface-hub = ">=0.19.4" multiprocess = "*" numpy = ">=1.17" packaging = "*" pandas = "*" pyarrow = ">=8.0.0" +pyarrow-hotfix = "*" pyyaml = ">=5.1" requests = ">=2.19.0" tqdm = ">=4.62.1" @@ -572,18 +593,35 @@ xxhash = "*" apache-beam = ["apache-beam (>=2.26.0,<2.44.0)"] audio = ["librosa", "soundfile (>=0.12.1)"] benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.1.5)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "torch (>=2.0.0)", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] docs = ["s3fs", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos", "torch", "transformers"] -jax = ["jax (>=0.2.8,!=0.3.2,<=0.3.25)", "jaxlib (>=0.1.65,<=0.3.25)"] +jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] -quality = ["black (>=23.1,<24.0)", "pyyaml (>=5.3.1)", "ruff (>=0.0.241)"] +quality = ["ruff (>=0.1.5)"] s3 = ["s3fs"] tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos"] tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch (>=2.0.0)", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] torch = ["torch"] vision = ["Pillow (>=6.2.1)"] +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + [[package]] name = "dill" version = "0.3.7" @@ -655,53 +693,53 @@ typing = ["typing-extensions (>=4.8)"] [[package]] name = "fonttools" -version = "4.47.0" +version = "4.47.2" description = "Tools to manipulate font files" optional = false python-versions = ">=3.8" files = [ - {file = "fonttools-4.47.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2d2404107626f97a221dc1a65b05396d2bb2ce38e435f64f26ed2369f68675d9"}, - {file = "fonttools-4.47.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c01f409be619a9a0f5590389e37ccb58b47264939f0e8d58bfa1f3ba07d22671"}, - {file = "fonttools-4.47.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d986b66ff722ef675b7ee22fbe5947a41f60a61a4da15579d5e276d897fbc7fa"}, - {file = "fonttools-4.47.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8acf6dd0434b211b3bd30d572d9e019831aae17a54016629fa8224783b22df8"}, - {file = "fonttools-4.47.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:495369c660e0c27233e3c572269cbe520f7f4978be675f990f4005937337d391"}, - {file = "fonttools-4.47.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c59227d7ba5b232281c26ae04fac2c73a79ad0e236bca5c44aae904a18f14faf"}, - {file = "fonttools-4.47.0-cp310-cp310-win32.whl", hash = "sha256:59a6c8b71a245800e923cb684a2dc0eac19c56493e2f896218fcf2571ed28984"}, - {file = "fonttools-4.47.0-cp310-cp310-win_amd64.whl", hash = "sha256:52c82df66201f3a90db438d9d7b337c7c98139de598d0728fb99dab9fd0495ca"}, - {file = "fonttools-4.47.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:854421e328d47d70aa5abceacbe8eef231961b162c71cbe7ff3f47e235e2e5c5"}, - {file = "fonttools-4.47.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:511482df31cfea9f697930f61520f6541185fa5eeba2fa760fe72e8eee5af88b"}, - {file = "fonttools-4.47.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0e2c88c8c985b7b9a7efcd06511fb0a1fe3ddd9a6cd2895ef1dbf9059719d7"}, - {file = "fonttools-4.47.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7a0a8848726956e9d9fb18c977a279013daadf0cbb6725d2015a6dd57527992"}, - {file = "fonttools-4.47.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e869da810ae35afb3019baa0d0306cdbab4760a54909c89ad8904fa629991812"}, - {file = "fonttools-4.47.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dd23848f877c3754f53a4903fb7a593ed100924f9b4bff7d5a4e2e8a7001ae11"}, - {file = "fonttools-4.47.0-cp311-cp311-win32.whl", hash = "sha256:bf1810635c00f7c45d93085611c995fc130009cec5abdc35b327156aa191f982"}, - {file = "fonttools-4.47.0-cp311-cp311-win_amd64.whl", hash = "sha256:61df4dee5d38ab65b26da8efd62d859a1eef7a34dcbc331299a28e24d04c59a7"}, - {file = "fonttools-4.47.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e3f4d61f3a8195eac784f1d0c16c0a3105382c1b9a74d99ac4ba421da39a8826"}, - {file = "fonttools-4.47.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:174995f7b057e799355b393e97f4f93ef1f2197cbfa945e988d49b2a09ecbce8"}, - {file = "fonttools-4.47.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea592e6a09b71cb7a7661dd93ac0b877a6228e2d677ebacbad0a4d118494c86d"}, - {file = "fonttools-4.47.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40bdbe90b33897d9cc4a39f8e415b0fcdeae4c40a99374b8a4982f127ff5c767"}, - {file = "fonttools-4.47.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:843509ae9b93db5aaf1a6302085e30bddc1111d31e11d724584818f5b698f500"}, - {file = "fonttools-4.47.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9acfa1cdc479e0dde528b61423855913d949a7f7fe09e276228298fef4589540"}, - {file = "fonttools-4.47.0-cp312-cp312-win32.whl", hash = "sha256:66c92ec7f95fd9732550ebedefcd190a8d81beaa97e89d523a0d17198a8bda4d"}, - {file = "fonttools-4.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:e8fa20748de55d0021f83754b371432dca0439e02847962fc4c42a0e444c2d78"}, - {file = "fonttools-4.47.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c75e19971209fbbce891ebfd1b10c37320a5a28e8d438861c21d35305aedb81c"}, - {file = "fonttools-4.47.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e79f1a3970d25f692bbb8c8c2637e621a66c0d60c109ab48d4a160f50856deff"}, - {file = "fonttools-4.47.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:562681188c62c024fe2c611b32e08b8de2afa00c0c4e72bed47c47c318e16d5c"}, - {file = "fonttools-4.47.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a77a60315c33393b2bd29d538d1ef026060a63d3a49a9233b779261bad9c3f71"}, - {file = "fonttools-4.47.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b4fabb8cc9422efae1a925160083fdcbab8fdc96a8483441eb7457235df625bd"}, - {file = "fonttools-4.47.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2a78dba8c2a1e9d53a0fb5382979f024200dc86adc46a56cbb668a2249862fda"}, - {file = "fonttools-4.47.0-cp38-cp38-win32.whl", hash = "sha256:e6b968543fde4119231c12c2a953dcf83349590ca631ba8216a8edf9cd4d36a9"}, - {file = "fonttools-4.47.0-cp38-cp38-win_amd64.whl", hash = "sha256:4a9a51745c0439516d947480d4d884fa18bd1458e05b829e482b9269afa655bc"}, - {file = "fonttools-4.47.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:62d8ddb058b8e87018e5dc26f3258e2c30daad4c87262dfeb0e2617dd84750e6"}, - {file = "fonttools-4.47.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5dde0eab40faaa5476133123f6a622a1cc3ac9b7af45d65690870620323308b4"}, - {file = "fonttools-4.47.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4da089f6dfdb822293bde576916492cd708c37c2501c3651adde39804630538"}, - {file = "fonttools-4.47.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:253bb46bab970e8aae254cebf2ae3db98a4ef6bd034707aa68a239027d2b198d"}, - {file = "fonttools-4.47.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1193fb090061efa2f9e2d8d743ae9850c77b66746a3b32792324cdce65784154"}, - {file = "fonttools-4.47.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:084511482dd265bce6dca24c509894062f0117e4e6869384d853f46c0e6d43be"}, - {file = "fonttools-4.47.0-cp39-cp39-win32.whl", hash = "sha256:97620c4af36e4c849e52661492e31dc36916df12571cb900d16960ab8e92a980"}, - {file = "fonttools-4.47.0-cp39-cp39-win_amd64.whl", hash = "sha256:e77bdf52185bdaf63d39f3e1ac3212e6cfa3ab07d509b94557a8902ce9c13c82"}, - {file = "fonttools-4.47.0-py3-none-any.whl", hash = "sha256:d6477ba902dd2d7adda7f0fd3bfaeb92885d45993c9e1928c9f28fc3961415f7"}, - {file = "fonttools-4.47.0.tar.gz", hash = "sha256:ec13a10715eef0e031858c1c23bfaee6cba02b97558e4a7bfa089dba4a8c2ebf"}, + {file = "fonttools-4.47.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b629108351d25512d4ea1a8393a2dba325b7b7d7308116b605ea3f8e1be88df"}, + {file = "fonttools-4.47.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c19044256c44fe299d9a73456aabee4b4d06c6b930287be93b533b4737d70aa1"}, + {file = "fonttools-4.47.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8be28c036b9f186e8c7eaf8a11b42373e7e4949f9e9f370202b9da4c4c3f56c"}, + {file = "fonttools-4.47.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f83a4daef6d2a202acb9bf572958f91cfde5b10c8ee7fb1d09a4c81e5d851fd8"}, + {file = "fonttools-4.47.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a5a5318ba5365d992666ac4fe35365f93004109d18858a3e18ae46f67907670"}, + {file = "fonttools-4.47.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8f57ecd742545362a0f7186774b2d1c53423ed9ece67689c93a1055b236f638c"}, + {file = "fonttools-4.47.2-cp310-cp310-win32.whl", hash = "sha256:a1c154bb85dc9a4cf145250c88d112d88eb414bad81d4cb524d06258dea1bdc0"}, + {file = "fonttools-4.47.2-cp310-cp310-win_amd64.whl", hash = "sha256:3e2b95dce2ead58fb12524d0ca7d63a63459dd489e7e5838c3cd53557f8933e1"}, + {file = "fonttools-4.47.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:29495d6d109cdbabe73cfb6f419ce67080c3ef9ea1e08d5750240fd4b0c4763b"}, + {file = "fonttools-4.47.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0a1d313a415eaaba2b35d6cd33536560deeebd2ed758b9bfb89ab5d97dc5deac"}, + {file = "fonttools-4.47.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90f898cdd67f52f18049250a6474185ef6544c91f27a7bee70d87d77a8daf89c"}, + {file = "fonttools-4.47.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3480eeb52770ff75140fe7d9a2ec33fb67b07efea0ab5129c7e0c6a639c40c70"}, + {file = "fonttools-4.47.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0255dbc128fee75fb9be364806b940ed450dd6838672a150d501ee86523ac61e"}, + {file = "fonttools-4.47.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f791446ff297fd5f1e2247c188de53c1bfb9dd7f0549eba55b73a3c2087a2703"}, + {file = "fonttools-4.47.2-cp311-cp311-win32.whl", hash = "sha256:740947906590a878a4bde7dd748e85fefa4d470a268b964748403b3ab2aeed6c"}, + {file = "fonttools-4.47.2-cp311-cp311-win_amd64.whl", hash = "sha256:63fbed184979f09a65aa9c88b395ca539c94287ba3a364517698462e13e457c9"}, + {file = "fonttools-4.47.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4ec558c543609e71b2275c4894e93493f65d2f41c15fe1d089080c1d0bb4d635"}, + {file = "fonttools-4.47.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e040f905d542362e07e72e03612a6270c33d38281fd573160e1003e43718d68d"}, + {file = "fonttools-4.47.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6dd58cc03016b281bd2c74c84cdaa6bd3ce54c5a7f47478b7657b930ac3ed8eb"}, + {file = "fonttools-4.47.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32ab2e9702dff0dd4510c7bb958f265a8d3dd5c0e2547e7b5f7a3df4979abb07"}, + {file = "fonttools-4.47.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a808f3c1d1df1f5bf39be869b6e0c263570cdafb5bdb2df66087733f566ea71"}, + {file = "fonttools-4.47.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac71e2e201df041a2891067dc36256755b1229ae167edbdc419b16da78732c2f"}, + {file = "fonttools-4.47.2-cp312-cp312-win32.whl", hash = "sha256:69731e8bea0578b3c28fdb43dbf95b9386e2d49a399e9a4ad736b8e479b08085"}, + {file = "fonttools-4.47.2-cp312-cp312-win_amd64.whl", hash = "sha256:b3e1304e5f19ca861d86a72218ecce68f391646d85c851742d265787f55457a4"}, + {file = "fonttools-4.47.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:254d9a6f7be00212bf0c3159e0a420eb19c63793b2c05e049eb337f3023c5ecc"}, + {file = "fonttools-4.47.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eabae77a07c41ae0b35184894202305c3ad211a93b2eb53837c2a1143c8bc952"}, + {file = "fonttools-4.47.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86a5ab2873ed2575d0fcdf1828143cfc6b977ac448e3dc616bb1e3d20efbafa"}, + {file = "fonttools-4.47.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13819db8445a0cec8c3ff5f243af6418ab19175072a9a92f6cc8ca7d1452754b"}, + {file = "fonttools-4.47.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4e743935139aa485fe3253fc33fe467eab6ea42583fa681223ea3f1a93dd01e6"}, + {file = "fonttools-4.47.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d49ce3ea7b7173faebc5664872243b40cf88814ca3eb135c4a3cdff66af71946"}, + {file = "fonttools-4.47.2-cp38-cp38-win32.whl", hash = "sha256:94208ea750e3f96e267f394d5588579bb64cc628e321dbb1d4243ffbc291b18b"}, + {file = "fonttools-4.47.2-cp38-cp38-win_amd64.whl", hash = "sha256:0f750037e02beb8b3569fbff701a572e62a685d2a0e840d75816592280e5feae"}, + {file = "fonttools-4.47.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3d71606c9321f6701642bd4746f99b6089e53d7e9817fc6b964e90d9c5f0ecc6"}, + {file = "fonttools-4.47.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:86e0427864c6c91cf77f16d1fb9bf1bbf7453e824589e8fb8461b6ee1144f506"}, + {file = "fonttools-4.47.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a00bd0e68e88987dcc047ea31c26d40a3c61185153b03457956a87e39d43c37"}, + {file = "fonttools-4.47.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5d77479fb885ef38a16a253a2f4096bc3d14e63a56d6246bfdb56365a12b20c"}, + {file = "fonttools-4.47.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5465df494f20a7d01712b072ae3ee9ad2887004701b95cb2cc6dcb9c2c97a899"}, + {file = "fonttools-4.47.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4c811d3c73b6abac275babb8aa439206288f56fdb2c6f8835e3d7b70de8937a7"}, + {file = "fonttools-4.47.2-cp39-cp39-win32.whl", hash = "sha256:5b60e3afa9635e3dfd3ace2757039593e3bd3cf128be0ddb7a1ff4ac45fa5a50"}, + {file = "fonttools-4.47.2-cp39-cp39-win_amd64.whl", hash = "sha256:7ee48bd9d6b7e8f66866c9090807e3a4a56cf43ffad48962725a190e0dd774c8"}, + {file = "fonttools-4.47.2-py3-none-any.whl", hash = "sha256:7eb7ad665258fba68fd22228a09f347469d95a97fb88198e133595947a20a184"}, + {file = "fonttools-4.47.2.tar.gz", hash = "sha256:7df26dd3650e98ca45f1e29883c96a0b9f5bb6af8d632a6a108bc744fa0bd9b3"}, ] [package.extras] @@ -806,13 +844,13 @@ files = [ [[package]] name = "fsspec" -version = "2023.12.2" +version = "2023.10.0" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, - {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, + {file = "fsspec-2023.10.0-py3-none-any.whl", hash = "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"}, + {file = "fsspec-2023.10.0.tar.gz", hash = "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5"}, ] [package.dependencies] @@ -972,13 +1010,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "huggingface-hub" -version = "0.20.1" +version = "0.20.2" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.20.1-py3-none-any.whl", hash = "sha256:ecfdea395a8bc68cd160106c5bd857f7e010768d95f9e1862a779010cc304831"}, - {file = "huggingface_hub-0.20.1.tar.gz", hash = "sha256:8c88c4c3c8853e22f2dfb4d84c3d493f4e1af52fb3856a90e1eeddcf191ddbb1"}, + {file = "huggingface_hub-0.20.2-py3-none-any.whl", hash = "sha256:53752eda2239d30a470c307a61cf9adcf136bc77b0a734338c7d04941af560d8"}, + {file = "huggingface_hub-0.20.2.tar.gz", hash = "sha256:215c5fceff631030c7a3d19ba7b588921c908b3f21eef31d160ebc245b200ff6"}, ] [package.dependencies] @@ -1026,13 +1064,13 @@ files = [ [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -1192,13 +1230,13 @@ files = [ [[package]] name = "langchain" -version = "0.0.352" +version = "0.1.0" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain-0.0.352-py3-none-any.whl", hash = "sha256:43ab580e1223e5d7c3495b3c0cb79e2f3a0ecb52caf8126271fb10d42cede2d0"}, - {file = "langchain-0.0.352.tar.gz", hash = "sha256:8928d7b63d73af9681fe1b2a2b99b84238efef61ed537de666160fd001f41efd"}, + {file = "langchain-0.1.0-py3-none-any.whl", hash = "sha256:8652e74b039333a55c79faff4400b077ba1bd0ddce5255574e42d301c05c1733"}, + {file = "langchain-0.1.0.tar.gz", hash = "sha256:d43119f8d3fda2c8ddf8c3a19bd5b94b347e27d1867ff14a921b90bdbed0668a"}, ] [package.dependencies] @@ -1206,9 +1244,9 @@ aiohttp = ">=3.8.3,<4.0.0" async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\""} dataclasses-json = ">=0.5.7,<0.7" jsonpatch = ">=1.33,<2.0" -langchain-community = ">=0.0.2,<0.1" -langchain-core = ">=0.1,<0.2" -langsmith = ">=0.0.70,<0.1.0" +langchain-community = ">=0.0.9,<0.1" +langchain-core = ">=0.1.7,<0.2" +langsmith = ">=0.0.77,<0.1.0" numpy = ">=1,<2" pydantic = ">=1,<3" PyYAML = ">=5.3" @@ -1223,7 +1261,7 @@ cli = ["typer (>=0.9.0,<0.10.0)"] cohere = ["cohere (>=4,<5)"] docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"] embeddings = ["sentence-transformers (>=2,<3)"] -extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "couchbase (>=4.1.9,<5.0.0)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] +extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "couchbase (>=4.1.9,<5.0.0)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "langchain-openai (>=0.0.2,<0.1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] javascript = ["esprima (>=4.0.1,<5.0.0)"] llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (<2)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"] openai = ["openai (<2)", "tiktoken (>=0.3.2,<0.6.0)"] @@ -1232,19 +1270,19 @@ text-helpers = ["chardet (>=5.1.0,<6.0.0)"] [[package]] name = "langchain-community" -version = "0.0.6" +version = "0.0.11" description = "Community contributed LangChain integrations." optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain_community-0.0.6-py3-none-any.whl", hash = "sha256:13b16da0f89c328df456911ff03069e4d919f647c7dd3bfc5062525cf956ed82"}, - {file = "langchain_community-0.0.6.tar.gz", hash = "sha256:b7deb63fd8205d54b51cf8b1702de15d1da77987f8465c356b158a65adff378c"}, + {file = "langchain_community-0.0.11-py3-none-any.whl", hash = "sha256:30ab1d7dbf35d0ebe684d8a1e8964e8dedd3d31a3703790436b39674cfa06f41"}, + {file = "langchain_community-0.0.11.tar.gz", hash = "sha256:eaeaa8d63427ecf0cb32fe2f1ba4d05ad6d5ef9f7019baf21dc2dde5b1403002"}, ] [package.dependencies] aiohttp = ">=3.8.3,<4.0.0" dataclasses-json = ">=0.5.7,<0.7" -langchain-core = ">=0.1,<0.2" +langchain-core = ">=0.1.8,<0.2" langsmith = ">=0.0.63,<0.1.0" numpy = ">=1,<2" PyYAML = ">=5.3" @@ -1254,17 +1292,17 @@ tenacity = ">=8.1.0,<9.0.0" [package.extras] cli = ["typer (>=0.9.0,<0.10.0)"] -extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "gradientai (>=1.4.0,<2.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "oracle-ads (>=2.9.1,<3.0.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] +extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "azure-ai-documentintelligence (>=1.0.0b1,<2.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "gradientai (>=1.4.0,<2.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "oracle-ads (>=2.9.1,<3.0.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)", "zhipuai (>=1.0.7,<2.0.0)"] [[package]] name = "langchain-core" -version = "0.1.3" +version = "0.1.10" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain_core-0.1.3-py3-none-any.whl", hash = "sha256:bfbbc5dfeb06cfe3fd078e7a12db3a4cfb9d28b715b200a64f7abb7ae1976b17"}, - {file = "langchain_core-0.1.3.tar.gz", hash = "sha256:d8898254dfea1c4ab614f470db40909969604775f7524175f6d9167ea58050c9"}, + {file = "langchain_core-0.1.10-py3-none-any.whl", hash = "sha256:d89952f6d0766cfc88d9f1e25b84d56f8d7bd63a45ad8ec1a9a038c9b49df16d"}, + {file = "langchain_core-0.1.10.tar.gz", hash = "sha256:3c9e1383264c102fcc6f865700dbb9416c4931a25d0ac2195f6311c6b867aa17"}, ] [package.dependencies] @@ -1280,21 +1318,76 @@ tenacity = ">=8.1.0,<9.0.0" [package.extras] extended-testing = ["jinja2 (>=3,<4)"] +[[package]] +name = "langchain-openai" +version = "0.0.2" +description = "An integration package connecting OpenAI and LangChain" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "langchain_openai-0.0.2-py3-none-any.whl", hash = "sha256:0a46067be13ce95a029fdca339cd1034a61be1a727786178fbad702668a060f9"}, + {file = "langchain_openai-0.0.2.tar.gz", hash = "sha256:713af4a638f65b3af2f741a9d61991011c31939b070d81ede5b2e3cba625e01a"}, +] + +[package.dependencies] +langchain-core = ">=0.1.7,<0.2" +numpy = ">=1,<2" +openai = ">=1.6.1,<2.0.0" +tiktoken = ">=0.5.2,<0.6.0" + [[package]] name = "langsmith" -version = "0.0.75" +version = "0.0.79" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langsmith-0.0.75-py3-none-any.whl", hash = "sha256:3e008854204c5eaae007f34c7e249059218605689c385c037f6a40cac044833b"}, - {file = "langsmith-0.0.75.tar.gz", hash = "sha256:3fd44c58bd53cb9366af3de129c7f11b6947914f1bb598a585240df0e2c566eb"}, + {file = "langsmith-0.0.79-py3-none-any.whl", hash = "sha256:be0374e913c36d9f6a13dd6b6e20a506066d5a0f3abfd476f9cf9e0b086ed744"}, + {file = "langsmith-0.0.79.tar.gz", hash = "sha256:d32639ccd18a92533b302f6f482255619afc8eb007fff91e37ee699d947c5e29"}, ] [package.dependencies] pydantic = ">=1,<3" requests = ">=2,<3" +[[package]] +name = "llama-index" +version = "0.9.29" +description = "Interface between LLMs and your data" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "llama_index-0.9.29-py3-none-any.whl", hash = "sha256:4ac6f589a2e8c5049882c185f25e6c57b68cf9d6e46a6f84102705a7dd357dd2"}, + {file = "llama_index-0.9.29.tar.gz", hash = "sha256:df5af84bf593cdf6a36da403f521bd5582eccbd6ca551da44d2e4e5b2b4f5619"}, +] + +[package.dependencies] +aiohttp = ">=3.8.6,<4.0.0" +beautifulsoup4 = ">=4.12.2,<5.0.0" +dataclasses-json = "*" +deprecated = ">=1.2.9.3" +fsspec = ">=2023.5.0" +httpx = "*" +nest-asyncio = ">=1.5.8,<2.0.0" +networkx = ">=3.0" +nltk = ">=3.8.1,<4.0.0" +numpy = "*" +openai = ">=1.1.0" +pandas = "*" +requests = ">=2.31.0" +SQLAlchemy = {version = ">=1.4.49", extras = ["asyncio"]} +tenacity = ">=8.2.0,<9.0.0" +tiktoken = ">=0.3.3" +typing-extensions = ">=4.5.0" +typing-inspect = ">=0.8.0" + +[package.extras] +gradientai = ["gradientai (>=1.4.0)"] +langchain = ["langchain (>=0.0.303)"] +local-models = ["optimum[onnxruntime] (>=1.13.2,<2.0.0)", "sentencepiece (>=0.1.99,<0.2.0)", "transformers[torch] (>=4.33.1,<5.0.0)"] +postgres = ["asyncpg (>=0.28.0,<0.29.0)", "pgvector (>=0.1.0,<0.2.0)", "psycopg-binary (>=3.1.12,<4.0.0)", "psycopg2 (>=2.9.9,<3.0.0)"] +query-tools = ["guidance (>=0.0.64,<0.0.65)", "jsonpath-ng (>=1.6.0,<2.0.0)", "lm-format-enforcer (>=0.4.3,<0.5.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "scikit-learn", "spacy (>=3.7.1,<4.0.0)"] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -1390,22 +1483,22 @@ files = [ [[package]] name = "marshmallow" -version = "3.20.1" +version = "3.20.2" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.8" files = [ - {file = "marshmallow-3.20.1-py3-none-any.whl", hash = "sha256:684939db93e80ad3561392f47be0230743131560a41c5110684c16e21ade0a5c"}, - {file = "marshmallow-3.20.1.tar.gz", hash = "sha256:5d2371bbe42000f2b3fb5eaa065224df7d8f8597bc19a1bbfa5bfe7fba8da889"}, + {file = "marshmallow-3.20.2-py3-none-any.whl", hash = "sha256:c21d4b98fee747c130e6bc8f45c4b3199ea66bc00c12ee1f639f0aeca034d5e9"}, + {file = "marshmallow-3.20.2.tar.gz", hash = "sha256:4c1daff273513dc5eb24b219a8035559dc573c8f322558ef85f5438ddd1236dd"}, ] [package.dependencies] packaging = ">=17.0" [package.extras] -dev = ["flake8 (==6.0.0)", "flake8-bugbear (==23.7.10)", "mypy (==1.4.1)", "pre-commit (>=2.4,<4.0)", "pytest", "pytz", "simplejson", "tox"] -docs = ["alabaster (==0.7.13)", "autodocsumm (==0.2.11)", "sphinx (==7.0.1)", "sphinx-issues (==3.0.1)", "sphinx-version-warning (==1.1.2)"] -lint = ["flake8 (==6.0.0)", "flake8-bugbear (==23.7.10)", "mypy (==1.4.1)", "pre-commit (>=2.4,<4.0)"] +dev = ["pre-commit (>=2.4,<4.0)", "pytest", "pytz", "simplejson", "tox"] +docs = ["alabaster (==0.7.15)", "autodocsumm (==0.2.12)", "sphinx (==7.2.6)", "sphinx-issues (==3.0.1)", "sphinx-version-warning (==1.1.2)"] +lint = ["pre-commit (>=2.4,<4.0)"] tests = ["pytest", "pytz", "simplejson"] [[package]] @@ -1662,47 +1755,47 @@ twitter = ["twython"] [[package]] name = "numpy" -version = "1.26.2" +version = "1.26.3" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "numpy-1.26.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3703fc9258a4a122d17043e57b35e5ef1c5a5837c3db8be396c82e04c1cf9b0f"}, - {file = "numpy-1.26.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc392fdcbd21d4be6ae1bb4475a03ce3b025cd49a9be5345d76d7585aea69440"}, - {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36340109af8da8805d8851ef1d74761b3b88e81a9bd80b290bbfed61bd2b4f75"}, - {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc008217145b3d77abd3e4d5ef586e3bdfba8fe17940769f8aa09b99e856c00"}, - {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ced40d4e9e18242f70dd02d739e44698df3dcb010d31f495ff00a31ef6014fe"}, - {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b272d4cecc32c9e19911891446b72e986157e6a1809b7b56518b4f3755267523"}, - {file = "numpy-1.26.2-cp310-cp310-win32.whl", hash = "sha256:22f8fc02fdbc829e7a8c578dd8d2e15a9074b630d4da29cda483337e300e3ee9"}, - {file = "numpy-1.26.2-cp310-cp310-win_amd64.whl", hash = "sha256:26c9d33f8e8b846d5a65dd068c14e04018d05533b348d9eaeef6c1bd787f9919"}, - {file = "numpy-1.26.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b96e7b9c624ef3ae2ae0e04fa9b460f6b9f17ad8b4bec6d7756510f1f6c0c841"}, - {file = "numpy-1.26.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa18428111fb9a591d7a9cc1b48150097ba6a7e8299fb56bdf574df650e7d1f1"}, - {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a"}, - {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96ca5482c3dbdd051bcd1fce8034603d6ebfc125a7bd59f55b40d8f5d246832b"}, - {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:854ab91a2906ef29dc3925a064fcd365c7b4da743f84b123002f6139bcb3f8a7"}, - {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f43740ab089277d403aa07567be138fc2a89d4d9892d113b76153e0e412409f8"}, - {file = "numpy-1.26.2-cp311-cp311-win32.whl", hash = "sha256:a2bbc29fcb1771cd7b7425f98b05307776a6baf43035d3b80c4b0f29e9545186"}, - {file = "numpy-1.26.2-cp311-cp311-win_amd64.whl", hash = "sha256:2b3fca8a5b00184828d12b073af4d0fc5fdd94b1632c2477526f6bd7842d700d"}, - {file = "numpy-1.26.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a4cd6ed4a339c21f1d1b0fdf13426cb3b284555c27ac2f156dfdaaa7e16bfab0"}, - {file = "numpy-1.26.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d5244aabd6ed7f312268b9247be47343a654ebea52a60f002dc70c769048e75"}, - {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3cdb4d9c70e6b8c0814239ead47da00934666f668426fc6e94cce869e13fd7"}, - {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa317b2325f7aa0a9471663e6093c210cb2ae9c0ad824732b307d2c51983d5b6"}, - {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:174a8880739c16c925799c018f3f55b8130c1f7c8e75ab0a6fa9d41cab092fd6"}, - {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f79b231bf5c16b1f39c7f4875e1ded36abee1591e98742b05d8a0fb55d8a3eec"}, - {file = "numpy-1.26.2-cp312-cp312-win32.whl", hash = "sha256:4a06263321dfd3598cacb252f51e521a8cb4b6df471bb12a7ee5cbab20ea9167"}, - {file = "numpy-1.26.2-cp312-cp312-win_amd64.whl", hash = "sha256:b04f5dc6b3efdaab541f7857351aac359e6ae3c126e2edb376929bd3b7f92d7e"}, - {file = "numpy-1.26.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4eb8df4bf8d3d90d091e0146f6c28492b0be84da3e409ebef54349f71ed271ef"}, - {file = "numpy-1.26.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1a13860fdcd95de7cf58bd6f8bc5a5ef81c0b0625eb2c9a783948847abbef2c2"}, - {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64308ebc366a8ed63fd0bf426b6a9468060962f1a4339ab1074c228fa6ade8e3"}, - {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baf8aab04a2c0e859da118f0b38617e5ee65d75b83795055fb66c0d5e9e9b818"}, - {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d73a3abcac238250091b11caef9ad12413dab01669511779bc9b29261dd50210"}, - {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b361d369fc7e5e1714cf827b731ca32bff8d411212fccd29ad98ad622449cc36"}, - {file = "numpy-1.26.2-cp39-cp39-win32.whl", hash = "sha256:bd3f0091e845164a20bd5a326860c840fe2af79fa12e0469a12768a3ec578d80"}, - {file = "numpy-1.26.2-cp39-cp39-win_amd64.whl", hash = "sha256:2beef57fb031dcc0dc8fa4fe297a742027b954949cabb52a2a376c144e5e6060"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1cc3d5029a30fb5f06704ad6b23b35e11309491c999838c31f124fee32107c79"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94cc3c222bb9fb5a12e334d0479b97bb2df446fbe622b470928f5284ffca3f8d"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841"}, - {file = "numpy-1.26.2.tar.gz", hash = "sha256:f65738447676ab5777f11e6bbbdb8ce11b785e105f690bc45966574816b6d3ea"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, + {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, + {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, + {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, + {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, + {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, + {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, + {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, + {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, + {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, ] [[package]] @@ -1848,13 +1941,13 @@ files = [ [[package]] name = "openai" -version = "1.6.1" +version = "1.7.1" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.6.1-py3-none-any.whl", hash = "sha256:bc9f774838d67ac29fb24cdeb2d58faf57de8b311085dcd1348f7aa02a96c7ee"}, - {file = "openai-1.6.1.tar.gz", hash = "sha256:d553ca9dbf9486b08e75b09e8671e4f638462aaadccfced632bf490fc3d75fa2"}, + {file = "openai-1.7.1-py3-none-any.whl", hash = "sha256:e52ad7ea015331edc584e6e9c98741c819d7ffbbd2ecc50bf1f55c33f9cb3f77"}, + {file = "openai-1.7.1.tar.gz", hash = "sha256:7556e6aa30e20254b1ad68de49bb5ef4d8106bfac5e8a78abdc1daa911fbb1fb"}, ] [package.dependencies] @@ -1960,70 +2053,88 @@ files = [ [[package]] name = "pillow" -version = "10.1.0" +version = "10.2.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.8" files = [ - {file = "Pillow-10.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106"}, - {file = "Pillow-10.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db"}, - {file = "Pillow-10.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f"}, - {file = "Pillow-10.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818"}, - {file = "Pillow-10.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57"}, - {file = "Pillow-10.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7"}, - {file = "Pillow-10.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061"}, - {file = "Pillow-10.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262"}, - {file = "Pillow-10.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992"}, - {file = "Pillow-10.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a"}, - {file = "Pillow-10.1.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b"}, - {file = "Pillow-10.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651"}, - {file = "Pillow-10.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b"}, - {file = "Pillow-10.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f"}, - {file = "Pillow-10.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996"}, - {file = "Pillow-10.1.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793"}, - {file = "Pillow-10.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d"}, - {file = "Pillow-10.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80"}, - {file = "Pillow-10.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212"}, - {file = "Pillow-10.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14"}, - {file = "Pillow-10.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099"}, - {file = "Pillow-10.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd"}, - {file = "Pillow-10.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28"}, - {file = "Pillow-10.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2"}, - {file = "Pillow-10.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f"}, - {file = "Pillow-10.1.0.tar.gz", hash = "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0"}, + {file = "pillow-10.2.0-cp310-cp310-win32.whl", hash = "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023"}, + {file = "pillow-10.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72"}, + {file = "pillow-10.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757"}, + {file = "pillow-10.2.0-cp311-cp311-win32.whl", hash = "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068"}, + {file = "pillow-10.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56"}, + {file = "pillow-10.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb"}, + {file = "pillow-10.2.0-cp312-cp312-win32.whl", hash = "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f"}, + {file = "pillow-10.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9"}, + {file = "pillow-10.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe"}, + {file = "pillow-10.2.0-cp38-cp38-win32.whl", hash = "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e"}, + {file = "pillow-10.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591"}, + {file = "pillow-10.2.0-cp39-cp39-win32.whl", hash = "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516"}, + {file = "pillow-10.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8"}, + {file = "pillow-10.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6"}, + {file = "pillow-10.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"}, + {file = "pillow-10.2.0.tar.gz", hash = "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e"}, ] [package.extras] docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] [[package]] name = "platformdirs" @@ -2076,22 +2187,22 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "protobuf" -version = "4.25.1" +version = "4.25.2" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.1-cp310-abi3-win32.whl", hash = "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7"}, - {file = "protobuf-4.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b"}, - {file = "protobuf-4.25.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd"}, - {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb"}, - {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"}, - {file = "protobuf-4.25.1-cp38-cp38-win32.whl", hash = "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd"}, - {file = "protobuf-4.25.1-cp38-cp38-win_amd64.whl", hash = "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0"}, - {file = "protobuf-4.25.1-cp39-cp39-win32.whl", hash = "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510"}, - {file = "protobuf-4.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10"}, - {file = "protobuf-4.25.1-py3-none-any.whl", hash = "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6"}, - {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, ] [[package]] @@ -2142,6 +2253,17 @@ files = [ [package.dependencies] numpy = ">=1.16.6" +[[package]] +name = "pyarrow-hotfix" +version = "0.6" +description = "" +optional = false +python-versions = ">=3.5" +files = [ + {file = "pyarrow_hotfix-0.6-py3-none-any.whl", hash = "sha256:dcc9ae2d220dff0083be6a9aa8e0cdee5182ad358d4931fce825c545e5c89178"}, + {file = "pyarrow_hotfix-0.6.tar.gz", hash = "sha256:79d3e030f7ff890d408a100ac16d6f00b14d44a502d7897cd9fc3e3a534e9945"}, +] + [[package]] name = "pydantic" version = "2.5.3" @@ -2319,13 +2441,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.3" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, - {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] @@ -2965,13 +3087,13 @@ files = [ [[package]] name = "sentry-sdk" -version = "1.39.1" +version = "1.39.2" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.39.1.tar.gz", hash = "sha256:320a55cdf9da9097a0bead239c35b7e61f53660ef9878861824fd6d9b2eaf3b5"}, - {file = "sentry_sdk-1.39.1-py2.py3-none-any.whl", hash = "sha256:81b5b9ffdd1a374e9eb0c053b5d2012155db9cbe76393a8585677b753bd5fdc1"}, + {file = "sentry-sdk-1.39.2.tar.gz", hash = "sha256:24c83b0b41c887d33328a9166f5950dc37ad58f01c9f2fbff6b87a6f1094170c"}, + {file = "sentry_sdk-1.39.2-py2.py3-none-any.whl", hash = "sha256:acaf597b30258fc7663063b291aa99e58f3096e91fe1e6634f4b79f9c1943e8e"}, ] [package.dependencies] @@ -2997,7 +3119,7 @@ huey = ["huey (>=2)"] loguru = ["loguru (>=0.5)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] -pure-eval = ["asttokens", "executing", "pure-eval"] +pure-eval = ["asttokens", "executing", "pure_eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -3046,72 +3168,83 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "sqlalchemy" -version = "2.0.23" +version = "2.0.25" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.23-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:638c2c0b6b4661a4fd264f6fb804eccd392745c5887f9317feb64bb7cb03b3ea"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3b5036aa326dc2df50cba3c958e29b291a80f604b1afa4c8ce73e78e1c9f01d"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:787af80107fb691934a01889ca8f82a44adedbf5ef3d6ad7d0f0b9ac557e0c34"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c14eba45983d2f48f7546bb32b47937ee2cafae353646295f0e99f35b14286ab"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0666031df46b9badba9bed00092a1ffa3aa063a5e68fa244acd9f08070e936d3"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:89a01238fcb9a8af118eaad3ffcc5dedaacbd429dc6fdc43fe430d3a941ff965"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-win32.whl", hash = "sha256:cabafc7837b6cec61c0e1e5c6d14ef250b675fa9c3060ed8a7e38653bd732ff8"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-win_amd64.whl", hash = "sha256:87a3d6b53c39cd173990de2f5f4b83431d534a74f0e2f88bd16eabb5667e65c6"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d5578e6863eeb998980c212a39106ea139bdc0b3f73291b96e27c929c90cd8e1"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:62d9e964870ea5ade4bc870ac4004c456efe75fb50404c03c5fd61f8bc669a72"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c80c38bd2ea35b97cbf7c21aeb129dcbebbf344ee01a7141016ab7b851464f8e"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75eefe09e98043cff2fb8af9796e20747ae870c903dc61d41b0c2e55128f958d"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd45a5b6c68357578263d74daab6ff9439517f87da63442d244f9f23df56138d"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a86cb7063e2c9fb8e774f77fbf8475516d270a3e989da55fa05d08089d77f8c4"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-win32.whl", hash = "sha256:b41f5d65b54cdf4934ecede2f41b9c60c9f785620416e8e6c48349ab18643855"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-win_amd64.whl", hash = "sha256:9ca922f305d67605668e93991aaf2c12239c78207bca3b891cd51a4515c72e22"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0f7fb0c7527c41fa6fcae2be537ac137f636a41b4c5a4c58914541e2f436b45"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7c424983ab447dab126c39d3ce3be5bee95700783204a72549c3dceffe0fc8f4"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f508ba8f89e0a5ecdfd3761f82dda2a3d7b678a626967608f4273e0dba8f07ac"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6463aa765cf02b9247e38b35853923edbf2f6fd1963df88706bc1d02410a5577"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e599a51acf3cc4d31d1a0cf248d8f8d863b6386d2b6782c5074427ebb7803bda"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fd54601ef9cc455a0c61e5245f690c8a3ad67ddb03d3b91c361d076def0b4c60"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-win32.whl", hash = "sha256:42d0b0290a8fb0165ea2c2781ae66e95cca6e27a2fbe1016ff8db3112ac1e846"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-win_amd64.whl", hash = "sha256:227135ef1e48165f37590b8bfc44ed7ff4c074bf04dc8d6f8e7f1c14a94aa6ca"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:14aebfe28b99f24f8a4c1346c48bc3d63705b1f919a24c27471136d2f219f02d"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e983fa42164577d073778d06d2cc5d020322425a509a08119bdcee70ad856bf"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e0dc9031baa46ad0dd5a269cb7a92a73284d1309228be1d5935dac8fb3cae24"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5f94aeb99f43729960638e7468d4688f6efccb837a858b34574e01143cf11f89"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:63bfc3acc970776036f6d1d0e65faa7473be9f3135d37a463c5eba5efcdb24c8"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-win32.whl", hash = "sha256:f48ed89dd11c3c586f45e9eec1e437b355b3b6f6884ea4a4c3111a3358fd0c18"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-win_amd64.whl", hash = "sha256:1e018aba8363adb0599e745af245306cb8c46b9ad0a6fc0a86745b6ff7d940fc"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:64ac935a90bc479fee77f9463f298943b0e60005fe5de2aa654d9cdef46c54df"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c4722f3bc3c1c2fcc3702dbe0016ba31148dd6efcd2a2fd33c1b4897c6a19693"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af79c06825e2836de21439cb2a6ce22b2ca129bad74f359bddd173f39582bf5"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:683ef58ca8eea4747737a1c35c11372ffeb84578d3aab8f3e10b1d13d66f2bc4"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d4041ad05b35f1f4da481f6b811b4af2f29e83af253bf37c3c4582b2c68934ab"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aeb397de65a0a62f14c257f36a726945a7f7bb60253462e8602d9b97b5cbe204"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-win32.whl", hash = "sha256:42ede90148b73fe4ab4a089f3126b2cfae8cfefc955c8174d697bb46210c8306"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-win_amd64.whl", hash = "sha256:964971b52daab357d2c0875825e36584d58f536e920f2968df8d581054eada4b"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:616fe7bcff0a05098f64b4478b78ec2dfa03225c23734d83d6c169eb41a93e55"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0e680527245895aba86afbd5bef6c316831c02aa988d1aad83c47ffe92655e74"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9585b646ffb048c0250acc7dad92536591ffe35dba624bb8fd9b471e25212a35"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4895a63e2c271ffc7a81ea424b94060f7b3b03b4ea0cd58ab5bb676ed02f4221"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:cc1d21576f958c42d9aec68eba5c1a7d715e5fc07825a629015fe8e3b0657fb0"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:967c0b71156f793e6662dd839da54f884631755275ed71f1539c95bbada9aaab"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-win32.whl", hash = "sha256:0a8c6aa506893e25a04233bc721c6b6cf844bafd7250535abb56cb6cc1368884"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-win_amd64.whl", hash = "sha256:f3420d00d2cb42432c1d0e44540ae83185ccbbc67a6054dcc8ab5387add6620b"}, - {file = "SQLAlchemy-2.0.23-py3-none-any.whl", hash = "sha256:31952bbc527d633b9479f5f81e8b9dfada00b91d6baba021a869095f1a97006d"}, - {file = "SQLAlchemy-2.0.23.tar.gz", hash = "sha256:c1bda93cbbe4aa2aa0aa8655c5aeda505cd219ff3e8da91d1d329e143e4aff69"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-win32.whl", hash = "sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-win_amd64.whl", hash = "sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-win32.whl", hash = "sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-win32.whl", hash = "sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-win_amd64.whl", hash = "sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-win32.whl", hash = "sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-win_amd64.whl", hash = "sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-win32.whl", hash = "sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-win_amd64.whl", hash = "sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-win32.whl", hash = "sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-win_amd64.whl", hash = "sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7"}, + {file = "SQLAlchemy-2.0.25-py3-none-any.whl", hash = "sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3"}, + {file = "SQLAlchemy-2.0.25.tar.gz", hash = "sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08"}, ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} -typing-extensions = ">=4.2.0" +greenlet = {version = "!=0.4.17", optional = true, markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\" or extra == \"asyncio\""} +typing-extensions = ">=4.6.0" [package.extras] aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] @@ -3121,7 +3254,7 @@ mssql-pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)"] mysql = ["mysqlclient (>=1.4.0)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=8)"] +oracle = ["cx_oracle (>=8)"] oracle-oracledb = ["oracledb (>=1.0.1)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] @@ -3131,7 +3264,7 @@ postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] pymysql = ["pymysql"] -sqlcipher = ["sqlcipher3-binary"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sympy" @@ -3617,13 +3750,13 @@ typing-extensions = ">=3.7.4" [[package]] name = "tzdata" -version = "2023.3" +version = "2023.4" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, - {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] @@ -3656,6 +3789,85 @@ files = [ [package.extras] test = ["pytest (>=6.0.0)", "setuptools (>=65)"] +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + [[package]] name = "xxhash" version = "3.4.1" @@ -3879,4 +4091,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "999515af5757ca9c0f111d90fe0bcb416dd62e1f9fc04bfa6183a5a87c49b10e" +content-hash = "1acc3ce5c08e0d3af483964b46a783bab32b37eb09f762b55d46826514210252" diff --git a/pyproject.toml b/pyproject.toml index e521dc652..0dc7e540d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "deepeval" -version = "0.20.42" +version = "0.20.46" description = "The Evaluation Framework for LLMs" authors = ["Jeffrey Ip "] license = "Apache-2.0" @@ -37,10 +37,13 @@ black = "*" portalocker = "*" openai = "*" langchain = "*" +langchain_openai = "*" protobuf = "*" typer = "*" setuptools = "*" wheel = "*" +aiohttp = "*" +llama-index = "*" [tool.black] line-length = 80 diff --git a/setup.py b/setup.py index ab5d4584d..135a26e56 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ "pytest-xdist", "portalocker", "langchain", + "langchain_openai", "rouge_score==0.1.2", "nltk==3.8.1", "ragas", diff --git a/tests/test_answer_relevancy.py b/tests/test_answer_relevancy.py index 8d828cc82..2863bf091 100644 --- a/tests/test_answer_relevancy.py +++ b/tests/test_answer_relevancy.py @@ -47,7 +47,7 @@ @pytest.mark.skip(reason="openai is expensive") def test_answer_relevancy(): - metric = AnswerRelevancyMetric(minimum_score=0.5) + metric = AnswerRelevancyMetric(threshold=0.5) test_case = LLMTestCase( input=question, actual_output=answer, diff --git a/tests/test_contextual_precision.py b/tests/test_contextual_precision.py new file mode 100644 index 000000000..0916f167a --- /dev/null +++ b/tests/test_contextual_precision.py @@ -0,0 +1,71 @@ +import pytest +from deepeval.test_case import LLMTestCase +from deepeval.metrics import ContextualPrecisionMetric +from deepeval import assert_test + +question = "What are the primary benefits of meditation?" +answer = """ +Meditation offers a rich tapestry of benefits that touch upon various aspects of well-being. On a mental level, +it greatly reduces stress and anxiety, fostering enhanced emotional health. This translates to better emotional +regulation and a heightened sense of overall well-being. Interestingly, the practice of meditation has been around +for centuries, evolving through various cultures and traditions, which underscores its timeless relevance. + +Physically, it contributes to lowering blood pressure and alleviating chronic pain, which is pivotal for long-term health. +Improved sleep quality is another significant benefit, aiding in overall physical restoration. Cognitively, meditation is a +boon for enhancing attention span, improving memory, and slowing down age-related cognitive decline. Amidst these benefits, +meditation's role in cultural and historical contexts is a fascinating side note, though not directly related to its health benefits. + +Such a comprehensive set of advantages makes meditation a valuable practice for individuals seeking holistic improvement i +n both mental and physical health, transcending its historical and cultural origins. +""" + +one = """ +Meditation is an ancient practice, rooted in various cultural traditions, where individuals +engage in mental exercises like mindfulness or concentration to promote mental clarity, emotional +calmness, and physical relaxation. This practice can range from techniques focusing on breath, visual +imagery, to movement-based forms like yoga. The goal is to bring about a sense of peace and self-awareness, +enabling individuals to deal with everyday stress more effectively. +""" + +two = """ +One of the key benefits of meditation is its impact on mental health. It's widely used as a tool to +reduce stress and anxiety. Meditation helps in managing emotions, leading to enhanced emotional health. +It can improve symptoms of anxiety and depression, fostering a general sense of well-being. Regular practice +is known to increase self-awareness, helping individuals understand their thoughts and emotions more clearly +and reduce negative reactions to challenging situations. +""" + +three = """ +Meditation has shown positive effects on various aspects of physical health. It can lower blood pressure, +reduce chronic pain, and improve sleep. From a cognitive perspective, meditation can sharpen the mind, increase +attention span, and improve memory. It's particularly beneficial in slowing down age-related cognitive decline and +enhancing brain functions related to concentration and attention. +""" + +four = """ +Understanding comets and asteroids is crucial in studying the solar system's formation +and evolution. Comets, which are remnants from the outer solar system, can provide +insights into its icy and volatile components. Asteroids, primarily remnants of the +early solar system's formation, offer clues about the materials that didn't form into +planets, mostly located in the asteroid belt. +""" + +five = """ +The physical characteristics and orbital paths of comets and asteroids vary significantly. +Comets often have highly elliptical orbits, taking them close to the Sun and then far into +the outer solar system. Their icy composition leads to distinctive features like tails and +comas. Asteroids, conversely, have more circular orbits and lack these visible features, +being composed mostly of rock and metal. +""" + + +@pytest.mark.skip(reason="openai is expensive") +def test_contextual_precision(): + metric = ContextualPrecisionMetric(threshold=0.5) + test_case = LLMTestCase( + input=question, + actual_output=answer, + expected_output=answer, + retrieval_context=[one, four, two, five, three], + ) + assert_test(test_case, [metric]) diff --git a/tests/test_custom_metric.py b/tests/test_custom_metric.py index f9e75db0a..9ec77aa9b 100644 --- a/tests/test_custom_metric.py +++ b/tests/test_custom_metric.py @@ -6,32 +6,37 @@ from deepeval import assert_test -class LengthMetric(BaseMetric): - """This metric checks if the output is more than 3 letters""" - - def __init__(self, minimum_score: int = 3): - self.minimum_score = minimum_score +class LatencyMetric(BaseMetric): + # This metric by default checks if the latency is greater than 10 seconds + def __init__(self, max_seconds: float = 10): + self.threshold = max_seconds def measure(self, test_case: LLMTestCase): - # sends to server - text = test_case.actual_output - score = len(text) - self.success = score > self.minimum_score - # Optional: Logs it to the server - return score + # Set self.success and self.score in the "measure" method + self.success = test_case.execution_time <= self.threshold + if self.success: + self.score = 1 + else: + self.score = 0 + + # You can also set a reason for the score returned. + # This is particularly useful for a score computed using LLMs + self.reason = None + return self.score def is_successful(self): return self.success @property def __name__(self): - return "Length" + return "Latency" def test_length_metric(): - metric = LengthMetric() + metric = LatencyMetric() test_case = LLMTestCase( input="placeholder", actual_output="This is a long sentence that is more than 3 letters", + execution_time=8.3, ) assert_test(test_case, [metric]) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 770bb05fc..d99ba8e98 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -41,5 +41,5 @@ def test_create_dataset(): # dataset, # ) # def test_customer_chatbot(test_case: LLMTestCase): -# hallucination_metric = HallucinationMetric(minimum_score=0.3) +# hallucination_metric = HallucinationMetric(threshold=0.3) # assert_test(test_case, [hallucination_metric]) diff --git a/tests/test_hallucination_metric.py b/tests/test_hallucination_metric.py index c37518785..13560d23c 100644 --- a/tests/test_hallucination_metric.py +++ b/tests/test_hallucination_metric.py @@ -5,7 +5,7 @@ def test_hallucination_metric(): - metric = HallucinationMetric(minimum_score=0.5) + metric = HallucinationMetric(threshold=0.5) test_case = LLMTestCase( input="placeholder", actual_output="A blond drinking water in public.", @@ -17,7 +17,7 @@ def test_hallucination_metric(): def test_hallucination_metric_2(): - metric = HallucinationMetric(minimum_score=0.6) + metric = HallucinationMetric(threshold=0.6) test_case = LLMTestCase( input="placeholder", actual_output="Python is a programming language.", @@ -28,7 +28,7 @@ def test_hallucination_metric_2(): def test_hallucination_metric_3(): - metric = HallucinationMetric(minimum_score=0.6) + metric = HallucinationMetric(threshold=0.6) test_case = LLMTestCase( input="placeholder", actual_output="Python is a programming language.", diff --git a/tests/test_judgemental.py b/tests/test_judgemental.py index 3b2181655..f7f8bfd62 100644 --- a/tests/test_judgemental.py +++ b/tests/test_judgemental.py @@ -20,7 +20,7 @@ def test_judgemntal(): LLMTestCaseParams.INPUT, ], language=Languages.SPANISH, - minimum_score=0.5, + threshold=0.5, ) assert_test(test_case, [metric]) diff --git a/tests/test_llm_metric.py b/tests/test_llm_metric.py index 4183ee2ff..54f34fb46 100644 --- a/tests/test_llm_metric.py +++ b/tests/test_llm_metric.py @@ -10,7 +10,7 @@ def test_chat_completion(): metric = LLMEvalMetric( name="Validity", criteria="The response is a valid response to the prompt.", - minimum_score=0.5, + threshold=0.5, evaluation_params=[ LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT, @@ -41,7 +41,7 @@ def test_chat_completion(): # metric = LLMEvalMetric( # name="Validity", # criteria="The response is a valid response to the prompt.", -# minimum_score=0.5, +# threshold=0.5, # evaluation_params=[ # LLMTestCaseParams.INPUT, # LLMTestCaseParams.ACTUAL_OUTPUT, diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py index 504cb2b87..d33a71148 100644 --- a/tests/test_quickstart.py +++ b/tests/test_quickstart.py @@ -19,7 +19,7 @@ def test_llm_output(): test_case = LLMTestCase( input=input, actual_output=generate_llm_output(input), context=context ) - assert_test(test_case, [HallucinationMetric(minimum_score=0.5)]) + assert_test(test_case, [HallucinationMetric(threshold=0.5)]) def test_llm_output_custom(): @@ -29,7 +29,7 @@ def test_llm_output_custom(): input="Placerholder", actual_output=actual_output, context=context ) with pytest.raises(AssertionError): - assert_test(test_case, [HallucinationMetric(minimum_score=0.5)]) + assert_test(test_case, [HallucinationMetric(threshold=0.5)]) def test_0(): diff --git a/tests/test_ragas.py b/tests/test_ragas.py index a2e38d306..caa93a868 100644 --- a/tests/test_ragas.py +++ b/tests/test_ragas.py @@ -2,17 +2,17 @@ from deepeval.test_case import LLMTestCase from deepeval.metrics import ( RagasMetric, - ContextualPrecisionMetric, - ContextualRelevancyMetric, + RAGASContextualPrecisionMetric, + RAGASContextualRelevancyMetric, RAGASFaithfulnessMetric, - ContextualRecallMetric, + RAGASContextualRecallMetric, ConcisenessMetric, CorrectnessMetric, CoherenceMetric, MaliciousnessMetric, + RAGASAnswerRelevancyMetric, ) -from deepeval.metrics.ragas_metric import RAGASAnswerRelevancyMetric -from deepeval import assert_test, evaluate +from deepeval import assert_test query = "Who won the FIFA World Cup in 2018 and what was the score?" output = "Winners of the FIFA world cup were the French national football team" @@ -51,28 +51,28 @@ def test_everything(): retrieval_context=context, context=context, ) - # metric1 = ContextualRelevancyMetric(model="gpt-4") - # metric2 = RAGASFaithfulnessMetric(model="gpt-4") - # metric3 = ContextualRecallMetric(model="gpt-4") - # metric4 = ConcisenessMetric(model="gpt-4") - # metric5 = CorrectnessMetric(model="gpt-4") - # metric6 = CoherenceMetric(model="gpt-4") - # metric7 = MaliciousnessMetric(model="gpt-4") - # metric8 = RAGASAnswerRelevancyMetric(model="gpt-4") - metric9 = ContextualPrecisionMetric() - # metric10 = RagasMetric() + metric1 = RAGASContextualRelevancyMetric(model="gpt-4") + metric2 = RAGASFaithfulnessMetric(model="gpt-4") + metric3 = RAGASContextualRecallMetric(model="gpt-4") + metric4 = ConcisenessMetric(model="gpt-4") + metric5 = CorrectnessMetric(model="gpt-4") + metric6 = CoherenceMetric(model="gpt-4") + metric7 = MaliciousnessMetric(model="gpt-4") + metric8 = RAGASAnswerRelevancyMetric(model="gpt-4") + metric9 = RAGASContextualPrecisionMetric() + metric10 = RagasMetric() assert_test( test_case, [ - # metric1, - # metric2, - # metric3, - # metric4, - # metric5, - # metric6, - # metric7, - # metric8, + metric1, + metric2, + metric3, + metric4, + metric5, + metric6, + metric7, + metric8, metric9, - # metric10, + metric10, ], )