From 2a0514636ae5d5c1db937c8def802d94e6e01c0c Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Thu, 28 Dec 2023 22:01:59 +0800 Subject: [PATCH 01/13] LLamaindex tracing --- deepeval/tracing/integrations/llama_index.py | 142 +++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 deepeval/tracing/integrations/llama_index.py diff --git a/deepeval/tracing/integrations/llama_index.py b/deepeval/tracing/integrations/llama_index.py new file mode 100644 index 000000000..748a5975d --- /dev/null +++ b/deepeval/tracing/integrations/llama_index.py @@ -0,0 +1,142 @@ +from typing import Any, Dict, List, Optional, Union +from time import perf_counter + +from llama_index.bridge.pydantic import BaseModel +from llama_index.callbacks.base_handler import BaseCallbackHandler +from llama_index.callbacks.schema import CBEventType, EventPayload +from llama_index.llms import ChatMessage + +from deepeval.tracing import ( + trace_manager, + get_trace_stack, + LlmTrace, + GenericTrace, + EmbeddingTrace, + TraceStatus, + LlmMetadata, + EmbeddingMetadata, +) +from deepeval.utils import dataclass_to_dict + +events_to_ignore = [ + CBEventType.CHUNKING, + CBEventType.NODE_PARSING, + CBEventType.EMBEDDING, + CBEventType.TREE, + CBEventType.SUB_QUESTION, + CBEventType.FUNCTION_CALL, + CBEventType.EXCEPTION, + CBEventType.AGENT_STEP, +] + + +class LlamaIndexCallbackHandler(BaseCallbackHandler): + def __init__(self) -> None: + self.event_map = {} + super().__init__( + event_starts_to_ignore=events_to_ignore, + event_ends_to_ignore=events_to_ignore, + ) + + def start_trace(self, trace_id: Optional[str] = None) -> None: + self.event_map = {} + trace_manager.clear_trace_stack() + return + + def end_trace( + self, + trace_id: Optional[str] = None, + trace_map: Optional[Dict[str, List[str]]] = None, + ) -> None: + # TODO: + print(get_trace_stack()) + return + + def on_event_start( + self, + event_type: CBEventType, + payload: Optional[Dict[str, Any]] = None, + event_id: str = "", + parent_id: str = "", + **kwargs: Any, + ) -> str: + trace_instance = self.create_trace_instance(event_type) + self.event_map[event_id] = trace_instance + trace_manager.append_to_trace_stack(trace_instance) + return + + def on_event_end( + self, + event_type: CBEventType, + payload: Optional[Dict[str, Any]] = None, + event_id: str = "", + **kwargs: Any, + ) -> None: + trace_instance = self.event_map[event_id] + trace_instance.executionTime = ( + perf_counter() - trace_instance.executionTime + ) + print(trace_instance.executionTime) + # TODO: get inputs and outputs fron payload into kwargs based on CBEventType + + current_trace_stack = trace_manager.get_trace_stack() + if len(current_trace_stack) > 1: + parent_trace = current_trace_stack[-2] + parent_trace.traces.append(trace_instance) + + if len(current_trace_stack) == 1: + dict_representation = dataclass_to_dict(current_trace_stack[0]) + trace_manager.set_dict_trace_stack(dict_representation) + trace_manager.clear_trace_stack() + else: + trace_manager.pop_trace_stack() + + return + + def create_trace_instance( + self, event_type: CBEventType + ) -> Union[EmbeddingTrace, LlmMetadata, GenericTrace]: + current_time = perf_counter() + type = self.convert_event_type_to_deepeval_trace_type(event_type) + name = event_type.capitalize() + trace_instance_input = {"args": None, "kwargs": None} + if event_type == CBEventType.LLM: + trace_instance = LlmTrace( + type=type, + executionTime=current_time, + name=name, + input=trace_instance_input, + output=None, + status=TraceStatus.SUCCESS, + traces=[], + llmMetadata=LlmMetadata(model="None"), + ) + elif event_type == CBEventType.EMBEDDING: + trace_instance = EmbeddingTrace( + type=type, + executionTime=current_time, + name=name, + input=trace_instance_input, + output=None, + status=TraceStatus.SUCCESS, + traces=[], + embeddingMetadata=EmbeddingMetadata(model="None"), + ) + else: + trace_instance = GenericTrace( + type=type, + executionTime=current_time, + name=name, + input=trace_instance_input, + output=None, + status=TraceStatus.SUCCESS, + traces=[], + ) + + return trace_instance + + # TODO + def convert_event_type_to_deepeval_trace_type( + self, event_type: CBEventType + ): + pass From 25bd4c9a88e8bccbbd39a3059492d06305f911b6 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Thu, 28 Dec 2023 22:03:53 +0800 Subject: [PATCH 02/13] Added llama --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c12a227af..86830629a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ typer = "*" setuptools = "*" wheel = "*" aiohttp = "*" - +llama-index = {path = "/Users/jeffreyip/mrgpt/repos/llama_index"} [tool.black] line-length = 80 From 8a5e382d5ae9fd6e237fda4be0dfe314bee42909 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Tue, 2 Jan 2024 03:52:37 +0800 Subject: [PATCH 03/13] llamaindex tracing --- deepeval/tracing/__init__.py | 13 +- deepeval/tracing/integrations/llama_index.py | 22 ++- llama_test/chatbot.py | 14 ++ llama_test/data/data.txt | 25 +++ llama_test/main.py | 6 + poetry.lock | 157 ++++++++++--------- pyproject.toml | 1 - 7 files changed, 152 insertions(+), 86 deletions(-) create mode 100644 llama_test/chatbot.py create mode 100644 llama_test/data/data.txt create mode 100644 llama_test/main.py diff --git a/deepeval/tracing/__init__.py b/deepeval/tracing/__init__.py index 76ea3893c..ce4c4642a 100644 --- a/deepeval/tracing/__init__.py +++ b/deepeval/tracing/__init__.py @@ -1 +1,12 @@ -from .tracing import trace, TraceType, get_trace_stack +from .tracing import ( + trace, + trace_manager, + get_trace_stack, + TraceType, + TraceStatus, + LlmTrace, + EmbeddingTrace, + GenericTrace, + LlmMetadata, + EmbeddingMetadata, +) diff --git a/deepeval/tracing/integrations/llama_index.py b/deepeval/tracing/integrations/llama_index.py index 748a5975d..0cd2d1736 100644 --- a/deepeval/tracing/integrations/llama_index.py +++ b/deepeval/tracing/integrations/llama_index.py @@ -15,6 +15,7 @@ TraceStatus, LlmMetadata, EmbeddingMetadata, + TraceType, ) from deepeval.utils import dataclass_to_dict @@ -48,8 +49,6 @@ def end_trace( trace_id: Optional[str] = None, trace_map: Optional[Dict[str, List[str]]] = None, ) -> None: - # TODO: - print(get_trace_stack()) return def on_event_start( @@ -76,8 +75,12 @@ def on_event_end( trace_instance.executionTime = ( perf_counter() - trace_instance.executionTime ) - print(trace_instance.executionTime) - # TODO: get inputs and outputs fron payload into kwargs based on CBEventType + input_kwargs = {} + if payload is not None: + for event in EventPayload: + value = payload.get(event.value) + if value is not None: + input_kwargs[event.value] = value current_trace_stack = trace_manager.get_trace_stack() if len(current_trace_stack) > 1: @@ -135,8 +138,15 @@ def create_trace_instance( return trace_instance - # TODO def convert_event_type_to_deepeval_trace_type( self, event_type: CBEventType ): - pass + # TODO: add more types + if event_type == CBEventType.LLM: + return TraceType.LLM + elif event_type == CBEventType.RETRIEVE: + return TraceType.RETRIEVER + elif event_type == CBEventType.EMBEDDING: + return TraceType.EMBEDDING + + return event_type.value.capitalize() diff --git a/llama_test/chatbot.py b/llama_test/chatbot.py new file mode 100644 index 000000000..a07724b5e --- /dev/null +++ b/llama_test/chatbot.py @@ -0,0 +1,14 @@ +from llama_index import VectorStoreIndex, SimpleDirectoryReader +from llama_index import ServiceContext +import llama_index + +llama_index.set_global_handler("deepeval") + +service_context = ServiceContext.from_defaults(chunk_size=1000) +documents = SimpleDirectoryReader("data").load_data() +index = VectorStoreIndex.from_documents(documents) +query_engine = index.as_query_engine(similarity_top_k=5) + + +def query(user_input): + return query_engine.query(user_input).response diff --git a/llama_test/data/data.txt b/llama_test/data/data.txt new file mode 100644 index 000000000..c2bd37022 --- /dev/null +++ b/llama_test/data/data.txt @@ -0,0 +1,25 @@ +About MadeUpCompany +MadeUpCompany is a pioneering technology firm founded in 2010, specializing in cloud computing, data analytics, and machine learning. Our headquarters is based in San Francisco, California, with satellite offices spread across New York, London, and Tokyo. We are committed to offering state-of-the-art solutions that help businesses and individuals achieve their full potential. With a diverse team of experts from various industries, we strive to redefine the boundaries of innovation and efficiency. + +Products and Services +We offer a suite of services ranging from cloud storage solutions, data analytics platforms, to custom machine learning models tailored for specific business needs. Our most popular product is CloudMate, a cloud storage solution designed for businesses of all sizes. It offers seamless data migration, top-tier security protocols, and an easy-to-use interface. Our data analytics service, DataWiz, helps companies turn raw data into actionable insights using advanced algorithms. + +Pricing +We have a variety of pricing options tailored to different needs. Our basic cloud storage package starts at $9.99 per month, with premium plans offering more storage and functionalities. We also provide enterprise solutions on a case-by-case basis, so it’s best to consult with our sales team for customized pricing. + +Technical Support +Our customer support team is available 24/7 to assist with any technical issues. We offer multiple channels for support including live chat, email, and a toll-free number. Most issues are typically resolved within 24 hours. We also have an extensive FAQ section on our website and a community forum for peer support. + +Security and Compliance +MadeUpCompany places the utmost importance on security and compliance. All our products are GDPR compliant and adhere to the highest security standards, including end-to-end encryption and multi-factor authentication. + +Account Management +Customers can easily manage their accounts through our online portal, which allows you to upgrade your service, view billing history, and manage users in your organization. If you encounter any issues or have questions about your account, our account management team is available weekdays from 9 AM to 6 PM. + +Refund and Cancellation Policy +We offer a 30-day money-back guarantee on all our products. If you're not satisfied for any reason, you can request a full refund within the first 30 days of your purchase. After that, you can still cancel your service at any time, but a prorated refund will be issued based on the remaining term of your subscription. + +Upcoming Features +We’re constantly working to improve our services and offer new features. Keep an eye out for updates on machine learning functionalities in DataWiz and more collaborative tools in CloudMate in the upcoming quarters. + +Your customer support staff can use these paragraphs to build their responses to customer inquiries, providing both detailed and precise information to address various questions. \ No newline at end of file diff --git a/llama_test/main.py b/llama_test/main.py new file mode 100644 index 000000000..9ddcfa104 --- /dev/null +++ b/llama_test/main.py @@ -0,0 +1,6 @@ +from chatbot import query + +while True: + user_input = input("Enter your question: ") + response = query(user_input) + print("Bot response:", response) diff --git a/poetry.lock b/poetry.lock index 161ba87c4..e12b30b58 100644 --- a/poetry.lock +++ b/poetry.lock @@ -167,21 +167,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "bert-score" @@ -544,13 +545,13 @@ typing-inspect = ">=0.4.0,<1" [[package]] name = "datasets" -version = "2.16.0" +version = "2.16.1" description = "HuggingFace community-driven open-source library of datasets" optional = false python-versions = ">=3.8.0" files = [ - {file = "datasets-2.16.0-py3-none-any.whl", hash = "sha256:301cc39b3d81cd751100b79c85f8ae8626c17b0b113819ba2831c204d90b43f2"}, - {file = "datasets-2.16.0.tar.gz", hash = "sha256:91b06f7a8f0329179e7d603004102a6cc7a424a2f599315297a061caa1f8fa64"}, + {file = "datasets-2.16.1-py3-none-any.whl", hash = "sha256:fafa300c78ff92d521473a3d47d60c2d3e0d6046212cc03ceb6caf6550737257"}, + {file = "datasets-2.16.1.tar.gz", hash = "sha256:ad3215e9b1984d1de4fda2123bc7319ccbdf1e17d0c3d5590d13debff308a080"}, ] [package.dependencies] @@ -1194,13 +1195,13 @@ files = [ [[package]] name = "langchain" -version = "0.0.352" +version = "0.0.353" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain-0.0.352-py3-none-any.whl", hash = "sha256:43ab580e1223e5d7c3495b3c0cb79e2f3a0ecb52caf8126271fb10d42cede2d0"}, - {file = "langchain-0.0.352.tar.gz", hash = "sha256:8928d7b63d73af9681fe1b2a2b99b84238efef61ed537de666160fd001f41efd"}, + {file = "langchain-0.0.353-py3-none-any.whl", hash = "sha256:54cac8b74fbefacddcdf0c443619a7331d6b59fe94fa2a48a4d7da2b59cf1f63"}, + {file = "langchain-0.0.353.tar.gz", hash = "sha256:a095ea819f13a3606ced699182a8369eb2d77034ec8c913983675d6dd9a98196"}, ] [package.dependencies] @@ -1209,7 +1210,7 @@ async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\ dataclasses-json = ">=0.5.7,<0.7" jsonpatch = ">=1.33,<2.0" langchain-community = ">=0.0.2,<0.1" -langchain-core = ">=0.1,<0.2" +langchain-core = ">=0.1.4,<0.2" langsmith = ">=0.0.70,<0.1.0" numpy = ">=1,<2" pydantic = ">=1,<3" @@ -1234,13 +1235,13 @@ text-helpers = ["chardet (>=5.1.0,<6.0.0)"] [[package]] name = "langchain-community" -version = "0.0.6" +version = "0.0.7" description = "Community contributed LangChain integrations." optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain_community-0.0.6-py3-none-any.whl", hash = "sha256:13b16da0f89c328df456911ff03069e4d919f647c7dd3bfc5062525cf956ed82"}, - {file = "langchain_community-0.0.6.tar.gz", hash = "sha256:b7deb63fd8205d54b51cf8b1702de15d1da77987f8465c356b158a65adff378c"}, + {file = "langchain_community-0.0.7-py3-none-any.whl", hash = "sha256:468af187bfffe753426cc4548132824be7df9404d38ceef2f873087290d8ff0e"}, + {file = "langchain_community-0.0.7.tar.gz", hash = "sha256:cfbeb25cac7dff3c021f3c82aa243fc80f80082d6f6fdcc79daf36b1408828cc"}, ] [package.dependencies] @@ -1256,17 +1257,17 @@ tenacity = ">=8.1.0,<9.0.0" [package.extras] cli = ["typer (>=0.9.0,<0.10.0)"] -extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "gradientai (>=1.4.0,<2.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "oracle-ads (>=2.9.1,<3.0.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] +extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "azure-ai-documentintelligence (>=1.0.0b1,<2.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "cohere (>=4,<5)", "dashvector (>=1.0.1,<2.0.0)", "databricks-vectorsearch (>=0.21,<0.22)", "datasets (>=2.15.0,<3.0.0)", "dgml-utils (>=0.3.0,<0.4.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "fireworks-ai (>=0.9.0,<0.10.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "gradientai (>=1.4.0,<2.0.0)", "hologres-vector (>=0.0.6,<0.0.7)", "html2text (>=2020.1.16,<2021.0.0)", "javelin-sdk (>=0.1.8,<0.2.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "jsonschema (>1)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "msal (>=1.25.0,<2.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (<2)", "openapi-pydantic (>=0.3.2,<0.4.0)", "oracle-ads (>=2.9.1,<3.0.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "praw (>=7.7.1,<8.0.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] [[package]] name = "langchain-core" -version = "0.1.3" +version = "0.1.4" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain_core-0.1.3-py3-none-any.whl", hash = "sha256:bfbbc5dfeb06cfe3fd078e7a12db3a4cfb9d28b715b200a64f7abb7ae1976b17"}, - {file = "langchain_core-0.1.3.tar.gz", hash = "sha256:d8898254dfea1c4ab614f470db40909969604775f7524175f6d9167ea58050c9"}, + {file = "langchain_core-0.1.4-py3-none-any.whl", hash = "sha256:c62bd362d5abf5359436a99b29629e12a4d1ede9f1704dc958cdb8530a791efd"}, + {file = "langchain_core-0.1.4.tar.gz", hash = "sha256:f700138689c9014e23d3c29796a892dccf7f2a42901cb8817671823e1a24724c"}, ] [package.dependencies] @@ -2332,13 +2333,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.3" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, - {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] @@ -3061,60 +3062,60 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.23" +version = "2.0.24" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.23-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:638c2c0b6b4661a4fd264f6fb804eccd392745c5887f9317feb64bb7cb03b3ea"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3b5036aa326dc2df50cba3c958e29b291a80f604b1afa4c8ce73e78e1c9f01d"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:787af80107fb691934a01889ca8f82a44adedbf5ef3d6ad7d0f0b9ac557e0c34"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c14eba45983d2f48f7546bb32b47937ee2cafae353646295f0e99f35b14286ab"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0666031df46b9badba9bed00092a1ffa3aa063a5e68fa244acd9f08070e936d3"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:89a01238fcb9a8af118eaad3ffcc5dedaacbd429dc6fdc43fe430d3a941ff965"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-win32.whl", hash = "sha256:cabafc7837b6cec61c0e1e5c6d14ef250b675fa9c3060ed8a7e38653bd732ff8"}, - {file = "SQLAlchemy-2.0.23-cp310-cp310-win_amd64.whl", hash = "sha256:87a3d6b53c39cd173990de2f5f4b83431d534a74f0e2f88bd16eabb5667e65c6"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d5578e6863eeb998980c212a39106ea139bdc0b3f73291b96e27c929c90cd8e1"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:62d9e964870ea5ade4bc870ac4004c456efe75fb50404c03c5fd61f8bc669a72"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c80c38bd2ea35b97cbf7c21aeb129dcbebbf344ee01a7141016ab7b851464f8e"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75eefe09e98043cff2fb8af9796e20747ae870c903dc61d41b0c2e55128f958d"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd45a5b6c68357578263d74daab6ff9439517f87da63442d244f9f23df56138d"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a86cb7063e2c9fb8e774f77fbf8475516d270a3e989da55fa05d08089d77f8c4"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-win32.whl", hash = "sha256:b41f5d65b54cdf4934ecede2f41b9c60c9f785620416e8e6c48349ab18643855"}, - {file = "SQLAlchemy-2.0.23-cp311-cp311-win_amd64.whl", hash = "sha256:9ca922f305d67605668e93991aaf2c12239c78207bca3b891cd51a4515c72e22"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0f7fb0c7527c41fa6fcae2be537ac137f636a41b4c5a4c58914541e2f436b45"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7c424983ab447dab126c39d3ce3be5bee95700783204a72549c3dceffe0fc8f4"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f508ba8f89e0a5ecdfd3761f82dda2a3d7b678a626967608f4273e0dba8f07ac"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6463aa765cf02b9247e38b35853923edbf2f6fd1963df88706bc1d02410a5577"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e599a51acf3cc4d31d1a0cf248d8f8d863b6386d2b6782c5074427ebb7803bda"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fd54601ef9cc455a0c61e5245f690c8a3ad67ddb03d3b91c361d076def0b4c60"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-win32.whl", hash = "sha256:42d0b0290a8fb0165ea2c2781ae66e95cca6e27a2fbe1016ff8db3112ac1e846"}, - {file = "SQLAlchemy-2.0.23-cp312-cp312-win_amd64.whl", hash = "sha256:227135ef1e48165f37590b8bfc44ed7ff4c074bf04dc8d6f8e7f1c14a94aa6ca"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:14aebfe28b99f24f8a4c1346c48bc3d63705b1f919a24c27471136d2f219f02d"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e983fa42164577d073778d06d2cc5d020322425a509a08119bdcee70ad856bf"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e0dc9031baa46ad0dd5a269cb7a92a73284d1309228be1d5935dac8fb3cae24"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5f94aeb99f43729960638e7468d4688f6efccb837a858b34574e01143cf11f89"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:63bfc3acc970776036f6d1d0e65faa7473be9f3135d37a463c5eba5efcdb24c8"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-win32.whl", hash = "sha256:f48ed89dd11c3c586f45e9eec1e437b355b3b6f6884ea4a4c3111a3358fd0c18"}, - {file = "SQLAlchemy-2.0.23-cp37-cp37m-win_amd64.whl", hash = "sha256:1e018aba8363adb0599e745af245306cb8c46b9ad0a6fc0a86745b6ff7d940fc"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:64ac935a90bc479fee77f9463f298943b0e60005fe5de2aa654d9cdef46c54df"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c4722f3bc3c1c2fcc3702dbe0016ba31148dd6efcd2a2fd33c1b4897c6a19693"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af79c06825e2836de21439cb2a6ce22b2ca129bad74f359bddd173f39582bf5"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:683ef58ca8eea4747737a1c35c11372ffeb84578d3aab8f3e10b1d13d66f2bc4"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d4041ad05b35f1f4da481f6b811b4af2f29e83af253bf37c3c4582b2c68934ab"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aeb397de65a0a62f14c257f36a726945a7f7bb60253462e8602d9b97b5cbe204"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-win32.whl", hash = "sha256:42ede90148b73fe4ab4a089f3126b2cfae8cfefc955c8174d697bb46210c8306"}, - {file = "SQLAlchemy-2.0.23-cp38-cp38-win_amd64.whl", hash = "sha256:964971b52daab357d2c0875825e36584d58f536e920f2968df8d581054eada4b"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:616fe7bcff0a05098f64b4478b78ec2dfa03225c23734d83d6c169eb41a93e55"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0e680527245895aba86afbd5bef6c316831c02aa988d1aad83c47ffe92655e74"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9585b646ffb048c0250acc7dad92536591ffe35dba624bb8fd9b471e25212a35"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4895a63e2c271ffc7a81ea424b94060f7b3b03b4ea0cd58ab5bb676ed02f4221"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:cc1d21576f958c42d9aec68eba5c1a7d715e5fc07825a629015fe8e3b0657fb0"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:967c0b71156f793e6662dd839da54f884631755275ed71f1539c95bbada9aaab"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-win32.whl", hash = "sha256:0a8c6aa506893e25a04233bc721c6b6cf844bafd7250535abb56cb6cc1368884"}, - {file = "SQLAlchemy-2.0.23-cp39-cp39-win_amd64.whl", hash = "sha256:f3420d00d2cb42432c1d0e44540ae83185ccbbc67a6054dcc8ab5387add6620b"}, - {file = "SQLAlchemy-2.0.23-py3-none-any.whl", hash = "sha256:31952bbc527d633b9479f5f81e8b9dfada00b91d6baba021a869095f1a97006d"}, - {file = "SQLAlchemy-2.0.23.tar.gz", hash = "sha256:c1bda93cbbe4aa2aa0aa8655c5aeda505cd219ff3e8da91d1d329e143e4aff69"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5f801d85ba4753d4ed97181d003e5d3fa330ac7c4587d131f61d7f968f416862"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b35c35e3923ade1e7ac44e150dec29f5863513246c8bf85e2d7d313e3832bcfb"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d9b3fd5eca3c0b137a5e0e468e24ca544ed8ca4783e0e55341b7ed2807518ee"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a6209e689d0ff206c40032b6418e3cfcfc5af044b3f66e381d7f1ae301544b4"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:37e89d965b52e8b20571b5d44f26e2124b26ab63758bf1b7598a0e38fb2c4005"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6910eb4ea90c0889f363965cd3c8c45a620ad27b526a7899f0054f6c1b9219e"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-win32.whl", hash = "sha256:d8e7e8a150e7b548e7ecd6ebb9211c37265991bf2504297d9454e01b58530fc6"}, + {file = "SQLAlchemy-2.0.24-cp310-cp310-win_amd64.whl", hash = "sha256:396f05c552f7fa30a129497c41bef5b4d1423f9af8fe4df0c3dcd38f3e3b9a14"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:adbd67dac4ebf54587198b63cd30c29fd7eafa8c0cab58893d9419414f8efe4b"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0f611b431b84f55779cbb7157257d87b4a2876b067c77c4f36b15e44ced65e2"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56a0e90a959e18ac5f18c80d0cad9e90cb09322764f536e8a637426afb1cae2f"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6db686a1d9f183c639f7e06a2656af25d4ed438eda581de135d15569f16ace33"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f0cc0b486a56dff72dddae6b6bfa7ff201b0eeac29d4bc6f0e9725dc3c360d71"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a1d4856861ba9e73bac05030cec5852eabfa9ef4af8e56c19d92de80d46fc34"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-win32.whl", hash = "sha256:a3c2753bf4f48b7a6024e5e8a394af49b1b12c817d75d06942cae03d14ff87b3"}, + {file = "SQLAlchemy-2.0.24-cp311-cp311-win_amd64.whl", hash = "sha256:38732884eabc64982a09a846bacf085596ff2371e4e41d20c0734f7e50525d01"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9f992e0f916201731993eab8502912878f02287d9f765ef843677ff118d0e0b1"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2587e108463cc2e5b45a896b2e7cc8659a517038026922a758bde009271aed11"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bb7cedcddffca98c40bb0becd3423e293d1fef442b869da40843d751785beb3"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83fa6df0e035689df89ff77a46bf8738696785d3156c2c61494acdcddc75c69d"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:cc889fda484d54d0b31feec409406267616536d048a450fc46943e152700bb79"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57ef6f2cb8b09a042d0dbeaa46a30f2df5dd1e1eb889ba258b0d5d7d6011b81c"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-win32.whl", hash = "sha256:ea490564435b5b204d8154f0e18387b499ea3cedc1e6af3b3a2ab18291d85aa7"}, + {file = "SQLAlchemy-2.0.24-cp312-cp312-win_amd64.whl", hash = "sha256:ccfd336f96d4c9bbab0309f2a565bf15c468c2d8b2d277a32f89c5940f71fcf9"}, + {file = "SQLAlchemy-2.0.24-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9aaaaa846b10dfbe1bda71079d0e31a7e2cebedda9409fa7dba3dfed1ae803e8"}, + {file = "SQLAlchemy-2.0.24-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95bae3d38f8808d79072da25d5e5a6095f36fe1f9d6c614dd72c59ca8397c7c0"}, + {file = "SQLAlchemy-2.0.24-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a04191a7c8d77e63f6fc1e8336d6c6e93176c0c010833e74410e647f0284f5a1"}, + {file = "SQLAlchemy-2.0.24-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:acc58b7c2e40235712d857fdfc8f2bda9608f4a850d8d9ac0dd1fc80939ca6ac"}, + {file = "SQLAlchemy-2.0.24-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00d76fe5d7cdb5d84d625ce002ce29fefba0bfd98e212ae66793fed30af73931"}, + {file = "SQLAlchemy-2.0.24-cp37-cp37m-win32.whl", hash = "sha256:29e51f848f843bbd75d74ae64ab1ab06302cb1dccd4549d1f5afe6b4a946edb2"}, + {file = "SQLAlchemy-2.0.24-cp37-cp37m-win_amd64.whl", hash = "sha256:e9d036e343a604db3f5a6c33354018a84a1d3f6dcae3673358b404286204798c"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9bafaa05b19dc07fa191c1966c5e852af516840b0d7b46b7c3303faf1a349bc9"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e69290b921b7833c04206f233d6814c60bee1d135b09f5ae5d39229de9b46cd4"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8398593ccc4440ce6dffcc4f47d9b2d72b9fe7112ac12ea4a44e7d4de364db1"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f073321a79c81e1a009218a21089f61d87ee5fa3c9563f6be94f8b41ff181812"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9036ebfd934813990c5b9f71f297e77ed4963720db7d7ceec5a3fdb7cd2ef6ce"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fcf84fe93397a0f67733aa2a38ed4eab9fc6348189fc950e656e1ea198f45668"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-win32.whl", hash = "sha256:6f5e75de91c754365c098ac08c13fdb267577ce954fa239dd49228b573ca88d7"}, + {file = "SQLAlchemy-2.0.24-cp38-cp38-win_amd64.whl", hash = "sha256:9f29c7f0f4b42337ec5a779e166946a9f86d7d56d827e771b69ecbdf426124ac"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07cc423892f2ceda9ae1daa28c0355757f362ecc7505b1ab1a3d5d8dc1c44ac6"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a479aa1ab199178ff1956b09ca8a0693e70f9c762875d69292d37049ffd0d8f"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b8d0e8578e7f853f45f4512b5c920f6a546cd4bed44137460b2a56534644205"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17e7e27af178d31b436dda6a596703b02a89ba74a15e2980c35ecd9909eea3a"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1ca7903d5e7db791a355b579c690684fac6304478b68efdc7f2ebdcfe770d8d7"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db09e424d7bb89b6215a184ca93b4f29d7f00ea261b787918a1af74143b98c06"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-win32.whl", hash = "sha256:a5cd7d30e47f87b21362beeb3e86f1b5886e7d9b0294b230dde3d3f4a1591375"}, + {file = "SQLAlchemy-2.0.24-cp39-cp39-win_amd64.whl", hash = "sha256:7ae5d44517fe81079ce75cf10f96978284a6db2642c5932a69c82dbae09f009a"}, + {file = "SQLAlchemy-2.0.24-py3-none-any.whl", hash = "sha256:8f358f5cfce04417b6ff738748ca4806fe3d3ae8040fb4e6a0c9a6973ccf9b6e"}, + {file = "SQLAlchemy-2.0.24.tar.gz", hash = "sha256:6db97656fd3fe3f7e5b077f12fa6adb5feb6e0b567a3e99f47ecf5f7ea0a09e3"}, ] [package.dependencies] @@ -3124,7 +3125,7 @@ typing-extensions = ">=4.2.0" [package.extras] aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] @@ -3134,7 +3135,7 @@ mssql-pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)"] mysql = ["mysqlclient (>=1.4.0)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=8)"] +oracle = ["cx_oracle (>=8)"] oracle-oracledb = ["oracledb (>=1.0.1)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] @@ -3144,7 +3145,7 @@ postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] pymysql = ["pymysql"] -sqlcipher = ["sqlcipher3-binary"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sympy" @@ -3630,13 +3631,13 @@ typing-extensions = ">=3.7.4" [[package]] name = "tzdata" -version = "2023.3" +version = "2023.4" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, - {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 86830629a..ab821bbe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ typer = "*" setuptools = "*" wheel = "*" aiohttp = "*" -llama-index = {path = "/Users/jeffreyip/mrgpt/repos/llama_index"} [tool.black] line-length = 80 From 903352ada676c551b55c6d4343ca927c594d2f7b Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Tue, 2 Jan 2024 03:54:46 +0800 Subject: [PATCH 04/13] added dependency --- poetry.lock | 166 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 165 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index e12b30b58..34c10e78c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -184,6 +184,24 @@ tests = ["attrs[tests-no-zope]", "zope-interface"] tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +[[package]] +name = "beautifulsoup4" +version = "4.12.2" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "bert-score" version = "0.3.13" @@ -587,6 +605,23 @@ tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elast torch = ["torch"] vision = ["Pillow (>=6.2.1)"] +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + [[package]] name = "dill" version = "0.3.7" @@ -1298,6 +1333,43 @@ files = [ pydantic = ">=1,<3" requests = ">=2,<3" +[[package]] +name = "llama-index" +version = "0.9.24" +description = "Interface between LLMs and your data" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [ + {file = "llama_index-0.9.24-py3-none-any.whl", hash = "sha256:aeef8a4fb478d45474261289046f37c2805e3bf3453c156c84088c0414465e5e"}, + {file = "llama_index-0.9.24.tar.gz", hash = "sha256:48175a35c30427f361068693d6f384baf76865831569ca4e04a1a8b6f10ba269"}, +] + +[package.dependencies] +aiohttp = ">=3.8.6,<4.0.0" +beautifulsoup4 = ">=4.12.2,<5.0.0" +dataclasses-json = "*" +deprecated = ">=1.2.9.3" +fsspec = ">=2023.5.0" +httpx = "*" +nest-asyncio = ">=1.5.8,<2.0.0" +nltk = ">=3.8.1,<4.0.0" +numpy = "*" +openai = ">=1.1.0" +pandas = "*" +requests = ">=2.31.0" +SQLAlchemy = {version = ">=1.4.49", extras = ["asyncio"]} +tenacity = ">=8.2.0,<9.0.0" +tiktoken = ">=0.3.3" +typing-extensions = ">=4.5.0" +typing-inspect = ">=0.8.0" + +[package.extras] +gradientai = ["gradientai (>=1.4.0)"] +langchain = ["langchain (>=0.0.303)"] +local-models = ["optimum[onnxruntime] (>=1.13.2,<2.0.0)", "sentencepiece (>=0.1.99,<0.2.0)", "transformers[torch] (>=4.34.0,<5.0.0)"] +postgres = ["asyncpg (>=0.28.0,<0.29.0)", "pgvector (>=0.1.0,<0.2.0)", "psycopg-binary (>=3.1.12,<4.0.0)"] +query-tools = ["guidance (>=0.0.64,<0.0.65)", "jsonpath-ng (>=1.6.0,<2.0.0)", "lm-format-enforcer (>=0.4.3,<0.5.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "scikit-learn", "spacy (>=3.7.1,<4.0.0)"] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -3060,6 +3132,17 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "sqlalchemy" version = "2.0.24" @@ -3119,7 +3202,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", optional = true, markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\" or extra == \"asyncio\""} typing-extensions = ">=4.2.0" [package.extras] @@ -3670,6 +3753,85 @@ files = [ [package.extras] test = ["pytest (>=6.0.0)", "setuptools (>=65)"] +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + [[package]] name = "xxhash" version = "3.4.1" @@ -3893,4 +4055,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "8b882533d1c305fc650e8609cb93c420c3becbff3abb398cb247f5bbd42a73ec" +content-hash = "2b0939b9cce5fa623a932eb89645a6ef3cc7384addd232339faded4a4c35cab7" diff --git a/pyproject.toml b/pyproject.toml index ab821bbe3..4fd145270 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ typer = "*" setuptools = "*" wheel = "*" aiohttp = "*" +llama-index = "*" [tool.black] line-length = 80 From 03d767f072af5fb01ffa768532a81ea90928b64c Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 03:18:11 +0800 Subject: [PATCH 05/13] updated docs --- docs/docs/confident-ai-evals-in-production.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/docs/confident-ai-evals-in-production.mdx b/docs/docs/confident-ai-evals-in-production.mdx index 7a1fc87b9..e08abb723 100644 --- a/docs/docs/confident-ai-evals-in-production.mdx +++ b/docs/docs/confident-ai-evals-in-production.mdx @@ -25,6 +25,10 @@ Simply add `deepeval.track(...)` in your application to start tracking events. T - [Optional] `additional_data`: type `dict` - [Optional] `fail_silently`: type `bool`, defaults to True +:::note +Please do **NOT** provide placeholder values for optional parameters. Leave it blank instead. +::: + ```python import deepeval From 4069536f34fb15a37dd252e4c969a6033f95ea93 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 18:42:01 +0800 Subject: [PATCH 06/13] updated docs --- docs/docs/metrics-toxicity.mdx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/docs/metrics-toxicity.mdx b/docs/docs/metrics-toxicity.mdx index 1604d30e7..0af3e030a 100644 --- a/docs/docs/metrics-toxicity.mdx +++ b/docs/docs/metrics-toxicity.mdx @@ -6,6 +6,14 @@ sidebar_label: Toxicity The toxicity metric is another **referenceless** metric that evaluates toxicness in your LLM's outputs. This is particularly useful for a fine-tuning use case. +## Installation + +Toxicity in `deepeval` requires an additional installation: + +``` +pip install detoxify +``` + ## Required Parameters To use the `NonToxicMetric`, you'll have to provide the following parameters when creating an `LLMTestCase`: From 6eb34e265261326fac9a5e78053ccaa8c9b47cc5 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 19:18:09 +0800 Subject: [PATCH 07/13] Sentry counter --- deepeval/__init__.py | 2 +- deepeval/cli/test.py | 2 ++ deepeval/evaluate.py | 3 +++ deepeval/telemetry.py | 50 ++++++++++++++++++++----------------------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/deepeval/__init__.py b/deepeval/__init__.py index 099bbdfb8..6d6c39dc6 100644 --- a/deepeval/__init__.py +++ b/deepeval/__init__.py @@ -2,13 +2,13 @@ import re # Optionally add telemtry -from .telemetry import * from ._version import __version__ from .decorators.hyperparameters import set_hyperparameters from deepeval.event import track from deepeval.evaluate import evaluate, run_test, assert_test from deepeval.test_run import on_test_run_end +from deepeval.telemetry import * __all__ = [ "set_hyperparameters", diff --git a/deepeval/cli/test.py b/deepeval/cli/test.py index 0187476e8..7ea86e1bb 100644 --- a/deepeval/cli/test.py +++ b/deepeval/cli/test.py @@ -6,6 +6,7 @@ from deepeval.test_run import test_run_manager, TEMP_FILE_NAME from deepeval.utils import delete_file_if_exists from deepeval.test_run import invoke_test_run_end_hook +from deepeval.telemetry import capture_evaluation_count app = typer.Typer(name="test") @@ -74,6 +75,7 @@ def run( pytest_args.extend(["-p", "plugins"]) retcode = pytest.main(pytest_args) + capture_evaluation_count() test_run_manager.wrap_up_test_run() invoke_test_run_end_hook() diff --git a/deepeval/evaluate.py b/deepeval/evaluate.py index 53ddf5a36..e433931d5 100644 --- a/deepeval/evaluate.py +++ b/deepeval/evaluate.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import copy +from deepeval.telemetry import capture_evaluation_count from deepeval.progress_context import progress_context from deepeval.metrics import BaseMetric from deepeval.test_case import LLMTestCase @@ -90,6 +91,7 @@ def run_test( test_run_manager.reset() with progress_context("Executing run_test()..."): test_result = execute_test([test_case], metrics, False)[0] + capture_evaluation_count() print_test_result(test_result) print("") print("-" * 70) @@ -120,6 +122,7 @@ def evaluate(test_cases: List[LLMTestCase], metrics: List[BaseMetric]): test_run_manager.reset() with progress_context("Evaluating testcases..."): test_results = execute_test(test_cases, metrics, True) + capture_evaluation_count() for test_result in test_results: print_test_result(test_result) print("") diff --git a/deepeval/telemetry.py b/deepeval/telemetry.py index c0c2e5364..d6915bf11 100644 --- a/deepeval/telemetry.py +++ b/deepeval/telemetry.py @@ -1,9 +1,10 @@ import os import socket import sys +import sentry_sdk -def check_firewall(): +def blocked_by_firewall(): try: socket.create_connection(("www.google.com", 80)) return False @@ -11,29 +12,24 @@ def check_firewall(): return True -if os.getenv("ERROR_REPORTING") == "YES" and not check_firewall(): - try: - import sentry_sdk - - sentry_sdk.init( - dsn="https://5ef587d58109ee45d6544f3657efdd1f@o4506098477236224.ingest.sentry.io/4506098479136768", - # Set traces_sample_rate to 1.0 to capture 100% - # of transactions for performance monitoring. - traces_sample_rate=1.0, - # Set profiles_sample_rate to 1.0 to profile 100% - # of sampled transactions. - # We recommend adjusting this value in production. - profiles_sample_rate=1.0, - ) - - # Add a global error handler - def handle_exception(exc_type, exc_value, exc_traceback): - print({"exc_type": exc_type, "exc_value": exc_value}) - sentry_sdk.capture_exception(exc_value) - sys.__excepthook__(exc_type, exc_value, exc_traceback) - - sys.excepthook = handle_exception - - except ModuleNotFoundError: - # sentry_sdk not installed - pass +def capture_evaluation_count(): + sentry_sdk.capture_message("evaluation ran!") + + +sentry_sdk.init( + dsn="https://5ef587d58109ee45d6544f3657efdd1f@o4506098477236224.ingest.sentry.io/4506098479136768", + profiles_sample_rate=1.0, + traces_sample_rate=1.0, # For performance monitoring + send_default_pii=False, # Don't send personally identifiable information + attach_stacktrace=False, # Don't attach stack traces to messages + default_integrations=False, # Disable Sentry's default integrations +) + +if os.getenv("ERROR_REPORTING") == "YES" and not blocked_by_firewall(): + + def handle_exception(exc_type, exc_value, exc_traceback): + print({"exc_type": exc_type, "exc_value": exc_value}) + sentry_sdk.capture_exception(exc_value) + sys.__excepthook__(exc_type, exc_value, exc_traceback) + + sys.excepthook = handle_exception From 5c7a40a6e22f766d6b184a7cbf3235cc67ef235e Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 19:21:52 +0800 Subject: [PATCH 08/13] Make threshold dynamic --- deepeval/test_run/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepeval/test_run/test_run.py b/deepeval/test_run/test_run.py index 4d1f3d1e0..7ef742632 100644 --- a/deepeval/test_run/test_run.py +++ b/deepeval/test_run/test_run.py @@ -91,7 +91,7 @@ def add_llm_test_case( metrics_metadata = MetricsMetadata( metric=metric.__name__, score=metric.score, - minimumScore=0.5, + minimumScore=metric.minimum_score, reason=metric.reason, ) From 8689b7b299a2ba1caa93ba6068e0b07930a75b33 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip <143328635+penguine-ip@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:32:35 +0800 Subject: [PATCH 09/13] Update README.md --- README.md | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cf3851ef2..286915e03 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@

-**DeepEval** is a simple-to-use, open-source evaluation framework for LLM applications. It is similar to Pytest but specialized for unit testing LLM applications. DeepEval evaluates performance based on metrics such as hallucination, answer relevancy, RAGAS, etc., using LLMs and various other NLP models **locally on your machine**. +**DeepEval** is a simple-to-use, open-source LLM evaluation framework for LLM applications. It is similar to Pytest but specialized for unit testing LLM applications. DeepEval evaluates performance based on metrics such as hallucination, answer relevancy, RAGAS, etc., using LLMs and various other NLP models that runs **locally on your machine**. Whether your application is implemented via RAG or fine-tuning, LangChain or LlamaIndex, DeepEval has you covered. With it, you can easily determine the optimal hyperparameters to improve your RAG pipeline, prevent prompt drifting, or even transition from OpenAI to hosting your own Llama2 with confidence. @@ -26,7 +26,7 @@ Whether your application is implemented via RAG or fine-tuning, LangChain or Lla # Features -- Large variety of ready-to-use evaluation metrics powered by LLMs (all with explanations), statistical methods, or NLP models that runs **locally on your machine**: +- Large variety of ready-to-use LLM evaluation metrics powered by LLMs (all with explanations), statistical methods, or NLP models that runs **locally on your machine**: - Hallucination - Summarization - Answer Relevancy @@ -38,8 +38,8 @@ Whether your application is implemented via RAG or fine-tuning, LangChain or Lla - Toxicity - Bias - etc. +- Evaluate your entire dataset in bulk in under 20 lines of Python code **in parallel**. Do this via the CLI in a Pytest-like manner, or through our `evaluate()` function. - Easily create your own custom metrics that are automatically integrated with DeepEval's ecosystem by inheriting DeepEval's base metric class. -- Evaluate your entire dataset in bulk in under 20 lines of Python code **in parallel**. - [Automatically integrated with Confident AI](https://app.confident-ai.com) for continous evaluation throughout the lifetime of your LLM (app): - log evaluation results and analyze metrics pass / fails - compare and pick the optimal hyperparameters (eg. prompt templates, chunk size, models used, etc.) based on evaluation results @@ -115,6 +115,29 @@ deepeval test run test_chatbot.py
+## Evaluating Without Pytest Integration + +Alternatively, you can evaluate without Pytest, which is more suited for a notebook environment. + +```python +from deepeval import evalate +from deepeval.metrics import HallucinationMetric +from deepeval.test_case import LLMTestCase + +input = "What if these shoes don't fit?" +context = ["All customers are eligible for a 30 day full refund at no extra costs."] +# Replace this with the actual output from your LLM application +actual_output = "We offer a 30-day full refund at no extra costs." + +hallucination_metric = HallucinationMetric(minimum_score=0.7) +test_case = LLMTestCase( + input=input, + actual_output=actual_output, + context=context +) +evalate([test_case], [hallucination_metric]) +``` + ## Evaluting a Dataset / Test Cases in Bulk In DeepEval, a dataset is simply a collection of test cases. Here is how you can evaluate things in bulk: @@ -148,7 +171,7 @@ deepeval test run test_.py -n 4
-Alternatively, although we recommend using `deepeval test run`, you can evaluate a dataset/test cases without using pytest: +Alternatively, although we recommend using `deepeval test run`, you can evaluate a dataset/test cases without using our Pytest integration: ```python from deepeval import evaluate @@ -168,6 +191,7 @@ We offer a [free web platform](https://app.confident-ai.com) for you to: 3. Compare and pick the optimal hyperparameteres (prompt templates, models, chunk size, etc.). 4. Create, manage, and centralize your evaluation datasets. 5. Track events in production and augment your evaluation dataset for continous evaluation in production. +6. Track events in production and view live evaluation results over time. Everything on Confident AI, including how to use Confident is available [here](https://docs.confident-ai.com/docs/confident-ai-introduction). From 7ec7044ef5a2fb210a21d9809a5d43981340c391 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 20:55:30 +0800 Subject: [PATCH 10/13] new release --- deepeval/_version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepeval/_version.py b/deepeval/_version.py index f81ac3edf..ee489183b 100644 --- a/deepeval/_version.py +++ b/deepeval/_version.py @@ -1 +1 @@ -__version__: str = "0.20.43" +__version__: str = "0.20.44" diff --git a/pyproject.toml b/pyproject.toml index 4fd145270..df3ada338 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "deepeval" -version = "0.20.43" +version = "0.20.44" description = "The Evaluation Framework for LLMs" authors = ["Jeffrey Ip "] license = "Apache-2.0" From 5497a1858c8db2788981494fd9b76bb2289458dd Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 21:23:16 +0800 Subject: [PATCH 11/13] added progress loading --- deepeval/metrics/answer_relevancy.py | 31 +++++++++++---------- deepeval/metrics/contextual_precision.py | 34 ++++++++++++------------ deepeval/metrics/contextual_recall.py | 27 ++++++++++--------- deepeval/metrics/contextual_relevancy.py | 29 ++++++++++---------- deepeval/metrics/faithfulness.py | 31 +++++++++++---------- deepeval/progress_context.py | 16 +++++++++++ deepeval/templates.py | 4 ++- tests/test_answer_relevancy.py | 2 +- 8 files changed, 95 insertions(+), 79 deletions(-) diff --git a/deepeval/metrics/answer_relevancy.py b/deepeval/metrics/answer_relevancy.py index 8380976ff..a4cdadbb2 100644 --- a/deepeval/metrics/answer_relevancy.py +++ b/deepeval/metrics/answer_relevancy.py @@ -7,6 +7,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import AnswerRelevancyTemplate +from deepeval.progress_context import metrics_progress_context class AnswerRelvancyVerdict(BaseModel): @@ -35,24 +36,22 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's latest Answer Relevancy Metric! This may take a minute..." - ) - self.key_points: List[str] = self._generate_key_points( - test_case.actual_output, "\n".join(test_case.retrieval_context) - ) - self.verdicts: List[AnswerRelvancyVerdict] = self._generate_verdicts( - test_case.input - ) + with metrics_progress_context(self.__name__): + self.key_points: List[str] = self._generate_key_points( + test_case.actual_output, "\n".join(test_case.retrieval_context) + ) + self.verdicts: List[ + AnswerRelvancyVerdict + ] = self._generate_verdicts(test_case.input) - answer_relevancy_score = self._generate_score() + answer_relevancy_score = self._generate_score() - self.reason = self._generate_reason( - test_case.input, test_case.actual_output, answer_relevancy_score - ) - self.success = answer_relevancy_score >= self.minimum_score - self.score = answer_relevancy_score - return self.score + self.reason = self._generate_reason( + test_case.input, test_case.actual_output, answer_relevancy_score + ) + self.success = answer_relevancy_score >= self.minimum_score + self.score = answer_relevancy_score + return self.score def _generate_score(self): relevant_count = 0 diff --git a/deepeval/metrics/contextual_precision.py b/deepeval/metrics/contextual_precision.py index 622c6c1b1..8c7f0b9d6 100644 --- a/deepeval/metrics/contextual_precision.py +++ b/deepeval/metrics/contextual_precision.py @@ -7,6 +7,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import ContextualPrecisionTemplate +from deepeval.progress_context import metrics_progress_context class ContextualPrecisionVerdict(BaseModel): @@ -36,25 +37,24 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, expected output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's latest Contextual Precision Metric! This may take a minute..." - ) - self.verdicts: List[ - ContextualPrecisionVerdict - ] = self._generate_verdicts( - test_case.input, - test_case.expected_output, - test_case.retrieval_context, - ) - contextual_precision_score = self._generate_score() - self.reason = self._generate_reason( - test_case.input, contextual_precision_score - ) + with metrics_progress_context(self.__name__): + self.verdicts: List[ + ContextualPrecisionVerdict + ] = self._generate_verdicts( + test_case.input, + test_case.expected_output, + test_case.retrieval_context, + ) + contextual_precision_score = self._generate_score() + + self.reason = self._generate_reason( + test_case.input, contextual_precision_score + ) - self.success = contextual_precision_score >= self.minimum_score - self.score = contextual_precision_score - return self.score + self.success = contextual_precision_score >= self.minimum_score + self.score = contextual_precision_score + return self.score def _generate_reason(self, input: str, score: float): if self.include_reason is False: diff --git a/deepeval/metrics/contextual_recall.py b/deepeval/metrics/contextual_recall.py index bdf65878a..347a97463 100644 --- a/deepeval/metrics/contextual_recall.py +++ b/deepeval/metrics/contextual_recall.py @@ -7,6 +7,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import ContextualRecallTemplate +from deepeval.progress_context import metrics_progress_context class ContextualRecallVerdict(BaseModel): @@ -36,22 +37,22 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, expected output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's latest Contextual Recall Metric! This may take a minute..." - ) - self.verdicts: List[ContextualRecallVerdict] = self._generate_verdicts( - test_case.expected_output, test_case.retrieval_context - ) + with metrics_progress_context(self.__name__): + self.verdicts: List[ + ContextualRecallVerdict + ] = self._generate_verdicts( + test_case.expected_output, test_case.retrieval_context + ) - contextual_recall_score = self._generate_score() + contextual_recall_score = self._generate_score() - self.reason = self._generate_reason( - test_case.expected_output, contextual_recall_score - ) + self.reason = self._generate_reason( + test_case.expected_output, contextual_recall_score + ) - self.success = contextual_recall_score >= self.minimum_score - self.score = contextual_recall_score - return self.score + self.success = contextual_recall_score >= self.minimum_score + self.score = contextual_recall_score + return self.score def _generate_reason(self, expected_output: str, score: float): if self.include_reason is False: diff --git a/deepeval/metrics/contextual_relevancy.py b/deepeval/metrics/contextual_relevancy.py index 73ccbfc43..556d0fe5b 100644 --- a/deepeval/metrics/contextual_relevancy.py +++ b/deepeval/metrics/contextual_relevancy.py @@ -8,6 +8,7 @@ from deepeval.metrics import BaseMetric from deepeval.models import GPTModel from deepeval.templates import ContextualRelevancyTemplate +from deepeval.progress_context import metrics_progress_context class ContextualRelevancyVerdict(BaseModel): @@ -35,24 +36,22 @@ def measure(self, test_case: LLMTestCase) -> float: raise ValueError( "Input, actual output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's latest Contextual Relevancy Metric! This may take a minute..." - ) - self.verdicts_list: List[ - List[ContextualRelevancyVerdict] - ] = self._generate_verdicts_list( - test_case.input, test_case.retrieval_context - ) - contextual_recall_score = self._generate_score() + with metrics_progress_context(self.__name__): + self.verdicts_list: List[ + List[ContextualRelevancyVerdict] + ] = self._generate_verdicts_list( + test_case.input, test_case.retrieval_context + ) + contextual_recall_score = self._generate_score() - self.reason = self._generate_reason( - test_case.input, contextual_recall_score - ) + self.reason = self._generate_reason( + test_case.input, contextual_recall_score + ) - self.success = contextual_recall_score >= self.minimum_score - self.score = contextual_recall_score + self.success = contextual_recall_score >= self.minimum_score + self.score = contextual_recall_score - return self.score + return self.score def _generate_reason(self, input: str, score: float): if self.include_reason is False: diff --git a/deepeval/metrics/faithfulness.py b/deepeval/metrics/faithfulness.py index 643ef2c40..74a5ddc45 100644 --- a/deepeval/metrics/faithfulness.py +++ b/deepeval/metrics/faithfulness.py @@ -8,6 +8,7 @@ from deepeval.utils import trimToJson from deepeval.models import GPTModel from deepeval.templates import FaithfulnessTemplate +from deepeval.progress_context import metrics_progress_context class FaithfulnessVerdict(BaseModel): @@ -37,22 +38,20 @@ def measure(self, test_case: LLMTestCase): raise ValueError( "Input, actual output, or retrieval context cannot be None" ) - print( - "✨ 🍰 ✨ You're using DeepEval's latest Faithfulness Metric! This may take a minute..." - ) - self.truths_list: List[List[str]] = self._generate_truths_list( - test_case.retrieval_context - ) - self.verdicts_list: List[ - List[FaithfulnessVerdict] - ] = self._generate_verdicts_list( - self.truths_list, test_case.actual_output - ) - faithfulness_score = self._generate_score() - self.reason = self._generate_reason(faithfulness_score) - self.success = faithfulness_score >= self.minimum_score - self.score = faithfulness_score - return self.score + with metrics_progress_context(self.__name__): + self.truths_list: List[List[str]] = self._generate_truths_list( + test_case.retrieval_context + ) + self.verdicts_list: List[ + List[FaithfulnessVerdict] + ] = self._generate_verdicts_list( + self.truths_list, test_case.actual_output + ) + faithfulness_score = self._generate_score() + self.reason = self._generate_reason(faithfulness_score) + self.success = faithfulness_score >= self.minimum_score + self.score = faithfulness_score + return self.score def _generate_score(self): total_verdicts = 0 diff --git a/deepeval/progress_context.py b/deepeval/progress_context.py index ed1b3c523..f7986a602 100644 --- a/deepeval/progress_context.py +++ b/deepeval/progress_context.py @@ -17,3 +17,19 @@ def progress_context( ) as progress: progress.add_task(description=description, total=total) yield + + +@contextmanager +def metrics_progress_context( + metric_name: str, total: int = 9999, transient: bool = True +): + description = f"✨ 🍰 ✨ You're using DeepEval's latest {metric_name} Metric! This may take a minute..." + console = Console(file=sys.stderr) # Direct output to standard error + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, # Use the custom console + transient=transient, + ) as progress: + progress.add_task(description=description, total=total) + yield diff --git a/deepeval/templates.py b/deepeval/templates.py index b4ae5d584..f32f4b10a 100644 --- a/deepeval/templates.py +++ b/deepeval/templates.py @@ -97,6 +97,7 @@ def generate_verdicts(truths, text): You should NOT incorporate any prior knowledge you have and take each context at face value. Since you are going to generate a verdict for each context, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to that of contexts. You DON'T have to provide a reason if the answer is 'yes'. +You should ONLY provide a 'no' answer if IT IS A CONTRADICTION. ** Retrieval Contexts: @@ -113,7 +114,7 @@ def generate_reason(score, contradictions): return f"""Below is a list of Contradictions. It is a list of JSON with the `contradiction` and `rank` key. The `contradiction` explains why the 'actual output' does not align with a certain node in the 'retrieval context'. Contradictions happen in the 'actual output', NOT the 'retrieval context'. The `rank` tells you which node in the 'retrieval context' the actual output contradicted with. -Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), concisely summarize the contradictions to justify the score. +Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), CONCISELY summarize the contradictions to justify the score. Faithfulness Score: {score} @@ -128,6 +129,7 @@ def generate_reason(score, contradictions): IMPORTANT: If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying). Your reason MUST use information in `contradiction` and the node RANK (eg., first node of the retrieval context) in your reason. +Be sure in your reason, as if you know what the actual output is from the contradictions. ** Reason: diff --git a/tests/test_answer_relevancy.py b/tests/test_answer_relevancy.py index 8d828cc82..c1de4e866 100644 --- a/tests/test_answer_relevancy.py +++ b/tests/test_answer_relevancy.py @@ -2,7 +2,7 @@ """ import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics import AnswerRelevancyMetric +from deepeval.metrics import AnswerRelevancyMetric, FaithfulnessMetric from deepeval import assert_test question = "What are the primary benefits of meditation?" From 7ea687e9eec24fd3836bf06f043099d254c83d40 Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 21:50:50 +0800 Subject: [PATCH 12/13] Remove import --- tests/test_answer_relevancy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_answer_relevancy.py b/tests/test_answer_relevancy.py index c1de4e866..8d828cc82 100644 --- a/tests/test_answer_relevancy.py +++ b/tests/test_answer_relevancy.py @@ -2,7 +2,7 @@ """ import pytest from deepeval.test_case import LLMTestCase -from deepeval.metrics import AnswerRelevancyMetric, FaithfulnessMetric +from deepeval.metrics import AnswerRelevancyMetric from deepeval import assert_test question = "What are the primary benefits of meditation?" From f476ea61332910a82b952f99abbf73e26ed112eb Mon Sep 17 00:00:00 2001 From: Jeffrey Ip Date: Wed, 3 Jan 2024 22:35:53 +0800 Subject: [PATCH 13/13] updated docs --- docs/docs/confident-ai-evals-in-production.mdx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/docs/confident-ai-evals-in-production.mdx b/docs/docs/confident-ai-evals-in-production.mdx index e08abb723..98db35258 100644 --- a/docs/docs/confident-ai-evals-in-production.mdx +++ b/docs/docs/confident-ai-evals-in-production.mdx @@ -23,7 +23,8 @@ Simply add `deepeval.track(...)` in your application to start tracking events. T - [Optional] `token_usage`: type `float` - [Optional] `token_cost`: type `float` - [Optional] `additional_data`: type `dict` -- [Optional] `fail_silently`: type `bool`, defaults to True +- [Optional] `fail_silently`: type `bool`, defaults to True. You should try setting this to `False` if your events are not logging properly. +- [Optional] `run_on_background_thread`: type `bool`, defaults to True. You should try setting this to `False` if your events are not logging properly. :::note Please do **NOT** provide placeholder values for optional parameters. Leave it blank instead. @@ -48,6 +49,7 @@ deepeval.track( token_cost=0.23, additional_data={"example": "example"}, fail_silently=True + run_on_background_thread=True ) ```