From 94dfff1d38c18d9de4adf663314f5e63d77cd094 Mon Sep 17 00:00:00 2001 From: lievan Date: Tue, 10 Dec 2024 23:32:17 -0500 Subject: [PATCH 01/15] ragas context precision implementation; need to refactor --- .../_evaluators/ragas/context_precision.py | 316 ++++++++++++++++++ ddtrace/llmobs/_evaluators/ragas/models.py | 7 + ddtrace/llmobs/_evaluators/runner.py | 2 + tests/llmobs/_utils.py | 42 ++- ...ator.py => test_llmobs_ragas_evaluator.py} | 225 ++++++++++++- 5 files changed, 588 insertions(+), 4 deletions(-) create mode 100644 ddtrace/llmobs/_evaluators/ragas/context_precision.py rename tests/llmobs/{test_llmobs_ragas_faithfulness_evaluator.py => test_llmobs_ragas_evaluator.py} (52%) diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py new file mode 100644 index 00000000000..92234cd9bc7 --- /dev/null +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -0,0 +1,316 @@ +import math +import traceback +from typing import Optional +from typing import Tuple +from typing import Union + +from ddtrace.internal.logger import get_logger +from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL +from ddtrace.internal.utils.version import parse_version +from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA +from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA +from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS +from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS +from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX + + +logger = get_logger(__name__) + + +class MiniRagas: + """ + A helper class to store instances of ragas classes and functions + that may or may not exist in a user's environment. + """ + + llm_factory = None + RagasoutputParser = None + ensembler = None + context_precision = None + ContextPrecisionVerification = None + + +def _get_ml_app_for_ragas_trace(span_event: dict) -> str: + """ + The `ml_app` spans generated from traces of ragas will be named as `dd-ragas-` + or `dd-ragas` if `ml_app` is not present in the span event. + """ + tags = span_event.get("tags", []) # list[str] + ml_app = None + for tag in tags: + if isinstance(tag, str) and tag.startswith("ml_app:"): + ml_app = tag.split(":")[1] + break + if not ml_app: + return RAGAS_ML_APP_PREFIX + return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) + + +def _get_context_precision_instance() -> Optional[object]: + """ + This helper function ensures the context precision instance used in + ragas evaluator is updated with the latest ragas context precision instance + instance AND has an non-null llm + """ + if MiniRagas.context_precision is None: + return None + ragas_context_precision_instance = MiniRagas.context_precision + if not ragas_context_precision_instance.llm: + ragas_context_precision_instance.llm = MiniRagas.llm_factory() + return ragas_context_precision_instance + + +class RagasContextPrecisionEvaluator: + """A class used by EvaluatorRunner to conduct ragas context precision evaluations + on LLM Observability span events. The job of an Evaluator is to take a span and + submit evaluation metrics based on the span's attributes. + """ + + LABEL = "ragas_context_precision" + METRIC_TYPE = "score" + + def __init__(self, llmobs_service): + """ + Initialize an evaluator that uses the ragas library to generate a context precision score on finished LLM spans. + + Context Precision is a metric that verifies if the context was useful in arriving at the given answer. + We compute this by dividing the number of relevant contexts by the total number of contexts. + Note that this is slightly modified from the original context precision metric in ragas, which computes + the mean of the precision @ rank k for each chunk in the context (where k is the number of + retrieved context chunks). + + For more information, see https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/context_precision/ + + The `ragas.metrics.context_precision` instance is used for context precision scores. + If there is no llm attribute set on this instance, it will be set to the + default `llm_factory()` which uses openai. + + :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and + submitting evaluation metrics. + + Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. + """ + self.llmobs_service = llmobs_service + self.ragas_version = "unknown" + telemetry_state = "ok" + try: + import ragas + + self.ragas_version = parse_version(ragas.__version__) + if self.ragas_version >= (0, 2, 0) or self.ragas_version < (0, 1, 10): + raise NotImplementedError( + "Ragas version: {} is not supported for `ragas_context_precision` evaluator".format( + self.ragas_version + ), + ) + + from ragas.llms import llm_factory + + MiniRagas.llm_factory = llm_factory + + from ragas.llms.output_parser import RagasoutputParser + + MiniRagas.RagasoutputParser = RagasoutputParser + + from ragas.metrics import context_precision + + MiniRagas.context_precision = context_precision + + from ragas.metrics.base import ensembler + + MiniRagas.ensembler = ensembler + + from ddtrace.llmobs._evaluators.ragas.models import ContextPrecisionVerification + + MiniRagas.ContextPrecisionVerification = ContextPrecisionVerification + + except Exception as e: + telemetry_state = "fail" + telemetry_writer.add_log( + level=TELEMETRY_LOG_LEVEL.ERROR, + message="Failed to import Ragas dependencies", + stack_trace=traceback.format_exc(), + tags={"ragas_version": self.ragas_version}, + ) + raise NotImplementedError("Failed to load dependencies for `ragas_context_precision` evaluator") from e + finally: + telemetry_writer.add_count_metric( + namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + name="evaluators.init", + value=1, + tags=( + ("evaluator_label", self.LABEL), + ("state", telemetry_state), + ("ragas_version", self.ragas_version), + ), + ) + + self.ragas_context_precision_instance = _get_context_precision_instance() + self.context_precision_output_parser = MiniRagas.RagasoutputParser( + pydantic_object=MiniRagas.ContextPrecisionVerification + ) + + def run_and_submit_evaluation(self, span_event: dict): + if not span_event: + return + score_result_or_failure, metric_metadata = self.evaluate(span_event) + telemetry_writer.add_count_metric( + TELEMETRY_APM_PRODUCT.LLMOBS, + "evaluators.run", + 1, + tags=( + ("evaluator_label", self.LABEL), + ("state", score_result_or_failure if isinstance(score_result_or_failure, str) else "success"), + ), + ) + if isinstance(score_result_or_failure, float): + self.llmobs_service.submit_evaluation( + span_context={"trace_id": span_event.get("trace_id"), "span_id": span_event.get("span_id")}, + label=self.LABEL, + metric_type=self.METRIC_TYPE, + value=score_result_or_failure, + metadata=metric_metadata, + ) + + def _extract_inputs(self, span_event: dict) -> Optional[dict]: + """ + Extracts the question, answer, and context used as inputs to faithfulness + evaluation from a span event. + + question - input.prompt.variables.question OR input.messages[-1].content + context - input.prompt.variables.context + answer - output.messages[-1].content + """ + with self.llmobs_service.workflow("dd-ragas.extract_context_precision_inputs") as extract_inputs_workflow: + self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) + question, answer, context = None, None, None + + meta_io = span_event.get("meta") + if meta_io is None: + return None + + meta_input = meta_io.get("input") + meta_output = meta_io.get("output") + + if not (meta_input and meta_output): + return None + + prompt = meta_input.get("prompt") + if prompt is None: + logger.debug("Failed to extract `prompt` from span for `ragas_faithfulness` evaluation") + return None + prompt_variables = prompt.get("variables") + + input_messages = meta_input.get("messages") + + messages = meta_output.get("messages") + if messages is not None and len(messages) > 0: + answer = messages[-1].get("content") + + if prompt_variables: + context_keys = prompt.get(INTERNAL_CONTEXT_VARIABLE_KEYS, ["context"]) + question_keys = prompt.get(INTERNAL_QUERY_VARIABLE_KEYS, ["question"]) + contexts = [prompt_variables.get(key) for key in context_keys if prompt_variables.get(key)] + question = " ".join([prompt_variables.get(key) for key in question_keys if prompt_variables.get(key)]) + + if not question and input_messages is not None and len(input_messages) > 0: + question = input_messages[-1].get("content") + + self.llmobs_service.annotate( + span=extract_inputs_workflow, output_data={"question": question, "context": context, "answer": answer} + ) + if any(field is None for field in (question, contexts, answer)): + logger.debug("Failed to extract inputs required for faithfulness evaluation") + return None + + return {"question": question, "contexts": contexts, "answer": answer} + + def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: + """ + Performs a context precision evaluation on a retrieval span event, returning either + - context precision score (float) OR failure reason (str) + - evaluation metadata (dict) + If the ragas context precision instance does not have `llm` set, we set `llm` using the `llm_factory()` + method from ragas which currently defaults to openai's gpt-4o-turbo. + """ + self.ragas_context_precision_instance = _get_context_precision_instance() + if not self.ragas_context_precision_instance: + return "fail_context_precision_is_none", {} + + evaluation_metadata = {EVALUATION_KIND_METADATA: "context_precision"} # type: dict[str, Union[str, dict, list]] + + # initialize data we annotate for tracing ragas + score, question, answer = ( + math.nan, + None, + None, + ) + + with self.llmobs_service.workflow( + "dd-ragas.context_precision", ml_app=_get_ml_app_for_ragas_trace(span_event) + ) as ragas_cp_workflow: + try: + evaluation_metadata[EVALUATION_SPAN_METADATA] = self.llmobs_service.export_span(span=ragas_cp_workflow) + + cp_inputs = self._extract_inputs(span_event) + if cp_inputs is None: + logger.debug( + "Failed to extract question and contexts from " + "span sampled for `ragas_context_precision` evaluation" + ) + return "fail_extract_context_precision_inputs", evaluation_metadata + + question = cp_inputs["question"] + contexts = cp_inputs["contexts"] + answer = cp_inputs["answer"] + + # create a prompt to evaluate each context chunk + ctx_precision_prompts = [ + self.ragas_context_precision_instance.context_precision_prompt.format( + question=question, context=c, answer=answer + ) + for c in contexts + ] + + responses = [] + + for prompt in ctx_precision_prompts: + result = self.ragas_context_precision_instance.llm.generate_text(prompt) + reproducibility = getattr(self.ragas_context_precision_instance, "_reproducibility", 1) + + results = [result.generations[0][i].text for i in range(reproducibility)] + responses.append( + [ + res.dict() + for res in [self.context_precision_output_parser.parse(text) for text in results] + if res is not None + ] + ) + + answers = [] + for response in responses: + agg_answer = MiniRagas.ensembler.from_discrete([response], "verdict") + if agg_answer: + try: + agg_answer = MiniRagas.ContextPrecisionVerification.parse_obj(agg_answer[0]) + except Exception as e: + logger.debug( + "Failed to parse context precision verification for `ragas_context_precision`", + exc_info=e, + ) + continue + answers.append(agg_answer) + + if len(answers) == 0: + return "fail_no_answers", evaluation_metadata + + verdict_list = [1 if ver.verdict else 0 for ver in answers] + return sum(verdict_list) / len(verdict_list), evaluation_metadata + finally: + self.llmobs_service.annotate( + span=ragas_cp_workflow, + input_data=span_event, + output_data=score, + ) diff --git a/ddtrace/llmobs/_evaluators/ragas/models.py b/ddtrace/llmobs/_evaluators/ragas/models.py index 5ee4d433c33..9886c7cf1d3 100644 --- a/ddtrace/llmobs/_evaluators/ragas/models.py +++ b/ddtrace/llmobs/_evaluators/ragas/models.py @@ -11,6 +11,13 @@ """ +class ContextPrecisionVerification(BaseModel): + """Answer for the verification task whether the context was useful.""" + + reason: str = Field(..., description="Reason for verification") + verdict: int = Field(..., description="Binary (0/1) verdict of verification") + + class StatementFaithfulnessAnswer(BaseModel): statement: str = Field(..., description="the original statement, word-by-word") reason: str = Field(..., description="the reason of the verdict") diff --git a/ddtrace/llmobs/_evaluators/runner.py b/ddtrace/llmobs/_evaluators/runner.py index bf45e618e01..babca5b5274 100644 --- a/ddtrace/llmobs/_evaluators/runner.py +++ b/ddtrace/llmobs/_evaluators/runner.py @@ -8,6 +8,7 @@ from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.telemetry import telemetry_writer from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.llmobs._evaluators.ragas.context_precision import RagasContextPrecisionEvaluator from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator from ddtrace.llmobs._evaluators.sampler import EvaluatorRunnerSampler @@ -17,6 +18,7 @@ SUPPORTED_EVALUATORS = { RagasFaithfulnessEvaluator.LABEL: RagasFaithfulnessEvaluator, + RagasContextPrecisionEvaluator.LABEL: RagasContextPrecisionEvaluator, } diff --git a/tests/llmobs/_utils.py b/tests/llmobs/_utils.py index d39e69808fb..8fbcbffa28e 100644 --- a/tests/llmobs/_utils.py +++ b/tests/llmobs/_utils.py @@ -516,7 +516,47 @@ def _dummy_evaluator_eval_metric_event(span_id, trace_id): ) -def _expected_ragas_spans(ragas_inputs=None): +def _expected_ragas_context_precision_spans(ragas_inputs=None): + if not ragas_inputs: + ragas_inputs = default_ragas_inputs + return [ + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": "undefined", + "name": "dd-ragas.context_precision", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": "1.0"}, + "metadata": {}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": mock.ANY, + "name": "dd-ragas.extract_context_precision_inputs", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + ] + + +def _expected_ragas_faithfulness_spans(ragas_inputs=None): if not ragas_inputs: ragas_inputs = default_ragas_inputs return [ diff --git a/tests/llmobs/test_llmobs_ragas_faithfulness_evaluator.py b/tests/llmobs/test_llmobs_ragas_evaluator.py similarity index 52% rename from tests/llmobs/test_llmobs_ragas_faithfulness_evaluator.py rename to tests/llmobs/test_llmobs_ragas_evaluator.py index 1f78b538f24..1a71d04d61c 100644 --- a/tests/llmobs/test_llmobs_ragas_faithfulness_evaluator.py +++ b/tests/llmobs/test_llmobs_ragas_evaluator.py @@ -3,10 +3,12 @@ import mock import pytest +from ddtrace.llmobs._evaluators.ragas.context_precision import RagasContextPrecisionEvaluator from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator from ddtrace.span import Span from tests.llmobs._utils import _expected_llmobs_llm_span_event -from tests.llmobs._utils import _expected_ragas_spans +from tests.llmobs._utils import _expected_ragas_context_precision_spans +from tests.llmobs._utils import _expected_ragas_faithfulness_spans from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_prompt @@ -15,7 +17,7 @@ def _llm_span_without_io(): return _expected_llmobs_llm_span_event(Span("dummy")) -def test_ragas_evaluator_init(ragas, LLMObs): +def test_ragas_faithfulness_evaluator_init(ragas, LLMObs): rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) assert rf_evaluator.llmobs_service == LLMObs assert rf_evaluator.ragas_faithfulness_instance == ragas.metrics.faithfulness @@ -176,7 +178,7 @@ def test_ragas_faithfulness_emits_traces(ragas, LLMObs): spans = [call[0][0] for call in calls] # check name, io, span kinds match - assert spans == _expected_ragas_spans() + assert spans == _expected_ragas_faithfulness_spans() # verify the trace structure root_span = spans[0] @@ -247,3 +249,220 @@ def test_llmobs_with_faithfulness_emits_traces_and_evals_on_exit(mock_writer_log assert status == 0, err assert out == b"" assert err == b"" + + +def test_ragas_context_precision_evaluator_init(ragas, LLMObs): + rcp_evaluator = RagasContextPrecisionEvaluator(LLMObs) + assert rcp_evaluator.llmobs_service == LLMObs + assert rcp_evaluator.ragas_context_precision_instance == ragas.metrics.faithfulness + assert rcp_evaluator.ragas_context_precision_instance.llm == ragas.llms.llm_factory() + + +def test_ragas_context_precision_throws_if_dependencies_not_present(LLMObs, mock_ragas_dependencies_not_present, ragas): + with pytest.raises( + NotImplementedError, match="Failed to load dependencies for `ragas_context_precision` evaluator" + ): + RagasContextPrecisionEvaluator(LLMObs) + + +def test_ragas_context_precision_returns_none_if_inputs_extraction_fails(ragas, mock_llmobs_submit_evaluation, LLMObs): + rcp_evaluator = RagasContextPrecisionEvaluator(LLMObs) + failure_msg, _ = rcp_evaluator.evaluate(_llm_span_without_io()) + assert failure_msg == "fail_extract_context_precision_inputs" + assert rcp_evaluator.llmobs_service.submit_evaluation.call_count == 0 + + +def test_ragas_context_precision_has_modified_context_precision_instance( + ragas, mock_llmobs_submit_evaluation, reset_ragas_context_precision_llm, LLMObs +): + """Context precision instance used in ragas evaluator should match the global ragas context precision instance""" + from ragas.llms import BaseRagasLLM + from ragas.metrics import context_precision + + class FirstDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "dummy llm" + + def agenerate_text(self) -> str: + return "dummy llm" + + context_precision.llm = FirstDummyLLM() + + rf_evaluator = RagasContextPrecisionEvaluator(LLMObs) + + assert rf_evaluator.ragas_context_precision_instance.llm.generate_text() == "dummy llm" + + class SecondDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "second dummy llm" + + def agenerate_text(self) -> str: + return "second dummy llm" + + context_precision.llm = SecondDummyLLM() + + rf_evaluator = RagasContextPrecisionEvaluator(LLMObs) + + assert rf_evaluator.ragas_context_precision_instance.llm.generate_text() == "second dummy llm" + + +@pytest.mark.vcr_logs +def test_ragas_context_precision_submits_evaluation(ragas, LLMObs, mock_llmobs_submit_evaluation): + """Test that evaluation is submitted for a valid llm span where question is in the prompt variables""" + rf_evaluator = RagasContextPrecisionEvaluator(LLMObs) + llm_span = _llm_span_with_expected_ragas_inputs_in_prompt() + rf_evaluator.run_and_submit_evaluation(llm_span) + rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasContextPrecisionEvaluator.LABEL, + metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, + value=1.0, + metadata={"_dd.evaluation_kind": "context_precision"}, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_context_precision_submits_evaluation_on_span_with_question_in_messages( + ragas, LLMObs, mock_llmobs_submit_evaluation +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rf_evaluator = RagasContextPrecisionEvaluator(LLMObs) + llm_span = _llm_span_with_expected_ragas_inputs_in_messages() + rf_evaluator.run_and_submit_evaluation(llm_span) + rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasContextPrecisionEvaluator.LABEL, + metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, + value=1.0, + metadata={"_dd.evaluation_kind": "context_precision"}, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys( + ragas, LLMObs, mock_llmobs_submit_evaluation +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rf_evaluator = RagasContextPrecisionEvaluator(LLMObs) + llm_span = _expected_llmobs_llm_span_event( + Span("dummy"), + prompt={ + "variables": { + "user_input": "Is france part of europe?", + "context_2": "irrelevant", + "context_3": "France is part of europe", + }, + "_dd_context_variable_keys": ["context_2", "context_3"], + "_dd_query_variable_keys": ["user_input"], + }, + output_messages=[{"content": "France is indeed part of europe"}], + ) + rf_evaluator.run_and_submit_evaluation(llm_span) + rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasContextPrecisionEvaluator.LABEL, + metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, + value=0.5, + metadata={"_dd.evaluation_kind": "context_precision"}, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_context_precision_emits_traces(ragas, LLMObs): + rf_evaluator = RagasContextPrecisionEvaluator(LLMObs) + rf_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) + assert rf_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_count == 2 + calls = rf_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_args_list + + spans = [call[0][0] for call in calls] + + # check name, io, span kinds match + assert spans == _expected_ragas_context_precision_spans() + + # verify the trace structure + root_span = spans[0] + root_span_id = root_span["span_id"] + assert root_span["parent_id"] == "undefined" + assert root_span["meta"] is not None + + root_span_trace_id = root_span["trace_id"] + for child_span in spans[1:]: + assert child_span["trace_id"] == root_span_trace_id + assert child_span["parent_id"] == root_span_id + + +def test_llmobs_with_context_precision_emits_traces_and_evals_on_exit(mock_writer_logs, run_python_code_in_subprocess): + env = os.environ.copy() + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] + if "PYTHONPATH" in env: + pypath.append(env["PYTHONPATH"]) + env.update( + { + "DD_API_KEY": os.getenv("DD_API_KEY", "dummy-api-key"), + "DD_SITE": "datad0g.com", + "PYTHONPATH": ":".join(pypath), + "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", "dummy-openai-api-key"), + "DD_LLMOBS_ML_APP": "unnamed-ml-app", + "_DD_LLMOBS_EVALUATOR_INTERVAL": "5", + "_DD_LLMOBS_EVALUATORS": "ragas_context_precision", + "DD_LLMOBS_AGENTLESS_ENABLED": "true", + } + ) + out, err, status, pid = run_python_code_in_subprocess( + """ +import os +import time +import atexit +import mock +from ddtrace.llmobs import LLMObs +from ddtrace.internal.utils.http import Response +from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages +from tests.llmobs._utils import logs_vcr + +ctx = logs_vcr.use_cassette( + "tests.llmobs.test_llmobs_ragas_context_precision_evaluator.emits_traces_and_evaluations_on_exit.yaml" +) +ctx.__enter__() +atexit.register(lambda: ctx.__exit__()) +with mock.patch( + "ddtrace.internal.writer.HTTPWriter._send_payload", + return_value=Response( + status=200, + body="{}", + ), +): + LLMObs.enable() + LLMObs._instance._evaluator_runner.enqueue(_llm_span_with_expected_ragas_inputs_in_messages(), None) +""", + env=env, + ) + assert status == 0, err + assert out == b"" + assert err == b"" From 72b71f0d5631bbe1faff86273a793d2e1eebaf68 Mon Sep 17 00:00:00 2001 From: lievan Date: Wed, 11 Dec 2024 16:15:14 -0500 Subject: [PATCH 02/15] tests wip --- ...lmobs_ragas_evaluator.py => test_llmobs_ragas_evaluators.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/llmobs/{test_llmobs_ragas_evaluator.py => test_llmobs_ragas_evaluators.py} (99%) diff --git a/tests/llmobs/test_llmobs_ragas_evaluator.py b/tests/llmobs/test_llmobs_ragas_evaluators.py similarity index 99% rename from tests/llmobs/test_llmobs_ragas_evaluator.py rename to tests/llmobs/test_llmobs_ragas_evaluators.py index 1a71d04d61c..3e8228d5c66 100644 --- a/tests/llmobs/test_llmobs_ragas_evaluator.py +++ b/tests/llmobs/test_llmobs_ragas_evaluators.py @@ -254,7 +254,7 @@ def test_llmobs_with_faithfulness_emits_traces_and_evals_on_exit(mock_writer_log def test_ragas_context_precision_evaluator_init(ragas, LLMObs): rcp_evaluator = RagasContextPrecisionEvaluator(LLMObs) assert rcp_evaluator.llmobs_service == LLMObs - assert rcp_evaluator.ragas_context_precision_instance == ragas.metrics.faithfulness + assert rcp_evaluator.ragas_context_precision_instance == ragas.metrics.context_precision assert rcp_evaluator.ragas_context_precision_instance.llm == ragas.llms.llm_factory() From 4d57d8305169a51ed6a86b63024ca1a3be513a40 Mon Sep 17 00:00:00 2001 From: lievan Date: Wed, 11 Dec 2024 20:49:39 -0500 Subject: [PATCH 03/15] working tests --- .../_evaluators/ragas/context_precision.py | 7 +- tests/llmobs/_utils.py | 1 - tests/llmobs/conftest.py | 9 + ....emits_traces_and_evaluations_on_exit.yaml | 197 +++++++++++ ..._ragas_context_precision_emits_traces.yaml | 160 +++++++++ ..._context_precision_submits_evaluation.yaml | 160 +++++++++ ...s_evaluation_on_span_with_custom_keys.yaml | 312 ++++++++++++++++++ ...ion_on_span_with_question_in_messages.yaml | 157 +++++++++ tests/llmobs/test_llmobs_ragas_evaluators.py | 25 +- 9 files changed, 1016 insertions(+), 12 deletions(-) create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_context_precision_evaluator.emits_traces_and_evaluations_on_exit.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_emits_traces.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_question_in_messages.yaml diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index 92234cd9bc7..f76b56e305d 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -48,7 +48,7 @@ def _get_ml_app_for_ragas_trace(span_event: dict) -> str: return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) -def _get_context_precision_instance() -> Optional[object]: +def _get_context_precision_instance(): """ This helper function ensures the context precision instance used in ragas evaluator is updated with the latest ragas context precision instance @@ -289,7 +289,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] ] ) - answers = [] + answers = [] # type: list[MiniRagas.ContextPrecisionVerification] for response in responses: agg_answer = MiniRagas.ensembler.from_discrete([response], "verdict") if agg_answer: @@ -307,7 +307,8 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] return "fail_no_answers", evaluation_metadata verdict_list = [1 if ver.verdict else 0 for ver in answers] - return sum(verdict_list) / len(verdict_list), evaluation_metadata + score = sum(verdict_list) / len(verdict_list) + return score, evaluation_metadata finally: self.llmobs_service.annotate( span=ragas_cp_workflow, diff --git a/tests/llmobs/_utils.py b/tests/llmobs/_utils.py index 8fbcbffa28e..07b35280772 100644 --- a/tests/llmobs/_utils.py +++ b/tests/llmobs/_utils.py @@ -532,7 +532,6 @@ def _expected_ragas_context_precision_spans(ragas_inputs=None): "span.kind": "workflow", "input": {"value": mock.ANY}, "output": {"value": "1.0"}, - "metadata": {}, }, "metrics": {}, "tags": expected_ragas_trace_tags(), diff --git a/tests/llmobs/conftest.py b/tests/llmobs/conftest.py index 0b0ce8b7964..acb79b5f413 100644 --- a/tests/llmobs/conftest.py +++ b/tests/llmobs/conftest.py @@ -205,6 +205,15 @@ def reset_ragas_faithfulness_llm(): ragas.metrics.faithfulness.llm = previous_llm +@pytest.fixture +def reset_ragas_context_precision_llm(): + import ragas + + previous_llm = ragas.metrics.context_precision.llm + yield + ragas.metrics.context_precision.llm = previous_llm + + @pytest.fixture def mock_ragas_evaluator(mock_llmobs_eval_metric_writer, ragas): patcher = mock.patch("ddtrace.llmobs._evaluators.ragas.faithfulness.RagasFaithfulnessEvaluator.evaluate") diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_context_precision_evaluator.emits_traces_and_evaluations_on_exit.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_context_precision_evaluator.emits_traces_and_evaluations_on_exit.yaml new file mode 100644 index 00000000000..1fbb302c105 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_context_precision_evaluator.emits_traces_and_evaluations_on_exit.yaml @@ -0,0 +1,197 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"What is the capital of France?\"\ncontext: \"The capital of France is Paris.\"\nanswer: + \"The capital of France is Paris\"\nverification: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4637' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFJNj9owFLznVzz5TFawrAjk1qLurdVKW7Vqmyox9gsxdWzLfnysEP+9 + cggE1D30EiVv3ozmzeSYADAlWQ5MNJxE63T6Qb5+W9ZfZPNz8/HF2/mrMPzH8vPy0/fn3Z6NIsOu + NijownoQtnUaSVlzhoVHThhVJ9l0uphl2WTeAa2VqCNt7Sh9smmrjEofx49P6ThLJ/Oe3VglMLAc + fiUAAMfuGX0aiQeWw3h0mbQYAl8jy69LAMxbHSeMh6ACcUNsNIDCGkLTWa+qahOsKcyxYB55fGU5 + FOxrg+C83SmJErr9A4FUHgXpNwjECQNQwwmoQRDcKeIabA3PnhuBoAK8cK/CCPaNEk38xgPvyB5r + jYJQgjIdm5uwR/9QsBEUbIdeKkHRxeRUmKqqbp17rLeBx/TMVut+frpGoe3aebsKPX6d18qo0JTn + A+PZgaxjHXpKAH53kW/vUmTO29ZRSfYPmii4mPWRs6HpAZ3OepAscX3DWlyAO71SInGlw01pTHDR + oByoQ8N8K5W9AZKbq/918572+XJl1v8jPwBCoCOUpfMYS7m7eFjzuOn6fH/tmnJnmIW3QNiWtTJr + 9M6r829Yu3K14lMxx2y8Yskp+QsAAP//AwD3kxRElAMAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f0a03dada4a1a07-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:41:59 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=4cLkRN3PY0ywDy6QMgqCvUo.CNS0TwsQcaD5jxK7_gY-1733967719-1.0.1.1-2.RXD8FxUii4MNFP7.rwVGA.kO1IWKZO2S1Tk_qg6JQnbvS1CdSa5Zs9iY2u.LeDL7LSdOj_wu_vSeownk1aaA; + path=/; expires=Thu, 12-Dec-24 02:11:59 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=vyJTsjtFxMQ60N5X2hl.15Ts.0C6n9TY2Udt2zwtF1k-1733967719343-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '1298' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998897' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_43f2a90eb7cbf539dca15be67231801c + status: + code: 200 + message: OK +- request: + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": + "171867700624094790", "trace_id": "675a3f650000000010166e6542ea0ba8", "label": + "ragas_context_precision", "metric_type": "score", "timestamp_ms": 1733967719288, + "score_value": 1.0, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.15.0.dev406+ge04c8acf7.d20241210", + "ml_app:unnamed-ml-app"], "metadata": {"_dd.evaluation_kind": "context_precision", + "_dd.evaluation_span": {"span_id": "8227163496056680773", "trace_id": "675a3f6500000000e42b67cfbbbba3b1"}}}]}}}' + headers: + Content-Type: + - application/json + DD-API-KEY: + - XXXXXX + method: POST + uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + response: + body: + string: '{"data":{"id":"92ff21e8-7018-4a29-b8c7-21c24b9fa74d","type":"evaluation_metric","attributes":{"metrics":[{"id":"giRg5dS154","join_on":{"span":{"trace_id":"675a3f650000000010166e6542ea0ba8","span_id":"171867700624094790"}},"trace_id":"675a3f650000000010166e6542ea0ba8","span_id":"171867700624094790","timestamp_ms":1733967719288,"ml_app":"unnamed-ml-app","metric_type":"score","label":"ragas_context_precision","score_value":1,"tags":["ddtrace.version:2.15.0.dev406+ge04c8acf7.d20241210","ml_app:unnamed-ml-app"]}]}}}' + headers: + content-length: + - '516' + content-security-policy: + - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com + content-type: + - application/vnd.api+json + date: + - Thu, 12 Dec 2024 01:42:00 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + status: + code: 202 + message: Accepted +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_emits_traces.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_emits_traces.yaml new file mode 100644 index 00000000000..c120a47a91a --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_emits_traces.yaml @@ -0,0 +1,160 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"What is the capital of France?\"\ncontext: \"The capital of France is Paris.\"\nanswer: + \"The capital of France is Paris\"\nverification: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4637' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA4xTy27bMBC86ysWPFuBFLt27VsvveQSIEaDoCokmlxJTCiSINexA8P/XlB+SEZS + oBeB2tkZzM6ShwSAKclWwETLSXROpz/k06OUv4zLvz2spdYPzy9PM77T+Tp7fmGTyLCbVxR0Yd0J + 2zmNpKw5wcIjJ4yq+WI6Xc4X08W0BzorUUda4yid2bRTRqX32f0szRZp/v3Mbq0SGNgKficAAIf+ + G30aiXu2gmxyqXQYAm+Qra5NAMxbHSuMh6ACcUNsMoDCGkLTW6+q6jVYU5hDwTzyeGQrKNi6RXDe + viuJEvr+PYFUHgXpDwjECQNQywmoRRDcKeIabA0/PTcCQQV45F6FCexaJdr4j3vek3cXFjdhhx48 + 1hoFhbuCTaBg7+ilEhRt5MfCVFU1tu6x3gYe4zNbrc/14zULbRvn7Sac8Wu9VkaFtjxNGOcOZB3r + 0WMC8KfPfHsTI3Pedo5Ksm9oouByfs6cDase0On8DJIlrkes5QW40SslElc6jLbGBBctyoE6rJhv + pbIjIBlN/dnNV9qnyZVp/kd+AIRARyhL5zEu5Wbioc1jfAn/arum3Btm4SMQdmWtTIPeeXW6h7Ur + 57XIM8wz3LDkmPwFAAD//wMAQ0FUzJUDAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09fb708ef90f6c-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:36:14 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=fCcmxnYO8u.R3nFktk2HHMIDMPviv2L8kLBoQUp.otM-1733967374-1.0.1.1-IJYA9.9mXpPiazN87fvwosH.W6jEkLbDR4NrfOTBCMdEGr4uneuGZBVyC3d3QPo0mSFlfaz6ngV4DYT5qDGsBw; + path=/; expires=Thu, 12-Dec-24 02:06:14 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=Qz70tOtm5D3_PIVD6i_zDOqY_g36ZmAJ8dpG7vQH9K4-1733967374352-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '872' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998897' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_7adcf8448902ffcd3434ae348eaac9d8 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation.yaml new file mode 100644 index 00000000000..34a4a99a372 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation.yaml @@ -0,0 +1,160 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"What is the capital of France?\"\ncontext: \"The capital of France is Paris.\"\nanswer: + \"The capital of France is Paris\"\nverification: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4637' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFJBbtswELzrFQuercCyAzvxrZf20qIpEqOHqpAociXRpUiCXMcODP+9 + IO1YNppDL4K0szOYndEhA2BKshUw0XMSg9P5J/n89fuP55/Ft123XTfFk5qpL6Zbv6z3irNJZNhm + g4LeWXfCDk4jKWtOsPDICaNqsZzPHxfLopgmYLASdaR1jvJ7mw/KqHw2nd3n02VePJzZvVUCA1vB + rwwA4JCe0aeRuGcrSFppMmAIvEO2uiwBMG91nDAeggrEDbHJCAprCE2yXtf1JlhTmkPJPPL4ylZQ + spcewXn7qiRKSPt7Aqk8CtJvEIgTBqCeE1CPILhTxDXYFj57bgSCCvDEvQoT2PVK9PEb9zyRPbYa + BaEEZRKbm7BDf1eyCZTsFb1UgqKL4liauq6vnXtst4HH9MxW6/P8eIlC285524Qzfpm3yqjQV6cD + 49mBrGMJPWYAv1Pk25sUmfN2cFSR/YMmCj4uzpGzsekRnS/OIFni+or1+A7c6FUSiSsdrkpjgose + 5UgdG+ZbqewVkF1d/a+bj7RPlyvT/Y/8CAiBjlBWzmMs5ebicc3jJvX58dol5WSYhbdAOFStMh16 + 59XpN2xd1TR8Lh5wOW1Ydsz+AgAA//8DAJLjX/OUAwAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09f507cc477281-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:31:51 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=2N0lRp5YNIBKY6AUc.tpQsJVlWEga7Ys924AChkX4qk-1733967111-1.0.1.1-IJEARyUXuMN2pbqt5jU4yaj77.QHaVM0uVSztZt49GpbAV1HXoPr6.uIdz2viIUlRExuu5tYN_.v5wUpYjyBSQ; + path=/; expires=Thu, 12-Dec-24 02:01:51 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=TvHcCPz7N_.kfviRP.Y0iD_HMeA.0uxvji5nzbbTR5w-1733967111302-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '564' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998898' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_db048be5fbcb3bc4136d9c89ace1249a + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys.yaml new file mode 100644 index 00000000000..9964826b6b2 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys.yaml @@ -0,0 +1,312 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"Is france part of europe?\"\ncontext: \"irrelevant\"\nanswer: \"France is + indeed part of europe\"\nverification: \n", "role": "user"}], "model": "gpt-4o-mini", + "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4612' + content-type: + - application/json + cookie: + - __cf_bm=2N0lRp5YNIBKY6AUc.tpQsJVlWEga7Ys924AChkX4qk-1733967111-1.0.1.1-IJEARyUXuMN2pbqt5jU4yaj77.QHaVM0uVSztZt49GpbAV1HXoPr6.uIdz2viIUlRExuu5tYN_.v5wUpYjyBSQ; + _cfuvid=TvHcCPz7N_.kfviRP.Y0iD_HMeA.0uxvji5nzbbTR5w-1733967111302-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFNNj5swFLzzK5586QVWZJNtPm49dC+tVKmteikVGPsFnBrbsR9Roij/ + vTLJBlbdSr0g9ObNaGYenBMApiTbABMtJ9E5nX2Q3z5//dTu9WH/vJD57stx79z6JFD+oBVLI8PW + OxT0wnoQtnMaSVlzhYVHThhVZ8v5fP1+OZvNB6CzEnWkNY6yhc06ZVT2mD8usnyZzW7iorVKYGAb + +JkAAJyHZ/RpJB7ZBvL0ZdJhCLxBtrkvATBvdZwwHoIKxA2xdASFNYRmsF5V1S5YU5hzwTzy+Mo2 + ULDvLcKwdiRw3h6URAkqgPIeNR64IeBGgrQYwFgadr2qe0Lg5gTKbK3veGwDPDbcS2UaePbcCHwX + oEHbeO5aJbiGQJz6AMrAx95bhw8FS6FgB/RSCYp28kthqqqaRvC47QOPNZpe69v8cu9E28Z5W4cb + fp9vlVGhLa9JY/5A1rEBvSQAv4bu+1d1Mudt56gk+xtNFFw/PV312HjyEZ0vbiBZ4nrCWq3TN/RK + icSVDpPrMcFFi3KkjqfmvVR2AiST1H+7eUv7mlyZ5n/kR0AIdISydB7jUV4lHtc8xj/iX2v3lgfD + LJwCYVdulWnQO6+u3+PWlXXN52KFy7xmySX5AwAA//8DAEriHaudAwAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09f515b8157281-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:31:53 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '976' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998903' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_2c2d33a1a025655db6ab9a62096b644a + status: + code: 200 + message: OK +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"Is france part of europe?\"\ncontext: \"France is part of europe\"\nanswer: + \"France is indeed part of europe\"\nverification: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4626' + content-type: + - application/json + cookie: + - __cf_bm=2N0lRp5YNIBKY6AUc.tpQsJVlWEga7Ys924AChkX4qk-1733967111-1.0.1.1-IJEARyUXuMN2pbqt5jU4yaj77.QHaVM0uVSztZt49GpbAV1HXoPr6.uIdz2viIUlRExuu5tYN_.v5wUpYjyBSQ; + _cfuvid=TvHcCPz7N_.kfviRP.Y0iD_HMeA.0uxvji5nzbbTR5w-1733967111302-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFLLbtswELzrKxY8S4EVu7Wtm4H0ccilSHtpVUg0uZLoSCRLrmKnhv+9 + oOxYCpoCvRDEzs5wZpfHCIApyTJgouEkOtsmG/lw//Bp83tffRH2+7duU9/dfT7gPS53j79YHBhm + u0NBL6wbYTrbIimjz7BwyAmDarqcz9fvl2m6GIDOSGwDrbaULEzSKa2S29ntIpktk3R1YTdGCfQs + gx8RAMBxOINPLfHAMpjFL5UOvec1suzaBMCcaUOFce+VJ66JxSMojCbUg/WyLHfe6Fwfc+aQhyvL + IGdfG4Sh7UAglUNB7TN44oQeqOEEHx3XAkF5sNwRmAo+9M5YjGHfKNFMSL21xlGgIXDt9+jAOvOk + JMqbnMWQsyd0UgkKL6enXJdlOXXrsOo9DxPTfdte6qdr/NbU1pmtv+DXeqW08k1xDhWiejKWDegp + Avg5jLl/NTlmneksFWQeUQfB9bvVWY+N2x3R+fwCkiHeTljrNH5Dr5BIXLV+sigmuGhQjtRxq7yX + ykyAaJL6bzdvaZ+TK13/j/wICIGWUBbWYVjKq8Rjm8Pw+f/Vdp3yYJj5Z0/YFZXSNTrr1PnrVbbY + bvlcrHA527LoFP0BAAD//wMAcOvPiIgDAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09f51cdbaf7281-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:31:54 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '779' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998900' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_4b6ae2d675e25726729e2577e662f691 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_question_in_messages.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_question_in_messages.yaml new file mode 100644 index 00000000000..e6673bfe3c0 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_submits_evaluation_on_span_with_question_in_messages.yaml @@ -0,0 +1,157 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"What is the capital of France?\"\ncontext: \"The capital of France is Paris.\"\nanswer: + \"The capital of France is Paris\"\nverification: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4637' + content-type: + - application/json + cookie: + - __cf_bm=2N0lRp5YNIBKY6AUc.tpQsJVlWEga7Ys924AChkX4qk-1733967111-1.0.1.1-IJEARyUXuMN2pbqt5jU4yaj77.QHaVM0uVSztZt49GpbAV1HXoPr6.uIdz2viIUlRExuu5tYN_.v5wUpYjyBSQ; + _cfuvid=TvHcCPz7N_.kfviRP.Y0iD_HMeA.0uxvji5nzbbTR5w-1733967111302-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFLLbtswELzrKxY8W4FtBXbsW1CkaNEc6qZAD1UhUeTKokORLLmOnRj+ + 94LyQzaaQy6CtLMzmJ3RLgFgSrI5MNFwEq3T6b18elxk9mH9a/Mo77Mvz5u3t79fFz++faoWT2wQ + GbZaoaAT60bY1mkkZc0BFh45YVQdTbNsNpmORuMOaK1EHWlLR+mtTVtlVDoejm/T4TQd3R3ZjVUC + A5vD7wQAYNc9o08jccvmMBycJi2GwJfI5uclAOatjhPGQ1CBuCE26EFhDaHprJdluQrW5GaXM488 + vrI55Oxng+C8fVESJXT7WwKpPArSrxCIEwaghhNQgyC4U8Q12Bo+e24EggrwnXsVBrBplGjiN255 + R/ZYaxSEEpTp2NyEDfqbnA0gZy/opRIUXYz2uSnL8tK5x3odeEzPrLU+zvfnKLRdOm+rcMTP81oZ + FZricGA8O5B1rEP3CcCfLvL1VYrMeds6Ksg+o4mCs8kxctY33aPZ5AiSJa4vWLMTcKVXSCSudLgo + jQkuGpQ9tW+Yr6WyF0BycfX/bt7TPlyuzPIj8j0gBDpCWTiPsZSri/s1j6uuz/fXzil3hll4DYRt + USuzRO+8OvyGtSuqimfiDqfDiiX75B8AAAD//wMAVJCtVZQDAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09f50f2dee7281-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:31:52 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '814' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998898' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_063db1deb138ded38b6a5ab9c2ff89c2 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/test_llmobs_ragas_evaluators.py b/tests/llmobs/test_llmobs_ragas_evaluators.py index 3e8228d5c66..7c5c3d02b63 100644 --- a/tests/llmobs/test_llmobs_ragas_evaluators.py +++ b/tests/llmobs/test_llmobs_ragas_evaluators.py @@ -251,7 +251,7 @@ def test_llmobs_with_faithfulness_emits_traces_and_evals_on_exit(mock_writer_log assert err == b"" -def test_ragas_context_precision_evaluator_init(ragas, LLMObs): +def test_ragas_context_precision_init(ragas, LLMObs): rcp_evaluator = RagasContextPrecisionEvaluator(LLMObs) assert rcp_evaluator.llmobs_service == LLMObs assert rcp_evaluator.ragas_context_precision_instance == ragas.metrics.context_precision @@ -328,7 +328,10 @@ def test_ragas_context_precision_submits_evaluation(ragas, LLMObs, mock_llmobs_s label=RagasContextPrecisionEvaluator.LABEL, metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, value=1.0, - metadata={"_dd.evaluation_kind": "context_precision"}, + metadata={ + "_dd.evaluation_kind": "context_precision", + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, ) ] ) @@ -352,7 +355,10 @@ def test_ragas_context_precision_submits_evaluation_on_span_with_question_in_mes label=RagasContextPrecisionEvaluator.LABEL, metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, value=1.0, - metadata={"_dd.evaluation_kind": "context_precision"}, + metadata={ + "_dd.evaluation_kind": "context_precision", + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, ) ] ) @@ -388,7 +394,10 @@ def test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys( label=RagasContextPrecisionEvaluator.LABEL, metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, value=0.5, - metadata={"_dd.evaluation_kind": "context_precision"}, + metadata={ + "_dd.evaluation_kind": "context_precision", + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, ) ] ) @@ -396,10 +405,10 @@ def test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys( @pytest.mark.vcr_logs def test_ragas_context_precision_emits_traces(ragas, LLMObs): - rf_evaluator = RagasContextPrecisionEvaluator(LLMObs) - rf_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) - assert rf_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_count == 2 - calls = rf_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_args_list + rcp_evaluator = RagasContextPrecisionEvaluator(LLMObs) + rcp_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) + assert rcp_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_count == 2 + calls = rcp_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_args_list spans = [call[0][0] for call in calls] From 06dbc55b8b96aa899cca0bc7ae40a97a881b65f2 Mon Sep 17 00:00:00 2001 From: lievan Date: Wed, 11 Dec 2024 21:18:06 -0500 Subject: [PATCH 04/15] fix cassette names --- ....emits_traces_and_evaluations_on_exit.yaml | 315 ++++++++++++++++++ ...test_ragas_faithfulness_emits_traces.yaml} | 0 ...agas_faithfulness_submits_evaluation.yaml} | 0 ..._evaluation_on_span_with_custom_keys.yaml} | 0 ...on_on_span_with_question_in_messages.yaml} | 0 ....emits_traces_and_evaluations_on_exit.yaml | 236 +------------ tests/llmobs/test_llmobs_ragas_evaluators.py | 2 +- 7 files changed, 333 insertions(+), 220 deletions(-) create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml rename tests/llmobs/llmobs_cassettes/{tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_emits_traces.yaml => tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_emits_traces.yaml} (100%) rename tests/llmobs/llmobs_cassettes/{tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation.yaml => tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation.yaml} (100%) rename tests/llmobs/llmobs_cassettes/{tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml => tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml} (100%) rename tests/llmobs/llmobs_cassettes/{tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml => tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml} (100%) diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml new file mode 100644 index 00000000000..757f875443f --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml @@ -0,0 +1,315 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given a question, an answer, and sentences from + the answer analyze the complexity of each sentence given under ''sentences'' + and break down each sentence into one or more fully understandable statements + while also ensuring no pronouns are used in each statement. Format the outputs + in JSON.\n\nThe output should be a well-formatted JSON instance that conforms + to the JSON schema below.\n\nAs an example, for the schema {\"properties\": + {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": + \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe + object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. + The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere + is the output JSON schema:\n```\n{\"type\": \"array\", \"items\": {\"$ref\": + \"#/definitions/Statements\"}, \"definitions\": {\"Statements\": {\"title\": + \"Statements\", \"type\": \"object\", \"properties\": {\"sentence_index\": {\"title\": + \"Sentence Index\", \"description\": \"Index of the sentence from the statement + list\", \"type\": \"integer\"}, \"simpler_statements\": {\"title\": \"Simpler + Statements\", \"description\": \"the simpler statements\", \"type\": \"array\", + \"items\": {\"type\": \"string\"}}}, \"required\": [\"sentence_index\", \"simpler_statements\"]}}}\n```\n\nDo + not return any preamble or explanations, return only a pure JSON string surrounded + by triple backticks (```).\n\nExamples:\n\nquestion: \"Who was Albert Einstein + and what is he best known for?\"\nanswer: \"He was a German-born theoretical + physicist, widely acknowledged to be one of the greatest and most influential + physicists of all time. He was best known for developing the theory of relativity, + he also made important contributions to the development of the theory of quantum + mechanics.\"\nsentences: \"\\n 0:He was a German-born theoretical physicist, + widely acknowledged to be one of the greatest and most influential physicists + of all time. \\n 1:He was best known for developing the theory of relativity, + he also made important contributions to the development of the theory of quantum + mechanics.\\n \"\nanalysis: ```[{\"sentence_index\": 0, \"simpler_statements\": + [\"Albert Einstein was a German-born theoretical physicist.\", \"Albert Einstein + is recognized as one of the greatest and most influential physicists of all + time.\"]}, {\"sentence_index\": 1, \"simpler_statements\": [\"Albert Einstein + was best known for developing the theory of relativity.\", \"Albert Einstein + also made important contributions to the development of the theory of quantum + mechanics.\"]}]```\n\nYour actual task:\n\nquestion: \"What is the capital of + France?\"\nanswer: \"The capital of France is Paris\"\nsentences: \"\"\nanalysis: + \n", "role": "user"}], "model": "gpt-4o-mini", "n": 1, "stream": false, "temperature": + 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '2919' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2yRW4vbMBCF3/0rxDzHi+2mySZvbUOhsPRGoSxxsBV5bKuVJaGZ9ELIfy9yvEnK + 7oseztH5NGd0TIQA3cBagOolq8Gb9M3D5tPi1zuz+r0qHjavv27a74+vPj5+sG8z8wVmMeH2P1Dx + U+pOucEbZO3s2VYBJWOk5stitchXy3w+GoNr0MRY5zmdu3TQVqdFVszTbJnm91O6d1ohwVpsEyGE + OI5nnNM2+AfWIps9KQMSyQ5hfbkkBARnogKSSBNLyzC7mspZRjuOXtf19lgCYVQUViO+HPmiBNKx + U6iIJeOAlila2xK+9SiU9JqlEa4V74O0CoUm8VkGTXcl7E67uq5vHw3YHkjG4vZgzKSfLi2M63xw + e5r8i95qq6mvAkpyNk5M7DyM7ikRYjdu6/DfAsAHN3iu2P1EG4GLLD/z4PpJV7dYTCY7luYmVSxn + L/CqBllqQzf7BiVVj801miU35Z4/+hLiXFDb7hklmUhAf4lxqFptOww+6PMPtr6a3xeqKORyryA5 + Jf8AAAD//wMAn6C7Cc8CAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d6b5b701f294367-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 22 Oct 2024 17:55:15 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=iQaF937ylY7BvvBCyWYQoxiJwi1nBp5.LILrHLw1uno-1729619715-1.0.1.1-jS4Dz7yc_ud.hKZlJ_CAZkSQesqzVkfrA5F30zI7CtJsbEKyAiuVlpX0CPf816UtlhXQEW8T5nsc.UvnsCOzOw; + path=/; expires=Tue, 22-Oct-24 18:25:15 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=wQzHCwLW6CPU768K_tlLklWp36I8zYCVJkKlAMtnMkk-1729619715162-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '496' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999323' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_33b8cddecaab8b8bc36e90f58f844636 + status: + code: 200 + message: OK +- request: + body: '{"messages": [{"content": "Your task is to judge the faithfulness of a + series of statements based on a given context. For each statement you must return + verdict as 1 if the statement can be directly inferred based on the context + or 0 if the statement can not be directly inferred based on the context.\n\nThe + output should be a well-formatted JSON instance that conforms to the JSON schema + below.\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": + \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": + {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", + \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": + {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output + JSON schema:\n```\n{\"type\": \"array\", \"items\": {\"$ref\": \"#/definitions/StatementFaithfulnessAnswer\"}, + \"definitions\": {\"StatementFaithfulnessAnswer\": {\"title\": \"StatementFaithfulnessAnswer\", + \"type\": \"object\", \"properties\": {\"statement\": {\"title\": \"Statement\", + \"description\": \"the original statement, word-by-word\", \"type\": \"string\"}, + \"reason\": {\"title\": \"Reason\", \"description\": \"the reason of the verdict\", + \"type\": \"string\"}, \"verdict\": {\"title\": \"Verdict\", \"description\": + \"the verdict(0/1) of the faithfulness.\", \"type\": \"integer\"}}, \"required\": + [\"statement\", \"reason\", \"verdict\"]}}}\n```\n\nDo not return any preamble + or explanations, return only a pure JSON string surrounded by triple backticks + (```).\n\nExamples:\n\ncontext: \"John is a student at XYZ University. He is + pursuing a degree in Computer Science. He is enrolled in several courses this + semester, including Data Structures, Algorithms, and Database Management. John + is a diligent student and spends a significant amount of time studying and completing + assignments. He often stays late in the library to work on his projects.\"\nstatements: + ```[\"John is majoring in Biology.\", \"John is taking a course on Artificial + Intelligence.\", \"John is a dedicated student.\", \"John has a part-time job.\"]```\nanswer: + ```[{\"statement\": \"John is majoring in Biology.\", \"reason\": \"John''s + major is explicitly mentioned as Computer Science. There is no information suggesting + he is majoring in Biology.\", \"verdict\": 0}, {\"statement\": \"John is taking + a course on Artificial Intelligence.\", \"reason\": \"The context mentions the + courses John is currently enrolled in, and Artificial Intelligence is not mentioned. + Therefore, it cannot be deduced that John is taking a course on AI.\", \"verdict\": + 0}, {\"statement\": \"John is a dedicated student.\", \"reason\": \"The context + states that he spends a significant amount of time studying and completing assignments. + Additionally, it mentions that he often stays late in the library to work on + his projects, which implies dedication.\", \"verdict\": 1}, {\"statement\": + \"John has a part-time job.\", \"reason\": \"There is no information given in + the context about John having a part-time job.\", \"verdict\": 0}]```\n\ncontext: + \"Photosynthesis is a process used by plants, algae, and certain bacteria to + convert light energy into chemical energy.\"\nstatements: ```[\"Albert Einstein + was a genius.\"]```\nanswer: ```[{\"statement\": \"Albert Einstein was a genius.\", + \"reason\": \"The context and statement are unrelated\", \"verdict\": 0}]```\n\nYour + actual task:\n\ncontext: \"The capital of France is Paris.\"\nstatements: \"[\\\"The + capital of France is Paris.\\\"]\"\nanswer: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '3661' + content-type: + - application/json + cookie: + - __cf_bm=iQaF937ylY7BvvBCyWYQoxiJwi1nBp5.LILrHLw1uno-1729619715-1.0.1.1-jS4Dz7yc_ud.hKZlJ_CAZkSQesqzVkfrA5F30zI7CtJsbEKyAiuVlpX0CPf816UtlhXQEW8T5nsc.UvnsCOzOw; + _cfuvid=wQzHCwLW6CPU768K_tlLklWp36I8zYCVJkKlAMtnMkk-1729619715162-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xSQW7bMBC86xWLPVuBpTqR7VuAoGiBAmmLHhrEgUVTK2tdiSTIdZDA8N8Lyorl + ILnwMLMznB3ykAAgV7gE1I0S3bk2vf1xd19M/c3u4frhu/37Tet73qm7383r7fwXTqLCbnak5U11 + pW3nWhK25kRrT0ooumZFvrjJFkV23ROdraiNsq2TdGbTjg2n+TSfpdMizeaDurGsKeASHhMAgEN/ + xpymohdcwnTyhnQUgtoSLs9DAOhtGxFUIXAQZQQnI6mtETJ99LIsHw8rDKKEOjKywiWs8E9DoJVj + US3YGr56ZTQBB/ipPIerFU5ghZ5UsGYUnD3ioIKKPWkBT46EYy3RSRoCNrX1neoh5+0zV1QBm57r + k73IcMMz+Yp1nyk7PpVlebmEp3ofVCzS7Nt2wI/nVlq7dd5uwsCf8ZoNh2Z9Ch8bCGId9uwxAXjq + 29+/KxSdt52Ttdh/ZKJhUcxPfjg++sh+WQykWFHtiM+zYvKJ37oiUdyGi/dDrXRD1SidJhfLfbz0 + M4vTgmy2H1ySwQnDaxDq1jWbLXnn+fQjareezXOd56rYaEyOyX8AAAD//wMAUtzROh8DAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d6b5b744e034367-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 22 Oct 2024 17:55:16 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '749' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999151' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_fbb01161a03eb6f478ff52314b72cfd6 + status: + code: 200 + message: OK +- request: + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": + "6877142543397072040", "trace_id": "6717e70200000000a99ea8ad36f4f36d", "label": + "ragas_faithfulness", "metric_type": "score", "timestamp_ms": 1729619716093, + "score_value": 1.0, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022", + "ml_app:unnamed-ml-app"]}]}}}' + headers: + Content-Type: + - application/json + DD-API-KEY: + - XXXXXX + method: POST + uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + response: + body: + string: '{"data":{"id":"99fa371c-457c-4d2b-8d4c-61657e0ffd48","type":"evaluation_metric","attributes":{"metrics":[{"id":"CbapxUnzcX","trace_id":"6717e70200000000a99ea8ad36f4f36d","span_id":"6877142543397072040","timestamp_ms":1729619716093,"ml_app":"unnamed-ml-app","metric_type":"score","label":"ragas_faithfulness","score_value":1,"tags":["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022","ml_app:unnamed-ml-app"]}]}}}' + headers: + content-length: + - '414' + content-security-policy: + - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com + content-type: + - application/vnd.api+json + date: + - Tue, 22 Oct 2024 17:55:17 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + status: + code: 202 + message: Accepted +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_emits_traces.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_emits_traces.yaml similarity index 100% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_emits_traces.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_emits_traces.yaml diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation.yaml similarity index 100% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation.yaml diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml similarity index 100% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml similarity index 100% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml index 757f875443f..b155dc2d5e4 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml @@ -72,244 +72,42 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: !!binary | - H4sIAAAAAAAAA2yRW4vbMBCF3/0rxDzHi+2mySZvbUOhsPRGoSxxsBV5bKuVJaGZ9ELIfy9yvEnK - 7oseztH5NGd0TIQA3cBagOolq8Gb9M3D5tPi1zuz+r0qHjavv27a74+vPj5+sG8z8wVmMeH2P1Dx - U+pOucEbZO3s2VYBJWOk5stitchXy3w+GoNr0MRY5zmdu3TQVqdFVszTbJnm91O6d1ohwVpsEyGE - OI5nnNM2+AfWIps9KQMSyQ5hfbkkBARnogKSSBNLyzC7mspZRjuOXtf19lgCYVQUViO+HPmiBNKx - U6iIJeOAlila2xK+9SiU9JqlEa4V74O0CoUm8VkGTXcl7E67uq5vHw3YHkjG4vZgzKSfLi2M63xw - e5r8i95qq6mvAkpyNk5M7DyM7ikRYjdu6/DfAsAHN3iu2P1EG4GLLD/z4PpJV7dYTCY7luYmVSxn - L/CqBllqQzf7BiVVj801miU35Z4/+hLiXFDb7hklmUhAf4lxqFptOww+6PMPtr6a3xeqKORyryA5 - Jf8AAAD//wMAn6C7Cc8CAAA= + string: "{\n \"error\": {\n \"message\": \"Incorrect API key provided: + dummy-op********-key. You can find your API key at https://platform.openai.com/account/api-keys.\",\n + \ \"type\": \"invalid_request_error\",\n \"param\": null,\n \"code\": + \"invalid_api_key\"\n }\n}\n" headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8d6b5b701f294367-EWR + - 8f0a37055b6a43c9-EWR Connection: - keep-alive - Content-Encoding: - - gzip + Content-Length: + - '270' Content-Type: - - application/json + - application/json; charset=utf-8 Date: - - Tue, 22 Oct 2024 17:55:15 GMT + - Thu, 12 Dec 2024 02:16:53 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=iQaF937ylY7BvvBCyWYQoxiJwi1nBp5.LILrHLw1uno-1729619715-1.0.1.1-jS4Dz7yc_ud.hKZlJ_CAZkSQesqzVkfrA5F30zI7CtJsbEKyAiuVlpX0CPf816UtlhXQEW8T5nsc.UvnsCOzOw; - path=/; expires=Tue, 22-Oct-24 18:25:15 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=tsTMtFLakdJEC6pZi9qDFxtRlPBCHLf1DYsmb.f7JSc-1733969813-1.0.1.1-cxUdAvt3rjIZsIJ889b37xrJXEJJFxpEoCgPkf1eU9cnDejs227o8rx9kNERGzD8XAVYXutat7XUAF8VVDD6_Q; + path=/; expires=Thu, 12-Dec-24 02:46:53 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=wQzHCwLW6CPU768K_tlLklWp36I8zYCVJkKlAMtnMkk-1729619715162-0.0.1.1-604800000; + - _cfuvid=nX_aH4E6WCOEIAbar7JHuQN_pvwvAgFxMBWBTr3bdgQ-1733969813447-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - Transfer-Encoding: - - chunked X-Content-Type-Options: - nosniff - access-control-expose-headers: - - X-Request-ID alt-svc: - h3=":443"; ma=86400 - openai-organization: - - datadog-staging - openai-processing-ms: - - '496' - openai-version: - - '2020-10-01' - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999323' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_33b8cddecaab8b8bc36e90f58f844636 - status: - code: 200 - message: OK -- request: - body: '{"messages": [{"content": "Your task is to judge the faithfulness of a - series of statements based on a given context. For each statement you must return - verdict as 1 if the statement can be directly inferred based on the context - or 0 if the statement can not be directly inferred based on the context.\n\nThe - output should be a well-formatted JSON instance that conforms to the JSON schema - below.\n\nAs an example, for the schema {\"properties\": {\"foo\": {\"title\": - \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": - {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", - \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": - {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n\nHere is the output - JSON schema:\n```\n{\"type\": \"array\", \"items\": {\"$ref\": \"#/definitions/StatementFaithfulnessAnswer\"}, - \"definitions\": {\"StatementFaithfulnessAnswer\": {\"title\": \"StatementFaithfulnessAnswer\", - \"type\": \"object\", \"properties\": {\"statement\": {\"title\": \"Statement\", - \"description\": \"the original statement, word-by-word\", \"type\": \"string\"}, - \"reason\": {\"title\": \"Reason\", \"description\": \"the reason of the verdict\", - \"type\": \"string\"}, \"verdict\": {\"title\": \"Verdict\", \"description\": - \"the verdict(0/1) of the faithfulness.\", \"type\": \"integer\"}}, \"required\": - [\"statement\", \"reason\", \"verdict\"]}}}\n```\n\nDo not return any preamble - or explanations, return only a pure JSON string surrounded by triple backticks - (```).\n\nExamples:\n\ncontext: \"John is a student at XYZ University. He is - pursuing a degree in Computer Science. He is enrolled in several courses this - semester, including Data Structures, Algorithms, and Database Management. John - is a diligent student and spends a significant amount of time studying and completing - assignments. He often stays late in the library to work on his projects.\"\nstatements: - ```[\"John is majoring in Biology.\", \"John is taking a course on Artificial - Intelligence.\", \"John is a dedicated student.\", \"John has a part-time job.\"]```\nanswer: - ```[{\"statement\": \"John is majoring in Biology.\", \"reason\": \"John''s - major is explicitly mentioned as Computer Science. There is no information suggesting - he is majoring in Biology.\", \"verdict\": 0}, {\"statement\": \"John is taking - a course on Artificial Intelligence.\", \"reason\": \"The context mentions the - courses John is currently enrolled in, and Artificial Intelligence is not mentioned. - Therefore, it cannot be deduced that John is taking a course on AI.\", \"verdict\": - 0}, {\"statement\": \"John is a dedicated student.\", \"reason\": \"The context - states that he spends a significant amount of time studying and completing assignments. - Additionally, it mentions that he often stays late in the library to work on - his projects, which implies dedication.\", \"verdict\": 1}, {\"statement\": - \"John has a part-time job.\", \"reason\": \"There is no information given in - the context about John having a part-time job.\", \"verdict\": 0}]```\n\ncontext: - \"Photosynthesis is a process used by plants, algae, and certain bacteria to - convert light energy into chemical energy.\"\nstatements: ```[\"Albert Einstein - was a genius.\"]```\nanswer: ```[{\"statement\": \"Albert Einstein was a genius.\", - \"reason\": \"The context and statement are unrelated\", \"verdict\": 0}]```\n\nYour - actual task:\n\ncontext: \"The capital of France is Paris.\"\nstatements: \"[\\\"The - capital of France is Paris.\\\"]\"\nanswer: \n", "role": "user"}], "model": - "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '3661' - content-type: - - application/json - cookie: - - __cf_bm=iQaF937ylY7BvvBCyWYQoxiJwi1nBp5.LILrHLw1uno-1729619715-1.0.1.1-jS4Dz7yc_ud.hKZlJ_CAZkSQesqzVkfrA5F30zI7CtJsbEKyAiuVlpX0CPf816UtlhXQEW8T5nsc.UvnsCOzOw; - _cfuvid=wQzHCwLW6CPU768K_tlLklWp36I8zYCVJkKlAMtnMkk-1729619715162-0.0.1.1-604800000 - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.52.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.52.0 - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.10.13 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: !!binary | - H4sIAAAAAAAAA2xSQW7bMBC86xWLPVuBpTqR7VuAoGiBAmmLHhrEgUVTK2tdiSTIdZDA8N8Lyorl - ILnwMLMznB3ykAAgV7gE1I0S3bk2vf1xd19M/c3u4frhu/37Tet73qm7383r7fwXTqLCbnak5U11 - pW3nWhK25kRrT0ooumZFvrjJFkV23ROdraiNsq2TdGbTjg2n+TSfpdMizeaDurGsKeASHhMAgEN/ - xpymohdcwnTyhnQUgtoSLs9DAOhtGxFUIXAQZQQnI6mtETJ99LIsHw8rDKKEOjKywiWs8E9DoJVj - US3YGr56ZTQBB/ipPIerFU5ghZ5UsGYUnD3ioIKKPWkBT46EYy3RSRoCNrX1neoh5+0zV1QBm57r - k73IcMMz+Yp1nyk7PpVlebmEp3ofVCzS7Nt2wI/nVlq7dd5uwsCf8ZoNh2Z9Ch8bCGId9uwxAXjq - 29+/KxSdt52Ttdh/ZKJhUcxPfjg++sh+WQykWFHtiM+zYvKJ37oiUdyGi/dDrXRD1SidJhfLfbz0 - M4vTgmy2H1ySwQnDaxDq1jWbLXnn+fQjareezXOd56rYaEyOyX8AAAD//wMAUtzROh8DAAA= - headers: - CF-Cache-Status: - - DYNAMIC - CF-RAY: - - 8d6b5b744e034367-EWR - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Tue, 22 Oct 2024 17:55:16 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - openai-organization: - - datadog-staging - openai-processing-ms: - - '749' - openai-version: - - '2020-10-01' - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999151' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_fbb01161a03eb6f478ff52314b72cfd6 - status: - code: 200 - message: OK -- request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "6877142543397072040", "trace_id": "6717e70200000000a99ea8ad36f4f36d", "label": - "ragas_faithfulness", "metric_type": "score", "timestamp_ms": 1729619716093, - "score_value": 1.0, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022", - "ml_app:unnamed-ml-app"]}]}}}' - headers: - Content-Type: - - application/json - DD-API-KEY: - - XXXXXX - method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric - response: - body: - string: '{"data":{"id":"99fa371c-457c-4d2b-8d4c-61657e0ffd48","type":"evaluation_metric","attributes":{"metrics":[{"id":"CbapxUnzcX","trace_id":"6717e70200000000a99ea8ad36f4f36d","span_id":"6877142543397072040","timestamp_ms":1729619716093,"ml_app":"unnamed-ml-app","metric_type":"score","label":"ragas_faithfulness","score_value":1,"tags":["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022","ml_app:unnamed-ml-app"]}]}}}' - headers: - content-length: - - '414' - content-security-policy: - - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com - content-type: - - application/vnd.api+json - date: - - Tue, 22 Oct 2024 17:55:17 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: - - Accept-Encoding - x-content-type-options: - - nosniff - x-frame-options: - - SAMEORIGIN + - Origin + x-request-id: + - req_c8d54071e4b1918edf93a566a7df96cd status: - code: 202 - message: Accepted + code: 401 + message: Unauthorized version: 1 diff --git a/tests/llmobs/test_llmobs_ragas_evaluators.py b/tests/llmobs/test_llmobs_ragas_evaluators.py index 7c5c3d02b63..4c008dea50e 100644 --- a/tests/llmobs/test_llmobs_ragas_evaluators.py +++ b/tests/llmobs/test_llmobs_ragas_evaluators.py @@ -230,7 +230,7 @@ def test_llmobs_with_faithfulness_emits_traces_and_evals_on_exit(mock_writer_log from tests.llmobs._utils import logs_vcr ctx = logs_vcr.use_cassette( - "tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml" + "tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml" ) ctx.__enter__() atexit.register(lambda: ctx.__exit__()) From cd31162960e1c99c629869400f521cfb64bd0e06 Mon Sep 17 00:00:00 2001 From: lievan Date: Thu, 12 Dec 2024 11:54:21 -0500 Subject: [PATCH 05/15] refactor to include base ragas eval --- ddtrace/llmobs/_evaluators/ragas/base.py | 139 ++++++++++++++++++ .../_evaluators/ragas/context_precision.py | 134 +++-------------- .../llmobs/_evaluators/ragas/faithfulness.py | 3 +- 3 files changed, 165 insertions(+), 111 deletions(-) create mode 100644 ddtrace/llmobs/_evaluators/ragas/base.py diff --git a/ddtrace/llmobs/_evaluators/ragas/base.py b/ddtrace/llmobs/_evaluators/ragas/base.py new file mode 100644 index 00000000000..84187164ed4 --- /dev/null +++ b/ddtrace/llmobs/_evaluators/ragas/base.py @@ -0,0 +1,139 @@ +from abc import ABC +from abc import abstractmethod +import traceback +from typing import Optional +from typing import Tuple +from typing import Union + +from ddtrace.internal.logger import get_logger +from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL +from ddtrace.internal.utils.version import parse_version +from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX + + +logger = get_logger(__name__) + + +class MiniRagas: + """ + A helper class to store instances of ragas classes and functions + that may or may not exist in a user's environment. + """ + + def __init__(self): + import ragas + + self.ragas_version = parse_version(ragas.__version__) + if self.ragas_version >= (0, 2, 0) or self.ragas_version < (0, 1, 10): + raise NotImplementedError( + "Ragas version: {} is not supported".format(self.ragas_version), + ) + + from ragas.llms import llm_factory + + self.llm_factory = llm_factory + + from ragas.llms.output_parser import RagasoutputParser + + self.RagasoutputParser = RagasoutputParser + + from ragas.metrics import context_precision + + self.context_precision = context_precision + + from ragas.metrics.base import ensembler + + self.ensembler = ensembler + + from ddtrace.llmobs._evaluators.ragas.models import ContextPrecisionVerification + + self.ContextPrecisionVerification = ContextPrecisionVerification + + +def _get_ml_app_for_ragas_trace(span_event: dict) -> str: + """ + The `ml_app` spans generated from traces of ragas will be named as `dd-ragas-` + or `dd-ragas` if `ml_app` is not present in the span event. + """ + tags = span_event.get("tags", []) # list[str] + ml_app = None + for tag in tags: + if isinstance(tag, str) and tag.startswith("ml_app:"): + ml_app = tag.split(":")[1] + break + if not ml_app: + return RAGAS_ML_APP_PREFIX + return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) + + +class RagasBaseEvaluator(ABC): + """A class used by EvaluatorRunner to conduct ragas evaluations + on LLM Observability span events. The job of an Evaluator is to take a span and + submit evaluation metrics based on the span's attributes. + """ + + LABEL = "ragas_context_precision" + METRIC_TYPE = "score" + + def __init__(self, llmobs_service): + """ + Initialize an evaluator that uses the ragas library to generate a score on finished LLM spans. + + :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and + submitting evaluation metrics. + + Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. + """ + self.llmobs_service = llmobs_service + self.ragas_version = "unknown" + telemetry_state = "ok" + try: + self.mini_ragas = MiniRagas() + except Exception as e: + telemetry_state = "fail" + telemetry_writer.add_log( + level=TELEMETRY_LOG_LEVEL.ERROR, + message="Failed to import Ragas dependencies", + stack_trace=traceback.format_exc(), + tags={"ragas_version": self.ragas_version}, + ) + raise NotImplementedError("Failed to load dependencies for `{}` evaluator".format(self.LABEL)) from e + finally: + telemetry_writer.add_count_metric( + namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + name="evaluators.init", + value=1, + tags=( + ("evaluator_label", self.LABEL), + ("state", telemetry_state), + ("ragas_version", self.ragas_version), + ), + ) + + def run_and_submit_evaluation(self, span_event: dict): + if not span_event: + return + score_result_or_failure, metric_metadata = self.evaluate(span_event) + telemetry_writer.add_count_metric( + TELEMETRY_APM_PRODUCT.LLMOBS, + "evaluators.run", + 1, + tags=( + ("evaluator_label", self.LABEL), + ("state", score_result_or_failure if isinstance(score_result_or_failure, str) else "success"), + ), + ) + if isinstance(score_result_or_failure, float): + self.llmobs_service.submit_evaluation( + span_context={"trace_id": span_event.get("trace_id"), "span_id": span_event.get("span_id")}, + label=self.LABEL, + metric_type=self.METRIC_TYPE, + value=score_result_or_failure, + metadata=metric_metadata, + ) + + @abstractmethod + def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: + pass diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index f76b56e305d..a2d80251ea8 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -1,5 +1,4 @@ import math -import traceback from typing import Optional from typing import Tuple from typing import Union @@ -7,62 +6,18 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT -from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.internal.utils.version import parse_version from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS -from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX +from ddtrace.llmobs._evaluators.ragas.base import RagasBaseEvaluator +from ddtrace.llmobs._evaluators.ragas.base import _get_ml_app_for_ragas_trace logger = get_logger(__name__) -class MiniRagas: - """ - A helper class to store instances of ragas classes and functions - that may or may not exist in a user's environment. - """ - - llm_factory = None - RagasoutputParser = None - ensembler = None - context_precision = None - ContextPrecisionVerification = None - - -def _get_ml_app_for_ragas_trace(span_event: dict) -> str: - """ - The `ml_app` spans generated from traces of ragas will be named as `dd-ragas-` - or `dd-ragas` if `ml_app` is not present in the span event. - """ - tags = span_event.get("tags", []) # list[str] - ml_app = None - for tag in tags: - if isinstance(tag, str) and tag.startswith("ml_app:"): - ml_app = tag.split(":")[1] - break - if not ml_app: - return RAGAS_ML_APP_PREFIX - return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) - - -def _get_context_precision_instance(): - """ - This helper function ensures the context precision instance used in - ragas evaluator is updated with the latest ragas context precision instance - instance AND has an non-null llm - """ - if MiniRagas.context_precision is None: - return None - ragas_context_precision_instance = MiniRagas.context_precision - if not ragas_context_precision_instance.llm: - ragas_context_precision_instance.llm = MiniRagas.llm_factory() - return ragas_context_precision_instance - - -class RagasContextPrecisionEvaluator: +class RagasContextPrecisionEvaluator(RagasBaseEvaluator): """A class used by EvaluatorRunner to conduct ragas context precision evaluations on LLM Observability span events. The job of an Evaluator is to take a span and submit evaluation metrics based on the span's attributes. @@ -92,66 +47,25 @@ def __init__(self, llmobs_service): Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. """ - self.llmobs_service = llmobs_service - self.ragas_version = "unknown" - telemetry_state = "ok" - try: - import ragas - - self.ragas_version = parse_version(ragas.__version__) - if self.ragas_version >= (0, 2, 0) or self.ragas_version < (0, 1, 10): - raise NotImplementedError( - "Ragas version: {} is not supported for `ragas_context_precision` evaluator".format( - self.ragas_version - ), - ) - - from ragas.llms import llm_factory - - MiniRagas.llm_factory = llm_factory - - from ragas.llms.output_parser import RagasoutputParser - - MiniRagas.RagasoutputParser = RagasoutputParser - - from ragas.metrics import context_precision - - MiniRagas.context_precision = context_precision - - from ragas.metrics.base import ensembler - - MiniRagas.ensembler = ensembler - - from ddtrace.llmobs._evaluators.ragas.models import ContextPrecisionVerification - - MiniRagas.ContextPrecisionVerification = ContextPrecisionVerification - - except Exception as e: - telemetry_state = "fail" - telemetry_writer.add_log( - level=TELEMETRY_LOG_LEVEL.ERROR, - message="Failed to import Ragas dependencies", - stack_trace=traceback.format_exc(), - tags={"ragas_version": self.ragas_version}, - ) - raise NotImplementedError("Failed to load dependencies for `ragas_context_precision` evaluator") from e - finally: - telemetry_writer.add_count_metric( - namespace=TELEMETRY_APM_PRODUCT.LLMOBS, - name="evaluators.init", - value=1, - tags=( - ("evaluator_label", self.LABEL), - ("state", telemetry_state), - ("ragas_version", self.ragas_version), - ), - ) - - self.ragas_context_precision_instance = _get_context_precision_instance() - self.context_precision_output_parser = MiniRagas.RagasoutputParser( - pydantic_object=MiniRagas.ContextPrecisionVerification + super().__init__(llmobs_service) + self.ragas_context_precision_instance = self._get_context_precision_instance() + self.context_precision_output_parser = self.mini_ragas.RagasoutputParser( + pydantic_object=self.mini_ragas.ContextPrecisionVerification ) + def _get_context_precision_instance(self): + """ + This helper function ensures the context precision instance used in + ragas evaluator is updated with the latest ragas context precision instance + instance AND has an non-null llm + """ + if self.mini_ragas.context_precision is None: + return None + ragas_context_precision_instance = self.mini_ragas.context_precision + if not ragas_context_precision_instance.llm: + ragas_context_precision_instance.llm = self.mini_ragas.llm_factory() + return ragas_context_precision_instance + def run_and_submit_evaluation(self, span_event: dict): if not span_event: return @@ -235,7 +149,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] If the ragas context precision instance does not have `llm` set, we set `llm` using the `llm_factory()` method from ragas which currently defaults to openai's gpt-4o-turbo. """ - self.ragas_context_precision_instance = _get_context_precision_instance() + self.ragas_context_precision_instance = self._get_context_precision_instance() if not self.ragas_context_precision_instance: return "fail_context_precision_is_none", {} @@ -289,12 +203,12 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] ] ) - answers = [] # type: list[MiniRagas.ContextPrecisionVerification] + answers = [] for response in responses: - agg_answer = MiniRagas.ensembler.from_discrete([response], "verdict") + agg_answer = self.mini_ragas.ensembler.from_discrete([response], "verdict") if agg_answer: try: - agg_answer = MiniRagas.ContextPrecisionVerification.parse_obj(agg_answer[0]) + agg_answer = self.mini_ragas.ContextPrecisionVerification.parse_obj(agg_answer[0]) except Exception as e: logger.debug( "Failed to parse context precision verification for `ragas_context_precision`", diff --git a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py index d651c2443a4..0de7dd3ab62 100644 --- a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py +++ b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py @@ -17,6 +17,7 @@ from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX +from ddtrace.llmobs._evaluators.ragas.base import RagasBaseEvaluator logger = get_logger(__name__) @@ -67,7 +68,7 @@ def _get_faithfulness_instance() -> Optional[object]: return ragas_faithfulness_instance -class RagasFaithfulnessEvaluator: +class RagasFaithfulnessEvaluator(RagasBaseEvaluator): """A class used by EvaluatorRunner to conduct ragas faithfulness evaluations on LLM Observability span events. The job of an Evaluator is to take a span and submit evaluation metrics based on the span's attributes. From dfd6c574c59b4ae33c1f189eb838c9df42b639cf Mon Sep 17 00:00:00 2001 From: lievan Date: Thu, 12 Dec 2024 14:52:45 -0500 Subject: [PATCH 06/15] refactor faithfulness to use base ragas --- ddtrace/llmobs/_evaluators/ragas/base.py | 16 +++ .../llmobs/_evaluators/ragas/faithfulness.py | 131 +++--------------- 2 files changed, 38 insertions(+), 109 deletions(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/base.py b/ddtrace/llmobs/_evaluators/ragas/base.py index 84187164ed4..1972adabe86 100644 --- a/ddtrace/llmobs/_evaluators/ragas/base.py +++ b/ddtrace/llmobs/_evaluators/ragas/base.py @@ -51,6 +51,22 @@ def __init__(self): self.ContextPrecisionVerification = ContextPrecisionVerification + from ragas.metrics import faithfulness + + self.faithfulness = faithfulness + + from ragas.metrics.base import get_segmenter + + self.get_segmenter = get_segmenter + + from ddtrace.llmobs._evaluators.ragas.models import StatementFaithfulnessAnswers + + self.StatementFaithfulnessAnswers = StatementFaithfulnessAnswers + + from ddtrace.llmobs._evaluators.ragas.models import StatementsAnswers + + self.StatementsAnswers = StatementsAnswers + def _get_ml_app_for_ragas_trace(span_event: dict) -> str: """ diff --git a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py index 0de7dd3ab62..f6e1b6a1b23 100644 --- a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py +++ b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py @@ -1,16 +1,11 @@ import json import math -import traceback from typing import List from typing import Optional from typing import Tuple from typing import Union from ddtrace.internal.logger import get_logger -from ddtrace.internal.telemetry import telemetry_writer -from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT -from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.internal.utils.version import parse_version from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA from ddtrace.llmobs._constants import FAITHFULNESS_DISAGREEMENTS_METADATA @@ -54,20 +49,6 @@ def _get_ml_app_for_ragas_trace(span_event: dict) -> str: return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) -def _get_faithfulness_instance() -> Optional[object]: - """ - This helper function ensures the faithfulness instance used in - ragas evaluator is updated with the latest ragas faithfulness - instance AND has an non-null llm - """ - if MiniRagas.faithfulness is None: - return None - ragas_faithfulness_instance = MiniRagas.faithfulness - if not ragas_faithfulness_instance.llm: - ragas_faithfulness_instance.llm = MiniRagas.llm_factory() - return ragas_faithfulness_instance - - class RagasFaithfulnessEvaluator(RagasBaseEvaluator): """A class used by EvaluatorRunner to conduct ragas faithfulness evaluations on LLM Observability span events. The job of an Evaluator is to take a span and @@ -96,98 +77,30 @@ def __init__(self, llmobs_service): Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. """ - self.llmobs_service = llmobs_service - self.ragas_version = "unknown" - telemetry_state = "ok" - try: - import ragas - - self.ragas_version = parse_version(ragas.__version__) - if self.ragas_version >= (0, 2, 0) or self.ragas_version < (0, 1, 10): - raise NotImplementedError( - "Ragas version: {} is not supported for `ragas_faithfulness` evaluator".format(self.ragas_version), - ) - - from ragas.llms import llm_factory - - MiniRagas.llm_factory = llm_factory - - from ragas.llms.output_parser import RagasoutputParser - - MiniRagas.RagasoutputParser = RagasoutputParser - - from ragas.metrics import faithfulness - - MiniRagas.faithfulness = faithfulness - - from ragas.metrics.base import ensembler - - MiniRagas.ensembler = ensembler - - from ragas.metrics.base import get_segmenter - - MiniRagas.get_segmenter = get_segmenter - - from ddtrace.llmobs._evaluators.ragas.models import StatementFaithfulnessAnswers - - MiniRagas.StatementFaithfulnessAnswers = StatementFaithfulnessAnswers - - from ddtrace.llmobs._evaluators.ragas.models import StatementsAnswers - - MiniRagas.StatementsAnswers = StatementsAnswers - except Exception as e: - telemetry_state = "fail" - telemetry_writer.add_log( - level=TELEMETRY_LOG_LEVEL.ERROR, - message="Failed to import Ragas dependencies", - stack_trace=traceback.format_exc(), - tags={"ragas_version": self.ragas_version}, - ) - raise NotImplementedError("Failed to load dependencies for `ragas_faithfulness` evaluator") from e - finally: - telemetry_writer.add_count_metric( - namespace=TELEMETRY_APM_PRODUCT.LLMOBS, - name="evaluators.init", - value=1, - tags=( - ("evaluator_label", self.LABEL), - ("state", telemetry_state), - ("ragas_version", self.ragas_version), - ), - ) - - self.ragas_faithfulness_instance = _get_faithfulness_instance() - self.llm_output_parser_for_generated_statements = MiniRagas.RagasoutputParser( - pydantic_object=MiniRagas.StatementsAnswers + super().__init__(llmobs_service) + self.ragas_faithfulness_instance = self._get_faithfulness_instance() + self.llm_output_parser_for_generated_statements = self.mini_ragas.RagasoutputParser( + pydantic_object=self.mini_ragas.StatementsAnswers ) - self.llm_output_parser_for_faithfulness_score = MiniRagas.RagasoutputParser( - pydantic_object=MiniRagas.StatementFaithfulnessAnswers + self.llm_output_parser_for_faithfulness_score = self.mini_ragas.RagasoutputParser( + pydantic_object=self.mini_ragas.StatementFaithfulnessAnswers ) - self.split_answer_into_sentences = MiniRagas.get_segmenter( + self.split_answer_into_sentences = self.mini_ragas.get_segmenter( language=self.ragas_faithfulness_instance.nli_statements_message.language, clean=False ) - def run_and_submit_evaluation(self, span_event: dict): - if not span_event: - return - score_result_or_failure, metric_metadata = self.evaluate(span_event) - telemetry_writer.add_count_metric( - TELEMETRY_APM_PRODUCT.LLMOBS, - "evaluators.run", - 1, - tags=( - ("evaluator_label", self.LABEL), - ("state", score_result_or_failure if isinstance(score_result_or_failure, str) else "success"), - ), - ) - if isinstance(score_result_or_failure, float): - self.llmobs_service.submit_evaluation( - span_context={"trace_id": span_event.get("trace_id"), "span_id": span_event.get("span_id")}, - label=RagasFaithfulnessEvaluator.LABEL, - metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, - value=score_result_or_failure, - metadata=metric_metadata, - ) + def _get_faithfulness_instance(self) -> Optional[object]: + """ + This helper function ensures the faithfulness instance used in + ragas evaluator is updated with the latest ragas faithfulness + instance AND has an non-null llm + """ + if self.mini_ragas.faithfulness is None: + return None + ragas_faithfulness_instance = self.mini_ragas.faithfulness + if not ragas_faithfulness_instance.llm: + ragas_faithfulness_instance.llm = self.mini_ragas.llm_factory() + return ragas_faithfulness_instance def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: """ @@ -197,7 +110,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] If the ragas faithfulness instance does not have `llm` set, we set `llm` using the `llm_factory()` method from ragas which defaults to openai's gpt-4o-turbo. """ - self.ragas_faithfulness_instance = _get_faithfulness_instance() + self.ragas_faithfulness_instance = self._get_faithfulness_instance() if not self.ragas_faithfulness_instance: return "fail_faithfulness_is_none", {} @@ -319,9 +232,9 @@ def _create_verdicts(self, context: str, statements: List[str]): return None # collapse multiple generations into a single faithfulness list - faithfulness_list = MiniRagas.ensembler.from_discrete(raw_faithfulness_list, "verdict") # type: ignore + faithfulness_list = self.mini_ragas.ensembler.from_discrete(raw_faithfulness_list, "verdict") try: - return MiniRagas.StatementFaithfulnessAnswers.parse_obj(faithfulness_list) # type: ignore + return self.mini_ragas.StatementFaithfulnessAnswers.parse_obj(faithfulness_list) except Exception as e: logger.debug("Failed to parse faithfulness_list", exc_info=e) return None From dd1abb4a2c5cb91fff64e8a3ba0ef4f3243a8ccc Mon Sep 17 00:00:00 2001 From: lievan Date: Thu, 12 Dec 2024 16:44:50 -0500 Subject: [PATCH 07/15] nit changes --- .../_evaluators/ragas/context_precision.py | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index a2d80251ea8..dad10d952c9 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -4,8 +4,6 @@ from typing import Union from ddtrace.internal.logger import get_logger -from ddtrace.internal.telemetry import telemetry_writer -from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS @@ -66,28 +64,6 @@ def _get_context_precision_instance(self): ragas_context_precision_instance.llm = self.mini_ragas.llm_factory() return ragas_context_precision_instance - def run_and_submit_evaluation(self, span_event: dict): - if not span_event: - return - score_result_or_failure, metric_metadata = self.evaluate(span_event) - telemetry_writer.add_count_metric( - TELEMETRY_APM_PRODUCT.LLMOBS, - "evaluators.run", - 1, - tags=( - ("evaluator_label", self.LABEL), - ("state", score_result_or_failure if isinstance(score_result_or_failure, str) else "success"), - ), - ) - if isinstance(score_result_or_failure, float): - self.llmobs_service.submit_evaluation( - span_context={"trace_id": span_event.get("trace_id"), "span_id": span_event.get("span_id")}, - label=self.LABEL, - metric_type=self.METRIC_TYPE, - value=score_result_or_failure, - metadata=metric_metadata, - ) - def _extract_inputs(self, span_event: dict) -> Optional[dict]: """ Extracts the question, answer, and context used as inputs to faithfulness @@ -180,7 +156,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] contexts = cp_inputs["contexts"] answer = cp_inputs["answer"] - # create a prompt to evaluate each context chunk + # create a prompt to evaluate the relevancy of each context chunk ctx_precision_prompts = [ self.ragas_context_precision_instance.context_precision_prompt.format( question=question, context=c, answer=answer From d2f5f2998473badb3789d4651bd0f8b3a47a5b68 Mon Sep 17 00:00:00 2001 From: lievan Date: Sun, 15 Dec 2024 16:17:34 -0500 Subject: [PATCH 08/15] touch up comments and trace --- ddtrace/llmobs/_evaluators/ragas/base.py | 2 +- ddtrace/llmobs/_evaluators/ragas/context_precision.py | 7 ++++--- ddtrace/llmobs/_evaluators/ragas/faithfulness.py | 3 ++- ddtrace/llmobs/_evaluators/runner.py | 2 ++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/base.py b/ddtrace/llmobs/_evaluators/ragas/base.py index 1972adabe86..7a2b3ab3298 100644 --- a/ddtrace/llmobs/_evaluators/ragas/base.py +++ b/ddtrace/llmobs/_evaluators/ragas/base.py @@ -90,7 +90,7 @@ class RagasBaseEvaluator(ABC): submit evaluation metrics based on the span's attributes. """ - LABEL = "ragas_context_precision" + LABEL = "ragas" METRIC_TYPE = "score" def __init__(self, llmobs_service): diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index dad10d952c9..7e34f2ad5e3 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -70,12 +70,13 @@ def _extract_inputs(self, span_event: dict) -> Optional[dict]: evaluation from a span event. question - input.prompt.variables.question OR input.messages[-1].content - context - input.prompt.variables.context + contexts - list of context prompt variables specified by + `input.prompt._dd_context_variable_keys` or defaults to `input.prompt.variables.context` answer - output.messages[-1].content """ with self.llmobs_service.workflow("dd-ragas.extract_context_precision_inputs") as extract_inputs_workflow: self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) - question, answer, context = None, None, None + question, answer, contexts = None, None, None meta_io = span_event.get("meta") if meta_io is None: @@ -109,7 +110,7 @@ def _extract_inputs(self, span_event: dict) -> Optional[dict]: question = input_messages[-1].get("content") self.llmobs_service.annotate( - span=extract_inputs_workflow, output_data={"question": question, "context": context, "answer": answer} + span=extract_inputs_workflow, output_data={"question": question, "contexts": contexts, "answer": answer} ) if any(field is None for field in (question, contexts, answer)): logger.debug("Failed to extract inputs required for faithfulness evaluation") diff --git a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py index f6e1b6a1b23..9bd68b6e1b7 100644 --- a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py +++ b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py @@ -250,7 +250,8 @@ def _extract_faithfulness_inputs(self, span_event: dict) -> Optional[dict]: evaluation from a span event. question - input.prompt.variables.question OR input.messages[-1].content - context - input.prompt.variables.context + context - joined string of context prompt variables specified by + `input.prompt._dd_context_variable_keys` or defaults to `input.prompt.variables.context` answer - output.messages[-1].content """ with self.llmobs_service.workflow("dd-ragas.extract_faithfulness_inputs") as extract_inputs_workflow: diff --git a/ddtrace/llmobs/_evaluators/runner.py b/ddtrace/llmobs/_evaluators/runner.py index babca5b5274..63c7a614b2a 100644 --- a/ddtrace/llmobs/_evaluators/runner.py +++ b/ddtrace/llmobs/_evaluators/runner.py @@ -66,6 +66,8 @@ def __init__(self, interval: float, llmobs_service=None, evaluators=None): ("state", evaluator_init_state), ), ) + else: + logger.warning("parsed unsupported evaluator: `%r`", evaluator) def start(self, *args, **kwargs): if not self.evaluators: From 9297b7d16c6cc41841d6b213af30bea3c1a13523 Mon Sep 17 00:00:00 2001 From: lievan Date: Sun, 15 Dec 2024 16:33:51 -0500 Subject: [PATCH 09/15] wip answer relevancy --- .../_evaluators/ragas/answer_relevancy.py | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py diff --git a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py new file mode 100644 index 00000000000..84aa8c708eb --- /dev/null +++ b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py @@ -0,0 +1,206 @@ +import math +from typing import Optional +from typing import Tuple +from typing import Union + +from ddtrace.internal.logger import get_logger +from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA +from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA +from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS +from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS +from ddtrace.llmobs._evaluators.ragas.base import RagasBaseEvaluator +from ddtrace.llmobs._evaluators.ragas.base import _get_ml_app_for_ragas_trace + + +logger = get_logger(__name__) + + +class RagasAnswerRelevancyEvaluator(RagasBaseEvaluator): + """A class used by EvaluatorRunner to conduct ragas answer relevancy evaluations + on LLM Observability span events. The job of an Evaluator is to take a span and + submit evaluation metrics based on the span's attributes. + """ + + LABEL = "ragas_context_precision" + METRIC_TYPE = "score" + + def __init__(self, llmobs_service): + """ + Initialize an evaluator that uses the ragas library to generate a context precision score on finished LLM spans. + + ResponseRelevancy metric focuses on assessing how pertinent the generated answer is to a given question. + A lower score is assigned to answers that are incomplete or contain redundant information and higher scores + indicate better relevancy. This metric is computed using the question, contexts, and answer. + + For more information, see https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/answer_relevance/ + + The `ragas.metrics.answer_relevancy` instance is used for context precision scores. + If there is no llm attribute set on this instance, it will be set to the + default `llm_factory()` which uses openai. + + :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and + submitting evaluation metrics. + + Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. + """ + super().__init__(llmobs_service) + self.ragas_answer_relevancy_instance = self._get_answer_relevancy_instance() + self.context_precision_output_parser = self.mini_ragas.RagasoutputParser( + pydantic_object=self.mini_ragas.ContextPrecisionVerification + ) + + def _get_answer_relevancy_instance(self): + """ + This helper function ensures the answer relevancy instance used in + ragas evaluator is updated with the latest ragas answer relevancy instance + instance AND has an non-null llm + """ + if self.mini_ragas.answer_relevancy is None: + return None + ragas_answer_relevancy_instance = self.mini_ragas.answer_relevancy + if not ragas_answer_relevancy_instance.llm: + ragas_answer_relevancy_instance.llm = self.mini_ragas.llm_factory() + return ragas_answer_relevancy_instance + + def _extract_inputs(self, span_event: dict) -> Optional[dict]: + """ + Extracts the question, answer, and context used as inputs to faithfulness + evaluation from a span event. + + question - input.prompt.variables.question OR input.messages[-1].content + contexts - list of context prompt variables specified by + `input.prompt._dd_context_variable_keys` or defaults to `input.prompt.variables.context` + answer - output.messages[-1].content + """ + with self.llmobs_service.workflow("dd-ragas.extract_answer_relevancy_inputs") as extract_inputs_workflow: + self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) + question, answer, contexts = None, None, None + + meta_io = span_event.get("meta") + if meta_io is None: + return None + + meta_input = meta_io.get("input") + meta_output = meta_io.get("output") + + if not (meta_input and meta_output): + return None + + prompt = meta_input.get("prompt") + if prompt is None: + logger.debug("Failed to extract `prompt` from span for `ragas_faithfulness` evaluation") + return None + prompt_variables = prompt.get("variables") + + input_messages = meta_input.get("messages") + + messages = meta_output.get("messages") + if messages is not None and len(messages) > 0: + answer = messages[-1].get("content") + + if prompt_variables: + context_keys = prompt.get(INTERNAL_CONTEXT_VARIABLE_KEYS, ["context"]) + question_keys = prompt.get(INTERNAL_QUERY_VARIABLE_KEYS, ["question"]) + contexts = [prompt_variables.get(key) for key in context_keys if prompt_variables.get(key)] + question = " ".join([prompt_variables.get(key) for key in question_keys if prompt_variables.get(key)]) + + if not question and input_messages is not None and len(input_messages) > 0: + question = input_messages[-1].get("content") + + self.llmobs_service.annotate( + span=extract_inputs_workflow, output_data={"question": question, "contexts": contexts, "answer": answer} + ) + if any(field is None for field in (question, contexts, answer)): + logger.debug("Failed to extract inputs required for faithfulness evaluation") + return None + + return {"question": question, "contexts": contexts, "answer": answer} + + def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: + """ + Performs a context precision evaluation on a retrieval span event, returning either + - context precision score (float) OR failure reason (str) + - evaluation metadata (dict) + If the ragas context precision instance does not have `llm` set, we set `llm` using the `llm_factory()` + method from ragas which currently defaults to openai's gpt-4o-turbo. + """ + self.ragas_context_precision_instance = self._get_context_precision_instance() + if not self.ragas_context_precision_instance: + return "fail_context_precision_is_none", {} + + evaluation_metadata = {EVALUATION_KIND_METADATA: "context_precision"} # type: dict[str, Union[str, dict, list]] + + # initialize data we annotate for tracing ragas + score, question, answer = ( + math.nan, + None, + None, + ) + + with self.llmobs_service.workflow( + "dd-ragas.context_precision", ml_app=_get_ml_app_for_ragas_trace(span_event) + ) as ragas_cp_workflow: + try: + evaluation_metadata[EVALUATION_SPAN_METADATA] = self.llmobs_service.export_span(span=ragas_cp_workflow) + + cp_inputs = self._extract_inputs(span_event) + if cp_inputs is None: + logger.debug( + "Failed to extract question and contexts from " + "span sampled for `ragas_context_precision` evaluation" + ) + return "fail_extract_context_precision_inputs", evaluation_metadata + + question = cp_inputs["question"] + contexts = cp_inputs["contexts"] + answer = cp_inputs["answer"] + + # create a prompt to evaluate the relevancy of each context chunk + ctx_precision_prompts = [ + self.ragas_context_precision_instance.context_precision_prompt.format( + question=question, context=c, answer=answer + ) + for c in contexts + ] + + responses = [] + + for prompt in ctx_precision_prompts: + result = self.ragas_context_precision_instance.llm.generate_text(prompt) + reproducibility = getattr(self.ragas_context_precision_instance, "_reproducibility", 1) + + results = [result.generations[0][i].text for i in range(reproducibility)] + responses.append( + [ + res.dict() + for res in [self.context_precision_output_parser.parse(text) for text in results] + if res is not None + ] + ) + + answers = [] + for response in responses: + agg_answer = self.mini_ragas.ensembler.from_discrete([response], "verdict") + if agg_answer: + try: + agg_answer = self.mini_ragas.ContextPrecisionVerification.parse_obj(agg_answer[0]) + except Exception as e: + logger.debug( + "Failed to parse context precision verification for `ragas_context_precision`", + exc_info=e, + ) + continue + answers.append(agg_answer) + + if len(answers) == 0: + return "fail_no_answers", evaluation_metadata + + verdict_list = [1 if ver.verdict else 0 for ver in answers] + score = sum(verdict_list) / len(verdict_list) + return score, evaluation_metadata + finally: + self.llmobs_service.annotate( + span=ragas_cp_workflow, + input_data=span_event, + output_data=score, + ) From 31f89c77ba2a98742871d2b8c8a40eaeac1a2151 Mon Sep 17 00:00:00 2001 From: lievan Date: Sun, 15 Dec 2024 16:34:28 -0500 Subject: [PATCH 10/15] fix comment --- ddtrace/llmobs/_evaluators/ragas/context_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index 7e34f2ad5e3..3d9d2e0e6fe 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -120,7 +120,7 @@ def _extract_inputs(self, span_event: dict) -> Optional[dict]: def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: """ - Performs a context precision evaluation on a retrieval span event, returning either + Performs a context precision evaluation on an llm span event, returning either - context precision score (float) OR failure reason (str) - evaluation metadata (dict) If the ragas context precision instance does not have `llm` set, we set `llm` using the `llm_factory()` From 9b19dab1f0d813362fc22ec7f98d8a4667e71558 Mon Sep 17 00:00:00 2001 From: lievan Date: Sun, 15 Dec 2024 16:46:34 -0500 Subject: [PATCH 11/15] more wip --- .../_evaluators/ragas/answer_relevancy.py | 39 ++++++++++--------- .../_evaluators/ragas/context_precision.py | 4 +- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py index 84aa8c708eb..757d84f0985 100644 --- a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py +++ b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py @@ -28,8 +28,8 @@ def __init__(self, llmobs_service): """ Initialize an evaluator that uses the ragas library to generate a context precision score on finished LLM spans. - ResponseRelevancy metric focuses on assessing how pertinent the generated answer is to a given question. - A lower score is assigned to answers that are incomplete or contain redundant information and higher scores + ResponseRelevancy metric focuses on assessing how pertinent the generated answer is to a given question. + A lower score is assigned to answers that are incomplete or contain redundant information and higher scores indicate better relevancy. This metric is computed using the question, contexts, and answer. For more information, see https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/answer_relevance/ @@ -64,7 +64,7 @@ def _get_answer_relevancy_instance(self): def _extract_inputs(self, span_event: dict) -> Optional[dict]: """ - Extracts the question, answer, and context used as inputs to faithfulness + Extracts the question, answer, and context used as inputs to a answer relevancy evaluation from a span event. question - input.prompt.variables.question OR input.messages[-1].content @@ -118,17 +118,17 @@ def _extract_inputs(self, span_event: dict) -> Optional[dict]: def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: """ - Performs a context precision evaluation on a retrieval span event, returning either - - context precision score (float) OR failure reason (str) + Performs a answer relevancy evaluation on an llm span event, returning either + - answer relevancy score (float) OR failure reason (str) - evaluation metadata (dict) - If the ragas context precision instance does not have `llm` set, we set `llm` using the `llm_factory()` + If the ragas answer relevancy instance does not have `llm` set, we set `llm` using the `llm_factory()` method from ragas which currently defaults to openai's gpt-4o-turbo. """ self.ragas_context_precision_instance = self._get_context_precision_instance() if not self.ragas_context_precision_instance: - return "fail_context_precision_is_none", {} + return "fail_answer_relevancy_is_none", {} - evaluation_metadata = {EVALUATION_KIND_METADATA: "context_precision"} # type: dict[str, Union[str, dict, list]] + evaluation_metadata = {} # type: dict[str, Union[str, dict, list]] # initialize data we annotate for tracing ragas score, question, answer = ( @@ -138,7 +138,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] ) with self.llmobs_service.workflow( - "dd-ragas.context_precision", ml_app=_get_ml_app_for_ragas_trace(span_event) + "dd-ragas.answer_relevancy", ml_app=_get_ml_app_for_ragas_trace(span_event) ) as ragas_cp_workflow: try: evaluation_metadata[EVALUATION_SPAN_METADATA] = self.llmobs_service.export_span(span=ragas_cp_workflow) @@ -147,21 +147,22 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] if cp_inputs is None: logger.debug( "Failed to extract question and contexts from " - "span sampled for `ragas_context_precision` evaluation" + "span sampled for `ragas_answer_relevancy` evaluation" ) - return "fail_extract_context_precision_inputs", evaluation_metadata + return "fail_extract_answer_relevancy_inputs", evaluation_metadata question = cp_inputs["question"] - contexts = cp_inputs["contexts"] + contexts = cp_inputs["context"] answer = cp_inputs["answer"] - # create a prompt to evaluate the relevancy of each context chunk - ctx_precision_prompts = [ - self.ragas_context_precision_instance.context_precision_prompt.format( - question=question, context=c, answer=answer - ) - for c in contexts - ] + prompt = self.ragas_answer_relevancy_instance.question_generation.format( + answer=answer, + context="\n".join(contexts), + ) + + result = self.ragas_answer_relevancy_instance.llm.generate_text( + prompt, n=self.ragas_answer_relevancy_instance.strictness + ) responses = [] diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index 7e34f2ad5e3..af36fc5e120 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -66,7 +66,7 @@ def _get_context_precision_instance(self): def _extract_inputs(self, span_event: dict) -> Optional[dict]: """ - Extracts the question, answer, and context used as inputs to faithfulness + Extracts the question, answer, and context used as inputs to a context precision evaluation from a span event. question - input.prompt.variables.question OR input.messages[-1].content @@ -120,7 +120,7 @@ def _extract_inputs(self, span_event: dict) -> Optional[dict]: def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: """ - Performs a context precision evaluation on a retrieval span event, returning either + Performs a context precision evaluation on an llm span event, returning either - context precision score (float) OR failure reason (str) - evaluation metadata (dict) If the ragas context precision instance does not have `llm` set, we set `llm` using the `llm_factory()` From 29422724ddeb03b80b9ae1c1478da623dd277d9a Mon Sep 17 00:00:00 2001 From: lievan Date: Sun, 15 Dec 2024 16:55:52 -0500 Subject: [PATCH 12/15] extract out input extraction --- ddtrace/llmobs/_evaluators/ragas/base.py | 57 +++++++++++++++++ .../_evaluators/ragas/context_precision.py | 60 +----------------- .../llmobs/_evaluators/ragas/faithfulness.py | 62 +------------------ tests/llmobs/_utils.py | 4 +- 4 files changed, 64 insertions(+), 119 deletions(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/base.py b/ddtrace/llmobs/_evaluators/ragas/base.py index 7a2b3ab3298..8cfe59eb6cf 100644 --- a/ddtrace/llmobs/_evaluators/ragas/base.py +++ b/ddtrace/llmobs/_evaluators/ragas/base.py @@ -10,6 +10,8 @@ from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL from ddtrace.internal.utils.version import parse_version +from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS +from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX @@ -128,6 +130,61 @@ def __init__(self, llmobs_service): ), ) + def _extract_evaluation_inputs_from_span(self, span_event: dict) -> Optional[dict]: + """ + Extracts the question, answer, and context used as inputs for a ragas evaluation on a span event. + + question - input.prompt.variables.question OR input.messages[-1].content + contexts - list of context prompt variables specified by + `input.prompt._dd_context_variable_keys` or defaults to `input.prompt.variables.context` + answer - output.messages[-1].content + """ + with self.llmobs_service.workflow("dd-ragas.extract_evaluation_inputs_from_span") as extract_inputs_workflow: + self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) + question, answer, contexts = None, None, None + + meta_io = span_event.get("meta") + if meta_io is None: + return None + + meta_input = meta_io.get("input") + meta_output = meta_io.get("output") + + if not (meta_input and meta_output): + return None + + prompt = meta_input.get("prompt") + if prompt is None: + logger.debug( + "Failed to extract `prompt` from span for ragas evaluation", + ) + return None + prompt_variables = prompt.get("variables") + + input_messages = meta_input.get("messages") + + messages = meta_output.get("messages") + if messages is not None and len(messages) > 0: + answer = messages[-1].get("content") + + if prompt_variables: + context_keys = prompt.get(INTERNAL_CONTEXT_VARIABLE_KEYS, ["context"]) + question_keys = prompt.get(INTERNAL_QUERY_VARIABLE_KEYS, ["question"]) + contexts = [prompt_variables.get(key) for key in context_keys if prompt_variables.get(key)] + question = " ".join([prompt_variables.get(key) for key in question_keys if prompt_variables.get(key)]) + + if not question and input_messages is not None and len(input_messages) > 0: + question = input_messages[-1].get("content") + + self.llmobs_service.annotate( + span=extract_inputs_workflow, output_data={"question": question, "contexts": contexts, "answer": answer} + ) + if any(field is None for field in (question, contexts, answer)): + logger.debug("Failed to extract inputs required for ragas evaluation") + return None + + return {"question": question, "contexts": contexts, "answer": answer} + def run_and_submit_evaluation(self, span_event: dict): if not span_event: return diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index 3d9d2e0e6fe..f36c0a825c4 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -6,8 +6,6 @@ from ddtrace.internal.logger import get_logger from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA -from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS -from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS from ddtrace.llmobs._evaluators.ragas.base import RagasBaseEvaluator from ddtrace.llmobs._evaluators.ragas.base import _get_ml_app_for_ragas_trace @@ -64,60 +62,6 @@ def _get_context_precision_instance(self): ragas_context_precision_instance.llm = self.mini_ragas.llm_factory() return ragas_context_precision_instance - def _extract_inputs(self, span_event: dict) -> Optional[dict]: - """ - Extracts the question, answer, and context used as inputs to faithfulness - evaluation from a span event. - - question - input.prompt.variables.question OR input.messages[-1].content - contexts - list of context prompt variables specified by - `input.prompt._dd_context_variable_keys` or defaults to `input.prompt.variables.context` - answer - output.messages[-1].content - """ - with self.llmobs_service.workflow("dd-ragas.extract_context_precision_inputs") as extract_inputs_workflow: - self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) - question, answer, contexts = None, None, None - - meta_io = span_event.get("meta") - if meta_io is None: - return None - - meta_input = meta_io.get("input") - meta_output = meta_io.get("output") - - if not (meta_input and meta_output): - return None - - prompt = meta_input.get("prompt") - if prompt is None: - logger.debug("Failed to extract `prompt` from span for `ragas_faithfulness` evaluation") - return None - prompt_variables = prompt.get("variables") - - input_messages = meta_input.get("messages") - - messages = meta_output.get("messages") - if messages is not None and len(messages) > 0: - answer = messages[-1].get("content") - - if prompt_variables: - context_keys = prompt.get(INTERNAL_CONTEXT_VARIABLE_KEYS, ["context"]) - question_keys = prompt.get(INTERNAL_QUERY_VARIABLE_KEYS, ["question"]) - contexts = [prompt_variables.get(key) for key in context_keys if prompt_variables.get(key)] - question = " ".join([prompt_variables.get(key) for key in question_keys if prompt_variables.get(key)]) - - if not question and input_messages is not None and len(input_messages) > 0: - question = input_messages[-1].get("content") - - self.llmobs_service.annotate( - span=extract_inputs_workflow, output_data={"question": question, "contexts": contexts, "answer": answer} - ) - if any(field is None for field in (question, contexts, answer)): - logger.debug("Failed to extract inputs required for faithfulness evaluation") - return None - - return {"question": question, "contexts": contexts, "answer": answer} - def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: """ Performs a context precision evaluation on an llm span event, returning either @@ -145,10 +89,10 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] try: evaluation_metadata[EVALUATION_SPAN_METADATA] = self.llmobs_service.export_span(span=ragas_cp_workflow) - cp_inputs = self._extract_inputs(span_event) + cp_inputs = self._extract_evaluation_inputs_from_span(span_event) if cp_inputs is None: logger.debug( - "Failed to extract question and contexts from " + "Failed to extract evaluation inputs from " "span sampled for `ragas_context_precision` evaluation" ) return "fail_extract_context_precision_inputs", evaluation_metadata diff --git a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py index 9bd68b6e1b7..ac4880dfa0b 100644 --- a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py +++ b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py @@ -9,8 +9,6 @@ from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA from ddtrace.llmobs._constants import FAITHFULNESS_DISAGREEMENTS_METADATA -from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS -from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX from ddtrace.llmobs._evaluators.ragas.base import RagasBaseEvaluator @@ -134,16 +132,16 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] span=ragas_faithfulness_workflow ) - faithfulness_inputs = self._extract_faithfulness_inputs(span_event) + faithfulness_inputs = self._extract_evaluation_inputs_from_span(span_event) if faithfulness_inputs is None: logger.debug( - "Failed to extract question and context from span sampled for ragas_faithfulness evaluation" + "Failed to extract evaluation inputs from span sampled for `ragas_faithfulness` evaluation" ) return "fail_extract_faithfulness_inputs", evaluation_metadata question = faithfulness_inputs["question"] answer = faithfulness_inputs["answer"] - context = faithfulness_inputs["context"] + context = " ".join(faithfulness_inputs["contexts"]) statements = self._create_statements(question, answer) if statements is None: @@ -244,60 +242,6 @@ def _create_verdicts(self, context: str, statements: List[str]): output_data=faithfulness_list, ) - def _extract_faithfulness_inputs(self, span_event: dict) -> Optional[dict]: - """ - Extracts the question, answer, and context used as inputs to faithfulness - evaluation from a span event. - - question - input.prompt.variables.question OR input.messages[-1].content - context - joined string of context prompt variables specified by - `input.prompt._dd_context_variable_keys` or defaults to `input.prompt.variables.context` - answer - output.messages[-1].content - """ - with self.llmobs_service.workflow("dd-ragas.extract_faithfulness_inputs") as extract_inputs_workflow: - self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) - question, answer, context = None, None, None - - meta_io = span_event.get("meta") - if meta_io is None: - return None - - meta_input = meta_io.get("input") - meta_output = meta_io.get("output") - - if not (meta_input and meta_output): - return None - - prompt = meta_input.get("prompt") - if prompt is None: - logger.debug("Failed to extract `prompt` from span for `ragas_faithfulness` evaluation") - return None - prompt_variables = prompt.get("variables") - - input_messages = meta_input.get("messages") - - messages = meta_output.get("messages") - if messages is not None and len(messages) > 0: - answer = messages[-1].get("content") - - if prompt_variables: - context_keys = prompt.get(INTERNAL_CONTEXT_VARIABLE_KEYS, ["context"]) - question_keys = prompt.get(INTERNAL_QUERY_VARIABLE_KEYS, ["question"]) - context = " ".join([prompt_variables.get(key) for key in context_keys if prompt_variables.get(key)]) - question = " ".join([prompt_variables.get(key) for key in question_keys if prompt_variables.get(key)]) - - if not question and input_messages is not None and len(input_messages) > 0: - question = input_messages[-1].get("content") - - self.llmobs_service.annotate( - span=extract_inputs_workflow, output_data={"question": question, "context": context, "answer": answer} - ) - if any(field is None for field in (question, context, answer)): - logger.debug("Failed to extract inputs required for faithfulness evaluation") - return None - - return {"question": question, "context": context, "answer": answer} - def _create_statements_prompt(self, answer, question): # Returns: `ragas.llms.PromptValue` object with self.llmobs_service.task("dd-ragas.create_statements_prompt"): diff --git a/tests/llmobs/_utils.py b/tests/llmobs/_utils.py index 07b35280772..925893937a3 100644 --- a/tests/llmobs/_utils.py +++ b/tests/llmobs/_utils.py @@ -540,7 +540,7 @@ def _expected_ragas_context_precision_spans(ragas_inputs=None): "trace_id": mock.ANY, "span_id": mock.ANY, "parent_id": mock.ANY, - "name": "dd-ragas.extract_context_precision_inputs", + "name": "dd-ragas.extract_evaluation_inputs_from_span", "start_ns": mock.ANY, "duration": mock.ANY, "status": "ok", @@ -583,7 +583,7 @@ def _expected_ragas_faithfulness_spans(ragas_inputs=None): "trace_id": mock.ANY, "span_id": mock.ANY, "parent_id": mock.ANY, - "name": "dd-ragas.extract_faithfulness_inputs", + "name": "dd-ragas.extract_evaluation_inputs_from_span", "start_ns": mock.ANY, "duration": mock.ANY, "status": "ok", From 8e4a4524c96dcc35ce5c3f5e0b67351cd838570a Mon Sep 17 00:00:00 2001 From: lievan Date: Sun, 15 Dec 2024 17:39:22 -0500 Subject: [PATCH 13/15] ctx prec --- .../_evaluators/ragas/context_precision.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py index f36c0a825c4..11ab5ac47a5 100644 --- a/ddtrace/llmobs/_evaluators/ragas/context_precision.py +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -116,13 +116,20 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] reproducibility = getattr(self.ragas_context_precision_instance, "_reproducibility", 1) results = [result.generations[0][i].text for i in range(reproducibility)] - responses.append( - [ - res.dict() - for res in [self.context_precision_output_parser.parse(text) for text in results] - if res is not None - ] - ) + try: + responses.append( + [ + res.dict() + for res in [self.context_precision_output_parser.parse(text) for text in results] + if res is not None + ] + ) + except Exception as e: + logger.debug( + "Failed to parse context precision verification for `ragas_context_precision`", + exc_info=e, + ) + return "fail_context_precision_parsing", evaluation_metadata answers = [] for response in responses: @@ -135,7 +142,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] "Failed to parse context precision verification for `ragas_context_precision`", exc_info=e, ) - continue + return "fail_context_precision_parsing", evaluation_metadata answers.append(agg_answer) if len(answers) == 0: From d246bc5add09ac008888b6c3475596dbbcdfaaee Mon Sep 17 00:00:00 2001 From: lievan Date: Sun, 15 Dec 2024 18:11:31 -0500 Subject: [PATCH 14/15] answer relevancy --- .../_evaluators/ragas/answer_relevancy.py | 20 +- ddtrace/llmobs/_evaluators/runner.py | 2 + tests/llmobs/_utils.py | 42 ++ tests/llmobs/conftest.py | 9 + ....emits_traces_and_evaluations_on_exit.yaml | 586 ++++++++++++++++++ ...t_ragas_answer_relevancy_emits_traces.yaml | 549 ++++++++++++++++ ...s_answer_relevancy_submits_evaluation.yaml | 549 ++++++++++++++++ ...s_evaluation_on_span_with_custom_keys.yaml | 543 ++++++++++++++++ ...ion_on_span_with_question_in_messages.yaml | 543 ++++++++++++++++ tests/llmobs/test_llmobs_ragas_evaluators.py | 231 +++++++ 10 files changed, 3064 insertions(+), 10 deletions(-) create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_answer_relevancy_evaluator.emits_traces_and_evaluations_on_exit.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml diff --git a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py index 6ea9509b5d8..64d6315c0eb 100644 --- a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py +++ b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py @@ -18,7 +18,7 @@ class RagasAnswerRelevancyEvaluator(RagasBaseEvaluator): submit evaluation metrics based on the span's attributes. """ - LABEL = "ragas_context_precision" + LABEL = "ragas_answer_relevancy" METRIC_TYPE = "score" def __init__(self, llmobs_service): @@ -76,10 +76,11 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] evaluation_metadata = {} # type: dict[str, Union[str, dict, list]] # initialize data we annotate for tracing ragas - score, question, answer = ( + score, question, answer, answer_classifications = ( math.nan, None, None, + None, ) with self.llmobs_service.workflow( @@ -118,13 +119,9 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] # calculate score gen_questions = [answer.question for answer in answers] - evaluation_metadata.update( - { - "answer_classifications": [ - {"question": answer.question, "noncommittal": answer.noncommittal} for answer in answers - ] - } - ) + answer_classifications = [ + {"question": answer.question, "noncommittal": answer.noncommittal} for answer in answers + ] if all(q == "" for q in gen_questions): logger.warning("Invalid JSON response. Expected dictionary with key 'question'") return "fail_parse_answer_relevancy_output", evaluation_metadata @@ -133,5 +130,8 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] return score, evaluation_metadata finally: self.llmobs_service.annotate( - span=ragas_ar_workflow, input_data=span_event, output_data=score, metadata=evaluation_metadata + span=ragas_ar_workflow, + input_data=span_event, + output_data=score, + metadata={"answer_classifications": answer_classifications}, ) diff --git a/ddtrace/llmobs/_evaluators/runner.py b/ddtrace/llmobs/_evaluators/runner.py index 63c7a614b2a..dfd1d233b22 100644 --- a/ddtrace/llmobs/_evaluators/runner.py +++ b/ddtrace/llmobs/_evaluators/runner.py @@ -8,6 +8,7 @@ from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.telemetry import telemetry_writer from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.llmobs._evaluators.ragas.answer_relevancy import RagasAnswerRelevancyEvaluator from ddtrace.llmobs._evaluators.ragas.context_precision import RagasContextPrecisionEvaluator from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator from ddtrace.llmobs._evaluators.sampler import EvaluatorRunnerSampler @@ -19,6 +20,7 @@ SUPPORTED_EVALUATORS = { RagasFaithfulnessEvaluator.LABEL: RagasFaithfulnessEvaluator, RagasContextPrecisionEvaluator.LABEL: RagasContextPrecisionEvaluator, + RagasAnswerRelevancyEvaluator.LABEL: RagasAnswerRelevancyEvaluator, } diff --git a/tests/llmobs/_utils.py b/tests/llmobs/_utils.py index 925893937a3..9c827e35fbd 100644 --- a/tests/llmobs/_utils.py +++ b/tests/llmobs/_utils.py @@ -516,6 +516,48 @@ def _dummy_evaluator_eval_metric_event(span_id, trace_id): ) +def _expected_ragas_answer_relevancy_spans(ragas_inputs=None): + if not ragas_inputs: + ragas_inputs = default_ragas_inputs + return [ + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": "undefined", + "name": "dd-ragas.answer_relevancy", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + "metadata": { + "answer_classifications": mock.ANY, + }, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": mock.ANY, + "name": "dd-ragas.extract_evaluation_inputs_from_span", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + ] + + def _expected_ragas_context_precision_spans(ragas_inputs=None): if not ragas_inputs: ragas_inputs = default_ragas_inputs diff --git a/tests/llmobs/conftest.py b/tests/llmobs/conftest.py index acb79b5f413..d1787f44206 100644 --- a/tests/llmobs/conftest.py +++ b/tests/llmobs/conftest.py @@ -214,6 +214,15 @@ def reset_ragas_context_precision_llm(): ragas.metrics.context_precision.llm = previous_llm +@pytest.fixture +def reset_ragas_answer_relevancy_llm(): + import ragas + + previous_llm = ragas.metrics.answer_relevancy.llm + yield + ragas.metrics.answer_relevancy.llm = previous_llm + + @pytest.fixture def mock_ragas_evaluator(mock_llmobs_eval_metric_writer, ragas): patcher = mock.patch("ddtrace.llmobs._evaluators.ragas.faithfulness.RagasFaithfulnessEvaluator.evaluate") diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_answer_relevancy_evaluator.emits_traces_and_evaluations_on_exit.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_answer_relevancy_evaluator.emits_traces_and_evaluations_on_exit.yaml new file mode 100644 index 00000000000..e7e856615ab --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_answer_relevancy_evaluator.emits_traces_and_evaluations_on_exit.yaml @@ -0,0 +1,586 @@ +interactions: +- request: + body: '{"messages": [{"content": "Generate a question for the given answer and + Identify if answer is noncommittal. Give noncommittal as 1 if the answer is + noncommittal and 0 if the answer is committal. A noncommittal answer is one + that is evasive, vague, or ambiguous. For example, \"I don''t know\" or \"I''m + not sure\" are noncommittal answers\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"type\": \"object\", + \"properties\": {\"question\": {\"title\": \"Question\", \"type\": \"string\"}, + \"noncommittal\": {\"title\": \"Noncommittal\", \"type\": \"integer\"}}, \"required\": + [\"question\", \"noncommittal\"]}\n```\n\nDo not return any preamble or explanations, + return only a pure JSON string surrounded by triple backticks (```).\n\nExamples:\n\nanswer: + \"Albert Einstein was born in Germany.\"\ncontext: \"Albert Einstein was a German-born + theoretical physicist who is widely held to be one of the greatest and most + influential scientists of all time\"\noutput: ```{\"question\": \"Where was + Albert Einstein born?\", \"noncommittal\": 0}```\n\nanswer: \"It can change + its skin color based on the temperature of its environment.\"\ncontext: \"A + recent scientific study has discovered a new species of frog in the Amazon rainforest + that has the unique ability to change its skin color based on the temperature + of its environment.\"\noutput: ```{\"question\": \"What unique ability does + the newly discovered species of frog have?\", \"noncommittal\": 0}```\n\nanswer: + \"Everest\"\ncontext: \"The tallest mountain on Earth, measured from sea level, + is a renowned peak located in the Himalayas.\"\noutput: ```{\"question\": \"What + is the tallest mountain on Earth?\", \"noncommittal\": 0}```\n\nanswer: \"I + don''t know about the groundbreaking feature of the smartphone invented in + 2023 as am unaware of information beyond 2022. \"\ncontext: \"In 2023, a groundbreaking + invention was announced: a smartphone with a battery life of one month, revolutionizing + the way people use mobile technology.\"\noutput: ```{\"question\": \"What was + the groundbreaking feature of the smartphone invented in 2023?\", \"noncommittal\": + 1}```\n\nYour actual task:\n\nanswer: \"The capital of France is Paris\"\ncontext: + \"The capital of France is Paris.\"\noutput: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 3, "stream": false, "temperature": 0.3}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '2795' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//7FRNr9MwELznV1h7blA+qvYpF8QBBBKCxwGBRFDiOpvED8c29kZ6UPW/ + I6dpk+qBxB0uOczsbGbHXh8jxkA2UDAQPScxWBW/QPezeZfl7cv7D2geW/r89vXHh/f7N+TuOWyC + whweUNBF9UyYwSokafSZFg45Yeia7vNtnmyTPJ+IwTSogqyzFG9NPEgt4yzJtnGyj9O7Wd0bKdBD + wb5EjDF2nL7Bp27wEQqWbC7IgN7zDqG4FjEGzqiAAPdeeuKaYLOQwmhCPVmv6/pYwvcRfXBeQsFK + +NRzYtIz6pEJbiVxxUzLXjmuBT4vYcNK0EYLMwySiKugSk51Xa//4bAdPQ9z6lGpGT9dTSvTWWcO + fuaveCu19H3lkHujg0FPxkK0Ej9JIv2fxJxE9u8lETH2dVqY8WZesM4Mlioy31CHhrtsXhhY9nTF + 7maSDHG1wu8uxE2/qkHiUvlVvCC46LFZpMt+8rGRZkWsj/Cpm9/1Pk8udfc37RdCCLSETWUdNlLc + TryUOQzP2J/KrilPhsH/8IRD1UrdobNOni9Ma6tdK9IE0wQPEJ2iXwAAAP//AwBI224lUgUAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a16b1f89b0f37-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 23:07:14 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=8KcW_M1tPh2CApqZ0TNX7xpcAB0hlLWlX5.K5HS_rHE-1734304034-1.0.1.1-O6kv0rYNRSmOEj7VH66J3KUXjvU1CxUy06JMgOGcURfGU33D32JEXc7DN8yDcmg.bwzDKEN1DacsWVdQiE.gJw; + path=/; expires=Sun, 15-Dec-24 23:37:14 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=e4R2HUk_6vXIJNizbWmKIaD9HRDvF5K4MKiIwpoGjOo-1734304034147-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '431' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999320' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_fda69a5550657c48eeeaed82de75e63e + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '114' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1R62a6zurbm/X6KpXWb2gpd8GDdESCQ0NhJIF2pVAKSECCE0NiAj867l8h/dKrq + ZkqT6QkGD3/d8H/866+//q6T4pH2f//z19/vvOv//h/ztXvcx3//89f//Ndff/3113/8fv5/Ix9V + 8rjf80/2G/77Y/65P8a///lL+O8r/3fQfO/usyW6cQiS8fqNqTqZDmFB9CzQZNaOAvxD18wig+yx + MtBK1X4rAQvr8BNO1X4QQBKbG1uX1xwNXnmJIVDSKxUgH/kkaS4FIigf4ir9J+GqqVbI3199XDcX + uajvtNRBWz01LMjftzeRozeg9egZzOXXtJjug5tDdO1dshW/q7BfSw2F69jrDBfHVzIF8SNCmVc/ + mL4LcsR72gpoO778eT4GF+8HyYLbsz6zLc6WbX8/SDZoi5YQm5Kl16XjykCR1Lzo8hI5RVfULAM1 + 2G7IzY8HNKqrAqB1rSs7ewprp+Pxe4TmuxswYGInw+pRZmgp1BzLrn3xuHURclAP+ZbgbKd4NIjP + Z7ADd0s2q3MecnzWDY0a4gMvF/d12Itn24fxmafEum1eaMhbIQMosxgP6XFop29oK7DgboPDe/nm + LTkiHRbZ4stcFB9bmpzUCC3Ec4ulnVeGVBTiWs1PTko2b/Hc8kXxluCwLUeC3fZk8hUyUgir20hc + +WO34+OUVdDot454R68xR6hfEaiK7WHqLb5FfT/YErqXR5eZPdTeoCaXXG1k9UIb59qF3f2NFogU + yyWehvclpO/ltVkthYYTV09lkwdsopqrZjqV8WntyYq/2cL0bY8UoQdGbDc1GDmP+kAOe10vpD3L + AGx1T5hX+0Eo58s8RocycpjFYuRNMOmNhjaWShfcKcKBrr0jSj+uybzrpQ07qoRbKMs6IoHSbc2h + iFcKxGKeM3N5jdCQfLcA/jhpZLMtP6hL754CR/oA4q2igvPL7WCjGj22xKlXZTH4bklhNEeHrkb3 + lPAKJkXL9T4mdo9q3pP7oQTZEnfMuMY7k+/umgVoVUVUk6tPMoivSIfs3QKe+pNXDBtf6lB/YhdC + FvdX+JsPOpRnh0o33U46NfiqoGp9hYXjrkvGg/wawNAvE7HwSi/kkzMA7CSjYuZRLBO2/FwjuCnX + hJGV+jK77NBF4B0si0QK26BJ2NwjEI6NQzbd2HlNRNI92iySFyO7rV0Mb9G1kJVkH7IzxTgsvFOe + L/Hm6VCl1HQkLfREQOI2Vsn6lQYhI0dkqJFd7EjwRkPLa6GOwPPpmhmXaJuMKdqfNdnHKV4mDm+n + ek0n+IaHN7HXHW0nenMewLRNSzxjtQ0nFLUpNF9voJNweJn9a31dgHBPBeK2730xkGVg/96fLpxk + F7JPoWNNJl+J6Dgswm67vrjgasuY6HJVmyOEgKERuzM5FMfcZLa9UqCJRkqHtpzMjjRPUKVjMeBX + ugq8QTkJDzgEg0n2R6kouHG5GMjZSyFxLwENK8/3MBDZrtl6FEjBPxdvCxcteJMgepqc2/aoqoHy + uBLjs2vCaeKiC+XKOOM6887tdHZ2HcpUc4EXix6F7QoZD8iEa8G274Ns1h9yPGt1qWyIrTiNyR26 + pKvSUwgzdlsnZGm9pkAVd0vco9Nzattuoy4e04f9vudwkKYc+lfas7un9uFUSlkFl4PusHsqDt7o + R76urks3JM7tY6DJ3Ly30F0/e+Ipa5IMd9oZiDlnhVlRvi6kgcY24CEJ6FJaeO34ukg1Mn3/Sfzd + 7o24WcsuCrreINaFdHzisftAy4fNiHO6Knys9WUGzW2pM33zunnTXntUEOXHJ1vbyo7TzAkG9BGm + ngVy5YSC2PiWGu6HM0mXyiX5LIq38MMnKqtbLfxIX0sCpjktIWRzLwZwIxWg39nM80jUStWjSuGj + 3W5kw921KT3WOUWy76fMJ9LB5O1XH7R5PkR3P6t2mBYhVd/de8vcgpTJsGkSFZ6ED5h3Nk+4/zEa + 4EZyYetndU2+mEYYRAoaO+jPiPfyxiyRmkYi2ZzuUTLgqyxAYvVHit5c8frLRa1QZwWYal8KRXXK + IIagtM94WHyffDSavoEvrL+41eTSm97La41mvCdOMvacviTTBvuQe+yHRxRUhYK+8gO2Nk67gq+m + +Lq6hZuR2A/b5h13TAmdxf3IrHD58oa5vlGhah+2U/enGe90QMNbHIgrPO4JP6yHK0o/W5NE+Jwl + o/42qj//T8SthNg3lFxYVaXBnO+AvL4kpQWumuvETOie88zRDJj5izj7oxLyASSANIl7YvSntpi2 + WSyo5OYdGF59YjQ9h9oADpZJ9u7nVvSP4uErvPUSKq7rYzI879/rSr4FHhZeCTVHSLs9qOrRp+Vr + gUP2TJstXCXlSMILsvjI+UYFGk42nfdTMpk1UVD6WoyY73yzEOOX7v7Gs3BPUDJMi4SixhmWbK5/ + 3rqWe0XZIVTx9OncduyH+KzW93zEKrfsZGrwOtLm/Ukcq8vC0ZJ1DDSNJ7J5fsWEBUztQKQLjZjT + s04YOXIdSJa9mad6cTJdrl8BUhgfdNVZEZomrrmoR/p6Xt+A860fATwmbhD9bJOQS+S4Rdp9/2B4 + eKlhP6UZRjN/Mt0WgnaQK9lCuegLWFQDBY3hYTK0UhJOzCovZjH9+LIS0zddbt9VMp605gy9q36Z + +VYnj7pqq4OxciT6JQ54Y/Aut3/W/6C6iPft2XWhX9wTtl6Fm5BXm8KAnKQ6Od6zb9uJixVW90zq + mDPlm2S0gh5UiJcbFsRpzKdpqWL1JSodHoZm4j1b9A91o4LDHm9ybSksF0d43k4bcp+yindusKLI + P1YB0+fx7My9CcS4RMRrvz7njtkpqPArh+DmdDOnS/VM0WN4r5m/bgaTf3BoaX2zuWExY2dzEnI1 + h9Pic2KbjJ29SX+vBM0n94I4eV8kfb7wFHgvxh378X2PvTH94dWMTyuTyW9ZAIo+OV6N6redmqG0 + EVvVB0KWkcwHS7oC0m1FwEhCuJUFDA0CiSZ4afsHs/vhJyVXE8tYWYfSimxj+ICrE3LEjin/7u+9 + HzYe01Xv8X1S2LAynzcKKebtRB66BMaFbSg/b/Ymm/EfJe3hROwqDU2mRBpe7fBLxSvl6/3RUyAN + T+cPf0vJST2jmY+Zva0tc3hnESC1oDuq7OKl2Vd3GUMXJgu2uYQ7ky7yRYnE0+1LgtM5Tj5ttqew + xnLNtt6eI66xRoWL31jM0kQ7mVQ1xDCowURlwbMT+dMqe7hR2aYvWhZoSCjO1PvR3hEcFbonGYK1 + AH3d58SxpTZkD+O2gMXWL3/43fapfloAkx/7mS/dhF2bEcOsD9nNXSft6LRZo23Kq0XuZrk2pdvD + V9BBoTvmXxTHHGtdzkB9Ky6GT/81OdN9A/pTf2Hrm702aSllJUh41xMrMFvEhzjFgBFRyWaRGmhi + 0VqFX/2706sy+ft57xA+3iViuqbqDT89+7X3EsGz/h12DziDcqdvWijLJ6f1upqAkWWJl69QQH2K + 9pGGfSkmjmikiAfJpgLho+9YPL1sc/KyIdLyQ2OwgAp3czzkUKtG1Ats565R24W8z1RZ3uZk1qec + B0o+we5+fLFgF+TelCh3+/e+FLldFA7iYsTa092esOQngzeeDAZoutwPDIM+eNOq5ramaqxivn8b + i6nolQztPDuj8OPDqK8pDHl8xiLv14nEllEHStNHbE3Lgg/S/nCF5aexKA+vVjFCWu7RXB9ke2Tf + oqduXmpO/WVU/qYW4hWoCjoOzYKOVR6g6XJ9CQiWF4f4EdP4GFNt+NUXrt8H2aMyzlXYmSwg7kUz + 2p8+VvPv88Hs82f0Jns8PWAtRRGz3Obcsvchs9G8n5i3P50Stu6sLeDjU6K587TD7iR4OqoXKCd+ + EwvtRCpFgR/fbe+P9re+FKTOsSjM+oUJ3UFSYSx2xNIfYjuZhxyD4mUl01/XgznzcQNqvBWIJ2Az + /P2O/FCsCDb958yXbqnScLCZs312Cdc3kor4bmiIP66akF/ytatlOc6JX66X7bgLaf7DF3pByQGN + 1++xg+z89HE58iGcDliaYPmpLaa/0SHpnx+9g3Pbp4RIi7ad9aWi5aFk4iF6lu0An8nW0p3fsV/9 + jmc7fiB6KDfMlN81H6wrH+BZv0SGT9oZsXCqjrA/H65Y+N4tUzCdsIHn7bJhO2NRoK8V9Atgfu4y + 67jzw17bfzDQ98KhXf6pi9eNZIb2yauOEOGgJ4JZKhJgsXoS91AG7XSO5AjG+zn55QV8WLt1rbru + tcalLTctj1dWBJ6TMSxeb18+WfiEwYdPw5zyXCajvPFKNNcb8RLB5aOVhzY4/mNPJ8XqPTbq3xwg + fE/ErEMn6X56zZkCi+2WRuZxS70coe1bl1lhqvP+KUgR2DRf4aftH7wRy/4RDZ50JMGhwOH0coc9 + zH6GFju/KLhE4i1Y0qOf13+c/cKo/PASa8aha0dxey0BPRqM5bJR227jLzrYbZUnlhTHNen3AwYa + 7FYnJA2icKybwwBFD8qvPkIuZumgktTNmTNOB69/dcUEr8vyRHT/TovvrL9U5/ZYEt/JLCQhD10h + PQ5rEv/8eVTEW4D74sx8zeu8qXWFMwjJkeOBXxbetBHyB4xHTWbWofK9sRRXZ/TzM/aHOcUAH9UG + vU9lrJRaxsdfHtJMbMMs+bvxaPrIr0g/OyWz62Pt0R9/zX533m9iwrVA32pz/kOHXWAgoTHVCah6 + 3lH5FVM0deOuRHJWPpn57R/mcCn0q5ZZoUd2yrf1OiPVJ3hx2pKAq1oxblrTheB0YmT7PlzMge3Q + +ee/sZxGOOGRhR8w5ynM7Rdv3j8f+wY+qVETXbg/+FTzlY0OBCyW2DvBG/GN2Wjej3TqJi+UZr0A + wijJbBf073CclgbA/VQU9JentHHCtuhWGjdmH15Dy81iVan7D9aZkQYnr5StNILiHflszg+8Yb9/ + H5FUrSZGivsTDUa4p4hegoz98pUhkvc6xLfRZ2czN5PJ30o+0rZq+ke/98jzXTjxqSbePF/uf9wG + wuOa0ekS1UnTlMxYbXTDIORxWYa/74+qlGbEQw/MqU5fqha6/pGYh5yh4apdK6QwpWLnh6aF03lK + M6jP3oPtiJN6Ux5m5z/7ybfMTTjtxgl+zyMVfideb6SDASTvKnLYRxLqBWm4auBNp59eCtnX1nNg + maWx3WPjtHxdRJaq+KPGdhZKTN56baWW2N4zz1jVyR+/O+drBOtPwRyuF++MZv1HZj1m0tn/o2ZT + LoivTmc++5cY0cA32HaRmIV494cMpOJxZYZxe/OfPljtz+H1D58OperUq1Y0HsxaVWk4xHgY4FxZ + a5LwbV5M+xhJSHraJ7azR8bH45PVaBnuDdrOfDHeJndCUfhSmGGAbcoZJxKU9YYS57JYc+k+GJmG + VLEnbpQVHg+SoIR0dbaIJ25uqBduCYa1kp3YmiqqOeXpNYZQ5Zx4UJJicq/6XnuScWDnZhmGTTeu + S0itdUyp3JhFPShD/cefWupq4N9N67molKQTXdCyMCdjq25V6fgayCZyl3xa32BYzXkDFY9B3bJT + k3QIbWwVPx+XZzImfqhDHgomHo5S0fLFafDR+01EvErJEU276DTB7D+ZmTdVSNPPMUUnPwQqaGIV + jqOnWwi84TTzd4lm/Smgnx7DylAmza2CBVx2Nad8etZhc0F+CXL/CahqvvVkADdVoWNQMGOlvlBf + OYdcrSq2IIbLJo8txQDQzrMyOqmjEtLIfGJI2vCERfrJ0LD6pNUPb7Hs3Lxi/OU9j6I8stCyTXPc + V+sGvp/28wePR398+QAfTcCLN7kWw2VV2aB+TyXZpshFP/6FwrmKs/6XTP6r35N/AGZadmG23ahI + 6EZFm2bu59ZOPtL/5Lu4TsTGaz4WmmBYJtasf0wkf9phDwdVcdn+sUz4hDwUr7YSfbA5X/Ck+XvC + 1puuZNNHXTI2QWyjk6cX5GaKccLybZwjQz9NeJrzMPUcLSOY8YbY+fdj9tV+V8KKYUq2p+Up5PZk + UQiX2ob5QyXysU96/4cnP7+IeG/edHXDS+EPvrYNHiqYXkZFtt4+5EN7ihXIK8MiplkYrSxfXR+G + q7gm61N8SfhGPFA4TtKSCh+hMMflssuRtHmtGaZl4fHi4FY/viVWtl0V0/K+MtBRazs8rGnvtWTY + Y4DwMzHTX5Ji6GMNYPdqSmJ1Pfnjb8F14/qnr/hYN7cBZv1HF20dFNOqRjaSDvsLuSW7ozdMmZSp + b8+/MEcTPuYvf4doeQjwoOzqYnpJngWOn+6ZIRxeHteC7RZumFFi3tbv4nub3AHN+Qlztb3TDsZX + itHmbgxso+1wyGvtGqveuX8yf7nOeDPsQQVciQrbGKji32NbUnDT6svcm2kmLe7eHcz4y3amqIaD + GrxU6M43guUqI8m4fbYlisJCweN4YOFYlAcM9eroUJ4exoTepsnQNEkviLUpJY+yhpWwxmJN3Ju6 + Ql14UwVIjOLAtv5ZaEdOt5GS7nBHtehpIumGGgvNeS+JZ74b/GG1QMGKjdh6Glf+yw8BvfGRLmc/ + 2q9cWoN0YQaxb3qVTPPz0CXzZYLnPGQoiy+AHJkrgs+9Z4rCLfShkZULVYWHFvbOQDHo80yFC+mQ + vPxczwD22ySmbNTFuKdjB/V6TEkadLY5ZvYjA7jDmZinU8L7U7bZ/vIcdrucbXNizadEkZBmZK0M + G3MULGqhGxJF9vNf7+s37kCNtJwu5jyIo1Pcodk/sY0aXPn3l9ekq8hiIdnF7ZDeTQUkWYiZ03X7 + lmvHDdWW02PCe8vJk3Hsc+EPHox8obacRqMLiy0uCTmvuTeJwrEBub359PHwdS4WtmBov/6CQcRj + 8TX36kL96enF5h0V/Lz/GjCa3JnzrzMabSiOEL7d66z34mIKqiRGw1seqDAVZdHoB5+iZNle6DTz + CSefevvLn+f+k4EmJfN02CePZq4XklDk8RguGnnP/atb8sdfzHiCFcIjr3Py9RmqdrVn6VCJiN9L + oOjZM4f88pgpXyBXLfuDzjaNpbd8rlckk1aiy5kPvkPqdnBxHtPcz6nasezXexQ6ZM2M5nY2R3FZ + pyD5SvTL39rx6ok5XFaDRx7fh1FIeWMb6Gwsr1gLdGp2IX/nYGcdop2hla34UMwIrHtXM2PW4xNM + 2xp839ixTWg9zcmfrim6RPcX2T02n6KLL5n0618Qa+5fdGryzOB8OcVse726RacqQgfvYpESN1j3 + 4RCn1RWSLK2ZJ11p0pFlYMFRLyqc5Lrvydz2S2B7NaDyzcqLcd2rzQ9PmbVcjiFF32sFM96yzeps + hPJmsZ5++TxV3J63060CgDE+uWTjPD7mSN2mgvjy8dgvvxdyoazgbp/5f/k9nb4Udc1PJubye8vF + 5Dv4YFm7iHnLg8p/eh42zfrJgt2QJ5NcXA1IUbZh21EKwqkLyxjNeRPVrGbNpa0h1nDB0oaQw+PI + x/ah+pAK7sTW4T3zJi9Ce7Ukp+/8fQNPVjL1CMWKYqwutSMfjHQ7gN48KzLXF+8Eab1F2+XJZT9/ + SX/8fKZdzwz/Gno8HcZJ0ySjIKYqRgW/weEMefpgzMdF2g5Xzz5qanoWiXmuh5DZk9+huT+J5Tnf + 43mj7ZErRpTM+iHpnNvtiAJd63E989VEe6EC1EsL5qYSS4bzpsVgq0fCdmfzUPCkJ3skX9YuO+z1 + rOgzpRbgHd8cZm3KsylfmxVG83qTzRR3nG++aQMvUe3+9HO50N0k+IbhG6ufqSyGrLqliOS0IltN + KcJuBesOZn1I37vFo+0sab/Qon7jk91bCYuRtA4GQj4w8xPj3f2ABfRbz+/MP115KRW4JIJATGW7 + S3hvHnTNiQ495pZ2K9jpVJXoW8UYo7XJC77TLwqc3ajFE611T0qP7zO00zqY+ZdzbhxvKcz3p6K2 + MlrR80KAm3eMmOPasjnOeAOHYDLJr98z+7UUAQkYRXx5M9kznGwt/94fdIi4yofzITsi4f4QsJYe + CZpefWige9yJdPlsOBpzMZHAIajBYuQ+0bgLq3xFacWZY4StSdNjH8ExsL0/7zPi6JsjQ5bucx50 + aFkVX7cw62UMs974ujhbwN+/UwH/+a+//vpfvxMGVX1/vOeDAf1j7P/930cF/h3f438LgvTnGALt + 4uzx9z//dQLh729bV9/+f/d1+fh0f//zF/pz1ODvvu7j9/9z+V/zg/7zX/8HAAD//wMAiTuukd4g + AAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a16b6985f421f-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 23:07:14 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=X1w.GQfHeIODW2_BxQz9ETij38rCIVXMXHPkk4eWFbQ-1734304034-1.0.1.1-C6EG1yhesXlt7N8Vw5rSNImKU.Lzn.fdFxgg8DVihfGv2tHS6oZmzKhvYZqqNUQ4dxprIU4VYbBWexdLftSuJQ; + path=/; expires=Sun, 15-Dec-24 23:37:14 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=znk2FBaREh8URm7ZNKJTgLttIGzsZI.qF0.YpiVJ4E8-1734304034768-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '83' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999993' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_bd640733f6c8733b95f3f6e449bb95df + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30], [3923, 374, 279, 6864, + 315, 9822, 30], [3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '192' + content-type: + - application/json + cookie: + - __cf_bm=X1w.GQfHeIODW2_BxQz9ETij38rCIVXMXHPkk4eWFbQ-1734304034-1.0.1.1-C6EG1yhesXlt7N8Vw5rSNImKU.Lzn.fdFxgg8DVihfGv2tHS6oZmzKhvYZqqNUQ4dxprIU4VYbBWexdLftSuJQ; + _cfuvid=znk2FBaREh8URm7ZNKJTgLttIGzsZI.qF0.YpiVJ4E8-1734304034768-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1RZyxKqvJaen6fY9U/tKhGQLM4MAUG5JCp465GgIiAilwSS8/JduKu6qydWiRFC + stZ3y3/+9efPP3VSPNL+n3//+eedd/0//zVdu9/62z///vPf//rz58+f//w+/9/IR5U87vf8k/2G + /37MP/fH+M+//0j/e+X/Bk337j4bYpj7MBkv3xvVuOUSFsbPAnGrdlUQH7piNhkUn5WhXmrOWw1Z + VEefiFe7QQJ50VzZqrzkaPDL8w1CNb1QCfJRcFn3KBBJ/RBP7T+J0CytQsHuEuC6OStFfaelAfry + qWNJ+b59Tg7+gFajbzJPXNKC3wcvh/jSe2Sz+C6jfiU3FC5jbzBcHF4JD2+PGGV+/WDGNsyR6Gkr + oc34Cqb5mGJx38s2XJ/1iW1wNm/7+152QJ+1hDiUzP0uHZcmiuXmRefn2C26omYZaOFmTa7BbUCj + tiwAWs++sJOvspYfDt8DNN/tgAETJxmWjzJDc6kWWPGcsy/ss5SDts83BGdb1afh7XQCJ/Q2ZL08 + 5ZHAJ8PUqbl44Pnsvor6xckJYHzmKbGv6xca8lbKAMrshof0MLT8GzkqzITX4OhevkVLDsiAWTb7 + Mg/dDi1NjlqMZotTi+WtX0Z0Id1qLT+6KVm/F6dWzIq3DPtNORLstUdLLJGZQlRdR+IpH6cdH8es + gsa4dsQ/+I01Qv2KQVMdH1N/9i3q+96R0b08eMzqofYHLTnnWqNoZ9q4ly7q7m80Q6SYzzEf3ueI + vueXZjmXGkE8I1UsETJOdU/LDKrg48pX1GC9Af5tDxShB0ZsyxuM3Ee9J/udYRTyjmUAjrYjzK+D + MFLyeX5D+zJ2mc1uyOfAjUZHa1ujM+EW0UBX/gGlH89i/uXcRh1Vow2UZR2TUO021lDclircFnnO + rPklRkPy3QAEI9fJelN+UJfefRUO9AHEX8aFEOfr3kE1emyIWy/LYgi8ksJojS5djt4xERVwVc+N + /kacHtWiJ/d9CYq92DLzcttaYnvXbUDLKqa6Un2SYfGKDcjeLWDeH/1iWAdyh/ojOxMyu7+i33zQ + vjy5VL4aTtJp4VcDTe8rLB22XTLuldcApnHmxMZLo1CO7gCwlc2KWYdFmbD55xLDVb0kjCy1l9Vl + +y4Gf2/bJFbZGnFpfY9BOjQuWXdj5zcxSXdoPUtejGw3TjG8F56N7CT7kK21uEWFf8zzOV4/XaqW + uoHkmZFIaLG5aWT1SsOIkQMytdgptiR8o6EVtVTH4Ad0xcxzvEnGFO1OuhLgFM8TV7S8XlEO32j/ + Js6qoy2nV/cBTF+3xDeXm4ijuE2h+foD5dL+ZfWv1WUG0j2ViNe+d8VA5qHze386c5NtxD6FgXWF + fGVi4KiIus3q7IGnz2/EUKraGiECDM2iO5F9ccgt5jhLFZp4pHRoS251pHmCJh+KAb/SZegP6lF6 + wD4cLLI7yEUhzPPZRO5Ojoh3DmlU+YGPgShOzVajRArxOfsbOOvhm4Tx0xLCcUZNC9XHhZifbRNx + LhYelEvzhOvMP7X85G47lGnWDM9mPYraJTIfkEmXgm3ee8WqP+Rw0utSXRNHdRtLuHROl6WvEmZu + N27E0npFgarehngHtxfUcbxGmz34h/3Wc9jLPIf+lfbs7mt9xEs5q+C8N1x2TxeDPwZxYGir0ouI + e/2YiFvr9wa6y2dHfHVFkuFOOxMx96QyO85XhTzQmwN4SEI6l2d+O77Oco2sIHiSYLt9I2HViofC + rjeJfSad4OLmPdD84TDiHi+qGGtjnkFznRvMWL+uPt/pjwri/PBkK0fdCpq54YA+Eu9ZqFRuJC2a + wNai3XAi6Vw9J59Z8ZZ++EQVbaNHH/lry8B0tyWErO/FAF6sAfRbh/k+iVu5elQpfPTrlayFt7Lk + xyqnSAmClAVE3lui/RqDPs2HGN5n2Q58FlHt3b03zCtImQzrJtHgScSAReeIRAQfswFhJme2elaX + 5ItpjGFBQWd74xmLXllbJdLSeEHWx3ucDPiiSJDY/YGit1D9/nzWKtTZIab6l0JRHTO4QVg6JzzM + vk8xmk3fwBdWX9zqSunz9/xSownviZuMvaAv2XLA2ec+++ERBU2lYCyDkK3M47YQS367LK/ReiTO + w3FEJ1xLRqfFbmR2NH/5w1TfqND0D9tqu+OEdwag4b0YiCc97onYr4YLSj8bi8T4lCWj8Tarv/8n + i42M2DeSPVhWpcnc74D8viSlDZ6WG8RK6E6IzNVNmPiLuLuDGokBZIA0ufXE7I9twTfZTdLI1d8z + vPzcEH8OtQkCbIvsvM+16B/FI1BF6yd0saoPyfC8fy9L5Rr6WHol1Boh7XagaYeAlq8ZjtgzbTZw + kdUDic7IFqMQaw1oxB069VPCrZqoKH3NRiy2gVUsbi/D+41n0Y6gZOCzhKLGHeZsqn/RerZ3Qdk+ + 0jD/dF479sPtpNX3fMSasJ2EN3gV61N/Etfusmi0FQMDTW+crJ/fRcJCpnWwoDOdWPxZJ4wchAEk + y97M1/xbws+XrwQpjA+67OwYcS50D/XIWE37GwqxCWKABxcmMU4OiYRMDhuk33cPhoeXFvU8zTCa + +JMZjhS2g1IpNsoXgYQXWqiiMdpzUy9l6cjs8mwV/MeX1SJ90/nmXSXjUW9O0Hval1lvjfvU01oD + zKUr0y9xwR/Dd7n5u/97zUOib0+eB/3snrDVMlpHoloXJuQkNcjhnn3bbjFbYm3H5I65PF8nox32 + oMFtvmbhLb0Jzuca1l4LtcPD0HDRs1n/0NYauOzxJpeWwnx2gOf1uCZ3nlWi88IlRcGhCpkxjWcn + 4XNY3EpE/PYbCOFanYqKoHIJbo5Xi5+rZ4oew3vFglUzWOKDI1vvm/UVLzJ2sriUazkcZ58jW2fs + 5HPjvZT0gNwL4uZ9kfT5zFfhPRu37Mf3PfbH9IdXEz4tLaa8FQko+uR4OWrfljdD6SC2rPeEzGNF + DLZ8AWQ4qoSRjHCrSBgaBDJN8NwJ9lb3w09KLhZWsLqK5CXZ3OADnkHIAbuW8ru//344eEyXvS92 + SeHA0npeKaRYtJw8DBnMM1tTcVrvLDbhP0ra/ZE4VRpZTI11vNzil4aX6tf/q6dAHp7uX/6Wk6N2 + QhMfM2dT29bwzmJAWkG3VN3e5lZf3RUMXZTM2PocbS06y2clWhyvXxIeT7fk02Y7Cius1Gzj7wQS + Oms0OAeNzWx94SRc0yIMgxZyqki+kyifVt3BlSoOfdGyQENCcabdD86W4LgwfNmU7BkYqz4nriO3 + EXuY1xnMNkH5w++2T43jDJjy2E186SXs0owYJn3Irt4qaUe3zRp9XV5scrfKlSVfH4GK9irdsuCs + utZYG0oG2lv1MHz6ryWYEZjQH/szW12dlUVLOStBxtue2KHVIjHcUgwYEY2sZ6mJOItXGvzq3+Ov + yhLv571D+HCXieVZmj/89OzX2ckET/p32D7gBOqdvmmhzp+C1quKAyPzEs9fkYT6FO1iHQfyjbgL + M0UiTNYVSB9jy2785Vjcz4ZYz/eNyUIq3a1xn0OtmXEvsa23Qm0XiT7TFGWTk0mfChGqOYft/fBi + 4TbMfZ6od+f3vhR5XRwNi9mI9ae3OWI5SAZ/PJoMED/f9wyDMfh8WQtH13RWsSC4jgUvejVDW9/J + KPz4MO5rCkN+O+GF6FeJzOZxB2rTx2xFy0IM8m5/gfmnsamILnYxQlru0FQfZHNg36KnXl7qbv1l + VPmmNhIVaCo6DM2MjlUeIn6+vCQE87NLgpjpYrxRffjVF67fe8WnCs412FosJN5ZN9ufPtby7/PB + nNNn9LkzHh+wkuOY2V5zatl7nzlo6ifm747HhK06ewP48JRp7j6dqDtKvoHqGcpJ0NyklpNKVeHH + d5v7o/3tLwW5c20Kk35hUreXNRiLLbGNx6Ll1j7HoPpZyYzXZW9NfNyAdttIxJewFf2+oyBaVARb + wXPiS6/UaDQ4zN08u0QYa1lDYjs0JBiXTSTO+crTsxznJChX83bcRjT/4Qs9o2SPxsv30EF2ega4 + HMUQ8T2WOcw/tc2MN9on/fNjdHBq+5QQeda2k75U9TySLTzEz7Id4MMdPd0GHfvV73hybg9E9+Wa + Wcq7FoN9EQM869eC4aN+Qizi1QF2p/0FS9+7bUmWGzXwvJ7XbGvOCvS1w34GLMg9Zh+2QdTruw8G + +p65tMs/dfG6kszUP3nVESLtjUSySlUGvKiexNuXYctPsRLDeD8lv7xADCuvrjXPu9S4dJSmFbel + HYPvZgwvLtev4DY+Ygjg0zC3PJXJqKz9Ek31RvxE8sRo55EDbvDYUa7avc9G45sDRG9OrDpyk+6n + 11we2mw7NzNf2Nr5AG3fesyOUkP0T0mOwaH5Ej+dYO+PWAkOaPDlAwn3BY74yxt2MPkZWmyDohAy + uW3Alh/9tP/j5BdG9YeXWDf3XTsuNpcS0KPBWCkbre3WwayD7UZ9Yll1PYt+P2CiwWkNQtIwjsa6 + 2Q9Q9KD+6iMSiywdNJJ6OXNHvvf7V1dweJ3nR2IEd1p8J/2ludfHnARuZiMZ+egC6WFYkdvPn8fF + bQNwn51YoPudz1tPOoGUHAQexHnm87WUP2A86Aqz91Xgj+VieUI/P+N8mFsM8NEcMPpUwWqpZ2L8 + 5SENZ2tmK9+1T9NHfkHGyS2ZUx9qn/74a/K7U78tEqGHxkaf8h86bEMTSY2lcaDaaUuV140i3o3b + EilZ+WTWt39Yw7kwLnpmRz7Zqt/W78zU4PAStCWh0PRiXLeWB+HxyMjmvT9bA9ui089/YyWNcSJi + Gz9gylOY18/eon8+dg18UrMmhnR/CF6LpYP2BGyWOFvJH/GVOWjqR8o77kfypBdAGmWFbcP+HY18 + bgLcj0VBf3lKe0vYBl1L88qc/WtohVUsK233wQYz0/Dol4qdxlC844BN+YE/7HbvA5KrJWekuD/R + YEY7iug5zNgvXxliZWfA7ToG7GTlVsKDjRwgfaOlf/V7j/zAg6PgNfGn+Yrg4zUQHVaM8nNcJ01T + MnO5NkyTkMd5Hv3WH1UpzYiPHlhQg740PfKCA7H2OUPDRb9USGVqxU4PXY/4iacZ1Cf/wbbETX2e + R9npbz8FtrWO+Hbk8HseqfA78XszHUwgeVeR/S6WUS/Jw0UHnx9/eiliX8fIgWW2zraPtduKVRHb + mhqMOtvaKLFE67eVVmJnx3xzWSd//e6UrxFsPCVruJz9E5r0H5n0mEUn/4+adTkjgcZPYvIvN0TD + wGSbWWIVi3swZCAXjwszzetb/PTBcneKLn/5dCg1t162C/PB7GWVRsMNDwOcKntFErHJC767IRnJ + T+fIts7IxHh4shrNo51J24kvxiv3OIqjl8pMExxLyQSRoazXlLjn2UrI98HMdKQteuLFWeGLMAlL + SJcnm/iL9RX10jXBsFKzI1tRVbN4nl5uEGlCEB9KUnDvYuz0JxkHdmrmUdR046qE1F7dKFUaq6gH + daj/+lNbWw7iu259D5WyfKQzWhYWNzfaRpMPr4GsY28u+OoKw3LKG+jiENYtOzZJh9Da0fDzcX4m + YxJEBuSRZOHhIBetmB2HAL3fZIGXKTkgvo2PHCb/yay8qSKafg4pOgYRUElfVNE4+oaNwB+OE3+X + aNKfEvrpMawOZdJcK5jBeVsLKvizjpozCkpQ+k9INettJAN4qQYdg4KZS+2F+srd51pVsRkxPcZ9 + Nl+EgLa+nVGujWpEY+uJIWmjI17QT4aG5SetfniLFffqF+Mv73kU5YFFtmNZ465aNfD9tJ+/eDwG + 4ysA+OgSnr3JpRjOy8oB7XssySZFHvrxLxTuZTHpf9kSv/o9Bntglu0UVtuNqoyudOHQzPtcWx4g + 42++i+tk0fjNx0YchnliT/rHQsqnHXaw11SP7R7zRHDko9tyI9MHm/IFX57WEzY+v5B1H3fJ2IQ3 + Bx19oyBXa3FLWL655cg0jhzzKQ/TTvE8hglviJN/P1Zf7bYlLBmmZHOcHyPhcJtCNNfXLBiqhRj7 + pA9+ePLzi0j01tXQ1qKU/uJr2+ChAv4yK7Lxd5EY2uNNhbwybWJZhdkqysULYLgsVmR1vJ0TsV7s + KRy4PKfSRyqscT7vciSvXyuGaVn4oth71Y9viZ1tlgWf35cmOuhth4cV7f2WDDsMEH04s4I5KYb+ + pgNsX01J7K4nf/0teN6t/ukrMdbNdYBJ/9FZW4cFX9bIQfJ+dybXZHvwB57Jmfb2gzNzdelj/fJ3 + iOf7EA/qti74S/ZtcIN0x0xp//KFHm42cMWMEuu6ehffK/cGNOUnzNN3bjuYX/mG1ndzYGt9iyNR + 65eb5p/6Jwvmq0w0ww40wNVCZWsTVeJ7aEsKXlp9mXe1rKTF3buDCX/Z1lpo0aCFLw2605VgpcpI + Mm6ebYniqFDxOO5ZNBblHkO9PLhUpPsxoVfOTV2XjYLY61L2KWtYCSu8qIl31Zaoi66aBIlZ7Nkm + OEntKOgmVtMt7qgePy0kX1FjoynvJbeJ74ZgWM5QuGQjtp/mRfzyQ0BvfKDzyY/2S4/WIJ+ZSZyr + USV8eh46Z4FC8JSHDGXxBVBia0nwqfethXSNAmgU9Uw16aFHvTtQDMY0U+lMOqTMP5cTgPO2iKWY + dTHu6NhBvRpTkoadY42Z88gA7nAi1vGYiP6YrTe/PIddzyfH4qz5lCiW0oys1GFtjZJNbXRFiwX7 + +a/35XvrQIv1nM6mPEig461Dk39iay28iO8vr0mXsc0isr21Q3q3VJAV6cbcrtu1Qj+sqT7nD453 + tpsn49jn0l88GMVMawWNRw9mG1wScloJny+kQwNKew3o4xEYYlE4kqn/zhdMsjgUX2unzbSfnp6t + 33EhTruvCaMl3Cn/OqHRgeIA0du7THrvVvCwSm5oeCsDlXhRFo2xDyhK5u2Z8olPBPnUm1/+PJ0/ + mYirmW/ALnk0U72QhCJf3OCsk/d0fnVN/vqLCU+wSkTsd26+OkHVLncsHaoFEvcSKHr2zCW/PIbn + M+RpZb832LqxjVZM9YoU0sp0PvHBd0i9Ds7ug0/nOVU7lv1qhyKXrJjZXE/WuJjXKciBGv/yt3a8 + +IsczsvBJ4/vwyzkvHFMdDLnF6yHBrW6SLxzcLIO0c7Uy3bxUK0Y7HtXM3PS4xz4poYgMLdsHdlP + iwf8kqJzfH+R7WP9KbrbOZN/5xfEns4vOi15ZnA6H29sc7l4RaepUgfvYpYSL1z10XBLqwskWVoz + X77QpCPz0IaDUVQ4yY3AV4QTlMB2WkiVq50X46rXmh+eMns+HyOKvpcKJrxl6+XJjJT1bMV/+TxV + vV60/FoBwHg7emT9PwAAAP//7N3LjqpIGADgfT/FSW97Og2oVDE7Ba+gVXIRJZlMENEWRRGoAio5 + 7z4puzOZ5OzOcvJvWUCgCP/tCzVLb1bL7EeO4+3N4c/+vXJWLjk+TEPxXe8N2WdfH4mNRUTvOhfq + vmiWeDxeBNz5cHXxzOfx5DE68tWiOe+7XrYzcYJOEz5vtZXfVf4lRrLfxIzxYyS0uane8ZZoE0rd + 1BNtmepLnCh2x0f+4eR0ToDW+oVuCvl8V06vf9I9nA0YIfqH4YnGTOYNHj6OOZXvl6gUbTRH84+N + zZ/1JXvG55BVNTeXO98RSdN2hqGZGbV0NchEhN0Qn5OU8yXJkrLZOVPP0JNQpVZ4b3w+7ZYVkvNJ + 0pP9PXF+GGtkqwGjMn/YV7Mo8tBqaNTkLuNVx2olx6jW3ridaHzfhJOS4KnuUb4ILTcT+5quUW87 + srm7Hp6y+tS/K/gaRzM+nlxCq7d7DAiS600nXVwJMSmSB/5U9eprniuUKtJw4ftXot+6S9ac8ihB + 9MxyOjf6mV8N8KjCMj9k18VbWlZjbf1mBPVkSRfXvp+1tJwRTOkNy/jERXVwiYKe61nI+FNdtpc+ + 3u4VhVr9+WIvassdGrPArYkYG1HGN5v8goo8JgSNLJGJxXDbx6EdlKRj96GjJd41xGU3Wsn4K4Qw + vSjB8vxMNQZmqTqOj3HkeAGf2dOe1crvDXZXnUWf8x5ZryUI0xVnSHxEFj/63dQ4F4eUNYHQRRO6 + Jw8ph1QhRuJR1H3WvokOcaWyj+NDoPas7jU8o+hB1MA+onbh5+cBY7ngM9MvLZZ4dYC91dT5up+W + BMUZmT3tIPtBbsnzeDfHMl8mWOYbhU1Ob/j1qQJ+/vEbokAFUQCiAEQBiAIQBSAKQBSAKABRAKIA + RAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQBiAIQBSAKQBSAKPhFFGggCkAUgCgA + UQCiAEQBiAIQBSAKQBSAKABRAKIARAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQB + iIL/uSh4+fHjr+cuCPn9kF4lDKjTtn7/lwq8x4f4XVG0r60SWBWf0tc/vwXCa1He86L+u75f0lsl + qcH33wte63sdX/97/EVe6ufLPwAAAP//AwArUdmvhGEAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a16b98cc9421f-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 23:07:15 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '108' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999979' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_66f760af9efd9cf406dba26638668e57 + status: + code: 200 + message: OK +- request: + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": + "2330430886409134037", "trace_id": "675f612100000000580c60cf6c651750", "label": + "ragas_answer_relevancy", "metric_type": "score", "timestamp_ms": 1734304035091, + "score_value": 0.9999999999999996, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.15.0.dev406+ge04c8acf7.d20241210", + "ml_app:unnamed-ml-app"], "metadata": {"_dd.evaluation_span": {"span_id": "10832341952143904572", + "trace_id": "675f61210000000082bd06f301825b86"}}}]}}}' + headers: + Content-Type: + - application/json + DD-API-KEY: + - XXXXXX + method: POST + uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + response: + body: + string: '{"data":{"id":"a37783cd-c126-4356-af92-6352c20b6b6e","type":"evaluation_metric","attributes":{"metrics":[{"id":"W-lQbvZhh1","join_on":{"span":{"trace_id":"675f612100000000580c60cf6c651750","span_id":"2330430886409134037"}},"trace_id":"675f612100000000580c60cf6c651750","span_id":"2330430886409134037","timestamp_ms":1734304035091,"ml_app":"unnamed-ml-app","metric_type":"score","label":"ragas_answer_relevancy","score_value":0.9999999999999996,"tags":["ddtrace.version:2.15.0.dev406+ge04c8acf7.d20241210","ml_app:unnamed-ml-app"]}]}}}' + headers: + content-length: + - '534' + content-security-policy: + - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com + content-type: + - application/vnd.api+json + date: + - Sun, 15 Dec 2024 23:07:15 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + status: + code: 202 + message: Accepted +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml new file mode 100644 index 00000000000..eb6e4de989c --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml @@ -0,0 +1,549 @@ +interactions: +- request: + body: '{"messages": [{"content": "Generate a question for the given answer and + Identify if answer is noncommittal. Give noncommittal as 1 if the answer is + noncommittal and 0 if the answer is committal. A noncommittal answer is one + that is evasive, vague, or ambiguous. For example, \"I don''t know\" or \"I''m + not sure\" are noncommittal answers\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"type\": \"object\", + \"properties\": {\"question\": {\"title\": \"Question\", \"type\": \"string\"}, + \"noncommittal\": {\"title\": \"Noncommittal\", \"type\": \"integer\"}}, \"required\": + [\"question\", \"noncommittal\"]}\n```\n\nDo not return any preamble or explanations, + return only a pure JSON string surrounded by triple backticks (```).\n\nExamples:\n\nanswer: + \"Albert Einstein was born in Germany.\"\ncontext: \"Albert Einstein was a German-born + theoretical physicist who is widely held to be one of the greatest and most + influential scientists of all time\"\noutput: ```{\"question\": \"Where was + Albert Einstein born?\", \"noncommittal\": 0}```\n\nanswer: \"It can change + its skin color based on the temperature of its environment.\"\ncontext: \"A + recent scientific study has discovered a new species of frog in the Amazon rainforest + that has the unique ability to change its skin color based on the temperature + of its environment.\"\noutput: ```{\"question\": \"What unique ability does + the newly discovered species of frog have?\", \"noncommittal\": 0}```\n\nanswer: + \"Everest\"\ncontext: \"The tallest mountain on Earth, measured from sea level, + is a renowned peak located in the Himalayas.\"\noutput: ```{\"question\": \"What + is the tallest mountain on Earth?\", \"noncommittal\": 0}```\n\nanswer: \"I + don''t know about the groundbreaking feature of the smartphone invented in + 2023 as am unaware of information beyond 2022. \"\ncontext: \"In 2023, a groundbreaking + invention was announced: a smartphone with a battery life of one month, revolutionizing + the way people use mobile technology.\"\noutput: ```{\"question\": \"What was + the groundbreaking feature of the smartphone invented in 2023?\", \"noncommittal\": + 1}```\n\nYour actual task:\n\nanswer: \"The capital of France is Paris\"\ncontext: + \"The capital of France is Paris.\"\noutput: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 3, "stream": false, "temperature": 0.3}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '2795' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//7FTBitswEL37K8Sc42I7qTfry1IoXSh0KRTaQ1NsRR7b2sqSKo1pl5B/ + L3Kc2GFb6H178eG9eeM3TxodIsZA1lAwEB0n0VsVv0H303zusk8fHz/kD8lth2/X9/dPlL+vH25h + FRRm/4iCzqpXwvRWIUmjT7RwyAlD1/RmvVkn6+3r7Uj0pkYVZK2leGPiXmoZZ0m2iZObON1O6s5I + gR4K9jVijLHD+A0+dY2/oGDJ6oz06D1vEYpLEWPgjAoIcO+lJ64JVjMpjCbUo/Wqqg47+DGgD853 + ULAdfOk4MekZdcgEt5K4YqZh7xzXAu92sGI70EYL0/eSiKugSo5VVS3/4bAZPA9z6kGpCT9eTCvT + Wmf2fuIveCO19F3pkHujg0FPxkK0ED9LIv2fxJRE9vKSiBj7Ni7McDUvWGd6SyWZ76hDwzybFgbm + PV2w+USSIa4W+PZMXPUrayQulV/EC4KLDutZOu8nH2ppFsTyCJ+7+VPv0+RSt//SfiaEQEtYl9Zh + LcX1xHOZw/CM/a3skvJoGPyTJ+zLRuoWnXXydGEaW+aNSBNME9xDdIx+AwAA//8DAGrumFFSBQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a126b0fcf8c5d-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 23:04:18 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=yoaDgBqIVg7q_qWICuV4sfI2HC68mSEiEJeGmauKwk8-1734303858-1.0.1.1-MYIlBP7O1Wu.r.hazt7JZrz7sO8oLIzVkD7otw3Ls8ovEJ1u4QDKgiMrWj.vAkY18tkpfJ9M3g0hUmnbiq2u6Q; + path=/; expires=Sun, 15-Dec-24 23:34:18 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=A1kfyxL1kDcyq.Uw7XGdceC3nZTqjxFlKdeMyNQICfw-1734303858928-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '410' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999320' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_0d1ac5d511c5de88fe61226b65f98608 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '114' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//VHrZrrO6tub9foqldZvaCl3wYN0RIJDQ2EkgXalUApIQIITQ2ICPzruX + yH90qupmSpPpCQYPf93wf/zrr7/+rpPikfZ///PX3++86//+H/O1e9zHf//z1//8119//fXXf/x+ + /n8jH1XyuN/zT/Yb/vtj/rk/xr//+Uv47yv/d9B87+6zJbpxCJLx+o2pOpkOYUH0LNBk1o4C/EPX + zCKD7LEy0ErVfisBC+vwE07VfhBAEpsbW5fXHA1eeYkhUNIrFSAf+SRpLgUiKB/iKv0n4aqpVsjf + X31cNxe5qO+01EFbPTUsyN+3N5GjN6D16BnM5de0mO6Dm0N07V2yFb+rsF9LDYXr2OsMF8dXMgXx + I0KZVz+YvgtyxHvaCmg7vvx5PgYX7wfJgtuzPrMtzpZtfz9INmiLlhCbkqXXpePKQJHUvOjyEjlF + V9QsAzXYbsjNjwc0qqsCoHWtKzt7Cmun4/F7hOa7GzBgYifD6lFmaCnUHMuuffG4dRFyUA/5luBs + p3g0iM9nsAN3Szarcx5yfNYNjRriAy8X93XYi2fbh/GZp8S6bV5oyFshAyizGA/pcWinb2grsOBu + g8N7+eYtOSIdFtniy1wUH1uanNQILcRzi6WdV4ZUFOJazU9OSjZv8dzyRfGW4LAtR4Ld9mTyFTJS + CKvbSFz5Y7fj45RV0Oi3jnhHrzFHqF8RqIrtYeotvkV9P9gSupdHl5k91N6gJpdcbWT1Qhvn2oXd + /Y0WiBTLJZ6G9yWk7+W1WS2FhhNXT2WTB2yimqtmOpXxae3Jir/ZwvRtjxShB0ZsNzUYOY/6QA57 + XS+kPcsAbHVPmFf7QSjnyzxGhzJymMVi5E0w6Y2GNpZKF9wpwoGuvSNKP67JvOulDTuqhFsoyzoi + gdJtzaGIVwrEYp4zc3mN0JB8twD+OGlksy0/qEvvngJH+gDiraKC88vtYKMaPbbEqVdlMfhuSWE0 + R4euRveU8AomRcv1PiZ2j2rek/uhBNkSd8y4xjuT7+6aBWhVRVSTq08yiK9Ih+zdAp76k1cMG1/q + UH9iF0IW91f4mw86lGeHSjfdTjo1+Kqgan2FheOuS8aD/BrA0C8TsfBKL+STMwDsJKNi5lEsE7b8 + XCO4KdeEkZX6Mrvs0EXgHSyLRArboEnY3CMQjo1DNt3YeU1E0j3aLJIXI7utXQxv0bWQlWQfsjPF + OCy8U54v8ebpUKXUdCQt9ERA4jZWyfqVBiEjR2SokV3sSPBGQ8troY7A8+maGZdom4wp2p812ccp + XiYOb6d6TSf4hoc3sdcdbSd6cx7AtE1LPGO1DScUtSk0X2+gk3B4mf1rfV2AcE8F4rbvfTGQZWD/ + 3p8unGQXsk+hY00mX4noOCzCbru+uOBqy5joclWbI4SAoRG7MzkUx9xktr1SoIlGSoe2nMyONE9Q + pWMx4Fe6CrxBOQkPOASDSfZHqSi4cbkYyNlLIXEvAQ0rz/cwENmu2XoUSME/F28LFy14kyB6mpzb + 9qiqgfK4EuOza8Jp4qIL5co44zrzzu10dnYdylRzgReLHoXtChkPyIRrwbbvg2zWH3I8a3WpbIit + OI3JHbqkq9JTCDN2Wydkab2mQBV3S9yj03Nq226jLh7Th/2+53CQphz6V9qzu6f24VRKWQWXg+6w + eyoO3uhHvq6uSzckzu1joMncvLfQXT974ilrkgx32hmIOWeFWVG+LqSBxjbgIQnoUlp47fi6SDUy + ff9J/N3ujbhZyy4Kut4g1oV0fOKx+0DLh82Ic7oqfKz1ZQbNbakzffO6edNee1QQ5ccnW9vKjtPM + CQb0EaaeBXLlhILY+JYa7oczSZfKJfksirfwwycqq1st/EhfSwKmOS0hZHMvBnAjFaDf2czzSNRK + 1aNK4aPdbmTD3bUpPdY5RbLvp8wn0sHk7VcftHk+RHc/q3aYFiFV3917y9yClMmwaRIVnoQPmHc2 + T7j/MRrgRnJh62d1Tb6YRhhECho76M+I9/LGLJGaRiLZnO5RMuCrLEBi9UeK3lzx+stFrVBnBZhq + XwpFdcoghqC0z3hYfJ98NJq+gS+sv7jV5NKb3strjWa8J04y9py+JNMG+5B77IdHFFSFgr7yA7Y2 + TruCr6b4urqFm5HYD9vmHXdMCZ3F/ciscPnyhrm+UaFqH7ZT96cZ73RAw1sciCs87gk/rIcrSj9b + k0T4nCWj/jaqP/9PxK2E2DeUXFhVpcGc74C8viSlBa6a68RM6J7zzNEMmPmLOPujEvIBJIA0iXti + 9Ke2mLZZLKjk5h0YXn1iND2H2gAOlkn27udW9I/i4Su89RIqrutjMjzv3+tKvgUeFl4JNUdIuz2o + 6tGn5WuBQ/ZMmy1cJeVIwguy+Mj5RgUaTjad91MymTVRUPpajJjvfLMQ45fu/sazcE9QMkyLhKLG + GZZsrn/eupZ7RdkhVPH06dx27If4rNb3fMQqt+xkavA60ub9SRyry8LRknUMNI0nsnl+xYQFTO1A + pAuNmNOzThg5ch1Ilr2Zp3pxMl2uXwFSGB901VkRmiauuahH+npe34DzrR8BPCZuEP1sk5BL5LhF + 2n3/YHh4qWE/pRlGM38y3RaCdpAr2UK56AtYVAMFjeFhMrRSEk7MKi9mMf34shLTN11u31UynrTm + DL2rfpn5ViePumqrg7FyJPolDnhj8C63f9b/oLqI9+3ZdaFf3BO2XoWbkFebwoCcpDo53rNv24mL + FVb3TOqYM+WbZLSCHlSIlxsWxGnMp2mpYvUlKh0ehmbiPVv0D3WjgsMeb3JtKSwXR3jeThtyn7KK + d26wosg/VgHT5/HszL0JxLhExGu/PueO2Smo8CuH4OZ0M6dL9UzRY3ivmb9uBpN/cGhpfbO5YTFj + Z3MScjWH0+JzYpuMnb1Jf68EzSf3gjh5XyR9vvAUeC/GHfvxfY+9Mf3h1YxPK5PJb1kAij45Xo3q + t52aobQRW9UHQpaRzAdLugLSbUXASEK4lQUMDQKJJnhp+wez++EnJVcTy1hZh9KKbGP4gKsTcsSO + Kf/u770fNh7TVe/xfVLYsDKfNwop5u1EHroExoVtKD9v9iab8R8l7eFE7CoNTaZEGl7t8EvFK+Xr + /dFTIA1P5w9/S8lJPaOZj5m9rS1zeGcRILWgO6rs4qXZV3cZQxcmC7a5hDuTLvJFicTT7UuC0zlO + Pm22p7DGcs223p4jrrFGhYvfWMzSRDuZVDXEMKjBRGXBsxP50yp7uFHZpi9aFmhIKM7U+9HeERwV + uicZgrUAfd3nxLGlNmQP47aAxdYvf/jd9ql+WgCTH/uZL92EXZsRw6wP2c1dJ+3otFmjbcqrRe5m + uTal28NX0EGhO+ZfFMcca13OQH0rLoZP/zU5030D+lN/YeubvTZpKWUlSHjXEyswW8SHOMWAEVHJ + ZpEaaGLRWoVf/bvTqzL5+3nvED7eJWK6puoNPz37tfcSwbP+HXYPOINyp29aKMsnp/W6moCRZYmX + r1BAfYr2kYZ9KSaOaKSIB8mmAuGj71g8vWxz8rIh0vJDY7CACndzPORQq0bUC2znrlHbhbzPVFne + 5mTWp5wHSj7B7n58sWAX5N6UKHf7974UuV0UDuJixNrT3Z6w5CeDN54MBmi63A8Mgz5406rmtqZq + rGK+fxuLqeiVDO08O6Pw48OorykMeXzGIu/XicSWUQdK00dsTcuCD9L+cIXlp7EoD69WMUJa7tFc + H2R7ZN+ip25eak79ZVT+phbiFagKOg7Ngo5VHqDpcn0JCJYXh/gR0/gYU2341Reu3wfZozLOVdiZ + LCDuRTPanz5W8+/zwezzZ/Qmezw9YC1FEbPc5tyy9yGz0byfmLc/nRK27qwt4ONTornztMPuJHg6 + qhcoJ34TC+1EKkWBH99t74/2t74UpM6xKMz6hQndQVJhLHbE0h9iO5mHHIPiZSXTX9eDOfNxA2q8 + FYgnYDP8/Y78UKwINv3nzJduqdJwsJmzfXYJ1zeSivhuaIg/rpqQX/K1q2U5zolfrpftuAtp/sMX + ekHJAY3X77GD7Pz0cTnyIZwOWJpg+aktpr/RIemfH72Dc9unhEiLtp31paLloWTiIXqW7QCfydbS + nd+xX/2OZzt+IHooN8yU3zUfrCsf4Fm/RIZP2hmxcKqOsD8frlj43i1TMJ2wgeftsmE7Y1GgrxX0 + C2B+7jLruPPDXtt/MND3wqFd/qmL141khvbJq44Q4aAnglkqEmCxehL3UAbtdI7kCMb7OfnlBXxY + u3Wtuu61xqUtNy2PV1YEnpMxLF5vXz5Z+ITBh0/DnPJcJqO88Uo01xvxEsHlo5WHNjj+Y08nxeo9 + NurfHCB8T8SsQyfpfnrNmQKL7ZZG5nFLvRyh7VuXWWGq8/4pSBHYNF/hp+0fvBHL/hENnnQkwaHA + 4fRyhz3MfoYWO78ouETiLVjSo5/Xf5z9wqj88BJrxqFrR3F7LQE9GozlslHbbuMvOthtlSeWFMc1 + 6fcDBhrsVickDaJwrJvDAEUPyq8+Qi5m6aCS1M2ZM04Hr391xQSvy/JEdP9Oi++sv1Tn9lgS38ks + JCEPXSE9DmsS//x5VMRbgPvizHzN67ypdYUzCMmR44FfFt60EfIHjEdNZtah8r2xFFdn9PMz9oc5 + xQAf1Qa9T2WslFrGx18e0kxswyz5u/Fo+sivSD87JbPrY+3RH3/Nfnfeb2LCtUDfanP+Q4ddYCCh + MdUJqHreUfkVUzR1465EclY+mfntH+ZwKfSrllmhR3bKt/U6I9UneHHakoCrWjFuWtOF4HRiZPs+ + XMyB7dD557+xnEY44ZGFHzDnKcztF2/ePx/7Bj6pURNduD/4VPOVjQ4ELJbYO8Eb8Y3ZaN6PdOom + L5RmvQDCKMlsF/TvcJyWBsD9VBT0l6e0ccK26FYaN2YfXkPLzWJVqfsP1pmRBievlK00guId+WzO + D7xhv38fkVStJkaK+xMNRriniF6CjP3ylSGS9zrEt9FnZzM3k8nfSj7Stmr6R7/3yPNdOPGpJt48 + X+5/3AbC45rR6RLVSdOUzFhtdMMg5HFZhr/vj6qUZsRDD8ypTl+qFrr+kZiHnKHhql0rpDClYueH + poXTeUozqM/eg+2Ik3pTHmbnP/vJt8xNOO3GCX7PIxV+J15vpIMBJO8qcthHEuoFabhq4E2nn14K + 2dfWc2CZpbHdY+O0fF1Elqr4o8Z2FkpM3nptpZbY3jPPWNXJH78752sE60/BHK4X74xm/UdmPWbS + 2f+jZlMuiK9OZz77lxjRwDfYdpGYhXj3hwyk4nFlhnF7858+WO3P4fUPnw6l6tSrVjQezFpVaTjE + eBjgXFlrkvBtXkz7GElIetontrNHxsfjk9VoGe4N2s58Md4md0JR+FKYYYBtyhknEpT1hhLnslhz + 6T4YmYZUsSdulBUeD5KghHR1tognbm6oF24JhrWSndiaKqo55ek1hlDlnHhQkmJyr/pee5JxYOdm + GYZNN65LSK11TKncmEU9KEP9x59a6mrg303ruaiUpBNd0LIwJ2OrblXp+BrIJnKXfFrfYFjNeQMV + j0HdslOTdAhtbBU/H5dnMiZ+qEMeCiYejlLR8sVp8NH7TUS8SskRTbvoNMHsP5mZN1VI088xRSc/ + BCpoYhWOo6dbCLzhNPN3iWb9KaCfHsPKUCbNrYIFXHY1p3x61mFzQX4Jcv8JqGq+9WQAN1WhY1Aw + Y6W+UF85h1ytKrYghssmjy3FANDOszI6qaMS0sh8Ykja8IRF+snQsPqk1Q9vsezcvGL85T2Pojyy + 0LJNc9xX6wa+n/bzB49Hf3z5AB9NwIs3uRbDZVXZoH5PJdmmyEU//oXCuYqz/pdM/qvfk38AZlp2 + YbbdqEjoRkWbZu7n1k4+0v/ku7hOxMZrPhaaYFgm1qx/TCR/2mEPB1Vx2f6xTPiEPBSvthJ9sDlf + 8KT5e8LWm65k00ddMjZBbKOTpxfkZopxwvJtnCNDP014mvMw9RwtI5jxhtj592P21X5XwophSran + 5Snk9mRRCJfahvlDJfKxT3r/hyc/v4h4b950dcNL4Q++tg0eKpheRkW23j7kQ3uKFcgrwyKmWRit + LF9dH4aruCbrU3xJ+EY8UDhO0pIKH6Ewx+Wyy5G0ea0ZpmXh8eLgVj++JVa2XRXT8r4y0FFrOzys + ae+1ZNhjgPAzMdNfkmLoYw1g92pKYnU9+eNvwXXj+qev+Fg3twFm/UcXbR0U06pGNpIO+wu5Jbuj + N0yZlKlvz78wRxM+5i9/h2h5CPCg7OpiekmeBY6f7pkhHF4e14LtFm6YUWLe1u/ie5vcAc35CXO1 + vdMOxleK0eZuDGyj7XDIa+0aq965fzJ/uc54M+xBBVyJCtsYqOLfY1tScNPqy9ybaSYt7t4dzPjL + dqaohoMavFTozjeC5Sojybh9tiWKwkLB43hg4ViUBwz16uhQnh7GhN6mydA0SS+ItSklj7KGlbDG + Yk3cm7pCXXhTBUiM4sC2/lloR063kZLucEe16Gki6YYaC815L4lnvhv8YbVAwYqN2HoaV/7LDwG9 + 8ZEuZz/ar1xag3RhBrFvepVM8/PQJfNlguc8ZCiLL4AcmSuCz71nisIt9KGRlQtVhYcW9s5AMejz + TIUL6ZC8/FzPAPbbJKZs1MW4p2MH9XpMSRp0tjlm9iMDuMOZmKdTwvtTttn+8hx2u5xtc2LNp0SR + kGZkrQwbcxQsaqEbEkX281/v6zfuQI20nC7mPIijU9yh2T+xjRpc+feX16SryGIh2cXtkN5NBSRZ + iJnTdfuWa8cN1ZbTY8J7y8mTcexz4Q8ejHyhtpxGowuLLS4JOa+5N4nCsQG5vfn08fB1Lha2YGi/ + /oJBxGPxNffqQv3p6cXmHRX8vP8aMJrcmfOvMxptKI4Qvt3rrPfiYgqqJEbDWx6oMBVl0egHn6Jk + 2V7oNPMJJ596+8uf5/6TgSYl83TYJ49mrheSUOTxGC4aec/9q1vyx1/MeIIVwiOvc/L1Gap2tWfp + UImI30ug6Nkzh/zymClfIFct+4PONo2lt3yuVySTVqLLmQ++Q+p2cHEe09zPqdqx7Nd7FDpkzYzm + djZHcVmnIPlK9Mvf2vHqiTlcVoNHHt+HUUh5YxvobCyvWAt0anYhf+dgZx2inaGVrfhQzAise1cz + Y9bjE0zbGnzf2LFNaD3NyZ+uKbpE9xfZPTafoosvmfTrXxBr7l90avLM4Hw5xWx7vbpFpypCB+9i + kRI3WPfhEKfVFZIsrZknXWnSkWVgwVEvKpzkuu/J3PZLYHs1oPLNyotx3avND0+ZtVyOIUXfawUz + 3rLN6myE8maxnn75PFXcnrfTrQKAMT65ZOM8PuZI3aaC+PLx2C+/F3KhrOBun/l/+T2dvhR1zU8m + 5vJ7y8XkO/hgWbuIecuDyn96HjbN+smC3ZAnk1xcDUhRtmHbUQrCqQvLGM15E9WsZs2lrSHWcMHS + hpDD48jH9qH6kAruxNbhPfMmL0J7tSSn7/x9A09WMvUIxYpirC61Ix+MdDuA3jwrMtcX7wRpvUXb + 5cllP39Jf/x8pl3PDP8aejwdxknTJKMgpipGBb/B4Qx5+mDMx0XaDlfPPmpqehaJea6HkNmT36G5 + P4nlOd/jeaPtkStGlMz6Iemc2+2IAl3rcT3z1UR7oQLUSwvmphJLhvOmxWCrR8J2Z/NQ8KQneyRf + 1i477PWs6DOlFuAd3xxmbcqzKV+bFUbzepPNFHecb75pAy9R7f70c7nQ3ST4huEbq5+pLIasuqWI + 5LQiW00pwm4F6w5mfUjfu8Wj7Sxpv9CifuOT3VsJi5G0DgZCPjDzE+Pd/YAF9FvP78w/XXkpFbgk + gkBMZbtLeG8edM2JDj3mlnYr2OlUlehbxRijtckLvtMvCpzdqMUTrXVPSo/vM7TTOpj5l3NuHG8p + zPenorYyWtHzQoCbd4yY49qyOc54A4dgMsmv3zP7tRQBCRhFfHkz2TOcbC3/3h90iLjKh/MhOyLh + /hCwlh4Jml59aKB73Il0+Ww4GnMxkcAhqMFi5D7RuAurfEVpxZljhK1J02MfwTGwvT/vM+LomyND + lu5zHnRoWRVftzDrZQyz3vi6OFvA379TAf/5r7/++l+/EwZVfX+854MB/WPs//3fRwX+Hd/jfwuC + 9OcYAu3i7PH3P/91AuHvb1tX3/5/93X5+HR///MX+nPU4O++7uP3/3P5X/OD/vNf/wcAAP//AwCJ + O66R3iAAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a126fde6180df-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 23:04:19 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=Gul7c8fExIU8FqFmLe7ibG6d2ui6aihEjGEbftB2HDo-1734303859-1.0.1.1-8CGeWlxDim_kuzwaWnQVnAJP0Er9WUj.qOwVjfGiTHyf0WAnOi3d7H8Wq9KweUcQneHQdvc56_9L2cKSSwpnhg; + path=/; expires=Sun, 15-Dec-24 23:34:19 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=jRamhH7v5XicVPgpi9Yoew5VMsO.2qrYeZTYALGr_4s-1734303859489-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '97' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999993' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_681a8e6776d55fd49c1653dd98e86efb + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30], [3923, 374, 279, 6864, + 315, 9822, 30], [3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '192' + content-type: + - application/json + cookie: + - __cf_bm=Gul7c8fExIU8FqFmLe7ibG6d2ui6aihEjGEbftB2HDo-1734303859-1.0.1.1-8CGeWlxDim_kuzwaWnQVnAJP0Er9WUj.qOwVjfGiTHyf0WAnOi3d7H8Wq9KweUcQneHQdvc56_9L2cKSSwpnhg; + _cfuvid=jRamhH7v5XicVPgpi9Yoew5VMsO.2qrYeZTYALGr_4s-1734303859489-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1RZyxKqvJaen6fY9U/tKhGQLM4MAUG5JCp465GgIiAilwSS8/JduKu6qydWiRFC + stZ3y3/+9efPP3VSPNL+n3//+eedd/0//zVdu9/62z///vPf//rz58+f//w+/9/IR5U87vf8k/2G + /37MP/fH+M+//0j/e+X/Bk337j4bYpj7MBkv3xvVuOUSFsbPAnGrdlUQH7piNhkUn5WhXmrOWw1Z + VEefiFe7QQJ50VzZqrzkaPDL8w1CNb1QCfJRcFn3KBBJ/RBP7T+J0CytQsHuEuC6OStFfaelAfry + qWNJ+b59Tg7+gFajbzJPXNKC3wcvh/jSe2Sz+C6jfiU3FC5jbzBcHF4JD2+PGGV+/WDGNsyR6Gkr + oc34Cqb5mGJx38s2XJ/1iW1wNm/7+152QJ+1hDiUzP0uHZcmiuXmRefn2C26omYZaOFmTa7BbUCj + tiwAWs++sJOvspYfDt8DNN/tgAETJxmWjzJDc6kWWPGcsy/ss5SDts83BGdb1afh7XQCJ/Q2ZL08 + 5ZHAJ8PUqbl44Pnsvor6xckJYHzmKbGv6xca8lbKAMrshof0MLT8GzkqzITX4OhevkVLDsiAWTb7 + Mg/dDi1NjlqMZotTi+WtX0Z0Id1qLT+6KVm/F6dWzIq3DPtNORLstUdLLJGZQlRdR+IpH6cdH8es + gsa4dsQ/+I01Qv2KQVMdH1N/9i3q+96R0b08eMzqofYHLTnnWqNoZ9q4ly7q7m80Q6SYzzEf3ueI + vueXZjmXGkE8I1UsETJOdU/LDKrg48pX1GC9Af5tDxShB0ZsyxuM3Ee9J/udYRTyjmUAjrYjzK+D + MFLyeX5D+zJ2mc1uyOfAjUZHa1ujM+EW0UBX/gGlH89i/uXcRh1Vow2UZR2TUO021lDclircFnnO + rPklRkPy3QAEI9fJelN+UJfefRUO9AHEX8aFEOfr3kE1emyIWy/LYgi8ksJojS5djt4xERVwVc+N + /kacHtWiJ/d9CYq92DLzcttaYnvXbUDLKqa6Un2SYfGKDcjeLWDeH/1iWAdyh/ojOxMyu7+i33zQ + vjy5VL4aTtJp4VcDTe8rLB22XTLuldcApnHmxMZLo1CO7gCwlc2KWYdFmbD55xLDVb0kjCy1l9Vl + +y4Gf2/bJFbZGnFpfY9BOjQuWXdj5zcxSXdoPUtejGw3TjG8F56N7CT7kK21uEWFf8zzOV4/XaqW + uoHkmZFIaLG5aWT1SsOIkQMytdgptiR8o6EVtVTH4Ad0xcxzvEnGFO1OuhLgFM8TV7S8XlEO32j/ + Js6qoy2nV/cBTF+3xDeXm4ijuE2h+foD5dL+ZfWv1WUG0j2ViNe+d8VA5qHze386c5NtxD6FgXWF + fGVi4KiIus3q7IGnz2/EUKraGiECDM2iO5F9ccgt5jhLFZp4pHRoS251pHmCJh+KAb/SZegP6lF6 + wD4cLLI7yEUhzPPZRO5Ojoh3DmlU+YGPgShOzVajRArxOfsbOOvhm4Tx0xLCcUZNC9XHhZifbRNx + LhYelEvzhOvMP7X85G47lGnWDM9mPYraJTIfkEmXgm3ee8WqP+Rw0utSXRNHdRtLuHROl6WvEmZu + N27E0npFgarehngHtxfUcbxGmz34h/3Wc9jLPIf+lfbs7mt9xEs5q+C8N1x2TxeDPwZxYGir0ouI + e/2YiFvr9wa6y2dHfHVFkuFOOxMx96QyO85XhTzQmwN4SEI6l2d+O77Oco2sIHiSYLt9I2HViofC + rjeJfSad4OLmPdD84TDiHi+qGGtjnkFznRvMWL+uPt/pjwri/PBkK0fdCpq54YA+Eu9ZqFRuJC2a + wNai3XAi6Vw9J59Z8ZZ++EQVbaNHH/lry8B0tyWErO/FAF6sAfRbh/k+iVu5elQpfPTrlayFt7Lk + xyqnSAmClAVE3lui/RqDPs2HGN5n2Q58FlHt3b03zCtImQzrJtHgScSAReeIRAQfswFhJme2elaX + 5ItpjGFBQWd74xmLXllbJdLSeEHWx3ucDPiiSJDY/YGit1D9/nzWKtTZIab6l0JRHTO4QVg6JzzM + vk8xmk3fwBdWX9zqSunz9/xSownviZuMvaAv2XLA2ec+++ERBU2lYCyDkK3M47YQS367LK/ReiTO + w3FEJ1xLRqfFbmR2NH/5w1TfqND0D9tqu+OEdwag4b0YiCc97onYr4YLSj8bi8T4lCWj8Tarv/8n + i42M2DeSPVhWpcnc74D8viSlDZ6WG8RK6E6IzNVNmPiLuLuDGokBZIA0ufXE7I9twTfZTdLI1d8z + vPzcEH8OtQkCbIvsvM+16B/FI1BF6yd0saoPyfC8fy9L5Rr6WHol1Boh7XagaYeAlq8ZjtgzbTZw + kdUDic7IFqMQaw1oxB069VPCrZqoKH3NRiy2gVUsbi/D+41n0Y6gZOCzhKLGHeZsqn/RerZ3Qdk+ + 0jD/dF479sPtpNX3fMSasJ2EN3gV61N/Etfusmi0FQMDTW+crJ/fRcJCpnWwoDOdWPxZJ4wchAEk + y97M1/xbws+XrwQpjA+67OwYcS50D/XIWE37GwqxCWKABxcmMU4OiYRMDhuk33cPhoeXFvU8zTCa + +JMZjhS2g1IpNsoXgYQXWqiiMdpzUy9l6cjs8mwV/MeX1SJ90/nmXSXjUW9O0Hval1lvjfvU01oD + zKUr0y9xwR/Dd7n5u/97zUOib0+eB/3snrDVMlpHoloXJuQkNcjhnn3bbjFbYm3H5I65PF8nox32 + oMFtvmbhLb0Jzuca1l4LtcPD0HDRs1n/0NYauOzxJpeWwnx2gOf1uCZ3nlWi88IlRcGhCpkxjWcn + 4XNY3EpE/PYbCOFanYqKoHIJbo5Xi5+rZ4oew3vFglUzWOKDI1vvm/UVLzJ2sriUazkcZ58jW2fs + 5HPjvZT0gNwL4uZ9kfT5zFfhPRu37Mf3PfbH9IdXEz4tLaa8FQko+uR4OWrfljdD6SC2rPeEzGNF + DLZ8AWQ4qoSRjHCrSBgaBDJN8NwJ9lb3w09KLhZWsLqK5CXZ3OADnkHIAbuW8ru//344eEyXvS92 + SeHA0npeKaRYtJw8DBnMM1tTcVrvLDbhP0ra/ZE4VRpZTI11vNzil4aX6tf/q6dAHp7uX/6Wk6N2 + QhMfM2dT29bwzmJAWkG3VN3e5lZf3RUMXZTM2PocbS06y2clWhyvXxIeT7fk02Y7Cius1Gzj7wQS + Oms0OAeNzWx94SRc0yIMgxZyqki+kyifVt3BlSoOfdGyQENCcabdD86W4LgwfNmU7BkYqz4nriO3 + EXuY1xnMNkH5w++2T43jDJjy2E186SXs0owYJn3Irt4qaUe3zRp9XV5scrfKlSVfH4GK9irdsuCs + utZYG0oG2lv1MHz6ryWYEZjQH/szW12dlUVLOStBxtue2KHVIjHcUgwYEY2sZ6mJOItXGvzq3+Ov + yhLv571D+HCXieVZmj/89OzX2ckET/p32D7gBOqdvmmhzp+C1quKAyPzEs9fkYT6FO1iHQfyjbgL + M0UiTNYVSB9jy2785Vjcz4ZYz/eNyUIq3a1xn0OtmXEvsa23Qm0XiT7TFGWTk0mfChGqOYft/fBi + 4TbMfZ6od+f3vhR5XRwNi9mI9ae3OWI5SAZ/PJoMED/f9wyDMfh8WQtH13RWsSC4jgUvejVDW9/J + KPz4MO5rCkN+O+GF6FeJzOZxB2rTx2xFy0IM8m5/gfmnsamILnYxQlru0FQfZHNg36KnXl7qbv1l + VPmmNhIVaCo6DM2MjlUeIn6+vCQE87NLgpjpYrxRffjVF67fe8WnCs412FosJN5ZN9ufPtby7/PB + nNNn9LkzHh+wkuOY2V5zatl7nzlo6ifm747HhK06ewP48JRp7j6dqDtKvoHqGcpJ0NyklpNKVeHH + d5v7o/3tLwW5c20Kk35hUreXNRiLLbGNx6Ll1j7HoPpZyYzXZW9NfNyAdttIxJewFf2+oyBaVARb + wXPiS6/UaDQ4zN08u0QYa1lDYjs0JBiXTSTO+crTsxznJChX83bcRjT/4Qs9o2SPxsv30EF2ega4 + HMUQ8T2WOcw/tc2MN9on/fNjdHBq+5QQeda2k75U9TySLTzEz7Id4MMdPd0GHfvV73hybg9E9+Wa + Wcq7FoN9EQM869eC4aN+Qizi1QF2p/0FS9+7bUmWGzXwvJ7XbGvOCvS1w34GLMg9Zh+2QdTruw8G + +p65tMs/dfG6kszUP3nVESLtjUSySlUGvKiexNuXYctPsRLDeD8lv7xADCuvrjXPu9S4dJSmFbel + HYPvZgwvLtev4DY+Ygjg0zC3PJXJqKz9Ek31RvxE8sRo55EDbvDYUa7avc9G45sDRG9OrDpyk+6n + 11we2mw7NzNf2Nr5AG3fesyOUkP0T0mOwaH5Ej+dYO+PWAkOaPDlAwn3BY74yxt2MPkZWmyDohAy + uW3Alh/9tP/j5BdG9YeXWDf3XTsuNpcS0KPBWCkbre3WwayD7UZ9Yll1PYt+P2CiwWkNQtIwjsa6 + 2Q9Q9KD+6iMSiywdNJJ6OXNHvvf7V1dweJ3nR2IEd1p8J/2ludfHnARuZiMZ+egC6WFYkdvPn8fF + bQNwn51YoPudz1tPOoGUHAQexHnm87WUP2A86Aqz91Xgj+VieUI/P+N8mFsM8NEcMPpUwWqpZ2L8 + 5SENZ2tmK9+1T9NHfkHGyS2ZUx9qn/74a/K7U78tEqGHxkaf8h86bEMTSY2lcaDaaUuV140i3o3b + EilZ+WTWt39Yw7kwLnpmRz7Zqt/W78zU4PAStCWh0PRiXLeWB+HxyMjmvT9bA9ui089/YyWNcSJi + Gz9gylOY18/eon8+dg18UrMmhnR/CF6LpYP2BGyWOFvJH/GVOWjqR8o77kfypBdAGmWFbcP+HY18 + bgLcj0VBf3lKe0vYBl1L88qc/WtohVUsK233wQYz0/Dol4qdxlC844BN+YE/7HbvA5KrJWekuD/R + YEY7iug5zNgvXxliZWfA7ToG7GTlVsKDjRwgfaOlf/V7j/zAg6PgNfGn+Yrg4zUQHVaM8nNcJ01T + MnO5NkyTkMd5Hv3WH1UpzYiPHlhQg740PfKCA7H2OUPDRb9USGVqxU4PXY/4iacZ1Cf/wbbETX2e + R9npbz8FtrWO+Hbk8HseqfA78XszHUwgeVeR/S6WUS/Jw0UHnx9/eiliX8fIgWW2zraPtduKVRHb + mhqMOtvaKLFE67eVVmJnx3xzWSd//e6UrxFsPCVruJz9E5r0H5n0mEUn/4+adTkjgcZPYvIvN0TD + wGSbWWIVi3swZCAXjwszzetb/PTBcneKLn/5dCg1t162C/PB7GWVRsMNDwOcKntFErHJC767IRnJ + T+fIts7IxHh4shrNo51J24kvxiv3OIqjl8pMExxLyQSRoazXlLjn2UrI98HMdKQteuLFWeGLMAlL + SJcnm/iL9RX10jXBsFKzI1tRVbN4nl5uEGlCEB9KUnDvYuz0JxkHdmrmUdR046qE1F7dKFUaq6gH + daj/+lNbWw7iu259D5WyfKQzWhYWNzfaRpMPr4GsY28u+OoKw3LKG+jiENYtOzZJh9Da0fDzcX4m + YxJEBuSRZOHhIBetmB2HAL3fZIGXKTkgvo2PHCb/yay8qSKafg4pOgYRUElfVNE4+oaNwB+OE3+X + aNKfEvrpMawOZdJcK5jBeVsLKvizjpozCkpQ+k9INettJAN4qQYdg4KZS+2F+srd51pVsRkxPcZ9 + Nl+EgLa+nVGujWpEY+uJIWmjI17QT4aG5SetfniLFffqF+Mv73kU5YFFtmNZ465aNfD9tJ+/eDwG + 4ysA+OgSnr3JpRjOy8oB7XssySZFHvrxLxTuZTHpf9kSv/o9Bntglu0UVtuNqoyudOHQzPtcWx4g + 42++i+tk0fjNx0YchnliT/rHQsqnHXaw11SP7R7zRHDko9tyI9MHm/IFX57WEzY+v5B1H3fJ2IQ3 + Bx19oyBXa3FLWL655cg0jhzzKQ/TTvE8hglviJN/P1Zf7bYlLBmmZHOcHyPhcJtCNNfXLBiqhRj7 + pA9+ePLzi0j01tXQ1qKU/uJr2+ChAv4yK7Lxd5EY2uNNhbwybWJZhdkqysULYLgsVmR1vJ0TsV7s + KRy4PKfSRyqscT7vciSvXyuGaVn4oth71Y9viZ1tlgWf35cmOuhth4cV7f2WDDsMEH04s4I5KYb+ + pgNsX01J7K4nf/0teN6t/ukrMdbNdYBJ/9FZW4cFX9bIQfJ+dybXZHvwB57Jmfb2gzNzdelj/fJ3 + iOf7EA/qti74S/ZtcIN0x0xp//KFHm42cMWMEuu6ehffK/cGNOUnzNN3bjuYX/mG1ndzYGt9iyNR + 65eb5p/6Jwvmq0w0ww40wNVCZWsTVeJ7aEsKXlp9mXe1rKTF3buDCX/Z1lpo0aCFLw2605VgpcpI + Mm6ebYniqFDxOO5ZNBblHkO9PLhUpPsxoVfOTV2XjYLY61L2KWtYCSu8qIl31Zaoi66aBIlZ7Nkm + OEntKOgmVtMt7qgePy0kX1FjoynvJbeJ74ZgWM5QuGQjtp/mRfzyQ0BvfKDzyY/2S4/WIJ+ZSZyr + USV8eh46Z4FC8JSHDGXxBVBia0nwqfethXSNAmgU9Uw16aFHvTtQDMY0U+lMOqTMP5cTgPO2iKWY + dTHu6NhBvRpTkoadY42Z88gA7nAi1vGYiP6YrTe/PIddzyfH4qz5lCiW0oys1GFtjZJNbXRFiwX7 + +a/35XvrQIv1nM6mPEig461Dk39iay28iO8vr0mXsc0isr21Q3q3VJAV6cbcrtu1Qj+sqT7nD453 + tpsn49jn0l88GMVMawWNRw9mG1wScloJny+kQwNKew3o4xEYYlE4kqn/zhdMsjgUX2unzbSfnp6t + 33EhTruvCaMl3Cn/OqHRgeIA0du7THrvVvCwSm5oeCsDlXhRFo2xDyhK5u2Z8olPBPnUm1/+PJ0/ + mYirmW/ALnk0U72QhCJf3OCsk/d0fnVN/vqLCU+wSkTsd26+OkHVLncsHaoFEvcSKHr2zCW/PIbn + M+RpZb832LqxjVZM9YoU0sp0PvHBd0i9Ds7ug0/nOVU7lv1qhyKXrJjZXE/WuJjXKciBGv/yt3a8 + +IsczsvBJ4/vwyzkvHFMdDLnF6yHBrW6SLxzcLIO0c7Uy3bxUK0Y7HtXM3PS4xz4poYgMLdsHdlP + iwf8kqJzfH+R7WP9KbrbOZN/5xfEns4vOi15ZnA6H29sc7l4RaepUgfvYpYSL1z10XBLqwskWVoz + X77QpCPz0IaDUVQ4yY3AV4QTlMB2WkiVq50X46rXmh+eMns+HyOKvpcKJrxl6+XJjJT1bMV/+TxV + vV60/FoBwHg7emT9PwAAAP//7N3LjqpIGADgfT/FSW97Og2oVDE7Ba+gVXIRJZlMENEWRRGoAio5 + 7z4puzOZ5OzOcvJvWUCgCP/tCzVLb1bL7EeO4+3N4c/+vXJWLjk+TEPxXe8N2WdfH4mNRUTvOhfq + vmiWeDxeBNz5cHXxzOfx5DE68tWiOe+7XrYzcYJOEz5vtZXfVf4lRrLfxIzxYyS0uane8ZZoE0rd + 1BNtmepLnCh2x0f+4eR0ToDW+oVuCvl8V06vf9I9nA0YIfqH4YnGTOYNHj6OOZXvl6gUbTRH84+N + zZ/1JXvG55BVNTeXO98RSdN2hqGZGbV0NchEhN0Qn5OU8yXJkrLZOVPP0JNQpVZ4b3w+7ZYVkvNJ + 0pP9PXF+GGtkqwGjMn/YV7Mo8tBqaNTkLuNVx2olx6jW3ridaHzfhJOS4KnuUb4ILTcT+5quUW87 + srm7Hp6y+tS/K/gaRzM+nlxCq7d7DAiS600nXVwJMSmSB/5U9eprniuUKtJw4ftXot+6S9ac8ihB + 9MxyOjf6mV8N8KjCMj9k18VbWlZjbf1mBPVkSRfXvp+1tJwRTOkNy/jERXVwiYKe61nI+FNdtpc+ + 3u4VhVr9+WIvassdGrPArYkYG1HGN5v8goo8JgSNLJGJxXDbx6EdlKRj96GjJd41xGU3Wsn4K4Qw + vSjB8vxMNQZmqTqOj3HkeAGf2dOe1crvDXZXnUWf8x5ZryUI0xVnSHxEFj/63dQ4F4eUNYHQRRO6 + Jw8ph1QhRuJR1H3WvokOcaWyj+NDoPas7jU8o+hB1MA+onbh5+cBY7ngM9MvLZZ4dYC91dT5up+W + BMUZmT3tIPtBbsnzeDfHMl8mWOYbhU1Ob/j1qQJ+/vEbokAFUQCiAEQBiAIQBSAKQBSAKABRAKIA + RAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQBiAIQBSAKQBSAKPhFFGggCkAUgCgA + UQCiAEQBiAIQBSAKQBSAKABRAKIARAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQB + iIL/uSh4+fHjr+cuCPn9kF4lDKjTtn7/lwq8x4f4XVG0r60SWBWf0tc/vwXCa1He86L+u75f0lsl + qcH33wte63sdX/97/EVe6ufLPwAAAP//AwArUdmvhGEAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a127268e180df-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 23:04:19 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '163' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999979' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_4086a3c988f643e3a10b3e2e9aa7a13c + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml new file mode 100644 index 00000000000..b35efdbbb4d --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml @@ -0,0 +1,549 @@ +interactions: +- request: + body: '{"messages": [{"content": "Generate a question for the given answer and + Identify if answer is noncommittal. Give noncommittal as 1 if the answer is + noncommittal and 0 if the answer is committal. A noncommittal answer is one + that is evasive, vague, or ambiguous. For example, \"I don''t know\" or \"I''m + not sure\" are noncommittal answers\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"type\": \"object\", + \"properties\": {\"question\": {\"title\": \"Question\", \"type\": \"string\"}, + \"noncommittal\": {\"title\": \"Noncommittal\", \"type\": \"integer\"}}, \"required\": + [\"question\", \"noncommittal\"]}\n```\n\nDo not return any preamble or explanations, + return only a pure JSON string surrounded by triple backticks (```).\n\nExamples:\n\nanswer: + \"Albert Einstein was born in Germany.\"\ncontext: \"Albert Einstein was a German-born + theoretical physicist who is widely held to be one of the greatest and most + influential scientists of all time\"\noutput: ```{\"question\": \"Where was + Albert Einstein born?\", \"noncommittal\": 0}```\n\nanswer: \"It can change + its skin color based on the temperature of its environment.\"\ncontext: \"A + recent scientific study has discovered a new species of frog in the Amazon rainforest + that has the unique ability to change its skin color based on the temperature + of its environment.\"\noutput: ```{\"question\": \"What unique ability does + the newly discovered species of frog have?\", \"noncommittal\": 0}```\n\nanswer: + \"Everest\"\ncontext: \"The tallest mountain on Earth, measured from sea level, + is a renowned peak located in the Himalayas.\"\noutput: ```{\"question\": \"What + is the tallest mountain on Earth?\", \"noncommittal\": 0}```\n\nanswer: \"I + don''t know about the groundbreaking feature of the smartphone invented in + 2023 as am unaware of information beyond 2022. \"\ncontext: \"In 2023, a groundbreaking + invention was announced: a smartphone with a battery life of one month, revolutionizing + the way people use mobile technology.\"\noutput: ```{\"question\": \"What was + the groundbreaking feature of the smartphone invented in 2023?\", \"noncommittal\": + 1}```\n\nYour actual task:\n\nanswer: \"The capital of France is Paris\"\ncontext: + \"The capital of France is Paris.\"\noutput: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 3, "stream": false, "temperature": 0.3}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '2795' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//7FRNi9swEL37VwxzjoudZJMll7L0g17aQyks23qxFXkcqytLqjSGlpD/ + XpQ4scO20Pv24sN788ZvnjTaJwCoatwAylaw7JxO78h7fic+bj/IzyTFp7dvHp7WD6pffRVf7nEW + FXb7nSSfVa+k7ZwmVtacaOlJMMWu+XqxXGSLm5vFkehsTTrKdo7TpU07ZVQ6z+bLNFun+e2gbq2S + FHAD3xIAgP3xG32amn7iBrLZGekoBLEj3FyKANBbHREUIajAwjDORlJaw2SO1quq2hf4o6cQnRe4 + gQLvW8GgAnBLIIVTLDTYBt57YSS9LnAGBRprpO06xSx0VGWHqqqm//DU9EHEOU2v9YAfLqa13Tlv + t2HgL3ijjApt6UkEa6LBwNZhMhE/SyL/n8SQxPzlJZEAPB4Xpr+aF523neOS7ROZ2HA1HxYGxz2d + sKuBZMtCT/DbM3HVr6yJhdJhEi9KIVuqR+m4n6KvlZ0Q0yN87uZPvU+TK7P7l/YjISU5prp0nmol + ryceyzzFZ+xvZZeUj4Yx/ApMXdkosyPvvDpdmMaVq0bmGeUZbTE5JL8BAAD//wMAfjk28VIFAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0afbbd5918b8-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:14 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=nxPGrcXhbymyBih2t7CgJx2kdUAj2c8FEUM6SgQuXQA-1734303554-1.0.1.1-0gC9PrWCtGQtTp5RVHnIIDJR3yD.DgFli7YVq20QDgOcIMxDrBJn9wXO3e.Fak8_HP_bRTKACpjVZuouGgX1ig; + path=/; expires=Sun, 15-Dec-24 23:29:14 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=GidLrZtG7TkkHj.qFUWUeIOOu8KdGLixVP6dtmMZgV4-1734303554457-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '452' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999320' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c1efe4eef4aaf287a93b88df4e64a2c0 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '114' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1R62a6zurbm/X6KpXWb2gpd8GDdESCQ0NhJIF2pVAKSECCE0NiAj867l8h/dKrq + ZkqT6QkGD3/d8H/866+//q6T4pH2f//z19/vvOv//h/ztXvcx3//89f//Ndff/3113/8fv5/Ix9V + 8rjf80/2G/77Y/65P8a///lL+O8r/3fQfO/usyW6cQiS8fqNqTqZDmFB9CzQZNaOAvxD18wig+yx + MtBK1X4rAQvr8BNO1X4QQBKbG1uX1xwNXnmJIVDSKxUgH/kkaS4FIigf4ir9J+GqqVbI3199XDcX + uajvtNRBWz01LMjftzeRozeg9egZzOXXtJjug5tDdO1dshW/q7BfSw2F69jrDBfHVzIF8SNCmVc/ + mL4LcsR72gpoO778eT4GF+8HyYLbsz6zLc6WbX8/SDZoi5YQm5Kl16XjykCR1Lzo8hI5RVfULAM1 + 2G7IzY8HNKqrAqB1rSs7ewprp+Pxe4TmuxswYGInw+pRZmgp1BzLrn3xuHURclAP+ZbgbKd4NIjP + Z7ADd0s2q3MecnzWDY0a4gMvF/d12Itn24fxmafEum1eaMhbIQMosxgP6XFop29oK7DgboPDe/nm + LTkiHRbZ4stcFB9bmpzUCC3Ec4ulnVeGVBTiWs1PTko2b/Hc8kXxluCwLUeC3fZk8hUyUgir20hc + +WO34+OUVdDot454R68xR6hfEaiK7WHqLb5FfT/YErqXR5eZPdTeoCaXXG1k9UIb59qF3f2NFogU + yyWehvclpO/ltVkthYYTV09lkwdsopqrZjqV8WntyYq/2cL0bY8UoQdGbDc1GDmP+kAOe10vpD3L + AGx1T5hX+0Eo58s8RocycpjFYuRNMOmNhjaWShfcKcKBrr0jSj+uybzrpQ07qoRbKMs6IoHSbc2h + iFcKxGKeM3N5jdCQfLcA/jhpZLMtP6hL754CR/oA4q2igvPL7WCjGj22xKlXZTH4bklhNEeHrkb3 + lPAKJkXL9T4mdo9q3pP7oQTZEnfMuMY7k+/umgVoVUVUk6tPMoivSIfs3QKe+pNXDBtf6lB/YhdC + FvdX+JsPOpRnh0o33U46NfiqoGp9hYXjrkvGg/wawNAvE7HwSi/kkzMA7CSjYuZRLBO2/FwjuCnX + hJGV+jK77NBF4B0si0QK26BJ2NwjEI6NQzbd2HlNRNI92iySFyO7rV0Mb9G1kJVkH7IzxTgsvFOe + L/Hm6VCl1HQkLfREQOI2Vsn6lQYhI0dkqJFd7EjwRkPLa6GOwPPpmhmXaJuMKdqfNdnHKV4mDm+n + ek0n+IaHN7HXHW0nenMewLRNSzxjtQ0nFLUpNF9voJNweJn9a31dgHBPBeK2730xkGVg/96fLpxk + F7JPoWNNJl+J6Dgswm67vrjgasuY6HJVmyOEgKERuzM5FMfcZLa9UqCJRkqHtpzMjjRPUKVjMeBX + ugq8QTkJDzgEg0n2R6kouHG5GMjZSyFxLwENK8/3MBDZrtl6FEjBPxdvCxcteJMgepqc2/aoqoHy + uBLjs2vCaeKiC+XKOOM6887tdHZ2HcpUc4EXix6F7QoZD8iEa8G274Ns1h9yPGt1qWyIrTiNyR26 + pKvSUwgzdlsnZGm9pkAVd0vco9Nzattuoy4e04f9vudwkKYc+lfas7un9uFUSlkFl4PusHsqDt7o + R76urks3JM7tY6DJ3Ly30F0/e+Ipa5IMd9oZiDlnhVlRvi6kgcY24CEJ6FJaeO34ukg1Mn3/Sfzd + 7o24WcsuCrreINaFdHzisftAy4fNiHO6Knys9WUGzW2pM33zunnTXntUEOXHJ1vbyo7TzAkG9BGm + ngVy5YSC2PiWGu6HM0mXyiX5LIq38MMnKqtbLfxIX0sCpjktIWRzLwZwIxWg39nM80jUStWjSuGj + 3W5kw921KT3WOUWy76fMJ9LB5O1XH7R5PkR3P6t2mBYhVd/de8vcgpTJsGkSFZ6ED5h3Nk+4/zEa + 4EZyYetndU2+mEYYRAoaO+jPiPfyxiyRmkYi2ZzuUTLgqyxAYvVHit5c8frLRa1QZwWYal8KRXXK + IIagtM94WHyffDSavoEvrL+41eTSm97La41mvCdOMvacviTTBvuQe+yHRxRUhYK+8gO2Nk67gq+m + +Lq6hZuR2A/b5h13TAmdxf3IrHD58oa5vlGhah+2U/enGe90QMNbHIgrPO4JP6yHK0o/W5NE+Jwl + o/42qj//T8SthNg3lFxYVaXBnO+AvL4kpQWumuvETOie88zRDJj5izj7oxLyASSANIl7YvSntpi2 + WSyo5OYdGF59YjQ9h9oADpZJ9u7nVvSP4uErvPUSKq7rYzI879/rSr4FHhZeCTVHSLs9qOrRp+Vr + gUP2TJstXCXlSMILsvjI+UYFGk42nfdTMpk1UVD6WoyY73yzEOOX7v7Gs3BPUDJMi4SixhmWbK5/ + 3rqWe0XZIVTx9OncduyH+KzW93zEKrfsZGrwOtLm/Ukcq8vC0ZJ1DDSNJ7J5fsWEBUztQKQLjZjT + s04YOXIdSJa9mad6cTJdrl8BUhgfdNVZEZomrrmoR/p6Xt+A860fATwmbhD9bJOQS+S4Rdp9/2B4 + eKlhP6UZRjN/Mt0WgnaQK9lCuegLWFQDBY3hYTK0UhJOzCovZjH9+LIS0zddbt9VMp605gy9q36Z + +VYnj7pqq4OxciT6JQ54Y/Aut3/W/6C6iPft2XWhX9wTtl6Fm5BXm8KAnKQ6Od6zb9uJixVW90zq + mDPlm2S0gh5UiJcbFsRpzKdpqWL1JSodHoZm4j1b9A91o4LDHm9ybSksF0d43k4bcp+yindusKLI + P1YB0+fx7My9CcS4RMRrvz7njtkpqPArh+DmdDOnS/VM0WN4r5m/bgaTf3BoaX2zuWExY2dzEnI1 + h9Pic2KbjJ29SX+vBM0n94I4eV8kfb7wFHgvxh378X2PvTH94dWMTyuTyW9ZAIo+OV6N6redmqG0 + EVvVB0KWkcwHS7oC0m1FwEhCuJUFDA0CiSZ4afsHs/vhJyVXE8tYWYfSimxj+ICrE3LEjin/7u+9 + HzYe01Xv8X1S2LAynzcKKebtRB66BMaFbSg/b/Ymm/EfJe3hROwqDU2mRBpe7fBLxSvl6/3RUyAN + T+cPf0vJST2jmY+Zva0tc3hnESC1oDuq7OKl2Vd3GUMXJgu2uYQ7ky7yRYnE0+1LgtM5Tj5ttqew + xnLNtt6eI66xRoWL31jM0kQ7mVQ1xDCowURlwbMT+dMqe7hR2aYvWhZoSCjO1PvR3hEcFbonGYK1 + AH3d58SxpTZkD+O2gMXWL3/43fapfloAkx/7mS/dhF2bEcOsD9nNXSft6LRZo23Kq0XuZrk2pdvD + V9BBoTvmXxTHHGtdzkB9Ky6GT/81OdN9A/pTf2Hrm702aSllJUh41xMrMFvEhzjFgBFRyWaRGmhi + 0VqFX/2706sy+ft57xA+3iViuqbqDT89+7X3EsGz/h12DziDcqdvWijLJ6f1upqAkWWJl69QQH2K + 9pGGfSkmjmikiAfJpgLho+9YPL1sc/KyIdLyQ2OwgAp3czzkUKtG1Ats565R24W8z1RZ3uZk1qec + B0o+we5+fLFgF+TelCh3+/e+FLldFA7iYsTa092esOQngzeeDAZoutwPDIM+eNOq5ramaqxivn8b + i6nolQztPDuj8OPDqK8pDHl8xiLv14nEllEHStNHbE3Lgg/S/nCF5aexKA+vVjFCWu7RXB9ke2Tf + oqduXmpO/WVU/qYW4hWoCjoOzYKOVR6g6XJ9CQiWF4f4EdP4GFNt+NUXrt8H2aMyzlXYmSwg7kUz + 2p8+VvPv88Hs82f0Jns8PWAtRRGz3Obcsvchs9G8n5i3P50Stu6sLeDjU6K587TD7iR4OqoXKCd+ + EwvtRCpFgR/fbe+P9re+FKTOsSjM+oUJ3UFSYSx2xNIfYjuZhxyD4mUl01/XgznzcQNqvBWIJ2Az + /P2O/FCsCDb958yXbqnScLCZs312Cdc3kor4bmiIP66akF/ytatlOc6JX66X7bgLaf7DF3pByQGN + 1++xg+z89HE58iGcDliaYPmpLaa/0SHpnx+9g3Pbp4RIi7ad9aWi5aFk4iF6lu0An8nW0p3fsV/9 + jmc7fiB6KDfMlN81H6wrH+BZv0SGT9oZsXCqjrA/H65Y+N4tUzCdsIHn7bJhO2NRoK8V9Atgfu4y + 67jzw17bfzDQ98KhXf6pi9eNZIb2yauOEOGgJ4JZKhJgsXoS91AG7XSO5AjG+zn55QV8WLt1rbru + tcalLTctj1dWBJ6TMSxeb18+WfiEwYdPw5zyXCajvPFKNNcb8RLB5aOVhzY4/mNPJ8XqPTbq3xwg + fE/ErEMn6X56zZkCi+2WRuZxS70coe1bl1lhqvP+KUgR2DRf4aftH7wRy/4RDZ50JMGhwOH0coc9 + zH6GFju/KLhE4i1Y0qOf13+c/cKo/PASa8aha0dxey0BPRqM5bJR227jLzrYbZUnlhTHNen3AwYa + 7FYnJA2icKybwwBFD8qvPkIuZumgktTNmTNOB69/dcUEr8vyRHT/TovvrL9U5/ZYEt/JLCQhD10h + PQ5rEv/8eVTEW4D74sx8zeu8qXWFMwjJkeOBXxbetBHyB4xHTWbWofK9sRRXZ/TzM/aHOcUAH9UG + vU9lrJRaxsdfHtJMbMMs+bvxaPrIr0g/OyWz62Pt0R9/zX533m9iwrVA32pz/kOHXWAgoTHVCah6 + 3lH5FVM0deOuRHJWPpn57R/mcCn0q5ZZoUd2yrf1OiPVJ3hx2pKAq1oxblrTheB0YmT7PlzMge3Q + +ee/sZxGOOGRhR8w5ynM7Rdv3j8f+wY+qVETXbg/+FTzlY0OBCyW2DvBG/GN2Wjej3TqJi+UZr0A + wijJbBf073CclgbA/VQU9JentHHCtuhWGjdmH15Dy81iVan7D9aZkQYnr5StNILiHflszg+8Yb9/ + H5FUrSZGivsTDUa4p4hegoz98pUhkvc6xLfRZ2czN5PJ30o+0rZq+ke/98jzXTjxqSbePF/uf9wG + wuOa0ekS1UnTlMxYbXTDIORxWYa/74+qlGbEQw/MqU5fqha6/pGYh5yh4apdK6QwpWLnh6aF03lK + M6jP3oPtiJN6Ux5m5z/7ybfMTTjtxgl+zyMVfideb6SDASTvKnLYRxLqBWm4auBNp59eCtnX1nNg + maWx3WPjtHxdRJaq+KPGdhZKTN56baWW2N4zz1jVyR+/O+drBOtPwRyuF++MZv1HZj1m0tn/o2ZT + LoivTmc++5cY0cA32HaRmIV494cMpOJxZYZxe/OfPljtz+H1D58OperUq1Y0HsxaVWk4xHgY4FxZ + a5LwbV5M+xhJSHraJ7azR8bH45PVaBnuDdrOfDHeJndCUfhSmGGAbcoZJxKU9YYS57JYc+k+GJmG + VLEnbpQVHg+SoIR0dbaIJ25uqBduCYa1kp3YmiqqOeXpNYZQ5Zx4UJJicq/6XnuScWDnZhmGTTeu + S0itdUyp3JhFPShD/cefWupq4N9N67molKQTXdCyMCdjq25V6fgayCZyl3xa32BYzXkDFY9B3bJT + k3QIbWwVPx+XZzImfqhDHgomHo5S0fLFafDR+01EvErJEU276DTB7D+ZmTdVSNPPMUUnPwQqaGIV + jqOnWwi84TTzd4lm/Smgnx7DylAmza2CBVx2Nad8etZhc0F+CXL/CahqvvVkADdVoWNQMGOlvlBf + OYdcrSq2IIbLJo8txQDQzrMyOqmjEtLIfGJI2vCERfrJ0LD6pNUPb7Hs3Lxi/OU9j6I8stCyTXPc + V+sGvp/28wePR398+QAfTcCLN7kWw2VV2aB+TyXZpshFP/6FwrmKs/6XTP6r35N/AGZadmG23ahI + 6EZFm2bu59ZOPtL/5Lu4TsTGaz4WmmBYJtasf0wkf9phDwdVcdn+sUz4hDwUr7YSfbA5X/Ck+XvC + 1puuZNNHXTI2QWyjk6cX5GaKccLybZwjQz9NeJrzMPUcLSOY8YbY+fdj9tV+V8KKYUq2p+Up5PZk + UQiX2ob5QyXysU96/4cnP7+IeG/edHXDS+EPvrYNHiqYXkZFtt4+5EN7ihXIK8MiplkYrSxfXR+G + q7gm61N8SfhGPFA4TtKSCh+hMMflssuRtHmtGaZl4fHi4FY/viVWtl0V0/K+MtBRazs8rGnvtWTY + Y4DwMzHTX5Ji6GMNYPdqSmJ1Pfnjb8F14/qnr/hYN7cBZv1HF20dFNOqRjaSDvsLuSW7ozdMmZSp + b8+/MEcTPuYvf4doeQjwoOzqYnpJngWOn+6ZIRxeHteC7RZumFFi3tbv4nub3AHN+Qlztb3TDsZX + itHmbgxso+1wyGvtGqveuX8yf7nOeDPsQQVciQrbGKji32NbUnDT6svcm2kmLe7eHcz4y3amqIaD + GrxU6M43guUqI8m4fbYlisJCweN4YOFYlAcM9eroUJ4exoTepsnQNEkviLUpJY+yhpWwxmJN3Ju6 + Ql14UwVIjOLAtv5ZaEdOt5GS7nBHtehpIumGGgvNeS+JZ74b/GG1QMGKjdh6Glf+yw8BvfGRLmc/ + 2q9cWoN0YQaxb3qVTPPz0CXzZYLnPGQoiy+AHJkrgs+9Z4rCLfShkZULVYWHFvbOQDHo80yFC+mQ + vPxczwD22ySmbNTFuKdjB/V6TEkadLY5ZvYjA7jDmZinU8L7U7bZ/vIcdrucbXNizadEkZBmZK0M + G3MULGqhGxJF9vNf7+s37kCNtJwu5jyIo1Pcodk/sY0aXPn3l9ekq8hiIdnF7ZDeTQUkWYiZ03X7 + lmvHDdWW02PCe8vJk3Hsc+EPHox8obacRqMLiy0uCTmvuTeJwrEBub359PHwdS4WtmBov/6CQcRj + 8TX36kL96enF5h0V/Lz/GjCa3JnzrzMabSiOEL7d66z34mIKqiRGw1seqDAVZdHoB5+iZNle6DTz + CSefevvLn+f+k4EmJfN02CePZq4XklDk8RguGnnP/atb8sdfzHiCFcIjr3Py9RmqdrVn6VCJiN9L + oOjZM4f88pgpXyBXLfuDzjaNpbd8rlckk1aiy5kPvkPqdnBxHtPcz6nasezXexQ6ZM2M5nY2R3FZ + pyD5SvTL39rx6ok5XFaDRx7fh1FIeWMb6Gwsr1gLdGp2IX/nYGcdop2hla34UMwIrHtXM2PW4xNM + 2xp839ixTWg9zcmfrim6RPcX2T02n6KLL5n0618Qa+5fdGryzOB8OcVse726RacqQgfvYpESN1j3 + 4RCn1RWSLK2ZJ11p0pFlYMFRLyqc5Lrvydz2S2B7NaDyzcqLcd2rzQ9PmbVcjiFF32sFM96yzeps + hPJmsZ5++TxV3J63060CgDE+uWTjPD7mSN2mgvjy8dgvvxdyoazgbp/5f/k9nb4Udc1PJubye8vF + 5Dv4YFm7iHnLg8p/eh42zfrJgt2QJ5NcXA1IUbZh21EKwqkLyxjNeRPVrGbNpa0h1nDB0oaQw+PI + x/ah+pAK7sTW4T3zJi9Ce7Ukp+/8fQNPVjL1CMWKYqwutSMfjHQ7gN48KzLXF+8Eab1F2+XJZT9/ + SX/8fKZdzwz/Gno8HcZJ0ySjIKYqRgW/weEMefpgzMdF2g5Xzz5qanoWiXmuh5DZk9+huT+J5Tnf + 43mj7ZErRpTM+iHpnNvtiAJd63E989VEe6EC1EsL5qYSS4bzpsVgq0fCdmfzUPCkJ3skX9YuO+z1 + rOgzpRbgHd8cZm3KsylfmxVG83qTzRR3nG++aQMvUe3+9HO50N0k+IbhG6ufqSyGrLqliOS0IltN + KcJuBesOZn1I37vFo+0sab/Qon7jk91bCYuRtA4GQj4w8xPj3f2ABfRbz+/MP115KRW4JIJATGW7 + S3hvHnTNiQ495pZ2K9jpVJXoW8UYo7XJC77TLwqc3ajFE611T0qP7zO00zqY+ZdzbhxvKcz3p6K2 + MlrR80KAm3eMmOPasjnOeAOHYDLJr98z+7UUAQkYRXx5M9kznGwt/94fdIi4yofzITsi4f4QsJYe + CZpefWige9yJdPlsOBpzMZHAIajBYuQ+0bgLq3xFacWZY4StSdNjH8ExsL0/7zPi6JsjQ5bucx50 + aFkVX7cw62UMs974ujhbwN+/UwH/+a+//vpfvxMGVX1/vOeDAf1j7P/930cF/h3f438LgvTnGALt + 4uzx9z//dQLh729bV9/+f/d1+fh0f//zF/pz1ODvvu7j9/9z+V/zg/7zX/8HAAD//wMAiTuukd4g + AAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b00ec6af78d-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:15 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; + path=/; expires=Sun, 15-Dec-24 23:29:15 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '149' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999993' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_bc5d97a0802a0175e3fa12eae77be9f2 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30], [3923, 374, 279, 6864, + 315, 9822, 30], [3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '192' + content-type: + - application/json + cookie: + - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; + _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA+x6yxKyOrTm/DzFrj21q+QmWZwZAoJySVQQtUeCioCIXBIgp/rdu/Dv6q6e9qgH + /8QqNWIgK98t67/+459//q2T4pH2//7nP/++867/97/Nn91v/e3f//znv//HP//8889//V7/r5GP + Knnc7/kn+w3/fZl/7o/x3//8R/jfn/yfQfO1u8+W6MYhSMbL90bVyXQIC6JngSazdhTgH7pmFhlk + j5WBVqr2WwlYWIefcKr2gwCS2FzZurzkaPDK8w0CJb1QAfKRT5LmUiCC8iGu0n8Srppqhfz9xcd1 + c5aL+k5LHbTVU8OC/H17Ezl6A1qPnsFcfkmL6T64OUSX3iVb8bsK+7XUULiMvc5wcXwlU3B7RCjz + 6gfTd0GOeE9bAW3Hlz/Px+Di/SBZcH3WMdvibNn294Nkg7ZoCbEpWXpdOq4MFEnNiy7PkVN0Rc0y + UIPthlz924BGdVUAtK51YbGnsHY6Hr9HaL67AQMmdjKsHmWGlkLNsezaZ49bZyEH9ZBvCc52ikeD + WxyDHbhbslnFechxrBsaNcQHXi7u67AXY9uH8ZmnxLpuXmjIWyEDKLMbHtLj0E7f0FZgwd0Gh/fy + zVtyRDosssWXueh2bGlyUiO0EOMWSzuvDKko3Go1Pzkp2bzFuOWL4i3BYVuOBLvtyeQrZKQQVteR + uPLHbsfHKaug0a8d8Y5eY45QvyJQFdvD1Ft8i/p+sCV0L48uM3uovUFNzrnayOqZNs6lC7v7Gy0Q + KZZLPA3vc0jfy0uzWgoNJ66eyiYP2EQ1V810KuPT2pMVf7OF6dseKUIPjNhuajByHvWBHPa6Xkh7 + lgHY6p4wr/aDUM6X+Q0dyshhFrshb4JJbzS0sVS64E4RDnTtHVH6cU3mXc5t2FEl3EJZ1hEJlG5r + DsVtpcBNzHNmLi8RGpLvFsAfJ41stuUHdendU+BIH0C8VVRwfr4ebFSjx5Y49aosBt8tKYzm6NDV + 6J4SXsGkaLne34jdo5r35H4oQbbEHTMut53Jd3fNArSqIqrJ1ScZxFekQ/ZuAU/9ySuGjS91qD+x + MyGL+yv8zQcdytih0lW3k04NviqoWl9h4bjrkvEgvwYw9PNELLzSC/nkDAA7yaiYeRTLhC0/lwiu + yiVhZKW+zC47dBF4B8sikcI2aBI29wiEY+OQTTd2XhORdI82i+TFyG5rF8NbdC1kJdmH7EzxFhbe + Kc+XePN0qFJqOpIWeiIgcXtTyfqVBiEjR2SokV3sSPBGQ8troY7A8+maGedom4wp2sea7OMULxOH + t1O9phN8w8Ob2OuOthO9Og9g2qYlnrHahhOK2hSarzfQSTi8zP61vixAuKcCcdv3vhjIMrB/908X + TrIL2afQsSaTr0R0HBZht12fXXC15Y3oclWbI4SAoRG7mByKY24y214p0EQjpUNbTmZHmieo0rEY + 8CtdBd6gnIQHHILBJPujVBTcOJ8N5OylkLjngIaV53sYiGzXbD0KpOCfs7eFsxa8SRA9Tc5te1TV + QHlciPHZNeE0cdGFcmXEuM68uJ1iZ9ehTDUXeLHoUdiukPGATLgUbPs+yGb9IcdYq0tlQ2zFaUzu + 0CVdlZ5CmLHbOiFL6zUFqrhb4h6dnlPbdht18Zg+7Pc8h4M05dC/0p7dPbUPp1LKKjgfdIfdU3Hw + Rj/ydXVduiFxrh8DTebmvYXu8tkTT1mTZLjTzkDMiRVmRfm6kAZ6swEPSUCX0sJrx9dZqpHp+0/i + 73ZvxM1adlHQ9QaxzqTjE7+5D7R82Iw4p4vCx1pfZtBclzrTN6+rN+21RwVRfnyyta3sOM2cYEAf + YepZIFdOKIiNb6nhfohJulTOyWdRvIUfPlFZ3WrhR/paEjDNaQkhm3sxgBupAP3OZp5HolaqHlUK + H+16JRvurk3psc4pkn0/ZT6RDiZvv/qgzfMhuvtZtcO0CKn67t5b5hakTIZNk6jwJHzAvLN5wv2P + 0QA3kjNbP6tL8sU0wiBS0NhBf0a8lzdmidQ0EsnmdI+SAV9kARKrP1L05orXn89qhTorwFT7Uiiq + UwY3CEo7xsPi++Sj0fQNfGH9xa0ml970Xl5qNOM9cZKx5/QlmTbYh9xjPzyioCoU9JUfsLVx2hV8 + Nd0uq2u4GYn9sG3ecceUUCzuR2aFy5c3zPWNClX7sJ26P814pwMa3uJAXOFxT/hhPVxQ+tmaJMJx + loz626j+/J6IWwmxbyi5sKpKgznfAXl9SUoLXDXXiZnQPeeZoxkw8xdx9kcl5ANIAGly64nRn9pi + 2mY3QSVX78Dw6nND03OoDeBgmWTvfq5F/ygevsJbL6Hiuj4mw/P+vazka+Bh4ZVQc4S024OqHn1a + vhY4ZM+02cJFUo4kPCOLj5xvVKDhZNN5PyWTWRMFpa/FiPnONwvx9tLd33gW7glKhmmRUNQ4w5LN + 9c9b13IvKDuEKp4+nduO/XCL1fqej1jllp1MDV5H2rw/iWN1WThaso6BpreJbJ5fMWEBUzsQ6UIj + 5vSsE0aOXAeSZW/mqd4tmc6XrwApjA+66qwITRPXXNQjfT2vb8D51o8AHhM3iB7bJOQSOW6Rdt8/ + GB5eathPaYbRzJ9Mt4WgHeRKtlAu+gIW1UBBY3iYDK2UhBOzyrNZTD++rMT0TZfbd5WMJ62JoXfV + LzPf6uRRV211MFaORL/EAW8M3uX2z/ofVBfxvo1dF/rFPWHrVbgJebUpDMhJqpPjPfu2nbhYYXXP + pI45U75JRivoQYXbcsOCW3rj07RUsfoSlQ4PQzPxni36h7pRwWGPN7m0FJaLIzyvpw25T1nFOzdY + UeQfq4Dp83gWc28C8VYi4rVfn3PH7BRU+JVDcHO6mtO5eqboMbzXzF83g8k/OLS0vtlcsZix2JyE + XM3htPic2CZjsTfp75Wg+eReECfvi6TPF54C78W4Yz++77E3pj+8mvFpZTL5LQtA0SfHq1H9tlMz + lDZiq/pAyDKS+WBJF0C6rQgYSQi3soChQSDRBC9t/2B2P/yk5GJiGSvrUFqR7Q0+4OqEHLFjyr/r + e++Hjcd01Xt8nxQ2rMznlUKKeTuRhy6BcWYbyuPN3mQz/qOkPZyIXaWhyZRIw6sdfql4pXy9P3oK + pOHp/OFvKTmpMZr5mNnb2jKHdxYBUgu6o8rutjT76i5j6MJkwTbncGfSRb4okXi6fklwim/Jp832 + FNZYrtnW23PENdaocPYbi1maaCeTqoYYBjWYqCx4diJ/WmUPVyrb9EXLAg0JxZl6P9o7gqNC9yRD + sBagr/ucOLbUhuxhXBew2PrlD7/bPtVPC2DyYz/zpZuwSzNimPUhu7rrpB2dNmu0TXmxyN0s16Z0 + ffgKOih0x/yz4phjrcsZqG/FxfDpvyZnum9Af+rPbH211yYtpawECe96YgVmi/hwSzFgRFSyWaQG + mli0VuFX/+70qkz+ft47hI93iZiuqXrDT89+7b1E8Kx/h90DYlDu9E0LZfnktF5XEzCyLPHyFQqo + T9E+0rAv3YgjGiniQbKpQPjoO3abXrY5edkQafmhMVhAhbs5HnKoVSPqBbZz16jtQt5nqixvczLr + U84DJZ9gdz++WLALcm9KlLv9u1+K3C4KB3ExYu3pbk9Y8pPBG08GAzSd7weGQR+8aVVzW1M1VjHf + v47FVPRKhnaenVH48WHU1xSG/BZjkffrRGLLqAOl6SO2pmXBB2l/uMDy01iUhxerGCEt92iuD7I9 + sm/RUzcvNaf+Mip/UwvxClQFHYdmQccqD9B0vrwEBMuzQ/yIaXy8UW341Reu3wfZozLOVdiZLCDu + WTPanz5W8+/zwez4M3qTPZ4esJaiiFluE7fsfchsNO8n5u1Pp4StO2sL+PiUaO487bA7CZ6O6gXK + id/chHYilaLAj++290f7W18KUudYFGb9woTuIKkwFjti6Q+xncxDjkHxspLpr8vBnPm4AfW2FYgn + YDP8vUd+KFYEm/5z5ku3VGk42MzZPruE6xtJRXw3NMQfV03Iz/na1bIc58Qv18t23IU0/+ELPaPk + gMbL99hBFj99XI58CKcDliZYfmqL6W90SPrnR+8gbvuUEGnRtrO+VLQ8lEw8RM+yHeAz2Vq68zv2 + q98xtm8PRA/lhpnyu+aDdeEDPOuXyPBJixELp+oI+/hwwcL3bpmC6YQNPK/nDdsZiwJ9raBfAPNz + l1nHnR/22v6Dgb4XDu3yT128riQztE9edYQIBz0RzFKRAIvVk7iHMminOJIjGO9x8ssL+LB261p1 + 3UuNS1tuWn5bWRF4TsaweLl++WThEwYfPg1zyrhMRnnjlWiuN+IlgstHKw9tcPzHnk6K1Xts1L85 + QPieiFmHTtL99JozBRbbLY3M45Z6PkLbty6zwlTn/VOQIrBpvsJP2z94I5b9Ixo86UiCQ4HD6eUO + e5j9DC12flFwidy2YEmPfl7/cfYLo/LDS6wZh64dxe2lBPRoMJbLRm27jb/oYLdVnlhSHNek3w8Y + aLBbnZA0iMKxbg4DFD0ov/oIuZilg0pSN2fOOB28/tUVE7zOyxPR/TstvrP+Up3rY0l8J7OQhDx0 + gfQ4rMnt58+j4rYFuC9i5mte502tK8QgJEeOB35eeNNGyB8wHjWZWYfK98ZSXMXo52fsD3OKAT6q + DXqfylgptYyPvzykmdiGWfJ349H0kV+QHjsls+tj7dEff81+d95vYsK1QN9qc/5Dh11gIKEx1Qmo + Gu+o/LpRNHXjrkRyVj6Z+e0f5nAu9IuWWaFHdsq39Toj1Sd4cdqSgKtaMW5a04XgdGJk+z6czYHt + UPzz31hOI5zwyMIPmPMU5vaLN++fj30Dn9SoiS7cH3yq+cpGBwIWS+yd4I34ymw070c6dZMXSrNe + AGGUZLYL+nc4TksD4H4qCvrLU9pbwrboWhpXZh9eQ8vNYlWp+w/WmZEGJ6+UrTSC4h35bM4PvGG/ + fx+RVK0mRor7Ew1GuKeInoOM/fKVIZL3Otyuo89iMzeTyd9KPtK2avpHv/fI81048akm3jxf7n/c + BsLjmtHpHNVJ05TMWG10wyDkcV6Gv+ePqpRmxEMPzKlOX6oWuv6RmIecoeGiXSqkMKVi8UPTwime + 0gzq2HuwHXFSb8rDLP6zn3zL3ITTbpzg93+kwu/E6410MIDkXUUO+0hCvSANFw286fTTSyH72noO + LLM0tntsnJavi8hSFX/U2M5Ciclbr63UEtt75hmrOvnjd+d8jWD9KZjD5ezFaNZ/ZNZjJp39P2o2 + 5YL46hTz2b/cEA18g20XiVmId3/IQCoeF2YY1zf/6YPVPg4vf/h0KFWnXrWi8WDWqkrD4YaHAeLK + WpOEb/Ni2t+QhKSnfWI7e2R8PD5ZjZbh3qDtzBfjdXInFIUvhRkG2KaccSJBWW8occ6LNZfug5Fp + SBV74kZZ4fEgCUpIV7FFPHFzRb1wTTCslezE1lRRzSlPLzcIVc6JByUpJvei77UnGQcWN8swbLpx + XUJqrW+Uyo1Z1IMy1H/8qaWuBv7dtJ6LSkk60QUtC3MytupWlY6vgWwid8mn9RWG1Zw3UPEY1C07 + NUmH0MZW8fNxfiZj4oc65KFg4uEoFS1fnAYfvd9ExKuUHNG0i04TzP6TmXlThTT9HFN08kOggiZW + 4Th6uoXAG04zf5do1p8C+ukxrAxl0lwrWMB5V3PKp2cdNmfklyD3n4Cq5ltPBnBTFToGBTNW6gv1 + lXPI1apiC2K4bPLYUgwA7Twro5M6KiGNzCeGpA1PWKSfDA2rT1r98BbLztUrxl/e8yjKIwst2zTH + fbVu4PtpP3/wePTHlw/w0QS8eJNLMZxXlQ3q91SSbYpc9ONfKJyLOOt/yeS/+j35B2CmZRdm242K + hK5UtGnmfq7t5CP9T76L60RsvOZjoQmGZWLN+sdE8qcd9nBQFZftH8uET8hDt9VWog825wueND9P + 2HrThWz6qEvGJrjZ6OTpBbma4i1h+faWI0M/TXia8zA1jpYRzHhD7Pz7MftqvythxTAl29PyFHJ7 + siiES23D/KES+dgnvf/Dk59fRLw3r7q64aXwB1/bBg8VTC+jIltvH/KhPd0UyCvDIqZZGK0sX1wf + hou4JuvT7ZzwjXigcJykJRU+QmGOy2WXI2nzWjNMy8LjxcGtfnxLrGy7KqblfWWgo9Z2eFjT3mvJ + sMcA4Wdipr8kxdDfNIDdqymJ1fXkj78F173VP33Fx7q5DjDrP7po66CYVjWykXTYn8k12R29Ycqk + TH17/pk5mvAxf/k7RMtDgAdlVxfTS/IscPx0zwzh8PK4Fmy3cMWMEvO6fhff6+QOaM5PmKvtnXYw + vtINbe7GwDbaDoe81i431Yv7J/OX64w3wx5UwJWosI2BKv49tiUFN62+zL2aZtLi7t3BjL9sZ4pq + OKjBS4UuvhIsVxlJxu2zLVEUFgoexwMLx6I8YKhXR4fy9DAm9DpNhqZJekGsTSl5lDWshDUWa+Je + 1RXqwqsqQGIUB7b1Y6EdOd1GSrrDHdWip4mkK2osNOe95Dbz3eAPqwUKVmzE1tO48F9+COiNj3Q5 + +9F+5dIapDMziH3Vq2Sa/w+dM18meM5DhrL4AsiRuSI47j1TFK6hD42snKkqPLSwdwaKQZ9nKpxJ + h+Tl5xID2G+TmLJRF+Oejh3U6zEladDZ5pjZjwzgDjExT6eE96dss/3lOex6jm1zYs2nRJGQZmSt + DBtzFCxqoSsSRfbzX+/L99aBGmk5Xcx5EEenW4dm/8Q2anDh319ek64ii4Vkd2uH9G4qIMnCjTld + t2+5dtxQbTk9Jry3nDwZxz4X/uDByBdqy2k0urDY4pKQeM29SRSODcjt1aePh69zsbAFQ/udLxhE + PBZfc68u1J+eXmzeUcHj/deA0eTOnH/FaLShOEL4di+z3rsVU1AlNzS85YEKU1EWjX7wKUqW7ZlO + M59w8qm3v/x5Pn8y0KRkng775NHM9UISijx+g7NG3vP51TX54y9mPMEK4ZHXOfk6hqpd7Vk6VCLi + 9xIoevbMIb88ZsoXyFXL/qCzTWPpLZ/rFcmklehy5oPvkLodnJ3HNJ/nVO1Y9us9Ch2yZkZzjc1R + XNYpSL4S/fK3drx4Yg7n1eCRx/dhFFLe2AaKjeUFa4FOzS7k7xzsrEO0M7SyFR+KGYF172pmzHp8 + gmlbg+8bO7YJrac5+dMlRefo/iK7x+ZTdLdzJv3OL4g1n190avLMID6fbmx7ubhFpypCB+9ikRI3 + WPfhcEurCyRZWjNPutCkI8vAgqNeVDjJdd+Tue2XwPZqQOWrlRfjulebH54ya7kcQ4q+lwpmvGWb + VWyE8maxnn75PFXcnrfTtQKA8XZyycZ5fMyRuk0Ft/PHY7/8XsiFsoK7HfP/5fd0+lLUNT+ZmMvv + LReT7+CDZe0i5i0PKv/pedg06ycLdkOeTHJxMSBF2YZtRykIpy4sb2jOm6hmNWsubQ2xhjOWNoQc + Hkc+tg/Vh1RwJ7YO75k3eRHaqyU5fefnG3iykqlHKFYUY3WpHflgpNsB9OZZkbm+eCdI6y3aLk8u + +/lL+uPnmHY9M/xL6PF0GCdNk4yCmKoYFfwKhxjy9MGYj4u0HS6efdTUNBaJGddDyOzJ79B8Ponl + Od/jeaPtkStGlMz6Iemc6/WIAl3rcT3z1UR7oQLUSwvmphJLhnjTYrDVI2G72DwUPOnJHsnntcsO + ez0r+kypBXjfrg6zNmVsypdmhdG83mQz3TrON9+0gZeodn/Oc7nQXSX4huEbq5+pLIasuqaI5LQi + W00pwm4F6w5mfUjfu8Wj7Sxpv9CifuOT3VsJi5G0DgZCPjDzE+Pd/YAF9FvP78w/XXkuFTgngkBM + ZbtLeG8edM2JDj3mlnYt2OlUlehb3TBGa5MXfKefFYjdqMUTrXVPSo/vGNppHcz8yzk3jtcU5utT + UVsZreh5IcDVO0bMcW3ZHGe8gUMwmeR33jP7tRQBCRhFfHk12TOcbC3/3h90iLjKh/iQHZFwfwhY + S48ETa8+NND91ol0+Ww4GnMxkcAhqMFi5D7RuAurfEVpxZljhK1J02MfwTGwvT/3M+LomyNDlu5z + HnRoWXW7bGHWyxhmvfF1cbaAf39dAf/jv/0/dBSIfzsK/nYU/O0o+NtR8Lej4G9Hwd+Ogr8dBX87 + Cv52FPztKPjbUfD/X0fB/wQAAP//7N0xCsJAAATAPq8I1wdCSv8icpDDwsQL5gQb/y6nQfxBQKbd + Zj8w7BIFRAFRQBQQBUQBUUAUEAVEAVFAFBAFRAFRQBQQBUQBUbC7KBiIAqKAKCAKiAKigCggCogC + ooAoIAqIAqKAKCAKiAKigCggCogCooAoIAqIAqKAKCAKiAKigCj4c1HQtO3x/YIw5zFNFQaU9Cjd + lwp0cYxd3w+fq4T7Gs8pHDaBEJZbnpdyKvmSrmulBtt6QSi5xOk3b2rVs3kBAAD//wMAK1HZr4Rh + AAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b048feff78d-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:15 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '205' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999979' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_8dcd051b01b6b87197e931e81edf1ce2 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml new file mode 100644 index 00000000000..f0a4cdc548f --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml @@ -0,0 +1,543 @@ +interactions: +- request: + body: '{"messages": [{"content": "Generate a question for the given answer and + Identify if answer is noncommittal. Give noncommittal as 1 if the answer is + noncommittal and 0 if the answer is committal. A noncommittal answer is one + that is evasive, vague, or ambiguous. For example, \"I don''t know\" or \"I''m + not sure\" are noncommittal answers\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"type\": \"object\", + \"properties\": {\"question\": {\"title\": \"Question\", \"type\": \"string\"}, + \"noncommittal\": {\"title\": \"Noncommittal\", \"type\": \"integer\"}}, \"required\": + [\"question\", \"noncommittal\"]}\n```\n\nDo not return any preamble or explanations, + return only a pure JSON string surrounded by triple backticks (```).\n\nExamples:\n\nanswer: + \"Albert Einstein was born in Germany.\"\ncontext: \"Albert Einstein was a German-born + theoretical physicist who is widely held to be one of the greatest and most + influential scientists of all time\"\noutput: ```{\"question\": \"Where was + Albert Einstein born?\", \"noncommittal\": 0}```\n\nanswer: \"It can change + its skin color based on the temperature of its environment.\"\ncontext: \"A + recent scientific study has discovered a new species of frog in the Amazon rainforest + that has the unique ability to change its skin color based on the temperature + of its environment.\"\noutput: ```{\"question\": \"What unique ability does + the newly discovered species of frog have?\", \"noncommittal\": 0}```\n\nanswer: + \"Everest\"\ncontext: \"The tallest mountain on Earth, measured from sea level, + is a renowned peak located in the Himalayas.\"\noutput: ```{\"question\": \"What + is the tallest mountain on Earth?\", \"noncommittal\": 0}```\n\nanswer: \"I + don''t know about the groundbreaking feature of the smartphone invented in + 2023 as am unaware of information beyond 2022. \"\ncontext: \"In 2023, a groundbreaking + invention was announced: a smartphone with a battery life of one month, revolutionizing + the way people use mobile technology.\"\noutput: ```{\"question\": \"What was + the groundbreaking feature of the smartphone invented in 2023?\", \"noncommittal\": + 1}```\n\nYour actual task:\n\nanswer: \"France is indeed part of europe\"\ncontext: + \"irrelevant\\nFrance is part of europe\"\noutput: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 3, "stream": false, "temperature": 0.3}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '2802' + content-type: + - application/json + cookie: + - __cf_bm=nxPGrcXhbymyBih2t7CgJx2kdUAj2c8FEUM6SgQuXQA-1734303554-1.0.1.1-0gC9PrWCtGQtTp5RVHnIIDJR3yD.DgFli7YVq20QDgOcIMxDrBJn9wXO3e.Fak8_HP_bRTKACpjVZuouGgX1ig; + _cfuvid=GidLrZtG7TkkHj.qFUWUeIOOu8KdGLixVP6dtmMZgV4-1734303554457-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//7FRNa9wwFLz7Vzze2S72ej+CLyXQphRKein9IC62Vn72KpElRZIhZdn/ + XuTdrB2SQu/JxYeZN8O8sZ/3EQCKBgtAvmOe90Yml2Ttw9dv9+vN3fer6+XtJft5rb7kv1bDpx8f + MA4Kvb0l7h9V77jujSQvtDrS3BLzFFyzTb7M03y12oxErxuSQdYZnyx10gslkkW6WCbpJskuTuqd + FpwcFnATAQDsx2fIqRp6wALS+BHpyTnWERbnIQC0WgYEmXPCeaY8xhPJtfKkxuh1Xe9LvB/IheQl + FlDiZwdXlilOYJj1oFv4OFht6H2JMZSotOK674X3TAZBeqjrem5vqR0cCyuqQcoTfjjnlbozVm/d + iT/jrVDC7SpLzGkVsjmvDUYz8bMSsrcSCli8qhIigN/jhQxPVkVjdW985fUdqWC4Xpx6wekwZ2x+ + Ir32TM7wi1X8gl/VkGdCulmzyBnfUTNJp4NkQyP0jJi/vedpXvI+bi5U9z/2E8E5GU9NZSw1gj/d + eBqzFP5b/xo7tzwGRvfHeeqrVqiOrLHi+K20plq3PEspS2mL0SH6CwAA//8DAO5kTllDBQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b13ef8b18b8-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:18 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '423' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999318' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_afefe5d59636c8e9a842ec74f937fbc1 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3957, 48687, 961, 315, 38634, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '111' + content-type: + - application/json + cookie: + - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; + _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SaWw+6PLvmz99P8eQ5dd6I7HrznCEgImCLgoqTyQQQERCRTctmZX33Cf5X1syc + mIAobXNvruvX/se//vrr7zou0qT/+5+//n7nXf/3/1juPaI++vufv/7nv/7666+//uP3+f89mVZx + +njkn+z3+O/L/PNIx7//+Yv77zv/96F//vr7O2UaedwO97bcvO4JVKEYMb8QOmfGku+CleINXa9z + PR7HnkRIiK2RXTM5nMdyMkpg8ntFO3Lg2mF1qXOkOoqJFUlfFbOTSA009n4g2rN10PC8BjYIFxbQ + 4bYf9HFwpA6FdrEhe5VVc92txxA0mgbMGNK46FAUNKi4hypeYcqK6VRdbZDV14pt1/xjLpFR8RCq + TMUoMs96fUzCDB4B1vC8jsV5vAZ5CcYmQPijFk7R3pVzCl+dGcS8p00xiqs1Rtv96DEjoHMxzyax + IL+gM1GloxfP14dswpYvMbt8jKGd0293Bpkb7nRNGwHRWXNtOGqiTKzOLeKGf5U53E/wIVt7RnGH + O5XC926/iOt1H5/i/IUVWq7W+Eq6Uh9pkXBwLsWcqLCR0FxcVgZoxm2PwctcZ+LL9gqIUxA7BuK+ + mDYSqZEvzRyzsqePhp0w1vDePr9E27O6bfefIoT5yR1J0PFnZ5CETQjiHWts74uGPx05k6JrRky8 + 8Z4GGkY/rZAnnyJGWvwtBuH8DOHlnf1lPHfUa5NtwBZ3e/Y4rR7tQCrHBO0+lZR/nFlcBrVNIeu2 + AoVtcWiHV7HLgZPuHrGT5O3P4WnFg7bjXkRzcxxv7GsGiGvkhpGc1IgVbSEjhTvPxDyNsT53b7EE + OeOvbF9tzHljXZ6pxK24nn7509OZfAcwXPfvM5a2OCkY6Z8raM3vh1kG+uh9pyqNpMqDgS/AXoj9 + /i/dOSeGj7cwnkrMeTAbik/02DDbIeERRW6WnMjRv3DzqOhKB6VkHWnmt9t5fha0gYGVJjFJVzo0 + SFsXXj5a4aGU+GK26dOCz30HbGvQss0+ijTAPiEvtn80DZrax/sMh311xbN01Zxuv+3O8jwPJxJk + sa4L/vutId4zLkR7abrfCMI5gIC/tWS3ve4dPh58FbgkNEgkP9b+VMV9LZnlLaaoFcWYpnVmovf2 + 8WXWQ38X7GK5HnDedsDc5iu0093eqCDsXUrcrePrbIln+TnkJpXMu1iwTfu25eEeARZqVXfG46zl + 63X+qmjD3tq8OajIgstFvOI4ZPI8lhs9BMUkOZYtq22HKSMAdswQ0zb2dp6drYmhAnOg6HmiDnN9 + BcNwD4FunKxAQ2iUARTWDWFlb251Ru9KAPeNpVB+P9/9oZlVFZ6cMxJtXxz9ydtZKWR16DKHTnk8 + FFZRQXeIaqIfzbczprxWgiS6ObnNRtQO95sfwMbwHBLSx8YfTs7LAnJKd1Q45G7ciW8cSgPl7gQP + NVdQg0a2/DqpKvGtfen3wuObQQX8g8ryYx03UjKE6Mi9eWYndVaM1ZbjUId2Jdk7a6lgonMsxUDx + cmIxriv6oDtSwKJRMr07x/HURTSB9limBHPJPZ5znZgwvXQP81181OczfaVwx1qBP0d7M3fz6XsG + b00FtnNbVR/M/UuUnNCfyPacNWgkH3KGa7aaqYTPRdyFRnmFfrd60mfcb/x5j1cDiipBYnuTe7Sj + ZYoDBPhZ4s/qPhfdyDQNCVlJiF8Irj4gg/JIv15j/KLeNA/G4Ymlpb6xXXpylvF9E7QLixPZpTuC + +gdYLpjfQWS76PtqaxtEEwpc7Nhv/qOb+ZmCbivGNKYaiN6rOgGxt3f49ZGjYhadXQVvE5nM1GrF + p/ZwGtAo1wXBScY77NuUHQysMpn2bNt5WLudCeuPohPbVIi/od9zgj4FCOyh7dV48+nDBKxnR4hl + m8VS/70MpuFUMGPcnNFYfyIOVo/aJ86+f8Xd0h9QvKsFhg+ocQYN+gmW92Ol3L7j8RLVGCZcVnhc + 8w9E181TROXbtVmUSud2xNYQwbOrNKKPDyFuWJtZcOy8iByWfKJ6XFhwC/cu23vTMx5vL38F101d + kO25q/R+wEOuSCIQzC/1rr7kRwO+Z9LgM0TBPA3vPEPmeqUxtfL6mRGyNaH4iBVb+lX73XiDDQe1 + fRLHObCiVhqDB0ySgalDZCP6vUoZ8E5Y0tXF0525lRMZrU6nNTlc7DWaHF3lkbzJTApnRYqHx8Ab + gGvuROx3bOrT21o1yN3YW9q8iY0GuF14mD6wJ7fNqiyoifQc3Ohkkp35KvShTO8h4EDGxPQE1Req + TZoguyp5lnqnPJ6ypORkc+1eyM3klGLe1nm5Ua/8lhm4vLaNNpBUHnzlyfRLT/zxlPs20i8VpS+2 + VtuBm3iA75RrODPsHtU6WSq2ftbJPhTecX19mme01Fvaoi2vz3ryuaI2vtxY6DqyMw+QZEgwH1uC + V9PJn84niYO5/5iUCQ8l7jtZWEGFLjpeba2yHQ0ENfDGsyc7VO4KDkVBDYKIGnJwmYcGSVBCoO+u + I2o3vWIaelOt9FFNlnxH8Qeuh0YK/MOFHL9q5bA95ic0bXqV2PN9XwyXtyOi3WcuiXq4HOZBOrYe + AlNT6RiMisO+fZGjdvXqGVYH6gxPhSVoXdOB4JQUaNEnNnieJDNjq3T+W/8ea3SIsztR/XaLJvk6 + TbCyoGCuI22L4TRxKfqoGsfw7bVxRq1zE1h1lkTOe1SgfvO88nAu5ZxK2sj0ufm8ZSgM6hFMzEof + 08cDEHeFD3u2Go7r9BlWcFhRRqdr1+pLfU5QdOB6YlxMWx8cXk7ByO0324WyP1dHMrnoI0UB2d4E + zXk37iBCY6Y7pj8Oj2JISB1BNKUvrOwK3I70+FHlUx20WLjoajsYh5sLfHFumXGXWDHQmbeUoq59 + 5l5aS5+OHKYwz1rJjOvjFY/8UQqU2/nLCGnxoZj0sAzASRNg9nx049lwtjxCWnGm3JCilvmrUyMP + lL8z3dob8ah/dzU8tXRDtjlLiz/zm85Zw7B+59vBQ6IMmd8l7MhPbjwXF95Am+PFZ+oqdH1uBTOP + wgJdiXv3x6KJh1hFwy6R8Host4jTW8uDX3wS33ScaeUeVBT1uk4l4qZO23raBHY3qMTvN/1MNQss + FDfBhzgKyvRffsD2JadsLys7f4B5ewb9ZliYy7uNzrr3UMLBNAC3Wf0u6DqWMJK56c6scv+IJ0E+ + 2UinX57gi6fr1CrqUPZzbDJ9+x7QWORBBAN8zoyItVTQWTNslOVHja4fnuXwghBdIRQ/Jl3zhdZy + Qr1zIZ1XhBz35svhUZWvFNe86MQ11rd5HrSzizQgR+KuNmvUK0keoEWvLvoNih7cyQZe7Cxm5OIQ + T6mAJ9hRGtPp+5j82QRC5XC6A+WD52EeN88rB1P7nhjBe9ufTuCsoHRMoK81ryBq92dA0kVzMf9J + XGc4RXseihdLsHQ7Gc4gtIYGLzqIf/zMvHqCi26wJXhU7kM7/urBUq8ZeRjuf+mZ+ycy6MhVjj66 + +0qE0r6Qpd6H7Vgm6QBGS3o8LPVp0nbKFfi0lNnNuHl+XYbxAFXV+pjTDvuCN33RlXvFFLEoTTRm + YoZCFKDRIvtD3vkjy64qDOREqWDchnjab44yOj8fBn33VanXtZfxyuO26Qlm37kdp/Uqkmd0W+P0 + eBPjZX2vCg+3me3f5l0fByxmyNwedkzdJk99jK8NRkdNlunA4gOafvHVtcKJ7bbXzx+/gA6FsGG2 + H678EQ5zoKiyeiOGkud+l/BzBxMJfKLrVhH35XEwfvqPrvO41Ju6vmlQSvaRmH5mzQPu1A52/o5j + WuuqejuEqgbH0z3Do2Wf0dTu14n48zeWXrJ2DDkN5PtKmehvPWfUvFc//0KFnZyjfuxJKPcK/yaa + 6V4dCraQw6JXibWs5/x6jYFym1Z79utn86MrTIV/Bh/2q9edFt49ec8JF7IPn3hm/XMC+PZmSV8/ + v/YqjjnUza0gGl7H7ZCfag0t+omKcTahaetMAKfceTHSajSeP9lkS4u/JPr27aE+vr805RzpH4zE + 8Nj+xq/cPpcn2/dSWkyJqYrQm51LnP2lmefrbpeiEkYRo9JjzvwqRRuKjy3RyaXfdubfLAOJlJQd + iLvSm3rl8mDgFWb22EjzgHjVhDsXe0xnYeYPLPdqNF5WMy3DXIxbiX/hXz3DaLfyC77jTzI85QAx + Vc3LePF7EdRZRpm6Kj/FoN7IBO+gXRP3IhQx9ed9DqfHcyA6X/vxVJxmEyWxz+PV1X6g4dzxHOSW + fafoIfOoN26ZBf4jv5CjYOdoWh00DKlrWOzgFpti7JKvicLgq7OtpGj+VMDRRaMjvIlxuMsta69W + BMmtcchBDV4zvXAnDZb+RdTXqXfmfdNgGJxGZ47CSjQY8pCCrYcZwwdGEX1Zag6huvTb69mKl3wG + ZFj6jdgrmhXzpxABKatdTdS0SOO5GL8ZXDmZZ+Th1frURVUCincdiPFMNGeC7B3A4ieZteiJ4dyt + OBB7a8fszGucEWvyhLirKxNdsauZZeet9dMP7Ej61qdHImOZfLczMTbfWztLO7JCz21vLf4jaKlx + eWeQ0nzEq/mkODOZHyWMV5oR8hbOcf+oziFkdeTignNMZ9YTdpW5JDKItsTHUHPKVV54ArN7fuMM + 0UfPQX1fP8Shk+Yv8SaDnHFXEp773v/lm4wOyQGL5/7o05dl5Yickh07NQnVqV/WPHr50oodT0aO + vlP8EhVWqB7RtLJrJ9/h8J/+2X3XL9S/gL8ibce/6FwInT7rWK2V5816E392d4t++aQopdlIBXTI + iulRPgdEtezKjk94Ou3HOuYgyvIKJ8WmRN2iZ5XLhHL2W88fP0CLXiPmoF/b7zZJzj+/xrxtsnb6 + dRWIaMlnDNviWwzkuPR6jYnMOl2v+sRkzMuL3sLQf9qYsa68/uKb7axohXq6tjyQT6VO1F2f+TMy + 7wkYCdszrTaR3qiSI6KS+mcqra5vfR4qStFeF022u5+beE69dYP64LEjx+osxE3ojwkEVCUMTzfN + 53qQK5nlXkEO2ttHo3p4exCL+4w5Pmz0YdEDaOm/S7zl+tRmpfnH/xq45AvKrXfnn59nO8//tmPi + dZVsrkFjNxXlziC+zfDnP9hjb26dOUlkEW6TG2FlheViHiDIYPcZS9LnzREt/SAHdNxzhGQd77Op + HEOFdbuQOBmnzosf5KSF77BlPeLfNXzH6cIO+Kz7Yzo7A7LRuWLb745H8+vgUvSZHo8fL5r5r4dk + 9A1Djh3X7xMaXUm1oSkeJ4KbVChYJAQ8VBupot9MYH6vvr0OPVJNoNNJehezWI0qqE90IOokhM74 + 3k8pyjsjYXrJ9/rc73GA5Mz4EkM77NupEk4GqFduy3DYnRb//sqRHZoePRiWWUzHd62i6/oZMvfy + 7OYx5e0Kto9VxPYqyvWZ9E+ATj+mhFjD7FD6uXFQ3t85nm31iUaU3hqAi+cz/BCO/s8/wXvwPuR+ + lfqiWvyrsvhP9sy6azyyLFVlrOKUekt9WcafwJM7jMReeEqnMohQN6nJT4/5rFHiTl54IV5rkTwz + Lx3PsPA9qrSPOR4jlQuheh15Oh24dTt+5S2noPHqMeKGSdtFjX2Fx+Nk46FKvs7YHmQT4gDZdNN0 + kT6VPKlkCLKKWc3Gmeel/yqs0Dymw6ptR85lHPrxJAklsT7aXnuFwzpN6KpJb+3kqJomCyWR8Wbh + n6N4a2U4ch+euZyQ+iNdW2dkPr8NIQ+jm6el3kEj3ANmW5bTTk31oso8eQIzeUWNx94vG9iv7wJ9 + cRFD9NoGFfzGY1m3S9zCXU0VvheOv/o+Dz2/SOrpLOA6MAz0R5/5Fv8mhmNZ8U//I1XyGrIPW1vn + PHEywREre+mPd8Qe7CUipHtrCgt/Hcd+H8JLOaV4bLiynWbNtZD5bBsqcusazfH4xPI0+AUzXhdl + nkvDq+DB1+3vGi38rQHUNU+y36yMlqefJ49WeWCQ883YtsJdOScojvkt/UrOwUHl9j6ANcYunm7G + tmjc86gqqHpeFl6s+1QQzleI7Cpiuz6y5lnARqgoJ+NB7IU/DVO1K+XFj9H5fNrqY15HmvyHl7/O + gT8VJ2TA0k/JriYdGvJTpirhFAPNNu8KLfk6IXw4VX94wyj1dwpIP6+pHRRRO9a4wtByVCRWjrfF + uI8vJvjY08gxyEa9LF4eVuK9dWLnMj0VS/6UEK4mj9hU8+I/PMqa5Iro7X6td7izOth80hczxFvl + z4u+hcZMdsQMngc0XbiTiqoX4enCb9sObg8OsTR4Et0OeX+OvG2mLPoHi9lzRhNV3Rxt1iOQndtm + DsX5F8Mynz9++4//X/gAw6HYFUu/SeAtXRjucvOmL/1mggqMge0VDfz6eL9TILNMmSXbBpqL8ZUB + mKpKp8HZ6jx3G1eAbkTDiqGC/i7QQYPFnxNLWn2LWSwmDPKp0ulq8YN1MoMMS/zjqUIpmoMnC6H/ + ijnZijhoB55qHFz5yFv898UZ1udXB8VHrqhyMjTEjyi8ou4RtOQkjXt9Eo8WhhNWbGLem93M8eOh + /vFduimSjz6e3neA6OnZJBbee8SGquogvIQDsY7yB43cO0lg8r0bcb2s04eoP9jAO1FJbPsdzrMg + pjn0QhtjqcgExDuKY0Nyem6JsQ+KYtxsjBrNtfMlxzC4IIbMewrpfbPF4sLXx0uUYRC2esl+PP9b + l4GGln6z/L+I+q3+aaA0pIkZgkrRH/5pb/YJVuycxaOQZQHCgfViONWtdlx3UY5iMyyZ64p5O2Td + dgXtquipbI6cM+WaJiuJzL3IYT8wNHqktGD1aHxKF7035uQdoufNfhMji3VHWHizHESGy8xvv4o7 + ZT01yqI/ia5qq3h6W3wDt5CT8IclJ6evHtlZmevDl2nXznEG9/DK0eDU+uJ/78VoUdf48X3al9ud + Pyr6hoJl7OqFr7yKWRFuARq16UElgTsVg+Q1BuTec0N+fGAUg7cJrTdtfn4DTXX91OQfHzgSyWy/ + xq22gTiezC6VuInZ1wwx5N5jQ7bP/UYfY7+NYKjbn57IfbrkO5Kjr4OnUNeK0eHWHLzqNyXb7+6K + mGcaNdxPqw/DWLbnIaBWAFb/sEjCRx4a0vsZUBTefOKaV6YPw2Rx6NPYW6ps0Vt/aUw/K3YWd1S8 + kwxx/blO4LIFn6k4sNtxbiMDuJJleP0Wprg/DnoH52j7IdskdYvxGF1EOAVRTY5btNNnLuQtMHju + wpzSI3pve23w2z8jeywVxcyN+QDRoOyJuvSv7lXsMsDCVSD7ZT9CEE6OC1mQHqhU4aoY2uAyAX93 + Hnj9fMk6lYxFo5MpZD+ev8SHLafKbLDDFkM7Npzrwju5FkSXrpq+WfSBsvh1doCP4P/x17c+/xJt + dciL+WDbDeJLLSS7u8n0rm2cEkkizskSz/OEn03087fM0ocGTdxNWiG/PWHmHifFp89n0cD3WEgM + l3d13oDJZfC2oozoMHn+z49AdNipxGjbC/rt58kXyeyYkcWFM7A8bNBK/j7J7/dTJdwNuHN3j5jj + ZPi/eotAkU1cafXDH6+4A9hUlGM//Unz3ZYq9+x5JFp/EnRqtP4VZjsa2C6hYTFsXqdUWfgCsY60 + a4uTuleRuQ8IMU5pVoy0ihuo+srGq1B02xl8m0eyK5+ZHe+O87xOlAgt+o3tb9kRTZKhT7DoN+Z2 + RjizRo0qAMJvmGbJJZpOVWoB6iOX2FZgzpv9qInwvIlntjMvn3k63N423IzDgR2pozmjc+xy+Dx9 + E4s6mPpgBbEHcmZ+iZluXojZxlFFnBR7zJhLteAub0eG6MD3WNjJ2sxmc29BcA+eP3+hjz8/KuXz + jhH7qaI/+zeDcl0TKzqgdqrid62claPCTPlOi37nlCLEq3NDVNQoxZ96d97dBDwNzsuZM2Jf4bPN + MNmOYuL/eAtwVrr9zR8NPx6/8EmCpc/Fp+OzlEFKhS359Z9x4QMQqyTCSHh/5iHfQA5rgVyxbCrM + H7r1GMFDtHZ0PZYvNOnpC2Tq3hxmBZ7qbN7sOMmp8JFx24qi318bL4Ifr/ntp45Bt+vAEYyBEAOw + v/hVHtLvLmB6Htx/PDhBP31xSC8IMYOlV/j7dyrgP//111//63fCoKof6Xs5GNCnY//v/z4q8O/o + Ef2b4/g/xxBoF2Xp3//81wmEv79tXX37/93XZfrp/v7nL/nPUYO/+7qP3v/P7X8tL/rPf/0fAAAA + //8DADOe/03eIAAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b17bf7cf78d-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:18 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '116' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999993' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_505443a8093770cec87f1314500371e0 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3957, 9822, 961, 315, 4606, 30], [3957, 9822, 961, 315, 4606, + 30], [3957, 9822, 961, 315, 4606, 30]], "model": "text-embedding-ada-002", "encoding_format": + "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + cookie: + - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; + _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SZSw+yzJaF5+dXvPmmdiKIUJsz4yZ3qxC89ggQEbwglyqgzp/v6Jt0pycmAgqp + WnvvZy3+868/f/5psrrIh3/+/eefZ9UP//zX99g1HdJ//v3nv//158+fP//5ff6/K4tXVlyv1bv8 + Xf47Wb2vxfTPv/8I/3vk/y76959/6rkpyB6KvGsxkfdg7R6EpYo0BOM9qUtgrGkZeYw8YA8vUcCL + +5SdJLHgI79pDZh8b+JeWC27scaXGIHnESz1OzDHYXBeAF4AJLg6WzT7xxWGY3qTqDx7QzKaY2xB + SpyWkEtaJ30rJzmsN6sXw5JZJ/3L2KbKc0j3tDeLoZ4wbmbYHouE6cnqnbClZC/A2rlLrM7YSp6t + eDpDlEt7TBelWk9OcSphFoMNbuvVruukxMOomByTOKM/1VO3nlK42mPOwsAY69lk0ai2RR0Ss1wd + stm7Ki2QaNyy3PFQMD43gwHyfb3HU73RszFR+xagiRpi62GefIypauEl54wYnoPReOTbGYUNFohz + OO1rKuTxUbU/OwGXz9c1Gasd6qFEZyCaSdSMLxT6ALFwU+J+PiWaKtK3cBI/Hdu01K55XhILdTIb + mPZ5vdD0gZ0Gb3HpEHJodMQMKuyhsFydnJ0gRONi/S7BXl8UZqUO1H2jpbMyr7SYBL27ROyWKgY6 + ZscT5dKY1yPtcgBNmU4En09PRKu4DQGP/M2w/nARfxuXFYz5u8fpm81JW8UtBun0dmj5ucVoapa8 + gM21O+Lx+OySWV9FhlonjoiXy5qgVZhkI/I1u2cby+MdGzpmKIpaTSQMoppzVW8iGOy2YZ6We6YU + 7Egp535k0eUhSTsu+/kCLmMqYnjCvftUu7yAo+wlzDxccs7samjXr3Z8YbZbadkkb4MImn0XMS87 + boI5SI4PcE5Qk+3sjFl/vbcGUjS3JL7ykflkPzOAIEtk2nqmV0/xQsVgPGuEF6XRBEwT0QPqGg5E + OxZSMjVQNHAxngnT4t7v2p3ZNuDtsoi50L67WTxtKdrr6zNxi80+68zuVCnCco1Jun/riVSksELH + bNwTK7Dbuo21QEP43t2J/pIMJO4lFVA6nFbEYBFKpnVUWih8n950DFynY5rUvNCqDUW2lbskobfs + EkG0pXcsLUsRTXk1PyA8kztWVosw6c+s268/leZ+97Phg3hQ/fWiCmIMx5uQUfE802Wm7Ao6JJnK + ZzAVFx7LoMLW8yGa42EdnyH3qU7MQzmiCVk3DTYIVWxzuvU1825uCncfYrZxPTebWvGWwqbnOV1Z + iyuacGDCX/1AZWq87adrD0jJL9S0d0E9aSvFQs9+aZHv75PxEX4K8IvpzpxleUDT7fiisBlwT8LP + u0bjmsR72JDzleTp2UMTGI8zqGIYkYsqPJK/3z1j59OxiKqsuZ3FEY2KFODllkl12+h6pLh2aJCI + 6G5C6206Qnx6FbT2zSH4LNfbXCFE7hl+v44m75ndoJO7u5DweBMSFhwve7T7HJe/9eafMwttMJN5 + oCrFKeIfeShBHp8+Xm5VH40VLhS42tzDMMW2OYGtY2AaVHg5brualie5/d2f2eftifNEj0bFcBJG + QrOoMh7GUIE4RCUL8VAE7MzmF0T9ucWq3PFkxNtEg1lI1ozYcdPNR+liwTgYBMvUO9dDuwas9Jc+ + IUcv3HCeQGgjy9BkKtmE1uNlWFPU8H3PrMldm2yYmwYVhvQgZNmygM715QGd8lCYuRPrbLD6cwov + uWDMXy36hKtLHqoPfX4we1/5WT9XtQVnuwxo4u/KZDrIjQ+nc9yzsPC7pL8fkj1S6LkjVoqjbpT9 + TQ+fynBZGA/vesTbzADprFvEEhScSHLoLVDNrYzttAChcWd1AI2dnvBSWp85MwQX0FefLJD4seMv + K8zBnZmARRoLqBOujYVWbbNhzq5/B+OJUR+KauXjO2FFN6WNtYJOcVUsEqHN+mbx7NGvXrJlr6FJ + imMK0vuVEteJw4CxhpxRSyQTC0/Zqf/q+7N7rZn1lN8db9SNDxrYd4zs/bMeumk5w91fxHihk2My + XK8DwDISdNwM2bYee8EL144Xpyw0n/d6iLzdEfxbODHjzfusU5ccg7xDOtlW62PSL1TBh97bmcw9 + NptsvB/OJRRJbzFbmVcBbdQ+/atXxydmJpbnSEGwUgvcjbqYzSFZATgrkmMlD6xkLi+fAj2hTOm8 + wTiYrtfnAlDAWxIY1cMchHx6QX4GhbifsTTnc69qsOXikvzmy5zoowZBF91Y/O7P2fwd20g9lg6J + ipNer4prsEb1NO2YVhhK0grpcFRen2vOXOdSmky4ljZSNL+k0uXkZbzIPwLgqRGw5Clgdtf7FEM5 + +CbRtbI2m/TpxCiURp+WAnTJND5pjC674cIOlQjZqPHlXs7fn5G4UCf1zNqkgTMvSgqBZ3XTTnsr + 4BnMwctvvXDFHVzIlHgi+FxsTJEXjxDKPKmJv5LO2cx6rVEnT12QYPkMEd00u1KVVn3J/Fp9dizt + z64CF7ci4Xc+MSENYlR2gkycLZ9q+lZZiFZr1BE3Tq/J6F4xIENcFFRWb2pAn3ZSKVq/yJjN8RtN + kdPv0WdMdOLYbtpNn4W2Un/9SAPao3wVSw8kbpmERS1aBHQd0hUi2l5lePF+JYO0NyrlV2+Op0Aw + GlzuIZRGiVxnDTIaHrQZ7pfOpeBEzBw/wG3ousIhgaC35vhwbg2KEAwsuQSL7JPfNy84BlynPTRt + ParguKjen2OC2U371j/C8CiNkXla3SS0JsZL+fVrOxpd1PByzCHCfsFCH+X1zG/yAuR7ccJgPvVu + 2uLXWV5qWYEhHpxuliI7B27RjIVwX9XjzqpBzZsyYvprxJyf6fYFI/gH5gjmO5i//Vs1nB0jdm5v + E66JqzM0m82W2V1roekojXs0ONaTOXej64YGjo38GAuN+ezuduPbnM8wvwoghvK68XkpZyE6nS43 + Zm7B6CTFbY8w5s+ehZ+3iUbZ31I5se4lC5/itpZidUGRUmRnYj6aFe964Zwrgk5KLExHLRA6qREA + 3uKZefKQdZy4h0b53p+uiWVkvRzMFPrmnJPTjr8SaghlCIbrEqK/qqzmQ/c2YFG9KqYFrlNzHKgx + yj1BwDKZ1zV7eEWP5pcSUkPynvUDWahQwDdyZnT8HszOPY0Rr9T9l3+ipN0voj16S/GO+eeYIp7V + r29/C3wWita7Ht5mdFRwDB2dayXofv0XaPKI2EZWtU7agx7BPvIWZEvrKpDelt6qKMcKFlIc1VOs + tREq7suWbNtJ6ybWzmuFTucbloyHbs7ncbcCXqcPyvt0CKZbNlbqfFRkPGj3LpnZayFI3l0oKT9c + gLPV7qQBeM+QBb7tmFw8ZwWkgEKqbHZuMOpcqJBnuie8yL24G8vD0Yfq5XKiEd3N5kuvYMQWo0nF + 2Vplw2e91NAusGy8PKS0m9xiT8FikkBnTyP1FB4aA778Tueu1znXpPoM+50eEOPsHrpRPN5CkE6B + hNdO0KMpQde//ZHllVnyz6ZBDxB2oU0sWTcScae918i0yAmj7dJB81TnZ5TMJwPL8jAms5A+9zBu + by/cXfA6m7/1g9KbFVHehxfegF0W6m+9DU9j3TQ+6wJ5uqjgSV573Ww/JkVtcS2z8Apn88eLcObX + 7bce3YTPt2uFbhTt6Frcmoh/VGOPAnl5YM5jGND07XdIiE4vZhq7FZ/v+1hTv/4CS1h/cPpRrBHs + cL8heEE3Xe8VkwuOF6VUPW1v2bDF9Qtsxd+TL6/x+dLPISBF9xgOlSFrF3JhoC+P43v7sbKZDihS + DnjVM72SaDAKWbdCy6UV0jman4j3c12A5Co7KjhbMaAnyholpVuTbLIdDeicnxZQ14sDMSRj6jja + GL0q72SdmcvRNr96U9TF1DfM7O/3gMn+HtCXR4n25UGGAzGCoOtzSiW/6mgdDDOcTtmNkKbad1MY + B1jePlhIxfb9zuYlMrUfTzMnR1I3jR83RrIxIqIJ4ikbcIBnuN148POj2VyRTar++NV8NEc+aev7 + GoLcvRDzGry/eurO8qrFIu5se84mnR9iuNqnkI5A7928hFFTqaVtmedqvTlYffbtr8XAvMxVzamT + vBf6rV+wXTf1JGFxjazlKqUjVjyT+mmEgS3mEU9WTPjqqw+wtIExs2/K7G//lE5P5+vv33zMnmEB + 9C55ZJvtk4zNt0MF8u5m/eWd8csbytd/4vVdOqDRv3x6+Oy8Bse1skL0ZUc9dFvcEO/0qLu5nw70 + x3dff8KTaa7HFXz5g+HuLNV9teM9EDVZkI3cXLKevcUjtLS6ES/USs4SsGwY+8Ag1n3m3Tg9UgUu + 42rD9JdUIa6L4wzq4zAy5yEMAV0l1Eb9JToS14xP2WxO7gOFH0snyU6sE15FmYvGbpiJ6ZvbYGy0 + dITQqzq2WVmdOUuHcwQpqRRiuDLqvs+zhy//0tVtKQfzV38Qek5GZ/1w7vg9ui7Q7XbAxFr3jPf2 + E4XoiZOGhevVx/xkrxQr5dUdiR19vG42BvpCLb7LhNw3z66fHvEa7v7dwONdQGgUj/kZrgelInir + 6xljHzgja7FU8PKQKF0fG9MRFfFKItrkf9DIy3WhCIY9sJ8fG+13ieGmVwnR7ojw0bu8jyhsGosc + Fbk1x50BLjpm8x63X7/7nfhHtN+ZATP7ZOA9v2kt+Iujz6zA9muavatRDZTzk2ys0wuNJm1jOSUL + Hyffeup/fqESqUfh+GqSOdG0UJ2CKCM5RnYwd4J4RLbcn5gRzU/OjXnnQnUqWrraJFHCNHGRgkLT + jlKdrDIW7iyqWjt/yX71MT+98Qin04oRctnkQfP05RYtzJvHgt3kZT9+Ur77geezK9YcbbYl3DTi + MX//vidjJ1iGglI04olWz2DQeVyAeiAtMy8XO5u8VLTgffbXWIAir7kurkcI8sBmVtxuu955uA9F + X/tA56jb8R9/Aa2iCyOj3WfztSwpFGY/kbDyS/Oj8ziHXXOb6Xram6bkXbMU/fID+MR5N3nFbg0K + SWJmR2PDeXDc7ZVWi/dE14KMT/fEiqF6axq5ZGQy6e0kxRBsi4hhT++7qUr0HOG4zpj9nqxg3Lyf + AtyiMGfXaul//VgWwlF+1fii3lRzXCzvLlyW6oM063qBej+7VIC2pUWKL9/OyCAAezYkxHK2HufE + 91slyP0LC674jsbaX0bg3VclCx2P8NG9Wi5Qy1CYrYeQ8dU5amHrHLdEJyxJxOIW+fD2tJkRaX1G + 4/2QHOF2mwJisMJM5mtZV3D8vCP6rERI+ry+CKjLbncKxHgms+JPZ8hCMyZ6W0UZF1O3UHxV15jp + iTLnrTSvoX+rIdFvodbxHw98/Qldp8nMh8tHaBCtEht/yDwkvZB9NPSd50y/bfp63m3S4q8/++Zr + fDLZqKn62oW/9TnpPM2hVdsnHs3rM5vA9jBsn48L0+XJT+bsOS9gYe2PJP+8Td4J2RipP32ff/PU + ei/X67eoChTuVO7Yl+/AWW1z4gi+nTTx8hYjQ3R2VDCzpGagD4qS+7GFH5XzMpkc6gvYkPSKjwzz + YF7KSQheeT9SwMOMJhLornpjc8EsPJ0Daj9kBbr8qGNp3D270StkHw7QIrp+fk7JvFi0zbdeJQrH + 88HkBl36oOV1wHS17YKpTN0ehB3mVHzB8+fnDHgsXzNdrGgezPrMscIiqyE46in6m3diMzv8/H/y + 889ozbdPgiW/qudkkfkwbq8vZh1rnH37ZQvXwzVm4ackaJzKlQ1oazv44ew/Qf/tT3CS0g0zuuKC + hjLVqJpLncn016IyZyGtKtCu5zMezeOUDdvYTeG7fkR3LTuYwhgbSDAsjQRXrHPhO78QUpQjc7/5 + Z79HioYCWT3Qqf08kjmrji4kD9ujQzuV3Wj1c45+vECuySMYw30cI+e0jJj2fKnJWOTp629eEGzX + HI15nbe/5yXBY7aQ+PPn+YqvCBFsPVhtkwiQ3T52uO0aB63P03kPp9dCxMKX/4bwUGrqN0/9mz9/ + wNqlILk3nW0qcWfy1Z4Larl9DMQSiYZEi7ohqo7PK5VPnfHNR8sIDYL5/s5HjY9elu9h8i+YaIVH + s6k8duvf/MbXlfZEXPFTH23tdYQHLTmbk3/xC/Dp7NDldhV0/Bm4yi8vJEZOPjVFzm6GpcoZFrXi + Zfbpk8Tw3X+W7ChL2FSJLtBplZMwMKKO3y5bA7K8qAjp9z7nq2Mj/PIRpsF7qOdhStewZKVHgs2y + yGaztQFN3nJB10XYZNTLDQHk8e1jsb0IJrcGvVfdh7bA6IKH4MtHKXqfXy7ZtBchGEi4XgCuSsrM + 5X5Z36MNVOj6LJ7Me/GGT+nrJEB/oQme2o+VjC2Msfo++wFzjOPFHObKM0C/KCPzLCwi1iwEgObR + 1PRO2KLm6gJjpKdHDQMeCrMnQZPCedMsyI+/pkdoYPjqjU6vRRWwOlwc4ed31sM2zbhqRpa6Fd2c + eDd3h3j6tvff81tme+ETTfFCDMFwdgbD+xNkU9p9WuTTvUT2X31Ml+FSQIm1lpDm49ZCL+gYBaHH + KTin2pz30JTwJI8N2Veajebj7ERwDA4WCT8lQ3MC1AUvjq7E8fXRHInHU3i/KSYW9I/6bx5LEzch + rmOtkfABZIOG45h887J6NobXA02Xek/8Wo0QK4pLCzzJFlg41rU5fufHj0eYbS1UPjxt9YVQ2EhE + M69jxw78tAI7KwK2uZmsmxdyocGZmxFWtXuXjdebqin7Zf9i2hq0YOznPUZJ8nkx3YofHa9iYwG/ + eTCm0RrNsT6v1FdoXYnX9p+O//brl0d3V2dA49PZC6jh84fogaoGk+J1LcpCPWbGux8CarDkpdZW + usRKSKVgEk+PBRTTZsSLWth33Wp3M2DsPYOZWxF3Xx6c0cntP3Rci6E5f99XwDc/os/atBKeLEsF + auu8ZBs1TfnEi88Z3RuO6dK4JObolPQBT+K6JA7lKOOg1zZMQZyRjWt4wWzMYwqJtRjJtz/x1rvO + LSyqR8V21sVE37yjgMs9ZsQ93qdk1lbp6uc/f/lfMjSLoEE7f/vCcnZ9JP0qXj7A5Y89CUXL6Wj6 + iR8QXmzKwuPLTXiwf+whlz4mSb68O+szCtHyVtfE2fXzl4/wSlne7jWmfLc1f35LZfRU0qWwLbOV + JjwF8Bd7n+HomaJxGD8PsIzcw2K2wwE7riYBUrjC9//iZB6bjwZeTFPiINFNxkubrGG3Xh8Y5rs7 + 75dKXoLxVAM8fd/fjRdWlvBeCXviqcMzGA5y48L2aByJJyOzE0DPBHDe1Ys2ov2ovzz/ghHv7Z8+ + +XTp+uKX/7LwTuV6/uZxKJeIzXTCFt03H8jB3RwqsrH+BwAA///smsuSojAUht9ltk5V24AkLOUu + IATBRtgBIhpaWwQSSNW8+1TwDWY1C54gWSTn/JcvtDHPKyKF+3WCSlHBPM9vuX4VeP92jkfvqxUA + Pw9t+3QXT8A+UHBRreQ9rynXn/y/jcRHz7IYO+q7APjGmlh4lPFQ3ma9cwIEfZSAsbthlO/5amuJ + z6bzLa3hQwwPKDhWYcGsJvqRuV8idndpPIpDxwI5qmXk8z6TqvTUwcjqV2gvCY7OQgtrsH4Vz2Dc + GYd4Ouh9+d7HyIp2L/bQDoLionZAWxtFOq0qLYdQzAeikTDU2Vas74rgXTOkkW8N3yk+7fn9beSa + 6SOeFKi0cKMlq0AujbDg/eQgf0IBk13v5/FkX88WsDbDiThKb3rdxjUHyN8TceLXoJMTywO40SZA + vGbTeGOVB0c4+x1Lvep43dMuBc41xWSL1J+YitG4gupTxVwvSqDjeRpMv5AaCFgKMHsq7hFm16FD + duYkXpeR+ibjCQXEVAQ1FnsqCVAoDBIIx1sbt/N+t4otIEbwUevMOccplBpr4vNAKub9O+eJ3E9A + j9H7kSq39eeT+BdrzZ5BmDZv/7R3AcT0KJsa70f2/D3i14T22QDHbNUEsM9CPCDbtCDXP8S4TvFr + anaWC4EsJWhbaXlcrfPvBPqJriPen+tsd/ECUPhJGYxF3sZj2pcRlJV6DF4/5ymekglFQNhQbxiN + DAM2vJo94P0c0UTHfIm8v5a7h/0RfPN9OZzYWM3+lJgeOAI69+f12aWzn2TD0FYV9NfdhVhqjjAz + n60B/GSbIENixOskv0/gr5kK+PP7H4iCz4UoWIiChShYiIKFKFiIgoUoWIiChShYiIKFKFiIgv+Q + KPgLAAD//+zdMQrCQAAEwP5eEdIfhFTiX0QOcliYeMGcYOPfJUbEHwgy7Tb7gWGXKCAKiAKigCgg + CogCooAoIAqIAqKAKCAKiAKigCggCoiCX4uCniggCogCooAoIAqIAqKAKCAKiAKigCggCogCooAo + IAqIAqKAKCAKiAKigCggCogCooAoIAqIgj8XBaFpDq8XhKkMeVxhQM33Gj9UIKYhxa7rt6uE25JO + ud2/BUI7X8s012Mt53xZ1vGC3WYN2lpqGr/zsFY9whMAAP//AwDXFfCAhGEAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b199e7cf78d-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:18 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '43' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999982' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_11005c81622ce1d9b6fe05c68fed52e8 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml new file mode 100644 index 00000000000..baa3ceca13c --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml @@ -0,0 +1,543 @@ +interactions: +- request: + body: '{"messages": [{"content": "Generate a question for the given answer and + Identify if answer is noncommittal. Give noncommittal as 1 if the answer is + noncommittal and 0 if the answer is committal. A noncommittal answer is one + that is evasive, vague, or ambiguous. For example, \"I don''t know\" or \"I''m + not sure\" are noncommittal answers\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"type\": \"object\", + \"properties\": {\"question\": {\"title\": \"Question\", \"type\": \"string\"}, + \"noncommittal\": {\"title\": \"Noncommittal\", \"type\": \"integer\"}}, \"required\": + [\"question\", \"noncommittal\"]}\n```\n\nDo not return any preamble or explanations, + return only a pure JSON string surrounded by triple backticks (```).\n\nExamples:\n\nanswer: + \"Albert Einstein was born in Germany.\"\ncontext: \"Albert Einstein was a German-born + theoretical physicist who is widely held to be one of the greatest and most + influential scientists of all time\"\noutput: ```{\"question\": \"Where was + Albert Einstein born?\", \"noncommittal\": 0}```\n\nanswer: \"It can change + its skin color based on the temperature of its environment.\"\ncontext: \"A + recent scientific study has discovered a new species of frog in the Amazon rainforest + that has the unique ability to change its skin color based on the temperature + of its environment.\"\noutput: ```{\"question\": \"What unique ability does + the newly discovered species of frog have?\", \"noncommittal\": 0}```\n\nanswer: + \"Everest\"\ncontext: \"The tallest mountain on Earth, measured from sea level, + is a renowned peak located in the Himalayas.\"\noutput: ```{\"question\": \"What + is the tallest mountain on Earth?\", \"noncommittal\": 0}```\n\nanswer: \"I + don''t know about the groundbreaking feature of the smartphone invented in + 2023 as am unaware of information beyond 2022. \"\ncontext: \"In 2023, a groundbreaking + invention was announced: a smartphone with a battery life of one month, revolutionizing + the way people use mobile technology.\"\noutput: ```{\"question\": \"What was + the groundbreaking feature of the smartphone invented in 2023?\", \"noncommittal\": + 1}```\n\nYour actual task:\n\nanswer: \"The capital of France is Paris\"\ncontext: + \"The capital of France is Paris.\"\noutput: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 3, "stream": false, "temperature": 0.3}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '2795' + content-type: + - application/json + cookie: + - __cf_bm=nxPGrcXhbymyBih2t7CgJx2kdUAj2c8FEUM6SgQuXQA-1734303554-1.0.1.1-0gC9PrWCtGQtTp5RVHnIIDJR3yD.DgFli7YVq20QDgOcIMxDrBJn9wXO3e.Fak8_HP_bRTKACpjVZuouGgX1ig; + _cfuvid=GidLrZtG7TkkHj.qFUWUeIOOu8KdGLixVP6dtmMZgV4-1734303554457-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//7FTBjtMwEL3nK6w5Nyhpu+kqF7QXBHvgCCoEJa4zSQyObewJsKr678hp + 2qTaReIOlxzemzd58+zxMWIMZA05A9FxEr1V8QM693P/7vHH+w/Np2zztE8P+zp7kHxn3j7CKijM + 4SsKuqheCdNbhSSNPtPCIScMXdPdZrtJNnd32Uj0pkYVZK2leGviXmoZr5P1Nk52cXo/qTsjBXrI + 2eeIMcaO4zf41DX+gpwlqwvSo/e8RcivRYyBMyogwL2XnrgmWM2kMJpQj9arqjoW8H1AH5wXkLMC + PnacmPSMOmSCW0lcMdOwN45rga8LWLECtNHC9L0k4iqoklNVVct/OGwGz8OcelBqwk9X08q01pmD + n/gr3kgtfVc65N7oYNCTsRAtxM+SSP8nMSWx/veSiBj7Mi7McDMvWGd6SyWZb6hDw2w9LQzMe7pg + s4kkQ1wt8PsLcdOvrJG4VH4RLwguOqxn6byffKilWRDLI3zu5qXe58mlbv+m/UwIgZawLq3DWorb + iecyh+EZ+1PZNeXRMPgnT9iXjdQtOuvk+cI0tswakSaYJniA6BT9BgAA//8DAI+legpSBQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b081a3818b8-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:16 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '595' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999320' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d2c1df90b6180c0a455b1e3d44a12030 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '114' + content-type: + - application/json + cookie: + - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; + _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1R62a6zurbm/X6KpXWb2gpd8GDdESCQ0NhJIF2pVAKSECCE0NiAj867l8h/dKrq + ZkqT6QkGD3/d8H/866+//q6T4pH2f//z19/vvOv//h/ztXvcx3//89f//Ndff/3113/8fv5/Ix9V + 8rjf80/2G/77Y/65P8a///lL+O8r/3fQfO/usyW6cQiS8fqNqTqZDmFB9CzQZNaOAvxD18wig+yx + MtBK1X4rAQvr8BNO1X4QQBKbG1uX1xwNXnmJIVDSKxUgH/kkaS4FIigf4ir9J+GqqVbI3199XDcX + uajvtNRBWz01LMjftzeRozeg9egZzOXXtJjug5tDdO1dshW/q7BfSw2F69jrDBfHVzIF8SNCmVc/ + mL4LcsR72gpoO778eT4GF+8HyYLbsz6zLc6WbX8/SDZoi5YQm5Kl16XjykCR1Lzo8hI5RVfULAM1 + 2G7IzY8HNKqrAqB1rSs7ewprp+Pxe4TmuxswYGInw+pRZmgp1BzLrn3xuHURclAP+ZbgbKd4NIjP + Z7ADd0s2q3MecnzWDY0a4gMvF/d12Itn24fxmafEum1eaMhbIQMosxgP6XFop29oK7DgboPDe/nm + LTkiHRbZ4stcFB9bmpzUCC3Ec4ulnVeGVBTiWs1PTko2b/Hc8kXxluCwLUeC3fZk8hUyUgir20hc + +WO34+OUVdDot454R68xR6hfEaiK7WHqLb5FfT/YErqXR5eZPdTeoCaXXG1k9UIb59qF3f2NFogU + yyWehvclpO/ltVkthYYTV09lkwdsopqrZjqV8WntyYq/2cL0bY8UoQdGbDc1GDmP+kAOe10vpD3L + AGx1T5hX+0Eo58s8RocycpjFYuRNMOmNhjaWShfcKcKBrr0jSj+uybzrpQ07qoRbKMs6IoHSbc2h + iFcKxGKeM3N5jdCQfLcA/jhpZLMtP6hL754CR/oA4q2igvPL7WCjGj22xKlXZTH4bklhNEeHrkb3 + lPAKJkXL9T4mdo9q3pP7oQTZEnfMuMY7k+/umgVoVUVUk6tPMoivSIfs3QKe+pNXDBtf6lB/YhdC + FvdX+JsPOpRnh0o33U46NfiqoGp9hYXjrkvGg/wawNAvE7HwSi/kkzMA7CSjYuZRLBO2/FwjuCnX + hJGV+jK77NBF4B0si0QK26BJ2NwjEI6NQzbd2HlNRNI92iySFyO7rV0Mb9G1kJVkH7IzxTgsvFOe + L/Hm6VCl1HQkLfREQOI2Vsn6lQYhI0dkqJFd7EjwRkPLa6GOwPPpmhmXaJuMKdqfNdnHKV4mDm+n + ek0n+IaHN7HXHW0nenMewLRNSzxjtQ0nFLUpNF9voJNweJn9a31dgHBPBeK2730xkGVg/96fLpxk + F7JPoWNNJl+J6Dgswm67vrjgasuY6HJVmyOEgKERuzM5FMfcZLa9UqCJRkqHtpzMjjRPUKVjMeBX + ugq8QTkJDzgEg0n2R6kouHG5GMjZSyFxLwENK8/3MBDZrtl6FEjBPxdvCxcteJMgepqc2/aoqoHy + uBLjs2vCaeKiC+XKOOM6887tdHZ2HcpUc4EXix6F7QoZD8iEa8G274Ns1h9yPGt1qWyIrTiNyR26 + pKvSUwgzdlsnZGm9pkAVd0vco9Nzattuoy4e04f9vudwkKYc+lfas7un9uFUSlkFl4PusHsqDt7o + R76urks3JM7tY6DJ3Ly30F0/e+Ipa5IMd9oZiDlnhVlRvi6kgcY24CEJ6FJaeO34ukg1Mn3/Sfzd + 7o24WcsuCrreINaFdHzisftAy4fNiHO6Knys9WUGzW2pM33zunnTXntUEOXHJ1vbyo7TzAkG9BGm + ngVy5YSC2PiWGu6HM0mXyiX5LIq38MMnKqtbLfxIX0sCpjktIWRzLwZwIxWg39nM80jUStWjSuGj + 3W5kw921KT3WOUWy76fMJ9LB5O1XH7R5PkR3P6t2mBYhVd/de8vcgpTJsGkSFZ6ED5h3Nk+4/zEa + 4EZyYetndU2+mEYYRAoaO+jPiPfyxiyRmkYi2ZzuUTLgqyxAYvVHit5c8frLRa1QZwWYal8KRXXK + IIagtM94WHyffDSavoEvrL+41eTSm97La41mvCdOMvacviTTBvuQe+yHRxRUhYK+8gO2Nk67gq+m + +Lq6hZuR2A/b5h13TAmdxf3IrHD58oa5vlGhah+2U/enGe90QMNbHIgrPO4JP6yHK0o/W5NE+Jwl + o/42qj//T8SthNg3lFxYVaXBnO+AvL4kpQWumuvETOie88zRDJj5izj7oxLyASSANIl7YvSntpi2 + WSyo5OYdGF59YjQ9h9oADpZJ9u7nVvSP4uErvPUSKq7rYzI879/rSr4FHhZeCTVHSLs9qOrRp+Vr + gUP2TJstXCXlSMILsvjI+UYFGk42nfdTMpk1UVD6WoyY73yzEOOX7v7Gs3BPUDJMi4SixhmWbK5/ + 3rqWe0XZIVTx9OncduyH+KzW93zEKrfsZGrwOtLm/Ukcq8vC0ZJ1DDSNJ7J5fsWEBUztQKQLjZjT + s04YOXIdSJa9mad6cTJdrl8BUhgfdNVZEZomrrmoR/p6Xt+A860fATwmbhD9bJOQS+S4Rdp9/2B4 + eKlhP6UZRjN/Mt0WgnaQK9lCuegLWFQDBY3hYTK0UhJOzCovZjH9+LIS0zddbt9VMp605gy9q36Z + +VYnj7pqq4OxciT6JQ54Y/Aut3/W/6C6iPft2XWhX9wTtl6Fm5BXm8KAnKQ6Od6zb9uJixVW90zq + mDPlm2S0gh5UiJcbFsRpzKdpqWL1JSodHoZm4j1b9A91o4LDHm9ybSksF0d43k4bcp+yindusKLI + P1YB0+fx7My9CcS4RMRrvz7njtkpqPArh+DmdDOnS/VM0WN4r5m/bgaTf3BoaX2zuWExY2dzEnI1 + h9Pic2KbjJ29SX+vBM0n94I4eV8kfb7wFHgvxh378X2PvTH94dWMTyuTyW9ZAIo+OV6N6redmqG0 + EVvVB0KWkcwHS7oC0m1FwEhCuJUFDA0CiSZ4afsHs/vhJyVXE8tYWYfSimxj+ICrE3LEjin/7u+9 + HzYe01Xv8X1S2LAynzcKKebtRB66BMaFbSg/b/Ymm/EfJe3hROwqDU2mRBpe7fBLxSvl6/3RUyAN + T+cPf0vJST2jmY+Zva0tc3hnESC1oDuq7OKl2Vd3GUMXJgu2uYQ7ky7yRYnE0+1LgtM5Tj5ttqew + xnLNtt6eI66xRoWL31jM0kQ7mVQ1xDCowURlwbMT+dMqe7hR2aYvWhZoSCjO1PvR3hEcFbonGYK1 + AH3d58SxpTZkD+O2gMXWL3/43fapfloAkx/7mS/dhF2bEcOsD9nNXSft6LRZo23Kq0XuZrk2pdvD + V9BBoTvmXxTHHGtdzkB9Ky6GT/81OdN9A/pTf2Hrm702aSllJUh41xMrMFvEhzjFgBFRyWaRGmhi + 0VqFX/2706sy+ft57xA+3iViuqbqDT89+7X3EsGz/h12DziDcqdvWijLJ6f1upqAkWWJl69QQH2K + 9pGGfSkmjmikiAfJpgLho+9YPL1sc/KyIdLyQ2OwgAp3czzkUKtG1Ats565R24W8z1RZ3uZk1qec + B0o+we5+fLFgF+TelCh3+/e+FLldFA7iYsTa092esOQngzeeDAZoutwPDIM+eNOq5ramaqxivn8b + i6nolQztPDuj8OPDqK8pDHl8xiLv14nEllEHStNHbE3Lgg/S/nCF5aexKA+vVjFCWu7RXB9ke2Tf + oqduXmpO/WVU/qYW4hWoCjoOzYKOVR6g6XJ9CQiWF4f4EdP4GFNt+NUXrt8H2aMyzlXYmSwg7kUz + 2p8+VvPv88Hs82f0Jns8PWAtRRGz3Obcsvchs9G8n5i3P50Stu6sLeDjU6K587TD7iR4OqoXKCd+ + EwvtRCpFgR/fbe+P9re+FKTOsSjM+oUJ3UFSYSx2xNIfYjuZhxyD4mUl01/XgznzcQNqvBWIJ2Az + /P2O/FCsCDb958yXbqnScLCZs312Cdc3kor4bmiIP66akF/ytatlOc6JX66X7bgLaf7DF3pByQGN + 1++xg+z89HE58iGcDliaYPmpLaa/0SHpnx+9g3Pbp4RIi7ad9aWi5aFk4iF6lu0An8nW0p3fsV/9 + jmc7fiB6KDfMlN81H6wrH+BZv0SGT9oZsXCqjrA/H65Y+N4tUzCdsIHn7bJhO2NRoK8V9Atgfu4y + 67jzw17bfzDQ98KhXf6pi9eNZIb2yauOEOGgJ4JZKhJgsXoS91AG7XSO5AjG+zn55QV8WLt1rbru + tcalLTctj1dWBJ6TMSxeb18+WfiEwYdPw5zyXCajvPFKNNcb8RLB5aOVhzY4/mNPJ8XqPTbq3xwg + fE/ErEMn6X56zZkCi+2WRuZxS70coe1bl1lhqvP+KUgR2DRf4aftH7wRy/4RDZ50JMGhwOH0coc9 + zH6GFju/KLhE4i1Y0qOf13+c/cKo/PASa8aha0dxey0BPRqM5bJR227jLzrYbZUnlhTHNen3AwYa + 7FYnJA2icKybwwBFD8qvPkIuZumgktTNmTNOB69/dcUEr8vyRHT/TovvrL9U5/ZYEt/JLCQhD10h + PQ5rEv/8eVTEW4D74sx8zeu8qXWFMwjJkeOBXxbetBHyB4xHTWbWofK9sRRXZ/TzM/aHOcUAH9UG + vU9lrJRaxsdfHtJMbMMs+bvxaPrIr0g/OyWz62Pt0R9/zX533m9iwrVA32pz/kOHXWAgoTHVCah6 + 3lH5FVM0deOuRHJWPpn57R/mcCn0q5ZZoUd2yrf1OiPVJ3hx2pKAq1oxblrTheB0YmT7PlzMge3Q + +ee/sZxGOOGRhR8w5ynM7Rdv3j8f+wY+qVETXbg/+FTzlY0OBCyW2DvBG/GN2Wjej3TqJi+UZr0A + wijJbBf073CclgbA/VQU9JentHHCtuhWGjdmH15Dy81iVan7D9aZkQYnr5StNILiHflszg+8Yb9/ + H5FUrSZGivsTDUa4p4hegoz98pUhkvc6xLfRZ2czN5PJ30o+0rZq+ke/98jzXTjxqSbePF/uf9wG + wuOa0ekS1UnTlMxYbXTDIORxWYa/74+qlGbEQw/MqU5fqha6/pGYh5yh4apdK6QwpWLnh6aF03lK + M6jP3oPtiJN6Ux5m5z/7ybfMTTjtxgl+zyMVfideb6SDASTvKnLYRxLqBWm4auBNp59eCtnX1nNg + maWx3WPjtHxdRJaq+KPGdhZKTN56baWW2N4zz1jVyR+/O+drBOtPwRyuF++MZv1HZj1m0tn/o2ZT + LoivTmc++5cY0cA32HaRmIV494cMpOJxZYZxe/OfPljtz+H1D58OperUq1Y0HsxaVWk4xHgY4FxZ + a5LwbV5M+xhJSHraJ7azR8bH45PVaBnuDdrOfDHeJndCUfhSmGGAbcoZJxKU9YYS57JYc+k+GJmG + VLEnbpQVHg+SoIR0dbaIJ25uqBduCYa1kp3YmiqqOeXpNYZQ5Zx4UJJicq/6XnuScWDnZhmGTTeu + S0itdUyp3JhFPShD/cefWupq4N9N67molKQTXdCyMCdjq25V6fgayCZyl3xa32BYzXkDFY9B3bJT + k3QIbWwVPx+XZzImfqhDHgomHo5S0fLFafDR+01EvErJEU276DTB7D+ZmTdVSNPPMUUnPwQqaGIV + jqOnWwi84TTzd4lm/Smgnx7DylAmza2CBVx2Nad8etZhc0F+CXL/CahqvvVkADdVoWNQMGOlvlBf + OYdcrSq2IIbLJo8txQDQzrMyOqmjEtLIfGJI2vCERfrJ0LD6pNUPb7Hs3Lxi/OU9j6I8stCyTXPc + V+sGvp/28wePR398+QAfTcCLN7kWw2VV2aB+TyXZpshFP/6FwrmKs/6XTP6r35N/AGZadmG23ahI + 6EZFm2bu59ZOPtL/5Lu4TsTGaz4WmmBYJtasf0wkf9phDwdVcdn+sUz4hDwUr7YSfbA5X/Ck+XvC + 1puuZNNHXTI2QWyjk6cX5GaKccLybZwjQz9NeJrzMPUcLSOY8YbY+fdj9tV+V8KKYUq2p+Up5PZk + UQiX2ob5QyXysU96/4cnP7+IeG/edHXDS+EPvrYNHiqYXkZFtt4+5EN7ihXIK8MiplkYrSxfXR+G + q7gm61N8SfhGPFA4TtKSCh+hMMflssuRtHmtGaZl4fHi4FY/viVWtl0V0/K+MtBRazs8rGnvtWTY + Y4DwMzHTX5Ji6GMNYPdqSmJ1Pfnjb8F14/qnr/hYN7cBZv1HF20dFNOqRjaSDvsLuSW7ozdMmZSp + b8+/MEcTPuYvf4doeQjwoOzqYnpJngWOn+6ZIRxeHteC7RZumFFi3tbv4nub3AHN+Qlztb3TDsZX + itHmbgxso+1wyGvtGqveuX8yf7nOeDPsQQVciQrbGKji32NbUnDT6svcm2kmLe7eHcz4y3amqIaD + GrxU6M43guUqI8m4fbYlisJCweN4YOFYlAcM9eroUJ4exoTepsnQNEkviLUpJY+yhpWwxmJN3Ju6 + Ql14UwVIjOLAtv5ZaEdOt5GS7nBHtehpIumGGgvNeS+JZ74b/GG1QMGKjdh6Glf+yw8BvfGRLmc/ + 2q9cWoN0YQaxb3qVTPPz0CXzZYLnPGQoiy+AHJkrgs+9Z4rCLfShkZULVYWHFvbOQDHo80yFC+mQ + vPxczwD22ySmbNTFuKdjB/V6TEkadLY5ZvYjA7jDmZinU8L7U7bZ/vIcdrucbXNizadEkZBmZK0M + G3MULGqhGxJF9vNf7+s37kCNtJwu5jyIo1Pcodk/sY0aXPn3l9ekq8hiIdnF7ZDeTQUkWYiZ03X7 + lmvHDdWW02PCe8vJk3Hsc+EPHox8obacRqMLiy0uCTmvuTeJwrEBub359PHwdS4WtmBov/6CQcRj + 8TX36kL96enF5h0V/Lz/GjCa3JnzrzMabSiOEL7d66z34mIKqiRGw1seqDAVZdHoB5+iZNle6DTz + CSefevvLn+f+k4EmJfN02CePZq4XklDk8RguGnnP/atb8sdfzHiCFcIjr3Py9RmqdrVn6VCJiN9L + oOjZM4f88pgpXyBXLfuDzjaNpbd8rlckk1aiy5kPvkPqdnBxHtPcz6nasezXexQ6ZM2M5nY2R3FZ + pyD5SvTL39rx6ok5XFaDRx7fh1FIeWMb6Gwsr1gLdGp2IX/nYGcdop2hla34UMwIrHtXM2PW4xNM + 2xp839ixTWg9zcmfrim6RPcX2T02n6KLL5n0618Qa+5fdGryzOB8OcVse726RacqQgfvYpESN1j3 + 4RCn1RWSLK2ZJ11p0pFlYMFRLyqc5Lrvydz2S2B7NaDyzcqLcd2rzQ9PmbVcjiFF32sFM96yzeps + hPJmsZ5++TxV3J63060CgDE+uWTjPD7mSN2mgvjy8dgvvxdyoazgbp/5f/k9nb4Udc1PJubye8vF + 5Dv4YFm7iHnLg8p/eh42zfrJgt2QJ5NcXA1IUbZh21EKwqkLyxjNeRPVrGbNpa0h1nDB0oaQw+PI + x/ah+pAK7sTW4T3zJi9Ce7Ukp+/8fQNPVjL1CMWKYqwutSMfjHQ7gN48KzLXF+8Eab1F2+XJZT9/ + SX/8fKZdzwz/Gno8HcZJ0ySjIKYqRgW/weEMefpgzMdF2g5Xzz5qanoWiXmuh5DZk9+huT+J5Tnf + 43mj7ZErRpTM+iHpnNvtiAJd63E989VEe6EC1EsL5qYSS4bzpsVgq0fCdmfzUPCkJ3skX9YuO+z1 + rOgzpRbgHd8cZm3KsylfmxVG83qTzRR3nG++aQMvUe3+9HO50N0k+IbhG6ufqSyGrLqliOS0IltN + KcJuBesOZn1I37vFo+0sab/Qon7jk91bCYuRtA4GQj4w8xPj3f2ABfRbz+/MP115KRW4JIJATGW7 + S3hvHnTNiQ495pZ2K9jpVJXoW8UYo7XJC77TLwqc3ajFE611T0qP7zO00zqY+ZdzbhxvKcz3p6K2 + MlrR80KAm3eMmOPasjnOeAOHYDLJr98z+7UUAQkYRXx5M9kznGwt/94fdIi4yofzITsi4f4QsJYe + CZpefWige9yJdPlsOBpzMZHAIajBYuQ+0bgLq3xFacWZY4StSdNjH8ExsL0/7zPi6JsjQ5bucx50 + aFkVX7cw62UMs974ujhbwN+/UwH/+a+//vpfvxMGVX1/vOeDAf1j7P/930cF/h3f438LgvTnGALt + 4uzx9z//dQLh729bV9/+f/d1+fh0f//zF/pz1ODvvu7j9/9z+V/zg/7zX/8HAAD//wMAiTuukd4g + AAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b0ccbbdf78d-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:17 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '108' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999993' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_8dc90305ed6ad294095c7572ef16918f + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30], [3923, 374, 279, 6864, + 315, 9822, 30], [3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '192' + content-type: + - application/json + cookie: + - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; + _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1RZyxKqvJaen6fY9U/tKhGQLM4MAUG5JCp465GgIiAilwSS8/JduKu6qydWiRFC + stZ3y3/+9efPP3VSPNL+n3//+eedd/0//zVdu9/62z///vPf//rz58+f//w+/9/IR5U87vf8k/2G + /37MP/fH+M+//0j/e+X/Bk337j4bYpj7MBkv3xvVuOUSFsbPAnGrdlUQH7piNhkUn5WhXmrOWw1Z + VEefiFe7QQJ50VzZqrzkaPDL8w1CNb1QCfJRcFn3KBBJ/RBP7T+J0CytQsHuEuC6OStFfaelAfry + qWNJ+b59Tg7+gFajbzJPXNKC3wcvh/jSe2Sz+C6jfiU3FC5jbzBcHF4JD2+PGGV+/WDGNsyR6Gkr + oc34Cqb5mGJx38s2XJ/1iW1wNm/7+152QJ+1hDiUzP0uHZcmiuXmRefn2C26omYZaOFmTa7BbUCj + tiwAWs++sJOvspYfDt8DNN/tgAETJxmWjzJDc6kWWPGcsy/ss5SDts83BGdb1afh7XQCJ/Q2ZL08 + 5ZHAJ8PUqbl44Pnsvor6xckJYHzmKbGv6xca8lbKAMrshof0MLT8GzkqzITX4OhevkVLDsiAWTb7 + Mg/dDi1NjlqMZotTi+WtX0Z0Id1qLT+6KVm/F6dWzIq3DPtNORLstUdLLJGZQlRdR+IpH6cdH8es + gsa4dsQ/+I01Qv2KQVMdH1N/9i3q+96R0b08eMzqofYHLTnnWqNoZ9q4ly7q7m80Q6SYzzEf3ueI + vueXZjmXGkE8I1UsETJOdU/LDKrg48pX1GC9Af5tDxShB0ZsyxuM3Ee9J/udYRTyjmUAjrYjzK+D + MFLyeX5D+zJ2mc1uyOfAjUZHa1ujM+EW0UBX/gGlH89i/uXcRh1Vow2UZR2TUO021lDclircFnnO + rPklRkPy3QAEI9fJelN+UJfefRUO9AHEX8aFEOfr3kE1emyIWy/LYgi8ksJojS5djt4xERVwVc+N + /kacHtWiJ/d9CYq92DLzcttaYnvXbUDLKqa6Un2SYfGKDcjeLWDeH/1iWAdyh/ojOxMyu7+i33zQ + vjy5VL4aTtJp4VcDTe8rLB22XTLuldcApnHmxMZLo1CO7gCwlc2KWYdFmbD55xLDVb0kjCy1l9Vl + +y4Gf2/bJFbZGnFpfY9BOjQuWXdj5zcxSXdoPUtejGw3TjG8F56N7CT7kK21uEWFf8zzOV4/XaqW + uoHkmZFIaLG5aWT1SsOIkQMytdgptiR8o6EVtVTH4Ad0xcxzvEnGFO1OuhLgFM8TV7S8XlEO32j/ + Js6qoy2nV/cBTF+3xDeXm4ijuE2h+foD5dL+ZfWv1WUG0j2ViNe+d8VA5qHze386c5NtxD6FgXWF + fGVi4KiIus3q7IGnz2/EUKraGiECDM2iO5F9ccgt5jhLFZp4pHRoS251pHmCJh+KAb/SZegP6lF6 + wD4cLLI7yEUhzPPZRO5Ojoh3DmlU+YGPgShOzVajRArxOfsbOOvhm4Tx0xLCcUZNC9XHhZifbRNx + LhYelEvzhOvMP7X85G47lGnWDM9mPYraJTIfkEmXgm3ee8WqP+Rw0utSXRNHdRtLuHROl6WvEmZu + N27E0npFgarehngHtxfUcbxGmz34h/3Wc9jLPIf+lfbs7mt9xEs5q+C8N1x2TxeDPwZxYGir0ouI + e/2YiFvr9wa6y2dHfHVFkuFOOxMx96QyO85XhTzQmwN4SEI6l2d+O77Oco2sIHiSYLt9I2HViofC + rjeJfSad4OLmPdD84TDiHi+qGGtjnkFznRvMWL+uPt/pjwri/PBkK0fdCpq54YA+Eu9ZqFRuJC2a + wNai3XAi6Vw9J59Z8ZZ++EQVbaNHH/lry8B0tyWErO/FAF6sAfRbh/k+iVu5elQpfPTrlayFt7Lk + xyqnSAmClAVE3lui/RqDPs2HGN5n2Q58FlHt3b03zCtImQzrJtHgScSAReeIRAQfswFhJme2elaX + 5ItpjGFBQWd74xmLXllbJdLSeEHWx3ucDPiiSJDY/YGit1D9/nzWKtTZIab6l0JRHTO4QVg6JzzM + vk8xmk3fwBdWX9zqSunz9/xSownviZuMvaAv2XLA2ec+++ERBU2lYCyDkK3M47YQS367LK/ReiTO + w3FEJ1xLRqfFbmR2NH/5w1TfqND0D9tqu+OEdwag4b0YiCc97onYr4YLSj8bi8T4lCWj8Tarv/8n + i42M2DeSPVhWpcnc74D8viSlDZ6WG8RK6E6IzNVNmPiLuLuDGokBZIA0ufXE7I9twTfZTdLI1d8z + vPzcEH8OtQkCbIvsvM+16B/FI1BF6yd0saoPyfC8fy9L5Rr6WHol1Boh7XagaYeAlq8ZjtgzbTZw + kdUDic7IFqMQaw1oxB069VPCrZqoKH3NRiy2gVUsbi/D+41n0Y6gZOCzhKLGHeZsqn/RerZ3Qdk+ + 0jD/dF479sPtpNX3fMSasJ2EN3gV61N/Etfusmi0FQMDTW+crJ/fRcJCpnWwoDOdWPxZJ4wchAEk + y97M1/xbws+XrwQpjA+67OwYcS50D/XIWE37GwqxCWKABxcmMU4OiYRMDhuk33cPhoeXFvU8zTCa + +JMZjhS2g1IpNsoXgYQXWqiiMdpzUy9l6cjs8mwV/MeX1SJ90/nmXSXjUW9O0Hval1lvjfvU01oD + zKUr0y9xwR/Dd7n5u/97zUOib0+eB/3snrDVMlpHoloXJuQkNcjhnn3bbjFbYm3H5I65PF8nox32 + oMFtvmbhLb0Jzuca1l4LtcPD0HDRs1n/0NYauOzxJpeWwnx2gOf1uCZ3nlWi88IlRcGhCpkxjWcn + 4XNY3EpE/PYbCOFanYqKoHIJbo5Xi5+rZ4oew3vFglUzWOKDI1vvm/UVLzJ2sriUazkcZ58jW2fs + 5HPjvZT0gNwL4uZ9kfT5zFfhPRu37Mf3PfbH9IdXEz4tLaa8FQko+uR4OWrfljdD6SC2rPeEzGNF + DLZ8AWQ4qoSRjHCrSBgaBDJN8NwJ9lb3w09KLhZWsLqK5CXZ3OADnkHIAbuW8ru//344eEyXvS92 + SeHA0npeKaRYtJw8DBnMM1tTcVrvLDbhP0ra/ZE4VRpZTI11vNzil4aX6tf/q6dAHp7uX/6Wk6N2 + QhMfM2dT29bwzmJAWkG3VN3e5lZf3RUMXZTM2PocbS06y2clWhyvXxIeT7fk02Y7Cius1Gzj7wQS + Oms0OAeNzWx94SRc0yIMgxZyqki+kyifVt3BlSoOfdGyQENCcabdD86W4LgwfNmU7BkYqz4nriO3 + EXuY1xnMNkH5w++2T43jDJjy2E186SXs0owYJn3Irt4qaUe3zRp9XV5scrfKlSVfH4GK9irdsuCs + utZYG0oG2lv1MHz6ryWYEZjQH/szW12dlUVLOStBxtue2KHVIjHcUgwYEY2sZ6mJOItXGvzq3+Ov + yhLv571D+HCXieVZmj/89OzX2ckET/p32D7gBOqdvmmhzp+C1quKAyPzEs9fkYT6FO1iHQfyjbgL + M0UiTNYVSB9jy2785Vjcz4ZYz/eNyUIq3a1xn0OtmXEvsa23Qm0XiT7TFGWTk0mfChGqOYft/fBi + 4TbMfZ6od+f3vhR5XRwNi9mI9ae3OWI5SAZ/PJoMED/f9wyDMfh8WQtH13RWsSC4jgUvejVDW9/J + KPz4MO5rCkN+O+GF6FeJzOZxB2rTx2xFy0IM8m5/gfmnsamILnYxQlru0FQfZHNg36KnXl7qbv1l + VPmmNhIVaCo6DM2MjlUeIn6+vCQE87NLgpjpYrxRffjVF67fe8WnCs412FosJN5ZN9ufPtby7/PB + nNNn9LkzHh+wkuOY2V5zatl7nzlo6ifm747HhK06ewP48JRp7j6dqDtKvoHqGcpJ0NyklpNKVeHH + d5v7o/3tLwW5c20Kk35hUreXNRiLLbGNx6Ll1j7HoPpZyYzXZW9NfNyAdttIxJewFf2+oyBaVARb + wXPiS6/UaDQ4zN08u0QYa1lDYjs0JBiXTSTO+crTsxznJChX83bcRjT/4Qs9o2SPxsv30EF2ega4 + HMUQ8T2WOcw/tc2MN9on/fNjdHBq+5QQeda2k75U9TySLTzEz7Id4MMdPd0GHfvV73hybg9E9+Wa + Wcq7FoN9EQM869eC4aN+Qizi1QF2p/0FS9+7bUmWGzXwvJ7XbGvOCvS1w34GLMg9Zh+2QdTruw8G + +p65tMs/dfG6kszUP3nVESLtjUSySlUGvKiexNuXYctPsRLDeD8lv7xADCuvrjXPu9S4dJSmFbel + HYPvZgwvLtev4DY+Ygjg0zC3PJXJqKz9Ek31RvxE8sRo55EDbvDYUa7avc9G45sDRG9OrDpyk+6n + 11we2mw7NzNf2Nr5AG3fesyOUkP0T0mOwaH5Ej+dYO+PWAkOaPDlAwn3BY74yxt2MPkZWmyDohAy + uW3Alh/9tP/j5BdG9YeXWDf3XTsuNpcS0KPBWCkbre3WwayD7UZ9Yll1PYt+P2CiwWkNQtIwjsa6 + 2Q9Q9KD+6iMSiywdNJJ6OXNHvvf7V1dweJ3nR2IEd1p8J/2ludfHnARuZiMZ+egC6WFYkdvPn8fF + bQNwn51YoPudz1tPOoGUHAQexHnm87WUP2A86Aqz91Xgj+VieUI/P+N8mFsM8NEcMPpUwWqpZ2L8 + 5SENZ2tmK9+1T9NHfkHGyS2ZUx9qn/74a/K7U78tEqGHxkaf8h86bEMTSY2lcaDaaUuV140i3o3b + EilZ+WTWt39Yw7kwLnpmRz7Zqt/W78zU4PAStCWh0PRiXLeWB+HxyMjmvT9bA9ui089/YyWNcSJi + Gz9gylOY18/eon8+dg18UrMmhnR/CF6LpYP2BGyWOFvJH/GVOWjqR8o77kfypBdAGmWFbcP+HY18 + bgLcj0VBf3lKe0vYBl1L88qc/WtohVUsK233wQYz0/Dol4qdxlC844BN+YE/7HbvA5KrJWekuD/R + YEY7iug5zNgvXxliZWfA7ToG7GTlVsKDjRwgfaOlf/V7j/zAg6PgNfGn+Yrg4zUQHVaM8nNcJ01T + MnO5NkyTkMd5Hv3WH1UpzYiPHlhQg740PfKCA7H2OUPDRb9USGVqxU4PXY/4iacZ1Cf/wbbETX2e + R9npbz8FtrWO+Hbk8HseqfA78XszHUwgeVeR/S6WUS/Jw0UHnx9/eiliX8fIgWW2zraPtduKVRHb + mhqMOtvaKLFE67eVVmJnx3xzWSd//e6UrxFsPCVruJz9E5r0H5n0mEUn/4+adTkjgcZPYvIvN0TD + wGSbWWIVi3swZCAXjwszzetb/PTBcneKLn/5dCg1t162C/PB7GWVRsMNDwOcKntFErHJC767IRnJ + T+fIts7IxHh4shrNo51J24kvxiv3OIqjl8pMExxLyQSRoazXlLjn2UrI98HMdKQteuLFWeGLMAlL + SJcnm/iL9RX10jXBsFKzI1tRVbN4nl5uEGlCEB9KUnDvYuz0JxkHdmrmUdR046qE1F7dKFUaq6gH + daj/+lNbWw7iu259D5WyfKQzWhYWNzfaRpMPr4GsY28u+OoKw3LKG+jiENYtOzZJh9Da0fDzcX4m + YxJEBuSRZOHhIBetmB2HAL3fZIGXKTkgvo2PHCb/yay8qSKafg4pOgYRUElfVNE4+oaNwB+OE3+X + aNKfEvrpMawOZdJcK5jBeVsLKvizjpozCkpQ+k9INettJAN4qQYdg4KZS+2F+srd51pVsRkxPcZ9 + Nl+EgLa+nVGujWpEY+uJIWmjI17QT4aG5SetfniLFffqF+Mv73kU5YFFtmNZ465aNfD9tJ+/eDwG + 4ysA+OgSnr3JpRjOy8oB7XssySZFHvrxLxTuZTHpf9kSv/o9Bntglu0UVtuNqoyudOHQzPtcWx4g + 42++i+tk0fjNx0YchnliT/rHQsqnHXaw11SP7R7zRHDko9tyI9MHm/IFX57WEzY+v5B1H3fJ2IQ3 + Bx19oyBXa3FLWL655cg0jhzzKQ/TTvE8hglviJN/P1Zf7bYlLBmmZHOcHyPhcJtCNNfXLBiqhRj7 + pA9+ePLzi0j01tXQ1qKU/uJr2+ChAv4yK7Lxd5EY2uNNhbwybWJZhdkqysULYLgsVmR1vJ0TsV7s + KRy4PKfSRyqscT7vciSvXyuGaVn4oth71Y9viZ1tlgWf35cmOuhth4cV7f2WDDsMEH04s4I5KYb+ + pgNsX01J7K4nf/0teN6t/ukrMdbNdYBJ/9FZW4cFX9bIQfJ+dybXZHvwB57Jmfb2gzNzdelj/fJ3 + iOf7EA/qti74S/ZtcIN0x0xp//KFHm42cMWMEuu6ehffK/cGNOUnzNN3bjuYX/mG1ndzYGt9iyNR + 65eb5p/6Jwvmq0w0ww40wNVCZWsTVeJ7aEsKXlp9mXe1rKTF3buDCX/Z1lpo0aCFLw2605VgpcpI + Mm6ebYniqFDxOO5ZNBblHkO9PLhUpPsxoVfOTV2XjYLY61L2KWtYCSu8qIl31Zaoi66aBIlZ7Nkm + OEntKOgmVtMt7qgePy0kX1FjoynvJbeJ74ZgWM5QuGQjtp/mRfzyQ0BvfKDzyY/2S4/WIJ+ZSZyr + USV8eh46Z4FC8JSHDGXxBVBia0nwqfethXSNAmgU9Uw16aFHvTtQDMY0U+lMOqTMP5cTgPO2iKWY + dTHu6NhBvRpTkoadY42Z88gA7nAi1vGYiP6YrTe/PIddzyfH4qz5lCiW0oys1GFtjZJNbXRFiwX7 + +a/35XvrQIv1nM6mPEig461Dk39iay28iO8vr0mXsc0isr21Q3q3VJAV6cbcrtu1Qj+sqT7nD453 + tpsn49jn0l88GMVMawWNRw9mG1wScloJny+kQwNKew3o4xEYYlE4kqn/zhdMsjgUX2unzbSfnp6t + 33EhTruvCaMl3Cn/OqHRgeIA0du7THrvVvCwSm5oeCsDlXhRFo2xDyhK5u2Z8olPBPnUm1/+PJ0/ + mYirmW/ALnk0U72QhCJf3OCsk/d0fnVN/vqLCU+wSkTsd26+OkHVLncsHaoFEvcSKHr2zCW/PIbn + M+RpZb832LqxjVZM9YoU0sp0PvHBd0i9Ds7ug0/nOVU7lv1qhyKXrJjZXE/WuJjXKciBGv/yt3a8 + +IsczsvBJ4/vwyzkvHFMdDLnF6yHBrW6SLxzcLIO0c7Uy3bxUK0Y7HtXM3PS4xz4poYgMLdsHdlP + iwf8kqJzfH+R7WP9KbrbOZN/5xfEns4vOi15ZnA6H29sc7l4RaepUgfvYpYSL1z10XBLqwskWVoz + X77QpCPz0IaDUVQ4yY3AV4QTlMB2WkiVq50X46rXmh+eMns+HyOKvpcKJrxl6+XJjJT1bMV/+TxV + vV60/FoBwHg7emT9PwAAAP//7N3LjqpIGADgfT/FSW97Og2oVDE7Ba+gVXIRJZlMENEWRRGoAio5 + 7z4puzOZ5OzOcvJvWUCgCP/tCzVLb1bL7EeO4+3N4c/+vXJWLjk+TEPxXe8N2WdfH4mNRUTvOhfq + vmiWeDxeBNz5cHXxzOfx5DE68tWiOe+7XrYzcYJOEz5vtZXfVf4lRrLfxIzxYyS0uane8ZZoE0rd + 1BNtmepLnCh2x0f+4eR0ToDW+oVuCvl8V06vf9I9nA0YIfqH4YnGTOYNHj6OOZXvl6gUbTRH84+N + zZ/1JXvG55BVNTeXO98RSdN2hqGZGbV0NchEhN0Qn5OU8yXJkrLZOVPP0JNQpVZ4b3w+7ZYVkvNJ + 0pP9PXF+GGtkqwGjMn/YV7Mo8tBqaNTkLuNVx2olx6jW3ridaHzfhJOS4KnuUb4ILTcT+5quUW87 + srm7Hp6y+tS/K/gaRzM+nlxCq7d7DAiS600nXVwJMSmSB/5U9eprniuUKtJw4ftXot+6S9ac8ihB + 9MxyOjf6mV8N8KjCMj9k18VbWlZjbf1mBPVkSRfXvp+1tJwRTOkNy/jERXVwiYKe61nI+FNdtpc+ + 3u4VhVr9+WIvassdGrPArYkYG1HGN5v8goo8JgSNLJGJxXDbx6EdlKRj96GjJd41xGU3Wsn4K4Qw + vSjB8vxMNQZmqTqOj3HkeAGf2dOe1crvDXZXnUWf8x5ZryUI0xVnSHxEFj/63dQ4F4eUNYHQRRO6 + Jw8ph1QhRuJR1H3WvokOcaWyj+NDoPas7jU8o+hB1MA+onbh5+cBY7ngM9MvLZZ4dYC91dT5up+W + BMUZmT3tIPtBbsnzeDfHMl8mWOYbhU1Ob/j1qQJ+/vEbokAFUQCiAEQBiAIQBSAKQBSAKABRAKIA + RAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQBiAIQBSAKQBSAKPhFFGggCkAUgCgA + UQCiAEQBiAIQBSAKQBSAKABRAKIARAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQB + iIL/uSh4+fHjr+cuCPn9kF4lDKjTtn7/lwq8x4f4XVG0r60SWBWf0tc/vwXCa1He86L+u75f0lsl + qcH33wte63sdX/97/EVe6ufLPwAAAP//AwArUdmvhGEAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f2a0b0fcdb3f78d-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 15 Dec 2024 22:59:17 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '105' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999979' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_483b922409dfd3fd0f1fbe0f18b132fe + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/test_llmobs_ragas_evaluators.py b/tests/llmobs/test_llmobs_ragas_evaluators.py index 4c008dea50e..c050b56a4df 100644 --- a/tests/llmobs/test_llmobs_ragas_evaluators.py +++ b/tests/llmobs/test_llmobs_ragas_evaluators.py @@ -3,10 +3,12 @@ import mock import pytest +from ddtrace.llmobs._evaluators.ragas.answer_relevancy import RagasAnswerRelevancyEvaluator from ddtrace.llmobs._evaluators.ragas.context_precision import RagasContextPrecisionEvaluator from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator from ddtrace.span import Span from tests.llmobs._utils import _expected_llmobs_llm_span_event +from tests.llmobs._utils import _expected_ragas_answer_relevancy_spans from tests.llmobs._utils import _expected_ragas_context_precision_spans from tests.llmobs._utils import _expected_ragas_faithfulness_spans from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages @@ -475,3 +477,232 @@ def test_llmobs_with_context_precision_emits_traces_and_evals_on_exit(mock_write assert status == 0, err assert out == b"" assert err == b"" + + +def test_ragas_answer_relevancy_init(ragas, LLMObs): + rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + assert rar_evaluator.llmobs_service == LLMObs + assert rar_evaluator.ragas_answer_relevancy_instance == ragas.metrics.answer_relevancy + assert rar_evaluator.ragas_answer_relevancy_instance.llm == ragas.llms.llm_factory() + assert ( + rar_evaluator.ragas_answer_relevancy_instance.embeddings.embeddings + == ragas.embeddings.embedding_factory().embeddings + ) + assert ( + rar_evaluator.ragas_answer_relevancy_instance.embeddings.run_config + == ragas.embeddings.embedding_factory().run_config + ) + + +def test_ragas_answer_relevancy_throws_if_dependencies_not_present(LLMObs, mock_ragas_dependencies_not_present, ragas): + with pytest.raises(NotImplementedError, match="Failed to load dependencies for `ragas_answer_relevancy` evaluator"): + RagasAnswerRelevancyEvaluator(LLMObs) + + +def test_ragas_answer_relevancy_returns_none_if_inputs_extraction_fails(ragas, mock_llmobs_submit_evaluation, LLMObs): + rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + failure_msg, _ = rar_evaluator.evaluate(_llm_span_without_io()) + assert failure_msg == "fail_extract_answer_relevancy_inputs" + assert rar_evaluator.llmobs_service.submit_evaluation.call_count == 0 + + +def test_ragas_answer_relevancy_has_modified_answer_relevancy_instance( + ragas, mock_llmobs_submit_evaluation, reset_ragas_answer_relevancy_llm, LLMObs +): + """Context precision instance used in ragas evaluator should match the global ragas context precision instance""" + from ragas.llms import BaseRagasLLM + from ragas.metrics import answer_relevancy + + class FirstDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "dummy llm" + + def agenerate_text(self) -> str: + return "dummy llm" + + answer_relevancy.llm = FirstDummyLLM() + + rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + + assert rar_evaluator.ragas_answer_relevancy_instance.llm.generate_text() == "dummy llm" + + class SecondDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "second dummy llm" + + def agenerate_text(self) -> str: + return "second dummy llm" + + answer_relevancy.llm = SecondDummyLLM() + + rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + + assert rar_evaluator.ragas_answer_relevancy_instance.llm.generate_text() == "second dummy llm" + + +@pytest.mark.vcr_logs +def test_ragas_answer_relevancy_submits_evaluation(ragas, LLMObs, mock_llmobs_submit_evaluation): + """Test that evaluation is submitted for a valid llm span where question is in the prompt variables""" + rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + llm_span = _llm_span_with_expected_ragas_inputs_in_prompt() + rar_evaluator.run_and_submit_evaluation(llm_span) + rar_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasAnswerRelevancyEvaluator.LABEL, + metric_type=RagasAnswerRelevancyEvaluator.METRIC_TYPE, + value=mock.ANY, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages( + ragas, LLMObs, mock_llmobs_submit_evaluation +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + llm_span = _llm_span_with_expected_ragas_inputs_in_messages() + rar_evaluator.run_and_submit_evaluation(llm_span) + rar_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasAnswerRelevancyEvaluator.LABEL, + metric_type=RagasAnswerRelevancyEvaluator.METRIC_TYPE, + value=mock.ANY, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys( + ragas, LLMObs, mock_llmobs_submit_evaluation +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rf_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + llm_span = _expected_llmobs_llm_span_event( + Span("dummy"), + prompt={ + "variables": { + "user_input": "Is france part of europe?", + "context_2": "irrelevant", + "context_3": "France is part of europe", + }, + "_dd_context_variable_keys": ["context_2", "context_3"], + "_dd_query_variable_keys": ["user_input"], + }, + output_messages=[{"content": "France is indeed part of europe"}], + ) + rf_evaluator.run_and_submit_evaluation(llm_span) + rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasAnswerRelevancyEvaluator.LABEL, + metric_type=RagasAnswerRelevancyEvaluator.METRIC_TYPE, + value=mock.ANY, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_answer_relevancy_emits_traces(ragas, LLMObs): + rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) + rar_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) + assert rar_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_count == 2 + calls = rar_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_args_list + + spans = [call[0][0] for call in calls] + + # check name, io, span kinds match + assert spans == _expected_ragas_answer_relevancy_spans() + + # verify the trace structure + root_span = spans[0] + root_span_id = root_span["span_id"] + assert root_span["parent_id"] == "undefined" + assert root_span["meta"] is not None + + root_span_trace_id = root_span["trace_id"] + for child_span in spans[1:]: + assert child_span["trace_id"] == root_span_trace_id + assert child_span["parent_id"] == root_span_id + + +def test_llmobs_with_answer_relevancy_emits_traces_and_evals_on_exit(mock_writer_logs, run_python_code_in_subprocess): + env = os.environ.copy() + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] + if "PYTHONPATH" in env: + pypath.append(env["PYTHONPATH"]) + env.update( + { + "DD_API_KEY": os.getenv("DD_API_KEY", "dummy-api-key"), + "DD_SITE": "datad0g.com", + "PYTHONPATH": ":".join(pypath), + "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", "dummy-openai-api-key"), + "DD_LLMOBS_ML_APP": "unnamed-ml-app", + "_DD_LLMOBS_EVALUATOR_INTERVAL": "5", + "_DD_LLMOBS_EVALUATORS": "ragas_answer_relevancy", + "DD_LLMOBS_AGENTLESS_ENABLED": "true", + } + ) + out, err, status, pid = run_python_code_in_subprocess( + """ +import os +import time +import atexit +import mock +from ddtrace.llmobs import LLMObs +from ddtrace.internal.utils.http import Response +from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages +from tests.llmobs._utils import logs_vcr + +ctx = logs_vcr.use_cassette( + "tests.llmobs.test_llmobs_ragas_answer_relevancy_evaluator.emits_traces_and_evaluations_on_exit.yaml" +) +ctx.__enter__() +atexit.register(lambda: ctx.__exit__()) +with mock.patch( + "ddtrace.internal.writer.HTTPWriter._send_payload", + return_value=Response( + status=200, + body="{}", + ), +): + LLMObs.enable() + LLMObs._instance._evaluator_runner.enqueue(_llm_span_with_expected_ragas_inputs_in_messages(), None) +""", + env=env, + ) + assert status == 0, err + assert out == b"" + assert err == b"" From 812a16222a74c08eb44179fb4d1e373e5f4f2875 Mon Sep 17 00:00:00 2001 From: lievan Date: Mon, 16 Dec 2024 10:21:42 -0500 Subject: [PATCH 15/15] add calc sim workflow, update tests --- .../_evaluators/ragas/answer_relevancy.py | 22 +- ddtrace/llmobs/_evaluators/ragas/base.py | 7 +- ddtrace/llmobs/_evaluators/runner.py | 4 +- tests/llmobs/_utils.py | 20 +- ...t_ragas_answer_relevancy_emits_traces.yaml | 504 +++++++++--------- ...s_answer_relevancy_submits_evaluation.yaml | 280 +++++----- ...s_evaluation_on_span_with_custom_keys.yaml | 276 +++++----- ...ion_on_span_with_question_in_messages.yaml | 52 +- tests/llmobs/test_llmobs_ragas_evaluators.py | 2 +- 9 files changed, 593 insertions(+), 574 deletions(-) diff --git a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py index 64d6315c0eb..7888643227e 100644 --- a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py +++ b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py @@ -33,7 +33,9 @@ def __init__(self, llmobs_service): The `ragas.metrics.answer_relevancy` instance is used for answer relevancy scores. If there is no llm attribute set on this instance, it will be set to the - default `llm_factory()` which uses openai. + default `llm_factory()` from ragas which uses openai. + If there is no embedding attribute set on this instance, it will be to to the + default `embedding_factory()` from ragas which uses openai :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and submitting evaluation metrics. @@ -74,6 +76,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] return "fail_answer_relevancy_is_none", {} evaluation_metadata = {} # type: dict[str, Union[str, dict, list]] + trace_metadata = {} # type: dict[str, Union[str, dict, list]] # initialize data we annotate for tracing ragas score, question, answer, answer_classifications = ( @@ -106,6 +109,8 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] context="\n".join(contexts), ) + # 'strictness' is a parameter that can be set to control the number of generations + trace_metadata["strictness"] = self.ragas_answer_relevancy_instance.strictness result = self.ragas_answer_relevancy_instance.llm.generate_text( prompt, n=self.ragas_answer_relevancy_instance.strictness ) @@ -117,15 +122,24 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] logger.debug("Failed to parse answer relevancy output: %s", e) return "fail_parse_answer_relevancy_output", evaluation_metadata - # calculate score gen_questions = [answer.question for answer in answers] answer_classifications = [ {"question": answer.question, "noncommittal": answer.noncommittal} for answer in answers ] + trace_metadata["answer_classifications"] = answer_classifications if all(q == "" for q in gen_questions): logger.warning("Invalid JSON response. Expected dictionary with key 'question'") return "fail_parse_answer_relevancy_output", evaluation_metadata - cosine_sim = self.ragas_answer_relevancy_instance.calculate_similarity(question, gen_questions) + + # calculate cosine similarity between the question and generated questions + with self.llmobs_service.workflow("dd-ragas.calculate_similarity") as ragas_cs_workflow: + cosine_sim = self.ragas_answer_relevancy_instance.calculate_similarity(question, gen_questions) + self.llmobs_service.annotate( + span=ragas_cs_workflow, + input_data={"question": question, "generated_questions": gen_questions}, + output_data=cosine_sim.mean(), + ) + score = cosine_sim.mean() * int(not any(answer.noncommittal for answer in answers)) return score, evaluation_metadata finally: @@ -133,5 +147,5 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] span=ragas_ar_workflow, input_data=span_event, output_data=score, - metadata={"answer_classifications": answer_classifications}, + metadata=trace_metadata, ) diff --git a/ddtrace/llmobs/_evaluators/ragas/base.py b/ddtrace/llmobs/_evaluators/ragas/base.py index 92387f1d3a1..0dff57d601f 100644 --- a/ddtrace/llmobs/_evaluators/ragas/base.py +++ b/ddtrace/llmobs/_evaluators/ragas/base.py @@ -1,5 +1,3 @@ -from abc import ABC -from abc import abstractmethod import traceback from typing import Optional from typing import Tuple @@ -98,7 +96,7 @@ def _get_ml_app_for_ragas_trace(span_event: dict) -> str: return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) -class RagasBaseEvaluator(ABC): +class RagasBaseEvaluator: """A class used by EvaluatorRunner to conduct ragas evaluations on LLM Observability span events. The job of an Evaluator is to take a span and submit evaluation metrics based on the span's attributes. @@ -219,6 +217,5 @@ def run_and_submit_evaluation(self, span_event: dict): metadata=metric_metadata, ) - @abstractmethod def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: - pass + raise NotImplementedError("evaluate method must be implemented by individual ragas metrics") diff --git a/ddtrace/llmobs/_evaluators/runner.py b/ddtrace/llmobs/_evaluators/runner.py index dfd1d233b22..46ed5572255 100644 --- a/ddtrace/llmobs/_evaluators/runner.py +++ b/ddtrace/llmobs/_evaluators/runner.py @@ -18,9 +18,9 @@ SUPPORTED_EVALUATORS = { + RagasAnswerRelevancyEvaluator.LABEL: RagasAnswerRelevancyEvaluator, RagasFaithfulnessEvaluator.LABEL: RagasFaithfulnessEvaluator, RagasContextPrecisionEvaluator.LABEL: RagasContextPrecisionEvaluator, - RagasAnswerRelevancyEvaluator.LABEL: RagasAnswerRelevancyEvaluator, } @@ -54,7 +54,7 @@ def __init__(self, interval: float, llmobs_service=None, evaluators=None): if evaluator in SUPPORTED_EVALUATORS: evaluator_init_state = "ok" try: - self.evaluators.append(SUPPORTED_EVALUATORS[evaluator](llmobs_service=llmobs_service)) + self.evaluators.append(SUPPORTED_EVALUATORS[evaluator](llmobs_service=llmobs_service)) # noqa: E501 except NotImplementedError as e: evaluator_init_state = "error" raise e diff --git a/tests/llmobs/_utils.py b/tests/llmobs/_utils.py index 9c827e35fbd..bbff6cf5b42 100644 --- a/tests/llmobs/_utils.py +++ b/tests/llmobs/_utils.py @@ -532,9 +532,7 @@ def _expected_ragas_answer_relevancy_spans(ragas_inputs=None): "span.kind": "workflow", "input": {"value": mock.ANY}, "output": {"value": mock.ANY}, - "metadata": { - "answer_classifications": mock.ANY, - }, + "metadata": {"answer_classifications": mock.ANY, "strictness": mock.ANY}, }, "metrics": {}, "tags": expected_ragas_trace_tags(), @@ -555,6 +553,22 @@ def _expected_ragas_answer_relevancy_spans(ragas_inputs=None): "metrics": {}, "tags": expected_ragas_trace_tags(), }, + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": mock.ANY, + "name": "dd-ragas.calculate_similarity", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, ] diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml index eb6e4de989c..33d7c5d557e 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_emits_traces.yaml @@ -46,6 +46,9 @@ interactions: - '2795' content-type: - application/json + cookie: + - __cf_bm=KxfM6APw09zrJass.4OtAFMoUu2vRIbwSxiTzDsvK18-1734361136-1.0.1.1-ziXwyO6Y2EVlj7avwtfxuCobMUBGVXaF4V0POP9dVYb7lePh2DTCi3wGND0Bfqx44OdTFy0.IruHQWWlbKLdYg; + _cfuvid=nZLW2HYfOt4n61DxTEcgN8TpbBbjLbo4wSNNVqPlytw-1734361136731-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -71,19 +74,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//7FTBitswEL37K8Sc42I7qTfry1IoXSh0KRTaQ1NsRR7b2sqSKo1pl5B/ - L3Kc2GFb6H178eG9eeM3TxodIsZA1lAwEB0n0VsVv0H303zusk8fHz/kD8lth2/X9/dPlL+vH25h - FRRm/4iCzqpXwvRWIUmjT7RwyAlD1/RmvVkn6+3r7Uj0pkYVZK2leGPiXmoZZ0m2iZObON1O6s5I - gR4K9jVijLHD+A0+dY2/oGDJ6oz06D1vEYpLEWPgjAoIcO+lJ64JVjMpjCbUo/Wqqg47+DGgD853 - ULAdfOk4MekZdcgEt5K4YqZh7xzXAu92sGI70EYL0/eSiKugSo5VVS3/4bAZPA9z6kGpCT9eTCvT - Wmf2fuIveCO19F3pkHujg0FPxkK0ED9LIv2fxJRE9vKSiBj7Ni7McDUvWGd6SyWZ76hDwzybFgbm - PV2w+USSIa4W+PZMXPUrayQulV/EC4KLDutZOu8nH2ppFsTyCJ+7+VPv0+RSt//SfiaEQEtYl9Zh - LcX1xHOZw/CM/a3skvJoGPyTJ+zLRuoWnXXydGEaW+aNSBNME9xDdIx+AwAA//8DAGrumFFSBQAA + H4sIAAAAAAAAAwAAAP//7FTRitQwFH3vV4T7PJV2drYz9EUE0RVZEAQVrLRpettG0ySb3MLKMP8u + 6XSnHVbBd33pwz33nJ57kptjxBjIBnIGouckBqviV232IO8+qPcfvxyE29/fvbX1p8f69n7/+t0t + bALD1N9R0BPrhTCDVUjS6DMsHHLCoJrub3Y3WZrukgkYTIMq0DpL8c7Eg9Qy3ibbXZzs4/Qws3sj + BXrI2deIMcaO0zf41A0+Qs4mrakyoPe8Q8gvTYyBMypUgHsvPXFNsFlAYTShnqxXVXUs4GFEH5wX + kLMCPvecmPSMemSCW0lcMdOyN45rgS8L2LACtNHCDIMk4iqwklNVVet/OGxHz8OcelRqrp8uppXp + rDO1n/FLvZVa+r50yL3RwaAnYyFakZ8lkf5PYk5i++8lETH2bVqY8WpesM4MlkoyP1AHwWw7Lwws + e7pCsxkkQ1yt6ocn4EqvbJC4VH4VLwguemwW6rKffGykWQHrI3zu5nfa58ml7v5GfgGEQEvYlNZh + I8X1xEubw/CM/antkvJkGPxPTziUrdQdOuvk+cK0tsxakSaYJlhDdIp+AQAA//8DAMXZlT1SBQAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a126b0fcf8c5d-EWR + - 8f2f88e85a272082-IAD Connection: - keep-alive Content-Encoding: @@ -91,15 +94,9 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 23:04:18 GMT + - Mon, 16 Dec 2024 14:59:01 GMT Server: - cloudflare - Set-Cookie: - - __cf_bm=yoaDgBqIVg7q_qWICuV4sfI2HC68mSEiEJeGmauKwk8-1734303858-1.0.1.1-MYIlBP7O1Wu.r.hazt7JZrz7sO8oLIzVkD7otw3Ls8ovEJ1u4QDKgiMrWj.vAkY18tkpfJ9M3g0hUmnbiq2u6Q; - path=/; expires=Sun, 15-Dec-24 23:34:18 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=A1kfyxL1kDcyq.Uw7XGdceC3nZTqjxFlKdeMyNQICfw-1734303858928-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked X-Content-Type-Options: @@ -111,7 +108,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '410' + - '572' openai-version: - '2020-10-01' strict-transport-security: @@ -129,7 +126,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_0d1ac5d511c5de88fe61226b65f98608 + - req_28a0d71749a3573803d202d8b441b265 status: code: 200 message: OK @@ -147,6 +144,9 @@ interactions: - '114' content-type: - application/json + cookie: + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -172,120 +172,120 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//VHrZrrO6tub9foqldZvaCl3wYN0RIJDQ2EkgXalUApIQIITQ2ICPzruX - yH90qupmSpPpCQYPf93wf/zrr7/+rpPikfZ///PX3++86//+H/O1e9zHf//z1//8119//fXXf/x+ - /n8jH1XyuN/zT/Yb/vtj/rk/xr//+Uv47yv/d9B87+6zJbpxCJLx+o2pOpkOYUH0LNBk1o4C/EPX - zCKD7LEy0ErVfisBC+vwE07VfhBAEpsbW5fXHA1eeYkhUNIrFSAf+SRpLgUiKB/iKv0n4aqpVsjf - X31cNxe5qO+01EFbPTUsyN+3N5GjN6D16BnM5de0mO6Dm0N07V2yFb+rsF9LDYXr2OsMF8dXMgXx - I0KZVz+YvgtyxHvaCmg7vvx5PgYX7wfJgtuzPrMtzpZtfz9INmiLlhCbkqXXpePKQJHUvOjyEjlF - V9QsAzXYbsjNjwc0qqsCoHWtKzt7Cmun4/F7hOa7GzBgYifD6lFmaCnUHMuuffG4dRFyUA/5luBs - p3g0iM9nsAN3Szarcx5yfNYNjRriAy8X93XYi2fbh/GZp8S6bV5oyFshAyizGA/pcWinb2grsOBu - g8N7+eYtOSIdFtniy1wUH1uanNQILcRzi6WdV4ZUFOJazU9OSjZv8dzyRfGW4LAtR4Ld9mTyFTJS - CKvbSFz5Y7fj45RV0Oi3jnhHrzFHqF8RqIrtYeotvkV9P9gSupdHl5k91N6gJpdcbWT1Qhvn2oXd - /Y0WiBTLJZ6G9yWk7+W1WS2FhhNXT2WTB2yimqtmOpXxae3Jir/ZwvRtjxShB0ZsNzUYOY/6QA57 - XS+kPcsAbHVPmFf7QSjnyzxGhzJymMVi5E0w6Y2GNpZKF9wpwoGuvSNKP67JvOulDTuqhFsoyzoi - gdJtzaGIVwrEYp4zc3mN0JB8twD+OGlksy0/qEvvngJH+gDiraKC88vtYKMaPbbEqVdlMfhuSWE0 - R4euRveU8AomRcv1PiZ2j2rek/uhBNkSd8y4xjuT7+6aBWhVRVSTq08yiK9Ih+zdAp76k1cMG1/q - UH9iF0IW91f4mw86lGeHSjfdTjo1+Kqgan2FheOuS8aD/BrA0C8TsfBKL+STMwDsJKNi5lEsE7b8 - XCO4KdeEkZX6Mrvs0EXgHSyLRArboEnY3CMQjo1DNt3YeU1E0j3aLJIXI7utXQxv0bWQlWQfsjPF - OCy8U54v8ebpUKXUdCQt9ERA4jZWyfqVBiEjR2SokV3sSPBGQ8troY7A8+maGZdom4wp2p812ccp - XiYOb6d6TSf4hoc3sdcdbSd6cx7AtE1LPGO1DScUtSk0X2+gk3B4mf1rfV2AcE8F4rbvfTGQZWD/ - 3p8unGQXsk+hY00mX4noOCzCbru+uOBqy5joclWbI4SAoRG7MzkUx9xktr1SoIlGSoe2nMyONE9Q - pWMx4Fe6CrxBOQkPOASDSfZHqSi4cbkYyNlLIXEvAQ0rz/cwENmu2XoUSME/F28LFy14kyB6mpzb - 9qiqgfK4EuOza8Jp4qIL5co44zrzzu10dnYdylRzgReLHoXtChkPyIRrwbbvg2zWH3I8a3WpbIit - OI3JHbqkq9JTCDN2Wydkab2mQBV3S9yj03Nq226jLh7Th/2+53CQphz6V9qzu6f24VRKWQWXg+6w - eyoO3uhHvq6uSzckzu1joMncvLfQXT974ilrkgx32hmIOWeFWVG+LqSBxjbgIQnoUlp47fi6SDUy - ff9J/N3ujbhZyy4Kut4g1oV0fOKx+0DLh82Ic7oqfKz1ZQbNbakzffO6edNee1QQ5ccnW9vKjtPM - CQb0EaaeBXLlhILY+JYa7oczSZfKJfksirfwwycqq1st/EhfSwKmOS0hZHMvBnAjFaDf2czzSNRK - 1aNK4aPdbmTD3bUpPdY5RbLvp8wn0sHk7VcftHk+RHc/q3aYFiFV3917y9yClMmwaRIVnoQPmHc2 - T7j/MRrgRnJh62d1Tb6YRhhECho76M+I9/LGLJGaRiLZnO5RMuCrLEBi9UeK3lzx+stFrVBnBZhq - XwpFdcoghqC0z3hYfJ98NJq+gS+sv7jV5NKb3strjWa8J04y9py+JNMG+5B77IdHFFSFgr7yA7Y2 - TruCr6b4urqFm5HYD9vmHXdMCZ3F/ciscPnyhrm+UaFqH7ZT96cZ73RAw1sciCs87gk/rIcrSj9b - k0T4nCWj/jaqP/9PxK2E2DeUXFhVpcGc74C8viSlBa6a68RM6J7zzNEMmPmLOPujEvIBJIA0iXti - 9Ke2mLZZLKjk5h0YXn1iND2H2gAOlkn27udW9I/i4Su89RIqrutjMjzv3+tKvgUeFl4JNUdIuz2o - 6tGn5WuBQ/ZMmy1cJeVIwguy+Mj5RgUaTjad91MymTVRUPpajJjvfLMQ45fu/sazcE9QMkyLhKLG - GZZsrn/eupZ7RdkhVPH06dx27If4rNb3fMQqt+xkavA60ub9SRyry8LRknUMNI0nsnl+xYQFTO1A - pAuNmNOzThg5ch1Ilr2Zp3pxMl2uXwFSGB901VkRmiauuahH+npe34DzrR8BPCZuEP1sk5BL5LhF - 2n3/YHh4qWE/pRlGM38y3RaCdpAr2UK56AtYVAMFjeFhMrRSEk7MKi9mMf34shLTN11u31UynrTm - DL2rfpn5ViePumqrg7FyJPolDnhj8C63f9b/oLqI9+3ZdaFf3BO2XoWbkFebwoCcpDo53rNv24mL - FVb3TOqYM+WbZLSCHlSIlxsWxGnMp2mpYvUlKh0ehmbiPVv0D3WjgsMeb3JtKSwXR3jeThtyn7KK - d26wosg/VgHT5/HszL0JxLhExGu/PueO2Smo8CuH4OZ0M6dL9UzRY3ivmb9uBpN/cGhpfbO5YTFj - Z3MScjWH0+JzYpuMnb1Jf68EzSf3gjh5XyR9vvAUeC/GHfvxfY+9Mf3h1YxPK5PJb1kAij45Xo3q - t52aobQRW9UHQpaRzAdLugLSbUXASEK4lQUMDQKJJnhp+wez++EnJVcTy1hZh9KKbGP4gKsTcsSO - Kf/u770fNh7TVe/xfVLYsDKfNwop5u1EHroExoVtKD9v9iab8R8l7eFE7CoNTaZEGl7t8EvFK+Xr - /dFTIA1P5w9/S8lJPaOZj5m9rS1zeGcRILWgO6rs4qXZV3cZQxcmC7a5hDuTLvJFicTT7UuC0zlO - Pm22p7DGcs223p4jrrFGhYvfWMzSRDuZVDXEMKjBRGXBsxP50yp7uFHZpi9aFmhIKM7U+9HeERwV - uicZgrUAfd3nxLGlNmQP47aAxdYvf/jd9ql+WgCTH/uZL92EXZsRw6wP2c1dJ+3otFmjbcqrRe5m - uTal28NX0EGhO+ZfFMcca13OQH0rLoZP/zU5030D+lN/YeubvTZpKWUlSHjXEyswW8SHOMWAEVHJ - ZpEaaGLRWoVf/bvTqzL5+3nvED7eJWK6puoNPz37tfcSwbP+HXYPOINyp29aKMsnp/W6moCRZYmX - r1BAfYr2kYZ9KSaOaKSIB8mmAuGj71g8vWxz8rIh0vJDY7CACndzPORQq0bUC2znrlHbhbzPVFne - 5mTWp5wHSj7B7n58sWAX5N6UKHf7974UuV0UDuJixNrT3Z6w5CeDN54MBmi63A8Mgz5406rmtqZq - rGK+fxuLqeiVDO08O6Pw48OorykMeXzGIu/XicSWUQdK00dsTcuCD9L+cIXlp7EoD69WMUJa7tFc - H2R7ZN+ip25eak79ZVT+phbiFagKOg7Ngo5VHqDpcn0JCJYXh/gR0/gYU2341Reu3wfZozLOVdiZ - LCDuRTPanz5W8+/zwezzZ/Qmezw9YC1FEbPc5tyy9yGz0byfmLc/nRK27qwt4ONTornztMPuJHg6 - qhcoJ34TC+1EKkWBH99t74/2t74UpM6xKMz6hQndQVJhLHbE0h9iO5mHHIPiZSXTX9eDOfNxA2q8 - FYgnYDP8/Y78UKwINv3nzJduqdJwsJmzfXYJ1zeSivhuaIg/rpqQX/K1q2U5zolfrpftuAtp/sMX - ekHJAY3X77GD7Pz0cTnyIZwOWJpg+aktpr/RIemfH72Dc9unhEiLtp31paLloWTiIXqW7QCfydbS - nd+xX/2OZzt+IHooN8yU3zUfrCsf4Fm/RIZP2hmxcKqOsD8frlj43i1TMJ2wgeftsmE7Y1GgrxX0 - C2B+7jLruPPDXtt/MND3wqFd/qmL141khvbJq44Q4aAnglkqEmCxehL3UAbtdI7kCMb7OfnlBXxY - u3Wtuu61xqUtNy2PV1YEnpMxLF5vXz5Z+ITBh0/DnPJcJqO88Uo01xvxEsHlo5WHNjj+Y08nxeo9 - NurfHCB8T8SsQyfpfnrNmQKL7ZZG5nFLvRyh7VuXWWGq8/4pSBHYNF/hp+0fvBHL/hENnnQkwaHA - 4fRyhz3MfoYWO78ouETiLVjSo5/Xf5z9wqj88BJrxqFrR3F7LQE9GozlslHbbuMvOthtlSeWFMc1 - 6fcDBhrsVickDaJwrJvDAEUPyq8+Qi5m6aCS1M2ZM04Hr391xQSvy/JEdP9Oi++sv1Tn9lgS38ks - JCEPXSE9DmsS//x5VMRbgPvizHzN67ypdYUzCMmR44FfFt60EfIHjEdNZtah8r2xFFdn9PMz9oc5 - xQAf1Qa9T2WslFrGx18e0kxswyz5u/Fo+sivSD87JbPrY+3RH3/Nfnfeb2LCtUDfanP+Q4ddYCCh - MdUJqHreUfkVUzR1465EclY+mfntH+ZwKfSrllmhR3bKt/U6I9UneHHakoCrWjFuWtOF4HRiZPs+ - XMyB7dD557+xnEY44ZGFHzDnKcztF2/ePx/7Bj6pURNduD/4VPOVjQ4ELJbYO8Eb8Y3ZaN6PdOom - L5RmvQDCKMlsF/TvcJyWBsD9VBT0l6e0ccK26FYaN2YfXkPLzWJVqfsP1pmRBievlK00guId+WzO - D7xhv38fkVStJkaK+xMNRriniF6CjP3ylSGS9zrEt9FnZzM3k8nfSj7Stmr6R7/3yPNdOPGpJt48 - X+5/3AbC45rR6RLVSdOUzFhtdMMg5HFZhr/vj6qUZsRDD8ypTl+qFrr+kZiHnKHhql0rpDClYueH - poXTeUozqM/eg+2Ik3pTHmbnP/vJt8xNOO3GCX7PIxV+J15vpIMBJO8qcthHEuoFabhq4E2nn14K - 2dfWc2CZpbHdY+O0fF1Elqr4o8Z2FkpM3nptpZbY3jPPWNXJH78752sE60/BHK4X74xm/UdmPWbS - 2f+jZlMuiK9OZz77lxjRwDfYdpGYhXj3hwyk4nFlhnF7858+WO3P4fUPnw6l6tSrVjQezFpVaTjE - eBjgXFlrkvBtXkz7GElIetontrNHxsfjk9VoGe4N2s58Md4md0JR+FKYYYBtyhknEpT1hhLnslhz - 6T4YmYZUsSdulBUeD5KghHR1tognbm6oF24JhrWSndiaKqo55ek1hlDlnHhQkmJyr/pee5JxYOdm - GYZNN65LSK11TKncmEU9KEP9x59a6mrg303ruaiUpBNd0LIwJ2OrblXp+BrIJnKXfFrfYFjNeQMV - j0HdslOTdAhtbBU/H5dnMiZ+qEMeCiYejlLR8sVp8NH7TUS8SskRTbvoNMHsP5mZN1VI088xRSc/ - BCpoYhWOo6dbCLzhNPN3iWb9KaCfHsPKUCbNrYIFXHY1p3x61mFzQX4Jcv8JqGq+9WQAN1WhY1Aw - Y6W+UF85h1ytKrYghssmjy3FANDOszI6qaMS0sh8Ykja8IRF+snQsPqk1Q9vsezcvGL85T2Pojyy - 0LJNc9xX6wa+n/bzB49Hf3z5AB9NwIs3uRbDZVXZoH5PJdmmyEU//oXCuYqz/pdM/qvfk38AZlp2 - YbbdqEjoRkWbZu7n1k4+0v/ku7hOxMZrPhaaYFgm1qx/TCR/2mEPB1Vx2f6xTPiEPBSvthJ9sDlf - 8KT5e8LWm65k00ddMjZBbKOTpxfkZopxwvJtnCNDP014mvMw9RwtI5jxhtj592P21X5XwophSran - 5Snk9mRRCJfahvlDJfKxT3r/hyc/v4h4b950dcNL4Q++tg0eKpheRkW23j7kQ3uKFcgrwyKmWRit - LF9dH4aruCbrU3xJ+EY8UDhO0pIKH6Ewx+Wyy5G0ea0ZpmXh8eLgVj++JVa2XRXT8r4y0FFrOzys - ae+1ZNhjgPAzMdNfkmLoYw1g92pKYnU9+eNvwXXj+qev+Fg3twFm/UcXbR0U06pGNpIO+wu5Jbuj - N0yZlKlvz78wRxM+5i9/h2h5CPCg7OpiekmeBY6f7pkhHF4e14LtFm6YUWLe1u/ie5vcAc35CXO1 - vdMOxleK0eZuDGyj7XDIa+0aq965fzJ/uc54M+xBBVyJCtsYqOLfY1tScNPqy9ybaSYt7t4dzPjL - dqaohoMavFTozjeC5Sojybh9tiWKwkLB43hg4ViUBwz16uhQnh7GhN6mydA0SS+ItSklj7KGlbDG - Yk3cm7pCXXhTBUiM4sC2/lloR063kZLucEe16Gki6YYaC815L4lnvhv8YbVAwYqN2HoaV/7LDwG9 - 8ZEuZz/ar1xag3RhBrFvepVM8/PQJfNlguc8ZCiLL4AcmSuCz71nisIt9KGRlQtVhYcW9s5AMejz - TIUL6ZC8/FzPAPbbJKZs1MW4p2MH9XpMSRp0tjlm9iMDuMOZmKdTwvtTttn+8hx2u5xtc2LNp0SR - kGZkrQwbcxQsaqEbEkX281/v6zfuQI20nC7mPIijU9yh2T+xjRpc+feX16SryGIh2cXtkN5NBSRZ - iJnTdfuWa8cN1ZbTY8J7y8mTcexz4Q8ejHyhtpxGowuLLS4JOa+5N4nCsQG5vfn08fB1Lha2YGi/ - /oJBxGPxNffqQv3p6cXmHRX8vP8aMJrcmfOvMxptKI4Qvt3rrPfiYgqqJEbDWx6oMBVl0egHn6Jk - 2V7oNPMJJ596+8uf5/6TgSYl83TYJ49mrheSUOTxGC4aec/9q1vyx1/MeIIVwiOvc/L1Gap2tWfp - UImI30ug6Nkzh/zymClfIFct+4PONo2lt3yuVySTVqLLmQ++Q+p2cHEe09zPqdqx7Nd7FDpkzYzm - djZHcVmnIPlK9Mvf2vHqiTlcVoNHHt+HUUh5YxvobCyvWAt0anYhf+dgZx2inaGVrfhQzAise1cz - Y9bjE0zbGnzf2LFNaD3NyZ+uKbpE9xfZPTafoosvmfTrXxBr7l90avLM4Hw5xWx7vbpFpypCB+9i - kRI3WPfhEKfVFZIsrZknXWnSkWVgwVEvKpzkuu/J3PZLYHs1oPLNyotx3avND0+ZtVyOIUXfawUz - 3rLN6myE8maxnn75PFXcnrfTrQKAMT65ZOM8PuZI3aaC+PLx2C+/F3KhrOBun/l/+T2dvhR1zU8m - 5vJ7y8XkO/hgWbuIecuDyn96HjbN+smC3ZAnk1xcDUhRtmHbUQrCqQvLGM15E9WsZs2lrSHWcMHS - hpDD48jH9qH6kAruxNbhPfMmL0J7tSSn7/x9A09WMvUIxYpirC61Ix+MdDuA3jwrMtcX7wRpvUXb - 5cllP39Jf/x8pl3PDP8aejwdxknTJKMgpipGBb/B4Qx5+mDMx0XaDlfPPmpqehaJea6HkNmT36G5 - P4nlOd/jeaPtkStGlMz6Iemc2+2IAl3rcT3z1UR7oQLUSwvmphJLhvOmxWCrR8J2Z/NQ8KQneyRf - 1i477PWs6DOlFuAd3xxmbcqzKV+bFUbzepPNFHecb75pAy9R7f70c7nQ3ST4huEbq5+pLIasuqWI - 5LQiW00pwm4F6w5mfUjfu8Wj7Sxpv9CifuOT3VsJi5G0DgZCPjDzE+Pd/YAF9FvP78w/XXkpFbgk - gkBMZbtLeG8edM2JDj3mlnYr2OlUlehbxRijtckLvtMvCpzdqMUTrXVPSo/vM7TTOpj5l3NuHG8p - zPenorYyWtHzQoCbd4yY49qyOc54A4dgMsmv3zP7tRQBCRhFfHkz2TOcbC3/3h90iLjKh/MhOyLh - /hCwlh4Jml59aKB73Il0+Ww4GnMxkcAhqMFi5D7RuAurfEVpxZljhK1J02MfwTGwvT/vM+LomyND - lu5zHnRoWRVftzDrZQyz3vi6OFvA379TAf/5r7/++l+/EwZVfX+854MB/WPs//3fRwX+Hd/jfwuC - 9OcYAu3i7PH3P/91AuHvb1tX3/5/93X5+HR///MX+nPU4O++7uP3/3P5X/OD/vNf/wcAAP//AwCJ - O66R3iAAAA== + H4sIAAAAAAAAA1R62a6zurbm/X6KpXWb2gpd8GDdESCQ0NhJIF2pVAKSECCE0NiAj867l8h/dKrq + ZkqT6QkGD3/d8H/866+//q6T4pH2f//z19/vvOv//h/ztXvcx3//89f//Ndff/3113/8fv5/Ix9V + 8rjf80/2G/77Y/65P8a///lL+O8r/3fQfO/usyW6cQiS8fqNqTqZDmFB9CzQZNaOAvxD18wig+yx + MtBK1X4rAQvr8BNO1X4QQBKbG1uX1xwNXnmJIVDSKxUgH/kkaS4FIigf4ir9J+GqqVbI3199XDcX + uajvtNRBWz01LMjftzeRozeg9egZzOXXtJjug5tDdO1dshW/q7BfSw2F69jrDBfHVzIF8SNCmVc/ + mL4LcsR72gpoO778eT4GF+8HyYLbsz6zLc6WbX8/SDZoi5YQm5Kl16XjykCR1Lzo8hI5RVfULAM1 + 2G7IzY8HNKqrAqB1rSs7ewprp+Pxe4TmuxswYGInw+pRZmgp1BzLrn3xuHURclAP+ZbgbKd4NIjP + Z7ADd0s2q3MecnzWDY0a4gMvF/d12Itn24fxmafEum1eaMhbIQMosxgP6XFop29oK7DgboPDe/nm + LTkiHRbZ4stcFB9bmpzUCC3Ec4ulnVeGVBTiWs1PTko2b/Hc8kXxluCwLUeC3fZk8hUyUgir20hc + +WO34+OUVdDot454R68xR6hfEaiK7WHqLb5FfT/YErqXR5eZPdTeoCaXXG1k9UIb59qF3f2NFogU + yyWehvclpO/ltVkthYYTV09lkwdsopqrZjqV8WntyYq/2cL0bY8UoQdGbDc1GDmP+kAOe10vpD3L + AGx1T5hX+0Eo58s8RocycpjFYuRNMOmNhjaWShfcKcKBrr0jSj+uybzrpQ07qoRbKMs6IoHSbc2h + iFcKxGKeM3N5jdCQfLcA/jhpZLMtP6hL754CR/oA4q2igvPL7WCjGj22xKlXZTH4bklhNEeHrkb3 + lPAKJkXL9T4mdo9q3pP7oQTZEnfMuMY7k+/umgVoVUVUk6tPMoivSIfs3QKe+pNXDBtf6lB/YhdC + FvdX+JsPOpRnh0o33U46NfiqoGp9hYXjrkvGg/wawNAvE7HwSi/kkzMA7CSjYuZRLBO2/FwjuCnX + hJGV+jK77NBF4B0si0QK26BJ2NwjEI6NQzbd2HlNRNI92iySFyO7rV0Mb9G1kJVkH7IzxTgsvFOe + L/Hm6VCl1HQkLfREQOI2Vsn6lQYhI0dkqJFd7EjwRkPLa6GOwPPpmhmXaJuMKdqfNdnHKV4mDm+n + ek0n+IaHN7HXHW0nenMewLRNSzxjtQ0nFLUpNF9voJNweJn9a31dgHBPBeK2730xkGVg/96fLpxk + F7JPoWNNJl+J6Dgswm67vrjgasuY6HJVmyOEgKERuzM5FMfcZLa9UqCJRkqHtpzMjjRPUKVjMeBX + ugq8QTkJDzgEg0n2R6kouHG5GMjZSyFxLwENK8/3MBDZrtl6FEjBPxdvCxcteJMgepqc2/aoqoHy + uBLjs2vCaeKiC+XKOOM6887tdHZ2HcpUc4EXix6F7QoZD8iEa8G274Ns1h9yPGt1qWyIrTiNyR26 + pKvSUwgzdlsnZGm9pkAVd0vco9Nzattuoy4e04f9vudwkKYc+lfas7un9uFUSlkFl4PusHsqDt7o + R76urks3JM7tY6DJ3Ly30F0/e+Ipa5IMd9oZiDlnhVlRvi6kgcY24CEJ6FJaeO34ukg1Mn3/Sfzd + 7o24WcsuCrreINaFdHzisftAy4fNiHO6Knys9WUGzW2pM33zunnTXntUEOXHJ1vbyo7TzAkG9BGm + ngVy5YSC2PiWGu6HM0mXyiX5LIq38MMnKqtbLfxIX0sCpjktIWRzLwZwIxWg39nM80jUStWjSuGj + 3W5kw921KT3WOUWy76fMJ9LB5O1XH7R5PkR3P6t2mBYhVd/de8vcgpTJsGkSFZ6ED5h3Nk+4/zEa + 4EZyYetndU2+mEYYRAoaO+jPiPfyxiyRmkYi2ZzuUTLgqyxAYvVHit5c8frLRa1QZwWYal8KRXXK + IIagtM94WHyffDSavoEvrL+41eTSm97La41mvCdOMvacviTTBvuQe+yHRxRUhYK+8gO2Nk67gq+m + +Lq6hZuR2A/b5h13TAmdxf3IrHD58oa5vlGhah+2U/enGe90QMNbHIgrPO4JP6yHK0o/W5NE+Jwl + o/42qj//T8SthNg3lFxYVaXBnO+AvL4kpQWumuvETOie88zRDJj5izj7oxLyASSANIl7YvSntpi2 + WSyo5OYdGF59YjQ9h9oADpZJ9u7nVvSP4uErvPUSKq7rYzI879/rSr4FHhZeCTVHSLs9qOrRp+Vr + gUP2TJstXCXlSMILsvjI+UYFGk42nfdTMpk1UVD6WoyY73yzEOOX7v7Gs3BPUDJMi4SixhmWbK5/ + 3rqWe0XZIVTx9OncduyH+KzW93zEKrfsZGrwOtLm/Ukcq8vC0ZJ1DDSNJ7J5fsWEBUztQKQLjZjT + s04YOXIdSJa9mad6cTJdrl8BUhgfdNVZEZomrrmoR/p6Xt+A860fATwmbhD9bJOQS+S4Rdp9/2B4 + eKlhP6UZRjN/Mt0WgnaQK9lCuegLWFQDBY3hYTK0UhJOzCovZjH9+LIS0zddbt9VMp605gy9q36Z + +VYnj7pqq4OxciT6JQ54Y/Aut3/W/6C6iPft2XWhX9wTtl6Fm5BXm8KAnKQ6Od6zb9uJixVW90zq + mDPlm2S0gh5UiJcbFsRpzKdpqWL1JSodHoZm4j1b9A91o4LDHm9ybSksF0d43k4bcp+yindusKLI + P1YB0+fx7My9CcS4RMRrvz7njtkpqPArh+DmdDOnS/VM0WN4r5m/bgaTf3BoaX2zuWExY2dzEnI1 + h9Pic2KbjJ29SX+vBM0n94I4eV8kfb7wFHgvxh378X2PvTH94dWMTyuTyW9ZAIo+OV6N6redmqG0 + EVvVB0KWkcwHS7oC0m1FwEhCuJUFDA0CiSZ4afsHs/vhJyVXE8tYWYfSimxj+ICrE3LEjin/7u+9 + HzYe01Xv8X1S2LAynzcKKebtRB66BMaFbSg/b/Ymm/EfJe3hROwqDU2mRBpe7fBLxSvl6/3RUyAN + T+cPf0vJST2jmY+Zva0tc3hnESC1oDuq7OKl2Vd3GUMXJgu2uYQ7ky7yRYnE0+1LgtM5Tj5ttqew + xnLNtt6eI66xRoWL31jM0kQ7mVQ1xDCowURlwbMT+dMqe7hR2aYvWhZoSCjO1PvR3hEcFbonGYK1 + AH3d58SxpTZkD+O2gMXWL3/43fapfloAkx/7mS/dhF2bEcOsD9nNXSft6LRZo23Kq0XuZrk2pdvD + V9BBoTvmXxTHHGtdzkB9Ky6GT/81OdN9A/pTf2Hrm702aSllJUh41xMrMFvEhzjFgBFRyWaRGmhi + 0VqFX/2706sy+ft57xA+3iViuqbqDT89+7X3EsGz/h12DziDcqdvWijLJ6f1upqAkWWJl69QQH2K + 9pGGfSkmjmikiAfJpgLho+9YPL1sc/KyIdLyQ2OwgAp3czzkUKtG1Ats565R24W8z1RZ3uZk1qec + B0o+we5+fLFgF+TelCh3+/e+FLldFA7iYsTa092esOQngzeeDAZoutwPDIM+eNOq5ramaqxivn8b + i6nolQztPDuj8OPDqK8pDHl8xiLv14nEllEHStNHbE3Lgg/S/nCF5aexKA+vVjFCWu7RXB9ke2Tf + oqduXmpO/WVU/qYW4hWoCjoOzYKOVR6g6XJ9CQiWF4f4EdP4GFNt+NUXrt8H2aMyzlXYmSwg7kUz + 2p8+VvPv88Hs82f0Jns8PWAtRRGz3Obcsvchs9G8n5i3P50Stu6sLeDjU6K587TD7iR4OqoXKCd+ + EwvtRCpFgR/fbe+P9re+FKTOsSjM+oUJ3UFSYSx2xNIfYjuZhxyD4mUl01/XgznzcQNqvBWIJ2Az + /P2O/FCsCDb958yXbqnScLCZs312Cdc3kor4bmiIP66akF/ytatlOc6JX66X7bgLaf7DF3pByQGN + 1++xg+z89HE58iGcDliaYPmpLaa/0SHpnx+9g3Pbp4RIi7ad9aWi5aFk4iF6lu0An8nW0p3fsV/9 + jmc7fiB6KDfMlN81H6wrH+BZv0SGT9oZsXCqjrA/H65Y+N4tUzCdsIHn7bJhO2NRoK8V9Atgfu4y + 67jzw17bfzDQ98KhXf6pi9eNZIb2yauOEOGgJ4JZKhJgsXoS91AG7XSO5AjG+zn55QV8WLt1rbru + tcalLTctj1dWBJ6TMSxeb18+WfiEwYdPw5zyXCajvPFKNNcb8RLB5aOVhzY4/mNPJ8XqPTbq3xwg + fE/ErEMn6X56zZkCi+2WRuZxS70coe1bl1lhqvP+KUgR2DRf4aftH7wRy/4RDZ50JMGhwOH0coc9 + zH6GFju/KLhE4i1Y0qOf13+c/cKo/PASa8aha0dxey0BPRqM5bJR227jLzrYbZUnlhTHNen3AwYa + 7FYnJA2icKybwwBFD8qvPkIuZumgktTNmTNOB69/dcUEr8vyRHT/TovvrL9U5/ZYEt/JLCQhD10h + PQ5rEv/8eVTEW4D74sx8zeu8qXWFMwjJkeOBXxbetBHyB4xHTWbWofK9sRRXZ/TzM/aHOcUAH9UG + vU9lrJRaxsdfHtJMbMMs+bvxaPrIr0g/OyWz62Pt0R9/zX533m9iwrVA32pz/kOHXWAgoTHVCah6 + 3lH5FVM0deOuRHJWPpn57R/mcCn0q5ZZoUd2yrf1OiPVJ3hx2pKAq1oxblrTheB0YmT7PlzMge3Q + +ee/sZxGOOGRhR8w5ynM7Rdv3j8f+wY+qVETXbg/+FTzlY0OBCyW2DvBG/GN2Wjej3TqJi+UZr0A + wijJbBf073CclgbA/VQU9JentHHCtuhWGjdmH15Dy81iVan7D9aZkQYnr5StNILiHflszg+8Yb9/ + H5FUrSZGivsTDUa4p4hegoz98pUhkvc6xLfRZ2czN5PJ30o+0rZq+ke/98jzXTjxqSbePF/uf9wG + wuOa0ekS1UnTlMxYbXTDIORxWYa/74+qlGbEQw/MqU5fqha6/pGYh5yh4apdK6QwpWLnh6aF03lK + M6jP3oPtiJN6Ux5m5z/7ybfMTTjtxgl+zyMVfideb6SDASTvKnLYRxLqBWm4auBNp59eCtnX1nNg + maWx3WPjtHxdRJaq+KPGdhZKTN56baWW2N4zz1jVyR+/O+drBOtPwRyuF++MZv1HZj1m0tn/o2ZT + LoivTmc++5cY0cA32HaRmIV494cMpOJxZYZxe/OfPljtz+H1D58OperUq1Y0HsxaVWk4xHgY4FxZ + a5LwbV5M+xhJSHraJ7azR8bH45PVaBnuDdrOfDHeJndCUfhSmGGAbcoZJxKU9YYS57JYc+k+GJmG + VLEnbpQVHg+SoIR0dbaIJ25uqBduCYa1kp3YmiqqOeXpNYZQ5Zx4UJJicq/6XnuScWDnZhmGTTeu + S0itdUyp3JhFPShD/cefWupq4N9N67molKQTXdCyMCdjq25V6fgayCZyl3xa32BYzXkDFY9B3bJT + k3QIbWwVPx+XZzImfqhDHgomHo5S0fLFafDR+01EvErJEU276DTB7D+ZmTdVSNPPMUUnPwQqaGIV + jqOnWwi84TTzd4lm/Smgnx7DylAmza2CBVx2Nad8etZhc0F+CXL/CahqvvVkADdVoWNQMGOlvlBf + OYdcrSq2IIbLJo8txQDQzrMyOqmjEtLIfGJI2vCERfrJ0LD6pNUPb7Hs3Lxi/OU9j6I8stCyTXPc + V+sGvp/28wePR398+QAfTcCLN7kWw2VV2aB+TyXZpshFP/6FwrmKs/6XTP6r35N/AGZadmG23ahI + 6EZFm2bu59ZOPtL/5Lu4TsTGaz4WmmBYJtasf0wkf9phDwdVcdn+sUz4hDwUr7YSfbA5X/Ck+XvC + 1puuZNNHXTI2QWyjk6cX5GaKccLybZwjQz9NeJrzMPUcLSOY8YbY+fdj9tV+V8KKYUq2p+Up5PZk + UQiX2ob5QyXysU96/4cnP7+IeG/edHXDS+EPvrYNHiqYXkZFtt4+5EN7ihXIK8MiplkYrSxfXR+G + q7gm61N8SfhGPFA4TtKSCh+hMMflssuRtHmtGaZl4fHi4FY/viVWtl0V0/K+MtBRazs8rGnvtWTY + Y4DwMzHTX5Ji6GMNYPdqSmJ1Pfnjb8F14/qnr/hYN7cBZv1HF20dFNOqRjaSDvsLuSW7ozdMmZSp + b8+/MEcTPuYvf4doeQjwoOzqYnpJngWOn+6ZIRxeHteC7RZumFFi3tbv4nub3AHN+Qlztb3TDsZX + itHmbgxso+1wyGvtGqveuX8yf7nOeDPsQQVciQrbGKji32NbUnDT6svcm2kmLe7eHcz4y3amqIaD + GrxU6M43guUqI8m4fbYlisJCweN4YOFYlAcM9eroUJ4exoTepsnQNEkviLUpJY+yhpWwxmJN3Ju6 + Ql14UwVIjOLAtv5ZaEdOt5GS7nBHtehpIumGGgvNeS+JZ74b/GG1QMGKjdh6Glf+yw8BvfGRLmc/ + 2q9cWoN0YQaxb3qVTPPz0CXzZYLnPGQoiy+AHJkrgs+9Z4rCLfShkZULVYWHFvbOQDHo80yFC+mQ + vPxczwD22ySmbNTFuKdjB/V6TEkadLY5ZvYjA7jDmZinU8L7U7bZ/vIcdrucbXNizadEkZBmZK0M + G3MULGqhGxJF9vNf7+s37kCNtJwu5jyIo1Pcodk/sY0aXPn3l9ekq8hiIdnF7ZDeTQUkWYiZ03X7 + lmvHDdWW02PCe8vJk3Hsc+EPHox8obacRqMLiy0uCTmvuTeJwrEBub359PHwdS4WtmBov/6CQcRj + 8TX36kL96enF5h0V/Lz/GjCa3JnzrzMabSiOEL7d66z34mIKqiRGw1seqDAVZdHoB5+iZNle6DTz + CSefevvLn+f+k4EmJfN02CePZq4XklDk8RguGnnP/atb8sdfzHiCFcIjr3Py9RmqdrVn6VCJiN9L + oOjZM4f88pgpXyBXLfuDzjaNpbd8rlckk1aiy5kPvkPqdnBxHtPcz6nasezXexQ6ZM2M5nY2R3FZ + pyD5SvTL39rx6ok5XFaDRx7fh1FIeWMb6Gwsr1gLdGp2IX/nYGcdop2hla34UMwIrHtXM2PW4xNM + 2xp839ixTWg9zcmfrim6RPcX2T02n6KLL5n0618Qa+5fdGryzOB8OcVse726RacqQgfvYpESN1j3 + 4RCn1RWSLK2ZJ11p0pFlYMFRLyqc5Lrvydz2S2B7NaDyzcqLcd2rzQ9PmbVcjiFF32sFM96yzeps + hPJmsZ5++TxV3J63060CgDE+uWTjPD7mSN2mgvjy8dgvvxdyoazgbp/5f/k9nb4Udc1PJubye8vF + 5Dv4YFm7iHnLg8p/eh42zfrJgt2QJ5NcXA1IUbZh21EKwqkLyxjNeRPVrGbNpa0h1nDB0oaQw+PI + x/ah+pAK7sTW4T3zJi9Ce7Ukp+/8fQNPVjL1CMWKYqwutSMfjHQ7gN48KzLXF+8Eab1F2+XJZT9/ + SX/8fKZdzwz/Gno8HcZJ0ySjIKYqRgW/weEMefpgzMdF2g5Xzz5qanoWiXmuh5DZk9+huT+J5Tnf + 43mj7ZErRpTM+iHpnNvtiAJd63E989VEe6EC1EsL5qYSS4bzpsVgq0fCdmfzUPCkJ3skX9YuO+z1 + rOgzpRbgHd8cZm3KsylfmxVG83qTzRR3nG++aQMvUe3+9HO50N0k+IbhG6ufqSyGrLqliOS0IltN + KcJuBesOZn1I37vFo+0sab/Qon7jk91bCYuRtA4GQj4w8xPj3f2ABfRbz+/MP115KRW4JIJATGW7 + S3hvHnTNiQ495pZ2K9jpVJXoW8UYo7XJC77TLwqc3ajFE611T0qP7zO00zqY+ZdzbhxvKcz3p6K2 + MlrR80KAm3eMmOPasjnOeAOHYDLJr98z+7UUAQkYRXx5M9kznGwt/94fdIi4yofzITsi4f4QsJYe + CZpefWige9yJdPlsOBpzMZHAIajBYuQ+0bgLq3xFacWZY4StSdNjH8ExsL0/7zPi6JsjQ5bucx50 + aFkVX7cw62UMs974ujhbwN+/UwH/+a+//vpfvxMGVX1/vOeDAf1j7P/930cF/h3f438LgvTnGALt + 4uzx9z//dQLh729bV9/+f/d1+fh0f//zF/pz1ODvvu7j9/9z+V/zg/7zX/8HAAD//wMAiTuukd4g + AAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a126fde6180df-EWR + - 8f2f88ecbeb1c974-IAD Connection: - keep-alive Content-Encoding: @@ -293,15 +293,9 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 23:04:19 GMT + - Mon, 16 Dec 2024 14:59:01 GMT Server: - cloudflare - Set-Cookie: - - __cf_bm=Gul7c8fExIU8FqFmLe7ibG6d2ui6aihEjGEbftB2HDo-1734303859-1.0.1.1-8CGeWlxDim_kuzwaWnQVnAJP0Er9WUj.qOwVjfGiTHyf0WAnOi3d7H8Wq9KweUcQneHQdvc56_9L2cKSSwpnhg; - path=/; expires=Sun, 15-Dec-24 23:34:19 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=jRamhH7v5XicVPgpi9Yoew5VMsO.2qrYeZTYALGr_4s-1734303859489-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked X-Content-Type-Options: @@ -317,7 +311,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '97' + - '113' openai-version: - '2020-10-01' strict-transport-security: @@ -335,7 +329,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_681a8e6776d55fd49c1653dd98e86efb + - req_a32fc9f9b22a429c164e503c60cdf87a status: code: 200 message: OK @@ -355,8 +349,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=Gul7c8fExIU8FqFmLe7ibG6d2ui6aihEjGEbftB2HDo-1734303859-1.0.1.1-8CGeWlxDim_kuzwaWnQVnAJP0Er9WUj.qOwVjfGiTHyf0WAnOi3d7H8Wq9KweUcQneHQdvc56_9L2cKSSwpnhg; - _cfuvid=jRamhH7v5XicVPgpi9Yoew5VMsO.2qrYeZTYALGr_4s-1734303859489-0.0.1.1-604800000 + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -382,123 +376,123 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA1RZyxKqvJaen6fY9U/tKhGQLM4MAUG5JCp465GgIiAilwSS8/JduKu6qydWiRFC - stZ3y3/+9efPP3VSPNL+n3//+eedd/0//zVdu9/62z///vPf//rz58+f//w+/9/IR5U87vf8k/2G - /37MP/fH+M+//0j/e+X/Bk337j4bYpj7MBkv3xvVuOUSFsbPAnGrdlUQH7piNhkUn5WhXmrOWw1Z - VEefiFe7QQJ50VzZqrzkaPDL8w1CNb1QCfJRcFn3KBBJ/RBP7T+J0CytQsHuEuC6OStFfaelAfry - qWNJ+b59Tg7+gFajbzJPXNKC3wcvh/jSe2Sz+C6jfiU3FC5jbzBcHF4JD2+PGGV+/WDGNsyR6Gkr - oc34Cqb5mGJx38s2XJ/1iW1wNm/7+152QJ+1hDiUzP0uHZcmiuXmRefn2C26omYZaOFmTa7BbUCj - tiwAWs++sJOvspYfDt8DNN/tgAETJxmWjzJDc6kWWPGcsy/ss5SDts83BGdb1afh7XQCJ/Q2ZL08 - 5ZHAJ8PUqbl44Pnsvor6xckJYHzmKbGv6xca8lbKAMrshof0MLT8GzkqzITX4OhevkVLDsiAWTb7 - Mg/dDi1NjlqMZotTi+WtX0Z0Id1qLT+6KVm/F6dWzIq3DPtNORLstUdLLJGZQlRdR+IpH6cdH8es - gsa4dsQ/+I01Qv2KQVMdH1N/9i3q+96R0b08eMzqofYHLTnnWqNoZ9q4ly7q7m80Q6SYzzEf3ueI - vueXZjmXGkE8I1UsETJOdU/LDKrg48pX1GC9Af5tDxShB0ZsyxuM3Ee9J/udYRTyjmUAjrYjzK+D - MFLyeX5D+zJ2mc1uyOfAjUZHa1ujM+EW0UBX/gGlH89i/uXcRh1Vow2UZR2TUO021lDclircFnnO - rPklRkPy3QAEI9fJelN+UJfefRUO9AHEX8aFEOfr3kE1emyIWy/LYgi8ksJojS5djt4xERVwVc+N - /kacHtWiJ/d9CYq92DLzcttaYnvXbUDLKqa6Un2SYfGKDcjeLWDeH/1iWAdyh/ojOxMyu7+i33zQ - vjy5VL4aTtJp4VcDTe8rLB22XTLuldcApnHmxMZLo1CO7gCwlc2KWYdFmbD55xLDVb0kjCy1l9Vl - +y4Gf2/bJFbZGnFpfY9BOjQuWXdj5zcxSXdoPUtejGw3TjG8F56N7CT7kK21uEWFf8zzOV4/XaqW - uoHkmZFIaLG5aWT1SsOIkQMytdgptiR8o6EVtVTH4Ad0xcxzvEnGFO1OuhLgFM8TV7S8XlEO32j/ - Js6qoy2nV/cBTF+3xDeXm4ijuE2h+foD5dL+ZfWv1WUG0j2ViNe+d8VA5qHze386c5NtxD6FgXWF - fGVi4KiIus3q7IGnz2/EUKraGiECDM2iO5F9ccgt5jhLFZp4pHRoS251pHmCJh+KAb/SZegP6lF6 - wD4cLLI7yEUhzPPZRO5Ojoh3DmlU+YGPgShOzVajRArxOfsbOOvhm4Tx0xLCcUZNC9XHhZifbRNx - LhYelEvzhOvMP7X85G47lGnWDM9mPYraJTIfkEmXgm3ee8WqP+Rw0utSXRNHdRtLuHROl6WvEmZu - N27E0npFgarehngHtxfUcbxGmz34h/3Wc9jLPIf+lfbs7mt9xEs5q+C8N1x2TxeDPwZxYGir0ouI - e/2YiFvr9wa6y2dHfHVFkuFOOxMx96QyO85XhTzQmwN4SEI6l2d+O77Oco2sIHiSYLt9I2HViofC - rjeJfSad4OLmPdD84TDiHi+qGGtjnkFznRvMWL+uPt/pjwri/PBkK0fdCpq54YA+Eu9ZqFRuJC2a - wNai3XAi6Vw9J59Z8ZZ++EQVbaNHH/lry8B0tyWErO/FAF6sAfRbh/k+iVu5elQpfPTrlayFt7Lk - xyqnSAmClAVE3lui/RqDPs2HGN5n2Q58FlHt3b03zCtImQzrJtHgScSAReeIRAQfswFhJme2elaX - 5ItpjGFBQWd74xmLXllbJdLSeEHWx3ucDPiiSJDY/YGit1D9/nzWKtTZIab6l0JRHTO4QVg6JzzM - vk8xmk3fwBdWX9zqSunz9/xSownviZuMvaAv2XLA2ec+++ERBU2lYCyDkK3M47YQS367LK/ReiTO - w3FEJ1xLRqfFbmR2NH/5w1TfqND0D9tqu+OEdwag4b0YiCc97onYr4YLSj8bi8T4lCWj8Tarv/8n - i42M2DeSPVhWpcnc74D8viSlDZ6WG8RK6E6IzNVNmPiLuLuDGokBZIA0ufXE7I9twTfZTdLI1d8z - vPzcEH8OtQkCbIvsvM+16B/FI1BF6yd0saoPyfC8fy9L5Rr6WHol1Boh7XagaYeAlq8ZjtgzbTZw - kdUDic7IFqMQaw1oxB069VPCrZqoKH3NRiy2gVUsbi/D+41n0Y6gZOCzhKLGHeZsqn/RerZ3Qdk+ - 0jD/dF479sPtpNX3fMSasJ2EN3gV61N/Etfusmi0FQMDTW+crJ/fRcJCpnWwoDOdWPxZJ4wchAEk - y97M1/xbws+XrwQpjA+67OwYcS50D/XIWE37GwqxCWKABxcmMU4OiYRMDhuk33cPhoeXFvU8zTCa - +JMZjhS2g1IpNsoXgYQXWqiiMdpzUy9l6cjs8mwV/MeX1SJ90/nmXSXjUW9O0Hval1lvjfvU01oD - zKUr0y9xwR/Dd7n5u/97zUOib0+eB/3snrDVMlpHoloXJuQkNcjhnn3bbjFbYm3H5I65PF8nox32 - oMFtvmbhLb0Jzuca1l4LtcPD0HDRs1n/0NYauOzxJpeWwnx2gOf1uCZ3nlWi88IlRcGhCpkxjWcn - 4XNY3EpE/PYbCOFanYqKoHIJbo5Xi5+rZ4oew3vFglUzWOKDI1vvm/UVLzJ2sriUazkcZ58jW2fs - 5HPjvZT0gNwL4uZ9kfT5zFfhPRu37Mf3PfbH9IdXEz4tLaa8FQko+uR4OWrfljdD6SC2rPeEzGNF - DLZ8AWQ4qoSRjHCrSBgaBDJN8NwJ9lb3w09KLhZWsLqK5CXZ3OADnkHIAbuW8ru//344eEyXvS92 - SeHA0npeKaRYtJw8DBnMM1tTcVrvLDbhP0ra/ZE4VRpZTI11vNzil4aX6tf/q6dAHp7uX/6Wk6N2 - QhMfM2dT29bwzmJAWkG3VN3e5lZf3RUMXZTM2PocbS06y2clWhyvXxIeT7fk02Y7Cius1Gzj7wQS - Oms0OAeNzWx94SRc0yIMgxZyqki+kyifVt3BlSoOfdGyQENCcabdD86W4LgwfNmU7BkYqz4nriO3 - EXuY1xnMNkH5w++2T43jDJjy2E186SXs0owYJn3Irt4qaUe3zRp9XV5scrfKlSVfH4GK9irdsuCs - utZYG0oG2lv1MHz6ryWYEZjQH/szW12dlUVLOStBxtue2KHVIjHcUgwYEY2sZ6mJOItXGvzq3+Ov - yhLv571D+HCXieVZmj/89OzX2ckET/p32D7gBOqdvmmhzp+C1quKAyPzEs9fkYT6FO1iHQfyjbgL - M0UiTNYVSB9jy2785Vjcz4ZYz/eNyUIq3a1xn0OtmXEvsa23Qm0XiT7TFGWTk0mfChGqOYft/fBi - 4TbMfZ6od+f3vhR5XRwNi9mI9ae3OWI5SAZ/PJoMED/f9wyDMfh8WQtH13RWsSC4jgUvejVDW9/J - KPz4MO5rCkN+O+GF6FeJzOZxB2rTx2xFy0IM8m5/gfmnsamILnYxQlru0FQfZHNg36KnXl7qbv1l - VPmmNhIVaCo6DM2MjlUeIn6+vCQE87NLgpjpYrxRffjVF67fe8WnCs412FosJN5ZN9ufPtby7/PB - nNNn9LkzHh+wkuOY2V5zatl7nzlo6ifm747HhK06ewP48JRp7j6dqDtKvoHqGcpJ0NyklpNKVeHH - d5v7o/3tLwW5c20Kk35hUreXNRiLLbGNx6Ll1j7HoPpZyYzXZW9NfNyAdttIxJewFf2+oyBaVARb - wXPiS6/UaDQ4zN08u0QYa1lDYjs0JBiXTSTO+crTsxznJChX83bcRjT/4Qs9o2SPxsv30EF2ega4 - HMUQ8T2WOcw/tc2MN9on/fNjdHBq+5QQeda2k75U9TySLTzEz7Id4MMdPd0GHfvV73hybg9E9+Wa - Wcq7FoN9EQM869eC4aN+Qizi1QF2p/0FS9+7bUmWGzXwvJ7XbGvOCvS1w34GLMg9Zh+2QdTruw8G - +p65tMs/dfG6kszUP3nVESLtjUSySlUGvKiexNuXYctPsRLDeD8lv7xADCuvrjXPu9S4dJSmFbel - HYPvZgwvLtev4DY+Ygjg0zC3PJXJqKz9Ek31RvxE8sRo55EDbvDYUa7avc9G45sDRG9OrDpyk+6n - 11we2mw7NzNf2Nr5AG3fesyOUkP0T0mOwaH5Ej+dYO+PWAkOaPDlAwn3BY74yxt2MPkZWmyDohAy - uW3Alh/9tP/j5BdG9YeXWDf3XTsuNpcS0KPBWCkbre3WwayD7UZ9Yll1PYt+P2CiwWkNQtIwjsa6 - 2Q9Q9KD+6iMSiywdNJJ6OXNHvvf7V1dweJ3nR2IEd1p8J/2ludfHnARuZiMZ+egC6WFYkdvPn8fF - bQNwn51YoPudz1tPOoGUHAQexHnm87WUP2A86Aqz91Xgj+VieUI/P+N8mFsM8NEcMPpUwWqpZ2L8 - 5SENZ2tmK9+1T9NHfkHGyS2ZUx9qn/74a/K7U78tEqGHxkaf8h86bEMTSY2lcaDaaUuV140i3o3b - EilZ+WTWt39Yw7kwLnpmRz7Zqt/W78zU4PAStCWh0PRiXLeWB+HxyMjmvT9bA9ui089/YyWNcSJi - Gz9gylOY18/eon8+dg18UrMmhnR/CF6LpYP2BGyWOFvJH/GVOWjqR8o77kfypBdAGmWFbcP+HY18 - bgLcj0VBf3lKe0vYBl1L88qc/WtohVUsK233wQYz0/Dol4qdxlC844BN+YE/7HbvA5KrJWekuD/R - YEY7iug5zNgvXxliZWfA7ToG7GTlVsKDjRwgfaOlf/V7j/zAg6PgNfGn+Yrg4zUQHVaM8nNcJ01T - MnO5NkyTkMd5Hv3WH1UpzYiPHlhQg740PfKCA7H2OUPDRb9USGVqxU4PXY/4iacZ1Cf/wbbETX2e - R9npbz8FtrWO+Hbk8HseqfA78XszHUwgeVeR/S6WUS/Jw0UHnx9/eiliX8fIgWW2zraPtduKVRHb - mhqMOtvaKLFE67eVVmJnx3xzWSd//e6UrxFsPCVruJz9E5r0H5n0mEUn/4+adTkjgcZPYvIvN0TD - wGSbWWIVi3swZCAXjwszzetb/PTBcneKLn/5dCg1t162C/PB7GWVRsMNDwOcKntFErHJC767IRnJ - T+fIts7IxHh4shrNo51J24kvxiv3OIqjl8pMExxLyQSRoazXlLjn2UrI98HMdKQteuLFWeGLMAlL - SJcnm/iL9RX10jXBsFKzI1tRVbN4nl5uEGlCEB9KUnDvYuz0JxkHdmrmUdR046qE1F7dKFUaq6gH - daj/+lNbWw7iu259D5WyfKQzWhYWNzfaRpMPr4GsY28u+OoKw3LKG+jiENYtOzZJh9Da0fDzcX4m - YxJEBuSRZOHhIBetmB2HAL3fZIGXKTkgvo2PHCb/yay8qSKafg4pOgYRUElfVNE4+oaNwB+OE3+X - aNKfEvrpMawOZdJcK5jBeVsLKvizjpozCkpQ+k9INettJAN4qQYdg4KZS+2F+srd51pVsRkxPcZ9 - Nl+EgLa+nVGujWpEY+uJIWmjI17QT4aG5SetfniLFffqF+Mv73kU5YFFtmNZ465aNfD9tJ+/eDwG - 4ysA+OgSnr3JpRjOy8oB7XssySZFHvrxLxTuZTHpf9kSv/o9Bntglu0UVtuNqoyudOHQzPtcWx4g - 42++i+tk0fjNx0YchnliT/rHQsqnHXaw11SP7R7zRHDko9tyI9MHm/IFX57WEzY+v5B1H3fJ2IQ3 - Bx19oyBXa3FLWL655cg0jhzzKQ/TTvE8hglviJN/P1Zf7bYlLBmmZHOcHyPhcJtCNNfXLBiqhRj7 - pA9+ePLzi0j01tXQ1qKU/uJr2+ChAv4yK7Lxd5EY2uNNhbwybWJZhdkqysULYLgsVmR1vJ0TsV7s - KRy4PKfSRyqscT7vciSvXyuGaVn4oth71Y9viZ1tlgWf35cmOuhth4cV7f2WDDsMEH04s4I5KYb+ - pgNsX01J7K4nf/0teN6t/ukrMdbNdYBJ/9FZW4cFX9bIQfJ+dybXZHvwB57Jmfb2gzNzdelj/fJ3 - iOf7EA/qti74S/ZtcIN0x0xp//KFHm42cMWMEuu6ehffK/cGNOUnzNN3bjuYX/mG1ndzYGt9iyNR - 65eb5p/6Jwvmq0w0ww40wNVCZWsTVeJ7aEsKXlp9mXe1rKTF3buDCX/Z1lpo0aCFLw2605VgpcpI - Mm6ebYniqFDxOO5ZNBblHkO9PLhUpPsxoVfOTV2XjYLY61L2KWtYCSu8qIl31Zaoi66aBIlZ7Nkm - OEntKOgmVtMt7qgePy0kX1FjoynvJbeJ74ZgWM5QuGQjtp/mRfzyQ0BvfKDzyY/2S4/WIJ+ZSZyr - USV8eh46Z4FC8JSHDGXxBVBia0nwqfethXSNAmgU9Uw16aFHvTtQDMY0U+lMOqTMP5cTgPO2iKWY - dTHu6NhBvRpTkoadY42Z88gA7nAi1vGYiP6YrTe/PIddzyfH4qz5lCiW0oys1GFtjZJNbXRFiwX7 - +a/35XvrQIv1nM6mPEig461Dk39iay28iO8vr0mXsc0isr21Q3q3VJAV6cbcrtu1Qj+sqT7nD453 - tpsn49jn0l88GMVMawWNRw9mG1wScloJny+kQwNKew3o4xEYYlE4kqn/zhdMsjgUX2unzbSfnp6t - 33EhTruvCaMl3Cn/OqHRgeIA0du7THrvVvCwSm5oeCsDlXhRFo2xDyhK5u2Z8olPBPnUm1/+PJ0/ - mYirmW/ALnk0U72QhCJf3OCsk/d0fnVN/vqLCU+wSkTsd26+OkHVLncsHaoFEvcSKHr2zCW/PIbn - M+RpZb832LqxjVZM9YoU0sp0PvHBd0i9Ds7ug0/nOVU7lv1qhyKXrJjZXE/WuJjXKciBGv/yt3a8 - +IsczsvBJ4/vwyzkvHFMdDLnF6yHBrW6SLxzcLIO0c7Uy3bxUK0Y7HtXM3PS4xz4poYgMLdsHdlP - iwf8kqJzfH+R7WP9KbrbOZN/5xfEns4vOi15ZnA6H29sc7l4RaepUgfvYpYSL1z10XBLqwskWVoz - X77QpCPz0IaDUVQ4yY3AV4QTlMB2WkiVq50X46rXmh+eMns+HyOKvpcKJrxl6+XJjJT1bMV/+TxV - vV60/FoBwHg7emT9PwAAAP//7N3LjqpIGADgfT/FSW97Og2oVDE7Ba+gVXIRJZlMENEWRRGoAio5 - 7z4puzOZ5OzOcvJvWUCgCP/tCzVLb1bL7EeO4+3N4c/+vXJWLjk+TEPxXe8N2WdfH4mNRUTvOhfq - vmiWeDxeBNz5cHXxzOfx5DE68tWiOe+7XrYzcYJOEz5vtZXfVf4lRrLfxIzxYyS0uane8ZZoE0rd - 1BNtmepLnCh2x0f+4eR0ToDW+oVuCvl8V06vf9I9nA0YIfqH4YnGTOYNHj6OOZXvl6gUbTRH84+N - zZ/1JXvG55BVNTeXO98RSdN2hqGZGbV0NchEhN0Qn5OU8yXJkrLZOVPP0JNQpVZ4b3w+7ZYVkvNJ - 0pP9PXF+GGtkqwGjMn/YV7Mo8tBqaNTkLuNVx2olx6jW3ridaHzfhJOS4KnuUb4ILTcT+5quUW87 - srm7Hp6y+tS/K/gaRzM+nlxCq7d7DAiS600nXVwJMSmSB/5U9eprniuUKtJw4ftXot+6S9ac8ihB - 9MxyOjf6mV8N8KjCMj9k18VbWlZjbf1mBPVkSRfXvp+1tJwRTOkNy/jERXVwiYKe61nI+FNdtpc+ - 3u4VhVr9+WIvassdGrPArYkYG1HGN5v8goo8JgSNLJGJxXDbx6EdlKRj96GjJd41xGU3Wsn4K4Qw - vSjB8vxMNQZmqTqOj3HkeAGf2dOe1crvDXZXnUWf8x5ZryUI0xVnSHxEFj/63dQ4F4eUNYHQRRO6 - Jw8ph1QhRuJR1H3WvokOcaWyj+NDoPas7jU8o+hB1MA+onbh5+cBY7ngM9MvLZZ4dYC91dT5up+W - BMUZmT3tIPtBbsnzeDfHMl8mWOYbhU1Ob/j1qQJ+/vEbokAFUQCiAEQBiAIQBSAKQBSAKABRAKIA - RAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQBiAIQBSAKQBSAKPhFFGggCkAUgCgA - UQCiAEQBiAIQBSAKQBSAKABRAKIARAGIAhAFIApAFIAoAFEAogBEAYgCEAUgCkAUgCgAUQCiAEQB - iIL/uSh4+fHjr+cuCPn9kF4lDKjTtn7/lwq8x4f4XVG0r60SWBWf0tc/vwXCa1He86L+u75f0lsl - qcH33wte63sdX/97/EVe6ufLPwAAAP//AwArUdmvhGEAAA== + H4sIAAAAAAAAA+ydW6+6zLbm79enePPeunfkJDVYdwgIClilgqdOpwOoCIjIoQqonf3dO/hf2d2d + vuvLjjczmcjkVMV4nvHUL9P/+Mdff/1dxfk96f7+519/v7K2+/vfpm23qIv+/udf/+0ff/3111// + 8f35f+x5L+P77Za90+/u3w+z9+0+/P3Pv4T/2vK/dpqO3b7XRDf223i4fCKqjqZD2DZ85Gg0K0cB + /qZLZpFe9lix1QrVfilbFlTBOxjLXS+AJNZXtiwuGeq94hzBVkkuVIBs4KOkuRSIoLyJq3TvmKum + WiJ/d/FxVZ/lvLrRQgdt8dCwIH9e3kgOXo+Wg2cwl1+SfLz1bgbhpXPJWvwsgm4p1RQuQ6cznB+e + 8biN7iFKverO9M02Q7yjjYDWw9Ofrsfg4m0vWXB9VCe2xum86W57yQZt1hBiUzL32mRYGCiU6ied + n0Mnb/OKpaBu1yty9aMeDeoiB2hc68JOnsKa8XD4HKD+bHoMmNhxv7gXKZoLFceya589bp2FDNR9 + tiY43Sge3UanE9hbd01Wi1MWcHzSDY0a4h3PZ7dl0Ikn24fhkSXEuq6eqM8aIQUo0gj3yaFvxk9g + KzDjbo2DW/HiDTkgHWbp7MNcFB0aGh/VEM3EU4OljVcEVBSiSs2OTkJWL/HU8Fn+kmC/LgaC3eZo + 8gUyEgjK60Bc+W03w/2YllDr15Z4B682B6ieIaiK7WHqzT55ddvbEroVB5eZHVRer8bnTK1l9Uxr + 59IG7e2FZojk8zke+9c5oK/5pV7MhZoTV09kk2/ZSDVXTXUq4+PSkxV/tYbx0xwoQneM2GasMXLu + 1Z7sd7qeSzuWAtjqjjCv8reBnM2zCO2L0GEWi5A3wqjXGlpZKp1xJw96uvQOKHm7JvMu5yZoqRKs + oSiqkGyVdm32ebRQIBKzjJnzS4j6+LMG8IdRI6t18UZtcvMUONA7EG8R5pyfr3sbVei+Jk61KPLe + dwsKgzk4dDG4x5iXMCpapncRsTtU8Y7c9gXIlrhhxiXamHxz0yxAizKkmly+4158hjqkrwbw2B29 + vF/5Uou6IzsTMrs9g+/1oH1xcqh01e24VbcfFVStK7Fw2LTxsJefPRj6eSQWXui5fHR6gI1klMw8 + iEXM5u9LCFflEjOyUJ9mm+7bELy9ZZFQYSs0CqtbCMKhdsiqHVqvDkmyQ6tZ/GRks7bz/iW6FrLi + 9E02phgFuXfMsjlePRyqFJqOpJkeC0hcRypZPpNtwMgBGWpo5xuyfaG+4ZVQheD5dMmMc7iOhwTt + Tprs4wTPY4c3Y7WkI3yC/YvYy5Y2I706d2DaqiGesVgHIwqbBOqP19NR2D/N7rm8zEC4JQJxm9cu + 78l8a3/vn86ceBOwd65jTSYfieg4yIN2vTy74GrziOhyWZkDBIChFtsT2eeHzGS2vVCgDgdK+6YY + zZbUD1ClQ97jZ7LYer1yFO6w3/Ym2R2kPOfG+WwgZycFxD1vaVB6voeByHbFloNAcv4+e2s4a9sX + 2YYPk3PbHlR1q9wvxHhv6mAcuehCsTBOuEq9UzOenE2LUtWc4dmsQ0GzQMYdUuGSs/VrL5vVmxxO + WlUoK2IrTm1yh87povAUwozN2glYUi0pUMVdE/fgdJzatlurs/v4Zt/n2e+lMYPumXTs5qldMBZS + WsJ5rzvsloi9N/ihr6vLwg2Ic30baDRXrzW0l/eOeMqSxP2NtgZizklhVpgtc6mnkQ24j7d0Ls28 + ZniepQqZvv8g/mbzQtysZBdt284g1pm0fOSRe0fzu82Ic7wofKj0eQr1da4zffW8euNOu5cQZocH + W9rKhtPU2fboLYwd28qlEwhi7VtqsOtPJJkr5/g9y1/Ctz5RWV1rwVv6WBIwzWkIIatb3oMbqgDd + xmaeR8JGKu9lAm/teiUr7i5N6b7MKJJ9P2E+kfYmbz56r03XQ3T3vWj6cRZQ9dW+1szNSRH3qzpW + 4UF4j3lr85j7b6MGbsRntnyUl/iDaYhBpKCxvf4IeSevzAKpSSiS1fEWxj2+yALEVneg6MUVrzuf + 1RK11hZT7UMhL48pRLAt7BPuZ58HH4y6q+EDyw9uNLnwxtf8UqGp3hMnHjpOn5Jpg73PPPatRxRU + hYK+8LdsaRw3OV+M0WVxDVYDse+2zVvumBI6ibuBWcH86fXT/Ea5qr3ZRt0dp3qnA+pfYk9c4X6L + +X7ZX1DyXpskxKc0HvSXUf75eyKuJcQ+geTCoiwM5nx65HUFKSxw1UwnZkx3nKeOZsCkX8TZHZSA + 9yABJHHUEaM7Nvm4TiNBJVdvz/DiHaHx0VcGcLBMsnPf17y753df4Y0XU3FZHeL+cftcFvJ162Hh + GVNzgKTdgaoefFo8Zzhgj6Rew0VSDiQ4I4sPnK9UoMFo0+l9ikezIgpKnrMB841v5mL01N3v/izY + ERT34yymqHb6OZvmP29cy72gdB+oeHy3bjN0fXRSq1s2YJVbdjzWeBlq0/tJHKtNg8GSdQw0iUay + enzEmG2Z2oJIZxoxx0cVM3LgOpA0fTFP9aJ4PF8+AiQw3OmitUI0jlxzUYf05TS+W87XfghwH7lB + 9JNNAi6Rwxppt92d4f6pBt2YpBhN+sl0W9g2vVzKFspEX8CiulXQEOxHQysk4cis4mzm41cvSzF5 + 0fn6VcbDUatP0Lnqh5kvdfSoqzY6GAtHoh/igDdsX8X6z/jvVRfxrjm5LnSzW8yWi2AV8HKVG5CR + RCeHW/ppWnG2wOqOSS1zxmwVD9a2AxWi+YptoyTi4zhXsfoUlRb3fT3yjs26u7pSwWH3F7k0FOaz + AzyuxxW5jWnJW3e7oMg/lFumT/uzE/dGEKMCEa/5+Jw7Zqug3C8dguvj1RzP5SNB9/61ZP6y7k3+ + xoGldfXqisWUncxRyNQMjrP3ka1SdvJG/bUQNJ/ccuJkXR532cxT4DUbNuyr9x32huRbr6b6tDCZ + /JIFoOid4cWgfpqx7gsbsUW1J2Qeyry3pAsg3VYEjCSEG1nAUCOQaIzntr8322/9pORiYhkry0Ba + kHUEb3B1Qg7YMeXv8b3X3cZDsug8votzGxbm40ohwbwZyV2XwDizFeWn1c5kU/1HcbM/ErtMApMp + oYYXG/xU8UL5eH/8FEj9w/mj31J8VE9o0mNmryvL7F9pCEjN6YYqm2huduVNxtAG8YytzsHGpLNs + ViDxeP2Q7fEUxe8m3VFYYrlia2/HEddYrcLZry1maaIdj6oaYOjV7UhlwbNj+d0oO7hS2aZPWuSo + jylO1dvB3hAc5ronGYI1A33ZZcSxpSZgd+M6g9naL771u+kS/TgDJt93k166MbvUA4bJH7Kru4yb + wWnSWlsVF4vczGJpSte7r6C9QjfMPyuOOVS6nIL6UlwM7+5jcqb7BnTH7syWV3tp0kJKC5DwpiPW + 1mwQ76MEA0ZEJatZYqCRhUsVvvPfHZ+lyV+PW4vw4SYR0zVVr//62Y+9kwie/G+/ucMJlBt90VyZ + PzitluUIjMwLPH8GAuoStAs17EsRcUQjQXwbr0oQ3vqGRePTNkcv7UMt29cG21LhZg77DCrVCDuB + bdwlatqAd6kqy+uMTP6U862SjbC5HZ5su9lm3hgrN/t7vxS5bRj04mzA2sNdH7Hkx703HA0GaDzf + 9gyD3nvjouK2pmqsZL5/HfIx75QUbTw7pfDVw7CrKPRZdMIi75axxOZhC0rdhWxJi5z30m5/gfm7 + tigPLlY+QFLs0DQ/yPrAPnlH3azQnOrDqPxJLMRLUBV06OsZHcpsi8bz5SkgmJ8d4odM40NEtf47 + v3D12sselXGmwsZkW+KeNaP5+mM1+zzuzD69B2+0h+MdllIYMsutTw177VMbTe8T83bHY8yWrbUG + fHhINHMedtAeBU9H1QxlxK8joRlJqSjw1bv17d58x5eC1DoWhcm/MKHdSyoM+YZY+l1sRnOfYVC8 + tGD687I3Jz2uQY3WAvEEbAbf35EfiCXBpv+Y9NItVBr0NnPWjzbm+kpSEd/0NfGHRR3wc7Z0tTTD + GfGL5bwZNgHNvvWFnlG8R8Plc2ghPT18XAy8D8Y9lkaYvyuL6S+0j7vHW2/h1HQJIdKsaSZ/qWhZ + IJm4Dx9F08N7tLVk47fsO3+Hkx3dEd0XK2bKr4r31oX38KieIsNH7YRYMJYH2J32Fyx8bpYpmE5Q + w+N6XrGNMcvRx9p2M2B+5jLrsPGDTtu9MdDXzKFt9q7y55WkhvbOypYQYa/HglkoEmCxfBB3X2yb + 8RTKIQy3U/zNC3i/dKtKdd1LhQtbrhseLawQPCdlWLxcP3y08BGDD++aOcWpiAd55RVomm/EiwWX + D1YW2OD49x0dFavz2KB/MoDgNRKzCpy4/fo1Z9xabDM3Uo9b6vkATde4zAoSnXcPQQrBptkCP2x/ + 7w1Y9g+o96QD2e5zHIxPt9/B1M/QfOPnOZdItAZLunfT+A9TvzAo33qJNWPfNoO4vhSA7jXGclGr + TbvyZy1s1soDS4rjmvTzBgP1dqMTkmzDYKjqfQ95B8p3fgRcTJNeJYmbMWcY9173bPMRnuf5kej+ + jeafyX+pzvU+J76TWkhCHrpAcuiXJPr252EerQFusxPzNa/1xsYVTiDEB457fp5540rI7jAcNJlZ + +9L3hkJcnNC3n7HfzMl7eKs26F0iY6XQUj5885B6ZCtmyZ+VR5N7dkH6ySmYXR0qj371a+p3p/dN + jLm21dfalP/QfrM1kFCb6ghUPW2o/IwoGtthUyA5LR7M/HR3sz/n+kVLrcAjG+XTeK2R6CM8OW3I + lqtaPqwa04Xt8cjI+rU/mz3boNO3/8ZyEuKYhxa+w5SnMLebvXj3uO9qeCdGRXThdudjxRc22hOw + WGxvBG/AV2aj6X2kYzt6gTT5BRAGSWabbfcKhnFuANyOeU6/eUoTxWyNroVxZfb+2TfczBeluntj + nRnJ9ugVspWEkL9Cn035gdfvdq8DksrFyEh+e6DeCHYU0fM2Zd98pQ/lnQ7RdfDZyczMePTXko+0 + tZr88e8d8nwXjnysiDddL/ffbg3BYcnoeA6ruK4LZixWumEQcj/Pg+/zR2VCU+KhO+ZUp09VC1z/ + QMx9xlB/0S4lUphSstNd04LxNCYpVCfvzjbESbwxC9LTn/fJt8xVMG6GEb7nIyV+xV5nJL0BJGtL + st+FEuoEqb9o4I3Hr18K2MfWM2CppbHNfeU0fJmHlqr4g8Y2FopN3nhNqRbY3jHPWFTxn353ytcI + 1h+C2V/O3glN/o9MfsykU/+P6lUxI746nvjUv0SIbn2DrWexmYs3v09Byu8XZhjXF//6g8XuFFz+ + 6GlfqE61aETjzqxFmQR9hPseTqW1JDFfZ/m4i5CEpId9ZBt7YHw4PFiF5sHOoM2kF8N1dEcUBk+F + GQbYppxyIkFRrShxzrMll269kWpIFTvihmnu8W28LSBZnCziiasr6oRrjGGppEe2pIpqjllyiSBQ + OSceFCQf3Yu+0x5k6NmpngdB3Q7LAhJrGVEq12Ze9Upf/elPLXXR88+q8VxUSNKRzmiRm6OxVteq + dHj2ZBW6cz4ur9AvpryBiodt1bBjHbcIrWwVP+7nRzzEfqBDFggm7g9S3vDZsffR60VEvEjIAY2b + 8DjC1H8yM6vLgCbvQ4KOfgBU0MQyGAZPtxB4/XHS7wJN/lNAXz+Glb6I62sJMzhvKk75+KiC+oz8 + AuTuvaWq+dLjHtxEhZZBzoyF+kRd6ewztSzZjBguGz02F7eANp6V0lEdlICG5gND3ARHLNJ3ivrF + Oym/9RbLztXLh2/ec8+LAwss2zSHXbms4fNu3n/q8eAPTx/grQl49iKXvD8vShvUz7Eg6wS56Ku/ + kDsXcfL/ksm/8/fo74GZlp2bTTsoErpS0aap+742o4/0P/kurmKx9uq3hUbo57E1+R8Tye+m38Fe + VVy2u89jPiIPRYu1RO9syhc8aXqesPbGC1l1YRsP9Tay0dHTc3I1xShm2TrKkKEfRzxOeZh6Cuch + TPWG2NnnbXblblPAgmFK1sf5MeD2aFEI5tqK+X0p8qGLO/9bT779IuKdedXVFS+EP/W1qXFfwvg0 + SrL2dgHvm2OkQFYaFjHN3Ghk+eL60F/EJVkeo3PMV+KewmGU5lR4C7k5zOdthqTVc8kwLXKP53u3 + /OotsdL1Ih/nt4WBDlrT4n5JO68h/Q4DBO+Rmf6c5H0XaQCbZ10Qq+3In/4WXDeqvv6KD1V97WHy + f3TWVNt8XFTIRtJ+dybXeHPw+jGVUvXl+WfmaMLb/ObvEM73W9wrmyofn5JngeMnO2YI+6fHte16 + DVfMKDGvy1f+uY5uj6b8hLnazml64yNFaHUzerbSNjjglXaJVO/UPZg/X6a87negAi5Fha0MVPLP + oSkouEn5Ye7VNOMGt68WpvrLNqaoBr26farQnq4Ey2VK4mH9aAoUBrmCh2HPgiEv9hiqxcGhPNkP + Mb2Oo6Fpkp4Ta1VIHmU1K2CJxYq4V3WB2uCqChAb+Z6t/ZPQDJyuQyXZ4JZq4cNE0hXVFpryXhJN + etf7/WKGtgs2YOthXPg3PwT0wgc6n/rRbuHSCqQzM4h91ct4nM6HzqkvEzzlIX2RfwDk0FwQfOo8 + UxSugQ+1rJypKty1oHN6ikGfrlQ4kxbJ8/flBGC/TGLKRpUPOzq0UC2HhCTb1jaH1L6nADc4EfN4 + jHl3TFfrb57DrueTbY6sfhcoFJKULJV+ZQ6CRS10RaLIvv3X6/KJWlBDLaOzKQ/i6Bi1aOqf2Erd + Xvjnm9cki9BiAdlETZ/cTAUkWYiY07a7hmuHFdXm433EO8vJ4mHoMuFPPRj4TG04DQcXZmtcEHJa + cm8UhUMNcnP16f3u61zMbcHQvusLBhEP+cfcqTP166dnq1eY89PuY8BgcmfKv05osCE/QPByL5Pf + i/JxW8YR6l9yT4UxL/Ja3/sUxfPmTMdJTzh5V+tv/jytPxloVFJPh118r6f5QmKKPB7BWSOvaf3q + Gv/pL6Z6ghXCQ691suUJymaxY0lfiojfCqDo0TGHfPOYMZshVy26vc5WtaU3fJqvSCaNROeTHnz6 + xG3h7NzHaT2nbIaiW+5Q4JAlM+rryRzEeZWA5CvhN39rhosnZnBe9B65f+5GLmW1baCTMb9gbatT + sw34KwM7bRFtDa1oxLtihmDd2ooZkx8fYVxX4PvGhq0C62GO/nhJ0Dm8PcnmvnrnbXROpe/6BbGm + 9YtWjR8pnM7HiK0vFzdvVUVo4ZXPEuJul13QR0l5gThNKuZJFxq3ZL614KDnJY4z3fdkbvsFsJ26 + pfLVyvJh2an1t54yaz4fAoo+lxKmestWi5MRyKvZcvzm81RxO96M1xIAhujokpVzf5sDdesSovPb + Y9/8XsiEooSbfeL/6vd0+lTUJT+amMuvNRfjT++DZW1C5s33Kv/6eVjVywfbbvosHuX8YkCC0hVb + D9I2GNugiNCUN1HNqpdcWhtiBWcsrQjZ3w98aO6qD4ngjmwZ3FJv9EK0Uwty/EzPd+vJSqoeIF9Q + jNW5duC9kax70OtHSab5xVtBWq7Ren502be/pF99PtG2Y4Z/CTye9MOoaZKRE1MVw5xfYX+CLLkz + 5uM8afqLZx80NTmJxDxVfcDs0W/RtD6J5Snf41mt7ZArhpRM/iFunev1gLa61uFq0quRdkIJqJNm + zE0kFvenVYPBVg+EbU7mPudxR3ZIPi9dtt/pad6lSiXAK7o6zFoVJ1O+1AuMpvEmqzFqOV99khqe + otr+Wc/lQnuV4BMEL6y+xyLv0/KaIJLRkqw1JQ/aBSxbmPwhfW1m96a1pN1MC7uVTzYvJcgH0jgY + CHnDpE+Mt7c9FtB3PD+T/rTFuVDgHAsCMZX1Juadudc1J9x3mFvaNWfHY1mgTxlhjJYmz/lGPytw + csMGj7TSPSk5vE7QjMvtpL+cc+NwTWA6PhW1hdGInhcAXL1DyBzXls1hqjew344m+a73TP1agoBs + GUV8fjXZIxhtLfvc7rQPucr70z49IOF2F7CWHAgan11goFvUinT+qDkaMjGWwCGoxmLoPtCwCcps + QWnJmWMEjUmTQxfCYWt7f+5nwOEnQ4Ys3aY8aN+wMrqsYfLLGCa/8XFxOoO/v1TAf/7b/wNRIP6I + gh9R8CMKfkTBjyj4EQU/ouBHFPyIgh9R8CMKfkTBjyj4EQU/ouBHFPyIgh9R8CMKfkTBjyj4EQU/ + ouBHFPyIgh9R8CMKfkTBjyj4EQU/ouBHFPxfRIH0Iwp+RMGPKPgRBT+i4EcU/IiCH1HwIwp+RMGP + KPgRBT+i4EcU/IiCH1HwIwp+RMGPKPgRBT+i4EcU/IiCH1HwIwp+RMGPKPgRBT+i4EcU/IiC/8+J + gn/89dd//34LQlnd7q8JDOjuQ/fv/4UK/Ht0i/5dEKQ/X5VA2yi9//3PfxEIf3+aqvx0/6Orivu7 + nVCDf/33gr+7qote//v2f0yn+s9//E8AAAD//wMAK1HZr4RhAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a127268e180df-EWR + - 8f2f88ee2b01c974-IAD Connection: - keep-alive Content-Encoding: @@ -506,7 +500,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 23:04:19 GMT + - Mon, 16 Dec 2024 14:59:01 GMT Server: - cloudflare Transfer-Encoding: @@ -524,7 +518,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '163' + - '91' openai-version: - '2020-10-01' strict-transport-security: @@ -542,7 +536,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_4086a3c988f643e3a10b3e2e9aa7a13c + - req_11007f90342989246b3de7a6067cd990 status: code: 200 message: OK diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml index b35efdbbb4d..9db40879f70 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation.yaml @@ -71,19 +71,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//7FRNi9swEL37VwxzjoudZJMll7L0g17aQyks23qxFXkcqytLqjSGlpD/ - XpQ4scO20Pv24sN788ZvnjTaJwCoatwAylaw7JxO78h7fic+bj/IzyTFp7dvHp7WD6pffRVf7nEW - FXb7nSSfVa+k7ZwmVtacaOlJMMWu+XqxXGSLm5vFkehsTTrKdo7TpU07ZVQ6z+bLNFun+e2gbq2S - FHAD3xIAgP3xG32amn7iBrLZGekoBLEj3FyKANBbHREUIajAwjDORlJaw2SO1quq2hf4o6cQnRe4 - gQLvW8GgAnBLIIVTLDTYBt57YSS9LnAGBRprpO06xSx0VGWHqqqm//DU9EHEOU2v9YAfLqa13Tlv - t2HgL3ijjApt6UkEa6LBwNZhMhE/SyL/n8SQxPzlJZEAPB4Xpr+aF523neOS7ROZ2HA1HxYGxz2d - sKuBZMtCT/DbM3HVr6yJhdJhEi9KIVuqR+m4n6KvlZ0Q0yN87uZPvU+TK7P7l/YjISU5prp0nmol - ryceyzzFZ+xvZZeUj4Yx/ApMXdkosyPvvDpdmMaVq0bmGeUZbTE5JL8BAAD//wMAfjk28VIFAAA= + H4sIAAAAAAAAA+xUwY6bMBC98xXWnEMFJGUjLtVe2kqVeugecigVOGYAb43ttYduqyj/XjkhgWi3 + Uu/thcN784Y3zx4fIsZANlAwED0nMVgV37f5Ez4/POe77Y+H5LP9+ClZZ1/e3g+Puw8drILC7B9R + 0EX1RpjBKiRp9JkWDjlh6JrerTfrPE3X+YkYTIMqyDpL8cbEg9QyzpJsEyd3cbqd1L2RAj0U7GvE + GGOH0zf41A3+hIIlqwsyoPe8QyiuRYyBMyogwL2XnrgmWM2kMJpQn6zXdX0o4WlEH5yXULASdj0n + Jj2jHpngVhJXzLTsveNa4LsSVqwEbbQwwyCJuAqq5FjX9fIfDtvR8zCnHpWa8OPVtDKddWbvJ/6K + t1JL31cOuTc6GPRkLEQL8Ysk0v9JTElk/14SEWPfTgsz3swL1pnBUkXmO+rQMM+mhYF5TxdsPpFk + iKsFvr0QN/2qBolL5RfxguCix2aWzvvJx0aaBbE8wpduXut9nlzq7m/az4QQaAmbyjpspLideC5z + GJ6xP5VdUz4ZBv/LEw5VK3WHzjp5vjCtrfJWpAmmCe4hOka/AQAA//8DAH3KQL5SBQAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0afbbd5918b8-EWR + - 8f2f88ba8f2e2082-IAD Connection: - keep-alive Content-Encoding: @@ -91,14 +91,14 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:14 GMT + - Mon, 16 Dec 2024 14:58:56 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=nxPGrcXhbymyBih2t7CgJx2kdUAj2c8FEUM6SgQuXQA-1734303554-1.0.1.1-0gC9PrWCtGQtTp5RVHnIIDJR3yD.DgFli7YVq20QDgOcIMxDrBJn9wXO3e.Fak8_HP_bRTKACpjVZuouGgX1ig; - path=/; expires=Sun, 15-Dec-24 23:29:14 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=KxfM6APw09zrJass.4OtAFMoUu2vRIbwSxiTzDsvK18-1734361136-1.0.1.1-ziXwyO6Y2EVlj7avwtfxuCobMUBGVXaF4V0POP9dVYb7lePh2DTCi3wGND0Bfqx44OdTFy0.IruHQWWlbKLdYg; + path=/; expires=Mon, 16-Dec-24 15:28:56 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=GidLrZtG7TkkHj.qFUWUeIOOu8KdGLixVP6dtmMZgV4-1734303554457-0.0.1.1-604800000; + - _cfuvid=nZLW2HYfOt4n61DxTEcgN8TpbBbjLbo4wSNNVqPlytw-1734361136731-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked @@ -111,7 +111,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '452' + - '848' openai-version: - '2020-10-01' strict-transport-security: @@ -129,7 +129,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_c1efe4eef4aaf287a93b88df4e64a2c0 + - req_6d31b20403f791a45054bca0cee0fb38 status: code: 200 message: OK @@ -285,7 +285,7 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b00ec6af78d-EWR + - 8f2f88d1eeb7c974-IAD Connection: - keep-alive Content-Encoding: @@ -293,14 +293,14 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:15 GMT + - Mon, 16 Dec 2024 14:58:57 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; - path=/; expires=Sun, 15-Dec-24 23:29:15 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + path=/; expires=Mon, 16-Dec-24 15:28:57 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000; + - _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked @@ -317,7 +317,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '149' + - '122' openai-version: - '2020-10-01' strict-transport-security: @@ -335,7 +335,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_bc5d97a0802a0175e3fa12eae77be9f2 + - req_72b8f94aaf88e5c0bc9f6d9ab26e7497 status: code: 200 message: OK @@ -355,8 +355,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; - _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -382,123 +382,123 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA+x6yxKyOrTm/DzFrj21q+QmWZwZAoJySVQQtUeCioCIXBIgp/rdu/Dv6q6e9qgH - /8QqNWIgK98t67/+459//q2T4pH2//7nP/++867/97/Nn91v/e3f//znv//HP//8889//V7/r5GP - Knnc7/kn+w3/fZl/7o/x3//8R/jfn/yfQfO1u8+W6MYhSMbL90bVyXQIC6JngSazdhTgH7pmFhlk - j5WBVqr2WwlYWIefcKr2gwCS2FzZurzkaPDK8w0CJb1QAfKRT5LmUiCC8iGu0n8Srppqhfz9xcd1 - c5aL+k5LHbTVU8OC/H17Ezl6A1qPnsFcfkmL6T64OUSX3iVb8bsK+7XUULiMvc5wcXwlU3B7RCjz - 6gfTd0GOeE9bAW3Hlz/Px+Di/SBZcH3WMdvibNn294Nkg7ZoCbEpWXpdOq4MFEnNiy7PkVN0Rc0y - UIPthlz924BGdVUAtK51YbGnsHY6Hr9HaL67AQMmdjKsHmWGlkLNsezaZ49bZyEH9ZBvCc52ikeD - WxyDHbhbslnFechxrBsaNcQHXi7u67AXY9uH8ZmnxLpuXmjIWyEDKLMbHtLj0E7f0FZgwd0Gh/fy - zVtyRDosssWXueh2bGlyUiO0EOMWSzuvDKko3Go1Pzkp2bzFuOWL4i3BYVuOBLvtyeQrZKQQVteR - uPLHbsfHKaug0a8d8Y5eY45QvyJQFdvD1Ft8i/p+sCV0L48uM3uovUFNzrnayOqZNs6lC7v7Gy0Q - KZZLPA3vc0jfy0uzWgoNJ66eyiYP2EQ1V810KuPT2pMVf7OF6dseKUIPjNhuajByHvWBHPa6Xkh7 - lgHY6p4wr/aDUM6X+Q0dyshhFrshb4JJbzS0sVS64E4RDnTtHVH6cU3mXc5t2FEl3EJZ1hEJlG5r - DsVtpcBNzHNmLi8RGpLvFsAfJ41stuUHdendU+BIH0C8VVRwfr4ebFSjx5Y49aosBt8tKYzm6NDV - 6J4SXsGkaLne34jdo5r35H4oQbbEHTMut53Jd3fNArSqIqrJ1ScZxFekQ/ZuAU/9ySuGjS91qD+x - MyGL+yv8zQcdytih0lW3k04NviqoWl9h4bjrkvEgvwYw9PNELLzSC/nkDAA7yaiYeRTLhC0/lwiu - yiVhZKW+zC47dBF4B8sikcI2aBI29wiEY+OQTTd2XhORdI82i+TFyG5rF8NbdC1kJdmH7EzxFhbe - Kc+XePN0qFJqOpIWeiIgcXtTyfqVBiEjR2SokV3sSPBGQ8troY7A8+maGedom4wp2sea7OMULxOH - t1O9phN8w8Ob2OuOthO9Og9g2qYlnrHahhOK2hSarzfQSTi8zP61vixAuKcCcdv3vhjIMrB/908X - TrIL2afQsSaTr0R0HBZht12fXXC15Y3oclWbI4SAoRG7mByKY24y214p0EQjpUNbTmZHmieo0rEY - 8CtdBd6gnIQHHILBJPujVBTcOJ8N5OylkLjngIaV53sYiGzXbD0KpOCfs7eFsxa8SRA9Tc5te1TV - QHlciPHZNeE0cdGFcmXEuM68uJ1iZ9ehTDUXeLHoUdiukPGATLgUbPs+yGb9IcdYq0tlQ2zFaUzu - 0CVdlZ5CmLHbOiFL6zUFqrhb4h6dnlPbdht18Zg+7Pc8h4M05dC/0p7dPbUPp1LKKjgfdIfdU3Hw - Rj/ydXVduiFxrh8DTebmvYXu8tkTT1mTZLjTzkDMiRVmRfm6kAZ6swEPSUCX0sJrx9dZqpHp+0/i - 73ZvxM1adlHQ9QaxzqTjE7+5D7R82Iw4p4vCx1pfZtBclzrTN6+rN+21RwVRfnyyta3sOM2cYEAf - YepZIFdOKIiNb6nhfohJulTOyWdRvIUfPlFZ3WrhR/paEjDNaQkhm3sxgBupAP3OZp5HolaqHlUK - H+16JRvurk3psc4pkn0/ZT6RDiZvv/qgzfMhuvtZtcO0CKn67t5b5hakTIZNk6jwJHzAvLN5wv2P - 0QA3kjNbP6tL8sU0wiBS0NhBf0a8lzdmidQ0EsnmdI+SAV9kARKrP1L05orXn89qhTorwFT7Uiiq - UwY3CEo7xsPi++Sj0fQNfGH9xa0ml970Xl5qNOM9cZKx5/QlmTbYh9xjPzyioCoU9JUfsLVx2hV8 - Nd0uq2u4GYn9sG3ecceUUCzuR2aFy5c3zPWNClX7sJ26P814pwMa3uJAXOFxT/hhPVxQ+tmaJMJx - loz626j+/J6IWwmxbyi5sKpKgznfAXl9SUoLXDXXiZnQPeeZoxkw8xdx9kcl5ANIAGly64nRn9pi - 2mY3QSVX78Dw6nND03OoDeBgmWTvfq5F/ygevsJbL6Hiuj4mw/P+vazka+Bh4ZVQc4S024OqHn1a - vhY4ZM+02cJFUo4kPCOLj5xvVKDhZNN5PyWTWRMFpa/FiPnONwvx9tLd33gW7glKhmmRUNQ4w5LN - 9c9b13IvKDuEKp4+nduO/XCL1fqej1jllp1MDV5H2rw/iWN1WThaso6BpreJbJ5fMWEBUzsQ6UIj - 5vSsE0aOXAeSZW/mqd4tmc6XrwApjA+66qwITRPXXNQjfT2vb8D51o8AHhM3iB7bJOQSOW6Rdt8/ - GB5eathPaYbRzJ9Mt4WgHeRKtlAu+gIW1UBBY3iYDK2UhBOzyrNZTD++rMT0TZfbd5WMJ62JoXfV - LzPf6uRRV211MFaORL/EAW8M3uX2z/ofVBfxvo1dF/rFPWHrVbgJebUpDMhJqpPjPfu2nbhYYXXP - pI45U75JRivoQYXbcsOCW3rj07RUsfoSlQ4PQzPxni36h7pRwWGPN7m0FJaLIzyvpw25T1nFOzdY - UeQfq4Dp83gWc28C8VYi4rVfn3PH7BRU+JVDcHO6mtO5eqboMbzXzF83g8k/OLS0vtlcsZix2JyE - XM3htPic2CZjsTfp75Wg+eReECfvi6TPF54C78W4Yz++77E3pj+8mvFpZTL5LQtA0SfHq1H9tlMz - lDZiq/pAyDKS+WBJF0C6rQgYSQi3soChQSDRBC9t/2B2P/yk5GJiGSvrUFqR7Q0+4OqEHLFjyr/r - e++Hjcd01Xt8nxQ2rMznlUKKeTuRhy6BcWYbyuPN3mQz/qOkPZyIXaWhyZRIw6sdfql4pXy9P3oK - pOHp/OFvKTmpMZr5mNnb2jKHdxYBUgu6o8rutjT76i5j6MJkwTbncGfSRb4okXi6fklwim/Jp832 - FNZYrtnW23PENdaocPYbi1maaCeTqoYYBjWYqCx4diJ/WmUPVyrb9EXLAg0JxZl6P9o7gqNC9yRD - sBagr/ucOLbUhuxhXBew2PrlD7/bPtVPC2DyYz/zpZuwSzNimPUhu7rrpB2dNmu0TXmxyN0s16Z0 - ffgKOih0x/yz4phjrcsZqG/FxfDpvyZnum9Af+rPbH211yYtpawECe96YgVmi/hwSzFgRFSyWaQG - mli0VuFX/+70qkz+ft47hI93iZiuqXrDT89+7b1E8Kx/h90DYlDu9E0LZfnktF5XEzCyLPHyFQqo - T9E+0rAv3YgjGiniQbKpQPjoO3abXrY5edkQafmhMVhAhbs5HnKoVSPqBbZz16jtQt5nqixvczLr - U84DJZ9gdz++WLALcm9KlLv9u1+K3C4KB3ExYu3pbk9Y8pPBG08GAzSd7weGQR+8aVVzW1M1VjHf - v47FVPRKhnaenVH48WHU1xSG/BZjkffrRGLLqAOl6SO2pmXBB2l/uMDy01iUhxerGCEt92iuD7I9 - sm/RUzcvNaf+Mip/UwvxClQFHYdmQccqD9B0vrwEBMuzQ/yIaXy8UW341Reu3wfZozLOVdiZLCDu - WTPanz5W8+/zwez4M3qTPZ4esJaiiFluE7fsfchsNO8n5u1Pp4StO2sL+PiUaO487bA7CZ6O6gXK - id/chHYilaLAj++290f7W18KUudYFGb9woTuIKkwFjti6Q+xncxDjkHxspLpr8vBnPm4AfW2FYgn - YDP8vUd+KFYEm/5z5ku3VGk42MzZPruE6xtJRXw3NMQfV03Iz/na1bIc58Qv18t23IU0/+ELPaPk - gMbL99hBFj99XI58CKcDliZYfmqL6W90SPrnR+8gbvuUEGnRtrO+VLQ8lEw8RM+yHeAz2Vq68zv2 - q98xtm8PRA/lhpnyu+aDdeEDPOuXyPBJixELp+oI+/hwwcL3bpmC6YQNPK/nDdsZiwJ9raBfAPNz - l1nHnR/22v6Dgb4XDu3yT128riQztE9edYQIBz0RzFKRAIvVk7iHMminOJIjGO9x8ssL+LB261p1 - 3UuNS1tuWn5bWRF4TsaweLl++WThEwYfPg1zyrhMRnnjlWiuN+IlgstHKw9tcPzHnk6K1Xts1L85 - QPieiFmHTtL99JozBRbbLY3M45Z6PkLbty6zwlTn/VOQIrBpvsJP2z94I5b9Ixo86UiCQ4HD6eUO - e5j9DC12flFwidy2YEmPfl7/cfYLo/LDS6wZh64dxe2lBPRoMJbLRm27jb/oYLdVnlhSHNek3w8Y - aLBbnZA0iMKxbg4DFD0ov/oIuZilg0pSN2fOOB28/tUVE7zOyxPR/TstvrP+Up3rY0l8J7OQhDx0 - gfQ4rMnt58+j4rYFuC9i5mte502tK8QgJEeOB35eeNNGyB8wHjWZWYfK98ZSXMXo52fsD3OKAT6q - DXqfylgptYyPvzykmdiGWfJ349H0kV+QHjsls+tj7dEff81+d95vYsK1QN9qc/5Dh11gIKEx1Qmo - Gu+o/LpRNHXjrkRyVj6Z+e0f5nAu9IuWWaFHdsq39Toj1Sd4cdqSgKtaMW5a04XgdGJk+z6czYHt - UPzz31hOI5zwyMIPmPMU5vaLN++fj30Dn9SoiS7cH3yq+cpGBwIWS+yd4I34ymw070c6dZMXSrNe - AGGUZLYL+nc4TksD4H4qCvrLU9pbwrboWhpXZh9eQ8vNYlWp+w/WmZEGJ6+UrTSC4h35bM4PvGG/ - fx+RVK0mRor7Ew1GuKeInoOM/fKVIZL3Otyuo89iMzeTyd9KPtK2avpHv/fI81048akm3jxf7n/c - BsLjmtHpHNVJ05TMWG10wyDkcV6Gv+ePqpRmxEMPzKlOX6oWuv6RmIecoeGiXSqkMKVi8UPTwime - 0gzq2HuwHXFSb8rDLP6zn3zL3ITTbpzg93+kwu/E6410MIDkXUUO+0hCvSANFw286fTTSyH72noO - LLM0tntsnJavi8hSFX/U2M5Ciclbr63UEtt75hmrOvnjd+d8jWD9KZjD5ezFaNZ/ZNZjJp39P2o2 - 5YL46hTz2b/cEA18g20XiVmId3/IQCoeF2YY1zf/6YPVPg4vf/h0KFWnXrWi8WDWqkrD4YaHAeLK - WpOEb/Ni2t+QhKSnfWI7e2R8PD5ZjZbh3qDtzBfjdXInFIUvhRkG2KaccSJBWW8occ6LNZfug5Fp - SBV74kZZ4fEgCUpIV7FFPHFzRb1wTTCslezE1lRRzSlPLzcIVc6JByUpJvei77UnGQcWN8swbLpx - XUJqrW+Uyo1Z1IMy1H/8qaWuBv7dtJ6LSkk60QUtC3MytupWlY6vgWwid8mn9RWG1Zw3UPEY1C07 - NUmH0MZW8fNxfiZj4oc65KFg4uEoFS1fnAYfvd9ExKuUHNG0i04TzP6TmXlThTT9HFN08kOggiZW - 4Th6uoXAG04zf5do1p8C+ukxrAxl0lwrWMB5V3PKp2cdNmfklyD3n4Cq5ltPBnBTFToGBTNW6gv1 - lXPI1apiC2K4bPLYUgwA7Twro5M6KiGNzCeGpA1PWKSfDA2rT1r98BbLztUrxl/e8yjKIwst2zTH - fbVu4PtpP3/wePTHlw/w0QS8eJNLMZxXlQ3q91SSbYpc9ONfKJyLOOt/yeS/+j35B2CmZRdm242K - hK5UtGnmfq7t5CP9T76L60RsvOZjoQmGZWLN+sdE8qcd9nBQFZftH8uET8hDt9VWog825wueND9P - 2HrThWz6qEvGJrjZ6OTpBbma4i1h+faWI0M/TXia8zA1jpYRzHhD7Pz7MftqvythxTAl29PyFHJ7 - siiES23D/KES+dgnvf/Dk59fRLw3r7q64aXwB1/bBg8VTC+jIltvH/KhPd0UyCvDIqZZGK0sX1wf - hou4JuvT7ZzwjXigcJykJRU+QmGOy2WXI2nzWjNMy8LjxcGtfnxLrGy7KqblfWWgo9Z2eFjT3mvJ - sMcA4Wdipr8kxdDfNIDdqymJ1fXkj78F173VP33Fx7q5DjDrP7po66CYVjWykXTYn8k12R29Ycqk - TH17/pk5mvAxf/k7RMtDgAdlVxfTS/IscPx0zwzh8PK4Fmy3cMWMEvO6fhff6+QOaM5PmKvtnXYw - vtINbe7GwDbaDoe81i431Yv7J/OX64w3wx5UwJWosI2BKv49tiUFN62+zL2aZtLi7t3BjL9sZ4pq - OKjBS4UuvhIsVxlJxu2zLVEUFgoexwMLx6I8YKhXR4fy9DAm9DpNhqZJekGsTSl5lDWshDUWa+Je - 1RXqwqsqQGIUB7b1Y6EdOd1GSrrDHdWip4mkK2osNOe95Dbz3eAPqwUKVmzE1tO48F9+COiNj3Q5 - +9F+5dIapDMziH3Vq2Sa/w+dM18meM5DhrL4AsiRuSI47j1TFK6hD42snKkqPLSwdwaKQZ9nKpxJ - h+Tl5xID2G+TmLJRF+Oejh3U6zEladDZ5pjZjwzgDjExT6eE96dss/3lOex6jm1zYs2nRJGQZmSt - DBtzFCxqoSsSRfbzX+/L99aBGmk5Xcx5EEenW4dm/8Q2anDh319ek64ii4Vkd2uH9G4qIMnCjTld - t2+5dtxQbTk9Jry3nDwZxz4X/uDByBdqy2k0urDY4pKQeM29SRSODcjt1aePh69zsbAFQ/udLxhE - PBZfc68u1J+eXmzeUcHj/deA0eTOnH/FaLShOEL4di+z3rsVU1AlNzS85YEKU1EWjX7wKUqW7ZlO - M59w8qm3v/x5Pn8y0KRkng775NHM9UISijx+g7NG3vP51TX54y9mPMEK4ZHXOfk6hqpd7Vk6VCLi - 9xIoevbMIb88ZsoXyFXL/qCzTWPpLZ/rFcmklehy5oPvkLodnJ3HNJ/nVO1Y9us9Ch2yZkZzjc1R - XNYpSL4S/fK3drx4Yg7n1eCRx/dhFFLe2AaKjeUFa4FOzS7k7xzsrEO0M7SyFR+KGYF172pmzHp8 - gmlbg+8bO7YJrac5+dMlRefo/iK7x+ZTdLdzJv3OL4g1n190avLMID6fbmx7ubhFpypCB+9ikRI3 - WPfhcEurCyRZWjNPutCkI8vAgqNeVDjJdd+Tue2XwPZqQOWrlRfjulebH54ya7kcQ4q+lwpmvGWb - VWyE8maxnn75PFXcnrfTtQKA8XZyycZ5fMyRuk0Ft/PHY7/8XsiFsoK7HfP/5fd0+lLUNT+ZmMvv - LReT7+CDZe0i5i0PKv/pedg06ycLdkOeTHJxMSBF2YZtRykIpy4sb2jOm6hmNWsubQ2xhjOWNoQc - Hkc+tg/Vh1RwJ7YO75k3eRHaqyU5fefnG3iykqlHKFYUY3WpHflgpNsB9OZZkbm+eCdI6y3aLk8u - +/lL+uPnmHY9M/xL6PF0GCdNk4yCmKoYFfwKhxjy9MGYj4u0HS6efdTUNBaJGddDyOzJ79B8Ponl - Od/jeaPtkStGlMz6Iemc6/WIAl3rcT3z1UR7oQLUSwvmphJLhnjTYrDVI2G72DwUPOnJHsnntcsO - ez0r+kypBXjfrg6zNmVsypdmhdG83mQz3TrON9+0gZeodn/Oc7nQXSX4huEbq5+pLIasuqaI5LQi - W00pwm4F6w5mfUjfu8Wj7Sxpv9CifuOT3VsJi5G0DgZCPjDzE+Pd/YAF9FvP78w/XXkuFTgngkBM - ZbtLeG8edM2JDj3mlnYt2OlUlehb3TBGa5MXfKefFYjdqMUTrXVPSo/vGNppHcz8yzk3jtcU5utT - UVsZreh5IcDVO0bMcW3ZHGe8gUMwmeR33jP7tRQBCRhFfHk12TOcbC3/3h90iLjKh/iQHZFwfwhY - S48ETa8+NND91ol0+Ww4GnMxkcAhqMFi5D7RuAurfEVpxZljhK1J02MfwTGwvT/3M+LomyNDlu5z - HnRoWXW7bGHWyxhmvfF1cbaAf39dAf/jv/0/dBSIfzsK/nYU/O0o+NtR8Lej4G9Hwd+Ogr8dBX87 - Cv52FPztKPjbUfD/X0fB/wQAAP//7N0xCsJAAATAPq8I1wdCSv8icpDDwsQL5gQb/y6nQfxBQKbd - Zj8w7BIFRAFRQBQQBUQBUUAUEAVEAVFAFBAFRAFRQBQQBUQBUbC7KBiIAqKAKCAKiAKigCggCogC - ooAoIAqIAqKAKCAKiAKigCggCogCooAoIAqIAqKAKCAKiAKigCj4c1HQtO3x/YIw5zFNFQaU9Cjd - lwp0cYxd3w+fq4T7Gs8pHDaBEJZbnpdyKvmSrmulBtt6QSi5xOk3b2rVs3kBAAD//wMAK1HZr4Rh - AAA= + H4sIAAAAAAAAA+ydW6+6zLbm79enePPeunfkJDVYdwgIClilgqdOpwOoCIjIoQqonf3dO/hf2d2d + vuvLjjczmcjkVMV4nvHUL9P/+Mdff/1dxfk96f7+519/v7K2+/vfpm23qIv+/udf/+0ff/3111// + 8f35f+x5L+P77Za90+/u3w+z9+0+/P3Pv4T/2vK/dpqO3b7XRDf223i4fCKqjqZD2DZ85Gg0K0cB + /qZLZpFe9lix1QrVfilbFlTBOxjLXS+AJNZXtiwuGeq94hzBVkkuVIBs4KOkuRSIoLyJq3TvmKum + WiJ/d/FxVZ/lvLrRQgdt8dCwIH9e3kgOXo+Wg2cwl1+SfLz1bgbhpXPJWvwsgm4p1RQuQ6cznB+e + 8biN7iFKverO9M02Q7yjjYDWw9Ofrsfg4m0vWXB9VCe2xum86W57yQZt1hBiUzL32mRYGCiU6ied + n0Mnb/OKpaBu1yty9aMeDeoiB2hc68JOnsKa8XD4HKD+bHoMmNhxv7gXKZoLFceya589bp2FDNR9 + tiY43Sge3UanE9hbd01Wi1MWcHzSDY0a4h3PZ7dl0Ikn24fhkSXEuq6eqM8aIQUo0gj3yaFvxk9g + KzDjbo2DW/HiDTkgHWbp7MNcFB0aGh/VEM3EU4OljVcEVBSiSs2OTkJWL/HU8Fn+kmC/LgaC3eZo + 8gUyEgjK60Bc+W03w/2YllDr15Z4B682B6ieIaiK7WHqzT55ddvbEroVB5eZHVRer8bnTK1l9Uxr + 59IG7e2FZojk8zke+9c5oK/5pV7MhZoTV09kk2/ZSDVXTXUq4+PSkxV/tYbx0xwoQneM2GasMXLu + 1Z7sd7qeSzuWAtjqjjCv8reBnM2zCO2L0GEWi5A3wqjXGlpZKp1xJw96uvQOKHm7JvMu5yZoqRKs + oSiqkGyVdm32ebRQIBKzjJnzS4j6+LMG8IdRI6t18UZtcvMUONA7EG8R5pyfr3sbVei+Jk61KPLe + dwsKgzk4dDG4x5iXMCpapncRsTtU8Y7c9gXIlrhhxiXamHxz0yxAizKkmly+4158hjqkrwbw2B29 + vF/5Uou6IzsTMrs9g+/1oH1xcqh01e24VbcfFVStK7Fw2LTxsJefPRj6eSQWXui5fHR6gI1klMw8 + iEXM5u9LCFflEjOyUJ9mm+7bELy9ZZFQYSs0CqtbCMKhdsiqHVqvDkmyQ6tZ/GRks7bz/iW6FrLi + 9E02phgFuXfMsjlePRyqFJqOpJkeC0hcRypZPpNtwMgBGWpo5xuyfaG+4ZVQheD5dMmMc7iOhwTt + Tprs4wTPY4c3Y7WkI3yC/YvYy5Y2I706d2DaqiGesVgHIwqbBOqP19NR2D/N7rm8zEC4JQJxm9cu + 78l8a3/vn86ceBOwd65jTSYfieg4yIN2vTy74GrziOhyWZkDBIChFtsT2eeHzGS2vVCgDgdK+6YY + zZbUD1ClQ97jZ7LYer1yFO6w3/Ym2R2kPOfG+WwgZycFxD1vaVB6voeByHbFloNAcv4+e2s4a9sX + 2YYPk3PbHlR1q9wvxHhv6mAcuehCsTBOuEq9UzOenE2LUtWc4dmsQ0GzQMYdUuGSs/VrL5vVmxxO + WlUoK2IrTm1yh87povAUwozN2glYUi0pUMVdE/fgdJzatlurs/v4Zt/n2e+lMYPumXTs5qldMBZS + WsJ5rzvsloi9N/ihr6vLwg2Ic30baDRXrzW0l/eOeMqSxP2NtgZizklhVpgtc6mnkQ24j7d0Ls28 + ZniepQqZvv8g/mbzQtysZBdt284g1pm0fOSRe0fzu82Ic7wofKj0eQr1da4zffW8euNOu5cQZocH + W9rKhtPU2fboLYwd28qlEwhi7VtqsOtPJJkr5/g9y1/Ctz5RWV1rwVv6WBIwzWkIIatb3oMbqgDd + xmaeR8JGKu9lAm/teiUr7i5N6b7MKJJ9P2E+kfYmbz56r03XQ3T3vWj6cRZQ9dW+1szNSRH3qzpW + 4UF4j3lr85j7b6MGbsRntnyUl/iDaYhBpKCxvf4IeSevzAKpSSiS1fEWxj2+yALEVneg6MUVrzuf + 1RK11hZT7UMhL48pRLAt7BPuZ58HH4y6q+EDyw9uNLnwxtf8UqGp3hMnHjpOn5Jpg73PPPatRxRU + hYK+8LdsaRw3OV+M0WVxDVYDse+2zVvumBI6ibuBWcH86fXT/Ea5qr3ZRt0dp3qnA+pfYk9c4X6L + +X7ZX1DyXpskxKc0HvSXUf75eyKuJcQ+geTCoiwM5nx65HUFKSxw1UwnZkx3nKeOZsCkX8TZHZSA + 9yABJHHUEaM7Nvm4TiNBJVdvz/DiHaHx0VcGcLBMsnPf17y753df4Y0XU3FZHeL+cftcFvJ162Hh + GVNzgKTdgaoefFo8Zzhgj6Rew0VSDiQ4I4sPnK9UoMFo0+l9ikezIgpKnrMB841v5mL01N3v/izY + ERT34yymqHb6OZvmP29cy72gdB+oeHy3bjN0fXRSq1s2YJVbdjzWeBlq0/tJHKtNg8GSdQw0iUay + enzEmG2Z2oJIZxoxx0cVM3LgOpA0fTFP9aJ4PF8+AiQw3OmitUI0jlxzUYf05TS+W87XfghwH7lB + 9JNNAi6Rwxppt92d4f6pBt2YpBhN+sl0W9g2vVzKFspEX8CiulXQEOxHQysk4cis4mzm41cvSzF5 + 0fn6VcbDUatP0Lnqh5kvdfSoqzY6GAtHoh/igDdsX8X6z/jvVRfxrjm5LnSzW8yWi2AV8HKVG5CR + RCeHW/ppWnG2wOqOSS1zxmwVD9a2AxWi+YptoyTi4zhXsfoUlRb3fT3yjs26u7pSwWH3F7k0FOaz + AzyuxxW5jWnJW3e7oMg/lFumT/uzE/dGEKMCEa/5+Jw7Zqug3C8dguvj1RzP5SNB9/61ZP6y7k3+ + xoGldfXqisWUncxRyNQMjrP3ka1SdvJG/bUQNJ/ccuJkXR532cxT4DUbNuyr9x32huRbr6b6tDCZ + /JIFoOid4cWgfpqx7gsbsUW1J2Qeyry3pAsg3VYEjCSEG1nAUCOQaIzntr8322/9pORiYhkry0Ba + kHUEb3B1Qg7YMeXv8b3X3cZDsug8votzGxbm40ohwbwZyV2XwDizFeWn1c5kU/1HcbM/ErtMApMp + oYYXG/xU8UL5eH/8FEj9w/mj31J8VE9o0mNmryvL7F9pCEjN6YYqm2huduVNxtAG8YytzsHGpLNs + ViDxeP2Q7fEUxe8m3VFYYrlia2/HEddYrcLZry1maaIdj6oaYOjV7UhlwbNj+d0oO7hS2aZPWuSo + jylO1dvB3hAc5ronGYI1A33ZZcSxpSZgd+M6g9naL771u+kS/TgDJt93k166MbvUA4bJH7Kru4yb + wWnSWlsVF4vczGJpSte7r6C9QjfMPyuOOVS6nIL6UlwM7+5jcqb7BnTH7syWV3tp0kJKC5DwpiPW + 1mwQ76MEA0ZEJatZYqCRhUsVvvPfHZ+lyV+PW4vw4SYR0zVVr//62Y+9kwie/G+/ucMJlBt90VyZ + PzitluUIjMwLPH8GAuoStAs17EsRcUQjQXwbr0oQ3vqGRePTNkcv7UMt29cG21LhZg77DCrVCDuB + bdwlatqAd6kqy+uMTP6U862SjbC5HZ5su9lm3hgrN/t7vxS5bRj04mzA2sNdH7Hkx703HA0GaDzf + 9gyD3nvjouK2pmqsZL5/HfIx75QUbTw7pfDVw7CrKPRZdMIi75axxOZhC0rdhWxJi5z30m5/gfm7 + tigPLlY+QFLs0DQ/yPrAPnlH3azQnOrDqPxJLMRLUBV06OsZHcpsi8bz5SkgmJ8d4odM40NEtf47 + v3D12sselXGmwsZkW+KeNaP5+mM1+zzuzD69B2+0h+MdllIYMsutTw177VMbTe8T83bHY8yWrbUG + fHhINHMedtAeBU9H1QxlxK8joRlJqSjw1bv17d58x5eC1DoWhcm/MKHdSyoM+YZY+l1sRnOfYVC8 + tGD687I3Jz2uQY3WAvEEbAbf35EfiCXBpv+Y9NItVBr0NnPWjzbm+kpSEd/0NfGHRR3wc7Z0tTTD + GfGL5bwZNgHNvvWFnlG8R8Plc2ghPT18XAy8D8Y9lkaYvyuL6S+0j7vHW2/h1HQJIdKsaSZ/qWhZ + IJm4Dx9F08N7tLVk47fsO3+Hkx3dEd0XK2bKr4r31oX38KieIsNH7YRYMJYH2J32Fyx8bpYpmE5Q + w+N6XrGNMcvRx9p2M2B+5jLrsPGDTtu9MdDXzKFt9q7y55WkhvbOypYQYa/HglkoEmCxfBB3X2yb + 8RTKIQy3U/zNC3i/dKtKdd1LhQtbrhseLawQPCdlWLxcP3y08BGDD++aOcWpiAd55RVomm/EiwWX + D1YW2OD49x0dFavz2KB/MoDgNRKzCpy4/fo1Z9xabDM3Uo9b6vkATde4zAoSnXcPQQrBptkCP2x/ + 7w1Y9g+o96QD2e5zHIxPt9/B1M/QfOPnOZdItAZLunfT+A9TvzAo33qJNWPfNoO4vhSA7jXGclGr + TbvyZy1s1soDS4rjmvTzBgP1dqMTkmzDYKjqfQ95B8p3fgRcTJNeJYmbMWcY9173bPMRnuf5kej+ + jeafyX+pzvU+J76TWkhCHrpAcuiXJPr252EerQFusxPzNa/1xsYVTiDEB457fp5540rI7jAcNJlZ + +9L3hkJcnNC3n7HfzMl7eKs26F0iY6XQUj5885B6ZCtmyZ+VR5N7dkH6ySmYXR0qj371a+p3p/dN + jLm21dfalP/QfrM1kFCb6ghUPW2o/IwoGtthUyA5LR7M/HR3sz/n+kVLrcAjG+XTeK2R6CM8OW3I + lqtaPqwa04Xt8cjI+rU/mz3boNO3/8ZyEuKYhxa+w5SnMLebvXj3uO9qeCdGRXThdudjxRc22hOw + WGxvBG/AV2aj6X2kYzt6gTT5BRAGSWabbfcKhnFuANyOeU6/eUoTxWyNroVxZfb+2TfczBeluntj + nRnJ9ugVspWEkL9Cn035gdfvdq8DksrFyEh+e6DeCHYU0fM2Zd98pQ/lnQ7RdfDZyczMePTXko+0 + tZr88e8d8nwXjnysiDddL/ffbg3BYcnoeA6ruK4LZixWumEQcj/Pg+/zR2VCU+KhO+ZUp09VC1z/ + QMx9xlB/0S4lUphSstNd04LxNCYpVCfvzjbESbwxC9LTn/fJt8xVMG6GEb7nIyV+xV5nJL0BJGtL + st+FEuoEqb9o4I3Hr18K2MfWM2CppbHNfeU0fJmHlqr4g8Y2FopN3nhNqRbY3jHPWFTxn353ytcI + 1h+C2V/O3glN/o9MfsykU/+P6lUxI746nvjUv0SIbn2DrWexmYs3v09Byu8XZhjXF//6g8XuFFz+ + 6GlfqE61aETjzqxFmQR9hPseTqW1JDFfZ/m4i5CEpId9ZBt7YHw4PFiF5sHOoM2kF8N1dEcUBk+F + GQbYppxyIkFRrShxzrMll269kWpIFTvihmnu8W28LSBZnCziiasr6oRrjGGppEe2pIpqjllyiSBQ + OSceFCQf3Yu+0x5k6NmpngdB3Q7LAhJrGVEq12Ze9Upf/elPLXXR88+q8VxUSNKRzmiRm6OxVteq + dHj2ZBW6cz4ur9AvpryBiodt1bBjHbcIrWwVP+7nRzzEfqBDFggm7g9S3vDZsffR60VEvEjIAY2b + 8DjC1H8yM6vLgCbvQ4KOfgBU0MQyGAZPtxB4/XHS7wJN/lNAXz+Glb6I62sJMzhvKk75+KiC+oz8 + AuTuvaWq+dLjHtxEhZZBzoyF+kRd6ewztSzZjBguGz02F7eANp6V0lEdlICG5gND3ARHLNJ3ivrF + Oym/9RbLztXLh2/ec8+LAwss2zSHXbms4fNu3n/q8eAPTx/grQl49iKXvD8vShvUz7Eg6wS56Ku/ + kDsXcfL/ksm/8/fo74GZlp2bTTsoErpS0aap+742o4/0P/kurmKx9uq3hUbo57E1+R8Tye+m38Fe + VVy2u89jPiIPRYu1RO9syhc8aXqesPbGC1l1YRsP9Tay0dHTc3I1xShm2TrKkKEfRzxOeZh6Cuch + TPWG2NnnbXblblPAgmFK1sf5MeD2aFEI5tqK+X0p8qGLO/9bT779IuKdedXVFS+EP/W1qXFfwvg0 + SrL2dgHvm2OkQFYaFjHN3Ghk+eL60F/EJVkeo3PMV+KewmGU5lR4C7k5zOdthqTVc8kwLXKP53u3 + /OotsdL1Ih/nt4WBDlrT4n5JO68h/Q4DBO+Rmf6c5H0XaQCbZ10Qq+3In/4WXDeqvv6KD1V97WHy + f3TWVNt8XFTIRtJ+dybXeHPw+jGVUvXl+WfmaMLb/ObvEM73W9wrmyofn5JngeMnO2YI+6fHte16 + DVfMKDGvy1f+uY5uj6b8hLnazml64yNFaHUzerbSNjjglXaJVO/UPZg/X6a87negAi5Fha0MVPLP + oSkouEn5Ye7VNOMGt68WpvrLNqaoBr26farQnq4Ey2VK4mH9aAoUBrmCh2HPgiEv9hiqxcGhPNkP + Mb2Oo6Fpkp4Ta1VIHmU1K2CJxYq4V3WB2uCqChAb+Z6t/ZPQDJyuQyXZ4JZq4cNE0hXVFpryXhJN + etf7/WKGtgs2YOthXPg3PwT0wgc6n/rRbuHSCqQzM4h91ct4nM6HzqkvEzzlIX2RfwDk0FwQfOo8 + UxSugQ+1rJypKty1oHN6ikGfrlQ4kxbJ8/flBGC/TGLKRpUPOzq0UC2HhCTb1jaH1L6nADc4EfN4 + jHl3TFfrb57DrueTbY6sfhcoFJKULJV+ZQ6CRS10RaLIvv3X6/KJWlBDLaOzKQ/i6Bi1aOqf2Erd + Xvjnm9cki9BiAdlETZ/cTAUkWYiY07a7hmuHFdXm433EO8vJ4mHoMuFPPRj4TG04DQcXZmtcEHJa + cm8UhUMNcnP16f3u61zMbcHQvusLBhEP+cfcqTP166dnq1eY89PuY8BgcmfKv05osCE/QPByL5Pf + i/JxW8YR6l9yT4UxL/Ja3/sUxfPmTMdJTzh5V+tv/jytPxloVFJPh118r6f5QmKKPB7BWSOvaf3q + Gv/pL6Z6ghXCQ691suUJymaxY0lfiojfCqDo0TGHfPOYMZshVy26vc5WtaU3fJqvSCaNROeTHnz6 + xG3h7NzHaT2nbIaiW+5Q4JAlM+rryRzEeZWA5CvhN39rhosnZnBe9B65f+5GLmW1baCTMb9gbatT + sw34KwM7bRFtDa1oxLtihmDd2ooZkx8fYVxX4PvGhq0C62GO/nhJ0Dm8PcnmvnrnbXROpe/6BbGm + 9YtWjR8pnM7HiK0vFzdvVUVo4ZXPEuJul13QR0l5gThNKuZJFxq3ZL614KDnJY4z3fdkbvsFsJ26 + pfLVyvJh2an1t54yaz4fAoo+lxKmestWi5MRyKvZcvzm81RxO96M1xIAhujokpVzf5sDdesSovPb + Y9/8XsiEooSbfeL/6vd0+lTUJT+amMuvNRfjT++DZW1C5s33Kv/6eVjVywfbbvosHuX8YkCC0hVb + D9I2GNugiNCUN1HNqpdcWhtiBWcsrQjZ3w98aO6qD4ngjmwZ3FJv9EK0Uwty/EzPd+vJSqoeIF9Q + jNW5duC9kax70OtHSab5xVtBWq7Ren502be/pF99PtG2Y4Z/CTye9MOoaZKRE1MVw5xfYX+CLLkz + 5uM8afqLZx80NTmJxDxVfcDs0W/RtD6J5Snf41mt7ZArhpRM/iFunev1gLa61uFq0quRdkIJqJNm + zE0kFvenVYPBVg+EbU7mPudxR3ZIPi9dtt/pad6lSiXAK7o6zFoVJ1O+1AuMpvEmqzFqOV99khqe + otr+Wc/lQnuV4BMEL6y+xyLv0/KaIJLRkqw1JQ/aBSxbmPwhfW1m96a1pN1MC7uVTzYvJcgH0jgY + CHnDpE+Mt7c9FtB3PD+T/rTFuVDgHAsCMZX1Juadudc1J9x3mFvaNWfHY1mgTxlhjJYmz/lGPytw + csMGj7TSPSk5vE7QjMvtpL+cc+NwTWA6PhW1hdGInhcAXL1DyBzXls1hqjew344m+a73TP1agoBs + GUV8fjXZIxhtLfvc7rQPucr70z49IOF2F7CWHAgan11goFvUinT+qDkaMjGWwCGoxmLoPtCwCcps + QWnJmWMEjUmTQxfCYWt7f+5nwOEnQ4Ys3aY8aN+wMrqsYfLLGCa/8XFxOoO/v1TAf/7b/wNRIP6I + gh9R8CMKfkTBjyj4EQU/ouBHFPyIgh9R8CMKfkTBjyj4EQU/ouBHFPyIgh9R8CMKfkTBjyj4EQU/ + ouBHFPyIgh9R8CMKfkTBjyj4EQU/ouBHFPxfRIH0Iwp+RMGPKPgRBT+i4EcU/IiCH1HwIwp+RMGP + KPgRBT+i4EcU/IiCH1HwIwp+RMGPKPgRBT+i4EcU/IiCH1HwIwp+RMGPKPgRBT+i4EcU/IiC/8+J + gn/89dd//34LQlnd7q8JDOjuQ/fv/4UK/Ht0i/5dEKQ/X5VA2yi9//3PfxEIf3+aqvx0/6Orivu7 + nVCDf/33gr+7qote//v2f0yn+s9//E8AAAD//wMAK1HZr4RhAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b048feff78d-EWR + - 8f2f88d31ab5c974-IAD Connection: - keep-alive Content-Encoding: @@ -506,7 +506,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:15 GMT + - Mon, 16 Dec 2024 14:58:57 GMT Server: - cloudflare Transfer-Encoding: @@ -524,7 +524,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '205' + - '40' openai-version: - '2020-10-01' strict-transport-security: @@ -542,7 +542,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_8dcd051b01b6b87197e931e81edf1ce2 + - req_e2da439b486dcc1a1dfcc9a1d68a9cde status: code: 200 message: OK diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml index f0a4cdc548f..1850fe248e2 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys.yaml @@ -47,8 +47,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=nxPGrcXhbymyBih2t7CgJx2kdUAj2c8FEUM6SgQuXQA-1734303554-1.0.1.1-0gC9PrWCtGQtTp5RVHnIIDJR3yD.DgFli7YVq20QDgOcIMxDrBJn9wXO3e.Fak8_HP_bRTKACpjVZuouGgX1ig; - _cfuvid=GidLrZtG7TkkHj.qFUWUeIOOu8KdGLixVP6dtmMZgV4-1734303554457-0.0.1.1-604800000 + - __cf_bm=KxfM6APw09zrJass.4OtAFMoUu2vRIbwSxiTzDsvK18-1734361136-1.0.1.1-ziXwyO6Y2EVlj7avwtfxuCobMUBGVXaF4V0POP9dVYb7lePh2DTCi3wGND0Bfqx44OdTFy0.IruHQWWlbKLdYg; + _cfuvid=nZLW2HYfOt4n61DxTEcgN8TpbBbjLbo4wSNNVqPlytw-1734361136731-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -74,19 +74,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//7FRNa9wwFLz7Vzze2S72ej+CLyXQphRKein9IC62Vn72KpElRZIhZdn/ - XuTdrB2SQu/JxYeZN8O8sZ/3EQCKBgtAvmOe90Yml2Ttw9dv9+vN3fer6+XtJft5rb7kv1bDpx8f - MA4Kvb0l7h9V77jujSQvtDrS3BLzFFyzTb7M03y12oxErxuSQdYZnyx10gslkkW6WCbpJskuTuqd - FpwcFnATAQDsx2fIqRp6wALS+BHpyTnWERbnIQC0WgYEmXPCeaY8xhPJtfKkxuh1Xe9LvB/IheQl - FlDiZwdXlilOYJj1oFv4OFht6H2JMZSotOK674X3TAZBeqjrem5vqR0cCyuqQcoTfjjnlbozVm/d - iT/jrVDC7SpLzGkVsjmvDUYz8bMSsrcSCli8qhIigN/jhQxPVkVjdW985fUdqWC4Xpx6wekwZ2x+ - Ir32TM7wi1X8gl/VkGdCulmzyBnfUTNJp4NkQyP0jJi/vedpXvI+bi5U9z/2E8E5GU9NZSw1gj/d - eBqzFP5b/xo7tzwGRvfHeeqrVqiOrLHi+K20plq3PEspS2mL0SH6CwAA//8DAO5kTllDBQAA + H4sIAAAAAAAAA+xUTY+UQBC98ys6dQYDzGe4GBM17sHoXQz0NAXTK3T1dhdRM5n/bpqZHdjsbuJd + Lxzeq/fy6kFxioQA3UAhQB0lq8H2ybt2+9BtPn3p3jeW2XP+eaM8ft39lHh3D3FQ0OEeFT+q3iga + bI+syVxo5VAyBtdst1qvtlm22k/EQA32QdZZTtaUDNroJE/zdZLukmx/VR9JK/RQiG+REEKcpmfI + aRr8BYVI40dkQO9lh1DchoQAR31AQHqvPUvDEM+kIsNopuh1XZ9KeBjRh+QlFKKEOy8+OmkUCisd + C2rFh9GRxbclxKIEQ0bRMGhm2QdBeq7remnvsB29DCuase+v+PmWt6fOOjr4K3/DW220P1YOpScT + snkmC9FC/KyE7H8Jhcj/qRIiIb5PFzI+WRWso8FyxfQDTTDc5tdeYD7MBbu6kkws+wW+38Qv+FUN + stS9XzQLSqojNrN0Pkg5NpoWxPLtPU/zkvdlc226v7GfCaXQMjaVddho9XTjecxh+G+9NnZreQoM + /rdnHKpWmw6ddfryrbS22rYqSzFL8QDROfoDAAD//wMAZVeAtkMFAAA= headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b13ef8b18b8-EWR + - 8f2f88dd9fe92082-IAD Connection: - keep-alive Content-Encoding: @@ -94,7 +94,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:18 GMT + - Mon, 16 Dec 2024 14:58:59 GMT Server: - cloudflare Transfer-Encoding: @@ -108,7 +108,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '423' + - '777' openai-version: - '2020-10-01' strict-transport-security: @@ -126,7 +126,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_afefe5d59636c8e9a842ec74f937fbc1 + - req_01b32c59bd2cc37839613583552ea676 status: code: 200 message: OK @@ -145,8 +145,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; - _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -285,7 +285,7 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b17bf7cf78d-EWR + - 8f2f88e328aec974-IAD Connection: - keep-alive Content-Encoding: @@ -293,7 +293,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:18 GMT + - Mon, 16 Dec 2024 14:58:59 GMT Server: - cloudflare Transfer-Encoding: @@ -329,7 +329,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_505443a8093770cec87f1314500371e0 + - req_0634dcdfc7d19e21f5c5f69ab333cca6 status: code: 200 message: OK @@ -349,8 +349,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; - _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -376,123 +376,123 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA1SZSw+yzJaF5+dXvPmmdiKIUJsz4yZ3qxC89ggQEbwglyqgzp/v6Jt0pycmAgqp - WnvvZy3+868/f/5psrrIh3/+/eefZ9UP//zX99g1HdJ//v3nv//158+fP//5ff6/K4tXVlyv1bv8 - Xf47Wb2vxfTPv/8I/3vk/y76959/6rkpyB6KvGsxkfdg7R6EpYo0BOM9qUtgrGkZeYw8YA8vUcCL - +5SdJLHgI79pDZh8b+JeWC27scaXGIHnESz1OzDHYXBeAF4AJLg6WzT7xxWGY3qTqDx7QzKaY2xB - SpyWkEtaJ30rJzmsN6sXw5JZJ/3L2KbKc0j3tDeLoZ4wbmbYHouE6cnqnbClZC/A2rlLrM7YSp6t - eDpDlEt7TBelWk9OcSphFoMNbuvVruukxMOomByTOKM/1VO3nlK42mPOwsAY69lk0ai2RR0Ss1wd - stm7Ki2QaNyy3PFQMD43gwHyfb3HU73RszFR+xagiRpi62GefIypauEl54wYnoPReOTbGYUNFohz - OO1rKuTxUbU/OwGXz9c1Gasd6qFEZyCaSdSMLxT6ALFwU+J+PiWaKtK3cBI/Hdu01K55XhILdTIb - mPZ5vdD0gZ0Gb3HpEHJodMQMKuyhsFydnJ0gRONi/S7BXl8UZqUO1H2jpbMyr7SYBL27ROyWKgY6 - ZscT5dKY1yPtcgBNmU4En09PRKu4DQGP/M2w/nARfxuXFYz5u8fpm81JW8UtBun0dmj5ucVoapa8 - gM21O+Lx+OySWV9FhlonjoiXy5qgVZhkI/I1u2cby+MdGzpmKIpaTSQMoppzVW8iGOy2YZ6We6YU - 7Egp535k0eUhSTsu+/kCLmMqYnjCvftUu7yAo+wlzDxccs7samjXr3Z8YbZbadkkb4MImn0XMS87 - boI5SI4PcE5Qk+3sjFl/vbcGUjS3JL7ykflkPzOAIEtk2nqmV0/xQsVgPGuEF6XRBEwT0QPqGg5E - OxZSMjVQNHAxngnT4t7v2p3ZNuDtsoi50L67WTxtKdrr6zNxi80+68zuVCnCco1Jun/riVSksELH - bNwTK7Dbuo21QEP43t2J/pIMJO4lFVA6nFbEYBFKpnVUWih8n950DFynY5rUvNCqDUW2lbskobfs - EkG0pXcsLUsRTXk1PyA8kztWVosw6c+s268/leZ+97Phg3hQ/fWiCmIMx5uQUfE802Wm7Ao6JJnK - ZzAVFx7LoMLW8yGa42EdnyH3qU7MQzmiCVk3DTYIVWxzuvU1825uCncfYrZxPTebWvGWwqbnOV1Z - iyuacGDCX/1AZWq87adrD0jJL9S0d0E9aSvFQs9+aZHv75PxEX4K8IvpzpxleUDT7fiisBlwT8LP - u0bjmsR72JDzleTp2UMTGI8zqGIYkYsqPJK/3z1j59OxiKqsuZ3FEY2KFODllkl12+h6pLh2aJCI - 6G5C6206Qnx6FbT2zSH4LNfbXCFE7hl+v44m75ndoJO7u5DweBMSFhwve7T7HJe/9eafMwttMJN5 - oCrFKeIfeShBHp8+Xm5VH40VLhS42tzDMMW2OYGtY2AaVHg5brualie5/d2f2eftifNEj0bFcBJG - QrOoMh7GUIE4RCUL8VAE7MzmF0T9ucWq3PFkxNtEg1lI1ozYcdPNR+liwTgYBMvUO9dDuwas9Jc+ - IUcv3HCeQGgjy9BkKtmE1uNlWFPU8H3PrMldm2yYmwYVhvQgZNmygM715QGd8lCYuRPrbLD6cwov - uWDMXy36hKtLHqoPfX4we1/5WT9XtQVnuwxo4u/KZDrIjQ+nc9yzsPC7pL8fkj1S6LkjVoqjbpT9 - TQ+fynBZGA/vesTbzADprFvEEhScSHLoLVDNrYzttAChcWd1AI2dnvBSWp85MwQX0FefLJD4seMv - K8zBnZmARRoLqBOujYVWbbNhzq5/B+OJUR+KauXjO2FFN6WNtYJOcVUsEqHN+mbx7NGvXrJlr6FJ - imMK0vuVEteJw4CxhpxRSyQTC0/Zqf/q+7N7rZn1lN8db9SNDxrYd4zs/bMeumk5w91fxHihk2My - XK8DwDISdNwM2bYee8EL144Xpyw0n/d6iLzdEfxbODHjzfusU5ccg7xDOtlW62PSL1TBh97bmcw9 - NptsvB/OJRRJbzFbmVcBbdQ+/atXxydmJpbnSEGwUgvcjbqYzSFZATgrkmMlD6xkLi+fAj2hTOm8 - wTiYrtfnAlDAWxIY1cMchHx6QX4GhbifsTTnc69qsOXikvzmy5zoowZBF91Y/O7P2fwd20g9lg6J - ipNer4prsEb1NO2YVhhK0grpcFRen2vOXOdSmky4ljZSNL+k0uXkZbzIPwLgqRGw5Clgdtf7FEM5 - +CbRtbI2m/TpxCiURp+WAnTJND5pjC674cIOlQjZqPHlXs7fn5G4UCf1zNqkgTMvSgqBZ3XTTnsr - 4BnMwctvvXDFHVzIlHgi+FxsTJEXjxDKPKmJv5LO2cx6rVEnT12QYPkMEd00u1KVVn3J/Fp9dizt - z64CF7ci4Xc+MSENYlR2gkycLZ9q+lZZiFZr1BE3Tq/J6F4xIENcFFRWb2pAn3ZSKVq/yJjN8RtN - kdPv0WdMdOLYbtpNn4W2Un/9SAPao3wVSw8kbpmERS1aBHQd0hUi2l5lePF+JYO0NyrlV2+Op0Aw - GlzuIZRGiVxnDTIaHrQZ7pfOpeBEzBw/wG3ousIhgaC35vhwbg2KEAwsuQSL7JPfNy84BlynPTRt - ParguKjen2OC2U371j/C8CiNkXla3SS0JsZL+fVrOxpd1PByzCHCfsFCH+X1zG/yAuR7ccJgPvVu - 2uLXWV5qWYEhHpxuliI7B27RjIVwX9XjzqpBzZsyYvprxJyf6fYFI/gH5gjmO5i//Vs1nB0jdm5v - E66JqzM0m82W2V1roekojXs0ONaTOXej64YGjo38GAuN+ezuduPbnM8wvwoghvK68XkpZyE6nS43 - Zm7B6CTFbY8w5s+ehZ+3iUbZ31I5se4lC5/itpZidUGRUmRnYj6aFe964Zwrgk5KLExHLRA6qREA - 3uKZefKQdZy4h0b53p+uiWVkvRzMFPrmnJPTjr8SaghlCIbrEqK/qqzmQ/c2YFG9KqYFrlNzHKgx - yj1BwDKZ1zV7eEWP5pcSUkPynvUDWahQwDdyZnT8HszOPY0Rr9T9l3+ipN0voj16S/GO+eeYIp7V - r29/C3wWita7Ht5mdFRwDB2dayXofv0XaPKI2EZWtU7agx7BPvIWZEvrKpDelt6qKMcKFlIc1VOs - tREq7suWbNtJ6ybWzmuFTucbloyHbs7ncbcCXqcPyvt0CKZbNlbqfFRkPGj3LpnZayFI3l0oKT9c - gLPV7qQBeM+QBb7tmFw8ZwWkgEKqbHZuMOpcqJBnuie8yL24G8vD0Yfq5XKiEd3N5kuvYMQWo0nF - 2Vplw2e91NAusGy8PKS0m9xiT8FikkBnTyP1FB4aA778Tueu1znXpPoM+50eEOPsHrpRPN5CkE6B - hNdO0KMpQde//ZHllVnyz6ZBDxB2oU0sWTcScae918i0yAmj7dJB81TnZ5TMJwPL8jAms5A+9zBu - by/cXfA6m7/1g9KbFVHehxfegF0W6m+9DU9j3TQ+6wJ5uqjgSV573Ww/JkVtcS2z8Apn88eLcObX - 7bce3YTPt2uFbhTt6Frcmoh/VGOPAnl5YM5jGND07XdIiE4vZhq7FZ/v+1hTv/4CS1h/cPpRrBHs - cL8heEE3Xe8VkwuOF6VUPW1v2bDF9Qtsxd+TL6/x+dLPISBF9xgOlSFrF3JhoC+P43v7sbKZDihS - DnjVM72SaDAKWbdCy6UV0jman4j3c12A5Co7KjhbMaAnyholpVuTbLIdDeicnxZQ14sDMSRj6jja - GL0q72SdmcvRNr96U9TF1DfM7O/3gMn+HtCXR4n25UGGAzGCoOtzSiW/6mgdDDOcTtmNkKbad1MY - B1jePlhIxfb9zuYlMrUfTzMnR1I3jR83RrIxIqIJ4ikbcIBnuN148POj2VyRTar++NV8NEc+aev7 - GoLcvRDzGry/eurO8qrFIu5se84mnR9iuNqnkI5A7928hFFTqaVtmedqvTlYffbtr8XAvMxVzamT - vBf6rV+wXTf1JGFxjazlKqUjVjyT+mmEgS3mEU9WTPjqqw+wtIExs2/K7G//lE5P5+vv33zMnmEB - 9C55ZJvtk4zNt0MF8u5m/eWd8csbytd/4vVdOqDRv3x6+Oy8Bse1skL0ZUc9dFvcEO/0qLu5nw70 - x3dff8KTaa7HFXz5g+HuLNV9teM9EDVZkI3cXLKevcUjtLS6ES/USs4SsGwY+8Ag1n3m3Tg9UgUu - 42rD9JdUIa6L4wzq4zAy5yEMAV0l1Eb9JToS14xP2WxO7gOFH0snyU6sE15FmYvGbpiJ6ZvbYGy0 - dITQqzq2WVmdOUuHcwQpqRRiuDLqvs+zhy//0tVtKQfzV38Qek5GZ/1w7vg9ui7Q7XbAxFr3jPf2 - E4XoiZOGhevVx/xkrxQr5dUdiR19vG42BvpCLb7LhNw3z66fHvEa7v7dwONdQGgUj/kZrgelInir - 6xljHzgja7FU8PKQKF0fG9MRFfFKItrkf9DIy3WhCIY9sJ8fG+13ieGmVwnR7ojw0bu8jyhsGosc - Fbk1x50BLjpm8x63X7/7nfhHtN+ZATP7ZOA9v2kt+Iujz6zA9muavatRDZTzk2ys0wuNJm1jOSUL - Hyffeup/fqESqUfh+GqSOdG0UJ2CKCM5RnYwd4J4RLbcn5gRzU/OjXnnQnUqWrraJFHCNHGRgkLT - jlKdrDIW7iyqWjt/yX71MT+98Qin04oRctnkQfP05RYtzJvHgt3kZT9+Ur77geezK9YcbbYl3DTi - MX//vidjJ1iGglI04olWz2DQeVyAeiAtMy8XO5u8VLTgffbXWIAir7kurkcI8sBmVtxuu955uA9F - X/tA56jb8R9/Aa2iCyOj3WfztSwpFGY/kbDyS/Oj8ziHXXOb6Xram6bkXbMU/fID+MR5N3nFbg0K - SWJmR2PDeXDc7ZVWi/dE14KMT/fEiqF6axq5ZGQy6e0kxRBsi4hhT++7qUr0HOG4zpj9nqxg3Lyf - AtyiMGfXaul//VgWwlF+1fii3lRzXCzvLlyW6oM063qBej+7VIC2pUWKL9/OyCAAezYkxHK2HufE - 91slyP0LC674jsbaX0bg3VclCx2P8NG9Wi5Qy1CYrYeQ8dU5amHrHLdEJyxJxOIW+fD2tJkRaX1G - 4/2QHOF2mwJisMJM5mtZV3D8vCP6rERI+ry+CKjLbncKxHgms+JPZ8hCMyZ6W0UZF1O3UHxV15jp - iTLnrTSvoX+rIdFvodbxHw98/Qldp8nMh8tHaBCtEht/yDwkvZB9NPSd50y/bfp63m3S4q8/++Zr - fDLZqKn62oW/9TnpPM2hVdsnHs3rM5vA9jBsn48L0+XJT+bsOS9gYe2PJP+8Td4J2RipP32ff/PU - ei/X67eoChTuVO7Yl+/AWW1z4gi+nTTx8hYjQ3R2VDCzpGagD4qS+7GFH5XzMpkc6gvYkPSKjwzz - YF7KSQheeT9SwMOMJhLornpjc8EsPJ0Daj9kBbr8qGNp3D270StkHw7QIrp+fk7JvFi0zbdeJQrH - 88HkBl36oOV1wHS17YKpTN0ehB3mVHzB8+fnDHgsXzNdrGgezPrMscIiqyE46in6m3diMzv8/H/y - 889ozbdPgiW/qudkkfkwbq8vZh1rnH37ZQvXwzVm4ackaJzKlQ1oazv44ew/Qf/tT3CS0g0zuuKC - hjLVqJpLncn016IyZyGtKtCu5zMezeOUDdvYTeG7fkR3LTuYwhgbSDAsjQRXrHPhO78QUpQjc7/5 - Z79HioYCWT3Qqf08kjmrji4kD9ujQzuV3Wj1c45+vECuySMYw30cI+e0jJj2fKnJWOTp629eEGzX - HI15nbe/5yXBY7aQ+PPn+YqvCBFsPVhtkwiQ3T52uO0aB63P03kPp9dCxMKX/4bwUGrqN0/9mz9/ - wNqlILk3nW0qcWfy1Z4Larl9DMQSiYZEi7ohqo7PK5VPnfHNR8sIDYL5/s5HjY9elu9h8i+YaIVH - s6k8duvf/MbXlfZEXPFTH23tdYQHLTmbk3/xC/Dp7NDldhV0/Bm4yi8vJEZOPjVFzm6GpcoZFrXi - Zfbpk8Tw3X+W7ChL2FSJLtBplZMwMKKO3y5bA7K8qAjp9z7nq2Mj/PIRpsF7qOdhStewZKVHgs2y - yGaztQFN3nJB10XYZNTLDQHk8e1jsb0IJrcGvVfdh7bA6IKH4MtHKXqfXy7ZtBchGEi4XgCuSsrM - 5X5Z36MNVOj6LJ7Me/GGT+nrJEB/oQme2o+VjC2Msfo++wFzjOPFHObKM0C/KCPzLCwi1iwEgObR - 1PRO2KLm6gJjpKdHDQMeCrMnQZPCedMsyI+/pkdoYPjqjU6vRRWwOlwc4ed31sM2zbhqRpa6Fd2c - eDd3h3j6tvff81tme+ETTfFCDMFwdgbD+xNkU9p9WuTTvUT2X31Ml+FSQIm1lpDm49ZCL+gYBaHH - KTin2pz30JTwJI8N2Veajebj7ERwDA4WCT8lQ3MC1AUvjq7E8fXRHInHU3i/KSYW9I/6bx5LEzch - rmOtkfABZIOG45h887J6NobXA02Xek/8Wo0QK4pLCzzJFlg41rU5fufHj0eYbS1UPjxt9YVQ2EhE - M69jxw78tAI7KwK2uZmsmxdyocGZmxFWtXuXjdebqin7Zf9i2hq0YOznPUZJ8nkx3YofHa9iYwG/ - eTCm0RrNsT6v1FdoXYnX9p+O//brl0d3V2dA49PZC6jh84fogaoGk+J1LcpCPWbGux8CarDkpdZW - usRKSKVgEk+PBRTTZsSLWth33Wp3M2DsPYOZWxF3Xx6c0cntP3Rci6E5f99XwDc/os/atBKeLEsF - auu8ZBs1TfnEi88Z3RuO6dK4JObolPQBT+K6JA7lKOOg1zZMQZyRjWt4wWzMYwqJtRjJtz/x1rvO - LSyqR8V21sVE37yjgMs9ZsQ93qdk1lbp6uc/f/lfMjSLoEE7f/vCcnZ9JP0qXj7A5Y89CUXL6Wj6 - iR8QXmzKwuPLTXiwf+whlz4mSb68O+szCtHyVtfE2fXzl4/wSlne7jWmfLc1f35LZfRU0qWwLbOV - JjwF8Bd7n+HomaJxGD8PsIzcw2K2wwE7riYBUrjC9//iZB6bjwZeTFPiINFNxkubrGG3Xh8Y5rs7 - 75dKXoLxVAM8fd/fjRdWlvBeCXviqcMzGA5y48L2aByJJyOzE0DPBHDe1Ys2ov2ovzz/ghHv7Z8+ - +XTp+uKX/7LwTuV6/uZxKJeIzXTCFt03H8jB3RwqsrH+BwAA///smsuSojAUht9ltk5V24AkLOUu - IATBRtgBIhpaWwQSSNW8+1TwDWY1C54gWSTn/JcvtDHPKyKF+3WCSlHBPM9vuX4VeP92jkfvqxUA - Pw9t+3QXT8A+UHBRreQ9rynXn/y/jcRHz7IYO+q7APjGmlh4lPFQ3ma9cwIEfZSAsbthlO/5amuJ - z6bzLa3hQwwPKDhWYcGsJvqRuV8idndpPIpDxwI5qmXk8z6TqvTUwcjqV2gvCY7OQgtrsH4Vz2Dc - GYd4Ouh9+d7HyIp2L/bQDoLionZAWxtFOq0qLYdQzAeikTDU2Vas74rgXTOkkW8N3yk+7fn9beSa - 6SOeFKi0cKMlq0AujbDg/eQgf0IBk13v5/FkX88WsDbDiThKb3rdxjUHyN8TceLXoJMTywO40SZA - vGbTeGOVB0c4+x1Lvep43dMuBc41xWSL1J+YitG4gupTxVwvSqDjeRpMv5AaCFgKMHsq7hFm16FD - duYkXpeR+ibjCQXEVAQ1FnsqCVAoDBIIx1sbt/N+t4otIEbwUevMOccplBpr4vNAKub9O+eJ3E9A - j9H7kSq39eeT+BdrzZ5BmDZv/7R3AcT0KJsa70f2/D3i14T22QDHbNUEsM9CPCDbtCDXP8S4TvFr - anaWC4EsJWhbaXlcrfPvBPqJriPen+tsd/ECUPhJGYxF3sZj2pcRlJV6DF4/5ymekglFQNhQbxiN - DAM2vJo94P0c0UTHfIm8v5a7h/0RfPN9OZzYWM3+lJgeOAI69+f12aWzn2TD0FYV9NfdhVhqjjAz - n60B/GSbIENixOskv0/gr5kK+PP7H4iCz4UoWIiChShYiIKFKFiIgoUoWIiChShYiIKFKFiIgv+Q - KPgLAAD//+zdMQrCQAAEwP5eEdIfhFTiX0QOcliYeMGcYOPfJUbEHwgy7Tb7gWGXKCAKiAKigCgg - CogCooAoIAqIAqKAKCAKiAKigCggCoiCX4uCniggCogCooAoIAqIAqKAKCAKiAKigCggCogCooAo - IAqIAqKAKCAKiAKigCggCogCooAoIAqIgj8XBaFpDq8XhKkMeVxhQM33Gj9UIKYhxa7rt6uE25JO - ud2/BUI7X8s012Mt53xZ1vGC3WYN2lpqGr/zsFY9whMAAP//AwDXFfCAhGEAAA== + H4sIAAAAAAAAA+ydSa+zzJal5/dXfPqmzitjsInNndEZ0zkCg9tSqQQYY3CDaSKASOV/T+G3lFWl + mtWw5MmRjs05INix91orHtn//o+//vq7Ssos7f7+119/P4u2+/vfpteucRf//a+//ts//vrrr7/+ + /fvz/zgyeyXZ9Vq88+/h3zeL9zUb/v7XX8J/vfK/DvrXX3+XY5WRPWRpU2Oy2oO5exAWy1Ln9feo + zIGxqmbk0XOPPZxIBidsY3aSFhnv+U2twOB7A7eCOG/6El9CBI5DsNTuwOi7bvMCcDwg3nWzRaN7 + FDEc45tEV6PTRb3RhybEZFMTconLqK1XUQrLtfhiWDLKqH3p21h+dvGetkbWlQPG1QjbYxYxLRLf + EZtL1gzMnT3HyojN6FkvTmcIUmmP6SxXymGTnXIYF94a16W4axopcjDKho1BNr07lEOzHGK4Wn3K + fE/vy9FgQa/UWekTIxcPyehc5RpI0G9ZunGQ1z/XnQ6r+3KPh3KtJX2ktDVAFVTE0vw0+uhDUcNr + lTKiOxuM+iPfjsivsEA2h9O+pEIaHhXrsxNw/nxdo77YoRZydAaiGkRJ+EymD1hkdkzszydHQ0Ha + Gk6LT8PWNbVKnubERM2KdUz9vF5o+MBOhfdiviHkUGmI6VTYQ2baGjlvPB/1s+U7B2t5kZkZb6Bs + KzUe5VFUQ+K19hyxWyzr6JgcT5RLfVr2tEkBVHk4EXw+PREtwtoH3PM3w9rDRvytX0To03eL4zcb + o7oIawzS6b2h+ecWoqGa8wzW1+aI++OziUZNDHSljDYLPJ+XBIl+lPTIVa2WrU2HN6xrmC7LSjEQ + 3wtKzhWtCqCz6oo5auoYkrcj+Sp1A5POD1Hc8JWbzuDSxwsMT7g3n2KXZnBcOREzDpeUM6vo6uWr + 7l+Y7UQ1GVZbL4Bq3wTMSY5rb/Si4wM2JyjJdtz0SXu91zqSVTsnrvxZ8cF6JgBeEq1o7RhOOYQz + BYP+LBGe5XrlMXWBHlCWcCDqMZOioYKsgov+jJgatm5T74y6AmeXBMyG+t2Mi9OWor22PBM7W++T + xmhOhSzMl5jE+7cWSVkMIjom/Z6YnlWXdah6KsL35k60l6SjxV5SAMXdSSQ6C1A0LIPcRP779Ka9 + Z28apkrVC4m1v2DbVRNF9JZcAgi29I6leb5AQ1qMD/DP5I5lceZH7Zk1++WnUO3peVa8WxwUdzkr + vBDD8SYkdHEe6TyRdxntokThIxiyDY+5V2Dz+VgY/WEZniF1qUaMQ96jAZk3FdYIFWx9urUlc252 + DHcXQra2HTsZ6sUthnXLUyqasysasGfAn/qBwlB53Q7XFpCcXqhh7bxyUEXZRM92bpLp76P+4X8y + cLPhzjbz/ICG2/FFYd3hlvifd4n6JQn3sCbnK0njs4MG0B9nUBZ+QC6K8Ij+/O7oO5f2WVAk1e28 + 6FEvSx6eb5lU1pWmBbJt+ToJiGZHtNzGPYSnV0ZL1+i8z3y5TWVCVi3D79fR4C2zKnSydxfiH29C + xLzjZY92n+P8e7/558x8C4xo7KhCcYz4Z9XlsOqfLp5vFRf1Bc5kuFrcwTCEljGApWFgKhR43m+b + kuanVf09P7PO2xPnkRb0sr6JGPGNrEi4H0IBiy7ImY+7zGNnNr4gaM81VlYNj3q8jVQYhWjJiBVW + zXiULib0nU7wijrnsquXgOX20kbk6PhrziPwLWTq6opKFqFlf+mWFFV83zJzsJcG68aqQpkuPQiZ + 18yjY3l5QCM/ZGbsFmXSme05htcqY8wVZ23ElTn3lYc2Ppi1L9ykHYvShLOVezRyd3k0HFaVC6dz + 2DI/c5uovR+iPZLpuSFmjIOmX7nrFj6FbjM/7N5lj7eJDtJZM4kpyDiSVr4zQyU3E7ZTPYT6ndkA + VFZ8wnNpeeZMF2xAU30yT+LHhr9MPwV7ZAJe0FBAjXCtTCTW1Zptdu3b60+MupAVoovvhGXNEFem + CI1sK3hBhDppq9mzRd/1ksxbFQ1SGFKQ3q+Y2JvQ9xiryBnVRDKw8Fxtyj/1/dm9lsx8rt4Nr5S1 + CypYd4ys/bPsmmE+wt2dhXimkWPUXa8dwDwQNFx1ybbsW8HxlxsnjJlvPO9lFzi7I7g3f2D6m7dJ + o8w5htUOaWRbLI9RO1MEF1pnZzD7WK2T/n4455BFrckseRQ9Wilt/KdeNy4xkkV+DmQEopLhptcW + yegTEWAjkhTLqWdGY375ZOgJeUzHNcbecL0+Z4A8XhNPLx5GJ6TDC9IzyMT+9LkxnltFhS1fzMl3 + voyR1qvgNcGNhe/2nIzT2EbKMd+QIDtppZhdvSUqh2HH1EyXo1qIu6P8+lxTZm8uucGEa24hWXVz + Kl1OTsKz9CMAHioBS44MRnO9DyHknWsQTc1Lo4qfmxD5Uu/SXIAmGvonDdFl113YoVhA0qt8vl+l + 709PbCijcmR1VMGZZzkFzzGbYae+ZXB0tsHzab1w2e5sSORwIPicrY0Fzx4+5GlUEleUzsnIWrVS + BkeZEW/+9BFdV7tckcQ2Z26pPBsWt2dbhotdEH+aT0yIvRDljbAimy0fSvpWmI/EJWqIHcbXqLev + GJC+mGV0pdwUjz6tqJDVdpYwi+M3GoJNu0efPtLIxrLjZvjMVFH59iMVaItSMZQeaLFlEl6owcyj + S5+KiKh7heHZ+xV10l4v5O962zgyeL3OVy34Ui+R66hCQv2DOsL90tgUNgEz+g9wC5om2xBP0Gqj + f2xuFQoQdCy6eLPkk97XLzh6XKMtVHXZK7CxUbk/hwSzmzqtf4Thkes9c9SyimhJ9Jf87ddW0Nuo + 4nmfQoDdjPkuSsuR31YzWN2zEwbjqTXDFr/Oq7maZBjCbtOMUmClwE2aMB/uYtnvzBKUtMoDpr16 + zPmZbl/Qg3tgG8F4e+PUvxV9s2PESq1txNWFeIZqvd4yq6lNNBylfo+6jflkm7veNF0Fx2r16DOV + uexuN/3bGM8wvjIguvy68XG+Snx0Ol1uzNiC3kiyXR+hT58t8z9vA/Urd0tXkXnPmf9cbEspVGYU + yVlyJsajEnnTCudUFjSSY2E4qp7QSJUA8F6cmbPqkoYT+1DJ0/npkph60q68kUJbnVNy2vFXRHUh + 90G3bUK0V5GUvGveOsyKV8FUz96UHHtKiFJHEPCKjMuSPZysReNL9qkuOc/ygUyUyeDqKdMbfvfG + zT0OES+U/aR/gqjez4I9ekvhjrnnkCKelK+pv3ku8xfmu+zeRnCUcQgNHUvZa779F2j0CNh6paiN + tActgH3gzMiWloUnvU2tVlCKZSzEOCiHUK0DlN3nNdnWg9oMrB6XMh3ONyzpD80Yz/1OBF7GD8rb + uPOGW9IXyniUV7hT7000stdMkJy7kFN+uABn4u6kAjhPn3mutTH44pxkEAPyqbze2V6vcaFAjmGf + 8Cx1wqbPD0cXipfNiUo0OxkvrYwRm/UGXYymmHSf5VxFO8+08PwQ02awsz0Fk0kCHR2VlIN/qHSY + 9Dsdm1bjXJXKM+x3mkf0s31o+sXx5oN08iS83HgtGiJ0/dMfWVoYOf+sK/QAYedbxFxperTYqe8l + Mkxywmg736BxKNMzisaTjlerro9GIX7uod/eXri54GUyTusHxTczoLz1L7wCK8+U7/3WHZU1Q/8s + M+RoCxkPq6XTjNZjkJUalyvmX+FsfPUinPl1O61HO+Lj7VqgG0U7ulxsDcQ/ir5H3mp+YJtH16Fh + 6ndICE4vZug7kY/3fagqk7/AEtYenH5kswfL368JntF10zrZYMPGCWKqnLa3pNvi8gWW7O7JpNf4 + eGlHH5CsOQz7cpfUs1Wmo0mP43v9MZORdiiQD1hsmVZI1OuFpBHRfG76dAzGJ+LtWGYg2fKOCpvt + wqMnyio5pluDrJMd9eiYnmZQlrMD0SV9aDha662y2q00Zsx7y5jqTVZmQ1sxo73fPbZy94AmPUrU + SQ8y7C0C8Jo2pVRyi4aWXjfC6ZTcCKmKfTP4oYdX2wfz6aJ+v5Nxjgz1q6fZJkVSM/QfO0QrvUdE + FRanpMMeHuF2497XjyZjQdax8tWvxqM68kFd3pfgpfaFGFfvPdVTc16JNV7gxrLGZND4IYSrdfJp + D/TejHPoVYWa6pY5ttoandkmU3/NOuYktmIMjeS80Pf+edtlVQ4SXiyRORdj2mPZMagbBxjYbOzx + YIaEi1N9gKl2jBltlSd/+qd0em4mf//mffL0M6B3ySHbZB8lbLwdCljtbuYfvdNPekOe/Cde3qUD + 6t3Lp4XPzqlwWMoioi8raKHZ4oo4p0fZjO1woF99N/kTHg1j2Ysw6Q+Gm7NUtsWOt0CUaEbWq+qS + tOy9OEJNixtxfDXnLALTgr71dGLeR970wyOW4dKLa6a9pAJxbdGPoDwOPds8hM6jYkQt1F6CI7GN + 8JSMxmA/kP8xNRLtFmXEiyCxUd90IzFcY+v1lRr34DtFw9ai2RijdDgHEJNCJrq9Qs10PXuY9C8V + b/OVN071B76zSeioHc4NvwfXGbrdDpiYy5bx1noiHz1xVDF/KX6MT/KKsZxf7Z5YwcdpRr2jL1Tj + +4qQ+/rZtMMjXMLdveu4vwsI9YtjeobrQS4I3mpawtgHzsiczWU8P0Ry04b6cERZKEpEHdwP6nm+ + zGRBtzr29WO99c4x3LQiIuodEd47l/cR+VVlkqO8qo1+p4ONjsm4x/Xkd6eJf0T7neExo4063vKb + WoM7O7rM9Cy3pMm76BVPPj/J2jy9UG/QOlzFZObiaFpP7dcvFAvqUDi+qmiMVNVXBi9ISIqR5Y2N + sDgia9WemB6MT871cWdDccpqKq6jIGLqYhaDTOOGUo2ICfN3JlXMnTtn3/UxPp3+CKeTyAi5rFOv + erqrGs2Mm8O83eAkX/0kT88Dj2d7UXK03uZwU4nD3P37HvWNYOoyilGPB1o8vU7jYQbKgdTMuFys + ZHDihQnvs7vEAmRpybXFsgcv9SxmhvW2aTcP+yFrSxfoGDQ7/tVfQIvgwkhvtcl4zXMKmdEOxC/c + 3PhoPExhV91Guhz2hiE51yRG3/wAPmHaDE62W4JMopBZQV9x7h13e7lWwz3RVC/hwz0yQyjeqkou + CRkMejtJIXjbLGDY0dpmKCItRTgsE2a9B9Pr1++nALfAT9m1mLuTH0t8OK5eJb4oN8XoZ/O7DZe5 + 8iDVspyh1k0uBaBtbpJs0rcj0gnAnnURMTdbh3PiurXspe6FeVd8R33pzgNw7mLO/I1DeG9fTRuo + qcvM0nxIuHgOathujluiERZFi+wWuPB21JERaXlG/f0QHeF2Gzyis8yIxmteFnD8vAP6LBYQtWl5 + EVCT3O4UiP6MRtkdzpD4Rki0uggSvojtTHYVTWWGs1hxXkvjEtq34hPt5qsN/+qByZ/QZRyNvLt8 + hArRIrLwh4xd1ArJR0XTPGfabd2W424dZ3/82ZSv8cFgvapoSxv+rM9B43EKtVI/cW9cn8kAloNh + +3xcmLYa3GhMnuMMZub+SNLP2+CNkPSB8q3v83eemu/5cvleKAKFO101bNJ3sBG3KdkIrhVV4fwW + In2x2VHBSKKSgdbJcuqGJn4Um5fBVr42gzWJr/jIMPfG+SrywcnvRwq4G9FAPM1WbmzMmImHs0et + x0qGJj1qWOp3z6Z3spULB6gRXT4/p2iczepqWq8SheP5YHCdzl1Q09JjmlI33pDHdgvCDnO6eMHz + 6+d0eMxfI52JNPVGbeRYZoFZERy0FP3JO7GRHL7+P/r6Z7Tk2yfBkluUYzRLXOi31xczjyVOpn5Z + w/VwDZn/yQnqh1y0AG2tDX5s9h+vnfoTnKR4zfQmu6Auj1WqpFJjMO01K4xRiIsC1Ov5jHvjOCTd + NrRjmO4f0WzT8gY/xDoSdFMl3hVrXJjmF0KyfGT2lH+2eySryFspBzrUn0c0JsXRhuhhObSrh7zp + zXZM0VcvkGv08Hp/H4Zoc5oHTH2+lKjP0vj1Jy/wtkuO+rRM6+/1Eu8xmmjx9eepyEVCBEvzxG0U + ALLqxw7XTbVBy/Nw3sPpNVtgYdJ/nX/IVWXKU//kzx8wdzFI9k1j62KxM7i454KSbx8dMRdERQuT + 2j4qjs8rXZ0afcpH8wB1gvGe5qPKeydJ9zC4F0zUzKHJkB+b5Xd+46uoPhGX3dhFW2sZ4E6Nzsbg + XtwMXDpu6Hwreg1/erb8zQuJnpJPSdFmN8Jc4Qwv1OxltPGThDA9fxbtKIvYUCxsoIOYEt/Tg4bf + LlsdkjQrCGn3LufisRK++QhT4d2VYzfES5iz3CHeep4lo1FbgAZnPqPLzK8S6qS6AKv+7eJFfREM + bnZaq9gPdYbRBXfepI9i9D6/bLKuL4LXEX85A1zklBnz/by8B2so0PWZPZnz4hUf4tdJgPZCIzzU + HzPqa+hD5X12PbbRjxejGwtHB+0i98wx8QKxaiYAVI+qpHfCZiVXZhgjLT6qGHCXGS3xqhjO62pG + vvprePg6hqne6PCaFR4r/dkRvn5n2W3jhCtGYCrbhZ0S52bvEI/f1n56f8ssx3+iIZwtfNA3O53h + /QmSIW4+NXLpXiL7qT6GS3fJIMdqTUj1sUuhFTSMPN/hFDan0hj3UOXwJI812ReqhcbjuAng6B1M + 4n9yhsYIqA1OGFzJxtV6oycOj+H9ppiY0D7KP3ksjeyI2BtziYQPIAtUHIZkysvKUe9eDzRcyj1x + SyVALMsuNfAomWHhWJZGP82Prx5hljlTePe0lBdCfiUR1bj2DTvwkwhWknlsfTNYM85WmQpnbgRY + Ue9N0l9viirv5+2LqUtQvb4d9xhF0efFNDN8NLwI9Rl850EfB0s0htooKi/fvBKnbj8N/z6vbx7d + XDcd6p+bvYAqPn6I5imKN8hOU6PE10Kmv9vOozqLXkppxnMs+1TyhsXpMYNsWPd4Vgr7phF3Nx36 + 1tGZsV3gZtKDIzrZ7Yf2y4VvjNN+BUz5EX2WhhnxaJ7LUJrnOVsrccwHnn3O6F5xTOf6JTL6TU4f + 8CS2TUJ/FSQctNKCwQsTsrZ1xxv1sY8hMmc9mfoTr53rWMOseBRsZ14MNOUdGVzuISP28T5EoyrG + 4td/fvO/qKtmXoV27vaFV8n1EbViOH+AzR974i/MTUPjT/gA/2JR5h9fdsS9/WMPqfQxSDTp3VEb + kY/mt7Ikm107TvoIi/L8di8x5but8fVbCqOnnM6FbZ6IqvAUwJ3tXYaDZ4z6rv88wNRTBy+SHfbY + URwEiOEK0/8Lo7GvPio4IY3JBi3sqL/U0RJ2y+WBYb6783YupznoT8XDw7R/119YnsNbFPbEUbqn + 1x1WlQ3bo34kzgoZjQBaIsDmXbxotbAe5aTnX9DjvfWtTz5cmjb75r/Mv9NVOU55HEolYjGNsFkz + 5QMp2OtDQdZmsCmnvCJUJr/OSCop5ZTn15N+Faf9t2s0eIdaRNP5iNqd7WhEm12Pbpp1/NOv+0l/ + TuttYFvySZOh7bcuQltTYFY5yCVNi6/eOSFG5ini/GWa6Z/+utGPWz5ei3MObynYEbzPgoRbj7CS + J7/ENu3t4fVl4FgoJrlMttN+Zq/1pxZCq5sRfyk6Bg+sUoe8ST54sM1dNO6MLv0zj4kV2g1/6ztR + cUlNibohodFnmR4DSDFlOgsCg6tS/lJE734hOnvq5asvT/50/Rvirs/vaFRAqWGlH2dYTs0gmfYn + qbwAsWR2t42jcXO/Wsha0RNzlG7ttSt3TWGqJ+ZEDTXYiccYVvqImPdYPbwhi/Eevn7H0u5GKXR9 + e0bO/VwylWhV1EvhMAPto5WTXlyidsrT4HwgGhbLJS75R3H3cLnTlmwuztFrLywv5HIkmK0VUYuk + rl+KICYmw+K+qKP6O9+tREXMxPPc4M41OsPyYY1TP1gm3/n7zRMnPwEe71/7XimExYdtb5bAPzg4 + P/74J99FUPZ7ea1P+yP+VI9lMxL/QmG4zB4YuktQUrJZWzDpH2bex6gZH7blApKXR6JmehxlQvw8 + wvZoGGTaPze4ffMwSrbHFA9JXEfDuUtDkJV8wE11HaPxOJIQiaveo4N5KRGnzcNH0/4c0yVn3UjT + /rXcvjdz/JzmJT3xIfv6U7b20B713/3z/Or2Xz/JKa2zDLZCe2OWFpOSrz+1ibZH9UjMJWdeu9x2 + R/j7SwX8x7/9PxAFix9R8CMKfkTBjyj4EQU/ouBHFPyIgh9R8CMKfkTBjyj4EQU/ouBHFPyIgh9R + 8CMKfkTBjyj4EQU/ouBHFPyIgh9R8CMKfkTBjyj4EQU/ouBHFPyIgv+LKBB/RMGPKPgRBT+i4EcU + /IiCH1HwIwp+RMGPKPgRBT+i4EcU/IiCH1HwIwp+RMGPKPgRBT+i4EcU/IiCH1HwIwp+RMGPKPgR + BT+i4EcU/IiCH1Hw/zlR8I+//vrv329BeFXX7DmBAV02dP/8L1Tgn/E1/qcgiH++KoG2cZ79/a// + SSD8/Wmq16f7H131yN7t9OEF8Ic1+Luruvj5v7/+j+lU//GP/wQAAP//AwDXFfCAhGEAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b199e7cf78d-EWR + - 8f2f88e4cdbcc974-IAD Connection: - keep-alive Content-Encoding: @@ -500,7 +500,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:18 GMT + - Mon, 16 Dec 2024 14:59:00 GMT Server: - cloudflare Transfer-Encoding: @@ -518,7 +518,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '43' + - '186' openai-version: - '2020-10-01' strict-transport-security: @@ -530,13 +530,13 @@ interactions: x-ratelimit-remaining-requests: - '9999' x-ratelimit-remaining-tokens: - - '9999982' + - '9999981' x-ratelimit-reset-requests: - 6ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_11005c81622ce1d9b6fe05c68fed52e8 + - req_5dd3a4b0968bb2d0844855f843dec04c status: code: 200 message: OK diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml index baa3ceca13c..2eb27c48dba 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages.yaml @@ -47,8 +47,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=nxPGrcXhbymyBih2t7CgJx2kdUAj2c8FEUM6SgQuXQA-1734303554-1.0.1.1-0gC9PrWCtGQtTp5RVHnIIDJR3yD.DgFli7YVq20QDgOcIMxDrBJn9wXO3e.Fak8_HP_bRTKACpjVZuouGgX1ig; - _cfuvid=GidLrZtG7TkkHj.qFUWUeIOOu8KdGLixVP6dtmMZgV4-1734303554457-0.0.1.1-604800000 + - __cf_bm=KxfM6APw09zrJass.4OtAFMoUu2vRIbwSxiTzDsvK18-1734361136-1.0.1.1-ziXwyO6Y2EVlj7avwtfxuCobMUBGVXaF4V0POP9dVYb7lePh2DTCi3wGND0Bfqx44OdTFy0.IruHQWWlbKLdYg; + _cfuvid=nZLW2HYfOt4n61DxTEcgN8TpbBbjLbo4wSNNVqPlytw-1734361136731-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -74,19 +74,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//7FTBjtMwEL3nK6w5Nyhpu+kqF7QXBHvgCCoEJa4zSQyObewJsKr678hp - 2qTaReIOlxzemzd58+zxMWIMZA05A9FxEr1V8QM693P/7vHH+w/Np2zztE8P+zp7kHxn3j7CKijM - 4SsKuqheCdNbhSSNPtPCIScMXdPdZrtJNnd32Uj0pkYVZK2leGviXmoZr5P1Nk52cXo/qTsjBXrI - 2eeIMcaO4zf41DX+gpwlqwvSo/e8RcivRYyBMyogwL2XnrgmWM2kMJpQj9arqjoW8H1AH5wXkLMC - PnacmPSMOmSCW0lcMdOwN45rga8LWLECtNHC9L0k4iqoklNVVct/OGwGz8OcelBqwk9X08q01pmD - n/gr3kgtfVc65N7oYNCTsRAtxM+SSP8nMSWx/veSiBj7Mi7McDMvWGd6SyWZb6hDw2w9LQzMe7pg - s4kkQ1wt8PsLcdOvrJG4VH4RLwguOqxn6byffKilWRDLI3zu5qXe58mlbv+m/UwIgZawLq3DWorb - iecyh+EZ+1PZNeXRMPgnT9iXjdQtOuvk+cI0tswakSaYJniA6BT9BgAA//8DAI+legpSBQAA + H4sIAAAAAAAAA+xUwY6bMBC98xXWnEMFJCIrLlV7aA89VatqD6UCxwzgrbEde5C6ivLvlQkJRLuV + em8vHN6bN7x59vgUMQaygYKB6DmJwar4Q5sfWy+/9J8fm28vffJVuEf/8fjcZE95BpugMIdnFHRV + vRNmsApJGn2hhUNOGLqm++1um6fpdj8Rg2lQBVlnKd6ZeJBaxlmS7eJkH6cPs7o3UqCHgn2PGGPs + NH2DT93gLyhYsrkiA3rPO4TiVsQYOKMCAtx76Ylrgs1CCqMJ9WS9rutTCccRfXBeQsFKeOo5MekZ + 9cgEt5K4YqZlnxzXAt+XsGElaKOFGQZJxFVQJee6rtf/cNiOnoc59ajUjJ9vppXprDMHP/M3vJVa + +r5yyL3RwaAnYyFaiV8lkf5PYk4i+/eSiBj7MS3MeDcvWGcGSxWZn6hDwzybFwaWPV2x+UySIa5W + +MOVuOtXNUhcKr+KFwQXPTaLdNlPPjbSrIj1Eb5281bvy+RSd3/TfiGEQEvYVNZhI8X9xEuZw/CM + /anslvJkGPyLJxyqVuoOnXXycmFaW+WtSBNMEzxAdI5+AwAA//8DAGwlKy5SBQAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b081a3818b8-EWR + - 8f2f88d44f252082-IAD Connection: - keep-alive Content-Encoding: @@ -94,7 +94,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:16 GMT + - Mon, 16 Dec 2024 14:58:58 GMT Server: - cloudflare Transfer-Encoding: @@ -108,7 +108,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '595' + - '667' openai-version: - '2020-10-01' strict-transport-security: @@ -126,7 +126,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_d2c1df90b6180c0a455b1e3d44a12030 + - req_7af26d0531fb33160e3e0eadd5629fbd status: code: 200 message: OK @@ -145,8 +145,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; - _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -285,7 +285,7 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b0ccbbdf78d-EWR + - 8f2f88d9d864c974-IAD Connection: - keep-alive Content-Encoding: @@ -293,7 +293,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:17 GMT + - Mon, 16 Dec 2024 14:58:58 GMT Server: - cloudflare Transfer-Encoding: @@ -311,7 +311,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '108' + - '84' openai-version: - '2020-10-01' strict-transport-security: @@ -329,7 +329,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_8dc90305ed6ad294095c7572ef16918f + - req_3d4e6cd066737fa1302df6d55657ea31 status: code: 200 message: OK @@ -349,8 +349,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=qFuKHUwUyX4U8CFs8XC220DWcjeeBGYLadwc.3OJfGM-1734303555-1.0.1.1-kO6Fp04B.IJjdga6J9PEtG3FtC9pfiZojy2nu8Zk.sqhCMhy9Id0Rv4M_EYAOukf7kitJDxq0xgjZ3NVlK9_bQ; - _cfuvid=YghfZ57MPwYwLm7Ol.1SqjHtkwctAfmkTEDH_.h6Ddk-1734303555238-0.0.1.1-604800000 + - __cf_bm=P4tnLTGTLXNDMgrxvNM3_Mym.RNoPLh465HIc1BqRnY-1734361137-1.0.1.1-K.ErNa61CTMep7IAqgYh.YjE25TB8fFqpil9ma2Mo8untSEJFHro3wcRrnRYdjciM0Rshlbix9iHwHzov4BgTA; + _cfuvid=yQv8PrlaGn8AboPjIVwjUoFWTLDWPv2DU63sj9U8D5k-1734361137103-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -492,7 +492,7 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8f2a0b0fcdb3f78d-EWR + - 8f2f88dbae80c974-IAD Connection: - keep-alive Content-Encoding: @@ -500,7 +500,7 @@ interactions: Content-Type: - application/json Date: - - Sun, 15 Dec 2024 22:59:17 GMT + - Mon, 16 Dec 2024 14:58:58 GMT Server: - cloudflare Transfer-Encoding: @@ -518,7 +518,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '105' + - '207' openai-version: - '2020-10-01' strict-transport-security: @@ -536,7 +536,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_483b922409dfd3fd0f1fbe0f18b132fe + - req_6440b538ec2a83849de7a8e33aad1c45 status: code: 200 message: OK diff --git a/tests/llmobs/test_llmobs_ragas_evaluators.py b/tests/llmobs/test_llmobs_ragas_evaluators.py index c050b56a4df..71a01053bfa 100644 --- a/tests/llmobs/test_llmobs_ragas_evaluators.py +++ b/tests/llmobs/test_llmobs_ragas_evaluators.py @@ -638,7 +638,7 @@ def test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys( def test_ragas_answer_relevancy_emits_traces(ragas, LLMObs): rar_evaluator = RagasAnswerRelevancyEvaluator(LLMObs) rar_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) - assert rar_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_count == 2 + assert rar_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_count == 3 calls = rar_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_args_list spans = [call[0][0] for call in calls]