Skip to content

Commit

Permalink
chore(llmobs): assert using span events instead of mocks (#11856)
Browse files Browse the repository at this point in the history
Follow up of #11781 to further clean up LLMObs tests, specifically
replacing potentially flaky LLMObs span writer mocks and assertions with
the test LLMObsSpanWriter dummy class. Also clean up the
`tests/llmobs/conftest.py` file which previously contained a ton of
rarely used and sometimes redundant fixtures.

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
  • Loading branch information
Yun-Kim authored Jan 9, 2025
1 parent 68fc8f0 commit 04ee68f
Show file tree
Hide file tree
Showing 6 changed files with 277 additions and 394 deletions.
50 changes: 1 addition & 49 deletions tests/llmobs/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,6 @@ def pytest_configure(config):
config.addinivalue_line("markers", "vcr_logs: mark test to use recorded request/responses")


@pytest.fixture
def mock_llmobs_span_writer():
patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsSpanWriter")
LLMObsSpanWriterMock = patcher.start()
m = mock.MagicMock()
LLMObsSpanWriterMock.return_value = m
yield m
patcher.stop()


@pytest.fixture
def mock_llmobs_eval_metric_writer():
patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsEvalMetricWriter")
Expand Down Expand Up @@ -127,44 +117,6 @@ def default_global_config():
return {"_dd_api_key": "<not-a-real-api_key>", "_llmobs_ml_app": "unnamed-ml-app"}


@pytest.fixture
def LLMObs(
mock_llmobs_span_writer, mock_llmobs_eval_metric_writer, mock_llmobs_evaluator_runner, ddtrace_global_config
):
global_config = default_global_config()
global_config.update(ddtrace_global_config)
with override_global_config(global_config):
dummy_tracer = DummyTracer()
llmobs_service.enable(_tracer=dummy_tracer)
yield llmobs_service
llmobs_service.disable()


@pytest.fixture
def AgentlessLLMObs(
mock_llmobs_span_writer,
mock_llmobs_eval_metric_writer,
mock_llmobs_evaluator_runner,
ddtrace_global_config,
):
global_config = default_global_config()
global_config.update(ddtrace_global_config)
global_config.update(dict(_llmobs_agentless_enabled=True))
with override_global_config(global_config):
dummy_tracer = DummyTracer()
llmobs_service.enable(_tracer=dummy_tracer)
yield llmobs_service
llmobs_service.disable()


@pytest.fixture
def disabled_llmobs():
prev = llmobs_service.enabled
llmobs_service.enabled = False
yield
llmobs_service.enabled = prev


@pytest.fixture
def mock_ragas_dependencies_not_present():
import ragas
Expand All @@ -177,7 +129,7 @@ def mock_ragas_dependencies_not_present():


@pytest.fixture
def ragas(mock_llmobs_span_writer, mock_llmobs_eval_metric_writer):
def ragas(mock_llmobs_eval_metric_writer):
with override_global_config(dict(_dd_api_key="<not-a-real-key>")):
try:
import ragas
Expand Down
Loading

0 comments on commit 04ee68f

Please sign in to comment.