diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md index 44725df52d..5b66ea7972 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md @@ -13,3 +13,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#3123](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3123)) - Add server attributes to Vertex AI spans ([#3208](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3208)) +- VertexAI emit user, system, and assistant events + ([#3203](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3203)) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py new file mode 100644 index 0000000000..5d011006de --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py @@ -0,0 +1,91 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Factories for event types described in +https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-events.md#system-event. + +Hopefully this code can be autogenerated by Weaver once Gen AI semantic conventions are +schematized in YAML and the Weaver tool supports it. +""" + +from opentelemetry._events import Event +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes +from opentelemetry.util.types import AnyValue + + +def user_event( + *, + role: str = "user", + content: AnyValue = None, +) -> Event: + """Creates a User event + https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#user-event + """ + body: dict[str, AnyValue] = { + "role": role, + } + if content is not None: + body["content"] = content + return Event( + name="gen_ai.user.message", + attributes={ + gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, + }, + body=body, + ) + + +def assistant_event( + *, + role: str = "assistant", + content: AnyValue = None, +) -> Event: + """Creates an Assistant event + https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#assistant-event + """ + body: dict[str, AnyValue] = { + "role": role, + } + if content is not None: + body["content"] = content + return Event( + name="gen_ai.assistant.message", + attributes={ + gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, + }, + body=body, + ) + + +def system_event( + *, + role: str = "system", + content: AnyValue = None, +) -> Event: + """Creates a System event + https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#system-event + """ + body: dict[str, AnyValue] = { + "role": role, + } + if content is not None: + body["content"] = content + return Event( + name="gen_ai.system.message", + attributes={ + gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, + }, + body=body, + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index ecb87e4360..fe0a9cdf60 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -27,6 +27,7 @@ get_genai_request_attributes, get_server_attributes, get_span_name, + request_to_events, ) from opentelemetry.trace import SpanKind, Tracer @@ -113,12 +114,10 @@ def traced_method( kind=SpanKind.CLIENT, attributes=span_attributes, ) as _span: - # TODO: emit request events - # if span.is_recording(): - # for message in kwargs.get("messages", []): - # event_logger.emit( - # message_to_event(message, capture_content) - # ) + for event in request_to_events( + params=params, capture_content=capture_content + ): + event_logger.emit(event) # TODO: set error.type attribute # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md @@ -130,10 +129,9 @@ def traced_method( # ) # TODO: add response attributes and events - # if span.is_recording(): - # _set_response_attributes( - # span, result, event_logger, capture_content - # ) + # _set_response_attributes( + # span, result, event_logger, capture_content + # ) return result return traced_method diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py index e4297bc878..3e6de918f9 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py @@ -19,16 +19,24 @@ from os import environ from typing import ( TYPE_CHECKING, + Iterable, Mapping, Sequence, + cast, ) from urllib.parse import urlparse +from opentelemetry._events import Event +from opentelemetry.instrumentation.vertexai.events import ( + assistant_event, + system_event, + user_event, +) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) from opentelemetry.semconv.attributes import server_attributes -from opentelemetry.util.types import AttributeValue +from opentelemetry.util.types import AnyValue, AttributeValue if TYPE_CHECKING: from google.cloud.aiplatform_v1.types import content, tool @@ -157,3 +165,46 @@ def get_span_name(span_attributes: Mapping[str, AttributeValue]) -> str: if not model: return f"{name}" return f"{name} {model}" + + +def request_to_events( + *, params: GenerateContentParams, capture_content: bool +) -> Iterable[Event]: + # System message + if params.system_instruction: + request_content = _parts_to_any_value( + capture_content=capture_content, + parts=params.system_instruction.parts, + ) + yield system_event( + role=params.system_instruction.role, content=request_content + ) + + for content in params.contents or []: + # Assistant message + if content.role == "model": + request_content = _parts_to_any_value( + capture_content=capture_content, parts=content.parts + ) + + yield assistant_event(role=content.role, content=request_content) + # Assume user event but role should be "user" + else: + request_content = _parts_to_any_value( + capture_content=capture_content, parts=content.parts + ) + yield user_event(role=content.role, content=request_content) + + +def _parts_to_any_value( + *, + capture_content: bool, + parts: Sequence[content.Part] | Sequence[content_v1beta1.Part], +) -> list[dict[str, AnyValue]] | None: + if not capture_content: + return None + + return [ + cast("dict[str, AnyValue]", type(part).to_dict(part)) # type: ignore[reportUnknownMemberType] + for part in parts + ] diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_all_input_events.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_all_input_events.yaml new file mode 100644 index 0000000000..47c5ce6645 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_all_input_events.yaml @@ -0,0 +1,94 @@ +interactions: +- request: + body: |- + { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "My name is OpenTelemetry" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "Hello OpenTelemetry!" + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "Address me by name and say this is a test" + } + ] + } + ], + "systemInstruction": { + "role": "user", + "parts": [ + { + "text": "You are a clever language model" + } + ] + } + } + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '548' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: |- + { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "OpenTelemetry, this is a test.\n" + } + ] + }, + "finishReason": 1, + "avgLogprobs": -1.1655389850299496e-06 + } + ], + "usageMetadata": { + "promptTokenCount": 25, + "candidatesTokenCount": 9, + "totalTokenCount": 34 + }, + "modelVersion": "gemini-1.5-flash-002" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + content-length: + - '422' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml new file mode 100644 index 0000000000..dcc40f2fdf --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml @@ -0,0 +1,56 @@ +interactions: +- request: + body: |- + { + "contents": [ + { + "role": "invalid_role", + "parts": [ + { + "text": "Say this is a test" + } + ] + } + ] + } + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '149' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: |- + { + "error": { + "code": 400, + "message": "Please use a valid role: user, model.", + "status": "INVALID_ARGUMENT", + "details": [] + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + content-length: + - '416' + status: + code: 400 + message: Bad Request +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_without_events.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_without_events.yaml new file mode 100644 index 0000000000..0a71d24512 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_without_events.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: |- + { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Say this is a test" + } + ] + } + ] + } + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '141' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: |- + { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "Okay, I understand. I'm ready for your test. Please proceed.\n" + } + ] + }, + "finishReason": 1, + "avgLogprobs": -0.005519990466142956 + } + ], + "usageMetadata": { + "promptTokenCount": 5, + "candidatesTokenCount": 19, + "totalTokenCount": 24 + }, + "modelVersion": "gemini-1.5-flash-002" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + content-length: + - '453' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py index 2582a086f6..4a1ab1beba 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py @@ -8,6 +8,9 @@ ) from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor +from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import ( + InMemoryLogExporter, +) from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, @@ -18,6 +21,7 @@ @pytest.mark.vcr def test_generate_content( span_exporter: InMemorySpanExporter, + log_exporter: InMemoryLogExporter, instrument_with_content: VertexAIInstrumentor, ): model = GenerativeModel("gemini-1.5-flash-002") @@ -27,6 +31,50 @@ def test_generate_content( ] ) + # Emits span + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "chat gemini-1.5-flash-002" + assert dict(spans[0].attributes) == { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "gemini-1.5-flash-002", + "gen_ai.system": "vertex_ai", + "server.address": "us-central1-aiplatform.googleapis.com", + "server.port": 443, + } + + # Emits content event + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + log_record = logs[0].log_record + span_context = spans[0].get_span_context() + assert log_record.trace_id == span_context.trace_id + assert log_record.span_id == span_context.span_id + assert log_record.trace_flags == span_context.trace_flags + assert log_record.attributes == { + "gen_ai.system": "vertex_ai", + "event.name": "gen_ai.user.message", + } + assert log_record.body == { + "content": [{"text": "Say this is a test"}], + "role": "user", + } + + +@pytest.mark.vcr +def test_generate_content_without_events( + span_exporter: InMemorySpanExporter, + log_exporter: InMemoryLogExporter, + instrument_no_content: VertexAIInstrumentor, +): + model = GenerativeModel("gemini-1.5-flash-002") + model.generate_content( + [ + Content(role="user", parts=[Part.from_text("Say this is a test")]), + ] + ) + + # Emits span spans = span_exporter.get_finished_spans() assert len(spans) == 1 assert spans[0].name == "chat gemini-1.5-flash-002" @@ -38,6 +86,16 @@ def test_generate_content( "server.port": 443, } + # Emits event without body.content + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + log_record = logs[0].log_record + assert log_record.attributes == { + "gen_ai.system": "vertex_ai", + "event.name": "gen_ai.user.message", + } + assert log_record.body == {"role": "user"} + @pytest.mark.vcr def test_generate_content_empty_model( @@ -134,6 +192,38 @@ def test_generate_content_invalid_temperature( assert_span_error(spans[0]) +@pytest.mark.vcr +def test_generate_content_invalid_role( + log_exporter: InMemoryLogExporter, + instrument_with_content: VertexAIInstrumentor, +): + model = GenerativeModel("gemini-1.5-flash-002") + try: + # Fails because role must be "user" or "model" + model.generate_content( + [ + Content( + role="invalid_role", + parts=[Part.from_text("Say this is a test")], + ) + ] + ) + except BadRequest: + pass + + # Emits the faulty content which caused the request to fail + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + assert logs[0].log_record.attributes == { + "gen_ai.system": "vertex_ai", + "event.name": "gen_ai.user.message", + } + assert logs[0].log_record.body == { + "content": [{"text": "Say this is a test"}], + "role": "invalid_role", + } + + @pytest.mark.vcr() def test_generate_content_extra_params(span_exporter, instrument_no_content): generation_config = GenerationConfig( @@ -181,3 +271,74 @@ def assert_span_error(span: ReadableSpan) -> None: # Records exception event error_events = [e for e in span.events if e.name == "exception"] assert error_events != [] + + +@pytest.mark.vcr +def test_generate_content_all_input_events( + log_exporter: InMemoryLogExporter, + instrument_with_content: VertexAIInstrumentor, +): + model = GenerativeModel( + "gemini-1.5-flash-002", + system_instruction=Part.from_text("You are a clever language model"), + ) + model.generate_content( + [ + Content( + role="user", parts=[Part.from_text("My name is OpenTelemetry")] + ), + Content( + role="model", parts=[Part.from_text("Hello OpenTelemetry!")] + ), + Content( + role="user", + parts=[ + Part.from_text("Address me by name and say this is a test") + ], + ), + ], + ) + + # Emits a system event, 2 users events, and a assistant event + logs = log_exporter.get_finished_logs() + assert len(logs) == 4 + system_log, user_log1, assistant_log, user_log2 = [ + log_data.log_record for log_data in logs + ] + + assert system_log.attributes == { + "gen_ai.system": "vertex_ai", + "event.name": "gen_ai.system.message", + } + assert system_log.body == { + "content": [{"text": "You are a clever language model"}], + # The API only allows user and model, so system instruction is considered a user role + "role": "user", + } + + assert user_log1.attributes == { + "gen_ai.system": "vertex_ai", + "event.name": "gen_ai.user.message", + } + assert user_log1.body == { + "content": [{"text": "My name is OpenTelemetry"}], + "role": "user", + } + + assert assistant_log.attributes == { + "gen_ai.system": "vertex_ai", + "event.name": "gen_ai.assistant.message", + } + assert assistant_log.body == { + "content": [{"text": "Hello OpenTelemetry!"}], + "role": "model", + } + + assert user_log2.attributes == { + "gen_ai.system": "vertex_ai", + "event.name": "gen_ai.user.message", + } + assert user_log2.body == { + "content": [{"text": "Address me by name and say this is a test"}], + "role": "user", + }