diff --git a/instrumentation-genai/README.md b/instrumentation-genai/README.md index 0b6fa13f8a..cb80b8166d 100644 --- a/instrumentation-genai/README.md +++ b/instrumentation-genai/README.md @@ -1,7 +1,7 @@ | Instrumentation | Supported Packages | Metrics support | Semconv status | | --------------- | ------------------ | --------------- | -------------- | -| [opentelemetry-instrumentation-anthropic](./opentelemetry-instrumentation-anthropic) | anthropic >= 0.3.0 | No | development +| [opentelemetry-instrumentation-anthropic](./opentelemetry-instrumentation-anthropic) | anthropic >= 0.16.0 | No | development | [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.0.0 | No | development | [opentelemetry-instrumentation-langchain](./opentelemetry-instrumentation-langchain) | langchain >= 0.3.21 | No | development | [opentelemetry-instrumentation-openai-agents-v2](./opentelemetry-instrumentation-openai-agents-v2) | openai-agents >= 0.3.3 | No | development diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md index 6637be8f8a..ba164c0ebc 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md @@ -10,5 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Initial implementation of Anthropic instrumentation - ([#ISSUE_NUMBER](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3978)) + ([#3978](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3978)) +- Implement sync `Messages.create` instrumentation with GenAI semantic convention attributes + ([#4034](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4034)) + - Captures request attributes: `gen_ai.request.model`, `gen_ai.request.max_tokens`, `gen_ai.request.temperature`, `gen_ai.request.top_p`, `gen_ai.request.top_k`, `gen_ai.request.stop_sequences` + - Captures response attributes: `gen_ai.response.id`, `gen_ai.response.model`, `gen_ai.response.finish_reasons`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` + - Error handling with `error.type` attribute + - Minimum supported anthropic version is 0.16.0 (SDK uses modern `anthropic.resources.messages` module structure introduced in this version) diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/manual/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/manual/requirements.txt index ad0601fd3e..4ebceac4b9 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/manual/requirements.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/manual/requirements.txt @@ -1,4 +1,4 @@ -anthropic~=0.3.0 +anthropic>=0.16.0 opentelemetry-sdk~=1.36.0 opentelemetry-instrumentation-anthropic #TODO: update to 2.1b0 when released opentelemetry-exporter-otlp-proto-grpc~=1.36.0 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/zero-code/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/zero-code/requirements.txt index b2ae3a99fe..1ac7827d5c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/zero-code/requirements.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/examples/zero-code/requirements.txt @@ -1,4 +1,4 @@ -anthropic~=0.3.0 +anthropic>=0.16.0 opentelemetry-sdk~=1.28.2 opentelemetry-distro~=0.49b2 opentelemetry-instrumentation-anthropic #TODO: update to 2.1b0 when released diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/pyproject.toml index 8bbad37d4a..2997cbd5d6 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/pyproject.toml +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ [project.optional-dependencies] instruments = [ - "anthropic >= 0.3.0", + "anthropic >= 0.16.0", ] [project.entry-points.opentelemetry_instrumentor] diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py index 2255253d1e..3cb4a3e348 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py @@ -49,8 +49,15 @@ from typing import Any, Collection +from wrapt import ( + wrap_function_wrapper, # pyright: ignore[reportUnknownVariableType] +) + from opentelemetry.instrumentation.anthropic.package import _instruments +from opentelemetry.instrumentation.anthropic.patch import messages_create +from opentelemetry.instrumentation.anthropic.utils import is_content_enabled from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.semconv.schemas import Schemas @@ -119,11 +126,21 @@ def _instrument(self, **kwargs: Any) -> None: self._logger = logger self._meter = meter - # Patching will be added in Ticket 3 + # Patch Messages.create + wrap_function_wrapper( + module="anthropic.resources.messages", + name="Messages.create", + wrapper=messages_create(tracer, is_content_enabled()), + ) def _uninstrument(self, **kwargs: Any) -> None: """Disable Anthropic instrumentation. This removes all patches applied during instrumentation. """ - # Unpatching will be added in Ticket 3 + import anthropic # pylint: disable=import-outside-toplevel # noqa: PLC0415 + + unwrap( + anthropic.resources.messages.Messages, # pyright: ignore[reportAttributeAccessIssue,reportUnknownMemberType,reportUnknownArgumentType] + "create", + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/package.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/package.py index 89dcba846f..576638a5f8 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/package.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/package.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -_instruments = ("anthropic >= 0.3.0",) +_instruments = ("anthropic >= 0.16.0",) diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py index b0a6f42841..82dff57866 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py @@ -11,3 +11,87 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Patching functions for Anthropic instrumentation.""" + +from typing import Any, Callable + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.trace import Span, SpanKind, Tracer + +from .utils import ( + get_llm_request_attributes, + handle_span_exception, + set_span_attribute, +) + + +def messages_create( + tracer: Tracer, + _capture_content: bool, +) -> Callable[..., Any]: + """Wrap the `create` method of the `Messages` class to trace it.""" + + def traced_method( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + span_attributes = {**get_llm_request_attributes(kwargs, instance)} + + span_name = f"{span_attributes[GenAIAttributes.GEN_AI_OPERATION_NAME]} {span_attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL]}" + with tracer.start_as_current_span( + name=span_name, + kind=SpanKind.CLIENT, + attributes=span_attributes, + end_on_exit=False, + ) as span: + try: + result = wrapped(*args, **kwargs) + + if span.is_recording(): + _set_response_attributes(span, result) + + span.end() + return result + + except Exception as error: + handle_span_exception(span, error) + raise + + return traced_method + + +def _set_response_attributes(span: Span, result: Any) -> None: + """Set span attributes from the Anthropic response.""" + if getattr(result, "model", None): + set_span_attribute( + span, GenAIAttributes.GEN_AI_RESPONSE_MODEL, result.model + ) + + if getattr(result, "stop_reason", None): + # Anthropic uses stop_reason (single string), semantic convention expects array + set_span_attribute( + span, + GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, + [result.stop_reason], + ) + + if getattr(result, "id", None): + set_span_attribute(span, GenAIAttributes.GEN_AI_RESPONSE_ID, result.id) + + # Get the usage + if getattr(result, "usage", None): + set_span_attribute( + span, + GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, + result.usage.input_tokens, + ) + set_span_attribute( + span, + GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, + result.usage.output_tokens, + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/utils.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/utils.py new file mode 100644 index 0000000000..f14becf915 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/utils.py @@ -0,0 +1,108 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility functions for Anthropic instrumentation.""" + +from __future__ import annotations + +from os import environ +from typing import Any, Optional +from urllib.parse import urlparse + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.semconv._incubating.attributes import ( + server_attributes as ServerAttributes, +) +from opentelemetry.semconv.attributes import ( + error_attributes as ErrorAttributes, +) +from opentelemetry.trace import Span +from opentelemetry.trace.status import Status, StatusCode + +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = ( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" +) + + +def is_content_enabled() -> bool: + """Check if content capture is enabled via environment variable.""" + capture_content = environ.get( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "false" + ) + return capture_content.lower() == "true" + + +def set_server_address_and_port( + client_instance: Any, attributes: dict[str, Any] +) -> None: + """Extract server address and port from the Anthropic client instance.""" + base_client = getattr(client_instance, "_client", None) + base_url = getattr(base_client, "base_url", None) + if not base_url: + return + + port: Optional[int] = None + if hasattr(base_url, "host"): + # httpx.URL object + attributes[ServerAttributes.SERVER_ADDRESS] = base_url.host + port = getattr(base_url, "port", None) + elif isinstance(base_url, str): + url = urlparse(base_url) + attributes[ServerAttributes.SERVER_ADDRESS] = url.hostname + port = url.port + + if port and port != 443 and port > 0: + attributes[ServerAttributes.SERVER_PORT] = port + + +def set_span_attribute(span: Span, name: str, value: Any) -> None: + """Set a span attribute if the value is not None.""" + if value is None: + return + span.set_attribute(name, value) + + +def get_llm_request_attributes( + kwargs: dict[str, Any], client_instance: Any +) -> dict[str, Any]: + """Extract LLM request attributes from kwargs.""" + attributes = { + GenAIAttributes.GEN_AI_OPERATION_NAME: GenAIAttributes.GenAiOperationNameValues.CHAT.value, + GenAIAttributes.GEN_AI_SYSTEM: GenAIAttributes.GenAiSystemValues.ANTHROPIC.value, # pyright: ignore[reportDeprecated] + GenAIAttributes.GEN_AI_REQUEST_MODEL: kwargs.get("model"), + GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS: kwargs.get("max_tokens"), + GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE: kwargs.get("temperature"), + GenAIAttributes.GEN_AI_REQUEST_TOP_P: kwargs.get("top_p"), + GenAIAttributes.GEN_AI_REQUEST_TOP_K: kwargs.get("top_k"), + GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES: kwargs.get( + "stop_sequences" + ), + } + + set_server_address_and_port(client_instance, attributes) + + # Filter out None values + return {k: v for k, v in attributes.items() if v is not None} + + +def handle_span_exception(span: Span, error: Exception) -> None: + """Handle an exception by setting span status and error attributes.""" + span.set_status(Status(StatusCode.ERROR, str(error))) + if span.is_recording(): + span.set_attribute( + ErrorAttributes.ERROR_TYPE, type(error).__qualname__ + ) + span.end() diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_api_error.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_api_error.yaml new file mode 100644 index 0000000000..bb950b45bc --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_api_error.yaml @@ -0,0 +1,60 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "model": "invalid-model-name" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '110' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "type": "error", + "error": { + "type": "not_found_error", + "message": "model: invalid-model-name" + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 15 Dec 2024 10:00:04 GMT + Server: + - cloudflare + content-length: + - '105' + status: + code: 404 + message: Not Found +version: 1 + diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_basic.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_basic.yaml new file mode 100644 index 0000000000..82e9d60aba --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_basic.yaml @@ -0,0 +1,73 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hello in one word." + } + ], + "model": "claude-3-5-sonnet-20241022" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '128' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_01XFDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20241022", + "content": [ + { + "type": "text", + "text": "Hello!" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 14, + "output_tokens": 4 + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 15 Dec 2024 10:00:00 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + content-length: + - '350' + status: + code: 200 + message: OK +version: 1 + diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_stop_reason.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_stop_reason.yaml new file mode 100644 index 0000000000..8cadc6bbee --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_stop_reason.yaml @@ -0,0 +1,73 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hi." + } + ], + "model": "claude-3-5-sonnet-20241022" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '114' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_04AGDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20241022", + "content": [ + { + "type": "text", + "text": "Hi!" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 10, + "output_tokens": 3 + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 15 Dec 2024 10:00:03 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + content-length: + - '340' + status: + code: 200 + message: OK +version: 1 + diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_token_usage.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_token_usage.yaml new file mode 100644 index 0000000000..6f6e1a379c --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_token_usage.yaml @@ -0,0 +1,73 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Count to 5." + } + ], + "model": "claude-3-5-sonnet-20241022" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '118' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_03ZGDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20241022", + "content": [ + { + "type": "text", + "text": "1, 2, 3, 4, 5" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 12, + "output_tokens": 14 + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 15 Dec 2024 10:00:02 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + content-length: + - '355' + status: + code: 200 + message: OK +version: 1 + diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_with_all_params.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_with_all_params.yaml new file mode 100644 index 0000000000..0436f93657 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_sync_messages_create_with_all_params.yaml @@ -0,0 +1,77 @@ +interactions: +- request: + body: |- + { + "max_tokens": 50, + "messages": [ + { + "role": "user", + "content": "Say hello." + } + ], + "model": "claude-3-5-sonnet-20241022", + "stop_sequences": ["STOP"], + "temperature": 0.7, + "top_k": 40, + "top_p": 0.9 + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '200' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_02YGDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20241022", + "content": [ + { + "type": "text", + "text": "Hello! How can I help you today?" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 10, + "output_tokens": 10 + } + } + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 15 Dec 2024 10:00:01 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + content-length: + - '380' + status: + code: 200 + message: OK +version: 1 + diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py index 21b6e0b0c1..350baea4da 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py @@ -15,9 +15,18 @@ """Test configuration and fixtures for Anthropic instrumentation tests.""" # pylint: disable=redefined-outer-name +import json +import os + import pytest +import yaml +from anthropic import Anthropic from opentelemetry import trace +from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor +from opentelemetry.instrumentation.anthropic.utils import ( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, +) from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk._logs.export import ( InMemoryLogExporter, @@ -79,14 +88,72 @@ def meter_provider(metric_reader): yield provider +@pytest.fixture(autouse=True) +def environment(): + """Set up environment variables for testing.""" + if not os.getenv("ANTHROPIC_API_KEY"): + os.environ["ANTHROPIC_API_KEY"] = "test_anthropic_api_key" + + @pytest.fixture -def instrument_anthropic(tracer_provider, logger_provider, meter_provider): - """Fixture to instrument Anthropic with test providers.""" - # pylint: disable=import-outside-toplevel - from opentelemetry.instrumentation.anthropic import ( # noqa: PLC0415 - AnthropicInstrumentor, +def anthropic_client(): + """Create and return an Anthropic client.""" + return Anthropic() + + +@pytest.fixture(scope="module") +def vcr_config(): + """Configure VCR for recording/replaying HTTP interactions.""" + return { + "filter_headers": [ + ("x-api-key", "test_anthropic_api_key"), + ("authorization", "Bearer test_anthropic_api_key"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers, + } + + +@pytest.fixture(scope="function") +def instrument_no_content(tracer_provider, logger_provider, meter_provider): + """Instrument Anthropic without content capture.""" + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "False"} + ) + + instrumentor = AnthropicInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, ) + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +@pytest.fixture(scope="function") +def instrument_with_content(tracer_provider, logger_provider, meter_provider): + """Instrument Anthropic with content capture enabled.""" + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"} + ) + instrumentor = AnthropicInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +@pytest.fixture +def instrument_anthropic(tracer_provider, logger_provider, meter_provider): + """Fixture to instrument Anthropic with test providers.""" instrumentor = AnthropicInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, @@ -101,9 +168,70 @@ def instrument_anthropic(tracer_provider, logger_provider, meter_provider): def uninstrument_anthropic(): """Fixture to ensure Anthropic is uninstrumented after test.""" yield - # pylint: disable=import-outside-toplevel - from opentelemetry.instrumentation.anthropic import ( # noqa: PLC0415 - AnthropicInstrumentor, - ) - AnthropicInstrumentor().uninstrument() + + +class LiteralBlockScalar(str): + """Formats the string as a literal block scalar.""" + + +def literal_block_scalar_presenter(dumper, data): + """Represents a scalar string as a literal block, via '|' syntax.""" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + +yaml.add_representer(LiteralBlockScalar, literal_block_scalar_presenter) + + +def process_string_value(string_value): + """Pretty-prints JSON or returns long strings as a LiteralBlockScalar.""" + try: + json_data = json.loads(string_value) + return LiteralBlockScalar(json.dumps(json_data, indent=2)) + except (ValueError, TypeError): + if len(string_value) > 80: + return LiteralBlockScalar(string_value) + return string_value + + +def convert_body_to_literal(data): + """Searches the data for body strings, attempting to pretty-print JSON.""" + if isinstance(data, dict): + for key, value in data.items(): + if key == "body" and isinstance(value, dict) and "string" in value: + value["string"] = process_string_value(value["string"]) + elif key == "body" and isinstance(value, str): + data[key] = process_string_value(value) + else: + convert_body_to_literal(value) + elif isinstance(data, list): + for idx, choice in enumerate(data): + data[idx] = convert_body_to_literal(choice) + return data + + +class PrettyPrintJSONBody: + """This makes request and response body recordings more readable.""" + + @staticmethod + def serialize(cassette_dict): + cassette_dict = convert_body_to_literal(cassette_dict) + return yaml.dump( + cassette_dict, default_flow_style=False, allow_unicode=True + ) + + @staticmethod + def deserialize(cassette_string): + return yaml.load(cassette_string, Loader=yaml.Loader) + + +@pytest.fixture(scope="module", autouse=True) +def fixture_vcr(vcr): + """Register the VCR serializer.""" + vcr.register_serializer("yaml", PrettyPrintJSONBody) + return vcr + + +def scrub_response_headers(response): + """Scrub sensitive response headers.""" + return response diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/requirements.oldest.txt b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/requirements.oldest.txt index 7ecf4d1702..a0843b7004 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/requirements.oldest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/requirements.oldest.txt @@ -15,7 +15,8 @@ # This variant of the requirements aims to test the system using # the oldest supported version of external dependencies. -anthropic==0.3.0 +anthropic==0.16.0 +httpx>=0.25.2,<0.28.0 # Pin to version compatible with anthropic 0.16.0 (proxies arg removed in 0.28) pytest==7.4.4 pytest-vcr==1.0.2 pytest-asyncio==0.21.0 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_instrumentor.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_instrumentor.py index 2c25973507..b3e97bdf79 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_instrumentor.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_instrumentor.py @@ -31,7 +31,7 @@ def test_instrumentation_dependencies(): assert dependencies is not None assert len(dependencies) > 0 - assert "anthropic >= 0.3.0" in dependencies + assert "anthropic >= 0.16.0" in dependencies def test_instrument_uninstrument_cycle( diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_sync_messages.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_sync_messages.py new file mode 100644 index 0000000000..b8916b1498 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_sync_messages.py @@ -0,0 +1,302 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for sync Messages.create instrumentation.""" + +import pytest +from anthropic import Anthropic, APIConnectionError, NotFoundError + +from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor +from opentelemetry.semconv._incubating.attributes import ( + error_attributes as ErrorAttributes, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.semconv._incubating.attributes import ( + server_attributes as ServerAttributes, +) + + +def assert_span_attributes( + span, + request_model, + response_id=None, + response_model=None, + input_tokens=None, + output_tokens=None, + finish_reasons=None, + operation_name="chat", + server_address="api.anthropic.com", +): + """Assert that a span has the expected attributes.""" + assert span.name == f"{operation_name} {request_model}" + assert ( + operation_name + == span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + ) + assert ( + GenAIAttributes.GenAiSystemValues.ANTHROPIC.value + == span.attributes[GenAIAttributes.GEN_AI_SYSTEM] + ) + assert ( + request_model == span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] + ) + assert server_address == span.attributes[ServerAttributes.SERVER_ADDRESS] + + if response_id is not None: + assert ( + response_id == span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + ) + + if response_model is not None: + assert ( + response_model + == span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + ) + + if input_tokens is not None: + assert ( + input_tokens + == span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + ) + + if output_tokens is not None: + assert ( + output_tokens + == span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + ) + + if finish_reasons is not None: + # OpenTelemetry converts lists to tuples when storing as attributes + assert ( + tuple(finish_reasons) + == span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] + ) + + +@pytest.mark.vcr() +def test_sync_messages_create_basic( + span_exporter, anthropic_client, instrument_no_content +): + """Test basic sync message creation produces correct span.""" + model = "claude-3-5-sonnet-20241022" + messages = [{"role": "user", "content": "Say hello in one word."}] + + response = anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + assert_span_attributes( + spans[0], + request_model=model, + response_id=response.id, + response_model=response.model, + input_tokens=response.usage.input_tokens, + output_tokens=response.usage.output_tokens, + finish_reasons=[response.stop_reason], + ) + + +@pytest.mark.vcr() +def test_sync_messages_create_with_all_params( + span_exporter, anthropic_client, instrument_no_content +): + """Test message creation with all optional parameters.""" + model = "claude-3-5-sonnet-20241022" + messages = [{"role": "user", "content": "Say hello."}] + + anthropic_client.messages.create( + model=model, + max_tokens=50, + messages=messages, + temperature=0.7, + top_p=0.9, + top_k=40, + stop_sequences=["STOP"], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS] == 50 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_K] == 40 + # OpenTelemetry converts lists to tuples when storing as attributes + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES] == ( + "STOP", + ) + + +@pytest.mark.vcr() +def test_sync_messages_create_token_usage( + span_exporter, anthropic_client, instrument_no_content +): + """Test that token usage is captured correctly.""" + model = "claude-3-5-sonnet-20241022" + messages = [{"role": "user", "content": "Count to 5."}] + + response = anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS in span.attributes + assert GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS in span.attributes + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + == response.usage.input_tokens + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + == response.usage.output_tokens + ) + + +@pytest.mark.vcr() +def test_sync_messages_create_stop_reason( + span_exporter, anthropic_client, instrument_no_content +): + """Test that stop reason is captured as finish_reasons array.""" + model = "claude-3-5-sonnet-20241022" + messages = [{"role": "user", "content": "Say hi."}] + + response = anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + # Anthropic's stop_reason should be wrapped in a tuple (OTel converts lists) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( + response.stop_reason, + ) + + +def test_sync_messages_create_connection_error( + span_exporter, instrument_no_content +): + """Test that connection errors are handled correctly.""" + model = "claude-3-5-sonnet-20241022" + messages = [{"role": "user", "content": "Hello"}] + + # Create client with invalid endpoint + client = Anthropic(base_url="http://localhost:9999") + + with pytest.raises(APIConnectionError): + client.messages.create( + model=model, + max_tokens=100, + messages=messages, + timeout=0.1, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + assert ErrorAttributes.ERROR_TYPE in span.attributes + assert "APIConnectionError" in span.attributes[ErrorAttributes.ERROR_TYPE] + + +@pytest.mark.vcr() +def test_sync_messages_create_api_error( + span_exporter, anthropic_client, instrument_no_content +): + """Test that API errors (e.g., invalid model) are handled correctly.""" + model = "invalid-model-name" + messages = [{"role": "user", "content": "Hello"}] + + with pytest.raises(NotFoundError): + anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + assert ErrorAttributes.ERROR_TYPE in span.attributes + assert "NotFoundError" in span.attributes[ErrorAttributes.ERROR_TYPE] + + +def test_uninstrument_removes_patching( + span_exporter, tracer_provider, logger_provider, meter_provider +): + """Test that uninstrument() removes the patching.""" + instrumentor = AnthropicInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + # Uninstrument + instrumentor.uninstrument() + + # Create a new client after uninstrumenting + # The actual API call won't work without a real API key, + # but we can verify no spans are created for a mocked scenario + # For this test, we'll just verify uninstrument doesn't raise + assert True + + +def test_multiple_instrument_uninstrument_cycles( + tracer_provider, logger_provider, meter_provider +): + """Test that multiple instrument/uninstrument cycles work correctly.""" + instrumentor = AnthropicInstrumentor() + + # First cycle + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + instrumentor.uninstrument() + + # Second cycle + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + instrumentor.uninstrument() + + # Third cycle - should still work + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + instrumentor.uninstrument() diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/metrics.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/metrics.py index 68f8d08d39..c1865ff01e 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/metrics.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/metrics.py @@ -69,7 +69,8 @@ def record( duration_seconds: Optional[float] = None if invocation.monotonic_start_s is not None: duration_seconds = max( - timeit.default_timer() - invocation.monotonic_start_s, 0.0 + timeit.default_timer() - invocation.monotonic_start_s, + 0.0, ) span_context = set_span_in_context(span) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py index 4fbb059e73..02fe1c371e 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py @@ -123,12 +123,10 @@ class LLMInvocation: max_tokens: int | None = None stop_sequences: list[str] | None = None seed: int | None = None + # Monotonic start time in seconds (from timeit.default_timer) used + # for duration calculations to avoid mixing clock sources. This is + # populated by the TelemetryHandler when starting an invocation. monotonic_start_s: float | None = None - """ - Monotonic start time in seconds (from timeit.default_timer) used - for duration calculations to avoid mixing clock sources. This is - populated by the TelemetryHandler when starting an invocation. - """ @dataclass diff --git a/uv.lock b/uv.lock index bde58946cc..277418da95 100644 --- a/uv.lock +++ b/uv.lock @@ -2801,7 +2801,7 @@ instruments = [ [package.metadata] requires-dist = [ - { name = "anthropic", marker = "extra == 'instruments'", specifier = ">=0.3.0" }, + { name = "anthropic", marker = "extra == 'instruments'", specifier = ">=0.16.0" }, { name = "opentelemetry-api", git = "https://github.com/open-telemetry/opentelemetry-python?subdirectory=opentelemetry-api&branch=main" }, { name = "opentelemetry-instrumentation", editable = "opentelemetry-instrumentation" }, { name = "opentelemetry-semantic-conventions", git = "https://github.com/open-telemetry/opentelemetry-python?subdirectory=opentelemetry-semantic-conventions&branch=main" },