diff --git a/python/instrumentation/openinference-instrumentation-smolagents/README.md b/python/instrumentation/openinference-instrumentation-smolagents/README.md index fc0b9f788..8f0971584 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/README.md +++ b/python/instrumentation/openinference-instrumentation-smolagents/README.md @@ -44,7 +44,6 @@ trace_provider = TracerProvider() trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint))) SmolagentsInstrumentor().instrument(tracer_provider=trace_provider) -SmolagentsInstrumentor()._instrument(tracer_provider=trace_provider) from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel diff --git a/python/instrumentation/openinference-instrumentation-smolagents/examples/managed_agent.py b/python/instrumentation/openinference-instrumentation-smolagents/examples/managed_agent.py index e82da0080..30b0cef03 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/examples/managed_agent.py +++ b/python/instrumentation/openinference-instrumentation-smolagents/examples/managed_agent.py @@ -10,7 +10,6 @@ trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint))) SmolagentsInstrumentor().instrument(tracer_provider=trace_provider) -SmolagentsInstrumentor()._instrument(tracer_provider=trace_provider) agent = ToolCallingAgent(tools=[DuckDuckGoSearchTool()], model=HfApiModel(), max_steps=3) diff --git a/python/instrumentation/openinference-instrumentation-smolagents/examples/openai_model.py b/python/instrumentation/openinference-instrumentation-smolagents/examples/openai_model.py new file mode 100644 index 000000000..eee9f669c --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-smolagents/examples/openai_model.py @@ -0,0 +1,22 @@ +import os + +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, +) +from smolagents import OpenAIServerModel + +from openinference.instrumentation.smolagents import SmolagentsInstrumentor + +endpoint = "http://0.0.0.0:6006/v1/traces" +trace_provider = TracerProvider() +trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint))) + +SmolagentsInstrumentor().instrument(tracer_provider=trace_provider, skip_dep_check=True) + +model = OpenAIServerModel( + model_id="gpt-4o", api_key=os.environ["OPENAI_API_KEY"], api_base="https://api.openai.com/v1" +) +output = model(messages=[{"role": "user", "content": "hello world"}]) +print(output) diff --git a/python/instrumentation/openinference-instrumentation-smolagents/examples/openai_model_tool_call.py b/python/instrumentation/openinference-instrumentation-smolagents/examples/openai_model_tool_call.py new file mode 100644 index 000000000..954fdad1b --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-smolagents/examples/openai_model_tool_call.py @@ -0,0 +1,42 @@ +import os + +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, +) +from smolagents import OpenAIServerModel +from smolagents.tools import Tool + +from openinference.instrumentation.smolagents import SmolagentsInstrumentor + +endpoint = "http://0.0.0.0:6006/v1/traces" +trace_provider = TracerProvider() +trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint))) + +SmolagentsInstrumentor().instrument(tracer_provider=trace_provider, skip_dep_check=True) + + +class GetWeatherTool(Tool): + name = "get_weather" + description = "Get the weather for a given city" + inputs = {"location": {"type": "string", "description": "The city to get the weather for"}} + output_type = "string" + + def forward(self, location: str) -> str: + return "sunny" + + +model = OpenAIServerModel( + model_id="gpt-4o", api_key=os.environ["OPENAI_API_KEY"], api_base="https://api.openai.com/v1" +) +output_message = model( + messages=[ + { + "role": "user", + "content": "What is the weather in Paris?", + } + ], + tools_to_call_from=[GetWeatherTool()], +) +print(output_message) diff --git a/python/instrumentation/openinference-instrumentation-smolagents/examples/rag.py b/python/instrumentation/openinference-instrumentation-smolagents/examples/rag.py index 5fcde41f9..006f29771 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/examples/rag.py +++ b/python/instrumentation/openinference-instrumentation-smolagents/examples/rag.py @@ -18,7 +18,6 @@ trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint))) SmolagentsInstrumentor().instrument(tracer_provider=trace_provider) -SmolagentsInstrumentor()._instrument(tracer_provider=trace_provider) knowledge_base = datasets.load_dataset("m-ric/huggingface_doc", split="train") knowledge_base = knowledge_base.filter( diff --git a/python/instrumentation/openinference-instrumentation-smolagents/examples/text2sql.py b/python/instrumentation/openinference-instrumentation-smolagents/examples/text2sql.py index eebecd657..e6a02ff3d 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/examples/text2sql.py +++ b/python/instrumentation/openinference-instrumentation-smolagents/examples/text2sql.py @@ -30,7 +30,6 @@ trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint))) SmolagentsInstrumentor().instrument(tracer_provider=trace_provider) -SmolagentsInstrumentor()._instrument(tracer_provider=trace_provider) engine = create_engine("sqlite:///:memory:") metadata_obj = MetaData() diff --git a/python/instrumentation/openinference-instrumentation-smolagents/examples/tool_calling_agent.py b/python/instrumentation/openinference-instrumentation-smolagents/examples/tool_calling_agent.py index 6246e35c7..156b5ec5d 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/examples/tool_calling_agent.py +++ b/python/instrumentation/openinference-instrumentation-smolagents/examples/tool_calling_agent.py @@ -17,8 +17,7 @@ trace_provider = TracerProvider() trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint))) -SmolagentsInstrumentor().instrument(tracer_provider=trace_provider) -SmolagentsInstrumentor()._instrument(tracer_provider=trace_provider) +SmolagentsInstrumentor().instrument(tracer_provider=trace_provider, skip_dep_check=True) # Choose which LLM engine to use! # model = HfApiModel(model_id="meta-llama/Llama-3.3-70B-Instruct") diff --git a/python/instrumentation/openinference-instrumentation-smolagents/pyproject.toml b/python/instrumentation/openinference-instrumentation-smolagents/pyproject.toml index 564ee8ee3..3c1b2c2c1 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/pyproject.toml +++ b/python/instrumentation/openinference-instrumentation-smolagents/pyproject.toml @@ -34,13 +34,12 @@ dependencies = [ [project.optional-dependencies] instruments = [ - "smolagents >= 1.1.0", + "smolagents>=1.2.2", ] test = [ - "smolagents == 1.1.0", + "smolagents>=1.2.2", "opentelemetry-sdk", - "responses", - "vcrpy", + "pytest-recording", ] [project.urls] diff --git a/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/__init__.py b/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/__init__.py index d77aab5d6..cfad7636b 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/__init__.py +++ b/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/__init__.py @@ -18,7 +18,7 @@ ) from openinference.instrumentation.smolagents.version import __version__ -_instruments = ("smolagents >= 1.1.0",) +_instruments = ("smolagents >= 1.2.2",) logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ class SmolagentsInstrumentor(BaseInstrumentor): # type: ignore "_original_run", "_original_step", "_original_tool_call", - "_original_model", + "_original_model_calls", "_tracer", ) @@ -76,8 +76,10 @@ def _instrument(self, **kwargs: Any) -> None: from smolagents import Model model_subclasses = Model.__subclasses__() + self._original_model_calls = {} for model_subclass in model_subclasses: model_subclass_wrapper = _ModelWrapper(tracer=self._tracer) + self._original_model_calls[model_subclass] = getattr(model_subclass, "__call__") wrap_function_wrapper( module="smolagents", name=model_subclass.__name__ + ".__call__", @@ -103,10 +105,11 @@ def _uninstrument(self, **kwargs: Any) -> None: smolagents_module.MultiStepAgent.step = self._original_step self._original_step = None - if self._original_model_generate is not None: + if self._original_model_calls is not None: smolagents_module = import_module("smolagents.models") - smolagents_module.MultimodelAgent.model = self._original_model_generate - self._original_model = None + for model_subclass, original_model_call in self._original_model_calls.items(): + setattr(model_subclass, "__call__", original_model_call) + self._original_model_calls = None if self._original_tool_call is not None: tool_usage_module = import_module("smolagents.tools") diff --git a/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/_wrappers.py b/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/_wrappers.py index 19baa0d1d..ef995f2df 100644 --- a/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/_wrappers.py +++ b/python/instrumentation/openinference-instrumentation-smolagents/src/openinference/instrumentation/smolagents/_wrappers.py @@ -13,23 +13,11 @@ OpenInferenceMimeTypeValues, OpenInferenceSpanKindValues, SpanAttributes, + ToolAttributes, + ToolCallAttributes, ) -class SafeJSONEncoder(json.JSONEncoder): - """ - Safely encodes non-JSON-serializable objects. - """ - - def default(self, o: Any) -> Any: - try: - return super().default(o) - except TypeError: - if hasattr(o, "dict") and callable(o.dict): # pydantic v1 models, e.g., from Cohere - return o.dict() - return repr(o) - - def _flatten(mapping: Optional[Mapping[str, Any]]) -> Iterator[Tuple[str, AttributeValue]]: if not mapping: return @@ -80,7 +68,6 @@ def _get_input_value(method: Callable[..., Any], *args: Any, **kwargs: Any) -> s }, **bound_arguments.arguments.get("kwargs", {}), }, - cls=SafeJSONEncoder, ) @@ -105,8 +92,8 @@ def __call__( attributes=dict( _flatten( { - OPENINFERENCE_SPAN_KIND: OpenInferenceSpanKindValues.AGENT, - SpanAttributes.INPUT_VALUE: _get_input_value( + OPENINFERENCE_SPAN_KIND: AGENT, + INPUT_VALUE: _get_input_value( wrapped, *args, **kwargs, @@ -194,13 +181,9 @@ def __call__( span_name, record_exception=False, set_status_on_exception=False, - attributes=dict( - _flatten( - { - OPENINFERENCE_SPAN_KIND: OpenInferenceSpanKindValues.CHAIN, - } - ) - ), + attributes={ + OPENINFERENCE_SPAN_KIND: CHAIN, + }, ) as span: try: result = wrapped(*args, **kwargs) @@ -236,35 +219,85 @@ def _bind_arguments(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Di def _llm_input_messages(arguments: Mapping[str, Any]) -> Iterator[Tuple[str, Any]]: if isinstance(prompt := arguments.get("prompt"), str): - yield f"{SpanAttributes.LLM_INPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_ROLE}", "user" - yield f"{SpanAttributes.LLM_INPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}", prompt + yield f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}", "user" + yield f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}", prompt elif isinstance(messages := arguments.get("messages"), list): for i, message in enumerate(messages): if not isinstance(message, dict): continue if (role := message.get("role", None)) is not None: yield ( - f"{SpanAttributes.LLM_INPUT_MESSAGES}.{i}.{MessageAttributes.MESSAGE_ROLE}", + f"{LLM_INPUT_MESSAGES}.{i}.{MESSAGE_ROLE}", role, ) if (content := message.get("content", None)) is not None: yield ( - f"{SpanAttributes.LLM_INPUT_MESSAGES}.{i}.{MessageAttributes.MESSAGE_CONTENT}", + f"{LLM_INPUT_MESSAGES}.{i}.{MESSAGE_CONTENT}", content, ) +def _llm_output_messages(output_message: Any) -> Iterator[Tuple[str, Any]]: + if (role := getattr(output_message, "role", None)) is not None: + yield ( + f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_ROLE}", + role, + ) + if (content := getattr(output_message, "content", None)) is not None: + yield ( + f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_CONTENT}", + content, + ) + if isinstance(tool_calls := getattr(output_message, "tool_calls", None), list): + for tool_call_index, tool_call in enumerate(tool_calls): + if (tool_call_id := getattr(tool_call, "id", None)) is not None: + yield ( + f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.{tool_call_index}.{TOOL_CALL_ID}", + tool_call_id, + ) + if (function := getattr(tool_call, "function", None)) is not None: + if (name := getattr(function, "name", None)) is not None: + yield ( + f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.{tool_call_index}.{TOOL_CALL_FUNCTION_NAME}", + name, + ) + if isinstance(arguments := getattr(function, "arguments", None), str): + yield ( + f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.{tool_call_index}.{TOOL_CALL_FUNCTION_ARGUMENTS_JSON}", + arguments, + ) + + +def _output_value_and_mime_type(output: Any) -> Iterator[Tuple[str, Any]]: + yield OUTPUT_MIME_TYPE, JSON + yield OUTPUT_VALUE, output.model_dump_json() + + def _llm_invocation_parameters( model: Any, arguments: Mapping[str, Any] ) -> Iterator[Tuple[str, Any]]: model_kwargs = _ if isinstance(_ := getattr(model, "kwargs", {}), dict) else {} kwargs = _ if isinstance(_ := arguments.get("kwargs"), dict) else {} - yield SpanAttributes.LLM_INVOCATION_PARAMETERS, safe_json_dumps(model_kwargs | kwargs) + yield LLM_INVOCATION_PARAMETERS, safe_json_dumps(model_kwargs | kwargs) + + +def _llm_tools(tools_to_call_from: List[Any]) -> Iterator[Tuple[str, Any]]: + from smolagents import Tool + from smolagents.models import get_json_schema + + if not isinstance(tools_to_call_from, list): + return + for tool_index, tool in enumerate(tools_to_call_from): + if isinstance(tool, Tool): + yield ( + f"{LLM_TOOLS}.{tool_index}.{TOOL_JSON_SCHEMA}", + safe_json_dumps(get_json_schema(tool)), + ) def _input_value_and_mime_type(arguments: Mapping[str, Any]) -> Iterator[Tuple[str, Any]]: - yield SpanAttributes.INPUT_MIME_TYPE, OpenInferenceMimeTypeValues.JSON.value - yield SpanAttributes.INPUT_VALUE, safe_json_dumps(arguments) + yield INPUT_MIME_TYPE, JSON + yield INPUT_VALUE, safe_json_dumps(arguments) class _ModelWrapper: @@ -282,28 +315,24 @@ def __call__( return wrapped(*args, **kwargs) arguments = _bind_arguments(wrapped, *args, **kwargs) if instance: - span_name = f"{instance.__class__.__name__}" + span_name = f"{instance.__class__.__name__}.__call__" else: span_name = wrapped.__name__ model = instance with self._tracer.start_as_current_span( span_name, - attributes=dict( - _flatten( - { - OPENINFERENCE_SPAN_KIND: OpenInferenceSpanKindValues.LLM, - **dict(_input_value_and_mime_type(arguments)), - **dict(_llm_invocation_parameters(instance, arguments)), - **dict(_llm_input_messages(arguments)), - **dict(get_attributes_from_context()), - } - ) - ), + attributes={ + OPENINFERENCE_SPAN_KIND: LLM, + **dict(_input_value_and_mime_type(arguments)), + **dict(_llm_invocation_parameters(instance, arguments)), + **dict(_llm_input_messages(arguments)), + **dict(get_attributes_from_context()), + }, record_exception=False, set_status_on_exception=False, ) as span: try: - response = wrapped(*args, **kwargs) + output_message = wrapped(*args, **kwargs) except Exception as exception: span.set_status(trace_api.Status(trace_api.StatusCode.ERROR, str(exception))) span.record_exception(exception) @@ -315,9 +344,11 @@ def __call__( LLM_TOKEN_COUNT_TOTAL, model.last_input_token_count + model.last_output_token_count ) span.set_status(trace_api.StatusCode.OK) - span.set_attribute(OUTPUT_VALUE, response) - span.set_attributes(dict(get_attributes_from_context())) - return response + span.set_attribute(OUTPUT_VALUE, output_message) + span.set_attributes(dict(_llm_output_messages(output_message))) + span.set_attributes(dict(_llm_tools(arguments.get("tools_to_call_from", [])))) + span.set_attributes(dict(_output_value_and_mime_type(output_message))) + return output_message class _ToolCallWrapper: @@ -342,8 +373,8 @@ def __call__( attributes=dict( _flatten( { - OPENINFERENCE_SPAN_KIND: OpenInferenceSpanKindValues.TOOL, - SpanAttributes.INPUT_VALUE: _get_input_value( + OPENINFERENCE_SPAN_KIND: TOOL, + INPUT_VALUE: _get_input_value( wrapped, *args, **kwargs, @@ -354,8 +385,8 @@ def __call__( record_exception=False, set_status_on_exception=False, ) as span: - span.set_attribute(SpanAttributes.TOOL_NAME, f"{instance.__class__.name}") - span.set_attribute(SpanAttributes.TOOL_PARAMETERS, json.dumps(kwargs)) + span.set_attribute(TOOL_CALL_FUNCTION_NAME, f"{instance.__class__.name}") + span.set_attribute(TOOL_CALL_FUNCTION_ARGUMENTS_JSON, json.dumps(kwargs)) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -368,11 +399,44 @@ def __call__( return response +# span attributes +INPUT_MIME_TYPE = SpanAttributes.INPUT_MIME_TYPE INPUT_VALUE = SpanAttributes.INPUT_VALUE -OPENINFERENCE_SPAN_KIND = SpanAttributes.OPENINFERENCE_SPAN_KIND -OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE -OUTPUT_MIME_TYPE = SpanAttributes.OUTPUT_MIME_TYPE +LLM_INPUT_MESSAGES = SpanAttributes.LLM_INPUT_MESSAGES +LLM_INVOCATION_PARAMETERS = SpanAttributes.LLM_INVOCATION_PARAMETERS LLM_MODEL_NAME = SpanAttributes.LLM_MODEL_NAME -LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT +LLM_OUTPUT_MESSAGES = SpanAttributes.LLM_OUTPUT_MESSAGES +LLM_PROMPTS = SpanAttributes.LLM_PROMPTS LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION +LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT LLM_TOKEN_COUNT_TOTAL = SpanAttributes.LLM_TOKEN_COUNT_TOTAL +LLM_TOOLS = SpanAttributes.LLM_TOOLS +OPENINFERENCE_SPAN_KIND = SpanAttributes.OPENINFERENCE_SPAN_KIND +OUTPUT_MIME_TYPE = SpanAttributes.OUTPUT_MIME_TYPE +OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE + +# message attributes +MESSAGE_CONTENT = MessageAttributes.MESSAGE_CONTENT +MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON = MessageAttributes.MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON +MESSAGE_FUNCTION_CALL_NAME = MessageAttributes.MESSAGE_FUNCTION_CALL_NAME +MESSAGE_NAME = MessageAttributes.MESSAGE_NAME +MESSAGE_ROLE = MessageAttributes.MESSAGE_ROLE +MESSAGE_TOOL_CALLS = MessageAttributes.MESSAGE_TOOL_CALLS + +# mime types +JSON = OpenInferenceMimeTypeValues.JSON.value +TEXT = OpenInferenceMimeTypeValues.TEXT.value + +# span kinds +AGENT = OpenInferenceSpanKindValues.AGENT.value +CHAIN = OpenInferenceSpanKindValues.CHAIN.value +LLM = OpenInferenceSpanKindValues.LLM.value +TOOL = OpenInferenceSpanKindValues.TOOL.value + +# tool attributes +TOOL_JSON_SCHEMA = ToolAttributes.TOOL_JSON_SCHEMA + +# tool call attributes +TOOL_CALL_FUNCTION_ARGUMENTS_JSON = ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON +TOOL_CALL_FUNCTION_NAME = ToolCallAttributes.TOOL_CALL_FUNCTION_NAME +TOOL_CALL_ID = ToolCallAttributes.TOOL_CALL_ID diff --git a/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/cassettes/test_instrumentor/TestModels.test_openai_server_model_has_expected_attributes.yaml b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/cassettes/test_instrumentor/TestModels.test_openai_server_model_has_expected_attributes.yaml new file mode 100644 index 000000000..503596857 --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/cassettes/test_instrumentor/TestModels.test_openai_server_model_has_expected_attributes.yaml @@ -0,0 +1,26 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Who won the World Cup in 2018? + Answer in one word with no punctuation."}], "model": "gpt-4o", "max_tokens": + 1500, "stop": null, "temperature": 0.7}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ap7ufxJG59lObqU3ZjqsDq0ryGo1M\",\n \"object\": + \"chat.completion\",\n \"created\": 1736748509,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"France\",\n \"refusal\": null\n + \ },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 25,\n \"completion_tokens\": + 2,\n \"total_tokens\": 27,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_703d4ff298\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/cassettes/test_instrumentor/TestModels.test_openai_server_model_with_tool_has_expected_attributes.yaml b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/cassettes/test_instrumentor/TestModels.test_openai_server_model_with_tool_has_expected_attributes.yaml new file mode 100644 index 000000000..4636a5895 --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/cassettes/test_instrumentor/TestModels.test_openai_server_model_with_tool_has_expected_attributes.yaml @@ -0,0 +1,33 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "What is the weather in Paris?"}], + "model": "gpt-4o", "max_tokens": 1500, "stop": null, "temperature": 0.7, "tool_choice": + "auto", "tools": [{"type": "function", "function": {"name": "get_weather", "description": + "Get the weather for a given city", "parameters": {"type": "object", "properties": + {"location": {"type": "string", "description": "The city to get the weather + for"}}, "required": ["location"]}}}]}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ap7uf7aUH5nQ3tOeCHnA7WP70vV23\",\n \"object\": + \"chat.completion\",\n \"created\": 1736748509,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_SJU40osDa7rxyCVRXc9wG2Vs\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"location\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null\n },\n \"logprobs\": + null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": + {\n \"prompt_tokens\": 61,\n \"completion_tokens\": 15,\n \"total_tokens\": + 76,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n + \ \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_703d4ff298\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/test_instrumentor.py b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/test_instrumentor.py new file mode 100644 index 000000000..a7ef24a54 --- /dev/null +++ b/python/instrumentation/openinference-instrumentation-smolagents/tests/openinference/instrumentation/smolagents/test_instrumentor.py @@ -0,0 +1,297 @@ +import json +import os +from typing import Any, Generator + +import pytest +from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall +from opentelemetry import trace as trace_api +from opentelemetry.sdk import trace as trace_sdk +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from smolagents import OpenAIServerModel +from smolagents.tools import Tool + +from openinference.instrumentation.smolagents import SmolagentsInstrumentor +from openinference.semconv.trace import ( + MessageAttributes, + OpenInferenceMimeTypeValues, + OpenInferenceSpanKindValues, + SpanAttributes, + ToolAttributes, + ToolCallAttributes, +) + + +def remove_all_vcr_request_headers(request: Any) -> Any: + """ + Removes all request headers. + + Example: + ``` + @pytest.mark.vcr( + before_record_response=remove_all_vcr_request_headers + ) + def test_openai() -> None: + # make request to OpenAI + """ + request.headers.clear() + return request + + +def remove_all_vcr_response_headers(response: dict[str, Any]) -> dict[str, Any]: + """ + Removes all response headers. + + Example: + ``` + @pytest.mark.vcr( + before_record_response=remove_all_vcr_response_headers + ) + def test_openai() -> None: + # make request to OpenAI + """ + response["headers"] = {} + return response + + +@pytest.fixture +def in_memory_span_exporter() -> InMemorySpanExporter: + return InMemorySpanExporter() + + +@pytest.fixture +def tracer_provider(in_memory_span_exporter: InMemorySpanExporter) -> trace_api.TracerProvider: + resource = Resource(attributes={}) + tracer_provider = trace_sdk.TracerProvider(resource=resource) + span_processor = SimpleSpanProcessor(span_exporter=in_memory_span_exporter) + tracer_provider.add_span_processor(span_processor=span_processor) + return tracer_provider + + +@pytest.fixture(autouse=True) +def instrument( + tracer_provider: trace_api.TracerProvider, + in_memory_span_exporter: InMemorySpanExporter, +) -> Generator[None, None, None]: + SmolagentsInstrumentor().instrument(tracer_provider=tracer_provider, skip_dep_check=True) + yield + SmolagentsInstrumentor().uninstrument() + in_memory_span_exporter.clear() + + +@pytest.fixture +def openai_api_key(monkeypatch: pytest.MonkeyPatch) -> str: + api_key = "sk-0123456789" + monkeypatch.setenv("OPENAI_API_KEY", api_key) + return api_key + + +class TestModels: + @pytest.mark.vcr( + decode_compressed_response=True, + before_record_request=remove_all_vcr_request_headers, + before_record_response=remove_all_vcr_response_headers, + ) + def test_openai_server_model_has_expected_attributes( + self, + openai_api_key: str, + in_memory_span_exporter: InMemorySpanExporter, + ) -> None: + model = OpenAIServerModel( + model_id="gpt-4o", + api_key=os.environ["OPENAI_API_KEY"], + api_base="https://api.openai.com/v1", + ) + input_message_content = ( + "Who won the World Cup in 2018? Answer in one word with no punctuation." + ) + output_message = model( + messages=[ + { + "role": "user", + "content": input_message_content, + } + ] + ) + output_message_content = output_message.content + assert output_message_content == "France" + + spans = in_memory_span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.name == "OpenAIServerModel.__call__" + assert span.status.is_ok + attributes = dict(span.attributes or {}) + assert attributes.pop(OPENINFERENCE_SPAN_KIND) == LLM + assert attributes.pop(INPUT_MIME_TYPE) == JSON + assert isinstance(input_value := attributes.pop(INPUT_VALUE), str) + input_data = json.loads(input_value) + assert "messages" in input_data + assert attributes.pop(OUTPUT_MIME_TYPE) == JSON + assert isinstance(output_value := attributes.pop(OUTPUT_VALUE), str) + assert isinstance(json.loads(output_value), dict) + assert attributes.pop(LLM_MODEL_NAME) == "gpt-4o" + assert isinstance(inv_params := attributes.pop(LLM_INVOCATION_PARAMETERS), str) + assert json.loads(inv_params) == {} + assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "user" + assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}") == input_message_content + assert isinstance(attributes.pop(LLM_TOKEN_COUNT_PROMPT), int) + assert isinstance(attributes.pop(LLM_TOKEN_COUNT_COMPLETION), int) + assert isinstance(attributes.pop(LLM_TOKEN_COUNT_TOTAL), int) + assert attributes.pop(f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "assistant" + assert ( + attributes.pop(f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_CONTENT}") == output_message_content + ) + assert not attributes + + @pytest.mark.vcr( + decode_compressed_response=True, + before_record_request=remove_all_vcr_request_headers, + before_record_response=remove_all_vcr_response_headers, + ) + def test_openai_server_model_with_tool_has_expected_attributes( + self, + openai_api_key: str, + in_memory_span_exporter: InMemorySpanExporter, + tracer_provider: trace_api.TracerProvider, + ) -> None: + model = OpenAIServerModel( + model_id="gpt-4o", + api_key=os.environ["OPENAI_API_KEY"], + api_base="https://api.openai.com/v1", + ) + input_message_content = "What is the weather in Paris?" + + class GetWeatherTool(Tool): + name = "get_weather" + description = "Get the weather for a given city" + inputs = { + "location": {"type": "string", "description": "The city to get the weather for"} + } + output_type = "string" + + def forward(self, location: str) -> str: + return "sunny" + + output_message = model( + messages=[ + { + "role": "user", + "content": input_message_content, + } + ], + tools_to_call_from=[GetWeatherTool()], + ) + output_message_content = output_message.content + assert output_message_content is None + tool_calls = output_message.tool_calls + assert len(tool_calls) == 1 + assert isinstance(tool_call := tool_calls[0], ChatCompletionMessageToolCall) + assert tool_call.function.name == "get_weather" + assert isinstance(tool_call_arguments := tool_call.function.arguments, str) + assert json.loads(tool_call_arguments) == {"location": "Paris"} + + spans = in_memory_span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.name == "OpenAIServerModel.__call__" + assert span.status.is_ok + attributes = dict(span.attributes or {}) + assert attributes.pop(OPENINFERENCE_SPAN_KIND) == LLM + assert attributes.pop(INPUT_MIME_TYPE) == JSON + assert isinstance(input_value := attributes.pop(INPUT_VALUE), str) + input_data = json.loads(input_value) + assert "messages" in input_data + assert attributes.pop(OUTPUT_MIME_TYPE) == JSON + assert isinstance(output_value := attributes.pop(OUTPUT_VALUE), str) + assert isinstance(json.loads(output_value), dict) + assert attributes.pop(LLM_MODEL_NAME) == "gpt-4o" + assert isinstance(inv_params := attributes.pop(LLM_INVOCATION_PARAMETERS), str) + assert json.loads(inv_params) == {} + assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "user" + assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}") == input_message_content + assert isinstance( + tool_json_schema := attributes.pop(f"{LLM_TOOLS}.0.{TOOL_JSON_SCHEMA}"), str + ) + assert json.loads(tool_json_schema) == { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + }, + }, + "required": ["location"], + }, + }, + } + assert isinstance(attributes.pop(LLM_TOKEN_COUNT_PROMPT), int) + assert isinstance(attributes.pop(LLM_TOKEN_COUNT_COMPLETION), int) + assert isinstance(attributes.pop(LLM_TOKEN_COUNT_TOTAL), int) + assert attributes.pop(f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "assistant" + assert ( + attributes.pop(f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.0.{TOOL_CALL_ID}") + == tool_call.id + ) + assert ( + attributes.pop( + f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.0.{TOOL_CALL_FUNCTION_NAME}" + ) + == "get_weather" + ) + assert isinstance( + tool_call_arguments_json := attributes.pop( + f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.0.{TOOL_CALL_FUNCTION_ARGUMENTS_JSON}" + ), + str, + ) + assert json.loads(tool_call_arguments_json) == {"location": "Paris"} + assert not attributes + + +# message attributes +MESSAGE_CONTENT = MessageAttributes.MESSAGE_CONTENT +MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON = MessageAttributes.MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON +MESSAGE_FUNCTION_CALL_NAME = MessageAttributes.MESSAGE_FUNCTION_CALL_NAME +MESSAGE_NAME = MessageAttributes.MESSAGE_NAME +MESSAGE_ROLE = MessageAttributes.MESSAGE_ROLE +MESSAGE_TOOL_CALLS = MessageAttributes.MESSAGE_TOOL_CALLS + +# mime types +JSON = OpenInferenceMimeTypeValues.JSON.value +TEXT = OpenInferenceMimeTypeValues.TEXT.value + +# span kinds +CHAIN = OpenInferenceSpanKindValues.CHAIN.value +LLM = OpenInferenceSpanKindValues.LLM.value +TOOL = OpenInferenceSpanKindValues.TOOL.value + +# span attributes +INPUT_MIME_TYPE = SpanAttributes.INPUT_MIME_TYPE +INPUT_VALUE = SpanAttributes.INPUT_VALUE +LLM_INPUT_MESSAGES = SpanAttributes.LLM_INPUT_MESSAGES +LLM_INVOCATION_PARAMETERS = SpanAttributes.LLM_INVOCATION_PARAMETERS +LLM_MODEL_NAME = SpanAttributes.LLM_MODEL_NAME +LLM_OUTPUT_MESSAGES = SpanAttributes.LLM_OUTPUT_MESSAGES +LLM_PROMPTS = SpanAttributes.LLM_PROMPTS +LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION +LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT +LLM_TOKEN_COUNT_TOTAL = SpanAttributes.LLM_TOKEN_COUNT_TOTAL +LLM_TOOLS = SpanAttributes.LLM_TOOLS +OPENINFERENCE_SPAN_KIND = SpanAttributes.OPENINFERENCE_SPAN_KIND +OUTPUT_MIME_TYPE = SpanAttributes.OUTPUT_MIME_TYPE +OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE + +# tool attributes +TOOL_JSON_SCHEMA = ToolAttributes.TOOL_JSON_SCHEMA + +# tool call attributes +TOOL_CALL_FUNCTION_ARGUMENTS_JSON = ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON +TOOL_CALL_FUNCTION_NAME = ToolCallAttributes.TOOL_CALL_FUNCTION_NAME +TOOL_CALL_ID = ToolCallAttributes.TOOL_CALL_ID