From 7eb78e836bf093e23131d16d6a19391b6482b235 Mon Sep 17 00:00:00 2001 From: Carson Date: Tue, 24 Jun 2025 12:15:35 -0500 Subject: [PATCH 1/3] wip support ContentToolResultImage/ContentToolResultResource where we can --- chatlas/__init__.py | 3 ++- chatlas/_anthropic.py | 24 ++++++++++++++++++++++-- chatlas/_content.py | 6 ++++++ chatlas/_google.py | 9 +++++++++ chatlas/_openai.py | 8 ++++++++ tests/test_provider_anthropic.py | 18 +++++------------- 6 files changed, 52 insertions(+), 16 deletions(-) diff --git a/chatlas/__init__.py b/chatlas/__init__.py index 54811dd8..99381d5e 100644 --- a/chatlas/__init__.py +++ b/chatlas/__init__.py @@ -2,7 +2,7 @@ from ._anthropic import ChatAnthropic, ChatBedrockAnthropic from ._auto import ChatAuto from ._chat import Chat -from ._content import ContentToolRequest, ContentToolResult +from ._content import ContentToolRequest, ContentToolResult, ContentToolResultImage from ._content_image import content_image_file, content_image_plot, content_image_url from ._content_pdf import content_pdf_file, content_pdf_url from ._databricks import ChatDatabricks @@ -46,6 +46,7 @@ "content_pdf_url", "ContentToolRequest", "ContentToolResult", + "ContentToolResultImage", "interpolate", "interpolate_file", "Provider", diff --git a/chatlas/_anthropic.py b/chatlas/_anthropic.py index 95a672c5..01a19919 100644 --- a/chatlas/_anthropic.py +++ b/chatlas/_anthropic.py @@ -17,6 +17,8 @@ ContentText, ContentToolRequest, ContentToolResult, + ContentToolResultImage, + ContentToolResultResource, ) from ._logging import log_model_default from ._provider import Provider @@ -496,8 +498,26 @@ def _as_content_block(content: Content) -> "ContentBlockParam": "tool_use_id": content.id, "is_error": content.error is not None, } - # Anthropic supports non-text contents like ImageBlockParam - res["content"] = content.get_model_value() # type: ignore + + if isinstance(content, ContentToolResultImage): + res["content"] = [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": content.mime_type, + "data": content.value, + }, + } + ] + elif isinstance(content, ContentToolResultResource): + raise NotImplementedError( + "ContentToolResultResource is not currently supported by Anthropic." + ) + else: + # Anthropic supports non-text contents like ImageBlockParam + res["content"] = content.get_model_value() # type: ignore + return res raise ValueError(f"Unknown content type: {type(content)}") diff --git a/chatlas/_content.py b/chatlas/_content.py index 24a2c967..ecc2cf62 100644 --- a/chatlas/_content.py +++ b/chatlas/_content.py @@ -455,6 +455,12 @@ def __str__(self): def _repr_markdown_(self): return f"![](data:{self.mime_type};base64,{self.value})" + def get_model_value(self): + raise NotImplementedError( + "The value actually sent to the model depends on the model " + "provider used (e.g., OpenAI, Anthropic, etc.). " + ) + class ContentToolResultResource(ContentToolResult): """ diff --git a/chatlas/_google.py b/chatlas/_google.py index f9f5b35d..926e04d9 100644 --- a/chatlas/_google.py +++ b/chatlas/_google.py @@ -16,6 +16,8 @@ ContentText, ContentToolRequest, ContentToolResult, + ContentToolResultImage, + ContentToolResultResource, ) from ._logging import log_model_default from ._merge import merge_dicts @@ -430,6 +432,13 @@ def _as_part_type(self, content: Content) -> "Part": ) ) elif isinstance(content, ContentToolResult): + if isinstance( + content.value, (ContentToolResultImage, ContentToolResultResource) + ): + raise NotImplementedError( + "Tool results with images or resources aren't supported by Google (Gemini). " + ) + if content.error: resp = {"error": content.error} else: diff --git a/chatlas/_openai.py b/chatlas/_openai.py index f76bc480..831138d9 100644 --- a/chatlas/_openai.py +++ b/chatlas/_openai.py @@ -17,6 +17,8 @@ ContentText, ContentToolRequest, ContentToolResult, + ContentToolResultImage, + ContentToolResultResource, ) from ._logging import log_model_default from ._merge import merge_dicts @@ -490,6 +492,12 @@ def _as_message_param(turns: list[Turn]) -> list["ChatCompletionMessageParam"]: } ) elif isinstance(x, ContentToolResult): + if isinstance( + x, (ContentToolResultImage, ContentToolResultResource) + ): + raise NotImplementedError( + "OpenAI does not support tool results with images or resources." + ) tool_results.append( ChatCompletionToolMessageParam( # Currently, OpenAI only allows for text content in tool results diff --git a/tests/test_provider_anthropic.py b/tests/test_provider_anthropic.py index 1b982a18..e022c8af 100644 --- a/tests/test_provider_anthropic.py +++ b/tests/test_provider_anthropic.py @@ -2,8 +2,7 @@ import httpx import pytest - -from chatlas import ChatAnthropic, ContentToolResult +from chatlas import ChatAnthropic, ContentToolResultImage from .conftest import ( assert_data_extraction, @@ -108,17 +107,10 @@ def get_picture(): # Local copy of https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png with open(test_images_dir / "dice.png", "rb") as image: bytez = image.read() - res = [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": base64.b64encode(bytez).decode("utf-8"), - }, - } - ] - return ContentToolResult(value=res, model_format="as_is") + return ContentToolResultImage( + value=base64.b64encode(bytez).decode("utf-8"), + mime_type="image/png", + ) chat = ChatAnthropic() chat.register_tool(get_picture) From f3490500e90d00f594d60a3d406ac6b946d8be91 Mon Sep 17 00:00:00 2001 From: Carson Sievert Date: Wed, 2 Jul 2025 15:53:57 -0500 Subject: [PATCH 2/3] Update chatlas/_google.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- chatlas/_google.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatlas/_google.py b/chatlas/_google.py index a182e509..f5b54bc1 100644 --- a/chatlas/_google.py +++ b/chatlas/_google.py @@ -427,7 +427,7 @@ def _as_part_type(self, content: Content) -> "Part": ) elif isinstance(content, ContentToolResult): if isinstance( - content.value, (ContentToolResultImage, ContentToolResultResource) + content, (ContentToolResultImage, ContentToolResultResource) ): raise NotImplementedError( "Tool results with images or resources aren't supported by Google (Gemini). " From e0c46c1857511d919fcdae3a22015fc1598deb70 Mon Sep 17 00:00:00 2001 From: Carson Date: Wed, 2 Jul 2025 16:19:05 -0500 Subject: [PATCH 3/3] Simplify --- chatlas/_content.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/chatlas/_content.py b/chatlas/_content.py index d57576f7..11fddc72 100644 --- a/chatlas/_content.py +++ b/chatlas/_content.py @@ -462,12 +462,6 @@ def __str__(self): def _repr_markdown_(self): return f"![](data:{self.mime_type};base64,{self.value})" - def get_model_value(self): - raise NotImplementedError( - "The value actually sent to the model depends on the model " - "provider used (e.g., OpenAI, Anthropic, etc.). " - ) - class ContentToolResultResource(ContentToolResult): """