fix: refactor messages to use BaseMessage.text() (#683)

Author: btuan

src/rai_bench/rai_bench/manipulation_o3de/benchmark.py

@@ -286,7 +286,7 @@ def run_next(self, agent: CompiledStateGraph, experiment_id: uuid.UUID) -> None:
 
                         for msg in new_messages:
                             if isinstance(msg, HumanMultimodalMessage):
-                                last_msg = msg.text
+                                last_msg = msg.text()
                             elif isinstance(msg, BaseMessage):
                                 if isinstance(msg.content, list):
                                     if len(msg.content) == 1:

src/rai_core/rai/communication/hri_connector.py

@@ -103,12 +103,11 @@ def from_langchain(
         seq_no: int = 0,
         seq_end: bool = False,
     ) -> "HRIMessage":
+        text = message.text()
         if isinstance(message, RAIMultimodalMessage):
-            text = message.text
             images = message.images
             audios = message.audios
         else:
-            text = str(message.content)
             images = None
             audios = None
         if message.type not in ["ai", "human"]:

src/rai_core/rai/messages/multimodal.py

@@ -63,10 +63,6 @@ def __init__(
             _content.extend(_image_content)
         self.content = _content
 
-    @property
-    def text(self) -> str:
-        return self.content[0]["text"]
-
 
 class HumanMultimodalMessage(HumanMessage, MultimodalMessage):
     def __repr_args__(self) -> Any:

tests/agents/langchain/test_langchain_agent.py

@@ -18,9 +18,13 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+from langchain_core.callbacks import BaseCallbackHandler
+from langchain_core.language_models.fake_chat_models import ParrotFakeChatModel
+from langchain_core.runnables import RunnableConfig
 from rai.agents.langchain import invoke_llm_with_tracing
 from rai.agents.langchain.agent import LangChainAgent, newMessageBehaviorType
 from rai.initialization import get_tracing_callbacks
+from rai.messages import HumanMultimodalMessage
 
 
 @pytest.mark.parametrize(
@@ -150,3 +154,12 @@ def test_invoke_llm_with_existing_config(self):
             assert "callbacks" in call_args[1]["config"]
             assert "existing_callback" in call_args[1]["config"]["callbacks"]
             assert "tracing_callback" in call_args[1]["config"]["callbacks"]
+
+    def test_invoke_llm_with_callback_integration(self):
+        """Test that invoke_llm_with_tracing works with a callback handler."""
+        llm = ParrotFakeChatModel()
+        human_msg = HumanMultimodalMessage(content="human")
+        response = llm.invoke(
+            [human_msg], config=RunnableConfig(callbacks=[BaseCallbackHandler()])
+        )
+        assert response.content == [{"type": "text", "text": "human"}]

tests/messages/test_multimodal_message.py

@@ -0,0 +1,37 @@
+# Copyright (C) 2025 Robotec.AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from rai.messages import HumanMultimodalMessage
+
+
+class TestMultimodalMessage:
+    """Test the MultimodalMessage class and expected behaviors."""
+
+    def test_human_multimodal_message_text_simple(self):
+        """Test text() method with simple text content."""
+        msg = HumanMultimodalMessage(content="Hello world")
+        assert msg.text() == "Hello world"
+        assert isinstance(msg.text(), str)
+
+    def test_human_multimodal_message_text_with_images(self):
+        """Test text() method with text and images."""
+        # Use a small valid base64 image (1x1 pixel PNG)
+        valid_base64_image = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+        msg = HumanMultimodalMessage(
+            content="Look at this image", images=[valid_base64_image]
+        )
+        assert msg.text() == "Look at this image"
+        # Should only return text type blocks, not image content
+        assert valid_base64_image not in msg.text()