feat: refine multimodal agent architecture

1. Support model providers
2. Support `volcengine` provider
3. Better interface design.

Author: ulivz

.editorconfig

@@ -6,4 +6,5 @@ indent_style = space
 indent_size = 2
 end_of_line = lf
 insert_final_newline = true
-trim_trailing_whitespace = true
+trim_trailing_whitespace = true
+max_line_length= 100

packages/multimodal/agent/examples/weather.ts renamed to packages/multimodal/agent/examples/agent/basic.ts

@@ -2,20 +2,8 @@
  * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
  * SPDX-License-Identifier: Apache-2.0
  */
-import {
-  Agent,
-  Tool,
-  Model,
-  OpenAI,
-  OpenAIToolCallProvider,
-  InstructionToolCallProvider,
-} from '../src';
-import { z } from 'zod';
-import { getModel } from './model';
-
-const model = getModel('qwen3:1.7b');
-// Create tool call provider explicitly
-const toolCallProvider = new OpenAIToolCallProvider();
+import { Agent, Tool, z } from '../../src';
+import { TEST_MODEL_PROVIDERS } from './model';
 
 const locationTool = new Tool({
   id: 'getCurrentLocation',
@@ -46,7 +34,6 @@ const weatherTool = new Tool({
 });
 
 const agent = new Agent({
-  model,
   name: 'Agent TARS',
   tools: [locationTool, weatherTool],
   instructions: `
@@ -55,8 +42,10 @@ const agent = new Agent({
   1. DO NOT make any fake informations
   2. "finish_reason" should always be "tool_calls"
   `,
-  toolCallProvider, // Pass the provider explicitly
   maxIterations: 10,
+  model: {
+    providers: TEST_MODEL_PROVIDERS,
+  },
 });
 
 async function main() {

packages/multimodal/agent/examples/agent/config.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { ModelProvider } from '../../src';
+
+export const TEST_MODEL_PROVIDERS: ModelProvider[] = [
+  {
+    name: 'volcengine',
+    apiKey: process.env.MM_TEST_API_KEY,
+    models: [
+      {
+        id: 'ep-20250510145437-5sxhs',
+        label: 'doubao-1.5-thinking-vision-pro',
+      },
+    ],
+  },
+  {
+    name: 'lm-studio',
+    models: [
+      {
+        id: 'qwen2.5-coder-3b-instruct',
+      },
+      {
+        id: 'qwen2.5-7b-instruct-1m',
+      },
+    ],
+  },
+  {
+    name: 'ollama',
+    models: [
+      {
+        id: 'qwen3:1.7b',
+      },
+    ],
+  },
+  {
+    name: 'openai',
+    baseURL: process.env.OPENAI_API_BASE_URL,
+    models: [
+      {
+        id: 'gpt-4o-2024-11-20',
+      },
+    ],
+  },
+  {
+    name: 'azure-openai',
+    baseURL: process.env.AWS_CLAUDE_API_BASE_URL,
+    models: [
+      {
+        id: 'aws_sdk_claude37_sonnet',
+      },
+    ],
+  },
+];

packages/multimodal/agent/examples/result/REVIEW_RESULT.md renamed to packages/multimodal/agent/examples/agent/filesystem/REVIEW_RESULT.md

@@ -3,14 +3,11 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import { join } from 'path';
-import { MCPAgent } from '../src';
-import { getModel } from './model';
-
-const model = getModel('gpt-4o-2024-11-20');
+import { MCPAgent } from '../../src';
+import { TEST_MODEL_PROVIDERS } from './config';
 
 async function main() {
   const agent = new MCPAgent({
-    model,
     instructions:
       'You are GitHub Reviewer, a specialized assistant designed to help with code review tasks. ' +
       'You excel at analyzing pull requests, identifying potential bugs, security issues, and suggesting code improvements. ' +
@@ -38,18 +35,17 @@ async function main() {
       'Take screenshots of specific code sections when they help illustrate complex issues or changes. ' +
       'Your reviews should be thorough yet easy to understand, with code examples making your feedback concrete and actionable. ' +
       "Remember that including actual code snippets makes your reports more vivid and helps the developer understand exactly what you're referring to.",
+    model: {
+      providers: TEST_MODEL_PROVIDERS,
+    },
     mcpServers: {
       playwright: {
         command: 'npx',
         args: ['@playwright/mcp@latest'],
       },
       filesystem: {
         command: 'npx',
-        args: [
-          '-y',
-          '@modelcontextprotocol/server-filesystem',
-          join(__dirname, 'result'),
-        ],
+        args: ['-y', '@modelcontextprotocol/server-filesystem', join(__dirname, 'filesystem')],
       },
     },
   });
@@ -63,9 +59,7 @@ async function main() {
     }
 
     // GitHub PR review related queries
-    const queries = [
-      'Review https://github.com/bytedance/UI-TARS-desktop/pull/534',
-    ];
+    const queries = ['Review https://github.com/bytedance/UI-TARS-desktop/pull/534'];
 
     for (const query of queries) {
       console.log('\n==================================================');

packages/multimodal/agent/examples/github-reviewer-agent.ts renamed to packages/multimodal/agent/examples/agent/github-reviewer-agent.ts

@@ -2,14 +2,11 @@
  * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
  * SPDX-License-Identifier: Apache-2.0
  */
-import { MCPAgent } from '../src';
-import { getModel } from './model';
-
-const model = getModel('qwen3:1.7b');
+import { MCPAgent } from '../../src';
+import { TEST_MODEL_PROVIDERS } from './config';
 
 async function main() {
   const agent = new MCPAgent({
-    model,
     instructions:
       'You are Agent TARS, a helpful assistant that can use the tools available to help users with their questions.',
     mcpServers: {
@@ -18,6 +15,9 @@ async function main() {
         args: ['@playwright/mcp@latest'],
       },
     },
+    model: {
+      providers: TEST_MODEL_PROVIDERS,
+    },
   });
 
   try {

packages/multimodal/agent/examples/mcp-agent.ts renamed to packages/multimodal/agent/examples/agent/mcp-agent.ts

@@ -64,7 +64,7 @@ <h3>Output:</h3>
         z,
         Model,
         AzureOpenAI,
-        InstructionToolCallProvider
+        PromptToolCallEngine
       } from '../dist/index.mjs';
 
       const originalConsoleLog = console.log;
@@ -138,14 +138,14 @@ <h3>Output:</h3>
       });
 
       // Create an instruction-based tool call provider for Claude models
-      const toolCallProvider = new InstructionToolCallProvider();
+      const ToolCallEngine = new PromptToolCallEngine();
 
       const agent = new Agent({
         model,
         name: 'Agent TARS',
         instructions: `You are Agent TARS, a general AI agent that is good at understanding user needs and helping people solve problems.`,
         tools: [locationTool, weatherTool],
-        toolCallProvider, // Pass the provider explicitly
+        ToolCallEngine, // Pass the provider explicitly
         maxIterations: 3,
       });
 

packages/multimodal/agent/examples/browser.html

@@ -0,0 +1,30 @@
+{
+    "choices": [
+        {
+            "finish_reason": "stop",
+            "index": 0,
+            "logprobs": null,
+            "message": {
+                "content": "你好呀～有什么我可以帮忙的地方吗？",
+                "reasoning_content": "用户现在打招呼说“你好”，需要友好回应。所以回复“你好呀～有什么我可以帮忙的地方吗？”这样亲切自然。",
+                "role": "assistant"
+            }
+        }
+    ],
+    "created": 1746900426,
+    "id": "021746900424215384cfaa31ec6325bb0f7a8325e5c6fc5bc9319",
+    "model": "doubao-1-5-thinking-vision-pro-250428",
+    "service_tier": "default",
+    "object": "chat.completion",
+    "usage": {
+        "completion_tokens": 48,
+        "prompt_tokens": 60,
+        "total_tokens": 108,
+        "prompt_tokens_details": {
+            "cached_tokens": 0
+        },
+        "completion_tokens_details": {
+            "reasoning_tokens": 35
+        }
+    }
+}

packages/multimodal/agent/examples/llm/1.json

@@ -0,0 +1,66 @@
+/* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */
+/* eslint-disable no-unsafe-optional-chaining */
+import { OpenAI } from 'openai';
+import { TokenJS } from 'token.js';
+
+export const tokenjs = new TokenJS({
+  apiKey: process.env.MM_TEST_API_KEY,
+  baseURL: process.env.MM_TEST_BASE_URL,
+});
+
+tokenjs.extendModelList('openai', 'ep-20250510145437-5sxhs', {
+  streaming: true,
+  json: true,
+  toolCalls: true,
+  images: true,
+});
+
+// FIXME: remove as, we need to fix the type issue of token.js
+const client = tokenjs as unknown as OpenAI;
+
+async function main() {
+  console.time('> TTFT');
+
+  // @ts-expect-error
+  const completion = await client.chat.completions.create({
+    provider: 'openai',
+    stream: true as const,
+    model: process.env.MM_TEST_MODEL!,
+    messages: [
+      {
+        role: 'user',
+        content: 'Hello!',
+      },
+    ],
+  });
+
+  console.timeEnd('> TTFT');
+
+  console.time('> Streaming Duration');
+
+  let reachReasoning = false;
+  let reachContent = false;
+  for await (const part of completion) {
+    // @ts-expect-error
+    const { content, reasoning_content } = part.choices[0]?.delta!;
+    if (reasoning_content) {
+      if (!reachReasoning) {
+        reachReasoning = true;
+        console.log('> Reasoning Content:');
+      }
+      process.stdout.write(reasoning_content);
+    }
+    if (content) {
+      if (!reachContent) {
+        reachContent = true;
+        console.log('\n> Content:');
+      }
+      process.stdout.write(content);
+    }
+  }
+
+  console.log('');
+  console.timeEnd('> Streaming Duration');
+}
+
+main();

packages/multimodal/agent/examples/llm/default.ts

@@ -0,0 +1,83 @@
+/* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */
+/* eslint-disable no-unsafe-optional-chaining */
+import { OpenAI } from 'openai';
+import { TokenJS } from 'token.js';
+
+export const tokenjs = new TokenJS({
+  apiKey: process.env.MM_TEST_API_KEY,
+  baseURL: process.env.MM_TEST_BASE_URL,
+});
+
+tokenjs.extendModelList('openai', 'ep-20250510145437-5sxhs', {
+  streaming: true,
+  json: true,
+  toolCalls: true,
+  images: true,
+});
+
+// FIXME: remove as, we need to fix the type issue of token.js
+const client = tokenjs as unknown as OpenAI;
+
+async function main() {
+  console.time('> TTFT');
+  console.log(process.env.MM_TEST_MODEL!);
+
+  const completion = await client.chat.completions.create({
+    // @ts-expect-error
+    provider: 'openai',
+    model: process.env.MM_TEST_MODEL!,
+    messages: [
+      {
+        role: 'system',
+        content:
+          '\n  You are a tool call agent, you MUST SELECT a TOOL to handle user\'s request.\n  \n  1. DO NOT make any fake informations\n  2. "finish_reason" should always be "tool_calls"\n  \n\nCurrent time: 5/11/2025, 3:41:25 PM',
+      },
+      {
+        role: 'user',
+        content: "How's the weather today?",
+      },
+    ],
+    tools: [
+      {
+        type: 'function',
+        function: {
+          name: 'getCurrentLocation',
+          description: "Get user's current location",
+          parameters: {
+            type: 'object',
+            properties: {},
+          },
+        },
+      },
+      {
+        type: 'function',
+        function: {
+          name: 'getWeather',
+          description: 'Get weather information for a specified location',
+          parameters: {
+            type: 'object',
+            properties: {
+              location: {
+                type: 'string',
+                description: 'Location name, such as city name',
+              },
+            },
+            required: ['location'],
+          },
+        },
+      },
+    ],
+    temperature: 0.7,
+  });
+
+  console.timeEnd('> TTFT');
+
+  console.time('> Request Duration');
+
+  console.log(JSON.stringify(completion));
+
+  console.log('');
+  console.timeEnd('> Request Duration');
+}
+
+main();