feat(mcp-browser): vision mode add browser_vision_click and fullPage (#637)

Author: ycjcl868

.changeset/pre.json

@@ -27,11 +27,11 @@
     "@multimodal/agent": "0.1.1-beta.3",
     "@multimodal/agent-interface": "0.1.1-beta.3",
     "@multimodal/agent-snapshot": "0.1.1-beta.3",
-    "@agent-tars/core": "0.0.1-beta.4",
-    "@agent-tars/cli": "0.0.1-beta.4",
-    "@agent-tars/server": "0.0.1-beta.4",
-    "@agent-tars/test-workspace": "0.0.1-beta.4",
-    "@agent-tars/web-ui": "0.0.1-beta.4",
+    "@agent-tars/core": "0.0.1-beta.5",
+    "@agent-tars/cli": "0.0.1-beta.5",
+    "@agent-tars/server": "0.0.1-beta.5",
+    "@agent-tars/test-workspace": "0.0.1-beta.5",
+    "@agent-tars/web-ui": "0.0.1-beta.5",
     "@multimodal/codeact-agent": "0.1.1-beta.3",
     "@multimodal/config-loader": "0.1.1-beta.3",
     "@multimodal/deep-research-agent": "0.1.1-beta.3",
@@ -56,6 +56,7 @@
     "metal-flies-ring",
     "serious-spiders-wash",
     "sour-rocks-sip",
-    "spicy-kids-allow"
+    "spicy-kids-allow",
+    "twenty-snakes-refuse"
   ]
 }

.changeset/twenty-snakes-refuse.md

@@ -0,0 +1,5 @@
+---
+'@agent-infra/mcp-server-browser': patch
+---
+
+feat: support vision mode

packages/agent-infra/mcp-client/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @agent-infra/mcp-client
 
+## 1.1.6-beta.3
+
+### Patch Changes
+
+- @agent-infra/mcp-shared@1.1.6-beta.3
+
 ## 1.1.6-beta.2
 
 ### Patch Changes

packages/agent-infra/mcp-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-infra/mcp-client",
-  "version": "1.1.6-beta.2",
+  "version": "1.1.6-beta.3",
   "description": "An MCP Client to run servers for Electron apps, support same-process approaching",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",

packages/agent-infra/mcp-servers/browser/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @agent-infra/mcp-server-browser
 
+## 1.1.6-beta.3
+
+### Patch Changes
+
+- feat: support vision mode
+
 ## 1.1.6-beta.2
 
 ### Patch Changes
@@ -10,6 +16,10 @@
 
 ## 1.1.6-beta.0
 
+### Patch Changes
+
+- feat: add fullPage
+
 ## 1.1.5
 
 ### Patch Changes

packages/agent-infra/mcp-servers/browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-infra/mcp-server-browser",
-  "version": "1.1.6-beta.2",
+  "version": "1.1.6-beta.3",
   "description": "MCP server for browser use access",
   "license": "MIT",
   "homepage": "https://github.com/bytedance/UI-TARS-desktop",
@@ -23,10 +23,12 @@
   "scripts": {
     "build": "shx rm -rf dist && rslib build && shx chmod +x dist/*.{js,cjs}",
     "dev": "npx -y @modelcontextprotocol/inspector tsx src/index.ts",
+    "dev:vision": "npx -y @modelcontextprotocol/inspector tsx src/index.ts --vision",
     "prepare": "npm run build",
     "watch": "rslib build --watch"
   },
   "dependencies": {
+    "@ui-tars/action-parser": "workspace:*",
     "mcp-http-server": "workspace:*",
     "@modelcontextprotocol/sdk": "^1.11.2"
   },

packages/agent-infra/mcp-servers/browser/src/index.ts

@@ -94,10 +94,10 @@ program
     '--viewport-size <size>',
     'specify browser viewport size in pixels, for example "1280, 720"',
   )
-  // .option(
-  //   '--vision',
-  //   'Run server that uses screenshots (Aria snapshots are used by default)',
-  // )
+  .option(
+    '--vision',
+    'Run server that uses screenshots (Aria snapshots are used by default)',
+  )
   .action(async (options) => {
     try {
       console.log('[mcp-server-browser] options', options);
@@ -110,6 +110,7 @@ program
               cdpEndpoint: options.cdpEndpoint,
             },
           }),
+          vision: options.vision,
           launchOptions: {
             headless: options.headless,
             executablePath: options.executablePath,

packages/agent-infra/mcp-servers/browser/src/server.ts

@@ -43,6 +43,8 @@ import merge from 'lodash.merge';
 import { parseProxyUrl } from './utils.js';
 import { ElementHandle, KeyInput } from 'puppeteer-core';
 import { keyInputValues } from './constants.js';
+import { getVisionTools, visionToolsMap } from './tools/vision.js';
+import { ToolContext } from './typings.js';
 
 const consoleLogs: string[] = [];
 
@@ -72,6 +74,11 @@ interface GlobalConfig {
    * @defaultValue true
    */
   enableAdBlocker?: boolean;
+  /**
+   * Whether to add vision tools
+   * @defaultValue false
+   */
+  vision?: boolean;
 }
 
 // Global state
@@ -81,6 +88,7 @@ let globalConfig: GlobalConfig = {
   },
   contextOptions: {},
   enableAdBlocker: true,
+  vision: false,
 };
 
 let globalBrowser: LocalBrowser['browser'] | undefined;
@@ -273,6 +281,10 @@ export const toolsMap = {
           .number()
           .optional()
           .describe('Height in pixels (default: viewport height)'),
+        fullPage: z
+          .boolean()
+          .optional()
+          .describe('Full page screenshot (default: false)'),
         highlight: z
           .boolean()
           .optional()
@@ -285,7 +297,8 @@ export const toolsMap = {
   },
   browser_click: {
     name: 'browser_click',
-    description: 'Click an element on the page',
+    description:
+      'Click an element on the page, before using the tool, use `browser_get_clickable_elements` to get the index of the element, but not call `browser_get_clickable_elements` multiple times',
     inputSchema: z.object({
       // selector: z
       //   .string()
@@ -432,9 +445,11 @@ export const toolsMap = {
   },
 };
 
-type ToolNames = keyof typeof toolsMap;
+type ToolNames = keyof typeof toolsMap | keyof typeof visionToolsMap;
 type ToolInputMap = {
-  [K in ToolNames]: (typeof toolsMap)[K] extends { inputSchema: infer S }
+  [K in ToolNames]: (typeof toolsMap & typeof visionToolsMap)[K] extends {
+    inputSchema: infer S;
+  }
     ? S extends z.ZodType<any, any, any>
       ? z.infer<S>
       : unknown
@@ -498,9 +513,13 @@ const handleToolCall = async ({
     };
   }
 
+  const ctx: ToolContext = { page, browser, logger };
+
   const handlers: {
     [K in ToolNames]: (args: ToolInputMap[K]) => Promise<CallToolResult>;
   } = {
+    // vision tools
+    ...getVisionTools(ctx),
     browser_go_back: async (args) => {
       try {
         await Promise.all([waitForPageAndFramesLoad(page), page.goBack()]);
@@ -658,7 +677,10 @@ const handleToolCall = async ({
       // if screenshot is still undefined, take a screenshot of the whole page
       screenshot =
         screenshot ||
-        (await page.screenshot({ encoding: 'base64', fullPage: false }));
+        (await page.screenshot({
+          encoding: 'base64',
+          fullPage: args.fullPage ?? false,
+        }));
 
       // if screenshot is still undefined, return an error
       if (!screenshot) {
@@ -702,6 +724,7 @@ const handleToolCall = async ({
 
       try {
         const { clickableElements } = (await buildDomTree(page)) || {};
+        await removeHighlights(page);
         if (clickableElements) {
           return {
             content: [
@@ -1295,8 +1318,13 @@ function createServer(config: GlobalConfig = {}): McpServer {
     version: process.env.VERSION || '0.0.1',
   });
 
+  const mergedToolsMap = {
+    ...toolsMap,
+    ...(config.vision ? visionToolsMap : {}),
+  };
+
   // === Tools ===
-  Object.entries(toolsMap).forEach(([name, tool]) => {
+  Object.entries(mergedToolsMap).forEach(([name, tool]) => {
     // @ts-ignore
     if (tool?.inputSchema) {
       server.tool(

packages/agent-infra/mcp-servers/browser/src/tools/vision.ts

@@ -0,0 +1,115 @@
+import {
+  CallToolResult,
+  TextContent,
+} from '@modelcontextprotocol/sdk/types.js';
+import { z } from 'zod';
+import { ToolContext } from '../typings.js';
+
+type ToolNames = keyof typeof visionToolsMap;
+type ToolInputMap = {
+  [K in ToolNames]: (typeof visionToolsMap)[K] extends {
+    inputSchema: infer S;
+  }
+    ? S extends z.ZodType<any, any, any>
+      ? z.infer<S>
+      : unknown
+    : unknown;
+};
+
+export const visionToolsMap = {
+  browser_vision_screen_capture: {
+    description: 'Take a screenshot of the current page for vision mode',
+  },
+  browser_vision_screen_click: {
+    description:
+      'Click left mouse button on the page with vision and snapshot, before calling this tool, you should call `browser_vision_screen_capture` first only once, fallback to `browser_click` if failed',
+    inputSchema: z.object({
+      factors: z
+        .array(z.number())
+        .optional()
+        .describe('Vision Model scaling factors, [width_scale, height_scale]'),
+      x: z.number().describe('X coordinate'),
+      y: z.number().describe('Y coordinate'),
+    }),
+  },
+};
+
+export const getVisionTools = (ctx: ToolContext) => {
+  const { page, logger } = ctx;
+
+  const visionTools: {
+    [K in ToolNames]: (args: ToolInputMap[K]) => Promise<CallToolResult>;
+  } = {
+    browser_vision_screen_capture: async () => {
+      const viewport = page.viewport();
+
+      const screenshot = await page.screenshot({
+        type: 'jpeg' as const,
+        quality: 50,
+        fullPage: false,
+        omitBackground: false,
+        encoding: 'base64',
+      });
+
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Screenshot taken at ${viewport?.width}x${viewport?.height}`,
+          } as TextContent,
+          {
+            type: 'image' as const,
+            data: screenshot,
+            mimeType: 'image/jpeg',
+          },
+        ],
+      };
+    },
+    browser_vision_screen_click: async (args) => {
+      try {
+        let x = args.x;
+        let y = args.y;
+
+        if (args.factors) {
+          const { actionParser } = await import('@ui-tars/action-parser');
+
+          const viewport = page.viewport();
+          const { parsed } = actionParser({
+            prediction: `Action: click(start_box='(${args.x},${args.y})')`,
+            factor: args.factors as [number, number],
+            screenContext: {
+              width: viewport?.width ?? 0,
+              height: viewport?.height ?? 0,
+            },
+          });
+
+          const { start_coords } = parsed?.[0]?.action_inputs ?? {};
+          logger.info('[vision] start_coords', start_coords);
+
+          x = start_coords?.[0] ?? x;
+          y = start_coords?.[1] ?? y;
+        }
+
+        await page.mouse.move(x, y);
+        await page.mouse.down();
+        await page.mouse.up();
+
+        return {
+          content: [
+            {
+              type: 'text',
+              text: `Vision click at ${args.x}, ${args.y}`,
+            },
+          ],
+          isError: false,
+        };
+      } catch (error) {
+        return {
+          content: [{ type: 'text', text: 'Error clicking on the page' }],
+          isError: true,
+        };
+      }
+    },
+  };
+  return visionTools;
+};

packages/agent-infra/mcp-servers/browser/src/typings.ts

@@ -0,0 +1,8 @@
+import { Logger } from '@agent-infra/logger';
+import { Browser, Page } from 'puppeteer-core';
+
+export type ToolContext = {
+  page: Page;
+  browser: Browser;
+  logger: Logger;
+};

packages/agent-infra/mcp-servers/commands/CHANGELOG.md

@@ -1,5 +1,7 @@
 ## 0.4.0 2024-12-10 - add logging
 
+## 1.1.6-beta.3
+
 ## 1.1.6-beta.2
 
 ### Patch Changes

packages/agent-infra/mcp-servers/commands/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-infra/mcp-server-commands",
-  "version": "1.1.6-beta.2",
+  "version": "1.1.6-beta.3",
   "description": "An MCP server to run arbitrary commands",
   "type": "module",
   "main": "./dist/server.cjs",

packages/agent-infra/mcp-servers/filesystem/CHANGELOG.md

@@ -1,5 +1,7 @@
 # @agent-infra/mcp-server-filesystem
 
+## 1.1.6-beta.3
+
 ## 1.1.6-beta.2
 
 ### Patch Changes

packages/agent-infra/mcp-servers/filesystem/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-infra/mcp-server-filesystem",
-  "version": "1.1.6-beta.2",
+  "version": "1.1.6-beta.3",
   "description": "MCP server for filesystem access",
   "license": "MIT",
   "type": "module",

packages/agent-infra/mcp-servers/search/CHANGELOG.md

@@ -1,5 +1,7 @@
 # @agent-infra/mcp-server-search
 
+## 1.1.6-beta.3
+
 ## 1.1.6-beta.2
 
 ### Patch Changes