feat: UI-tars-1.5 (#502)

Co-authored-by: skychx <skychx@hotmail.com>
Co-authored-by: ZhaoHeh <zhaoqinghao@bytedance.com>
Co-authored-by: ULIVZ <chenhaoli@bytedance.com>
Co-authored-by: Charles <jinxin001@bytedance.com>

Author: 4 people

README.md

@@ -19,9 +19,9 @@ UI-TARS Desktop is a GUI Agent application based on [UI-TARS (Vision-Language Mo
 
 <p align="center">
         &nbsp&nbsp 📑 <a href="https://arxiv.org/abs/2501.12326">Paper</a> &nbsp&nbsp
-        | 🤗 <a href="https://huggingface.co/bytedance-research/UI-TARS-7B-DPO">Hugging Face Models</a>&nbsp&nbsp
+        | 🤗 <a href="https://huggingface.co/ByteDance-Seed/UI-TARS-1.5-7B">Hugging Face Models</a>&nbsp&nbsp
         | &nbsp&nbsp🫨 <a href="https://discord.gg/pTXwYVjfcs">Discord</a>&nbsp&nbsp
-        | &nbsp&nbsp🤖 <a href="https://www.modelscope.cn/models/bytedance-research/UI-TARS-7B-DPO">ModelScope</a>&nbsp&nbsp
+        | &nbsp&nbsp🤖 <a href="https://www.modelscope.cn/collections/UI-TARS-bccb56fa1ef640">ModelScope</a>&nbsp&nbsp
 <br>
 🖥️ Desktop Application &nbsp&nbsp
 | &nbsp&nbsp 👓 <a href="https://github.com/web-infra-dev/midscene">Midscene (use in browser)</a>
@@ -31,13 +31,13 @@ UI-TARS Desktop is a GUI Agent application based on [UI-TARS (Vision-Language Mo
 
 | Instruction  | Video |
 | :---:  | :---: |
-| Get the current weather in SF using the web browser      |    <video src="https://github.com/user-attachments/assets/5235418c-ac61-4895-831d-68c1c749fc87" height="300" />    |
-| Send a twitter with the content "hello world"   | <video src="https://github.com/user-attachments/assets/737ccc11-9124-4464-b4be-3514cbced85c" height="300" />        |
+| Please help me open the autosave feature of VS Code and delay AutoSave operations for 500 milliseconds in the VS Code setting.      |    <video src="https://github.com/user-attachments/assets/e0914ce9-ad33-494b-bdec-0c25c1b01a27" height="300" />    |
+| Could you help me check the latest open issue of the UI-TARS-Desktop project on GitHub?   | <video src="https://github.com/user-attachments/assets/3d159f54-d24a-4268-96c0-e149607e9199" height="300" />        |
 
 
 ## News
 
-- **\[2025-04-16\]** - We shared the latest progress of the **UI-TARS-1.5** model in our blog ([seed-tars.com/1.5](https://seed-tars.com/1.5)), which excels in playing games and performing GUI tasks, and we open-sourced the [UI-TARS-1.5-7B](https://huggingface.co/ByteDance-Seed/UI-TARS-1.5-7B), checkout the showcase here: https://seed-tars.com/showcase
+- **\[2025-04-17\]** - 🎉 We're thrilled to announce the release of new UI-TARS Desktop application v0.1.0, featuring a redesigned Agent UI. The application enhances the computer using experience, introduces new browser operation features, and supports [the advanced UI-TARS-1.5 model](https://seed-tars.com/1.5) for improved performance and precise control.
 - **\[2025-02-20\]** - 📦 Introduced [UI TARS SDK](./docs/sdk.md), is a powerful cross-platform toolkit for building GUI automation agents.
 - **\[2025-01-23\]** - 🚀 We updated the **[Cloud Deployment](./docs/deployment.md#cloud-deployment)** section in the 中文版: [GUI模型部署教程](https://bytedance.sg.larkoffice.com/docx/TCcudYwyIox5vyxiSDLlgIsTgWf#U94rdCxzBoJMLex38NPlHL21gNb) with new information related to the ModelScope platform. You can now use the ModelScope platform for deployment.
 
@@ -47,7 +47,7 @@ UI-TARS Desktop is a GUI Agent application based on [UI-TARS (Vision-Language Mo
 - 🤖 Natural language control powered by Vision-Language Model
 - 🖥️ Screenshot and visual recognition support
 - 🎯 Precise mouse and keyboard control
-- 💻 Cross-platform support (Windows/MacOS)
+- 💻 Cross-platform support (Windows/MacOS/Browser)
 - 🔄 Real-time feedback and status display
 - 🔐 Private and secure - fully local processing
 
@@ -57,7 +57,7 @@ See [Quick Start](./docs/quick-start.md).
 
 ## Deployment
 
-See [Deployment](./docs/deployment.md).
+See [Deployment](https://github.com/bytedance/UI-TARS/blob/main/README_deploy.md).
 
 ## Contributing
 

apps/agent-tars/tsconfig.base.json

@@ -3,6 +3,8 @@
     "baseUrl": ".",
     "moduleResolution": "bundler",
     "paths": {
+      "react": ["./src/renderer/node_modules/@types/react"],
+      "react-dom": ["./src/renderer/node_modules/@types/react-dom"],
       "@shared/*": ["./src/shared/*"],
       "@main/*": ["./src/main/*"],
       "@renderer/*": ["./src/renderer/src/*"],

apps/ui-tars/electron.vite.config.ts

@@ -10,6 +10,7 @@ import tsconfigPaths from 'vite-tsconfig-paths';
 
 import pkg from './package.json';
 import { getExternalPkgs } from './scripts/getExternalPkgs';
+import tailwindcss from '@tailwindcss/vite';
 
 export default defineConfig({
   main: {
@@ -66,7 +67,7 @@ export default defineConfig({
         },
       },
     },
-    plugins: [react(), tsconfigPaths()],
+    plugins: [react(), tsconfigPaths(), tailwindcss()],
     define: {
       APP_VERSION: JSON.stringify(pkg.version),
     },

apps/ui-tars/images/download-report.png

@@ -1,6 +1,6 @@
 {
   "name": "ui-tars-desktop",
-  "version": "0.0.9",
+  "version": "0.1.0-preview",
   "private": true,
   "main": "./dist/main/main.js",
   "packageManager": "pnpm@9.10.0",
@@ -69,65 +69,68 @@
     "sharp": "0.33.3"
   },
   "devDependencies": {
+    "@common/electron-build": "workspace:*",
     "@computer-use/mac-screen-capture-permissions": "^1.0.2",
     "@computer-use/nut-js": "^4.2.0",
-    "@ui-tars/operator-nut-js": "workspace:*",
-    "@electron-toolkit/utils": "^3.0.0",
-    "@common/electron-build": "workspace:*",
-    "@ui-tars/sdk": "workspace:*",
-    "@ui-tars/action-parser": "workspace:*",
-    "@ui-tars/shared": "workspace:*",
-    "@ui-tars/utio": "workspace:*",
-    "@ui-tars/electron-ipc": "workspace:*",
-    "dotenv": "^16.4.7",
-    "electron-debug": "^3.2.0",
-    "electron-devtools-installer": "^3.2.0",
-    "electron-log": "5.2.4",
-    "electron-squirrel-startup": "^1.0.1",
-    "electron-store": "^10.0.0",
-    "js-yaml": "^4.1.0",
-    "ms": "^2.1.3",
-    "electron-updater": "^6.3.9",
-    "zustand": "^5.0.0",
-    "@electron-forge/plugin-base": "^7.7.0",
-    "@electron-forge/shared-types": "^7.7.0",
     "@electron-forge/cli": "^7.7.0",
     "@electron-forge/maker-dmg": "^7.7.0",
     "@electron-forge/maker-pkg": "^7.7.0",
     "@electron-forge/maker-squirrel": "^7.7.0",
     "@electron-forge/maker-zip": "^7.7.0",
     "@electron-forge/plugin-auto-unpack-natives": "^7.7.0",
+    "@electron-forge/plugin-base": "^7.7.0",
     "@electron-forge/plugin-fuses": "^7.7.0",
     "@electron-forge/plugin-vite": "^7.7.0",
     "@electron-forge/publisher-github": "^7.7.0",
+    "@electron-forge/shared-types": "^7.7.0",
     "@electron-toolkit/eslint-config-prettier": "^2.0.0",
     "@electron-toolkit/eslint-config-ts": "^2.0.0",
     "@electron-toolkit/preload": "^3.0.1",
     "@electron-toolkit/tsconfig": "^1.0.1",
+    "@electron-toolkit/utils": "^3.0.0",
     "@electron/asar": "^3.2.18",
     "@electron/fuses": "^1.8.0",
     "@trivago/prettier-plugin-sort-imports": "^5.2.1",
     "@types/node": "^20.14.8",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",
+    "@ui-tars/action-parser": "workspace:*",
+    "@ui-tars/electron-ipc": "workspace:*",
+    "@ui-tars/operator-nut-js": "workspace:*",
+    "@ui-tars/operator-browser": "workspace:*",
+    "@ui-tars/sdk": "workspace:*",
+    "@ui-tars/shared": "workspace:*",
+    "@ui-tars/utio": "workspace:*",
     "@vitejs/plugin-react": "^4.3.4",
     "@vitest/coverage-istanbul": "^3.0.3",
     "cross-env": "^7.0.3",
+    "dotenv": "^16.4.7",
     "electron": "34.1.1",
+    "electron-debug": "^3.2.0",
+    "electron-devtools-installer": "^3.2.0",
+    "electron-log": "5.2.4",
     "electron-packager-languages": "0.5.0",
+    "electron-squirrel-startup": "^1.0.1",
+    "electron-store": "^10.0.0",
     "electron-vite": "^3.0.0",
     "eslint": "^8.57.0",
     "eslint-plugin-import": "^2.25.0",
     "eslint-plugin-react": "^7.34.3",
+    "js-yaml": "^4.1.0",
+    "ms": "^2.1.3",
     "prettier": "^3.3.2",
     "rimraf": "^6.0.1",
     "sass-embedded": "^1.83.1",
     "ts-node": "^10.9.2",
     "tsx": "^4.19.2",
     "typescript": "^5.7.2",
+    "update-electron-app": "^3.1.0",
     "vite": "^6.1.0",
     "vite-tsconfig-paths": "^5.1.4",
-    "vitest": "^3.0.8"
+    "vitest": "^3.0.8",
+    "zustand": "^5.0.0",
+    "@tailwindcss/vite": "4.1.3",
+    "tailwindcss": "4.1.3"
   },
   "engines": {
     "node": ">=20.x"

apps/ui-tars/images/mac_app.png

@@ -23,3 +23,65 @@ ${NutJSElectronOperator.MANUAL.ACTION_SPACES.join('\n')}
 
 ## User Instruction
 `;
+
+export const getSystemPromptV1_5 = (
+  language: 'zh' | 'en',
+  useCase: 'normal' | 'poki',
+) => `You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
+right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='ctrl c') # Split keys with a space and use lowercase. Also, do not use more than 3 keys in one hotkey action.
+type(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content.
+scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') # Show more information on the \`direction\` side.
+wait() # Sleep for 5s and take a screenshot to check for any changes.
+finished()
+call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
+
+
+## Note
+- Use ${language === 'zh' ? 'Chinese' : 'English'} in \`Thought\` part.
+- ${useCase === 'normal' ? 'Generate a well-defined and practical strategy in the `Thought` section, summarizing your next move and its objective.' : 'Compose a step-by-step approach in the `Thought` part, specifying your next action and its focus.'}
+
+## User Instruction
+`;
+
+export const getSystemPromptPoki = `
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
+right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='ctrl c') # Split keys with a space and use lowercase. Also, do not use more than 3 keys in one hotkey action.
+type(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content.
+scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') # Show more information on the \`direction\` side.
+wait() # Sleep for 5s and take a screenshot to check for any changes.
+finished()
+call_user() # Submit the task and call the user when the task is unsolvable, or when you need the user's help.
+
+
+## Note
+- Use Chinese in \`Thought\` part.
+- Compose a step-by-step approach in the \`Thought\` part, specifying your next action and its focus.
+
+## User Instruction
+`;

apps/ui-tars/images/mac_broken.png

@@ -9,9 +9,38 @@ import { runAgent } from '@main/services/runAgent';
 import { showWindow } from '@main/window/index';
 
 import { closeScreenMarker } from '@main/window/ScreenMarker';
+import { GUIAgent } from '@ui-tars/sdk';
+import { Operator } from '@ui-tars/sdk/core';
 
 const t = initIpc.create();
 
+export class GUIAgentManager {
+  private static instance: GUIAgentManager;
+  private currentAgent: GUIAgent<Operator> | null = null;
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private constructor() {}
+
+  public static getInstance(): GUIAgentManager {
+    if (!GUIAgentManager.instance) {
+      GUIAgentManager.instance = new GUIAgentManager();
+    }
+    return GUIAgentManager.instance;
+  }
+
+  public setAgent(agent: GUIAgent<Operator>) {
+    this.currentAgent = agent;
+  }
+
+  public getAgent(): GUIAgent<Operator> | null {
+    return this.currentAgent;
+  }
+
+  public clearAgent() {
+    this.currentAgent = null;
+  }
+}
+
 export const agentRoute = t.router({
   runAgent: t.procedure.input<void>().handle(async () => {
     const { thinking } = store.getState();
@@ -29,13 +58,31 @@ export const agentRoute = t.router({
 
     store.setState({ thinking: false });
   }),
+  pauseRun: t.procedure.input<void>().handle(async () => {
+    const guiAgent = GUIAgentManager.getInstance().getAgent();
+    if (guiAgent instanceof GUIAgent) {
+      guiAgent.pause();
+      store.setState({ thinking: false });
+    }
+  }),
+  resumeRun: t.procedure.input<void>().handle(async () => {
+    const guiAgent = GUIAgentManager.getInstance().getAgent();
+    if (guiAgent instanceof GUIAgent) {
+      guiAgent.resume();
+      store.setState({ thinking: false });
+    }
+  }),
   stopRun: t.procedure.input<void>().handle(async () => {
     const { abortController } = store.getState();
     store.setState({ status: StatusEnum.END, thinking: false });
 
     showWindow();
 
     abortController?.abort();
+    const guiAgent = GUIAgentManager.getInstance().getAgent();
+    if (guiAgent instanceof GUIAgent) {
+      guiAgent.stop();
+    }
 
     closeScreenMarker();
   }),

apps/ui-tars/images/mac_install.png

@@ -33,6 +33,7 @@ import { SettingStore } from './store/setting';
 import { createTray } from './tray';
 import { registerSettingsHandlers } from './services/settings';
 import { sanitizeState } from './utils/sanitizeState';
+import { windowManager } from './services/windowManager';
 
 const { isProd } = env;
 
@@ -185,18 +186,17 @@ const registerIPCHandlers = (
     return sanitizeState(state);
   });
 
+  // 初始化时注册已有窗口
+  wrappers.forEach((wrapper) => {
+    if (wrapper instanceof BrowserWindow) {
+      windowManager.registerWindow(wrapper);
+    }
+  });
+
   // only send state to the wrappers that are not destroyed
   ipcMain.on('subscribe', (state: unknown) => {
-    for (const wrapper of wrappers) {
-      const webContents = wrapper?.webContents;
-      if (webContents?.isDestroyed()) {
-        break;
-      }
-      webContents?.send(
-        'subscribe',
-        sanitizeState(state as Record<string, unknown>),
-      );
-    }
+    const sanitizedState = sanitizeState(state as Record<string, unknown>);
+    windowManager.broadcast('subscribe', sanitizedState);
   });
 
   const unsubscribe = store.subscribe((state: unknown) =>
@@ -243,4 +243,5 @@ app
 
     logger.info('app.whenReady end');
   })
+
   .catch(console.log);