Support doubao-1.5-thinking-vision-pro (#598)

ZhaoHeh · web-flow · commit b1c67722db4a · 2025-05-16T17:04:55.000+08:00
diff --git a/apps/ui-tars/src/main/agent/prompts.ts b/apps/ui-tars/src/main/agent/prompts.ts
@@ -85,3 +85,84 @@ call_user() # Submit the task and call the user when the task is unsolvable, or
 
 ## User Instruction
 `;
+
+export const getSystemPromptDoubao_15_15B = `
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(start_box='[x1, y1, x2, y2]')
+left_double(start_box='[x1, y1, x2, y2]')
+right_single(start_box='[x1, y1, x2, y2]')
+drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')
+hotkey(key='')
+type(content='xxx') # Use escape characters \\', \\", and \n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content.
+scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished(content='xxx') # Use escape characters \\', \\", and \n in content part to ensure we can parse the content in normal python string format.
+
+
+## Note
+- Use Chinese in \`Thought\` part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in \`Thought\` part.
+
+## User Instruction
+`;
+
+export const getSystemPromptDoubao_15_20B = `You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(point='<point>x1 y1</point>')
+left_double(point='<point>x1 y1</point>')
+right_single(point='<point>x1 y1</point>')
+drag(start_point='<point>x1 y1</point>', end_point='<point>x2 y2</point>')
+scroll(point='<point>x1 y1</point>', direction='down or up or right or left') # Show more information on the \`direction\` side.
+hotkey(key='ctrl c') # Split keys with a space and use lowercase. Also, do not use more than 3 keys in one hotkey action.
+press(key='ctrl') # Presses and holds down ONE key (e.g., ctrl). Use this action in combination with release(). You can perform other actions between press and release. For example, click elements while holding the ctrl key.
+release(key='ctrl') # Releases the key previously pressed. All actions between press and release will execute with the key held down. Note: Ensure all keys are released by the end of the step.
+type(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content.
+wait() # Sleep for 5s and take a screenshot to check for any changes.
+call_user() # Call the user when the task is unsolvable, or when you need the user's help. Then, user will see and answer your question in \`user_resp\`.
+finished(content='xxx') # Submit the task with an report to the user. Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
+
+
+## Note
+- Use Chinese in \`Thought\` part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in \`Thought\` part.
+- You may stumble upon new rules or features while playing the game or executing GUI tasks for the first time. Make sure to record them in your \`Thought\` and utilize them later.
+- Your thought style should follow the style of thought Examples.
+- You can provide multiple actions in one step, separated by "\n\n".
+- Ensure all keys you pressed are released by the end of the step.
+
+## Thought Examples
+- Example1. Thought: 第一行、第三列出现了一个数字2；第二列原有数字4与第四列新出现的数字4合并后变为8。注意观察第二列数字8与左边数字8的颜色比较浅一点，数字2的颜色看起来没有数字8的深。我猜测不同的颜色深的程度代表数值的大小，颜色较深的代表数值较大。这不，为了验证这个，我继续按下向左键让这两个8合并成为更大的数。
+- Example2. Thought: 真好！第一行第三列的数字2向左移动了两格合并到了第一行第一列，并且颜色比原先数字8的颜色深了许多。证明我的猜想没错，确实是这样！所以只有同样颜色深浅的数字才能够进行合并，而合并后的数字将变为原来数字的二倍并且颜色深度较深。而且!第一行第三列的2向左移动了两格，但是并没有和第一行第一列的2进行合并！由此可得，只有相同连续的格子才能够进行数字的合并。我按下向下键，16可以一步步进行合并得到2048，但是过程可能有些难。像我这样所做的操作并不是一步一步合并得到的。我这样做是为了更好的后续进行合并，得到更加大的数。
+- Example3. Thought: 又重新再来了。刚才的下键并没有起到什么作用。新格子还是刷到了第三行第四列的位置，表明下键此时并没有什么太大作用，我猜测是不是特定的布局无法支持一些方位的操作，为了验证，我得多尝试一些方位，我按下左键看看。
+- Example4. Thought: 哦，我知道了，同样的位置选择了同样的操作时不会发生改变的。除非是选择不同的方位！点击向上键以后，3、4行的数字都向上移动了一格，而它们原来所在的位置都被刷新出来了新数字，分别是4和2。同样，第三行第四列的数字2没有发生移动也刷新了新格子。明白了这一切后，我操作向左键试试看。
+- Example5. Thought: 经过我不懈的努力，在我的仔细观察选中的策略下，我成功地获得了胜利。这验证了我之前的猜想，移动按键只有我的头部移动到含数字的区域才会改变移动按键，蛇的身体移动到含数字的区域并不会影响移动按键。
+- Example6. Thought: 小蛇还是没动，我再次选择让它向右一步，希望这一次能成功移动，并且我猜测移动的间隔应该是蛇的长度，按动的次数也应该是蛇的长度。我或许需要将它记录下来，如果按一次它因为前方有障碍而动不了，但前方需要移动的话，需要按两次或者以上，按照蛇的长度来计算要按几次。
+- Example7. Thought: 我觉得我的猜测是正确的，小蛇的移动是根据手部的长度是否能达成这一条件进行前进，这对我之后的操作提供了很多帮助，也是游戏的通性。不过现在小蛇离苹果拿走只有一个格子，太过去了，所以后面还需要。再次往前走我们应该先走出道这个限制然后来到中间这个地方然后我们应该是绕一圈然后把这两道门选择开阔住然后使得这样才能让这个墙消失。那么我可以现在向左，尝试不触碰障碍的迈进，这似乎能改变小蛇的操作，使其改变路数。
+- Example8. Thought: 我观察到在出口管道里面，红苹果的前方还有一个阻挡物。那个阻挡物是一张带有浅褐和深褐色的老鼠皮，看起来随着红苹果的自然移动，它也在向着出口移动，但是对比旁边的方块框架显得很慢。目前这些都是我猜测的，我要看看推动这个老鼠皮要多少的力道。就在这时我刚好要按向右了，现在我按住 “D”键。
+- Example9. Thought: 太好了，我的做法是正确的，但是我发现激光点发射出来的激光这个时候并没有发光，看来我刚刚的猜测是不太全面的，还有新的知识，需要我再次了解一下激光的规则，回忆起来，刚刚似乎这个红色激光点发射出来的激光，别上是黄色，但上面的并没有什么波动，我需要新的条件，才能发现它的规律，将上一步的最后一格步骤拿出来，我发现刚刚不仅是激光颜色改变了，重要的是上面的箭头也改变了方向，也就是说激光点跟着太阳光一样，会有方向改变，这应该会是个关键消息，那我需要思考一下。
+- Example10. Thought: 我继续观察发光装置箭头方向和角度，我猜测离发射装置近的那个白方块，只能被移动到与发射装置相邻的中上方蓝色方块位置，那么此时下方的白方块只能位于最右边一列蓝色方块中的其中一个位置并与位于一条直线上的左下方的黑色圆圈重合，我只能在右下角和正下方的两个蓝色方块中选择，似乎，看起来右下角的这个方块的位置更能满足与两列黑色圆圈的距离的重合，但是到底是否正确的呢，那么我一定要去验证了。
+- Example11. Thought: 我们第一关是一个四边形,这个四边形内部的红绳是交织在一起的,我们根据以上经验如果要挪动一个毛线团的话,没有办法挪动任何一个上方有绳子限制的毛线团。所以从解题思路上我们可以打破这四边形的限制方向，那我们就可以挪动上方的毛线团。
+
+## Output Examples
+Thought: 在这里输出你的中文思考，你的思考样式应该参考上面的Thought Examples...
+Action: click(point='<point>10 20</point>')
+
+## User Instruction
+`;
diff --git a/apps/ui-tars/src/main/services/runAgent.ts b/apps/ui-tars/src/main/services/runAgent.ts
@@ -16,7 +16,12 @@ import {
   DefaultBrowserOperator,
   SearchEngine,
 } from '@ui-tars/operator-browser';
-import { getSystemPrompt, getSystemPromptV1_5 } from '../agent/prompts';
+import {
+  getSystemPrompt,
+  getSystemPromptV1_5,
+  getSystemPromptDoubao_15_15B,
+  getSystemPromptDoubao_15_20B,
+} from '../agent/prompts';
 import {
   closeScreenMarker,
   hideWidgetWindow,
@@ -44,6 +49,8 @@ const getModelVersion = (
       return UITarsModelVersion.V1_0;
     case VLMProviderV2.doubao_1_5:
       return UITarsModelVersion.DOUBAO_1_5_15B;
+    case VLMProviderV2.doubao_1_5_vl:
+      return UITarsModelVersion.DOUBAO_1_5_20B;
     default:
       return UITarsModelVersion.V1_0;
   }
@@ -166,16 +173,28 @@ export const runAgent = async (
     );
   }
 
+  const modelVersion = getModelVersion(settings.vlmProvider);
+
+  const getSpByModelVersion = (modelVersion: UITarsModelVersion) => {
+    switch (modelVersion) {
+      case UITarsModelVersion.DOUBAO_1_5_20B:
+        return getSystemPromptDoubao_15_20B;
+      case UITarsModelVersion.DOUBAO_1_5_15B:
+        return getSystemPromptDoubao_15_15B;
+      case UITarsModelVersion.V1_5:
+        return getSystemPromptV1_5(language, 'normal');
+      default:
+        return getSystemPrompt(language);
+    }
+  };
+
   const guiAgent = new GUIAgent({
     model: {
       baseURL: settings.vlmBaseUrl,
       apiKey: settings.vlmApiKey,
       model: settings.vlmModelName,
     },
-    systemPrompt:
-      getModelVersion(settings.vlmProvider) === UITarsModelVersion.V1_5
-        ? getSystemPromptV1_5(language, 'normal')
-        : getSystemPrompt(language),
+    systemPrompt: getSpByModelVersion(modelVersion),
     logger,
     signal: abortController?.signal,
     operator: operator,
@@ -206,7 +225,7 @@ export const runAgent = async (
     },
     maxLoopCount: settings.maxLoopCount,
     loopIntervalInMs: settings.loopIntervalInMs,
-    uiTarsVersion: getModelVersion(settings.vlmProvider),
+    uiTarsVersion: modelVersion,
   });
 
   GUIAgentManager.getInstance().setAgent(guiAgent);
diff --git a/apps/ui-tars/src/main/store/types.ts b/apps/ui-tars/src/main/store/types.ts
@@ -44,6 +44,7 @@ export enum VLMProviderV2 {
   ui_tars_1_0 = 'Hugging Face for UI-TARS-1.0',
   ui_tars_1_5 = 'Hugging Face for UI-TARS-1.5',
   doubao_1_5 = 'VolcEngine Ark for Doubao-1.5-UI-TARS',
+  doubao_1_5_vl = 'VolcEngine Ark for Doubao-1.5-thinking-vision-pro',
 }
 
 export enum SearchEngineForSettings {
diff --git a/packages/ui-tars/operators/browser-operator/src/browser-operator.ts b/packages/ui-tars/operators/browser-operator/src/browser-operator.ts
@@ -231,6 +231,14 @@ export class BrowserOperator extends Operator {
           await this.handleHotkey(action_inputs);
           break;
 
+        case 'press':
+          await this.handlePress(action_inputs);
+          break;
+
+        case 'release':
+          await this.handleRelease(action_inputs);
+          break;
+
         case 'scroll':
           await this.handleScroll(action_inputs);
           break;
@@ -405,6 +413,84 @@ export class BrowserOperator extends Operator {
     this.logger.info('Hotkey execution completed');
   }
 
+  private async handlePress(inputs: Record<string, any>) {
+    const page = await this.getActivePage();
+
+    const keyStr = inputs?.key;
+    if (!keyStr) {
+      this.logger.warn('No key specified for press');
+      throw new Error(`No key specified for press`);
+    }
+
+    this.logger.info(`Pressing key: ${keyStr}`);
+
+    const keys = keyStr.split(/[\s+]/);
+    const normalizedKeys: KeyInput[] = keys.map((key: string) => {
+      const lowercaseKey = key.toLowerCase();
+      const keyInput = KEY_MAPPINGS[lowercaseKey];
+
+      if (keyInput) {
+        return keyInput;
+      }
+
+      throw new Error(`Unsupported key: ${key}`);
+    });
+
+    this.logger.info(`Normalized keys for press:`, normalizedKeys);
+
+    // Only press the keys
+    for (const key of normalizedKeys) {
+      await page.keyboard.down(key);
+      await this.delay(50); // 添加小延迟确保按键稳定
+    }
+
+    this.logger.info('Press operation completed');
+  }
+
+  private async handleRelease(inputs: Record<string, any>) {
+    const page = await this.getActivePage();
+
+    const keyStr = inputs?.key;
+    if (!keyStr) {
+      this.logger.warn('No key specified for release');
+      throw new Error(`No key specified for release`);
+    }
+
+    this.logger.info(`Releasing key: ${keyStr}`);
+
+    const keys = keyStr.split(/[\s+]/);
+    const normalizedKeys: KeyInput[] = keys.map((key: string) => {
+      const lowercaseKey = key.toLowerCase();
+      const keyInput = KEY_MAPPINGS[lowercaseKey];
+
+      if (keyInput) {
+        return keyInput;
+      }
+
+      throw new Error(`Unsupported key: ${key}`);
+    });
+
+    this.logger.info(`Normalized keys for release:`, normalizedKeys);
+
+    // Release the keys
+    for (const key of normalizedKeys) {
+      await page.keyboard.up(key);
+      await this.delay(50); // 添加小延迟确保按键稳定
+    }
+
+    // For hotkey combinations that may trigger navigation,
+    // wait for navigation to complete
+    const navigationKeys = ['Enter', 'F5'];
+    if (normalizedKeys.some((key: string) => navigationKeys.includes(key))) {
+      this.logger.info('Waiting for possible navigation after key release');
+      await this.waitForPossibleNavigation(page);
+    } else {
+      await this.delay(500);
+    }
+
+    this.logger.info('Release operation completed');
+  }
+
   private async handleScroll(inputs: Record<string, any>) {
     const page = await this.getActivePage();
 
diff --git a/packages/ui-tars/operators/nut-js/src/index.ts b/packages/ui-tars/operators/nut-js/src/index.ts
@@ -117,6 +117,59 @@ export class NutJSOperator extends Operator {
     //   logger.info('[execute] [Region]', region);
     // }
 
+    const getHotkeys = (keyStr: string | undefined): Key[] => {
+      if (keyStr) {
+        const platformCommandKey =
+          process.platform === 'darwin' ? Key.LeftCmd : Key.LeftWin;
+        const platformCtrlKey =
+          process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl;
+        const keyMap: Record<string, Key> = {
+          return: Key.Enter,
+          enter: Key.Enter,
+          backspace: Key.Backspace,
+          delete: Key.Delete,
+          ctrl: platformCtrlKey,
+          shift: Key.LeftShift,
+          alt: Key.LeftAlt,
+          space: Key.Space,
+          'page down': Key.PageDown,
+          pagedown: Key.PageDown,
+          'page up': Key.PageUp,
+          pageup: Key.PageUp,
+          meta: platformCommandKey,
+          win: platformCommandKey,
+          command: platformCommandKey,
+          cmd: platformCommandKey,
+          comma: Key.Comma,
+          ',': Key.Comma,
+          up: Key.Up,
+          down: Key.Down,
+          left: Key.Left,
+          right: Key.Right,
+          arrowup: Key.Up,
+          arrowdown: Key.Down,
+          arrowleft: Key.Left,
+          arrowright: Key.Right,
+        };
+
+        const keys = keyStr
+          .split(/[\s+]/)
+          .map(
+            (k) =>
+              keyMap[k.toLowerCase()] ||
+              Key[k.toUpperCase() as keyof typeof Key],
+          );
+        logger.info('[NutjsOperator] hotkey: ', keys);
+        return keys;
+      } else {
+        logger.error(
+          '[NutjsOperator] hotkey error: ',
+          `${keyStr} is not a valid key`,
+        );
+        return [];
+      }
+    };
+
     switch (action_type) {
       case 'wait':
         logger.info('[NutjsOperator] wait', action_inputs);
@@ -215,50 +268,28 @@ export class NutJSOperator extends Operator {
 
       case 'hotkey': {
         const keyStr = action_inputs?.key || action_inputs?.hotkey;
-        if (keyStr) {
-          const platformCommandKey =
-            process.platform === 'darwin' ? Key.LeftCmd : Key.LeftWin;
-          const platformCtrlKey =
-            process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl;
-          const keyMap: Record<string, Key> = {
-            return: Key.Enter,
-            enter: Key.Enter,
-            backspace: Key.Backspace,
-            delete: Key.Delete,
-            ctrl: platformCtrlKey,
-            shift: Key.LeftShift,
-            alt: Key.LeftAlt,
-            space: Key.Space,
-            'page down': Key.PageDown,
-            pagedown: Key.PageDown,
-            'page up': Key.PageUp,
-            pageup: Key.PageUp,
-            meta: platformCommandKey,
-            win: platformCommandKey,
-            command: platformCommandKey,
-            cmd: platformCommandKey,
-            comma: Key.Comma,
-            ',': Key.Comma,
-            up: Key.Up,
-            down: Key.Down,
-            left: Key.Left,
-            right: Key.Right,
-            arrowup: Key.Up,
-            arrowdown: Key.Down,
-            arrowleft: Key.Left,
-            arrowright: Key.Right,
-          };
-
-          const keys = keyStr
-            .split(/[\s+]/)
-            .map(
-              (k) =>
-                keyMap[k.toLowerCase()] ||
-                Key[k.toUpperCase() as keyof typeof Key],
-            );
-          logger.info('[NutjsOperator] hotkey: ', keys);
+        const keys = getHotkeys(keyStr);
+        if (keys.length > 0) {
           await keyboard.pressKey(...keys);
-          await keyboard.releaseKey(...keys.reverse());
+          await keyboard.releaseKey(...keys);
+        }
+        break;
+      }
+
+      case 'press': {
+        const keyStr = action_inputs?.key || action_inputs?.hotkey;
+        const keys = getHotkeys(keyStr);
+        if (keys.length > 0) {
+          await keyboard.pressKey(...keys);
+        }
+        break;
+      }
+
+      case 'release': {
+        const keyStr = action_inputs?.key || action_inputs?.hotkey;
+        const keys = getHotkeys(keyStr);
+        if (keys.length > 0) {
+          await keyboard.releaseKey(...keys);
         }
         break;
       }

Original file line number	Diff line number	Diff line change
`@@ -44,6 +44,7 @@ export enum VLMProviderV2 {`
`44`	`44`	`ui_tars_1_0 = 'Hugging Face for UI-TARS-1.0',`
`45`	`45`	`ui_tars_1_5 = 'Hugging Face for UI-TARS-1.5',`
`46`	`46`	`doubao_1_5 = 'VolcEngine Ark for Doubao-1.5-UI-TARS',`
	`47`	`+ doubao_1_5_vl = 'VolcEngine Ark for Doubao-1.5-thinking-vision-pro',`
`47`	`48`	`}`
`48`	`49`
`49`	`50`	`export enum SearchEngineForSettings {`