fix(execute): replace tail newline when typing

Author: ycjcl868

packages/action-parser/src/index.test.ts

@@ -1,3 +1,8 @@
+/**
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+// @prettier
 import { describe, expect, it } from 'vitest';
 
 import { actionParser } from './index';
@@ -91,25 +96,19 @@ describe('actionParser', () => {
     });
   });
 
-  it('should remove trailing \\n in content value', () => {
+  it('should return parsed action with newline', () => {
     const result = actionParser({
-      prediction:
-        'Thought: To proceed with the task of accessing "doubao.com," I need to type the correct URL into the address bar. Since the address bar is already active, the next logical step is to input the URL "doubao.com" to navigate to the desired website.\nType "doubao.com" into the address bar to initiate navigation to the website.\nAction: type(content=\'doubao.com\\n\')',
+      // prettier-ignore
+      prediction: "Thought: 我已经点击了地址栏，现在需要输入网址doubao.com。地址栏已经被激活，可以直接输入网址。\nAction: type(content='doubao.com\n')",
       factor: 1000,
     });
 
-    expect(result).toEqual({
-      parsed: [
-        {
-          action_inputs: {
-            content: 'doubao.com',
-          },
-          action_type: 'type',
-          reflection: '',
-          thought:
-            'To proceed with the task of accessing "doubao.com," I need to type the correct URL into the address bar. Since the address bar is already active, the next logical step is to input the URL "doubao.com" to navigate to the desired website.\nType "doubao.com" into the address bar to initiate navigation to the website.',
-        },
-      ],
-    });
+    expect(result.parsed[0].thought).toBe(
+      '我已经点击了地址栏，现在需要输入网址doubao.com。地址栏已经被激活，可以直接输入网址。',
+    );
+    expect(result.parsed[0].action_type).toBe('type');
+    expect(result.parsed[0].action_inputs.content).toEqual(
+      String.raw`doubao.com\n`,
+    );
   });
 });

packages/action-parser/src/index.ts

@@ -79,7 +79,8 @@ function parseActionVlm(
   const actions: PredictionParsed[] = [];
 
   for (const rawStr of allActions) {
-    const actionInstance = parseAction(rawStr.replace(/\n/g, '\\n').trim());
+    // prettier-ignore
+    const actionInstance = parseAction(rawStr.replace(/\n/g, String.raw`\n`).trimStart());
     if (!actionInstance) {
       console.log(`Action can't parse: ${rawStr}`);
       continue;
@@ -152,16 +153,11 @@ function parseAction(actionStr: string) {
         if (!key) continue;
 
         // Join value parts back together in case there were = signs in the value
-        let value = valueParts
+        const value = valueParts
           .join('=')
           .trim()
           .replace(/^['"]|['"]$/g, ''); // Remove surrounding quotes
 
-        // Remove trailing \n in content value
-        if (key.trim() === 'content' && value.endsWith('\\n')) {
-          value = value.slice(0, -2);
-        }
-
         //@ts-ignore
         kwargs[key.trim()] = value;
       }

src/main/agent/execute.test.ts

@@ -0,0 +1,113 @@
+import { Key, keyboard } from '@computer-use/nut-js';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { ExecuteParams, execute } from './execute';
+
+// Mock @computer-use/nut-js
+vi.mock('@computer-use/nut-js', async (importOriginal) => {
+  const actual: any = await importOriginal();
+  return {
+    mouse: {
+      move: vi.fn(),
+      click: vi.fn(),
+      config: {
+        mouseSpeed: 1500,
+      },
+    },
+    Key: actual.Key,
+    keyboard: {
+      type: vi.fn(),
+      pressKey: vi.fn(),
+      releaseKey: vi.fn(),
+      config: {
+        autoDelayMs: 0,
+      },
+    },
+    Button: {
+      LEFT: 'left',
+      RIGHT: 'right',
+      MIDDLE: 'middle',
+    },
+    Point: vi.fn(),
+    straightTo: vi.fn((point) => point),
+    sleep: vi.fn(),
+  };
+});
+
+describe('execute', () => {
+  const mockLogger = {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('type doubao.com\n', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To proceed with the task of accessing doubao.com, I need to type the URL into the address bar. This will allow me to navigate to the website and continue with the subsequent steps of the task.\n' +
+          `Type "doubao.com" into the browser's address bar.`,
+        action_type: 'type',
+        action_inputs: { content: 'doubao.com\\n' },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(keyboard.type).toHaveBeenCalledWith('doubao.com');
+    expect(keyboard.pressKey).toHaveBeenCalledWith(Key.Enter);
+  });
+
+  it('type doubao.com', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To proceed with the task of accessing doubao.com, I need to type the URL into the address bar. This will allow me to navigate to the website and continue with the subsequent steps of the task.\n' +
+          `Type "doubao.com" into the browser's address bar.`,
+        action_type: 'type',
+        action_inputs: { content: 'doubao.com' },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(keyboard.type).toHaveBeenCalledWith('doubao.com');
+    expect(keyboard.pressKey).not.toHaveBeenCalledWith(Key.Enter);
+  });
+
+  it('type Hello World\nUI-TARS\n', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To proceed with the task of accessing doubao.com, I need to type the URL into the address bar. This will allow me to navigate to the website and continue with the subsequent steps of the task.\n' +
+          `Type "Hello World\nUI-TARS\n" into the browser's address bar.`,
+        action_type: 'type',
+        action_inputs: { content: 'Hello World\\nUI-TARS\\n' },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(keyboard.type).toHaveBeenCalledWith('Hello World\\nUI-TARS');
+    expect(keyboard.pressKey).toHaveBeenCalledWith(Key.Enter);
+  });
+});

src/main/agent/execute.ts

@@ -25,13 +25,15 @@ const moveStraightTo = async (startX: number | null, startY: number | null) => {
   await mouse.move(straightTo(new Point(startX, startY)));
 };
 
-export const execute = async (executeParams: {
+export interface ExecuteParams {
   scaleFactor?: number;
   prediction: PredictionParsed;
   screenWidth: number;
   screenHeight: number;
   logger?: any;
-}) => {
+}
+
+export const execute = async (executeParams: ExecuteParams) => {
   const {
     prediction,
     screenWidth,
@@ -141,17 +143,24 @@ export const execute = async (executeParams: {
       const content = action_inputs.content?.trim();
       logger.info('[device] type', content);
       if (content) {
+        const stripContent = content.replace(/\\n$/, '').replace(/\n$/, '');
         keyboard.config.autoDelayMs = 0;
         if (env.isWindows) {
           const originalClipboard = clipboard.readText();
-          clipboard.writeText(content);
+          clipboard.writeText(stripContent);
           await keyboard.pressKey(Key.LeftControl, Key.V);
           await keyboard.releaseKey(Key.LeftControl, Key.V);
-          await sleep(100);
+          await sleep(500);
           clipboard.writeText(originalClipboard);
         } else {
-          await keyboard.type(content);
+          await keyboard.type(stripContent);
+        }
+
+        if (content.endsWith('\n') || content.endsWith('\\n')) {
+          await keyboard.pressKey(Key.Enter);
+          await keyboard.releaseKey(Key.Enter);
         }
+
         keyboard.config.autoDelayMs = 500;
       }
       break;