fix(execute): type workaround for unexpected newline in content value (#6)

* feat(action-parser): workaround for unexpected newline in content value

* fix(execute): replace tail newline when typing

* feat(execute): page up and page down

---------

Co-authored-by: jinxin001 <jinxin001@bytedance.com>

Author: ulivz

packages/action-parser/src/index.test.ts

@@ -1,3 +1,8 @@
+/**
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+// @prettier
 import { describe, expect, it } from 'vitest';
 
 import { actionParser } from './index';
@@ -90,4 +95,20 @@ describe('actionParser', () => {
       ],
     });
   });
+
+  it('should return parsed action with newline', () => {
+    const result = actionParser({
+      // prettier-ignore
+      prediction: "Thought: 我已经点击了地址栏，现在需要输入网址doubao.com。地址栏已经被激活，可以直接输入网址。\nAction: type(content='doubao.com\n')",
+      factor: 1000,
+    });
+
+    expect(result.parsed[0].thought).toBe(
+      '我已经点击了地址栏，现在需要输入网址doubao.com。地址栏已经被激活，可以直接输入网址。',
+    );
+    expect(result.parsed[0].action_type).toBe('type');
+    expect(result.parsed[0].action_inputs.content).toEqual(
+      String.raw`doubao.com\n`,
+    );
+  });
 });

packages/action-parser/src/index.ts

@@ -79,7 +79,8 @@ function parseActionVlm(
   const actions: PredictionParsed[] = [];
 
   for (const rawStr of allActions) {
-    const actionInstance = parseAction(rawStr.replace(/\n/g, '\\n').trim());
+    // prettier-ignore
+    const actionInstance = parseAction(rawStr.replace(/\n/g, String.raw`\n`).trimStart());
     if (!actionInstance) {
       console.log(`Action can't parse: ${rawStr}`);
       continue;

src/main/agent/execute.test.ts

@@ -0,0 +1,113 @@
+import { Key, keyboard } from '@computer-use/nut-js';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { ExecuteParams, execute } from './execute';
+
+// Mock @computer-use/nut-js
+vi.mock('@computer-use/nut-js', async (importOriginal) => {
+  const actual: any = await importOriginal();
+  return {
+    mouse: {
+      move: vi.fn(),
+      click: vi.fn(),
+      config: {
+        mouseSpeed: 1500,
+      },
+    },
+    Key: actual.Key,
+    keyboard: {
+      type: vi.fn(),
+      pressKey: vi.fn(),
+      releaseKey: vi.fn(),
+      config: {
+        autoDelayMs: 0,
+      },
+    },
+    Button: {
+      LEFT: 'left',
+      RIGHT: 'right',
+      MIDDLE: 'middle',
+    },
+    Point: vi.fn(),
+    straightTo: vi.fn((point) => point),
+    sleep: vi.fn(),
+  };
+});
+
+describe('execute', () => {
+  const mockLogger = {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('type doubao.com\n', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To proceed with the task of accessing doubao.com, I need to type the URL into the address bar. This will allow me to navigate to the website and continue with the subsequent steps of the task.\n' +
+          `Type "doubao.com" into the browser's address bar.`,
+        action_type: 'type',
+        action_inputs: { content: 'doubao.com\\n' },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(keyboard.type).toHaveBeenCalledWith('doubao.com');
+    expect(keyboard.pressKey).toHaveBeenCalledWith(Key.Enter);
+  });
+
+  it('type doubao.com', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To proceed with the task of accessing doubao.com, I need to type the URL into the address bar. This will allow me to navigate to the website and continue with the subsequent steps of the task.\n' +
+          `Type "doubao.com" into the browser's address bar.`,
+        action_type: 'type',
+        action_inputs: { content: 'doubao.com' },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(keyboard.type).toHaveBeenCalledWith('doubao.com');
+    expect(keyboard.pressKey).not.toHaveBeenCalledWith(Key.Enter);
+  });
+
+  it('type Hello World\nUI-TARS\n', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To proceed with the task of accessing doubao.com, I need to type the URL into the address bar. This will allow me to navigate to the website and continue with the subsequent steps of the task.\n' +
+          `Type "Hello World\nUI-TARS\n" into the browser's address bar.`,
+        action_type: 'type',
+        action_inputs: { content: 'Hello World\\nUI-TARS\\n' },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(keyboard.type).toHaveBeenCalledWith('Hello World\\nUI-TARS');
+    expect(keyboard.pressKey).toHaveBeenCalledWith(Key.Enter);
+  });
+});

src/main/agent/execute.ts

@@ -25,13 +25,15 @@ const moveStraightTo = async (startX: number | null, startY: number | null) => {
   await mouse.move(straightTo(new Point(startX, startY)));
 };
 
-export const execute = async (executeParams: {
+export interface ExecuteParams {
   scaleFactor?: number;
   prediction: PredictionParsed;
   screenWidth: number;
   screenHeight: number;
   logger?: any;
-}) => {
+}
+
+export const execute = async (executeParams: ExecuteParams) => {
   const {
     prediction,
     screenWidth,
@@ -141,17 +143,24 @@ export const execute = async (executeParams: {
       const content = action_inputs.content?.trim();
       logger.info('[device] type', content);
       if (content) {
+        const stripContent = content.replace(/\\n$/, '').replace(/\n$/, '');
         keyboard.config.autoDelayMs = 0;
         if (env.isWindows) {
           const originalClipboard = clipboard.readText();
-          clipboard.writeText(content);
+          clipboard.writeText(stripContent);
           await keyboard.pressKey(Key.LeftControl, Key.V);
           await keyboard.releaseKey(Key.LeftControl, Key.V);
-          await sleep(100);
+          await sleep(500);
           clipboard.writeText(originalClipboard);
         } else {
-          await keyboard.type(content);
+          await keyboard.type(stripContent);
+        }
+
+        if (content.endsWith('\n') || content.endsWith('\\n')) {
+          await keyboard.pressKey(Key.Enter);
+          await keyboard.releaseKey(Key.Enter);
         }
+
         keyboard.config.autoDelayMs = 500;
       }
       break;
@@ -167,27 +176,35 @@ export const execute = async (executeParams: {
           shift: Key.LeftShift,
           alt: Key.LeftAlt,
           space: Key.Space,
+          'page down': Key.PageDown,
+          pagedown: Key.PageDown,
+          'page up': Key.PageUp,
+          pageup: Key.PageUp,
         };
 
         const keys = keyStr
           .split(/[\s+]/)
           .map((k) => keyMap[k.toLowerCase()] || Key[k as keyof typeof Key]);
         logger.info('[hotkey]: ', keys);
         await keyboard.pressKey(...keys);
+        await keyboard.releaseKey(...keys);
       }
       break;
     }
 
     case 'scroll': {
       const { direction } = action_inputs;
-      await moveStraightTo(startX, startY);
+      // if startX and startY is not null, move mouse to startX, startY
+      if (startX !== null && startY !== null) {
+        await moveStraightTo(startX, startY);
+      }
 
       switch (direction?.toLowerCase()) {
         case 'up':
-          await mouse.scrollUp(5); // 向上滚动为正数
+          await mouse.scrollUp(5);
           break;
         case 'down':
-          await mouse.scrollDown(-5); // 向下滚动为负数
+          await mouse.scrollDown(5);
           break;
         default:
           console.warn(`Unsupported scroll direction: ${direction}`);

src/renderer/src/components/RunMessages/index.tsx

@@ -72,6 +72,11 @@ const RunMessages: React.FC<RunMessagesProps> = (props) => {
         borderColor="rgba(112, 107, 87, 0.5)"
         p={4}
         overflow="auto"
+        css={{
+          '&::-webkit-scrollbar': 'initial',
+          '&::-webkit-scrollbar-track': 'initial',
+          '&::-webkit-scrollbar-thumb': 'border-radius: 4px',
+        }}
       >
         {Boolean(loading) && (
           <Center h="100%">