fix(execute): drag not work (#26)

Author: ycjcl868

package.json

@@ -46,6 +46,7 @@
     "@ui-tars/action-parser": "workspace:*",
     "@ui-tars/shared": "workspace:*",
     "async-retry": "^1.3.3",
+    "big.js": "^6.2.2",
     "dotenv": "^16.4.7",
     "electron-debug": "^3.2.0",
     "electron-devtools-installer": "^3.2.0",

pnpm-lock.yaml

@@ -1,4 +1,13 @@
-import { Key, keyboard } from '@computer-use/nut-js';
+import {
+  Button,
+  Key,
+  Point,
+  Region,
+  centerOf,
+  keyboard,
+  mouse,
+  straightTo,
+} from '@computer-use/nut-js';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { ExecuteParams, execute } from './execute';
@@ -13,6 +22,7 @@ vi.mock('@computer-use/nut-js', async (importOriginal) => {
       config: {
         mouseSpeed: 1500,
       },
+      drag: vi.fn(),
     },
     Key: actual.Key,
     keyboard: {
@@ -28,8 +38,11 @@ vi.mock('@computer-use/nut-js', async (importOriginal) => {
       RIGHT: 'right',
       MIDDLE: 'middle',
     },
-    Point: vi.fn(),
+    Point: actual.Point,
+    Region: actual.Region,
     straightTo: vi.fn((point) => point),
+    centerOf: vi.fn((region) => region),
+    randomPointIn: vi.fn((region) => region),
     sleep: vi.fn(),
   };
 });
@@ -45,6 +58,56 @@ describe('execute', () => {
     vi.clearAllMocks();
   });
 
+  it('Click on the search bar at the top of the screen', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought: 'Click on the search bar at the top of the screen\n',
+        action_type: 'click',
+        action_inputs: {
+          start_box: '[0.072,0.646,0.072,0.646]',
+        },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(mouse.move).toHaveBeenCalledWith(
+      straightTo(new Point(138.24, 697.68)),
+    );
+
+    expect(mouse.click).toHaveBeenCalledWith(Button.LEFT);
+  });
+
+  it('Click on the search bar at the top of the screen with scaleFactor', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought: 'Click on the search bar at the top of the screen\n',
+        action_type: 'click',
+        action_inputs: {
+          start_box: '[0.072,0.646,0.072,0.646]',
+        },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1.5,
+    };
+
+    await execute(executeParams);
+
+    expect(mouse.move).toHaveBeenCalledWith(
+      straightTo(new Point(207.36, 1046.52)),
+    );
+
+    expect(mouse.click).toHaveBeenCalledWith(Button.LEFT);
+  });
+
   it('type doubao.com\n', async () => {
     const executeParams: ExecuteParams = {
       prediction: {
@@ -110,4 +173,56 @@ describe('execute', () => {
     expect(keyboard.type).toHaveBeenCalledWith('Hello World\\nUI-TARS');
     expect(keyboard.pressKey).toHaveBeenCalledWith(Key.Enter);
   });
+
+  it('drag slider horizontally', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To narrow down the search results to cat litters within the specified price range of $18 to $32, I need to adjust the price filter. The next logical step is to drag the left handle of the price slider to set the minimum price to $18, ensuring that only products within the desired range are displayed.\n' +
+          'Drag the left handle of the price slider to set the minimum price to $18.',
+        action_type: 'drag',
+        action_inputs: {
+          start_box: '[0.072,0.646,0.072,0.646]',
+          end_box: '[0.175,0.647,0.175,0.647]',
+        },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(mouse.drag).toHaveBeenCalledWith(
+      straightTo(centerOf(new Region(138.24, 697.68, 197.76, 1.08))),
+    );
+  });
+
+  it('drag slider vertically', async () => {
+    const executeParams: ExecuteParams = {
+      prediction: {
+        reflection: '',
+        thought:
+          'To narrow down the search results to cat litters within the specified price range of $18 to $32, I need to adjust the price filter. The next logical step is to drag the left handle of the price slider to set the minimum price to $18, ensuring that only products within the desired range are displayed.\n' +
+          'Drag the left handle of the price slider to set the minimum price to $18.',
+        action_type: 'drag',
+        action_inputs: {
+          start_box: '[0.072,0.646,0.072,0.646]',
+          end_box: '[0.072,0.546,0.072,0.546]',
+        },
+      },
+      screenWidth: 1920,
+      screenHeight: 1080,
+      logger: mockLogger,
+      scaleFactor: 1,
+    };
+
+    await execute(executeParams);
+
+    expect(mouse.drag).toHaveBeenCalledWith(
+      straightTo(centerOf(new Region(138.24, 697.68, 0, -108))),
+    );
+  });
 });

src/main/agent/execute.test.ts

@@ -6,11 +6,14 @@ import {
   Button,
   Key,
   Point,
+  Region,
+  centerOf,
   keyboard,
   mouse,
   sleep,
   straightTo,
 } from '@computer-use/nut-js';
+import Big from 'big.js';
 import { clipboard } from 'electron';
 
 import { PredictionParsed } from '@ui-tars/shared/types';
@@ -133,7 +136,13 @@ export const execute = async (executeParams: ExecuteParams) => {
         );
 
         if (startX && startY && endX && endY) {
-          await mouse.drag([new Point(startX, startY), new Point(endX, endY)]);
+          // calculate x and y direction difference
+          const diffX = Big(endX).minus(startX).toNumber();
+          const diffY = Big(endY).minus(startY).toNumber();
+
+          await mouse.drag(
+            straightTo(centerOf(new Region(startX, startY, diffX, diffY))),
+          );
         }
       }
       break;