fix: scale factor

Author: ycjcl868

packages/operators/nut-js/test/execute.test.ts

@@ -29,6 +29,7 @@ vi.mock('@ui-tars/sdk/core', async (importOriginal) => {
       factor: 1000,
     }),
     Operator: actual.Operator,
+    parseBoxToScreenCoords: actual.parseBoxToScreenCoords,
   };
 });
 
@@ -85,6 +86,7 @@ describe('execute', () => {
       },
       screenWidth: 1920,
       screenHeight: 1080,
+      scaleFactor: 1,
     };
 
     await nutJSOperator.execute(executeParams);
@@ -96,30 +98,6 @@ describe('execute', () => {
     expect(mouse.click).toHaveBeenCalledWith(Button.LEFT);
   });
 
-  it('Click on the search bar at the top of the screen with scaleFactor', async () => {
-    const nutJSOperator = new NutJSOperator({ scaleFactor: 1.5 });
-    const executeParams: ExecuteParams = {
-      prediction: {
-        reflection: '',
-        thought: 'Click on the search bar at the top of the screen\n',
-        action_type: 'click',
-        action_inputs: {
-          start_box: '[0.072,0.646,0.072,0.646]',
-        },
-      },
-      screenWidth: 1920,
-      screenHeight: 1080,
-    };
-
-    await nutJSOperator.execute(executeParams);
-
-    expect(mouse.move).toHaveBeenCalledWith(
-      straightTo(new Point(207.36, 1046.52)),
-    );
-
-    expect(mouse.click).toHaveBeenCalledWith(Button.LEFT);
-  });
-
   it('type doubao.com\n', async () => {
     const nutJSOperator = new NutJSOperator();
     const executeParams: ExecuteParams = {
@@ -133,6 +111,7 @@ describe('execute', () => {
       },
       screenWidth: 1920,
       screenHeight: 1080,
+      scaleFactor: 1,
     };
 
     await nutJSOperator.execute(executeParams);
@@ -154,6 +133,7 @@ describe('execute', () => {
       },
       screenWidth: 1920,
       screenHeight: 1080,
+      scaleFactor: 1,
     };
 
     await nutJSOperator.execute(executeParams);
@@ -175,6 +155,7 @@ describe('execute', () => {
       },
       screenWidth: 1920,
       screenHeight: 1080,
+      scaleFactor: 1,
     };
 
     await nutJSOperator.execute(executeParams);
@@ -199,6 +180,7 @@ describe('execute', () => {
       },
       screenWidth: 1920,
       screenHeight: 1080,
+      scaleFactor: 1,
     };
 
     await nutJSOperator.execute(executeParams);
@@ -224,6 +206,7 @@ describe('execute', () => {
       },
       screenWidth: 1920,
       screenHeight: 1080,
+      scaleFactor: 1,
     };
 
     await nutJSOperator.execute(executeParams);

packages/sdk/src/GUIAgent.ts

@@ -127,6 +127,7 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent<
                   width: snapshot.width,
                   height: snapshot.height,
                 },
+                scaleFactor: snapshot.scaleFactor,
               },
               timing: {
                 start,
@@ -202,6 +203,7 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent<
                   width: snapshot.width,
                   height: snapshot.height,
                 },
+                scaleFactor: snapshot.scaleFactor,
               },
               predictionParsed: parsedPredictions,
             });

packages/shared/src/types/data.ts

@@ -16,9 +16,13 @@ export interface Conversation extends Message {
   screenshotBase64?: string;
   screenshotContext?: {
     size: {
+      /** physical device width */
       width: number;
+      /** physical device height */
       height: number;
     };
+    /** screenshot scale factor(DPR) */
+    scaleFactor?: number;
   };
   predictionParsed?: PredictionParsed[];
 }

src/main/agent/operator.ts

@@ -6,24 +6,23 @@ import { Key, keyboard } from '@computer-use/nut-js';
 import { type ScreenshotOutput, type ExecuteParams } from '@ui-tars/sdk/core';
 import { NutJSOperator } from '@ui-tars/operator-nut-js';
 import { clipboard } from 'electron';
-import { desktopCapturer, screen } from 'electron';
+import { desktopCapturer } from 'electron';
 
 import * as env from '@main/env';
 import { logger } from '@main/logger';
 import { sleep } from '@ui-tars/shared/utils';
+import { getScreenSize } from '@main/utils/screen';
 
 export class NutJSElectronOperator extends NutJSOperator {
   public async screenshot(): Promise<ScreenshotOutput> {
-    const primaryDisplay = screen.getPrimaryDisplay();
-
-    const logicalSize = primaryDisplay.size; // Logical = Physical / scaleX
+    const { physicalSize, logicalSize, scaleFactor } = getScreenSize(); // Logical = Physical / scaleX
 
     logger.info(
       '[screenshot] [primaryDisplay]',
-      'size:',
-      primaryDisplay.size,
+      'logicalSize:',
+      logicalSize,
       'scaleFactor:',
-      primaryDisplay.scaleFactor,
+      scaleFactor,
     );
 
     const sources = await desktopCapturer.getSources({
@@ -36,14 +35,6 @@ export class NutJSElectronOperator extends NutJSOperator {
     const primarySource = sources[0];
     const screenshot = primarySource.thumbnail;
 
-    // Mac retina display scaleFactor = 1
-    const scaleFactor = env.isMacOS ? 1 : primaryDisplay.scaleFactor;
-
-    const physicalSize = {
-      width: Math.round(logicalSize.width * scaleFactor),
-      height: Math.round(logicalSize.height * scaleFactor),
-    };
-
     const resized = screenshot.resize({
       width: physicalSize.width,
       height: physicalSize.height,

src/main/ipcRoutes/screen.ts

@@ -2,17 +2,19 @@
  * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
  * SPDX-License-Identifier: Apache-2.0
  */
-import { screen } from 'electron';
 import { initIpc } from '@ui-tars/electron-ipc/main';
+import { getScreenSize } from '@main/utils/screen';
 
 const t = initIpc.create();
 
 export const screenRoute = t.router({
   getScreenSize: t.procedure.input<void>().handle(async () => {
-    const primaryDisplay = screen.getPrimaryDisplay();
+    const primaryDisplay = getScreenSize();
+
     return {
-      screenWidth: primaryDisplay.size.width,
-      screenHeight: primaryDisplay.size.height,
+      screenWidth: primaryDisplay.physicalSize.width,
+      screenHeight: primaryDisplay.physicalSize.height,
+      scaleFactor: primaryDisplay.scaleFactor,
     };
   }),
 });

src/main/services/runAgent.ts

@@ -55,7 +55,7 @@ export const runAgent = async (
           predictionParsed
         ) {
           const screenshotBase64WithElementMarker = await markClickPosition({
-            ...screenshotContext.size,
+            screenshotContext,
             base64: lastConv?.screenshotBase64,
             parsed: predictionParsed,
           }).catch((e) => {
@@ -95,7 +95,7 @@ export const runAgent = async (
       screenshotContext?.size &&
       !abortController?.signal?.aborted
     ) {
-      showPredictionMarker(predictionParsed, screenshotContext.size);
+      showPredictionMarker(predictionParsed, screenshotContext);
     }
 
     setState({

src/main/shared/setOfMarks.test.ts

@@ -64,8 +64,11 @@ const testMakeScreenMarker = () => {
     const { overlays } = setOfMarksOverlays({
       predictions: [action],
       screenshotContext: {
-        width: 2560,
-        height: 1440,
+        size: {
+          width: 2560,
+          height: 1440,
+        },
+        scaleFactor: 1,
       },
       xPos,
       yPos,

src/main/shared/setOfMarks.ts

@@ -30,14 +30,14 @@ export const setOfMarksOverlays = ({
   yPos,
 }: {
   predictions: PredictionParsed[];
-  screenshotContext: NonNullable<Conversation['screenshotContext']>['size'];
+  screenshotContext: NonNullable<Conversation['screenshotContext']>;
   xPos?: number;
   yPos?: number;
 }): {
   overlays: Overlay[];
 } => {
   const overlays: Overlay[] = [];
-  const { width, height } = screenshotContext;
+  const { width, height } = screenshotContext?.size || {};
 
   for (const prediction of predictions) {
     let boxWidth: number;

src/main/utils/image.ts

@@ -4,16 +4,15 @@
  */
 import sharp from 'sharp';
 
-import { type PredictionParsed } from '@ui-tars/shared/types';
+import { Conversation, type PredictionParsed } from '@ui-tars/shared/types';
 
 import { logger } from '@main/logger';
 import { setOfMarksOverlays } from '@main/shared/setOfMarks';
 
 // TODO: use jimp to mark click position
 export async function markClickPosition(data: {
   base64: string;
-  width: number;
-  height: number;
+  screenshotContext: NonNullable<Conversation['screenshotContext']>;
   parsed: PredictionParsed[];
 }): Promise<string> {
   if (!data?.parsed?.length) {
@@ -23,10 +22,7 @@ export async function markClickPosition(data: {
     const imageBuffer = Buffer.from(data.base64, 'base64');
     const { overlays = [] } = setOfMarksOverlays({
       predictions: data.parsed,
-      screenshotContext: {
-        width: data.width,
-        height: data.height,
-      },
+      screenshotContext: data.screenshotContext,
     });
     const imageOverlays: sharp.OverlayOptions[] = overlays
       .map((o) => {

src/main/utils/screen.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { screen } from 'electron';
+
+import * as env from '@main/env';
+
+export const getScreenSize = () => {
+  const primaryDisplay = screen.getPrimaryDisplay();
+
+  const logicalSize = primaryDisplay.size; // Logical = Physical / scaleX
+  // Mac retina display scaleFactor = 1
+  const scaleFactor = env.isMacOS ? 1 : primaryDisplay.scaleFactor;
+
+  const physicalSize = {
+    width: Math.round(logicalSize.width * scaleFactor),
+    height: Math.round(logicalSize.height * scaleFactor),
+  };
+
+  return {
+    physicalSize,
+    logicalSize,
+    scaleFactor,
+  };
+};

src/main/window/ScreenMarker.ts

@@ -199,14 +199,15 @@ class ScreenMarker {
   // show Screen Marker in screen for prediction
   showPredictionMarker(
     predictions: PredictionParsed[],
-    screenshotContext: NonNullable<Conversation['screenshotContext']>['size'],
+    screenshotContext: NonNullable<Conversation['screenshotContext']>,
   ) {
     const { overlays } = setOfMarksOverlays({
       predictions,
       screenshotContext,
       xPos: this.lastShowPredictionMarkerPos?.xPos,
       yPos: this.lastShowPredictionMarkerPos?.yPos,
     });
+    const { scaleFactor = 1 } = screenshotContext;
 
     // loop predictions
     for (let i = 0; i < overlays.length; i++) {
@@ -230,8 +231,9 @@ class ScreenMarker {
           webPreferences: { nodeIntegration: true, contextIsolation: false },
           ...(overlay.xPos &&
             overlay.yPos && {
-              x: overlay.xPos + overlay.offsetX,
-              y: overlay.yPos + overlay.offsetY,
+              // Logical Resolution
+              x: (overlay.xPos + overlay.offsetX) * scaleFactor,
+              y: (overlay.yPos + overlay.offsetY) * scaleFactor,
             }),
         });
 
@@ -309,7 +311,7 @@ export const closeScreenMarker = () => {
 
 export const showPredictionMarker = (
   predictions: PredictionParsed[],
-  screenshotContext: NonNullable<Conversation['screenshotContext']>['size'],
+  screenshotContext: NonNullable<Conversation['screenshotContext']>,
 ) => {
   ScreenMarker.getInstance().showPredictionMarker(
     predictions,

src/renderer/src/hooks/useScreenRecord.ts

@@ -22,7 +22,7 @@ export const useScreenRecord = (
 
   const drawSetOfMarkOverlays = (
     ctx: CanvasRenderingContext2D,
-    screenshotContext: NonNullable<Conversation['screenshotContext']>['size'],
+    screenshotContext: NonNullable<Conversation['screenshotContext']>,
   ) => {
     const messages = getState().messages;
     // console.log('[messages]', messages);
@@ -64,7 +64,8 @@ export const useScreenRecord = (
     try {
       recordedChunksRef.current = [];
 
-      const { screenWidth, screenHeight } = await api.getScreenSize();
+      const { screenWidth, screenHeight, scaleFactor } =
+        await api.getScreenSize();
 
       const stream = await navigator.mediaDevices.getDisplayMedia({
         video: {
@@ -131,8 +132,11 @@ export const useScreenRecord = (
 
             // draw set of mark overlays
             drawSetOfMarkOverlays(ctx, {
-              width: screenWidth,
-              height: screenHeight,
+              size: {
+                width: screenWidth,
+                height: screenHeight,
+              },
+              scaleFactor,
             });
           }
           animationFrameId = requestAnimationFrame(drawFrame);