refactor(sdk): screenshot only return base64 and scaleFactor

Author: ycjcl868

packages/operators/browserbase/package.json

@@ -36,8 +36,7 @@
   "dependencies": {
     "@ui-tars/shared": "workspace:*",
     "@browserbasehq/stagehand": "^1.13.0",
-    "big.js": "^6.2.2",
-    "jimp": "1.6.0"
+    "big.js": "^6.2.2"
   },
   "peerDependencies": {
     "@ui-tars/sdk": "workspace:*"

packages/operators/browserbase/src/index.ts

@@ -9,7 +9,6 @@ import {
   type ExecuteOutput,
   StatusEnum,
 } from '@ui-tars/sdk/core';
-import { Jimp } from 'jimp';
 import { Stagehand, ConstructorParams } from '@browserbasehq/stagehand';
 
 export class BrowserbaseOperator extends Operator {
@@ -35,7 +34,7 @@ export class BrowserbaseOperator extends Operator {
 User Instruction: Who is the top GitHub contributor to Stagehand by Browserbase?
 1. Thought: The best starting point to find the top GitHub contributor to the Stagehand project by Browserbase is the project's GitHub repository itself. This will provide direct access to the contributors' list, where you can see who has made the most contributions.\nAction: GOTO(url='https://github.com/browserbase/stagehand')
 2. Thought: The 'Insights' tab on a GitHub repository page provides detailed information about the project's contributors, including the number of contributions each has made. By accessing this tab, we can find the top contributor.\nAction: ACT(description='Click on the 'Insights' tab.')
-3. The 'Contributors' section under the 'Insights' tab will show a list of contributors along with the number of contributions each has made. This is the most direct way to identify the top contributor to the project.\nAction: ACT(description='Click on the \'Contributors\' link in the left sidebar.')
+3. The 'Contributors' section under the 'Insights' tab will show a list of contributors along with the number of contributions each has made. This is the most direct way to identify the top contributor to the project.\nAction: ACT(description='Click on the 'Contributors' link in the left sidebar.')
 4. The screenshot shows the contributors to the Stagehand project, with 'kamath' listed as the top contributor with 69 commits. This information is directly visible in the 'Contributors' section under the 'Insights' tab.\nAction: CLOSE()
 
 \`\`\``,
@@ -60,13 +59,8 @@ User Instruction: Who is the top GitHub contributor to Stagehand by Browserbase?
     const cdpSession = await page.context().newCDPSession(page);
     const { data: base64 } = await cdpSession.send('Page.captureScreenshot');
 
-    const buffer = Buffer.from(base64, 'base64');
-    const image = await Jimp.fromBuffer(buffer);
-
     return {
       base64,
-      width: image.width,
-      height: image.height,
       scaleFactor: 1,
     };
   }

packages/operators/nut-js/src/index.ts

@@ -73,35 +73,27 @@ export class NutJSOperator extends Operator {
     const width = screenWithScale.width / screenWithScale.pixelDensity.scaleX;
     const height = screenWithScale.height / screenWithScale.pixelDensity.scaleY;
 
-    const realScreenImage = await screenWithScaleImage
+    const physicalScreenImage = await screenWithScaleImage
       .resize({
         w: width,
         h: height,
       })
       .getBuffer('image/png', { quality: 75 });
 
     const output = {
-      base64: realScreenImage.toString('base64'),
-      width,
-      height,
+      base64: physicalScreenImage.toString('base64'),
       scaleFactor,
     };
 
     logger?.info(
-      `[NutjsOperator] screenshot: ${output.width}x${output.height}, scaleFactor: ${scaleFactor}`,
+      `[NutjsOperator] screenshot: ${width}x${height}, scaleFactor: ${scaleFactor}`,
     );
     return output;
   }
 
   async execute(params: ExecuteParams): Promise<ExecuteOutput> {
     const { logger } = useContext();
-    const {
-      parsedPrediction,
-      screenWidth,
-      screenHeight,
-      scaleFactor,
-      factors,
-    } = params;
+    const { parsedPrediction, screenWidth, screenHeight, scaleFactor } = params;
 
     const { action_type, action_inputs } = parsedPrediction;
     const startBoxStr = action_inputs?.start_box || '';
@@ -111,7 +103,6 @@ export class NutJSOperator extends Operator {
       boxStr: startBoxStr,
       screenWidth,
       screenHeight,
-      factors,
     });
 
     logger.info(`[NutjsOperator Position]: (${startX}, ${startY})`);

packages/sdk/README.md

@@ -225,10 +225,10 @@ When implementing a custom operator, you need to implement two core methods: `sc
     "jimp": "^1.6.0"
   },
   "peerDependencies": {
-    "@ui-tars/sdk": "latest"
+    "@ui-tars/sdk": "^1.2.0-beta.17"
   },
   "devDependencies": {
-    "@ui-tars/sdk": "latest",
+    "@ui-tars/sdk": "^1.2.0-beta.17",
     "tsup": "^8.3.5",
     "typescript": "^5.7.2",
     "vitest": "^3.0.2"

packages/sdk/src/GUIAgent.ts

@@ -7,17 +7,13 @@ import { GUIAgentData, StatusEnum, ShareVersion } from '@ui-tars/shared/types';
 import { IMAGE_PLACEHOLDER, MAX_LOOP_COUNT } from '@ui-tars/shared/constants';
 import { sleep } from '@ui-tars/shared/utils';
 import asyncRetry from 'async-retry';
+import { Jimp } from 'jimp';
 
 import { setContext } from './context/useContext';
 import { Operator, GUIAgentConfig } from './types';
 import { UITarsModel } from './Model';
 import { BaseGUIAgent } from './base';
-import {
-  getSummary,
-  parseBoxToScreenCoords,
-  processVlmParams,
-  toVlmModelFormat,
-} from './utils';
+import { getSummary, processVlmParams, toVlmModelFormat } from './utils';
 import {
   INTERNAL_ACTION_SPACES_ENUM,
   MAX_SNAPSHOT_ERR_CNT,
@@ -123,11 +119,15 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent<
           onRetry: retry?.screenshot?.onRetry,
         });
 
-        const isValidImage = !!(
-          snapshot?.base64 &&
-          snapshot?.width &&
-          snapshot?.height
-        );
+        const { width, height } =
+          (await Jimp.read(
+            Buffer.from(
+              snapshot.base64.replace(/^data:image\/\w+;base64,/, ''),
+              'base64',
+            ),
+          )) || {};
+
+        const isValidImage = !!(snapshot?.base64 && width && height);
 
         if (!isValidImage) {
           loopCnt -= 1;
@@ -145,8 +145,8 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent<
             screenshotBase64: snapshot.base64,
             screenshotContext: {
               size: {
-                width: snapshot.width,
-                height: snapshot.height,
+                width,
+                height,
               },
               scaleFactor: snapshot.scaleFactor,
             },
@@ -224,8 +224,8 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent<
           },
           screenshotContext: {
             size: {
-              width: snapshot.width,
-              height: snapshot.height,
+              width,
+              height,
             },
             scaleFactor: snapshot.scaleFactor,
           },
@@ -271,8 +271,8 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent<
                 operator.execute({
                   prediction,
                   parsedPrediction,
-                  screenWidth: snapshot.width,
-                  screenHeight: snapshot.height,
+                  screenWidth: width,
+                  screenHeight: height,
                   scaleFactor: snapshot.scaleFactor,
                   factors: this.model.factors,
                 }),

packages/sdk/src/utils.ts

@@ -24,12 +24,10 @@ export const parseBoxToScreenCoords = ({
   boxStr,
   screenWidth,
   screenHeight,
-  factors = DEFUALT_FACTORS,
 }: {
   boxStr: string;
   screenWidth: number;
   screenHeight: number;
-  factors?: Factors;
 }) => {
   if (!boxStr) {
     return { x: null, y: null };
@@ -41,11 +39,10 @@ export const parseBoxToScreenCoords = ({
     .map((num) => parseFloat(num.trim()));
 
   const [x1, y1, x2 = x1, y2 = y1] = coords;
-  const [widthFactor, heightFactor] = factors;
 
   return {
-    x: Math.round(((x1 + x2) / 2) * screenWidth * widthFactor) / widthFactor,
-    y: Math.round(((y1 + y2) / 2) * screenHeight * heightFactor) / heightFactor,
+    x: Math.round(((x1 + x2) / 2) * screenWidth * 1000) / 1000,
+    y: Math.round(((y1 + y2) / 2) * screenHeight * 1000) / 1000,
   };
 };
 

packages/shared/src/types/agent.ts

@@ -26,11 +26,7 @@ export interface VlmResponse {
 export interface ScreenshotResult {
   /** screenshot base64, `keep screenshot size as physical resolution` */
   base64: string;
-  /** real screenshot pixel width, `Physical Resolution` */
-  width: number;
-  /** real screenshot pixel height, `Physical Resolution` */
-  height: number;
-  /** screenshot scale factor(DPR) */
+  /** screenshot scale factor(DPR), physical_resolution = logical_resolution * scaleFactor */
   scaleFactor: number;
 }
 

pnpm-lock.yaml

@@ -79,8 +79,6 @@ export class NutJSElectronOperator extends NutJSOperator {
 
     return {
       base64: resized.toPNG().toString('base64'),
-      width: physicalSize.width,
-      height: physicalSize.height,
       scaleFactor,
     };
   }