feat: add ui-tars Android operator for cli

Author: ZhaoHeh

packages/cli/package.json

@@ -37,6 +37,7 @@
     "js-yaml": "^4.1.0",
     "@clack/prompts": "^0.10.0",
     "@ui-tars/operator-nut-js": "workspace:*",
+    "@ui-tars/operator-adb": "workspace:*",
     "@ui-tars/sdk": "workspace:*",
     "node-fetch": "^2.7.0"
   },

packages/cli/src/cli/commands.ts

@@ -14,6 +14,8 @@ export const run = () => {
     .command('start')
     .description('starting the ui-tars agent...')
     .option('-p, --presets <url>', 'Model Config Presets')
+    .option('-t, --target <target>', 'The target operator')
+    .option('-q, --query <query>', "Use's query")
     .action(async (options: CliOptions) => {
       try {
         await start(options);

packages/cli/src/cli/start.ts

@@ -12,9 +12,12 @@ import * as p from '@clack/prompts';
 import yaml from 'js-yaml';
 
 import { NutJSOperator } from '@ui-tars/operator-nut-js';
+import { getAndroidDeviceId, AdbOperator } from '@ui-tars/operator-adb';
 
 export interface CliOptions {
   presets?: string;
+  target?: string;
+  query?: string;
 }
 export const start = async (options: CliOptions) => {
   const CONFIG_PATH = path.join(os.homedir(), '.ui-tars-cli.json');
@@ -73,17 +76,50 @@ export const start = async (options: CliOptions) => {
     }
   }
 
-  const answers = await p.group(
-    {
-      instruction: () => p.text({ message: 'Input your instruction' }),
-    },
-    {
-      onCancel: () => {
-        p.cancel('操作已取消');
+  let targetOperator = null;
+  const targetType =
+    options.target ||
+    ((await p.select({
+      message: 'Please select your operator target:',
+      options: [
+        { value: 'nut-js', label: 'nut-js' },
+        { value: 'adb', label: 'adb' },
+      ],
+    })) as string);
+
+  switch (targetType) {
+    case 'adb':
+      const deviceId = await getAndroidDeviceId();
+      if (deviceId == null) {
+        console.error(
+          'No Android devices found. Please connect a device and try again.',
+        );
         process.exit(0);
-      },
-    },
-  );
+      }
+      targetOperator = new AdbOperator(deviceId);
+      break;
+    // case 'browser':
+    //   // TODO: implement browser operator
+    //   break;
+    case 'nut-js':
+    default:
+      targetOperator = new NutJSOperator();
+      break;
+  }
+
+  const answers = options.query
+    ? { instruction: options.query }
+    : await p.group(
+        {
+          instruction: () => p.text({ message: 'Input your instruction' }),
+        },
+        {
+          onCancel: () => {
+            p.cancel('操作已取消');
+            process.exit(0);
+          },
+        },
+      );
 
   const abortController = new AbortController();
   process.on('SIGINT', () => {
@@ -96,7 +132,7 @@ export const start = async (options: CliOptions) => {
       apiKey: config.apiKey,
       model: config.model,
     },
-    operator: new NutJSOperator(),
+    operator: targetOperator,
     signal: abortController.signal,
     // onData: ({ data }) => {
     // console.log(

packages/operators/adb/CHANGELOG.md

@@ -0,0 +1 @@
+# @ui-tars/operator-adb

packages/operators/adb/README.md

@@ -0,0 +1,17 @@
+# @ui-tars/operator-adb
+
+Operator Android SDK for UI-TARS using ADB.
+
+To enable Chinese text input via ADB, please install [ADBKeyBoard](https://github.com/senzhk/ADBKeyBoard/blob/master/ADBKeyboard.apk) and activate it in your device's Settings.
+
+**Install ADBKeyBoard:**
+
+```Shell
+adb install /path/to/AdbKeyboard.apk
+```
+
+**Activate ADBKeyBoard:**
+
+```Shell
+adb shell ime set com.android.adbkeyboard/.AdbIME
+```

packages/operators/adb/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@ui-tars/operator-adb",
+  "version": "1.0.0",
+  "description": "Operator Android SDK for UI-TARS",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bytedance/UI-TARS-desktop"
+  },
+  "bugs": {
+    "url": "https://github.com/bytedance/UI-TARS-desktop/issues"
+  },
+  "keywords": [
+    "AI",
+    "Core",
+    "SDK",
+    "Operator",
+    "UI-TARS"
+  ],
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "tsup --watch",
+    "prepare": "npm run build",
+    "build": "tsup",
+    "test": "vitest"
+  },
+  "license": "Apache-2.0",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "dependencies": {
+    "@ui-tars/shared": "workspace:*",
+    "execa": "5.0.1",
+    "inquirer": "8.2.4",
+    "jimp": "1.6.0"
+  },
+  "devDependencies": {
+    "@ui-tars/sdk": "workspace:*",
+    "tsup": "^8.3.5",
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.2",
+    "@types/big.js": "^6.2.2",
+    "@types/inquirer": "9.0.7"
+  }
+}