From 263a31eb6db38ba9874465cff5c4fc959203a9f8 Mon Sep 17 00:00:00 2001 From: ycjcl868 Date: Fri, 21 Mar 2025 14:47:46 +0800 Subject: [PATCH 1/3] fix(agent-tars): need macos accessibility --- apps/agent-tars/forge.config.ts | 6 ++- apps/agent-tars/package.json | 1 + apps/agent-tars/src/main/index.ts | 17 +++++- .../src/main/utils/systemPermissions.ts | 52 +++++++++++++++++++ pnpm-lock.yaml | 3 ++ 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 apps/agent-tars/src/main/utils/systemPermissions.ts diff --git a/apps/agent-tars/forge.config.ts b/apps/agent-tars/forge.config.ts index 6de65e05d..b7babb490 100644 --- a/apps/agent-tars/forge.config.ts +++ b/apps/agent-tars/forge.config.ts @@ -23,7 +23,11 @@ const keepModules = new Set([ '@mixmark-io/domino', '@modelcontextprotocol/sdk', ]); -const needSubDependencies = ['@tavily/core', '@modelcontextprotocol/sdk']; +const needSubDependencies = [ + '@tavily/core', + '@modelcontextprotocol/sdk', + '@computer-use/node-mac-permissions', +]; const keepLanguages = new Set(['en', 'en_GB', 'en-US', 'en_US']); const ignorePattern = new RegExp( `^/node_modules/(?!${[...keepModules].join('|')})`, diff --git a/apps/agent-tars/package.json b/apps/agent-tars/package.json index 8bb3a7d07..0db4a50c6 100644 --- a/apps/agent-tars/package.json +++ b/apps/agent-tars/package.json @@ -39,6 +39,7 @@ }, "dependencies": { "@tavily/core": "0.3.1", + "@computer-use/node-mac-permissions": "2.2.2", "@agent-infra/mcp-server-commands": "workspace:*", "@agent-infra/mcp-server-filesystem": "workspace:*", "@agent-infra/mcp-server-browser": "workspace:*" diff --git a/apps/agent-tars/src/main/index.ts b/apps/agent-tars/src/main/index.ts index c5ca3c715..c85913286 100644 --- a/apps/agent-tars/src/main/index.ts +++ b/apps/agent-tars/src/main/index.ts @@ -63,10 +63,23 @@ function createWindow(): void { } } +const initializeApp = async () => { + if (process.platform === 'darwin') { + app.setAccessibilitySupportEnabled(true); + const { ensurePermissions } = await import('@main/utils/systemPermissions'); + + const ensureScreenCapturePermission = ensurePermissions(); + console.info( + 'ensureScreenCapturePermission', + ensureScreenCapturePermission, + ); + } +}; + // This method will be called when Electron has finished // initialization and is ready to create browser windows. // Some APIs can only be used after this event occurs. -app.whenReady().then(() => { +app.whenReady().then(async () => { // Set app user model id for windows electronApp.setAppUserModelId('com.electron'); @@ -77,6 +90,8 @@ app.whenReady().then(() => { optimizer.watchWindowShortcuts(window); }); + await initializeApp(); + // IPC test ipcMain.on('ping', () => console.log('pong')); registerIpcMain(ipcRoutes); diff --git a/apps/agent-tars/src/main/utils/systemPermissions.ts b/apps/agent-tars/src/main/utils/systemPermissions.ts new file mode 100644 index 000000000..0018d293a --- /dev/null +++ b/apps/agent-tars/src/main/utils/systemPermissions.ts @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2025 Bytedance, Inc. and its affiliates. + * SPDX-License-Identifier: Apache-2.0 + */ +import permissions from '@computer-use/node-mac-permissions'; + +let hasAccessibilityPermission = false; + +const wrapWithWarning = + (message, nativeFunction) => + (...args) => { + console.warn(message); + return nativeFunction(...args); + }; + +const askForAccessibility = (nativeFunction, functionName) => { + const accessibilityStatus = permissions.getAuthStatus('accessibility'); + console.info('[accessibilityStatus]', accessibilityStatus); + + if (accessibilityStatus === 'authorized') { + hasAccessibilityPermission = true; + return nativeFunction; + } else if ( + accessibilityStatus === 'not determined' || + accessibilityStatus === 'denied' + ) { + hasAccessibilityPermission = false; + permissions.askForAccessibilityAccess(); + return wrapWithWarning( + `##### WARNING! The application running this script tries to access accessibility features to execute ${functionName}! Please grant requested access and visit https://github.com/nut-tree/nut.js#macos for further information. #####`, + nativeFunction, + ); + } +}; + +export const ensurePermissions = (): { + accessibility: boolean; +} => { + if (process.env.CI === 'e2e') { + return { + accessibility: true, + }; + } + + askForAccessibility(() => {}, 'execute accessibility'); + + console.info('hasAccessibilityPermission', hasAccessibilityPermission); + + return { + accessibility: hasAccessibilityPermission, + }; +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 855aeb30b..76c1c714a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -98,6 +98,9 @@ importers: '@agent-infra/mcp-server-filesystem': specifier: workspace:* version: link:../../packages/agent-infra/mcp-servers/filesystem + '@computer-use/node-mac-permissions': + specifier: 2.2.2 + version: 2.2.2 '@tavily/core': specifier: 0.3.1 version: 0.3.1 From c17ba147fe92f2cbb42090a8dfff26ef4663dab8 Mon Sep 17 00:00:00 2001 From: ycjcl868 Date: Fri, 21 Mar 2025 14:56:26 +0800 Subject: [PATCH 2/3] docs: typo --- apps/agent-tars/docs/quick-start.md | 8 +++++++- apps/agent-tars/src/main/utils/systemPermissions.ts | 2 +- apps/ui-tars/src/main/utils/systemPermissions.ts | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/agent-tars/docs/quick-start.md b/apps/agent-tars/docs/quick-start.md index 8fac049de..a2d7b35c3 100644 --- a/apps/agent-tars/docs/quick-start.md +++ b/apps/agent-tars/docs/quick-start.md @@ -8,6 +8,12 @@ This guide will walk you through the process of setting up your first Agent TARS Before you begin, you will need to set some necessary configuration. +Enable the Accessibility permission of **Agent TARS** in MacOS: + - System Settings -> Privacy & Security -> **Accessibility** + +![accessibility-permission.png](https://github.com/user-attachments/assets/77e171d2-dffb-4905-98c0-92c5ab00e333) + + You can click the left-bottom button to open the configuration page: ![setting-icon.png](https://lf3-static.bytednsdoc.com/obj/eden-cn/uhbfnupenuhf/agent-tars/setting-icon.jpeg) @@ -38,7 +44,7 @@ We also support **Human In the Loop**, that means you can interact with the agen ![human-in-the-loop.jpeg](https://lf3-static.bytednsdoc.com/obj/eden-cn/uhbfnupenuhf/agent-tars/human-in-the-loop.jpeg) -## Share Your Thread +## Share Your Thead You can share your thread with others by the share button on the top menu. diff --git a/apps/agent-tars/src/main/utils/systemPermissions.ts b/apps/agent-tars/src/main/utils/systemPermissions.ts index 0018d293a..d61780bb5 100644 --- a/apps/agent-tars/src/main/utils/systemPermissions.ts +++ b/apps/agent-tars/src/main/utils/systemPermissions.ts @@ -27,7 +27,7 @@ const askForAccessibility = (nativeFunction, functionName) => { hasAccessibilityPermission = false; permissions.askForAccessibilityAccess(); return wrapWithWarning( - `##### WARNING! The application running this script tries to access accessibility features to execute ${functionName}! Please grant requested access and visit https://github.com/nut-tree/nut.js#macos for further information. #####`, + `##### WARNING! The application running this script tries to access accessibility features to execute ${functionName}! Please grant requested access for further information. #####`, nativeFunction, ); } diff --git a/apps/ui-tars/src/main/utils/systemPermissions.ts b/apps/ui-tars/src/main/utils/systemPermissions.ts index 705a34bc0..3e53fca98 100644 --- a/apps/ui-tars/src/main/utils/systemPermissions.ts +++ b/apps/ui-tars/src/main/utils/systemPermissions.ts @@ -54,7 +54,7 @@ const askForScreenRecording = (nativeFunction, functionName) => { hasScreenRecordingPermission = false; permissions.askForScreenCaptureAccess(); return wrapWithWarning( - `##### WARNING! The application running this script tries to screen recording features to execute ${functionName}! Please grant the requested access and visit https://github.com/nut-tree/nut.js#macos for further information. #####`, + `##### WARNING! The application running this script tries to screen recording features to execute ${functionName}! Please grant the requested access for further information. #####`, nativeFunction, ); } From f77e91b4125829c4ae8bddd772d07841d55be962 Mon Sep 17 00:00:00 2001 From: ycjcl868 Date: Fri, 21 Mar 2025 14:57:44 +0800 Subject: [PATCH 3/3] chore: ci --- .github/workflows/e2e.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0faf8a866..0a5628967 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,6 +51,6 @@ jobs: if: matrix.os == 'macos-latest' run: npm install -g appdmg - name: Install dependencies - run: npm exec turbo run bootstrap + run: pnpm install - name: Run e2e run: npm exec turbo run ui-tars-desktop#test:e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66aa8fcd4..eb5a0a5e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,7 @@ jobs: node-version: 20 cache: 'pnpm' - name: Install dependencies - run: npm exec turbo run bootstrap + run: pnpm install - name: Run typecheck run: npm exec turbo run typecheck - name: Run test