diff --git a/apps/agent-tars/package.json b/apps/agent-tars/package.json index 0db4a50c6..2a487104a 100644 --- a/apps/agent-tars/package.json +++ b/apps/agent-tars/package.json @@ -1,6 +1,6 @@ { "name": "agent-tars-app", - "version": "1.0.0-alpha.2", + "version": "1.0.0-alpha.3", "description": "A multimodal AI agent that revolutionizes GUI interaction", "main": "./dist/main/index.js", "author": "ByteDance", diff --git a/apps/agent-tars/src/main/customTools/search.ts b/apps/agent-tars/src/main/customTools/search.ts index 0853de2d6..83609fe96 100644 --- a/apps/agent-tars/src/main/customTools/search.ts +++ b/apps/agent-tars/src/main/customTools/search.ts @@ -56,6 +56,31 @@ export async function search(toolCall: ToolCall): Promise { count: args.count, query: args.query, }); + } else if ( + currentSearchConfig.provider === SearchProvider.DUCKDUCKGO_SEARCH + ) { + const client = new SearchClient({ + provider: SearchProviderEnum.DuckduckgoSearch, + providerConfig: {}, + }); + results = await client.search({ + query: args.query, + count: args.count, + }); + // } else if (currentSearchConfig.provider === SearchProvider.BROWSER_SEARCH) { + // const client = new SearchClient({ + // provider: SearchProviderEnum.BrowserSearch, + // providerConfig: { + // browserOptions: { + // headless: true, + // }, + // defaultEngine: 'bing', + // }, + // }); + // results = await client.search({ + // query: args.query, + // count: args.count || 10, + // }); } else { // Only for Bing Search, because Tavily is not supported in the bundle of this packages // Error info: trvily is not defined diff --git a/apps/agent-tars/src/main/index.ts b/apps/agent-tars/src/main/index.ts index c85913286..10e682733 100644 --- a/apps/agent-tars/src/main/index.ts +++ b/apps/agent-tars/src/main/index.ts @@ -1,10 +1,28 @@ import { app, shell, BrowserWindow, ipcMain } from 'electron'; import { join } from 'path'; import { registerIpcMain } from '@ui-tars/electron-ipc/main'; +import { updateElectronApp, UpdateSourceType } from 'update-electron-app'; import { electronApp, optimizer, is } from '@electron-toolkit/utils'; import { ipcRoutes } from './ipcRoutes'; import icon from '../../resources/icon.png?asset'; +class AppUpdater { + constructor() { + // autoUpdater.logger = logger; + // autoUpdater.checkForUpdatesAndNotify(); + if (process.env.CI !== 'e2e') { + updateElectronApp({ + updateSource: { + type: UpdateSourceType.ElectronPublicUpdateService, + repo: 'bytedance/UI-TARS-desktop', + host: 'https://update.electronjs.org', + }, + updateInterval: '20 minutes', + }); + } + } +} + function createWindow(): void { // Create the browser window. const mainWindow = new BrowserWindow({ @@ -90,6 +108,8 @@ app.whenReady().then(async () => { optimizer.watchWindowShortcuts(window); }); + new AppUpdater(); + await initializeApp(); // IPC test diff --git a/apps/agent-tars/src/main/ipcRoutes/action.ts b/apps/agent-tars/src/main/ipcRoutes/action.ts index 4e33b59c1..0f3d71c6d 100644 --- a/apps/agent-tars/src/main/ipcRoutes/action.ts +++ b/apps/agent-tars/src/main/ipcRoutes/action.ts @@ -90,7 +90,9 @@ export const actionRoute = t.router({ }); } } else { + console.log('executeCustomTool_toolCall', toolCall); const result = await executeCustomTool(toolCall); + console.log('executeCustomTool_result', result); if (result) { results.push(...result); } diff --git a/apps/agent-tars/src/main/ipcRoutes/llm.ts b/apps/agent-tars/src/main/ipcRoutes/llm.ts index b645924d0..ec69ef942 100644 --- a/apps/agent-tars/src/main/ipcRoutes/llm.ts +++ b/apps/agent-tars/src/main/ipcRoutes/llm.ts @@ -62,6 +62,7 @@ export const llmRoute = t.router({ const messages = input.messages.map((msg) => new Message(msg)); const llm = createLLM(currentLLMConfigRef.current); console.log('current llm config', currentLLMConfigRef.current); + console.log('current search config', SettingStore.get('search')); console.log('input.tools', input.tools); const response = await llm.askTool({ messages, diff --git a/apps/agent-tars/src/main/store/setting.ts b/apps/agent-tars/src/main/store/setting.ts index 3760839aa..73a46da48 100644 --- a/apps/agent-tars/src/main/store/setting.ts +++ b/apps/agent-tars/src/main/store/setting.ts @@ -26,7 +26,7 @@ const DEFAULT_FILESYSTEM_SETTINGS: FileSystemSettings = { }; const DEFAULT_SEARCH_SETTINGS: SearchSettings = { - provider: SearchProvider.DUCKDUCKGO_SEARCH, + provider: SearchProvider.TAVILY, apiKey: '', }; diff --git a/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/SearchSettingsTab.tsx b/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/SearchSettingsTab.tsx index 6fbf71f2d..73caf39d5 100644 --- a/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/SearchSettingsTab.tsx +++ b/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/SearchSettingsTab.tsx @@ -24,6 +24,18 @@ export function SearchSettingsTab({ }} startContent={getSearchProviderLogo(settings.provider)} > + + Tavily Search + + {/* + Browser Search + */} Bing Search - - Tavily Search - - {settings.provider !== SearchProvider.DUCKDUCKGO_SEARCH && ( + {[SearchProvider.TAVILY, SearchProvider.BING_SEARCH].includes( + settings.provider, + ) && ( Advanced Settings (Optional)

) : null} + {/* {settings.provider === SearchProvider.BROWSER_SEARCH && ( + + )} */} + {settings.provider === SearchProvider.BING_SEARCH && ( ; case SearchProvider.DUCKDUCKGO_SEARCH: return ; + // case SearchProvider.BROWSER_SEARCH: + // return ; default: return null; } diff --git a/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/useAppSettings.ts b/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/useAppSettings.ts index 0e222dbe9..09ef8be1c 100644 --- a/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/useAppSettings.ts +++ b/apps/agent-tars/src/renderer/src/components/LeftSidebar/Settings/useAppSettings.ts @@ -24,7 +24,7 @@ const DEFAULT_FILESYSTEM_SETTINGS: FileSystemSettings = { }; const DEFAULT_SEARCH_SETTINGS: SearchSettings = { - provider: SearchProvider.DUCKDUCKGO_SEARCH, + provider: SearchProvider.TAVILY, apiKey: '', }; @@ -87,7 +87,9 @@ export function useAppSettings() { } console.log('searchSettings.provider', searchSettings.provider); if ( - searchSettings.provider !== SearchProvider.DUCKDUCKGO_SEARCH && + [SearchProvider.BING_SEARCH, SearchProvider.TAVILY].includes( + searchSettings.provider, + ) && !searchSettings.apiKey ) { return 'API Key is required'; diff --git a/apps/ui-tars/src/renderer/src/pages/settings/index.tsx b/apps/ui-tars/src/renderer/src/pages/settings/index.tsx index 2a018859b..288acc911 100644 --- a/apps/ui-tars/src/renderer/src/pages/settings/index.tsx +++ b/apps/ui-tars/src/renderer/src/pages/settings/index.tsx @@ -45,8 +45,6 @@ export default function Settings() { settings.presetSource.type === 'remote' && settings.presetSource.autoUpdate; - console.log('settings', settings); - const handleSubmit = async (values) => { updateSetting(values); console.log('values', values); diff --git a/packages/agent-infra/browser/src/base-browser.ts b/packages/agent-infra/browser/src/base-browser.ts index aae4c6a8b..2d4148cca 100644 --- a/packages/agent-infra/browser/src/base-browser.ts +++ b/packages/agent-infra/browser/src/base-browser.ts @@ -139,14 +139,18 @@ export abstract class BaseBrowser implements BrowserInterface { pageFunction, pageFunctionParams, beforePageLoad, + afterPageLoad, beforeSendResult, + waitForOptions, } = options; const page = await this.browser!.newPage(); try { await beforePageLoad?.(page); await page.goto(url, { waitUntil: 'networkidle2', + ...waitForOptions, }); + await afterPageLoad?.(page); const _window = await page.evaluateHandle(() => window); const result = await page.evaluate( pageFunction, diff --git a/packages/agent-infra/browser/src/local-browser.ts b/packages/agent-infra/browser/src/local-browser.ts index bd41fcdb4..074e414aa 100644 --- a/packages/agent-infra/browser/src/local-browser.ts +++ b/packages/agent-infra/browser/src/local-browser.ts @@ -46,6 +46,8 @@ export class LocalBrowser extends BaseBrowser { }, args: [ '--no-sandbox', + '--mute-audio', + '--disable-gpu', '--disable-http2', '--disable-blink-features=AutomationControlled', '--disable-infobars', diff --git a/packages/agent-infra/browser/src/types.ts b/packages/agent-infra/browser/src/types.ts index b24ade2ee..4771a734a 100644 --- a/packages/agent-infra/browser/src/types.ts +++ b/packages/agent-infra/browser/src/types.ts @@ -3,7 +3,7 @@ * Copyright (c) 2025 Bytedance, Inc. and its affiliates. * SPDX-License-Identifier: Apache-2.0 */ -import { Page } from 'puppeteer-core'; +import { Page, WaitForOptions } from 'puppeteer-core'; /** * Options for launching a browser instance @@ -64,6 +64,11 @@ export interface EvaluateOnNewPageOptions { */ url: string; + /** + * Options for waiting for the page to load + */ + waitForOptions?: WaitForOptions; + /** * Function to be evaluated in the page context * This function runs in the context of the browser page, not Node.js @@ -87,6 +92,14 @@ export interface EvaluateOnNewPageOptions { */ beforePageLoad?: (page: Page) => void | Promise; + /** + * Optional function to execute after page navigation + * Useful for setting up page configuration after loading the URL + * @param {Page} page - Puppeteer page instance + * @returns {void | Promise} + */ + afterPageLoad?: (page: Page) => void | Promise; + /** * Optional function to process the result before returning * Can be used to transform or validate the result from page evaluation diff --git a/packages/agent-infra/search/browser-search/examples/bing-search.ts b/packages/agent-infra/search/browser-search/examples/bing-search.ts index 02f0a7c41..3ec072eb1 100644 --- a/packages/agent-infra/search/browser-search/examples/bing-search.ts +++ b/packages/agent-infra/search/browser-search/examples/bing-search.ts @@ -12,6 +12,7 @@ export async function bingSearch() { browserOptions: { headless: false, }, + defaultEngine: 'bing', }); try { diff --git a/packages/agent-infra/search/browser-search/src/browser-search.ts b/packages/agent-infra/search/browser-search/src/browser-search.ts index a361d3cfe..2e3e0519b 100644 --- a/packages/agent-infra/search/browser-search/src/browser-search.ts +++ b/packages/agent-infra/search/browser-search/src/browser-search.ts @@ -69,6 +69,7 @@ export class BrowserSearch { visitedUrls, excludeDomains, truncate: options.truncate, + needVisitedUrls: options.needVisitedUrls, engine, }), ), @@ -102,6 +103,7 @@ export class BrowserSearch { options: { query: string; count?: number; + needVisitedUrls?: boolean; excludeDomains: string[]; queue: PromiseQueue; visitedUrls: Set; @@ -119,11 +121,17 @@ export class BrowserSearch { let links = await browser.evaluateOnNewPage({ url, + waitForOptions: { + waitUntil: 'networkidle0', + }, pageFunction: searchEngine.extractSearchResults, pageFunctionParams: [], beforePageLoad: async (page) => { await interceptRequest(page); }, + afterPageLoad: async (page) => { + await page.waitForSelector('.b_pag'); + }, }); this.logger.info('Fetched links:', links); @@ -143,9 +151,11 @@ export class BrowserSearch { // Visit each link and extract content const results = await Promise.allSettled( - links.map((item) => - options.queue.add(() => this.visitLink(this.browser, item)), - ), + options.needVisitedUrls + ? links.map((item) => + options.queue.add(() => this.visitLink(this.browser, item)), + ) + : links, ); return results diff --git a/packages/agent-infra/search/browser-search/src/engines/get-search-engine.ts b/packages/agent-infra/search/browser-search/src/engines/get-search-engine.ts new file mode 100644 index 000000000..841970a6f --- /dev/null +++ b/packages/agent-infra/search/browser-search/src/engines/get-search-engine.ts @@ -0,0 +1,23 @@ +import { GoogleSearchEngine } from './google-engine'; +import { BingSearchEngine } from './bing-engine'; +import { BaiduSearchEngine } from './baidu-engine'; +import type { SearchEngine, SearchEngineAdapter } from '../types'; + +/** + * Factory function to get the appropriate search engine adapter instance. + * + * @param engine - The search engine identifier ('google', 'bing', or 'baidu') + * @returns An instance of the requested search engine adapter + */ +export function getSearchEngine(engine: SearchEngine): SearchEngineAdapter { + switch (engine) { + case 'google': + return new GoogleSearchEngine(); + case 'bing': + return new BingSearchEngine(); + case 'baidu': + return new BaiduSearchEngine(); + default: + return new GoogleSearchEngine(); + } +} diff --git a/packages/agent-infra/search/browser-search/src/engines/index.ts b/packages/agent-infra/search/browser-search/src/engines/index.ts index 337c39cc3..af3ecaa84 100644 --- a/packages/agent-infra/search/browser-search/src/engines/index.ts +++ b/packages/agent-infra/search/browser-search/src/engines/index.ts @@ -2,30 +2,8 @@ * Copyright (c) 2025 Bytedance, Inc. and its affiliates. * SPDX-License-Identifier: Apache-2.0 */ -import { GoogleSearchEngine } from './google-engine'; -import { BingSearchEngine } from './bing-engine'; -import { BaiduSearchEngine } from './baidu-engine'; -import type { SearchEngine, SearchEngineAdapter } from '../types'; export * from './google-engine'; export * from './bing-engine'; export * from './baidu-engine'; - -/** - * Factory function to get the appropriate search engine adapter instance. - * - * @param engine - The search engine identifier ('google', 'bing', or 'baidu') - * @returns An instance of the requested search engine adapter - */ -export function getSearchEngine(engine: SearchEngine): SearchEngineAdapter { - switch (engine) { - case 'google': - return new GoogleSearchEngine(); - case 'bing': - return new BingSearchEngine(); - case 'baidu': - return new BaiduSearchEngine(); - default: - return new GoogleSearchEngine(); - } -} +export { getSearchEngine } from './get-search-engine'; diff --git a/packages/agent-infra/search/browser-search/src/types.ts b/packages/agent-infra/search/browser-search/src/types.ts index cb63a4770..77a5d5ae3 100644 --- a/packages/agent-infra/search/browser-search/src/types.ts +++ b/packages/agent-infra/search/browser-search/src/types.ts @@ -42,6 +42,11 @@ export interface BrowserSearchOptions { * Search engine to use (default: 'google') */ engine?: SearchEngine; + /** + * need visited urls + * @default false + */ + needVisitedUrls?: boolean; } export interface BrowserSearchConfig { diff --git a/packages/agent-infra/shared/src/agent-tars-types/search.ts b/packages/agent-infra/shared/src/agent-tars-types/search.ts index 5b7114305..06cfba68f 100644 --- a/packages/agent-infra/shared/src/agent-tars-types/search.ts +++ b/packages/agent-infra/shared/src/agent-tars-types/search.ts @@ -22,12 +22,14 @@ export enum SearchProvider { BING_SEARCH = 'bing_search', TAVILY = 'tavily', DUCKDUCKGO_SEARCH = 'duckduckgo_search', + BROWSER_SEARCH = 'browser_search', } export interface SearchSettings { provider: SearchProvider; apiKey: string; baseUrl?: string; + defaultEngine?: 'bing'; } export interface AppSettings {