From 9e23563d2879a497bb2a01d77b79329605f97358 Mon Sep 17 00:00:00 2001 From: ycjcl868 Date: Tue, 18 Mar 2025 20:12:31 +0800 Subject: [PATCH 1/2] fix(mcp): browser navigate waitForPageAndFramesLoad --- .../browser-use/src/browser/page.ts | 233 +--------------- .../browser-use/src/browser/types.ts | 3 + .../browser-use/src/browser/utils.ts | 254 ++++++++++++++++++ packages/agent-infra/browser-use/src/index.ts | 1 + .../mcp-servers/browser/package.json | 2 +- .../mcp-servers/browser/src/server.ts | 139 ++++++++-- .../mcp-servers/filesystem/package.json | 2 +- 7 files changed, 382 insertions(+), 252 deletions(-) create mode 100644 packages/agent-infra/browser-use/src/browser/utils.ts diff --git a/packages/agent-infra/browser-use/src/browser/page.ts b/packages/agent-infra/browser-use/src/browser/page.ts index 3cc903110..5cdbace79 100644 --- a/packages/agent-infra/browser-use/src/browser/page.ts +++ b/packages/agent-infra/browser-use/src/browser/page.ts @@ -33,6 +33,7 @@ import { type PageState, } from './types'; import { createLogger, getBuildDomTreeScript } from '../utils'; +import { waitForPageAndFramesLoad } from './utils'; const logger = createLogger('Page'); @@ -1038,235 +1039,11 @@ export default class Page { return false; } - async waitForPageLoadState(timeout?: number) { - const timeoutValue = timeout || 8000; - await this._puppeteerPage?.waitForNavigation({ timeout: timeoutValue }); - } - - private async _waitForStableNetwork() { - if (!this._puppeteerPage) { - throw new Error('Puppeteer page is not connected'); - } - - const RELEVANT_RESOURCE_TYPES = new Set([ - 'document', - 'stylesheet', - 'image', - 'font', - 'script', - 'iframe', - ]); - - const RELEVANT_CONTENT_TYPES = new Set([ - 'text/html', - 'text/css', - 'application/javascript', - 'image/', - 'font/', - 'application/json', - ]); - - const IGNORED_URL_PATTERNS = new Set([ - // Analytics and tracking - 'analytics', - 'tracking', - 'telemetry', - 'beacon', - 'metrics', - // Ad-related - 'doubleclick', - 'adsystem', - 'adserver', - 'advertising', - // Social media widgets - 'facebook.com/plugins', - 'platform.twitter', - 'linkedin.com/embed', - // Live chat and support - 'livechat', - 'zendesk', - 'intercom', - 'crisp.chat', - 'hotjar', - // Push notifications - 'push-notifications', - 'onesignal', - 'pushwoosh', - // Background sync/heartbeat - 'heartbeat', - 'ping', - 'alive', - // WebRTC and streaming - 'webrtc', - 'rtmp://', - 'wss://', - // Common CDNs - 'cloudfront.net', - 'fastly.net', - ]); - - const pendingRequests = new Set(); - let lastActivity = Date.now(); - - const onRequest = (request: HTTPRequest) => { - // Filter by resource type - const resourceType = request.resourceType(); - if (!RELEVANT_RESOURCE_TYPES.has(resourceType)) { - return; - } - - // Filter out streaming, websocket, and other real-time requests - if ( - ['websocket', 'media', 'eventsource', 'manifest', 'other'].includes( - resourceType, - ) - ) { - return; - } - - // Filter out by URL patterns - const url = request.url().toLowerCase(); - if ( - Array.from(IGNORED_URL_PATTERNS).some((pattern) => - url.includes(pattern), - ) - ) { - return; - } - - // Filter out data URLs and blob URLs - if (url.startsWith('data:') || url.startsWith('blob:')) { - return; - } - - // Filter out requests with certain headers - const headers = request.headers(); - if ( - // biome-ignore lint/complexity/useLiteralKeys: - headers['purpose'] === 'prefetch' || - headers['sec-fetch-dest'] === 'video' || - headers['sec-fetch-dest'] === 'audio' - ) { - return; - } - - pendingRequests.add(request); - lastActivity = Date.now(); - }; - - const onResponse = (response: HTTPResponse) => { - const request = response.request(); - if (!pendingRequests.has(request)) { - return; - } - - // Filter by content type - const contentType = - response.headers()['content-type']?.toLowerCase() || ''; - - // Skip streaming content - if ( - [ - 'streaming', - 'video', - 'audio', - 'webm', - 'mp4', - 'event-stream', - 'websocket', - 'protobuf', - ].some((t) => contentType.includes(t)) - ) { - pendingRequests.delete(request); - return; - } - - // Only process relevant content types - if ( - !Array.from(RELEVANT_CONTENT_TYPES).some((ct) => - contentType.includes(ct), - ) - ) { - pendingRequests.delete(request); - return; - } - - // Skip large responses - const contentLength = response.headers()['content-length']; - if (contentLength && Number.parseInt(contentLength) > 5 * 1024 * 1024) { - // 5MB - pendingRequests.delete(request); - return; - } - - pendingRequests.delete(request); - lastActivity = Date.now(); - }; - - // Add event listeners - this._puppeteerPage.on('request', onRequest as any); - this._puppeteerPage.on('response', onResponse as any); - - try { - const startTime = Date.now(); - - // eslint-disable-next-line no-constant-condition - while (true) { - await new Promise((resolve) => setTimeout(resolve, 100)); - - const now = Date.now(); - const timeSinceLastActivity = (now - lastActivity) / 1000; // Convert to seconds - - if ( - pendingRequests.size === 0 && - timeSinceLastActivity >= this._config.waitForNetworkIdlePageLoadTime - ) { - break; - } - - const elapsedTime = (now - startTime) / 1000; // Convert to seconds - if (elapsedTime > this._config.maximumWaitPageLoadTime) { - console.debug( - `Network timeout after ${this._config.maximumWaitPageLoadTime}s with ${pendingRequests.size} pending requests:`, - Array.from(pendingRequests).map((r) => (r as HTTPRequest).url()), - ); - break; - } - } - } finally { - // Clean up event listeners - this._puppeteerPage.off('request', onRequest as any); - this._puppeteerPage.off('response', onResponse as any); - } - console.debug( - `Network stabilized for ${this._config.waitForNetworkIdlePageLoadTime} seconds`, - ); - } - async waitForPageAndFramesLoad(timeoutOverwrite?: number): Promise { - // Start timing - const startTime = Date.now(); - - // Wait for page load - try { - await this._waitForStableNetwork(); - } catch (error) { - console.warn('Page load failed, continuing...'); - } - - // Calculate remaining time to meet minimum wait time - const elapsed = (Date.now() - startTime) / 1000; // Convert to seconds - const minWaitTime = - timeoutOverwrite || this._config.minimumWaitPageLoadTime; - const remaining = Math.max(minWaitTime - elapsed, 0); - - console.debug( - `--Page loaded in ${elapsed.toFixed(2)} seconds, waiting for additional ${remaining.toFixed(2)} seconds`, + await waitForPageAndFramesLoad( + this._puppeteerPage, + timeoutOverwrite, + this._config, ); - - // Sleep remaining time if needed - if (remaining > 0) { - await new Promise((resolve) => setTimeout(resolve, remaining * 1000)); // Convert seconds to milliseconds - } } } diff --git a/packages/agent-infra/browser-use/src/browser/types.ts b/packages/agent-infra/browser-use/src/browser/types.ts index 600107003..012bf4f09 100644 --- a/packages/agent-infra/browser-use/src/browser/types.ts +++ b/packages/agent-infra/browser-use/src/browser/types.ts @@ -8,6 +8,9 @@ */ import type { DOMState } from '../dom/views'; +export type PartialWithRequired = Required> & + Partial>; + export interface BrowserContextWindowSize { width: number; height: number; diff --git a/packages/agent-infra/browser-use/src/browser/utils.ts b/packages/agent-infra/browser-use/src/browser/utils.ts new file mode 100644 index 000000000..b739163e6 --- /dev/null +++ b/packages/agent-infra/browser-use/src/browser/utils.ts @@ -0,0 +1,254 @@ +import { + type HTTPRequest, + type HTTPResponse, +} from 'puppeteer-core/lib/esm/puppeteer/puppeteer-core-browser.js'; +import { Page as PuppeteerPage } from 'puppeteer-core'; +import { + BrowserContextConfig, + DEFAULT_BROWSER_CONTEXT_CONFIG, + PartialWithRequired, +} from './types'; + +export async function waitForStableNetwork( + page: PuppeteerPage | null, + _options?: Partial, +) { + const options = { + ...DEFAULT_BROWSER_CONTEXT_CONFIG, + ..._options, + }; + if (!page) { + throw new Error('Puppeteer page is not connected'); + } + + const RELEVANT_RESOURCE_TYPES = new Set([ + 'document', + 'stylesheet', + 'image', + 'font', + 'script', + 'iframe', + ]); + + const RELEVANT_CONTENT_TYPES = new Set([ + 'text/html', + 'text/css', + 'application/javascript', + 'image/', + 'font/', + 'application/json', + ]); + + const IGNORED_URL_PATTERNS = new Set([ + // Analytics and tracking + 'analytics', + 'tracking', + 'telemetry', + 'beacon', + 'metrics', + // Ad-related + 'doubleclick', + 'adsystem', + 'adserver', + 'advertising', + // Social media widgets + 'facebook.com/plugins', + 'platform.twitter', + 'linkedin.com/embed', + // Live chat and support + 'livechat', + 'zendesk', + 'intercom', + 'crisp.chat', + 'hotjar', + // Push notifications + 'push-notifications', + 'onesignal', + 'pushwoosh', + // Background sync/heartbeat + 'heartbeat', + 'ping', + 'alive', + // WebRTC and streaming + 'webrtc', + 'rtmp://', + 'wss://', + // Common CDNs + 'cloudfront.net', + 'fastly.net', + ]); + + const pendingRequests = new Set(); + let lastActivity = Date.now(); + + const onRequest = (request: HTTPRequest) => { + // Filter by resource type + const resourceType = request.resourceType(); + if (!RELEVANT_RESOURCE_TYPES.has(resourceType)) { + return; + } + + // Filter out streaming, websocket, and other real-time requests + if ( + ['websocket', 'media', 'eventsource', 'manifest', 'other'].includes( + resourceType, + ) + ) { + return; + } + + // Filter out by URL patterns + const url = request.url().toLowerCase(); + if ( + Array.from(IGNORED_URL_PATTERNS).some((pattern) => url.includes(pattern)) + ) { + return; + } + + // Filter out data URLs and blob URLs + if (url.startsWith('data:') || url.startsWith('blob:')) { + return; + } + + // Filter out requests with certain headers + const headers = request.headers(); + if ( + // biome-ignore lint/complexity/useLiteralKeys: + headers['purpose'] === 'prefetch' || + headers['sec-fetch-dest'] === 'video' || + headers['sec-fetch-dest'] === 'audio' + ) { + return; + } + + pendingRequests.add(request); + lastActivity = Date.now(); + }; + + const onResponse = (response: HTTPResponse) => { + const request = response.request(); + if (!pendingRequests.has(request)) { + return; + } + + // Filter by content type + const contentType = response.headers()['content-type']?.toLowerCase() || ''; + + // Skip streaming content + if ( + [ + 'streaming', + 'video', + 'audio', + 'webm', + 'mp4', + 'event-stream', + 'websocket', + 'protobuf', + ].some((t) => contentType.includes(t)) + ) { + pendingRequests.delete(request); + return; + } + + // Only process relevant content types + if ( + !Array.from(RELEVANT_CONTENT_TYPES).some((ct) => contentType.includes(ct)) + ) { + pendingRequests.delete(request); + return; + } + + // Skip large responses + const contentLength = response.headers()['content-length']; + if (contentLength && Number.parseInt(contentLength) > 5 * 1024 * 1024) { + // 5MB + pendingRequests.delete(request); + return; + } + + pendingRequests.delete(request); + lastActivity = Date.now(); + }; + + // Add event listeners + page.on('request', onRequest as any); + page.on('response', onResponse as any); + + try { + const startTime = Date.now(); + + // eslint-disable-next-line no-constant-condition + while (true) { + await new Promise((resolve) => setTimeout(resolve, 100)); + + const now = Date.now(); + const timeSinceLastActivity = (now - lastActivity) / 1000; // Convert to seconds + + if ( + pendingRequests.size === 0 && + timeSinceLastActivity >= options.waitForNetworkIdlePageLoadTime + ) { + break; + } + + const elapsedTime = (now - startTime) / 1000; // Convert to seconds + if (elapsedTime > options.maximumWaitPageLoadTime) { + console.debug( + `Network timeout after ${options.maximumWaitPageLoadTime}s with ${pendingRequests.size} pending requests:`, + Array.from(pendingRequests).map((r) => (r as HTTPRequest).url()), + ); + break; + } + } + } finally { + // Clean up event listeners + page.off('request', onRequest as any); + page.off('response', onResponse as any); + } + console.debug( + `Network stabilized for ${options.waitForNetworkIdlePageLoadTime} seconds`, + ); +} + +export async function waitForPageAndFramesLoad( + page: PuppeteerPage | null, + timeoutOverwrite?: number, + _options?: Partial, +): Promise { + const options = { + ...DEFAULT_BROWSER_CONTEXT_CONFIG, + ..._options, + }; + // Start timing + const startTime = Date.now(); + + // Wait for page load + try { + await waitForStableNetwork(page, options); + } catch (error) { + console.warn('Page load failed, continuing...'); + } + + // Calculate remaining time to meet minimum wait time + const elapsed = (Date.now() - startTime) / 1000; // Convert to seconds + const minWaitTime = timeoutOverwrite || options.minimumWaitPageLoadTime; + const remaining = Math.max(minWaitTime - elapsed, 0); + + console.debug( + `--Page loaded in ${elapsed.toFixed(2)} seconds, waiting for additional ${remaining.toFixed(2)} seconds`, + ); + + // Sleep remaining time if needed + if (remaining > 0) { + await new Promise((resolve) => setTimeout(resolve, remaining * 1000)); // Convert seconds to milliseconds + } +} + +export async function waitForPageLoadState( + page: PuppeteerPage, + timeout?: number, +) { + const timeoutValue = timeout || 8000; + await page?.waitForNavigation({ timeout: timeoutValue }); +} diff --git a/packages/agent-infra/browser-use/src/index.ts b/packages/agent-infra/browser-use/src/index.ts index fc340d370..7658c438c 100644 --- a/packages/agent-infra/browser-use/src/index.ts +++ b/packages/agent-infra/browser-use/src/index.ts @@ -7,3 +7,4 @@ export { getBuildDomTreeScript } from './utils'; export { createSelectorMap, parseNode, removeHighlights } from './dom/service'; export type { RawDomTreeNode } from './dom/raw_types'; export { DOMElementNode } from './dom/views'; +export * from './browser/utils'; diff --git a/packages/agent-infra/mcp-servers/browser/package.json b/packages/agent-infra/mcp-servers/browser/package.json index eb039ce8c..3b04f8ee2 100644 --- a/packages/agent-infra/mcp-servers/browser/package.json +++ b/packages/agent-infra/mcp-servers/browser/package.json @@ -17,7 +17,7 @@ ], "scripts": { "build": "rm -rf dist && rslib build && shx chmod +x dist/*.{js,cjs}", - "dev": "npx -y @modelcontextprotocol/inspector tsx index.ts", + "dev": "npx -y @modelcontextprotocol/inspector tsx src/index.ts", "prepare": "npm run build", "watch": "rslib build --watch" }, diff --git a/packages/agent-infra/mcp-servers/browser/src/server.ts b/packages/agent-infra/mcp-servers/browser/src/server.ts index 79f9aa54a..cd30e37f0 100644 --- a/packages/agent-infra/mcp-servers/browser/src/server.ts +++ b/packages/agent-infra/mcp-servers/browser/src/server.ts @@ -22,6 +22,7 @@ import { DOMElementNode, createSelectorMap, removeHighlights, + waitForPageAndFramesLoad, } from '@agent-infra/browser-use'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import TurndownService from 'turndown'; @@ -378,32 +379,122 @@ const handleToolCall: Client['callTool'] = async ({ [K in ToolNames]: (args: ToolInputMap[K]) => Promise; } = { browser_go_back: async (args) => { - await page.goBack(); - return { - content: [{ type: 'text', text: 'Navigated back' }], - isError: false, - }; + try { + await Promise.all([waitForPageAndFramesLoad(page), page.goBack()]); + logger.info('Navigation back completed'); + return { + content: [{ type: 'text', text: 'Navigated back' }], + isError: false, + }; + } catch (error) { + if (error instanceof Error && error.message.includes('timeout')) { + logger.warn( + 'Back navigation timeout, but page might still be usable:', + error, + ); + return { + content: [ + { + type: 'text', + text: 'Back navigation timeout, but page might still be usable:', + }, + ], + isError: false, + }; + } else { + logger.error('Could not navigate back:', error); + return { + content: [ + { + type: 'text', + text: 'Could not navigate back', + }, + ], + isError: true, + }; + } + } }, browser_go_forward: async (args) => { - await page.goForward(); - return { - content: [{ type: 'text', text: 'Navigated forward' }], - isError: false, - }; + try { + await Promise.all([waitForPageAndFramesLoad(page), page.goForward()]); + logger.info('Navigation back completed'); + return { + content: [{ type: 'text', text: 'Navigated forward' }], + isError: false, + }; + } catch (error) { + if (error instanceof Error && error.message.includes('timeout')) { + logger.warn( + 'forward navigation timeout, but page might still be usable:', + error, + ); + return { + content: [ + { + type: 'text', + text: 'forward navigation timeout, but page might still be usable:', + }, + ], + isError: false, + }; + } else { + logger.error('Could not navigate forward:', error); + return { + content: [ + { + type: 'text', + text: 'Could not navigate forward', + }, + ], + isError: true, + }; + } + } }, browser_navigate: async (args) => { + try { + await Promise.all([ + waitForPageAndFramesLoad(page), + page.goto(args.url), + ]); + logger.info('navigateTo complete'); + const { clickableElements } = (await buildDomTree(page)) || {}; + return { + content: [ + { + type: 'text', + text: `Navigated to ${args.url}\nclickable elements: ${clickableElements}`, + }, + ], + isError: false, + }; + } catch (error) { + // Check if it's a timeout error + if (error instanceof Error && error.message.includes('timeout')) { + logger.warn( + 'Navigation timeout, but page might still be usable:', + error, + ); + // You might want to check if the page is actually loaded despite the timeout + return { + content: [ + { + type: 'text', + text: 'Navigation timeout, but page might still be usable:', + }, + ], + isError: false, + }; + } else { + logger.error('NavigationTo failed:', error); + return { + content: [{ type: 'text', text: 'Navigation failed' }], + isError: true, + }; + } + } // need to wait for the page to load - await page.goto(args.url); - const { clickableElements } = (await buildDomTree(page)) || {}; - return { - content: [ - { - type: 'text', - text: `Navigated to ${args.url}\nclickable elements: ${clickableElements}`, - }, - ], - isError: false, - }; }, browser_screenshot: async (args) => { // if highlight is true, build the dom tree with highlights @@ -745,7 +836,11 @@ const handleToolCall: Client['callTool'] = async ({ try { const scrollResult = await page.evaluate((amount) => { const beforeScrollY = window.scrollY; - window.scrollBy(0, amount); + if (amount) { + window.scrollBy(0, amount); + } else { + window.scrollBy(0, window.innerHeight); + } // check if the page is scrolled the expected distance const actualScroll = window.scrollY - beforeScrollY; diff --git a/packages/agent-infra/mcp-servers/filesystem/package.json b/packages/agent-infra/mcp-servers/filesystem/package.json index 0c279fe37..c51865db6 100644 --- a/packages/agent-infra/mcp-servers/filesystem/package.json +++ b/packages/agent-infra/mcp-servers/filesystem/package.json @@ -15,7 +15,7 @@ ], "scripts": { "build": "rm -rf dist && rslib build && shx chmod +x dist/*.js", - "dev": "npx -y @modelcontextprotocol/inspector tsx index.ts", + "dev": "npx -y @modelcontextprotocol/inspector tsx src/index.ts", "prepare": "npm run build", "watch": "rslib build --watch" }, From 2ffc4225b86f9c594a53cc0c54ea07e8f83737d0 Mon Sep 17 00:00:00 2001 From: ycjcl868 Date: Tue, 18 Mar 2025 20:15:04 +0800 Subject: [PATCH 2/2] chore: publish mac before build reporter --- apps/omega/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/omega/package.json b/apps/omega/package.json index 1309bea19..96694ef6e 100644 --- a/apps/omega/package.json +++ b/apps/omega/package.json @@ -16,7 +16,7 @@ "package": "electron-forge package", "build": "rimraf dist out && npm run typecheck && npm run build:reporter && electron-vite build && electron-forge make", "test": "vitest run", - "publish:mac": "electron-vite build && electron-forge publish --arch=universal --platform=darwin", + "publish:mac": "npm run build:reporter && electron-vite build && electron-forge publish --arch=universal --platform=darwin", "build:reporter": "vite build" }, "peerDependencies": {