feat: add support for Google LLM provider and update environment configuration

Author: gurvinder-dhillon

example.env

@@ -1,2 +1,4 @@
+LLM_PROVIDER=google
 OPENAI_API_KEY=OPENAI_API_KEY
-LLM_MODEL=gpt-4o-mini
+AUTOBROWSE_LLM_MODEL=gpt-4o-mini
+GOOGLE_API_KEY=GOOGLE_API_KEY

package-lock.json

@@ -39,6 +39,7 @@
   "license": "MIT",
   "dependencies": {
     "@langchain/core": "^0.3.43",
+    "@langchain/google-genai": "^0.2.1",
     "@langchain/langgraph": "^0.2.60",
     "@langchain/openai": "^0.5.0",
     "@modelcontextprotocol/sdk": "^1.8.0",
@@ -48,8 +49,8 @@
     "zod": "^3.24.2"
   },
   "devDependencies": {
-    "@types/node": "^20.0.0",
     "@types/dotenv": "^6.1.1",
+    "@types/node": "^20.0.0",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",
     "eslint": "^8.0.0",

package.json

@@ -1,26 +1,16 @@
 import { test as base } from '@playwright/test';
 import { AutoConfig } from './types';
 import { sessionManager, context } from './browser';
-import { ChatOpenAI } from "@langchain/openai";
 import { createReactAgent } from "@langchain/langgraph/prebuilt";
 import { HumanMessage } from "@langchain/core/messages";
-import dotenv from 'dotenv';
+import { createLLMModel } from './llm';
 import {
     browser_click, browser_type, browser_get_text, browser_navigate, browser_snapshot,
     browser_hover, browser_drag, browser_select_option, browser_take_screenshot,
     browser_go_back, browser_wait, browser_press_key, browser_save_pdf, browser_choose_file,
     browser_go_forward, browser_assert
 } from './tools';
 
-// Load environment variables
-dotenv.config();
-const openai_llm_model = process.env.LLM_MODEL || 'gpt-4o-mini';
-
-const openai_model = new ChatOpenAI({
-    modelName: openai_llm_model,
-    temperature: 0,
-});
-
 // Extend base test to automatically track page
 export const test = base.extend({
     page: async ({ page }, use) => {
@@ -31,7 +21,7 @@ export const test = base.extend({
 
 // Initialize the LangChain agent with more detailed instructions
 const initializeAgent = () => {
-    const model = openai_model;
+    const model = createLLMModel();
 
     const prompt =
         `You are a web automation assistant. When given a natural language instruction:
@@ -67,8 +57,6 @@ const initializeAgent = () => {
     return { agent };
 };
 
-
-
 // Main auto function that processes instructions
 export async function auto(instruction: string, config?: AutoConfig): Promise<any> {
     console.log(`[Auto] Processing instruction: "${instruction}"`);

src/auto.ts

@@ -0,0 +1,23 @@
+import { ChatOpenAI } from "@langchain/openai";
+import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+export function createLLMModel() {
+    const provider = process.env.LLM_PROVIDER || 'openai';
+    const model = process.env.AUTOBROWSE_LLM_MODEL || 'gpt-4o-mini';
+
+    if (provider === 'google')
+    {
+        return new ChatGoogleGenerativeAI({
+            model: model
+        });
+    }
+
+    return new ChatOpenAI({
+        modelName: model,
+        temperature: 0,
+    });
+}

src/llm.ts

@@ -5,8 +5,11 @@ import { context } from '../browser/context';
 
 /**
  * Schema for going back in browser history
+ * Includes dummy property to satisfy Gemini's API requirement for non-empty object properties
  */
-const goBackSchema = z.object({});
+const goBackSchema = z.object({
+    _: z.string().optional().describe('No parameters required for this operation')
+});
 
 export const browser_go_back = tool(
     async () => {

src/tools/browser_go_back.ts

@@ -4,39 +4,42 @@ import { runAndWait } from './utils';
 import { context } from '../browser/context';
 
 /**
- * Schema for going back in browser history
+ * Schema for going forward in browser history
+ * Includes dummy property to satisfy Gemini's API requirement for non-empty object properties
  */
-const goBackSchema = z.object({});
+const goForwardSchema = z.object({
+    _: z.string().optional().describe('No parameters required for this operation')
+});
 
 export const browser_go_forward = tool(
     async () => {
         try
         {
-            console.log(`[Go Back Tool] Starting operation`);
+            console.log(`[Go Forward Tool] Starting operation`);
 
             const result = await runAndWait(
                 context,
-                'Navigated back',
+                'Navigated forward',
                 async (page) => {
-                    console.log(`[Go Back Tool] Going back to previous page`);
+                    console.log(`[Go Forward Tool] Going forward to next page`);
                     await page.goForward();
-                    console.log(`[Go Back Tool] Operation successful`);
+                    console.log(`[Go Forward Tool] Operation successful`);
                 },
                 true
             );
 
-            console.log(`[Go Back Tool] Operation completed`);
+            console.log(`[Go Forward Tool] Operation completed`);
             return result;
         } catch (error)
         {
-            const errorMessage = `Failed to go back: ${error instanceof Error ? error.message : 'Unknown error'}`;
-            console.error(`[Go Back Tool] Error:`, errorMessage);
+            const errorMessage = `Failed to go forward: ${error instanceof Error ? error.message : 'Unknown error'}`;
+            console.error(`[Go Forward Tool] Error:`, errorMessage);
             return errorMessage;
         }
     },
     {
-        name: "goBack",
-        description: "Go back to the previous page",
-        schema: goBackSchema
+        name: "goForward",
+        description: "Go forward to the next page",
+        schema: goForwardSchema
     }
 );

src/tools/browser_go_forward.ts

@@ -5,9 +5,12 @@ import os from 'os';
 import path from 'path';
 
 /**
- * Schema for saving page as PDF
+ * Schema for saving PDF
+ * Includes dummy property to satisfy Gemini's API requirement for non-empty object properties
  */
-const pdfSchema = z.object({});
+const pdfSchema = z.object({
+    _: z.string().optional().describe('No parameters required for this operation')
+});
 
 export const browser_save_pdf = tool(
     async () => {

src/tools/browser_save_pdf.ts

@@ -3,7 +3,12 @@ import { z } from "zod";
 import { context } from '../browser/context';
 import { captureAriaSnapshot } from './utils';
 
-const snapshotSchema = z.object({});
+/**
+ * Schema with dummy property to satisfy Gemini's API requirement for non-empty object properties
+ */
+const snapshotSchema = z.object({
+    _: z.string().optional().describe('No parameters required for this operation')
+});
 
 export const browser_snapshot = tool(
     async () => {

src/tools/browser_snapshot.ts

@@ -10,7 +10,7 @@ const typeSchema = z.object({
     element: z.string().describe('Human-readable element description for the target field'),
     ref: z.string().describe('Element reference from page snapshot to locate the field'),
     text: z.string().describe('The text to type into the element'),
-    submit: z.boolean().optional().describe('Whether to submit by pressing Enter after typing').default(false)
+    submit: z.boolean().optional().describe('Whether to submit by pressing Enter after typing')
 });
 
 export const browser_type = tool(