Skip to content

Add better handling for native dropdowns #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions stagehand/handlers/act_handler_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,27 @@ async def press_key(ctx: MethodHandlerContext) -> None:
raise e


async def select_option(ctx: MethodHandlerContext) -> None:
try:
text = str(ctx.args[0]) if ctx.args and ctx.args[0] is not None else ""
await ctx.locator.select_option(text, timeout=5_000)
except Exception as e:
ctx.logger.error(
message="error selecting option",
category="action",
auxiliary={
"error": {"value": str(e), "type": "string"},
"trace": {
"value": getattr(e, "__traceback__", ""),
"type": "string",
},
"xpath": {"value": ctx.xpath, "type": "string"},
"args": {"value": json.dumps(ctx.args), "type": "object"},
},
)
raise e


async def click_element(ctx: MethodHandlerContext) -> None:
ctx.logger.debug(
message=f"page URL before click {ctx.stagehand_page._page.url}",
Expand Down Expand Up @@ -500,4 +521,5 @@ async def handle_possible_page_navigation(
"click": click_element,
"nextChunk": scroll_to_next_chunk,
"prevChunk": scroll_to_previous_chunk,
"selectOptionFromDropdown": select_option,
}
4 changes: 3 additions & 1 deletion stagehand/llm/prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,9 @@ def build_act_observe_prompt(
ONLY return one action. If multiple actions are relevant, return the most relevant one.
If the user is asking to scroll to a position on the page, e.g., 'halfway' or 0.75, etc, you must return the argument formatted as the correct percentage, e.g., '50%' or '75%', etc.
If the user is asking to scroll to the next chunk/previous chunk, choose the nextChunk/prevChunk method. No arguments are required here.
If the action implies a key press, e.g., 'press enter', 'press a', 'press space', etc., always choose the press method with the appropriate key as argument — e.g. 'a', 'Enter', 'Space'. Do not choose a click action on an on-screen keyboard. Capitalize the first character like 'Enter', 'Tab', 'Escape' only for special keys."""
If the action implies a key press, e.g., 'press enter', 'press a', 'press space', etc., always choose the press method with the appropriate key as argument — e.g. 'a', 'Enter', 'Space'. Do not choose a click action on an on-screen keyboard. Capitalize the first character like 'Enter', 'Tab', 'Escape' only for special keys.
If the action implies choosing an option from a dropdown, AND the corresponding element is a 'select' element, choose the selectOptionFromDropdown method. The argument should be the text of the option to select.
If the action implies choosing an option from a dropdown, and the corresponding element is NOT a 'select' element, choose the click method."""

if variables and len(variables) > 0:
variables_prompt = f"The following variables are available to use in the action: {', '.join(variables.keys())}. Fill the argument variables with the variable name."
Expand Down
86 changes: 86 additions & 0 deletions tests/e2e/test_act_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,92 @@ async def test_form_filling_browserbase(self, browserbase_stagehand):
assert filled_name is not None
assert len(filled_name) > 0

@pytest.mark.asyncio
@pytest.mark.local
async def test_selecting_option_local(self, local_stagehand):
"""Test option selecting capability in LOCAL mode"""
stagehand = local_stagehand

# Navigate to a page with a form containing a dropdown
await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/nested-dropdown/")

# Select an option from the dropdown.
await stagehand.page.act("Choose 'Smog Check Technician' from the 'License Type' dropdown")

# Verify the selected option.
selected_option = await stagehand.page.locator(
"xpath=/html/body/form/div[1]/div[3]/article/div[2]/div[1]/select[2] >> option:checked"
).text_content()

assert selected_option == "Smog Check Technician"

@pytest.mark.asyncio
@pytest.mark.browserbase
@pytest.mark.skipif(
not (os.getenv("BROWSERBASE_API_KEY") and os.getenv("BROWSERBASE_PROJECT_ID")),
reason="Browserbase credentials not available"
)
async def test_selecting_option_browserbase(self, browserbase_stagehand):
"""Test option selecting capability in BROWSERBASE mode"""
stagehand = browserbase_stagehand

# Navigate to a page with a form containing a dropdown
await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/nested-dropdown/")

# Select an option from the dropdown.
await stagehand.page.act("Choose 'Smog Check Technician' from the 'License Type' dropdown")

# Verify the selected option.
selected_option = await stagehand.page.locator(
"xpath=/html/body/form/div[1]/div[3]/article/div[2]/div[1]/select[2] >> option:checked"
).text_content()

assert selected_option == "Smog Check Technician"

@pytest.mark.asyncio
@pytest.mark.local
async def test_selecting_option_custom_input_local(self, local_stagehand):
"""Test not selecting option on custom select input in LOCAL mode"""
stagehand = local_stagehand

# Navigate to a page with a form containing a dropdown
await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/expand-dropdown/")

# Select an option from the dropdown.
await stagehand.page.act("Click the 'Select a Country' dropdown")

# Wait for dropdown to expand
await asyncio.sleep(1)

# We are expecting stagehand to click the dropdown to expand it, and therefore
# the available options should now be contained in the full a11y tree.

# To test, we'll grab the full a11y tree, and make sure it contains 'Canada'
extraction = await stagehand.page.extract()
assert "Canada" in extraction.data

@pytest.mark.asyncio
@pytest.mark.local
async def test_selecting_option_hidden_input_local(self, local_stagehand):
"""Test not selecting option on hidden input in LOCAL mode"""
stagehand = local_stagehand

# Navigate to a page with a form containing a dropdown
await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/hidden-input-dropdown/")

# Select an option from the dropdown.
await stagehand.page.act("Click to expand the 'Favourite Colour' dropdown")

# Wait for dropdown to expand
await asyncio.sleep(1)

# We are expecting stagehand to click the dropdown to expand it, and therefore
# the available options should now be contained in the full a11y tree.

# To test, we'll grab the full a11y tree, and make sure it contains 'Green'
extraction = await stagehand.page.extract()
assert "Green" in extraction.data

@pytest.mark.asyncio
@pytest.mark.local
async def test_button_clicking_local(self, local_stagehand):
Expand Down