Add better handling for native dropdowns (#140) (#147)

* Add better handling for native dropdowns (#140)

* Add better handling for native dropdowns

* Update comment

* Add role replacement

* patch and changeset

* clean up

---------

Co-authored-by: Zach Loza <Zach10za@users.noreply.github.com>

Author: miguelg719

.changeset/amusing-ivory-termite.md

@@ -0,0 +1,5 @@
+---
+"stagehand": patch
+---
+
+Add support for handling OS-level dropdowns

stagehand/a11y/utils.py

@@ -57,7 +57,7 @@ async def _clean_structural_nodes(
         page
         and logger
         and backend_node_id is not None
-        and node_role in ("generic", "none")
+        and node_role in ("generic", "combobox", "none")
     ):
         try:
             resolved_node = await page.send_cdp(
@@ -78,6 +78,8 @@ async def _clean_structural_nodes(
                     )
                     result_value = tag_name_result.get("result", {}).get("value")
                     if result_value:
+                        if node_role == "combobox" and result_value == "select":
+                            result_value = "select"
                         node["role"] = result_value
                         node_role = result_value
                 except Exception as tag_name_error:

stagehand/handlers/act_handler_utils.py

@@ -339,6 +339,27 @@ async def press_key(ctx: MethodHandlerContext) -> None:
         raise e
 
 
+async def select_option(ctx: MethodHandlerContext) -> None:
+    try:
+        text = str(ctx.args[0]) if ctx.args and ctx.args[0] is not None else ""
+        await ctx.locator.select_option(text, timeout=5_000)
+    except Exception as e:
+        ctx.logger.error(
+            message="error selecting option",
+            category="action",
+            auxiliary={
+                "error": {"value": str(e), "type": "string"},
+                "trace": {
+                    "value": getattr(e, "__traceback__", ""),
+                    "type": "string",
+                },
+                "xpath": {"value": ctx.xpath, "type": "string"},
+                "args": {"value": json.dumps(ctx.args), "type": "object"},
+            },
+        )
+        raise e
+
+
 async def click_element(ctx: MethodHandlerContext) -> None:
     ctx.logger.debug(
         message=f"page URL before click {ctx.stagehand_page._page.url}",
@@ -500,4 +521,5 @@ async def handle_possible_page_navigation(
     "click": click_element,
     "nextChunk": scroll_to_next_chunk,
     "prevChunk": scroll_to_previous_chunk,
+    "selectOptionFromDropdown": select_option,
 }

stagehand/llm/prompts.py

@@ -200,7 +200,9 @@ def build_act_observe_prompt(
 ONLY return one action. If multiple actions are relevant, return the most relevant one.
 If the user is asking to scroll to a position on the page, e.g., 'halfway' or 0.75, etc, you must return the argument formatted as the correct percentage, e.g., '50%' or '75%', etc.
 If the user is asking to scroll to the next chunk/previous chunk, choose the nextChunk/prevChunk method. No arguments are required here.
-If the action implies a key press, e.g., 'press enter', 'press a', 'press space', etc., always choose the press method with the appropriate key as argument — e.g. 'a', 'Enter', 'Space'. Do not choose a click action on an on-screen keyboard. Capitalize the first character like 'Enter', 'Tab', 'Escape' only for special keys."""
+If the action implies a key press, e.g., 'press enter', 'press a', 'press space', etc., always choose the press method with the appropriate key as argument — e.g. 'a', 'Enter', 'Space'. Do not choose a click action on an on-screen keyboard. Capitalize the first character like 'Enter', 'Tab', 'Escape' only for special keys.
+If the action implies choosing an option from a dropdown, AND the corresponding element is a 'select' element, choose the selectOptionFromDropdown method. The argument should be the text of the option to select.
+If the action implies choosing an option from a dropdown, and the corresponding element is NOT a 'select' element, choose the click method."""
 
     if variables and len(variables) > 0:
         variables_prompt = f"The following variables are available to use in the action: {', '.join(variables.keys())}. Fill the argument variables with the variable name."

tests/e2e/test_act_integration.py

@@ -109,6 +109,92 @@ async def test_form_filling_browserbase(self, browserbase_stagehand):
         assert filled_name is not None
         assert len(filled_name) > 0
 
+    @pytest.mark.asyncio
+    @pytest.mark.local
+    async def test_selecting_option_local(self, local_stagehand):
+        """Test option selecting capability in LOCAL mode"""
+        stagehand = local_stagehand
+        
+        # Navigate to a page with a form containing a dropdown
+        await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/nested-dropdown/")
+        
+        # Select an option from the dropdown.
+        await stagehand.page.act("Choose 'Smog Check Technician' from the 'License Type' dropdown")
+        
+        # Verify the selected option.
+        selected_option = await stagehand.page.locator(
+            "xpath=/html/body/form/div[1]/div[3]/article/div[2]/div[1]/select[2] >> option:checked"
+        ).text_content()
+
+        assert selected_option == "Smog Check Technician"
+
+    @pytest.mark.asyncio
+    @pytest.mark.browserbase
+    @pytest.mark.skipif(
+        not (os.getenv("BROWSERBASE_API_KEY") and os.getenv("BROWSERBASE_PROJECT_ID")),
+        reason="Browserbase credentials not available"
+    )
+    async def test_selecting_option_browserbase(self, browserbase_stagehand):
+        """Test option selecting capability in BROWSERBASE mode"""
+        stagehand = browserbase_stagehand
+        
+        # Navigate to a page with a form containing a dropdown
+        await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/nested-dropdown/")
+        
+        # Select an option from the dropdown.
+        await stagehand.page.act("Choose 'Smog Check Technician' from the 'License Type' dropdown")
+        
+        # Verify the selected option.
+        selected_option = await stagehand.page.locator(
+            "xpath=/html/body/form/div[1]/div[3]/article/div[2]/div[1]/select[2] >> option:checked"
+        ).text_content()
+
+        assert selected_option == "Smog Check Technician"
+
+    @pytest.mark.asyncio
+    @pytest.mark.local
+    async def test_selecting_option_custom_input_local(self, local_stagehand):
+        """Test not selecting option on custom select input in LOCAL mode"""
+        stagehand = local_stagehand
+        
+        # Navigate to a page with a form containing a dropdown
+        await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/expand-dropdown/")
+        
+        # Select an option from the dropdown.
+        await stagehand.page.act("Click the 'Select a Country' dropdown")
+
+        # Wait for dropdown to expand
+        await asyncio.sleep(1)
+        
+        # We are expecting stagehand to click the dropdown to expand it, and therefore
+        # the available options should now be contained in the full a11y tree.
+
+        # To test, we'll grab the full a11y tree, and make sure it contains 'Canada'
+        extraction = await stagehand.page.extract()
+        assert "Canada" in extraction.data
+
+    @pytest.mark.asyncio
+    @pytest.mark.local
+    async def test_selecting_option_hidden_input_local(self, local_stagehand):
+        """Test not selecting option on hidden input in LOCAL mode"""
+        stagehand = local_stagehand
+        
+        # Navigate to a page with a form containing a dropdown
+        await stagehand.page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/hidden-input-dropdown/")
+        
+        # Select an option from the dropdown.
+        await stagehand.page.act("Click to expand the 'Favourite Colour' dropdown")
+        
+        # Wait for dropdown to expand
+        await asyncio.sleep(1)
+        
+        # We are expecting stagehand to click the dropdown to expand it, and therefore
+        # the available options should now be contained in the full a11y tree.
+
+        # To test, we'll grab the full a11y tree, and make sure it contains 'Green'
+        extraction = await stagehand.page.extract()
+        assert "Green" in extraction.data
+
     @pytest.mark.asyncio
     @pytest.mark.local
     async def test_button_clicking_local(self, local_stagehand):