feat: get screenshots and save locally

Author: lynchee-owo

benchmark.py

@@ -1,3 +1,4 @@
+import base64
 import os
 import json
 import time
@@ -49,6 +50,27 @@ def history():
         json.dump(task_history, file, indent=4)
     return jsonify({"message": "Got history"}), 200
 
+@app.route('/screenshot/', methods=['POST'])
+def save_screenshot():
+    global task_id
+    # Decode the image data
+    image_data = request.json.get('value', '')
+    image_bytes = base64.b64decode(image_data)
+
+    # Ensure the directory exists
+    task_dir = os.path.join(results_dir, f"test{task_id}")
+    os.makedirs(task_dir, exist_ok=True)
+
+    # Create a unique filename for each screenshot
+    timestamp = int(time.time())
+    file_path = os.path.join(task_dir, f'screenshot_{timestamp}.png')
+    
+    # Save the screenshot to the file
+    with open(file_path, 'wb') as file:
+        file.write(image_bytes)
+    
+    return jsonify({"message": f"Screenshot saved to {file_path}"}), 200
+
 def run_server():
     app.run(port=5000, debug=False, use_reloader=False)
 

results/testGoogle--0/interact_messages.json

@@ -1,9 +1,9 @@
 [
     {
-        "prompt": "The user requests the following task:\n\n  Search for bananas\n\n\nCurrent time: 5/8/2024, 12:39:03 PM\nCurrent URL: https://www.google.com/\nCurrent page scrolling position: 100.0%\n\n\nUse the following data as a reference of the annotated elements (using `===` as a delimiter between each annotation):\n\nlabel = 1\nname = About\ntagName = A\n===\nlabel = 2\nname = Store\ntagName = A\n===\nlabel = 3\nname = Gmail\ntagName = A\n===\nlabel = 4\nname = Search for Images\ntagName = A\n===\nlabel = 5\nname = Google apps\ntagName = A\nrole = button\n===\nlabel = 6\nname = Sign in\ntagName = A\n===\nlabel = 7\nname = Search\ntagName = TEXTAREA\ncurrentValue = \nplaceholder = \nrole = combobox\n===\nlabel = 8\nname = Search by voice\ntagName = DIV\nrole = button\n===\nlabel = 9\nname = Search by image\ntagName = DIV\nrole = button\n===\nlabel = 10\nname = Google Search\ntagName = INPUT\ncurrentValue = Google Search\nplaceholder = \nrole = button\n===\nlabel = 11\nname = I'm Feeling Lucky\ntagName = INPUT\ncurrentValue = I'm Feeling Lucky\nplaceholder = \nrole = button\n===\nlabel = 12\nname = Advertising\ntagName = A\n===\nlabel = 13\nname = Business\ntagName = A\n===\nlabel = 14\nname = How Search works\ntagName = A\n===\nlabel = 15\nname = Our third decade of climate action: join us\ntagName = A\n===\nlabel = 16\nname = Privacy\ntagName = A\n===\nlabel = 17\nname = Terms\ntagName = A\n===\nlabel = 18\nname = Settings\ntagName = DIV\nrole = button\n\nThis textarea currently has focus:\nlabel = 7\nname = Search\ntagName = TEXTAREA\ncurrentValue = \nplaceholder = \nrole = combobox\n",
-        "response": "{\n  \"thought\": \"I am setting the value 'bananas' in the search field and pressing ENTER to perform the search.\",\n  \"action\": {\n    \"name\": \"setValueAndEnter\",\n    \"args\": {\n      \"label\": \"7\",\n      \"value\": \"bananas\"\n    }\n  }\n}",
+        "prompt": "The user requests the following task:\n\n  Search for bananas\n\n\nCurrent time: 5/8/2024, 2:18:03 PM\nCurrent URL: https://www.google.com/\nCurrent page scrolling position: 100.0%\n\n\nUse the following data as a reference of the annotated elements (using `===` as a delimiter between each annotation):\n\nlabel = 1\nname = About\ntagName = A\n===\nlabel = 2\nname = Store\ntagName = A\n===\nlabel = 3\nname = Gmail\ntagName = A\n===\nlabel = 4\nname = Search for Images\ntagName = A\n===\nlabel = 5\nname = Google apps\ntagName = A\nrole = button\n===\nlabel = 6\nname = Sign in\ntagName = A\n===\nlabel = 7\nname = Search\ntagName = TEXTAREA\ncurrentValue = \nplaceholder = \nrole = combobox\n===\nlabel = 8\nname = Search by voice\ntagName = DIV\nrole = button\n===\nlabel = 9\nname = Search by image\ntagName = DIV\nrole = button\n===\nlabel = 10\nname = Google Search\ntagName = INPUT\ncurrentValue = Google Search\nplaceholder = \nrole = button\n===\nlabel = 11\nname = I'm Feeling Lucky\ntagName = INPUT\ncurrentValue = I'm Feeling Lucky\nplaceholder = \nrole = button\n===\nlabel = 12\nname = Advertising\ntagName = A\n===\nlabel = 13\nname = Business\ntagName = A\n===\nlabel = 14\nname = How Search works\ntagName = A\n===\nlabel = 15\nname = Our third decade of climate action: join us\ntagName = A\n===\nlabel = 16\nname = Privacy\ntagName = A\n===\nlabel = 17\nname = Terms\ntagName = A\n===\nlabel = 18\nname = Settings\ntagName = DIV\nrole = button\n\nThis textarea currently has focus:\nlabel = 7\nname = Search\ntagName = TEXTAREA\ncurrentValue = \nplaceholder = \nrole = combobox\n",
+        "response": "{\n  \"thought\": \"I am entering 'bananas' into the search field and pressing ENTER to perform the search.\",\n  \"action\": {\n    \"name\": \"setValueAndEnter\",\n    \"args\": {\n      \"label\": \"7\",\n      \"value\": \"bananas\"\n    }\n  }\n}",
         "action": {
-            "thought": "I am setting the value 'bananas' in the search field and pressing ENTER to perform the search.",
+            "thought": "I am entering 'bananas' into the search field and pressing ENTER to perform the search.",
             "operation": {
                 "name": "setValueAndEnter",
                 "args": {
@@ -14,23 +14,23 @@
         },
         "usage": {
             "prompt_tokens": 2501,
-            "completion_tokens": 66,
-            "total_tokens": 2567
+            "completion_tokens": 64,
+            "total_tokens": 2565
         }
     },
     {
-        "prompt": "The user requests the following task:\n\n  Search for bananas\nYou have already taken the following actions: \nThought: I am setting the value 'bananas' in the search field and pressing ENTER to perform the search.\nAction:{\"name\":\"setValueAndEnter\",\"args\":{\"label\":\"7\",\"value\":\"bananas\"}}\n\n\n\nCurrent time: 5/8/2024, 12:39:17 PM\nCurrent URL: https://www.google.com/search\nCurrent page scrolling position: 27.7%\n\n\nUse the following data as a reference of the annotated elements (using `===` as a delimiter between each annotation):\n\nlabel = 1\nname = Google\ntagName = A\n===\nlabel = 2\nname = Search\ntagName = TEXTAREA\ncurrentValue = bananas\nplaceholder = \nrole = combobox\n===\nlabel = 3\nname = Clear\ntagName = DIV\nrole = button\n===\nlabel = 4\nname = Search by voice\ntagName = DIV\nrole = button\n===\nlabel = 5\nname = Search by image\ntagName = DIV\nrole = button\n===\nlabel = 6\nname = Search\ntagName = BUTTON\n===\nlabel = 7\nname = Images\ntagName = A\nrole = link\n===\nlabel = 8\nname = Shopping\ntagName = A\nrole = link\n===\nlabel = 9\nname = Videos\ntagName = A\nrole = link\n===\nlabel = 10\nname = Forums\ntagName = A\nrole = link\n===\nlabel = 11\nname = More\ntagName = DIV\nrole = button\n===\nlabel = 12\nname = Tools\ntagName = DIV\nrole = button\n===\nlabel = 13\nname = Add Nutrition facts\ntagName = A\nrole = link\n===\nlabel = 14\nname = Add Benefits\ntagName = A\nrole = link\n===\nlabel = 15\nname = Add Recipes\ntagName = A\nrole = link\n===\nlabel = 16\nname = Add Types\ntagName = A\nrole = link\n===\nlabel = 17\nname = Add Tree\ntagName = A\nrole = link\n===\nlabel = 18\nname = Add Fruit\ntagName = A\nrole = link\n===\nlabel = 19\nname = Add Digestion\ntagName = A\nrole = link\n===\nlabel = 20\nname = Add Who should avoid\ntagName = A\nrole = link\n===\nlabel = 21\nname = Banana Wikipedia https://en.wikipedia.org \u203a wiki \u203a Banana\ntagName = A\n===\nlabel = 22\nname = About this result\ntagName = DIV\nrole = button\n===\nlabel = 23\nname = List of banana cultivars\ntagName = A\n===\nlabel = 24\nname = Cavendish banana\ntagName = A\n===\nlabel = 25\nname = Cooking banana\ntagName = A\n===\nlabel = 26\nname = Gros Michel\ntagName = A\n===\nlabel = 27\nname = About this result\ntagName = DIV\nrole = button\n===\nlabel = 28\nname = What are the benefits of eating a banana?\ntagName = DIV\nrole = button\n===\nlabel = 29\nname = Is it okay to eat a banana every day?\ntagName = DIV\nrole = button\n===\nlabel = 30\nname = What does \ud83c\udf4c mean in chat from girl?\ntagName = DIV\nrole = button\n===\nlabel = 31\nname = Are bananas the healthiest food?\ntagName = DIV\nrole = button\n===\nlabel = 32\nname = Feedback\ntagName = DIV\nrole = button\n===\nlabel = 33\nname = About this result\ntagName = DIV\nrole = button\n",
-        "response": "{\n  \"thought\": \"The search for 'bananas' has been successfully performed on Google. The results include a Wikipedia link about bananas, various questions related to bananas under the 'People also ask' section, and links to additional information such as banana cultivars and types.\",\n  \"action\": {\n    \"name\": \"finish\"\n  }\n}",
+        "prompt": "The user requests the following task:\n\n  Search for bananas\nYou have already taken the following actions: \nThought: I am entering 'bananas' into the search field and pressing ENTER to perform the search.\nAction:{\"name\":\"setValueAndEnter\",\"args\":{\"label\":\"7\",\"value\":\"bananas\"}}\n\n\n\nCurrent time: 5/8/2024, 2:18:26 PM\nCurrent URL: https://www.google.com/search\nCurrent page scrolling position: 27.9%\n\n\nUse the following data as a reference of the annotated elements (using `===` as a delimiter between each annotation):\n\nlabel = 1\nname = Google\ntagName = A\n===\nlabel = 2\nname = Search\ntagName = TEXTAREA\ncurrentValue = bananas\nplaceholder = \nrole = combobox\n===\nlabel = 3\nname = Clear\ntagName = DIV\nrole = button\n===\nlabel = 4\nname = Search by voice\ntagName = DIV\nrole = button\n===\nlabel = 5\nname = Search by image\ntagName = DIV\nrole = button\n===\nlabel = 6\nname = Search\ntagName = BUTTON\n===\nlabel = 7\nname = Images\ntagName = A\nrole = link\n===\nlabel = 8\nname = Shopping\ntagName = A\nrole = link\n===\nlabel = 9\nname = Videos\ntagName = A\nrole = link\n===\nlabel = 10\nname = Forums\ntagName = A\nrole = link\n===\nlabel = 11\nname = More\ntagName = DIV\nrole = button\n===\nlabel = 12\nname = Tools\ntagName = DIV\nrole = button\n===\nlabel = 13\nname = Add Nutrition facts\ntagName = A\nrole = link\n===\nlabel = 14\nname = Vibe\ntagName = DIV\nrole = button\n===\nlabel = 15\nname = Add Benefits\ntagName = A\nrole = link\n===\nlabel = 16\nname = Add Recipes\ntagName = A\nrole = link\n===\nlabel = 17\nname = Add Types\ntagName = A\nrole = link\n===\nlabel = 18\nname = Add Open now\ntagName = A\nrole = link\n===\nlabel = 19\nname = Add Takeout\ntagName = A\nrole = link\n===\nlabel = 20\nname = Add Tree\ntagName = A\nrole = link\n===\nlabel = 21\nname = Add Delivery\ntagName = A\nrole = link\n===\nlabel = 22\nname = Banana Wikipedia https://en.wikipedia.org \u203a wiki \u203a Banana\ntagName = A\n===\nlabel = 23\nname = About this result\ntagName = DIV\nrole = button\n===\nlabel = 24\nname = List of banana cultivars\ntagName = A\n===\nlabel = 25\nname = Cavendish banana\ntagName = A\n===\nlabel = 26\nname = Cooking banana\ntagName = A\n===\nlabel = 27\nname = Gros Michel\ntagName = A\n===\nlabel = 28\nname = About this result\ntagName = DIV\nrole = button\n===\nlabel = 29\nname = What are the benefits of eating bananas?\ntagName = DIV\nrole = button\n===\nlabel = 30\nname = Are bananas good to eat everyday?\ntagName = DIV\nrole = button\n===\nlabel = 31\nname = What does \ud83c\udf4c mean in chat from girl?\ntagName = DIV\nrole = button\n===\nlabel = 32\nname = What are bananas high in?\ntagName = DIV\nrole = button\n===\nlabel = 33\nname = Feedback\ntagName = DIV\nrole = button\n===\nlabel = 34\nname = About this result\ntagName = DIV\nrole = button\n",
+        "response": "{\n  \"thought\": \"The search for 'bananas' has been successfully performed. The results include a Wikipedia link about bananas, various questions related to bananas such as their benefits and nutritional content, and links to different types of bananas like Cavendish and Cooking bananas.\",\n  \"action\": {\n    \"name\": \"finish\"\n  }\n}",
         "action": {
-            "thought": "The search for 'bananas' has been successfully performed on Google. The results include a Wikipedia link about bananas, various questions related to bananas under the 'People also ask' section, and links to additional information such as banana cultivars and types.",
+            "thought": "The search for 'bananas' has been successfully performed. The results include a Wikipedia link about bananas, various questions related to bananas such as their benefits and nutritional content, and links to different types of bananas like Cavendish and Cooking bananas.",
             "operation": {
                 "name": "finish"
             }
         },
         "usage": {
-            "prompt_tokens": 2838,
-            "completion_tokens": 71,
-            "total_tokens": 2909
+            "prompt_tokens": 2849,
+            "completion_tokens": 70,
+            "total_tokens": 2919
         }
     }
 ]