Rework to use the new screenshot/video generation script

Author: melissawm

docs/_scripts/generate_screenshots/README.md

@@ -0,0 +1,23 @@
+# Autogenerate screenshots and videos from pre-recorded napari interactions
+
+This folder contains scripts designed to automate the generation of screenshots
+and videos from pre-recorded interactions with napari. It uses `pyautogui` and
+`pynput` to record the screen and mouse interactions, exports the results to a
+json file, and then generates screenshots and videos based on this data.
+
+## Usage
+
+To use these scripts, follow these steps:
+1. **Install Dependencies**: Ensure you have the required Python packages installed. You can do this by running:
+   ```bash
+   pip install pyautogui pynput
+   ```
+2. **Launch napari**: Start napari in a separate terminal or environment. Ensure it is running and ready to accept interactions.
+3. **Record Interactions**: Use the `record.py` script to record your interactions with napari. This will create a JSON file containing the recorded mouse and keyboard events.
+4. **Convert to Screenshots**: Use the `convert.py` script to convert the recorded interactions into screenshots. This will generate a series of PNG files in the `screenshots` directory.
+5. The conversion will be saved as `play.py`. Run python play.py to play back the actions
+
+## Attribution
+
+These scripts are inspired by the code from https://github.com/shedloadofcode/mouse-and-keyboard-recorder
+and its accompanying blog post at https://www.shedloadofcode.com/blog/record-mouse-and-keyboard-for-automation-scripts-with-python.

docs/_scripts/generate_screenshots/convert.py

@@ -0,0 +1,103 @@
+"""
+Converts the recording.json file to a Python script
+'play.py' to use with PyAutoGUI.
+
+The 'play.py' script may require editing and adapting
+before use.
+
+Always review 'play.py' before running with PyAutoGUI!
+"""
+
+import json
+
+key_mappings = {
+    "cmd": "win",
+    "alt_l": "alt",
+    "alt_r": "alt",
+    "ctrl_l": "ctrl",
+    "ctrl_r": "ctrl",
+}
+
+
+def read_json_file():
+    """
+    Takes the JSON output 'recording.json'
+
+    Excludes released and scrolling events to
+    keep things simple.
+    """
+    with open("recording.json") as f:
+        recording = json.load(f)
+
+    def excluded_actions(object):
+        return "released" not in object["action"] and "scroll" not in object["action"]
+
+    recording = list(filter(excluded_actions, recording))
+
+    return recording
+
+
+def convert_to_pyautogui_script(recording):
+    """
+    Converts to a Python template script 'play.py' to
+    use with PyAutoGUI.
+
+    Converts the:
+
+    - Mouse clicks
+    - Keyboard input
+    - Time between actions calculated
+    """
+    if not recording:
+        return
+
+    output = open("play.py", "w")
+    output.write("import time\n")
+    output.write("import pyautogui\n\n")
+
+    for i, step in enumerate(recording):
+        print(step)
+
+        not_first_element = (i - 1) > 0
+        if not_first_element:
+            ## compare time to previous time for the 'sleep' with a 10% buffer
+            pause_in_seconds = (step["_time"] - recording[i - 1]["_time"]) * 1.1
+
+            output.write(f"time.sleep({pause_in_seconds})\n\n")
+        else:
+            output.write("time.sleep(1)\n\n")
+
+        if step["action"] == "pressed_key":
+            key = (
+                step["key"].replace("Key.", "")
+                if "Key." in step["key"]
+                else step["key"]
+            )
+
+            if key in key_mappings.keys():
+                key = key_mappings[key]
+
+            output.write(f"pyautogui.press('{key}')\n")
+
+        if step["action"] == "clicked":
+            output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n")
+
+            if step["button"] == "Button.right":
+                output.write("pyautogui.mouseDown(button='right')\n")
+            else:
+                output.write("pyautogui.mouseDown()\n")
+
+        if step["action"] == "unclicked":
+            output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n")
+
+            if step["button"] == "Button.right":
+                output.write("pyautogui.mouseUp(button='right')\n")
+            else:
+                output.write("pyautogui.mouseUp()\n")
+
+    print("Recording converted. Saved to 'play.py'")
+
+
+if __name__ == "__main__":
+    recording = read_json_file()
+    convert_to_pyautogui_script(recording)

docs/_scripts/generate_screenshots/coordinates.py

@@ -0,0 +1,168 @@
+"""
+Opens a napari window and records mouse and keyboard interactions to a JSON file
+
+To begin recording:
+- Run `python record.py`
+
+To end recording:
+- Hold right click for 2 seconds then release to end the recording for mouse.
+- Press 'ESC' to end the recording for keyboard.
+- Both are needed to finish recording.
+"""
+
+import time
+import json
+import napari
+import argparse
+from pynput import mouse, keyboard
+
+
+class InteractionRecorder:
+    def __init__(self, output_file="recording.json"):
+        self.recording = []
+        self.output_file = output_file
+        self.keyboard_listener = None
+        self.mouse_listener = None
+        self.viewer = None
+
+    def on_press(self, key):
+        try:
+            json_object = {"action": "pressed_key", "key": key.char, "_time": time.time()}
+        except AttributeError:
+            if key == keyboard.Key.esc:
+                print("Keyboard recording ended.")
+                self.stop_recording()
+                return False
+
+            json_object = {"action": "pressed_key", "key": str(key), "_time": time.time()}
+
+        self.recording.append(json_object)
+
+    def on_release(self, key):
+        try:
+            json_object = {"action": "released_key", "key": key.char, "_time": time.time()}
+        except AttributeError:
+            json_object = {"action": "released_key", "key": str(key), "_time": time.time()}
+
+        self.recording.append(json_object)
+
+    def on_move(self, x, y):
+        if len(self.recording) >= 1:
+            if (
+                self.recording[-1]["action"] == "pressed"
+                and self.recording[-1]["button"] == "Button.left"
+            ) or (
+                self.recording[-1]["action"] == "moved"
+                and time.time() - self.recording[-1]["_time"] > 0.02
+            ):
+                json_object = {"action": "moved", "x": x, "y": y, "_time": time.time()}
+                self.recording.append(json_object)
+
+    def on_click(self, x, y, button, pressed):
+        json_object = {
+            "action": "clicked" if pressed else "unclicked",
+            "button": str(button),
+            "x": x,
+            "y": y,
+            "_time": time.time(),
+        }
+
+        self.recording.append(json_object)
+
+        if len(self.recording) > 1:
+            if (
+                self.recording[-1]["action"] == "unclicked"
+                and self.recording[-1]["button"] == "Button.right"
+                and self.recording[-1]["_time"] - self.recording[-2]["_time"] > 2
+            ):
+                self.save_recording()
+                print("Mouse recording ended.")
+                self.stop_recording()
+                return False
+
+    def on_scroll(self, x, y, dx, dy):
+        json_object = {
+            "action": "scroll",
+            "vertical_direction": int(dy),
+            "horizontal_direction": int(dx),
+            "x": x,
+            "y": y,
+            "_time": time.time(),
+        }
+
+        self.recording.append(json_object)
+
+    def save_recording(self):
+        """Save the recorded interactions to a JSON file."""
+        with open(self.output_file, "w") as f:
+            json.dump(self.recording, f)
+        print(f"Recording saved to {self.output_file}")
+
+    def start_recording(self):
+        """Creates a napari window and starts recording mouse and keyboard interactions."""
+        napari.Viewer.close_all()
+        # Open napari with standard window size (800x600)
+        self.viewer = napari.Viewer()
+        self.viewer.window._qt_window.setGeometry(0, 0, 800, 600)
+
+        # Reset recording for this session
+        self.recording = []
+
+        print("Press 'ESC' to end the keyboard recording")
+        print("Hold right click for 2 seconds then release to end the mouse recording")
+
+        self.keyboard_listener = keyboard.Listener(
+            on_press=self.on_press,
+            on_release=self.on_release
+        )
+        self.mouse_listener = mouse.Listener(
+            on_click=self.on_click,
+            on_scroll=self.on_scroll,
+            on_move=self.on_move
+        )
+
+        self.keyboard_listener.start()
+        self.mouse_listener.start()
+
+        # Show the napari window
+        napari.run()
+
+        # Wait for listeners to finish
+        self.keyboard_listener.join()
+        self.mouse_listener.join()
+
+    def stop_recording(self):
+        """Stop the recording and save the data."""
+        if self.keyboard_listener:
+            self.keyboard_listener.stop()
+        if self.mouse_listener:
+            self.mouse_listener.stop()
+        self.save_recording()
+
+        if self.viewer:
+            self.viewer.close()
+
+
+def start_recording(output: str = "recording.json"):
+    """Creates a napari window and starts recording mouse and keyboard interactions.
+
+    Args:
+        output (str): The name of the output JSON file to save the recording.
+    """
+    recorder = InteractionRecorder(output)
+    recorder.start_recording()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Record mouse and keyboard interactions in a napari window."
+    )
+    parser.add_argument(
+        "output",
+        type=str,
+        nargs='?',
+        default="recording.json",
+        help="Output JSON file name"
+    )
+    args = parser.parse_args()
+    start_recording(args.output)