From 8df97e376748734a7e00000afb2c4ec2b0993358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melissa=20Weber=20Mendon=C3=A7a?= Date: Tue, 5 Dec 2023 10:26:36 -0300 Subject: [PATCH 1/4] Add pyautogui workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniel Althviz Moré <16781833+dalthviz@users.noreply.github.com> --- .../generate_screenshots/coordinates.py | 11 ++ .../generate_screenshots/webm_add_points.py | 136 ++++++++++++++++++ docs/images/ok.png | Bin 0 -> 952 bytes docs/images/point-adding-tool.png | Bin 0 -> 1234 bytes docs/images/point-deleting-tool.png | Bin 0 -> 612 bytes docs/images/point-selecting-tool.png | Bin 0 -> 978 bytes 6 files changed, 147 insertions(+) create mode 100644 docs/_scripts/generate_screenshots/coordinates.py create mode 100644 docs/_scripts/generate_screenshots/webm_add_points.py create mode 100644 docs/images/ok.png create mode 100644 docs/images/point-adding-tool.png create mode 100644 docs/images/point-deleting-tool.png create mode 100644 docs/images/point-selecting-tool.png diff --git a/docs/_scripts/generate_screenshots/coordinates.py b/docs/_scripts/generate_screenshots/coordinates.py new file mode 100644 index 000000000..5afe6c9ae --- /dev/null +++ b/docs/_scripts/generate_screenshots/coordinates.py @@ -0,0 +1,11 @@ +import pyautogui + +print('Press Ctrl-C to quit.') +try: + while True: + x, y = pyautogui.position() + positionStr = 'X: ' + str(x).rjust(4) + ' Y: ' + str(y).rjust(4) + print(positionStr, end='') + print('\b' * len(positionStr), end='', flush=True) +except KeyboardInterrupt: + print('\n') diff --git a/docs/_scripts/generate_screenshots/webm_add_points.py b/docs/_scripts/generate_screenshots/webm_add_points.py new file mode 100644 index 000000000..4738b43d1 --- /dev/null +++ b/docs/_scripts/generate_screenshots/webm_add_points.py @@ -0,0 +1,136 @@ +# This script generates the video for the points layer tutorial. + +# Before running, make sure to: +# 1. Install napari in development mode from the main branch +# 2. Run `napari --reset` + +# To generate the video, install pyautogui and run: +# python webm_add_points.py + +from pyautogui import click, alert, locateCenterOnScreen, moveTo, dragTo, screenshot +import numpy as np +from qtpy import QtCore +from skimage import data +from qtpy.QtWidgets import QApplication +import napari +import time + + +def apply_event(app, event, loc, msg=""): + duration = 0.3 + if event == "click_button": + button = locateCenterOnScreen(loc) + click(button, duration=duration) + if event == "click": + click(*loc, duration=duration) + if event == "move": + moveTo(*loc, duration=duration) + if event == "drag": + dragTo(*loc, button="left", duration=duration) + app.processEvents() + print(msg) + app.processEvents() + + +def run_actions(): + # Function that will be triggered from QTimer to run code even when running + # `napari.run` + print("Initial screenshot") + app = QApplication.instance() + time.sleep(1) + print("Done") + + # 1. Click add points icon + # Locate coordinates for the `Add points` buttons by using a image of the + # button + apply_event( + app, + "click_button", + "../../images/point-adding-tool.png", + msg="Click add points icon", + ) + + # 2. Add three new points + apply_event(app, "click", (700, 400), msg="Click add point 1") + apply_event(app, "click", (750, 200), msg="Click add point 2") + apply_event(app, "click", (650, 300), msg="Click add point 3") + + # 3. Click select points icon + apply_event( + app, + "click_button", + "../../images/point-selecting-tool.png", + msg="Click select points icon", + ) + + # 4. Select two points individually + apply_event(app, "click", (750, 200), msg="Click select point 1") + apply_event(app, "click", (650, 300), msg="Click select point 2") + + # 5. Drag mouse to select group of points + apply_event(app, "move", (400, 100), msg="Move to selection start") + apply_event(app, "drag", (600, 300), msg="Drag and select") + + # 6. Change face color + apply_event(app, "move", (150, 240), msg="Move to face color selection") + apply_event(app, "click", (150, 240), msg="Click face color selection") + apply_event(app, "click", (200, 240), msg="Click face color grid") + apply_event(app, "click_button", "../../images/ok.png", msg="Click OK") + + # 7. Change edge color + apply_event(app, "move", (150, 270), msg="Move to edge color selection") + apply_event(app, "click", (150, 270), msg="Click edge color selection") + apply_event(app, "click", (220, 310), msg="Click edge color grid") + apply_event(app, "click_button", "../../images/ok.png", msg="Click OK") + + # 8. Select group of points with different colors + apply_event(app, "move", (400, 200), msg="Move to selection start") + apply_event(app, "drag", (600, 400), msg="Drag and select") + + # 9. Use slider to increase point size + apply_event(app, "move", (175, 155), msg="Move to point size slider") + apply_event(app, "drag", (210, 155), msg="Drag slider") + + # 10. Select a group of points, click symbol dropdown and select cross + apply_event(app, "move", (480, 215), msg="Move to selection start") + apply_event(app, "drag", (680, 345), msg="Drag and select") + apply_event(app, "click", (295, 200), msg="Click symbol dropdown") + apply_event(app, "click", (295, 170), msg="Click cross symbol") + + # 11. Use slider to decrease and increase opacity + apply_event(app, "move", (270, 132), msg="Move to opacity slider") + apply_event(app, "drag", (180, 132), msg="Drag slider down") + apply_event(app, "drag", (270, 132), msg="Drag slider up") + + # 12. Select point and click the "delete selected points" icon + apply_event(app, "click", (440, 380), msg="Select one point") + apply_event(app, "click_button", "../../images/point-deleting-tool.png", msg="Delete point") + + # 13. Click the add points icon + apply_event(app, "click_button", "../../images/point-adding-tool.png", msg="Click add points icon") + + # 14. Use the face color dropdown to select a different color + apply_event(app, "click", (150, 240), msg="Click face color selection") + apply_event(app, "click", (310, 180), msg="Click face color grid") + apply_event(app, "click_button", "../../images/ok.png", msg="Click OK") + + # 15. Use the slider to increase point size and add new points + apply_event(app, "click", (210, 155), msg="Click point size slider") + apply_event(app, "drag", (170, 155), msg="Drag slider down") + apply_event(app, "click", (500, 500), msg="Click add point") + apply_event(app, "move", (800, 600), msg="Move mouse") + + screenshot("fallback.png") + + +viewer = napari.Viewer() +viewer.window.set_geometry(0, 0, 800, 600) +viewer.add_image(data.astronaut(), rgb=True) +points = np.array([[100, 100], [200, 200], [300, 100]]) +points_layer = viewer.add_points(points, size=30) +# Wait a bit so that the user has time to move the app to the foreground +print("Make sure the qt app is in the foreground... waiting 3s to trigger actions") +QtCore.QTimer.singleShot(3000, run_actions) + +print("Launched napari") +napari.run() diff --git a/docs/images/ok.png b/docs/images/ok.png new file mode 100644 index 0000000000000000000000000000000000000000..aa01573bddfb4b1bf9326d776319683a56446014 GIT binary patch literal 952 zcmV;p14sOcP)e}WGpN}-n*YfC|k6lW1zIG9B-`;yWL?d znp3E1n#koa(fi}mF;iv~f(QV+-GR5(HzQ_}Q3xWYF!t3XqmZ#==&)qyuw>}4Wa#+U zWN4Zo9*Y{o|0zt1uGv5ng3!ePvMl4TFEthe3gr!N(cZe6#+owRvdV+o*XaNKG(Yu? zkV#cbHwf%n3F>Z2HA!@V3m5ujxK+pI2{*-o-8 zpk+rVtFE8qi|)u|=H!7{lOaTg(C8qFqTpZNoLvMUl`h3sewU%)zYvD9hv89z!b{y^ z-@eQhPWoXMG-Wz4T$M9q?yxftjt9*+?m z8NlImP+QmZ)S;nAdvJ|QZ0Ek=s);&ydYbRUZ@KWJYkZm5S8d~^ZOEb}) zGxVEY3>vBMFC0Ci($dvNW45e%0rEDrRxVhG%jKr1xa8SI0H853c#=cub+or`=HpFe zxaBkxwS!FS$0biE(pz}ExZH#&PxB;yC+*WO&KKb`ZOJNw7< zabAYWT)~o|!;+!HlA*(rp~ECYRr9g``+s2Ut4T)Wa@bU9E;98KNsnorQOKAwqmZ#= a==d9#e-RQdY2i-*0000Xl$ETy^Dl63Of)WxF>zsBs0+j>3rSp=)B=UU7FsB6nSr4w6@~ep_Rc*nI&(Yc z(ix@+Ri0#$d*0vu_}%xs=RI@9;UlAxrvS|!_`i5+jOfZp2wMDp+#WBQwlyA2(+@HKo;aGOS2lvq56Cu#LP0{ZE{eT;bH;GNnkV>T$?Qt}0MAI}5 z9USIBbP&D#Hb$vLE}H`(@OeE1TDK8sjdCD5$YgAaU&k+Cnx^ee8`F>xm($7UTSo|W z?Ltb)%=H`mer1;1$-BsMu+zCAqG*TX9ZqU_$;!KojArcf-}?)WH}WdGg)bY18A+#+}Gs+nkn#O);WHxuZ(gQ2|xc3GjWk^bHY08>|IZA#-zU%3WA ze^0k6*r>T`G$CjUY)48-BDrM6b$$I3M^yuK=nk~1imWuZ_^l)W?d<`C5Y{TQQ`3mo z>p=)XCY!T*fzRt7(ZYj3t0BHJ7=7=}@*x)NHqKf|%3Z_=dejGw>6!p(%D!zTBd-_BoR zF>wpG%gwvTM~Rf@yh5?KLb14_#3JD!?;L*{x64U9k!1YbMMd9;x*4f3zZhpnTN^vt z13W+QEW0{(U>FAJOqNo4r8GA+(HjYK`1OMv8X2ZZ*NG>RoIZ0Fqoii5LUl$&^#|6j z&h^z{M+m{oFAgy>9HrUWY{m2CsdBrVR?ILACMGX4e(qx3m2!9Kj&)qKkJyn?GBFur z>gp_mdk5(6?WUu>-7;WG$-U+K%rC~dJUvslXKX~nMk?g;1tu=Un79x_2!Y$}LX_k6 z-u`hYMoiP()IWJCCAnPQ9&U(|t!|X3=5rfwNLxW6uTGtkkyN>gU2e}K-yKhiQmI5C wpR=A-L9G!fNu^TCr{I4@;{~zBJT*rC1#8E>G^Ou0ga7~l07*qoM6N<$f^P{`C;$Ke literal 0 HcmV?d00001 diff --git a/docs/images/point-deleting-tool.png b/docs/images/point-deleting-tool.png new file mode 100644 index 0000000000000000000000000000000000000000..ef06d9537e95a067a1083945a9581c38c17aa182 GIT binary patch literal 612 zcmV-q0-ODbP)Q+x7x%T98Y zqhgg-ZEIY(acQE7#>AMo5Xu0q%0iuIabarU&NPhE2aVy|-QT@u?w6B0xw$f1y4U3h z5mAF-bL51gY7mddNhHRJXqq~;uWY+6oz519Vf5vm4jqOJKt$6>B*xX)D|8q#JNWJ8 z5gIllp+Ss<2JwFgMKz79S1uqT>J%cv)Wx9>k-nHHIJl5%kt5kW*)n7=`GI*n~Ry!-e`EenL_E(?O*YExNRd?|cfpl^b!!&tVeSzL;Q>*JNufD`_TrOoZ z6lSigWq}AK^!dwIe*FB!qbJX4^_-z^^-Z2W+rYFeYMafTir|R)>NtL4qIb9}$6_%o z+g4)$;$veNrm4ohH1$T`I8=v#U1fm4GJb>>Iw(d$gBS@7!Xvcpx@vS_`0eEp>U6ez y7w$JZPiYm;U1k^thOxIBdoOr8G;BsfgZK^9Capz*Jj2xh0000PB)$Ds?%vp!EtaxAUGI|4Jp{Nk(^kTyD)Yv1qQ^c`Bu93T;KbR zbbO8^JpA&j2JZkK=fwYoXd4LvvLxa22k>~V$I-GZ4E+eLT!zM*hhY#>YTcA zJRVQ`3{BJC%(t%qpFhw!yjHq>e%sh4a$S$(B)nd)ZG3YlpTU_Tk)c68|Lg(%eZ9`m zm16(boWSBjoRP>dA0=kkE$p$h{Fo=t)-lbxbLpkwgMPyOcgg4X`QfKuc%I23dcA}~Dxn^gi4R7JPfgGziX0Y8 zysp)p+nk6i5r9&;!u-q>y&;vqmLE~l%B*c{vYbq#8wNe9f+9;qhVHR2ALq{P9t^`k zD<3(xvf*09Fseib!-PUAC9O^aX{4X7^Jw)cHPa;2qoODhBhg_Jv(xzf z-MrY^w&hW$$eDGd#oERul_QJnL)i{lU2%X%kg z){z#LWsyv!`Q+mSb2F3V@`ZLR7LBl&m?b>Whv+EXFaJH@#dePMjSQ8_={a+55rCCc zng@&XjE}|m^|#+Kss;dGJp2?{Zg$kx!P#t%>`o5dsCI5~Dy~Gda)oElGmOQe#3#r2 zdF3$xv++p;L11_=48V47_relz0g+60hr?nC zRgsAf4>5EsvbCLataOKKkw&AzKgm@9=4L0`cVsJj!y*8w)hC#yN$mbG%5i?=_x7F3 zTyY^0qxy>U+6ICk0L}ccEX%RV9TyS-SV}gN7A~J3a0!uoey?5BHZxgA)IF~1h?DPs z{25hIDC{?z;AOChSeE7LOO%EH#82><{907*qoM6N<$f~acN A Date: Wed, 18 Dec 2024 13:42:01 -0300 Subject: [PATCH 2/4] Rework to use the new screenshot/video generation script --- docs/_scripts/generate_screenshots/README.md | 23 +++ docs/_scripts/generate_screenshots/convert.py | 103 +++++++++++ .../generate_screenshots/coordinates.py | 11 -- docs/_scripts/generate_screenshots/record.py | 168 ++++++++++++++++++ .../generate_screenshots/webm_add_points.py | 136 -------------- 5 files changed, 294 insertions(+), 147 deletions(-) create mode 100644 docs/_scripts/generate_screenshots/README.md create mode 100644 docs/_scripts/generate_screenshots/convert.py delete mode 100644 docs/_scripts/generate_screenshots/coordinates.py create mode 100644 docs/_scripts/generate_screenshots/record.py delete mode 100644 docs/_scripts/generate_screenshots/webm_add_points.py diff --git a/docs/_scripts/generate_screenshots/README.md b/docs/_scripts/generate_screenshots/README.md new file mode 100644 index 000000000..57c8f75da --- /dev/null +++ b/docs/_scripts/generate_screenshots/README.md @@ -0,0 +1,23 @@ +# Autogenerate screenshots and videos from pre-recorded napari interactions + +This folder contains scripts designed to automate the generation of screenshots +and videos from pre-recorded interactions with napari. It uses `pyautogui` and +`pynput` to record the screen and mouse interactions, exports the results to a +json file, and then generates screenshots and videos based on this data. + +## Usage + +To use these scripts, follow these steps: +1. **Install Dependencies**: Ensure you have the required Python packages installed. You can do this by running: + ```bash + pip install pyautogui pynput + ``` +2. **Launch napari**: Start napari in a separate terminal or environment. Ensure it is running and ready to accept interactions. +3. **Record Interactions**: Use the `record.py` script to record your interactions with napari. This will create a JSON file containing the recorded mouse and keyboard events. +4. **Convert to Screenshots**: Use the `convert.py` script to convert the recorded interactions into screenshots. This will generate a series of PNG files in the `screenshots` directory. +5. The conversion will be saved as `play.py`. Run python play.py to play back the actions + +## Attribution + +These scripts are inspired by the code from https://github.com/shedloadofcode/mouse-and-keyboard-recorder +and its accompanying blog post at https://www.shedloadofcode.com/blog/record-mouse-and-keyboard-for-automation-scripts-with-python. diff --git a/docs/_scripts/generate_screenshots/convert.py b/docs/_scripts/generate_screenshots/convert.py new file mode 100644 index 000000000..ba7af71c4 --- /dev/null +++ b/docs/_scripts/generate_screenshots/convert.py @@ -0,0 +1,103 @@ +""" +Converts the recording.json file to a Python script +'play.py' to use with PyAutoGUI. + +The 'play.py' script may require editing and adapting +before use. + +Always review 'play.py' before running with PyAutoGUI! +""" + +import json + +key_mappings = { + "cmd": "win", + "alt_l": "alt", + "alt_r": "alt", + "ctrl_l": "ctrl", + "ctrl_r": "ctrl", +} + + +def read_json_file(): + """ + Takes the JSON output 'recording.json' + + Excludes released and scrolling events to + keep things simple. + """ + with open("recording.json") as f: + recording = json.load(f) + + def excluded_actions(object): + return "released" not in object["action"] and "scroll" not in object["action"] + + recording = list(filter(excluded_actions, recording)) + + return recording + + +def convert_to_pyautogui_script(recording): + """ + Converts to a Python template script 'play.py' to + use with PyAutoGUI. + + Converts the: + + - Mouse clicks + - Keyboard input + - Time between actions calculated + """ + if not recording: + return + + output = open("play.py", "w") + output.write("import time\n") + output.write("import pyautogui\n\n") + + for i, step in enumerate(recording): + print(step) + + not_first_element = (i - 1) > 0 + if not_first_element: + ## compare time to previous time for the 'sleep' with a 10% buffer + pause_in_seconds = (step["_time"] - recording[i - 1]["_time"]) * 1.1 + + output.write(f"time.sleep({pause_in_seconds})\n\n") + else: + output.write("time.sleep(1)\n\n") + + if step["action"] == "pressed_key": + key = ( + step["key"].replace("Key.", "") + if "Key." in step["key"] + else step["key"] + ) + + if key in key_mappings.keys(): + key = key_mappings[key] + + output.write(f"pyautogui.press('{key}')\n") + + if step["action"] == "clicked": + output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n") + + if step["button"] == "Button.right": + output.write("pyautogui.mouseDown(button='right')\n") + else: + output.write("pyautogui.mouseDown()\n") + + if step["action"] == "unclicked": + output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n") + + if step["button"] == "Button.right": + output.write("pyautogui.mouseUp(button='right')\n") + else: + output.write("pyautogui.mouseUp()\n") + + print("Recording converted. Saved to 'play.py'") + + +if __name__ == "__main__": + recording = read_json_file() + convert_to_pyautogui_script(recording) diff --git a/docs/_scripts/generate_screenshots/coordinates.py b/docs/_scripts/generate_screenshots/coordinates.py deleted file mode 100644 index 5afe6c9ae..000000000 --- a/docs/_scripts/generate_screenshots/coordinates.py +++ /dev/null @@ -1,11 +0,0 @@ -import pyautogui - -print('Press Ctrl-C to quit.') -try: - while True: - x, y = pyautogui.position() - positionStr = 'X: ' + str(x).rjust(4) + ' Y: ' + str(y).rjust(4) - print(positionStr, end='') - print('\b' * len(positionStr), end='', flush=True) -except KeyboardInterrupt: - print('\n') diff --git a/docs/_scripts/generate_screenshots/record.py b/docs/_scripts/generate_screenshots/record.py new file mode 100644 index 000000000..e98c57fbe --- /dev/null +++ b/docs/_scripts/generate_screenshots/record.py @@ -0,0 +1,168 @@ +""" +Opens a napari window and records mouse and keyboard interactions to a JSON file + +To begin recording: +- Run `python record.py` + +To end recording: +- Hold right click for 2 seconds then release to end the recording for mouse. +- Press 'ESC' to end the recording for keyboard. +- Both are needed to finish recording. +""" + +import time +import json +import napari +import argparse +from pynput import mouse, keyboard + + +class InteractionRecorder: + def __init__(self, output_file="recording.json"): + self.recording = [] + self.output_file = output_file + self.keyboard_listener = None + self.mouse_listener = None + self.viewer = None + + def on_press(self, key): + try: + json_object = {"action": "pressed_key", "key": key.char, "_time": time.time()} + except AttributeError: + if key == keyboard.Key.esc: + print("Keyboard recording ended.") + self.stop_recording() + return False + + json_object = {"action": "pressed_key", "key": str(key), "_time": time.time()} + + self.recording.append(json_object) + + def on_release(self, key): + try: + json_object = {"action": "released_key", "key": key.char, "_time": time.time()} + except AttributeError: + json_object = {"action": "released_key", "key": str(key), "_time": time.time()} + + self.recording.append(json_object) + + def on_move(self, x, y): + if len(self.recording) >= 1: + if ( + self.recording[-1]["action"] == "pressed" + and self.recording[-1]["button"] == "Button.left" + ) or ( + self.recording[-1]["action"] == "moved" + and time.time() - self.recording[-1]["_time"] > 0.02 + ): + json_object = {"action": "moved", "x": x, "y": y, "_time": time.time()} + self.recording.append(json_object) + + def on_click(self, x, y, button, pressed): + json_object = { + "action": "clicked" if pressed else "unclicked", + "button": str(button), + "x": x, + "y": y, + "_time": time.time(), + } + + self.recording.append(json_object) + + if len(self.recording) > 1: + if ( + self.recording[-1]["action"] == "unclicked" + and self.recording[-1]["button"] == "Button.right" + and self.recording[-1]["_time"] - self.recording[-2]["_time"] > 2 + ): + self.save_recording() + print("Mouse recording ended.") + self.stop_recording() + return False + + def on_scroll(self, x, y, dx, dy): + json_object = { + "action": "scroll", + "vertical_direction": int(dy), + "horizontal_direction": int(dx), + "x": x, + "y": y, + "_time": time.time(), + } + + self.recording.append(json_object) + + def save_recording(self): + """Save the recorded interactions to a JSON file.""" + with open(self.output_file, "w") as f: + json.dump(self.recording, f) + print(f"Recording saved to {self.output_file}") + + def start_recording(self): + """Creates a napari window and starts recording mouse and keyboard interactions.""" + napari.Viewer.close_all() + # Open napari with standard window size (800x600) + self.viewer = napari.Viewer() + self.viewer.window._qt_window.setGeometry(0, 0, 800, 600) + + # Reset recording for this session + self.recording = [] + + print("Press 'ESC' to end the keyboard recording") + print("Hold right click for 2 seconds then release to end the mouse recording") + + self.keyboard_listener = keyboard.Listener( + on_press=self.on_press, + on_release=self.on_release + ) + self.mouse_listener = mouse.Listener( + on_click=self.on_click, + on_scroll=self.on_scroll, + on_move=self.on_move + ) + + self.keyboard_listener.start() + self.mouse_listener.start() + + # Show the napari window + napari.run() + + # Wait for listeners to finish + self.keyboard_listener.join() + self.mouse_listener.join() + + def stop_recording(self): + """Stop the recording and save the data.""" + if self.keyboard_listener: + self.keyboard_listener.stop() + if self.mouse_listener: + self.mouse_listener.stop() + self.save_recording() + + if self.viewer: + self.viewer.close() + + +def start_recording(output: str = "recording.json"): + """Creates a napari window and starts recording mouse and keyboard interactions. + + Args: + output (str): The name of the output JSON file to save the recording. + """ + recorder = InteractionRecorder(output) + recorder.start_recording() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Record mouse and keyboard interactions in a napari window." + ) + parser.add_argument( + "output", + type=str, + nargs='?', + default="recording.json", + help="Output JSON file name" + ) + args = parser.parse_args() + start_recording(args.output) diff --git a/docs/_scripts/generate_screenshots/webm_add_points.py b/docs/_scripts/generate_screenshots/webm_add_points.py deleted file mode 100644 index 4738b43d1..000000000 --- a/docs/_scripts/generate_screenshots/webm_add_points.py +++ /dev/null @@ -1,136 +0,0 @@ -# This script generates the video for the points layer tutorial. - -# Before running, make sure to: -# 1. Install napari in development mode from the main branch -# 2. Run `napari --reset` - -# To generate the video, install pyautogui and run: -# python webm_add_points.py - -from pyautogui import click, alert, locateCenterOnScreen, moveTo, dragTo, screenshot -import numpy as np -from qtpy import QtCore -from skimage import data -from qtpy.QtWidgets import QApplication -import napari -import time - - -def apply_event(app, event, loc, msg=""): - duration = 0.3 - if event == "click_button": - button = locateCenterOnScreen(loc) - click(button, duration=duration) - if event == "click": - click(*loc, duration=duration) - if event == "move": - moveTo(*loc, duration=duration) - if event == "drag": - dragTo(*loc, button="left", duration=duration) - app.processEvents() - print(msg) - app.processEvents() - - -def run_actions(): - # Function that will be triggered from QTimer to run code even when running - # `napari.run` - print("Initial screenshot") - app = QApplication.instance() - time.sleep(1) - print("Done") - - # 1. Click add points icon - # Locate coordinates for the `Add points` buttons by using a image of the - # button - apply_event( - app, - "click_button", - "../../images/point-adding-tool.png", - msg="Click add points icon", - ) - - # 2. Add three new points - apply_event(app, "click", (700, 400), msg="Click add point 1") - apply_event(app, "click", (750, 200), msg="Click add point 2") - apply_event(app, "click", (650, 300), msg="Click add point 3") - - # 3. Click select points icon - apply_event( - app, - "click_button", - "../../images/point-selecting-tool.png", - msg="Click select points icon", - ) - - # 4. Select two points individually - apply_event(app, "click", (750, 200), msg="Click select point 1") - apply_event(app, "click", (650, 300), msg="Click select point 2") - - # 5. Drag mouse to select group of points - apply_event(app, "move", (400, 100), msg="Move to selection start") - apply_event(app, "drag", (600, 300), msg="Drag and select") - - # 6. Change face color - apply_event(app, "move", (150, 240), msg="Move to face color selection") - apply_event(app, "click", (150, 240), msg="Click face color selection") - apply_event(app, "click", (200, 240), msg="Click face color grid") - apply_event(app, "click_button", "../../images/ok.png", msg="Click OK") - - # 7. Change edge color - apply_event(app, "move", (150, 270), msg="Move to edge color selection") - apply_event(app, "click", (150, 270), msg="Click edge color selection") - apply_event(app, "click", (220, 310), msg="Click edge color grid") - apply_event(app, "click_button", "../../images/ok.png", msg="Click OK") - - # 8. Select group of points with different colors - apply_event(app, "move", (400, 200), msg="Move to selection start") - apply_event(app, "drag", (600, 400), msg="Drag and select") - - # 9. Use slider to increase point size - apply_event(app, "move", (175, 155), msg="Move to point size slider") - apply_event(app, "drag", (210, 155), msg="Drag slider") - - # 10. Select a group of points, click symbol dropdown and select cross - apply_event(app, "move", (480, 215), msg="Move to selection start") - apply_event(app, "drag", (680, 345), msg="Drag and select") - apply_event(app, "click", (295, 200), msg="Click symbol dropdown") - apply_event(app, "click", (295, 170), msg="Click cross symbol") - - # 11. Use slider to decrease and increase opacity - apply_event(app, "move", (270, 132), msg="Move to opacity slider") - apply_event(app, "drag", (180, 132), msg="Drag slider down") - apply_event(app, "drag", (270, 132), msg="Drag slider up") - - # 12. Select point and click the "delete selected points" icon - apply_event(app, "click", (440, 380), msg="Select one point") - apply_event(app, "click_button", "../../images/point-deleting-tool.png", msg="Delete point") - - # 13. Click the add points icon - apply_event(app, "click_button", "../../images/point-adding-tool.png", msg="Click add points icon") - - # 14. Use the face color dropdown to select a different color - apply_event(app, "click", (150, 240), msg="Click face color selection") - apply_event(app, "click", (310, 180), msg="Click face color grid") - apply_event(app, "click_button", "../../images/ok.png", msg="Click OK") - - # 15. Use the slider to increase point size and add new points - apply_event(app, "click", (210, 155), msg="Click point size slider") - apply_event(app, "drag", (170, 155), msg="Drag slider down") - apply_event(app, "click", (500, 500), msg="Click add point") - apply_event(app, "move", (800, 600), msg="Move mouse") - - screenshot("fallback.png") - - -viewer = napari.Viewer() -viewer.window.set_geometry(0, 0, 800, 600) -viewer.add_image(data.astronaut(), rgb=True) -points = np.array([[100, 100], [200, 200], [300, 100]]) -points_layer = viewer.add_points(points, size=30) -# Wait a bit so that the user has time to move the app to the foreground -print("Make sure the qt app is in the foreground... waiting 3s to trigger actions") -QtCore.QTimer.singleShot(3000, run_actions) - -print("Launched napari") -napari.run() From 00503054ca7ba9748d7c6725dd9f68740901daf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melissa=20Weber=20Mendon=C3=A7a?= Date: Thu, 31 Jul 2025 18:27:28 -0300 Subject: [PATCH 3/4] Update script and instructions --- docs/_scripts/generate_screenshots/README.md | 15 +- docs/_scripts/generate_screenshots/convert.py | 103 ----------- .../{record.py => record_interactions.py} | 160 ++++++++++++++++-- 3 files changed, 154 insertions(+), 124 deletions(-) delete mode 100644 docs/_scripts/generate_screenshots/convert.py rename docs/_scripts/generate_screenshots/{record.py => record_interactions.py} (50%) diff --git a/docs/_scripts/generate_screenshots/README.md b/docs/_scripts/generate_screenshots/README.md index 57c8f75da..47ff76f8e 100644 --- a/docs/_scripts/generate_screenshots/README.md +++ b/docs/_scripts/generate_screenshots/README.md @@ -5,6 +5,9 @@ and videos from pre-recorded interactions with napari. It uses `pyautogui` and `pynput` to record the screen and mouse interactions, exports the results to a json file, and then generates screenshots and videos based on this data. +NOTE: Make sure the Qt version on your system is compatible with the PyQt version +you are using. + ## Usage To use these scripts, follow these steps: @@ -12,10 +15,14 @@ To use these scripts, follow these steps: ```bash pip install pyautogui pynput ``` -2. **Launch napari**: Start napari in a separate terminal or environment. Ensure it is running and ready to accept interactions. -3. **Record Interactions**: Use the `record.py` script to record your interactions with napari. This will create a JSON file containing the recorded mouse and keyboard events. -4. **Convert to Screenshots**: Use the `convert.py` script to convert the recorded interactions into screenshots. This will generate a series of PNG files in the `screenshots` directory. -5. The conversion will be saved as `play.py`. Run python play.py to play back the actions +2. **Install napari**: You probably want to have a [development installation of napari](hhttps://napari.org/stable/developers/contributing/dev_install.html). +3. **Record Interactions**: Use the `record_interactions.py` script to record your interactions with napari. This will + a. Open a napari window, + b. Record mouse and keyboard actions, + c. Save the recorded actions to a JSON file named `recording.json`, + d. Convert the recorded actions into a Python script named `play.py`. + You can use `python record_interactions.py --help` to see the available options for naming output files. +4. Run python `play.py` to play back the actions. ## Attribution diff --git a/docs/_scripts/generate_screenshots/convert.py b/docs/_scripts/generate_screenshots/convert.py deleted file mode 100644 index ba7af71c4..000000000 --- a/docs/_scripts/generate_screenshots/convert.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Converts the recording.json file to a Python script -'play.py' to use with PyAutoGUI. - -The 'play.py' script may require editing and adapting -before use. - -Always review 'play.py' before running with PyAutoGUI! -""" - -import json - -key_mappings = { - "cmd": "win", - "alt_l": "alt", - "alt_r": "alt", - "ctrl_l": "ctrl", - "ctrl_r": "ctrl", -} - - -def read_json_file(): - """ - Takes the JSON output 'recording.json' - - Excludes released and scrolling events to - keep things simple. - """ - with open("recording.json") as f: - recording = json.load(f) - - def excluded_actions(object): - return "released" not in object["action"] and "scroll" not in object["action"] - - recording = list(filter(excluded_actions, recording)) - - return recording - - -def convert_to_pyautogui_script(recording): - """ - Converts to a Python template script 'play.py' to - use with PyAutoGUI. - - Converts the: - - - Mouse clicks - - Keyboard input - - Time between actions calculated - """ - if not recording: - return - - output = open("play.py", "w") - output.write("import time\n") - output.write("import pyautogui\n\n") - - for i, step in enumerate(recording): - print(step) - - not_first_element = (i - 1) > 0 - if not_first_element: - ## compare time to previous time for the 'sleep' with a 10% buffer - pause_in_seconds = (step["_time"] - recording[i - 1]["_time"]) * 1.1 - - output.write(f"time.sleep({pause_in_seconds})\n\n") - else: - output.write("time.sleep(1)\n\n") - - if step["action"] == "pressed_key": - key = ( - step["key"].replace("Key.", "") - if "Key." in step["key"] - else step["key"] - ) - - if key in key_mappings.keys(): - key = key_mappings[key] - - output.write(f"pyautogui.press('{key}')\n") - - if step["action"] == "clicked": - output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n") - - if step["button"] == "Button.right": - output.write("pyautogui.mouseDown(button='right')\n") - else: - output.write("pyautogui.mouseDown()\n") - - if step["action"] == "unclicked": - output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n") - - if step["button"] == "Button.right": - output.write("pyautogui.mouseUp(button='right')\n") - else: - output.write("pyautogui.mouseUp()\n") - - print("Recording converted. Saved to 'play.py'") - - -if __name__ == "__main__": - recording = read_json_file() - convert_to_pyautogui_script(recording) diff --git a/docs/_scripts/generate_screenshots/record.py b/docs/_scripts/generate_screenshots/record_interactions.py similarity index 50% rename from docs/_scripts/generate_screenshots/record.py rename to docs/_scripts/generate_screenshots/record_interactions.py index e98c57fbe..1d66c6a2d 100644 --- a/docs/_scripts/generate_screenshots/record.py +++ b/docs/_scripts/generate_screenshots/record_interactions.py @@ -13,6 +13,7 @@ import time import json import napari +import platform import argparse from pynput import mouse, keyboard @@ -31,7 +32,7 @@ def on_press(self, key): except AttributeError: if key == keyboard.Key.esc: print("Keyboard recording ended.") - self.stop_recording() + self.stop_recording("keyboard") return False json_object = {"action": "pressed_key", "key": str(key), "_time": time.time()} @@ -47,7 +48,7 @@ def on_release(self, key): self.recording.append(json_object) def on_move(self, x, y): - if len(self.recording) >= 1: + if len(self.recording) > 1: if ( self.recording[-1]["action"] == "pressed" and self.recording[-1]["button"] == "Button.left" @@ -69,7 +70,7 @@ def on_click(self, x, y, button, pressed): self.recording.append(json_object) - if len(self.recording) > 1: + if len(self.recording) > 2: if ( self.recording[-1]["action"] == "unclicked" and self.recording[-1]["button"] == "Button.right" @@ -77,7 +78,7 @@ def on_click(self, x, y, button, pressed): ): self.save_recording() print("Mouse recording ended.") - self.stop_recording() + self.stop_recording("mouse") return False def on_scroll(self, x, y, dx, dy): @@ -106,7 +107,14 @@ def start_recording(self): self.viewer.window._qt_window.setGeometry(0, 0, 800, 600) # Reset recording for this session - self.recording = [] + self.recording = [{ + "metadata": { + "start_time": time.time(), + "napari_version": napari.__version__, + "python_version": platform.python_version(), + "os": platform.system() + } + }] print("Press 'ESC' to end the keyboard recording") print("Hold right click for 2 seconds then release to end the mouse recording") @@ -131,38 +139,156 @@ def start_recording(self): self.keyboard_listener.join() self.mouse_listener.join() - def stop_recording(self): - """Stop the recording and save the data.""" - if self.keyboard_listener: + def stop_recording(self, mode: str): + """Stop the recording and save the data. + + Args: + mode (str): The mode of recording ("mouse" or "keyboard"). + """ + if self.keyboard_listener and mode == "keyboard": self.keyboard_listener.stop() - if self.mouse_listener: + if self.mouse_listener and mode == "mouse": self.mouse_listener.stop() - self.save_recording() - if self.viewer: - self.viewer.close() + if not self.keyboard_listener.running and not self.mouse_listener.running: + self.save_recording() + # if self.viewer: + # self.viewer.close() + print("Recording stopped. Please close the napari window.") + + +def read_json_file(json_input: str) -> tuple: + """ + Takes the JSON output 'recording.json' + + Excludes released and scrolling events to + keep things simple. + """ + with open(json_input) as f: + recording = json.load(f) + metadata = recording.pop(0) + + def excluded_actions(object): + return "released" not in object["action"] and "scroll" not in object["action"] + + recording = list(filter(excluded_actions, recording)) + + return metadata, recording + + +def convert_recording(json_input: str, output_file: str = "play.py"): + """Converts the recorded interactions from JSON to a Python script using pyautogui. + + Converts the: + + - Mouse clicks + - Keyboard input + - Time between actions calculated + + Args: + json_input (str): The path to the JSON file containing the recorded interactions. + output_file (str): The name of the output Python script file. + """ + + key_mappings = { + "cmd": "win", + "alt_l": "alt", + "alt_r": "alt", + "ctrl_l": "ctrl", + "ctrl_r": "ctrl", + } + metadata, recording = read_json_file(json_input) -def start_recording(output: str = "recording.json"): + if not recording: + return + + output = open(output_file, "w") + output.write(f"# {metadata}\n") + output.write("import time\n") + output.write("import pyautogui\n\n") + output.write("# Open napari with standard window size (800x600)\n") + output.write("self.viewer = napari.Viewer()\n") + output.write("self.viewer.window._qt_window.setGeometry(0, 0, 800, 600)\n") + + for i, step in enumerate(recording): + print(step) + + not_first_element = (i - 1) > 0 + if not_first_element: + ## compare time to previous time for the 'sleep' with a 10% buffer + pause_in_seconds = (step["_time"] - recording[i - 1]["_time"]) * 1.1 + + output.write(f"time.sleep({pause_in_seconds})\n\n") + else: + output.write("time.sleep(1)\n\n") + + if step["action"] == "pressed_key": + key = ( + step["key"].replace("Key.", "") + if "Key." in step["key"] + else step["key"] + ) + + if key in key_mappings.keys(): + key = key_mappings[key] + + output.write(f"pyautogui.press('{key}')\n") + + if step["action"] == "clicked": + output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n") + + if step["button"] == "Button.right": + output.write("pyautogui.mouseDown(button='right')\n") + else: + output.write("pyautogui.mouseDown()\n") + + if step["action"] == "unclicked": + output.write(f"pyautogui.moveTo({step['x']}, {step['y']})\n") + + if step["button"] == "Button.right": + output.write("pyautogui.mouseUp(button='right')\n") + else: + output.write("pyautogui.mouseUp()\n") + + print(f"Recording converted. Saved to '{output_file}'") + + +def start_recording(json_output: str = "recording.json"): """Creates a napari window and starts recording mouse and keyboard interactions. Args: output (str): The name of the output JSON file to save the recording. """ - recorder = InteractionRecorder(output) + recorder = InteractionRecorder(json_output) recorder.start_recording() if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Record mouse and keyboard interactions in a napari window." + description="Record mouse and keyboard interactions in a napari window.", + epilog="Use --json-output to specify the temporary json output file name. Use --output to specify the output Python script file name." ) parser.add_argument( - "output", + "--json-output", type=str, nargs='?', default="recording.json", help="Output JSON file name" ) + parser.add_argument( + "--output", + type=str, + nargs='?', + default="play.py", + help="Output Python script file name (default: play.py)" + ) + parser.add_argument( + "--no-convert", + action='store_true', + help="Do not convert the recording to a Python script after recording" + ) args = parser.parse_args() - start_recording(args.output) + start_recording(args.json_output) + if not args.no_convert: + convert_recording(args.json_output, args.output) From 8c5c7b01dad16ecc997a02d552beba88bb3ed141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melissa=20Weber=20Mendon=C3=A7a?= Date: Fri, 1 Aug 2025 10:38:35 -0300 Subject: [PATCH 4/4] Fix link and remove screenshot autogeneration readme from main docs --- docs/_scripts/generate_screenshots/README.md | 2 +- docs/conf.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/_scripts/generate_screenshots/README.md b/docs/_scripts/generate_screenshots/README.md index 47ff76f8e..f3b606faf 100644 --- a/docs/_scripts/generate_screenshots/README.md +++ b/docs/_scripts/generate_screenshots/README.md @@ -15,7 +15,7 @@ To use these scripts, follow these steps: ```bash pip install pyautogui pynput ``` -2. **Install napari**: You probably want to have a [development installation of napari](hhttps://napari.org/stable/developers/contributing/dev_install.html). +2. **Install napari**: You probably want to have a [development installation of napari](https://napari.org/stable/developers/contributing/dev_install.html). 3. **Record Interactions**: Use the `record_interactions.py` script to record your interactions with napari. This will a. Open a napari window, b. Record mouse and keyboard actions, diff --git a/docs/conf.py b/docs/conf.py index aff7a3bc7..af25b78aa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -236,6 +236,7 @@ "plugins/building_a_plugin/_layer_data_guide.md", "gallery/index.rst", "_scripts/README.md", + "_scripts/generate_screenshots/README.md", ] # -- Versions and switcher -------------------------------------------------