From e54e32770c9158e8ac6ddbddce3f1482696cf434 Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 12:33:48 -0800 Subject: [PATCH 1/9] Stream stdout/stderr to parent process --- .../cluster/test_reproducibility.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py index f099afd120..491f1403df 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py @@ -1692,23 +1692,29 @@ def run_subprocess(cmd, module_dir=None, cwd=None): else: env = None try: - p = subprocess.run( + # Use live output streaming for GitHub Actions visibility + process = subprocess.Popen( shlex.split(cmd), cwd=cwd, env=env, - check=True, shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=sys.stdout, + stderr=sys.stderr, text=True, ) + returncode = process.wait() + if returncode != 0: + raise subprocess.CalledProcessError(returncode, cmd) except subprocess.CalledProcessError as ex: - print(f"Subprocess command failed (return code: {ex.returncode}): {cmd}\nStdout:\n{ex.stdout}\nStderr:\n{ex.stderr}") + print(f"Subprocess command failed (return code: {ex.returncode}): {cmd}", flush=True) raise + except Exception as ex: + print(f"Unexpected error in subprocess: {ex}", flush=True) + raise + else: + print(f"Return code: {returncode}", flush=True) - print("Return code: {0}".format(p.returncode)) - - return p.returncode + return returncode def recreate_environment_test(self, environment_manager="conda"): """Test for PyRosettaCluster decoy reproducibility in a recreated virtual environment.""" From 4aafbef9b5ec218110e971488dde6965eb784801 Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 15:30:59 -0800 Subject: [PATCH 2/9] Speed up unit test --- .../distributed/cluster/recreate_environment_test_runs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py index ceea3cdb0a..953dbefbb4 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py @@ -22,7 +22,7 @@ def create_tasks(): "options": "-ex1 0 -ex1aro 0 -ex1aro_exposed 0 -ex2 0 -ex2aro 0 -ex2aro_exposed 0 -ex3 0 -ex4 0 -lazy_ig 1", "extra_options": "-out:level 300 -multithreading:total_threads 1 -ignore_unrecognized_res 1 -load_PDB_components 0", "set_logging_handler": "logging", - "seq": "TESTING/A/NEW/ENV", + "seq": "NEW/ENV", } @@ -41,7 +41,7 @@ def my_protocol(packed_pose, **kwargs): pack_rotamers = PackRotamersMover( scorefxn=scorefxn, task=pyrosetta.standard_packer_task(pose), - nloop=10, + nloop=3, ) pack_rotamers.apply(pose) scorefxn(pose) From 2ffa8d0d6be1a9f7045bded1af0b6ebb825c77a7 Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 15:36:13 -0800 Subject: [PATCH 3/9] Run python unbuffered --- .../distributed/cluster/test_reproducibility.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py index 491f1403df..af514b0e4d 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/test_reproducibility.py @@ -1744,14 +1744,14 @@ def recreate_environment_test(self, environment_manager="conda"): original_output_path = os.path.join(original_env_dir, f"{environment_manager}_original_outputs") original_scorefile_name = "test_scores.json" if environment_manager == "pixi": - cmd = "pixi run python {0} --env_manager '{1}' --output_path '{2}' --scorefile_name '{3}'".format( + cmd = "pixi run python -u {0} --env_manager '{1}' --output_path '{2}' --scorefile_name '{3}'".format( test_script, environment_manager, original_output_path, original_scorefile_name, ) elif environment_manager == "uv": - cmd = "uv run -p {0} python {1} --env_manager '{2}' --output_path '{3}' --scorefile_name '{4}'".format( + cmd = "uv run -p {0} python -u {1} --env_manager '{2}' --output_path '{3}' --scorefile_name '{4}'".format( original_env_dir, test_script, environment_manager, @@ -1759,7 +1759,7 @@ def recreate_environment_test(self, environment_manager="conda"): original_scorefile_name, ) elif environment_manager in ("conda", "mamba"): - cmd = "conda run -p {0} python {1} --env_manager '{2}' --output_path '{3}' --scorefile_name '{4}'".format( + cmd = "conda run -p {0} python -u {1} --env_manager '{2}' --output_path '{3}' --scorefile_name '{4}'".format( original_env_dir, test_script, environment_manager, @@ -1837,7 +1837,7 @@ def recreate_environment_test(self, environment_manager="conda"): reproduce_scorefile_name = "test_scores.json" if environment_manager == "pixi": cmd = ( - f"pixi run python {test_script} " + f"pixi run python -u {test_script} " f"--env_manager '{environment_manager}' " f"--output_path '{reproduce_output_path}' " f"--scorefile_name '{reproduce_scorefile_name}' " @@ -1847,7 +1847,7 @@ def recreate_environment_test(self, environment_manager="conda"): ) elif environment_manager == "uv": cmd = ( - f"uv run -p {reproduce_env_dir} python {test_script} " + f"uv run -p {reproduce_env_dir} python -u {test_script} " f"--env_manager '{environment_manager}' " f"--output_path '{reproduce_output_path}' " f"--scorefile_name '{reproduce_scorefile_name}' " @@ -1857,7 +1857,7 @@ def recreate_environment_test(self, environment_manager="conda"): ) elif environment_manager in ("conda", "mamba"): cmd = ( - f"conda run -p {reproduce_env_dir} python {test_script} " + f"conda run -p {reproduce_env_dir} python -u {test_script} " f"--env_manager '{environment_manager}' " f"--output_path '{reproduce_output_path}' " f"--scorefile_name '{reproduce_scorefile_name}' " From 720c114503148cfdce8b45244838dcfbd3f36696 Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 15:36:31 -0800 Subject: [PATCH 4/9] uv add pip --- .../src/pyrosetta/tests/distributed/cluster/setup_envs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py index 434259b53a..dab9c43a68 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py @@ -115,9 +115,9 @@ def setup_uv_environment(env_dir): ) # Install pyrosetta-installer - print("Adding 'pyrosetta-installer' to uv environment...") + print("Adding 'pyrosetta-installer' and 'pip' to uv environment...") subprocess.run( - ["uv", "add", "-p", str(env_path), "pyrosetta-installer"], + ["uv", "add", "-p", str(env_path), "pyrosetta-installer", "pip"], check=True, ) From 10e9d4340be0c3510d01a00083edcf0796637019 Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 15:37:20 -0800 Subject: [PATCH 5/9] uv add pip --- .../src/pyrosetta/tests/distributed/cluster/setup_envs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py index dab9c43a68..05a2e5f22b 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/setup_envs.py @@ -117,7 +117,11 @@ def setup_uv_environment(env_dir): # Install pyrosetta-installer print("Adding 'pyrosetta-installer' and 'pip' to uv environment...") subprocess.run( - ["uv", "add", "-p", str(env_path), "pyrosetta-installer", "pip"], + ["uv", "add", "-p", str(env_path), "pyrosetta-installer"], + check=True, + ) + subprocess.run( + ["uv", "add", "-p", str(env_path), "pip"], check=True, ) From f1700cf6905a26156b0b1888afd642f694a2c0ee Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 15:54:31 -0800 Subject: [PATCH 6/9] Add warning to docstring --- .../PyRosetta/src/pyrosetta/distributed/cluster/tools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/src/python/PyRosetta/src/pyrosetta/distributed/cluster/tools.py b/source/src/python/PyRosetta/src/pyrosetta/distributed/cluster/tools.py index 4f3019dd6d..73e27a44ad 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/distributed/cluster/tools.py +++ b/source/src/python/PyRosetta/src/pyrosetta/distributed/cluster/tools.py @@ -318,6 +318,14 @@ def recreate_environment( base_dir: Optional[str] = None, ) -> Optional[NoReturn]: """ + *Warning*: This function runs a subprocess with one of the following commands: + - `conda env create ...`: when 'conda' is an executable + - `mamba env create ...`: when 'mamba' is an executable + - `uv pip ...`: when 'uv' is an executable + - `pixi install ...`: when 'pixi' is an executable + Installing certain packages may not be secure, so please only run with input files you trust. + Learn more about PyPI security `here `_ and conda security `here `_. + Given an input file that was written by PyRosettaCluster, or a scorefile and a decoy name that was written by PyRosettaCluster, recreate the environment that was used to generate the decoy with a new environment name. From 67c72b048cd40d93683b04d6e9ef32a844c43e75 Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 16:03:31 -0800 Subject: [PATCH 7/9] Set min_workers/max_workers --- .../distributed/cluster/recreate_environment_test_runs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py index 953dbefbb4..065f703518 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py @@ -73,8 +73,8 @@ def run_original_simulation( cores=None, processes=None, memory=None, - min_workers=2, - max_workers=8, + min_workers=1, + max_workers=1, nstruct=1, dashboard_address=None, compressed=False, From 397c432f5f2b4784e626f4eb8dbde87940a576eb Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 16:03:51 -0800 Subject: [PATCH 8/9] Set logging level --- .../tests/distributed/cluster/recreate_environment_test_runs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py index 065f703518..aa99524191 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py @@ -79,7 +79,7 @@ def run_original_simulation( dashboard_address=None, compressed=False, compression=True, - logging_level="DEBUG", + logging_level="INFO", scorefile_name=scorefile_name, project_name="Original_Environment_Simulation", simulation_name=None, From 2b69ec3f7eac86532cd0b35544fa8f08ac668d15 Mon Sep 17 00:00:00 2001 From: "Jason C. Klima" Date: Tue, 4 Nov 2025 16:17:42 -0800 Subject: [PATCH 9/9] Maybe print logs --- .../cluster/recreate_environment_test_runs.py | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py index aa99524191..5161d5df6b 100644 --- a/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py +++ b/source/src/python/PyRosetta/src/pyrosetta/tests/distributed/cluster/recreate_environment_test_runs.py @@ -49,10 +49,20 @@ def my_protocol(packed_pose, **kwargs): return pose +def print_logs(prc_log, protocol_log): + for log_file in (prc_log, protocol_log): + if os.path.isfile(log_file): + with open(log_file, "r") as f: + print(f"Output: '{log_file}':", f.read(), sep="\n") + else: + print(f"Warning: missing PyRosettaCluster log file: '{log_file}'") + + def run_original_simulation( env_manager, output_path, scorefile_name, + verbose=True, ): # Set environment manager os.environ[get_environment_var()] = env_manager @@ -61,6 +71,9 @@ def run_original_simulation( scratch_dir = os.path.join(tmp_dir, "scratch") tasks = list(create_tasks()) decoy_dir_name = "test_decoys" + logs_dir_name = "logs" + project_name = "Original_Environment_Simulation" + simulation_name = env_manager protocols = [my_protocol,] instance_kwargs = dict( tasks=tasks, @@ -81,13 +94,13 @@ def run_original_simulation( compression=True, logging_level="INFO", scorefile_name=scorefile_name, - project_name="Original_Environment_Simulation", - simulation_name=None, + project_name=project_name, + simulation_name=simulation_name, environment=None, output_path=output_path, simulation_records_in_scorefile=True, decoy_dir_name=decoy_dir_name, - logs_dir_name="logs", + logs_dir_name=logs_dir_name, ignore_errors=False, timeout=0.1, max_delay_time=1.0, @@ -109,6 +122,11 @@ def run_original_simulation( ) # Run simulation run(**instance_kwargs) + # Maybe print log files + if verbose: + protocol_log = os.path.join(output_path, logs_dir_name, f"{project_name}_{simulation_name}.log") + prc_log = os.path.join(output_path, logs_dir_name, "PyRosettaCluster.log") + print_logs(prc_log, protocol_log) def run_reproduce_simulation( @@ -117,12 +135,16 @@ def run_reproduce_simulation( scorefile_name, original_scorefile, original_decoy_name, + verbose=True, ): # Set environment manager os.environ[get_environment_var()] = env_manager with tempfile.TemporaryDirectory() as tmp_dir: # Setup simulation scratch_dir = os.path.join(tmp_dir, "scratch") + logs_dir_name = "logs" + project_name = "Recreated_Environment_Simulation" + simulation_name = env_manager # Run simulation reproduce( input_file=None, @@ -138,7 +160,9 @@ def run_reproduce_simulation( "scratch_dir": scratch_dir, "sha1": None, "scorefile_name": scorefile_name, - "project_name": "Recreated_Environment_Simulation", + "logs_dir_name": logs_dir_name, + "project_name": project_name, + "simulation_name": simulation_name, "output_decoy_types": [".pdb", ".pkl_pose", ".b64_pose"], "output_scorefile_types": [".json",], "author": "Bob", @@ -146,6 +170,11 @@ def run_reproduce_simulation( "license": "LICENSE.PyRosetta.md" }, ) + # Maybe print log files + if verbose: + protocol_log = os.path.join(output_path, logs_dir_name, f"{project_name}_{simulation_name}.log") + prc_log = os.path.join(output_path, logs_dir_name, "PyRosettaCluster.log") + print_logs(prc_log, protocol_log) if __name__ == "__main__": @@ -157,7 +186,7 @@ def run_reproduce_simulation( parser.add_argument('--original_scorefile', type=str, default=None) parser.add_argument('--original_decoy_name', type=str, default=None) parser.add_argument('--reproduce', dest='reproduce', action='store_true') - parser.set_defaults(reproduce=False) + parser.set_defaults(reproduce=False) args = parser.parse_args() if not args.reproduce: run_original_simulation(