From 0f05d9cca083ca236d0a23df69ff3e36bc55c142 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 15 Jul 2025 14:51:58 +0200 Subject: [PATCH 01/25] write files when submitted function fails --- executorlib/backend/cache_parallel.py | 7 +++++++ executorlib/backend/interactive_parallel.py | 6 ++++++ executorlib/backend/interactive_serial.py | 6 +++++- executorlib/task_scheduler/file/backend.py | 7 ++++++- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index 9b1d25d8..2b560f04 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -10,6 +10,9 @@ ) +write_error_file = False + + def main() -> None: """ Main function for executing the cache_parallel script. @@ -53,6 +56,10 @@ def main() -> None: output={"error": error}, runtime=time.time() - time_start, ) + if write_error_file: + file_name_error = os.path.splitext(os.path.basename(file_name))[0] + with open(os.path.join(os.path.dirname(file_name), "error_" + file_name_error + ".out"), "a") as f: + f.write(error.output) else: if mpi_rank_zero: backend_write_file( diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index 5ae2c320..862ed839 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -15,6 +15,9 @@ ) +write_error_file = False + + def main() -> None: """ Entry point of the program. @@ -82,6 +85,9 @@ def main() -> None: socket=socket, result_dict={"error": error}, ) + if write_error_file: + with open("error.out", "a") as f: + f.write(error.output) else: # Send output if mpi_rank_zero: diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index 34fb2288..5533c4a5 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -11,12 +11,13 @@ ) -def main(argument_lst: Optional[list[str]] = None): +def main(argument_lst: Optional[list[str]] = None, write_error_file: bool = False): """ The main function of the program. Args: argument_lst (Optional[List[str]]): List of command line arguments. If None, sys.argv will be used. + write_error_file (bool): Returns: None @@ -58,6 +59,9 @@ def main(argument_lst: Optional[list[str]] = None): socket=socket, result_dict={"error": error}, ) + if write_error_file: + with open("error.out", "a") as f: + f.write(error.output) else: # Send output interface_send(socket=socket, result_dict={"result": output}) diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index 4ea27c17..797b54b9 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -57,12 +57,13 @@ def backend_write_file(file_name: str, output: Any, runtime: float) -> None: os.rename(file_name_out + "_r.h5", file_name_out + "_o.h5") -def backend_execute_task_in_file(file_name: str) -> None: +def backend_execute_task_in_file(file_name: str, write_error_file: bool = False) -> None: """ Execute the task stored in a given HDF5 file. Args: file_name (str): The file name of the HDF5 file as an absolute path. + write_error_file (bool): Returns: None @@ -77,6 +78,10 @@ def backend_execute_task_in_file(file_name: str) -> None: } except Exception as error: result = {"error": error} + if write_error_file: + file_name_error = os.path.splitext(os.path.basename(file_name))[0] + with open(os.path.join(os.path.dirname(file_name), "error_" + file_name_error + ".out"), "a") as f: + f.write(error.output) backend_write_file( file_name=file_name, From 54211e1c58c6116e5f76b7ac3f16e0ddc876b7ea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:52:30 +0000 Subject: [PATCH 02/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/backend/cache_parallel.py | 8 ++++++-- executorlib/backend/interactive_parallel.py | 1 - executorlib/task_scheduler/file/backend.py | 11 +++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index 2b560f04..4cf4a83a 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -9,7 +9,6 @@ backend_write_file, ) - write_error_file = False @@ -58,7 +57,12 @@ def main() -> None: ) if write_error_file: file_name_error = os.path.splitext(os.path.basename(file_name))[0] - with open(os.path.join(os.path.dirname(file_name), "error_" + file_name_error + ".out"), "a") as f: + with open( + os.path.join( + os.path.dirname(file_name), "error_" + file_name_error + ".out" + ), + "a", + ) as f: f.write(error.output) else: if mpi_rank_zero: diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index 862ed839..39bba565 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -14,7 +14,6 @@ interface_shutdown, ) - write_error_file = False diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index 797b54b9..fd8a4add 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -57,7 +57,9 @@ def backend_write_file(file_name: str, output: Any, runtime: float) -> None: os.rename(file_name_out + "_r.h5", file_name_out + "_o.h5") -def backend_execute_task_in_file(file_name: str, write_error_file: bool = False) -> None: +def backend_execute_task_in_file( + file_name: str, write_error_file: bool = False +) -> None: """ Execute the task stored in a given HDF5 file. @@ -80,7 +82,12 @@ def backend_execute_task_in_file(file_name: str, write_error_file: bool = False) result = {"error": error} if write_error_file: file_name_error = os.path.splitext(os.path.basename(file_name))[0] - with open(os.path.join(os.path.dirname(file_name), "error_" + file_name_error + ".out"), "a") as f: + with open( + os.path.join( + os.path.dirname(file_name), "error_" + file_name_error + ".out" + ), + "a", + ) as f: f.write(error.output) backend_write_file( From 174c1f671859cc94daf17ee8a7499aa9c18f6180 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 15 Jul 2025 15:04:47 +0200 Subject: [PATCH 03/25] get information from input_dict --- executorlib/backend/cache_parallel.py | 5 ++--- executorlib/backend/interactive_parallel.py | 7 ++----- executorlib/backend/interactive_serial.py | 6 +++--- executorlib/task_scheduler/file/backend.py | 8 +++----- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index 2b560f04..a36a6d72 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -56,9 +56,8 @@ def main() -> None: output={"error": error}, runtime=time.time() - time_start, ) - if write_error_file: - file_name_error = os.path.splitext(os.path.basename(file_name))[0] - with open(os.path.join(os.path.dirname(file_name), "error_" + file_name_error + ".out"), "a") as f: + if apply_dict.get("write_error_file", False): + with open(apply_dict.get("error_file_name", "error.out"), "a") as f: f.write(error.output) else: if mpi_rank_zero: diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index 862ed839..a01771e6 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -15,9 +15,6 @@ ) -write_error_file = False - - def main() -> None: """ Entry point of the program. @@ -85,8 +82,8 @@ def main() -> None: socket=socket, result_dict={"error": error}, ) - if write_error_file: - with open("error.out", "a") as f: + if input_dict.get("write_error_file", False): + with open(input_dict.get("error_file_name", "error.out"), "a") as f: f.write(error.output) else: # Send output diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index 5533c4a5..59f9d6d3 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -11,7 +11,7 @@ ) -def main(argument_lst: Optional[list[str]] = None, write_error_file: bool = False): +def main(argument_lst: Optional[list[str]] = None): """ The main function of the program. @@ -59,8 +59,8 @@ def main(argument_lst: Optional[list[str]] = None, write_error_file: bool = Fals socket=socket, result_dict={"error": error}, ) - if write_error_file: - with open("error.out", "a") as f: + if input_dict.get("write_error_file", False): + with open(input_dict.get("error_file_name", "error.out"), "a") as f: f.write(error.output) else: # Send output diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index 797b54b9..6f077fba 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -57,13 +57,12 @@ def backend_write_file(file_name: str, output: Any, runtime: float) -> None: os.rename(file_name_out + "_r.h5", file_name_out + "_o.h5") -def backend_execute_task_in_file(file_name: str, write_error_file: bool = False) -> None: +def backend_execute_task_in_file(file_name: str) -> None: """ Execute the task stored in a given HDF5 file. Args: file_name (str): The file name of the HDF5 file as an absolute path. - write_error_file (bool): Returns: None @@ -78,9 +77,8 @@ def backend_execute_task_in_file(file_name: str, write_error_file: bool = False) } except Exception as error: result = {"error": error} - if write_error_file: - file_name_error = os.path.splitext(os.path.basename(file_name))[0] - with open(os.path.join(os.path.dirname(file_name), "error_" + file_name_error + ".out"), "a") as f: + if apply_dict.get("write_error_file", False): + with open(apply_dict.get("error_file_name", "error.out"), "a") as f: f.write(error.output) backend_write_file( From 4b4e7e5e8eb603e70b36fff62807d736e06027b8 Mon Sep 17 00:00:00 2001 From: pyiron-runner Date: Tue, 15 Jul 2025 13:05:55 +0000 Subject: [PATCH 04/25] Format black --- executorlib/backend/interactive_parallel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index a01771e6..eb70ee46 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -83,7 +83,9 @@ def main() -> None: result_dict={"error": error}, ) if input_dict.get("write_error_file", False): - with open(input_dict.get("error_file_name", "error.out"), "a") as f: + with open( + input_dict.get("error_file_name", "error.out"), "a" + ) as f: f.write(error.output) else: # Send output From 5b49d660174aaa695286e4f151134a6212d8ba3b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 13:08:48 +0000 Subject: [PATCH 05/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/backend/cache_parallel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index a36a6d72..84b0dfc1 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -9,7 +9,6 @@ backend_write_file, ) - write_error_file = False From 2b7fb929e3e7238e2d2eb7f26e6951c85f09e985 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 15 Jul 2025 15:09:36 +0200 Subject: [PATCH 06/25] write either error.output or error --- executorlib/backend/cache_parallel.py | 5 ++++- executorlib/backend/interactive_parallel.py | 5 ++++- executorlib/backend/interactive_serial.py | 5 ++++- executorlib/task_scheduler/file/backend.py | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index a36a6d72..601f9f8e 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -58,7 +58,10 @@ def main() -> None: ) if apply_dict.get("write_error_file", False): with open(apply_dict.get("error_file_name", "error.out"), "a") as f: - f.write(error.output) + if hasattr(error, "output"): + f.write(error.output) + else: + f.write(error) else: if mpi_rank_zero: backend_write_file( diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index a01771e6..18bc886d 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -84,7 +84,10 @@ def main() -> None: ) if input_dict.get("write_error_file", False): with open(input_dict.get("error_file_name", "error.out"), "a") as f: - f.write(error.output) + if hasattr(error, "output"): + f.write(error.output) + else: + f.write(error) else: # Send output if mpi_rank_zero: diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index 59f9d6d3..88443c1d 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -61,7 +61,10 @@ def main(argument_lst: Optional[list[str]] = None): ) if input_dict.get("write_error_file", False): with open(input_dict.get("error_file_name", "error.out"), "a") as f: - f.write(error.output) + if hasattr(error, "output"): + f.write(error.output) + else: + f.write(error) else: # Send output interface_send(socket=socket, result_dict={"result": output}) diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index 6f077fba..05a4e892 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -79,7 +79,10 @@ def backend_execute_task_in_file(file_name: str) -> None: result = {"error": error} if apply_dict.get("write_error_file", False): with open(apply_dict.get("error_file_name", "error.out"), "a") as f: - f.write(error.output) + if hasattr(error, "output"): + f.write(error.output) + else: + f.write(error) backend_write_file( file_name=file_name, From 5bd926c8e0912cf146fa491653acd41213660eb7 Mon Sep 17 00:00:00 2001 From: pyiron-runner Date: Tue, 15 Jul 2025 13:11:17 +0000 Subject: [PATCH 07/25] Format black --- executorlib/backend/interactive_parallel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index 18bc886d..71d6ee9b 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -83,7 +83,9 @@ def main() -> None: result_dict={"error": error}, ) if input_dict.get("write_error_file", False): - with open(input_dict.get("error_file_name", "error.out"), "a") as f: + with open( + input_dict.get("error_file_name", "error.out"), "a" + ) as f: if hasattr(error, "output"): f.write(error.output) else: From 4f51280555ff2858cd08c576302f645b8cbbda23 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 15 Jul 2025 15:13:16 +0200 Subject: [PATCH 08/25] write exception as string --- executorlib/backend/cache_parallel.py | 2 +- executorlib/backend/interactive_parallel.py | 2 +- executorlib/backend/interactive_serial.py | 2 +- executorlib/task_scheduler/file/backend.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index d71a47ad..4afb5375 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -60,7 +60,7 @@ def main() -> None: if hasattr(error, "output"): f.write(error.output) else: - f.write(error) + f.write(str(error)) else: if mpi_rank_zero: backend_write_file( diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index 71d6ee9b..04d0c00b 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -89,7 +89,7 @@ def main() -> None: if hasattr(error, "output"): f.write(error.output) else: - f.write(error) + f.write(str(error)) else: # Send output if mpi_rank_zero: diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index 88443c1d..ed0a02d2 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -64,7 +64,7 @@ def main(argument_lst: Optional[list[str]] = None): if hasattr(error, "output"): f.write(error.output) else: - f.write(error) + f.write(str(error)) else: # Send output interface_send(socket=socket, result_dict={"result": output}) diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index 05a4e892..8ce32921 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -82,7 +82,7 @@ def backend_execute_task_in_file(file_name: str) -> None: if hasattr(error, "output"): f.write(error.output) else: - f.write(error) + f.write(str(error)) backend_write_file( file_name=file_name, From 0f1e8d358b82f5e1c8a3acd3e6815672f6d992df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 16 Jul 2025 13:31:43 +0200 Subject: [PATCH 09/25] shared function --- executorlib/backend/cache_parallel.py | 11 ++++---- executorlib/backend/interactive_parallel.py | 14 ++++------ executorlib/backend/interactive_serial.py | 12 ++++---- executorlib/task_scheduler/file/backend.py | 31 +++++++++++++++++---- 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index 4afb5375..35447db5 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -7,6 +7,7 @@ from executorlib.task_scheduler.file.backend import ( backend_load_file, backend_write_file, + backend_write_error_file ) write_error_file = False @@ -55,12 +56,10 @@ def main() -> None: output={"error": error}, runtime=time.time() - time_start, ) - if apply_dict.get("write_error_file", False): - with open(apply_dict.get("error_file_name", "error.out"), "a") as f: - if hasattr(error, "output"): - f.write(error.output) - else: - f.write(str(error)) + backend_write_error_file( + error=error, + apply_dict=apply_dict, + ) else: if mpi_rank_zero: backend_write_file( diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index 04d0c00b..daecd36f 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -6,7 +6,7 @@ import cloudpickle import zmq -from executorlib.standalone.interactive.backend import call_funct, parse_arguments +from executorlib.standalone.interactive.backend import backend_write_error_file, call_funct, parse_arguments from executorlib.standalone.interactive.communication import ( interface_connect, interface_receive, @@ -82,14 +82,10 @@ def main() -> None: socket=socket, result_dict={"error": error}, ) - if input_dict.get("write_error_file", False): - with open( - input_dict.get("error_file_name", "error.out"), "a" - ) as f: - if hasattr(error, "output"): - f.write(error.output) - else: - f.write(str(error)) + backend_write_error_file( + error=error, + apply_dict=input_dict, + ) else: # Send output if mpi_rank_zero: diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index ed0a02d2..fe851c28 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -2,7 +2,7 @@ from os.path import abspath from typing import Optional -from executorlib.standalone.interactive.backend import call_funct, parse_arguments +from executorlib.standalone.interactive.backend import backend_write_error_file, call_funct, parse_arguments from executorlib.standalone.interactive.communication import ( interface_connect, interface_receive, @@ -59,12 +59,10 @@ def main(argument_lst: Optional[list[str]] = None): socket=socket, result_dict={"error": error}, ) - if input_dict.get("write_error_file", False): - with open(input_dict.get("error_file_name", "error.out"), "a") as f: - if hasattr(error, "output"): - f.write(error.output) - else: - f.write(str(error)) + backend_write_error_file( + error=error, + apply_dict=input_dict, + ) else: # Send output interface_send(socket=socket, result_dict={"result": output}) diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index 8ce32921..e994a784 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -77,12 +77,10 @@ def backend_execute_task_in_file(file_name: str) -> None: } except Exception as error: result = {"error": error} - if apply_dict.get("write_error_file", False): - with open(apply_dict.get("error_file_name", "error.out"), "a") as f: - if hasattr(error, "output"): - f.write(error.output) - else: - f.write(str(error)) + backend_write_error_file( + error=error, + apply_dict=apply_dict, + ) backend_write_file( file_name=file_name, @@ -91,5 +89,26 @@ def backend_execute_task_in_file(file_name: str) -> None: ) +def backend_write_error_file( + error: Exception, apply_dict: dict +) -> None: + """ + Write an error to a file if specified in the apply_dict. + + Args: + error (Exception): The error to be written. + apply_dict (dict): Dictionary containing additional parameters. + + Returns: + None + """ + if apply_dict.get("write_error_file", False): + with open(apply_dict.get("error_file_name", "error.out"), "a") as f: + if hasattr(error, "output"): + f.write(error.output) + else: + f.write(str(error)) + + def _isinstance(obj: Any, cls: type) -> bool: return str(obj.__class__) == str(cls) From a5f05a5989c18ee94551a454a4e989c9a9f25ffa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 11:31:59 +0000 Subject: [PATCH 10/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/backend/cache_parallel.py | 2 +- executorlib/backend/interactive_parallel.py | 6 +++++- executorlib/backend/interactive_serial.py | 6 +++++- executorlib/task_scheduler/file/backend.py | 4 +--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index 35447db5..4d3a43ce 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -6,8 +6,8 @@ from executorlib.task_scheduler.file.backend import ( backend_load_file, + backend_write_error_file, backend_write_file, - backend_write_error_file ) write_error_file = False diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index daecd36f..f9bff967 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -6,7 +6,11 @@ import cloudpickle import zmq -from executorlib.standalone.interactive.backend import backend_write_error_file, call_funct, parse_arguments +from executorlib.standalone.interactive.backend import ( + backend_write_error_file, + call_funct, + parse_arguments, +) from executorlib.standalone.interactive.communication import ( interface_connect, interface_receive, diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index fe851c28..1da346d2 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -2,7 +2,11 @@ from os.path import abspath from typing import Optional -from executorlib.standalone.interactive.backend import backend_write_error_file, call_funct, parse_arguments +from executorlib.standalone.interactive.backend import ( + backend_write_error_file, + call_funct, + parse_arguments, +) from executorlib.standalone.interactive.communication import ( interface_connect, interface_receive, diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index e994a784..255b59d8 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -89,9 +89,7 @@ def backend_execute_task_in_file(file_name: str) -> None: ) -def backend_write_error_file( - error: Exception, apply_dict: dict -) -> None: +def backend_write_error_file(error: Exception, apply_dict: dict) -> None: """ Write an error to a file if specified in the apply_dict. From 87000cfbc6d6ec9b5efdf28cac509943e4e224f9 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 14:07:14 +0200 Subject: [PATCH 11/25] move function --- executorlib/backend/cache_parallel.py | 2 +- executorlib/backend/interactive_parallel.py | 2 +- executorlib/backend/interactive_serial.py | 1 + executorlib/standalone/error.py | 17 +++++++++++++++++ executorlib/task_scheduler/file/backend.py | 20 +------------------- 5 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 executorlib/standalone/error.py diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index 4d3a43ce..2cc9ae82 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -4,9 +4,9 @@ import cloudpickle +from executorlib.standalone.error import backend_write_error_file from executorlib.task_scheduler.file.backend import ( backend_load_file, - backend_write_error_file, backend_write_file, ) diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index f9bff967..6e50ecf0 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -6,8 +6,8 @@ import cloudpickle import zmq +from executorlib.standalone.error import backend_write_error_file from executorlib.standalone.interactive.backend import ( - backend_write_error_file, call_funct, parse_arguments, ) diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index 1da346d2..27609107 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -2,6 +2,7 @@ from os.path import abspath from typing import Optional +from executorlib.standalone.error import backend_write_error_file from executorlib.standalone.interactive.backend import ( backend_write_error_file, call_funct, diff --git a/executorlib/standalone/error.py b/executorlib/standalone/error.py new file mode 100644 index 00000000..608e6b6a --- /dev/null +++ b/executorlib/standalone/error.py @@ -0,0 +1,17 @@ +def backend_write_error_file(error: Exception, apply_dict: dict) -> None: + """ + Write an error to a file if specified in the apply_dict. + + Args: + error (Exception): The error to be written. + apply_dict (dict): Dictionary containing additional parameters. + + Returns: + None + """ + if apply_dict.get("write_error_file", False): + with open(apply_dict.get("error_file_name", "error.out"), "a") as f: + if hasattr(error, "output"): + f.write(error.output) + else: + f.write(str(error)) \ No newline at end of file diff --git a/executorlib/task_scheduler/file/backend.py b/executorlib/task_scheduler/file/backend.py index 255b59d8..cbe869cc 100644 --- a/executorlib/task_scheduler/file/backend.py +++ b/executorlib/task_scheduler/file/backend.py @@ -2,6 +2,7 @@ import time from typing import Any +from executorlib.standalone.error import backend_write_error_file from executorlib.task_scheduler.file.hdf import dump, load from executorlib.task_scheduler.file.shared import FutureItem @@ -89,24 +90,5 @@ def backend_execute_task_in_file(file_name: str) -> None: ) -def backend_write_error_file(error: Exception, apply_dict: dict) -> None: - """ - Write an error to a file if specified in the apply_dict. - - Args: - error (Exception): The error to be written. - apply_dict (dict): Dictionary containing additional parameters. - - Returns: - None - """ - if apply_dict.get("write_error_file", False): - with open(apply_dict.get("error_file_name", "error.out"), "a") as f: - if hasattr(error, "output"): - f.write(error.output) - else: - f.write(str(error)) - - def _isinstance(obj: Any, cls: type) -> bool: return str(obj.__class__) == str(cls) From f2a3abfbe35ff018a8d36af93aef0b83ed1ac59b Mon Sep 17 00:00:00 2001 From: pyiron-runner Date: Wed, 16 Jul 2025 12:08:09 +0000 Subject: [PATCH 12/25] Format black --- executorlib/standalone/error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executorlib/standalone/error.py b/executorlib/standalone/error.py index 608e6b6a..3655f141 100644 --- a/executorlib/standalone/error.py +++ b/executorlib/standalone/error.py @@ -14,4 +14,4 @@ def backend_write_error_file(error: Exception, apply_dict: dict) -> None: if hasattr(error, "output"): f.write(error.output) else: - f.write(str(error)) \ No newline at end of file + f.write(str(error)) From cf1fef72ea780f7e96e04332f9287b72b71fb460 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 14:13:27 +0200 Subject: [PATCH 13/25] bug fix --- executorlib/backend/interactive_serial.py | 1 - 1 file changed, 1 deletion(-) diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index 27609107..8b2f9187 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -4,7 +4,6 @@ from executorlib.standalone.error import backend_write_error_file from executorlib.standalone.interactive.backend import ( - backend_write_error_file, call_funct, parse_arguments, ) From e6f76053d92085425257b6bca2cee9296d4921a2 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 15:13:23 +0200 Subject: [PATCH 14/25] check error file is written --- executorlib/executor/flux.py | 13 ++++++++++++ executorlib/executor/single.py | 13 ++++++++++++ executorlib/executor/slurm.py | 13 ++++++++++++ executorlib/standalone/cache.py | 1 + executorlib/standalone/error.py | 8 ++++---- executorlib/task_scheduler/file/hdf.py | 2 ++ executorlib/task_scheduler/file/shared.py | 3 +++ .../task_scheduler/file/task_scheduler.py | 5 +++++ .../task_scheduler/interactive/shared.py | 3 +++ tests/test_cache_fileexecutor_serial.py | 20 ++++++++++++++++--- tests/test_singlenodeexecutor_cache.py | 14 ++++++++++++- 11 files changed, 87 insertions(+), 8 deletions(-) diff --git a/executorlib/executor/flux.py b/executorlib/executor/flux.py index 9291fa40..9ab2882d 100644 --- a/executorlib/executor/flux.py +++ b/executorlib/executor/flux.py @@ -65,6 +65,7 @@ class FluxJobExecutor(BaseExecutor): plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Examples: ``` @@ -106,6 +107,7 @@ def __init__( plot_dependency_graph_filename: Optional[str] = None, log_obj_size: bool = False, terminate_tasks_on_shutdown: bool = True, + write_error_file: bool = False, ): """ The executorlib.FluxJobExecutor leverages either the message passing interface (MPI), the SLURM workload manager @@ -152,6 +154,7 @@ def __init__( plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ default_resource_dict: dict = { @@ -186,6 +189,7 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, + write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -210,6 +214,7 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, + write_error_file=write_error_file, ) ) @@ -256,6 +261,7 @@ class FluxClusterExecutor(BaseExecutor): plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Examples: ``` @@ -294,6 +300,7 @@ def __init__( plot_dependency_graph_filename: Optional[str] = None, log_obj_size: bool = False, terminate_tasks_on_shutdown: bool = True, + write_error_file: bool = False, ): """ The executorlib.FluxClusterExecutor leverages either the message passing interface (MPI), the SLURM workload @@ -337,6 +344,7 @@ def __init__( plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ default_resource_dict: dict = { @@ -377,6 +385,7 @@ def __init__( init_function=init_function, disable_dependencies=disable_dependencies, terminate_tasks_on_shutdown=terminate_tasks_on_shutdown, + write_error_file=write_error_file, ) ) else: @@ -394,6 +403,7 @@ def __init__( hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, + write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -416,6 +426,7 @@ def create_flux_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, log_obj_size: bool = False, + write_error_file: bool = False, ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: """ Create a flux executor @@ -452,6 +463,7 @@ def create_flux_executor( of the individual function. init_function (None): optional function to preset arguments for functions which are submitted later log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: InteractiveStepExecutor/ InteractiveExecutor @@ -467,6 +479,7 @@ def create_flux_executor( resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost resource_dict["log_obj_size"] = log_obj_size + resource_dict["write_error_file"] = write_error_file check_init_function(block_allocation=block_allocation, init_function=init_function) check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode) check_oversubscribe(oversubscribe=resource_dict.get("openmpi_oversubscribe", False)) diff --git a/executorlib/executor/single.py b/executorlib/executor/single.py index 30037505..3ff6f57d 100644 --- a/executorlib/executor/single.py +++ b/executorlib/executor/single.py @@ -57,6 +57,7 @@ class SingleNodeExecutor(BaseExecutor): debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Examples: ``` @@ -93,6 +94,7 @@ def __init__( plot_dependency_graph: bool = False, plot_dependency_graph_filename: Optional[str] = None, log_obj_size: bool = False, + write_error_file: bool = False, ): """ The executorlib.SingleNodeExecutor leverages either the message passing interface (MPI), the SLURM workload @@ -134,6 +136,7 @@ def __init__( debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ default_resource_dict: dict = { @@ -161,6 +164,7 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, + write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -181,6 +185,7 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, + write_error_file=write_error_file, ) ) @@ -220,6 +225,7 @@ class TestClusterExecutor(BaseExecutor): debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Examples: ``` @@ -256,6 +262,7 @@ def __init__( plot_dependency_graph: bool = False, plot_dependency_graph_filename: Optional[str] = None, log_obj_size: bool = False, + write_error_file: bool = False, ): """ The executorlib.api.TestClusterExecutor is designed to test the file based communication used in the @@ -291,6 +298,7 @@ def __init__( debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ default_resource_dict: dict = { @@ -330,6 +338,7 @@ def __init__( init_function=init_function, disable_dependencies=disable_dependencies, execute_function=execute_in_subprocess, + write_error_file=write_error_file, ) ) else: @@ -344,6 +353,7 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, + write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -362,6 +372,7 @@ def create_single_node_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, log_obj_size: bool = False, + write_error_file: bool = False, ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: """ Create a single node executor @@ -394,6 +405,7 @@ def create_single_node_executor( of the individual function. init_function (None): optional function to preset arguments for functions which are submitted later log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: InteractiveStepExecutor/ InteractiveExecutor @@ -404,6 +416,7 @@ def create_single_node_executor( resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost resource_dict["log_obj_size"] = log_obj_size + resource_dict["write_error_file"] = write_error_file check_init_function(block_allocation=block_allocation, init_function=init_function) check_gpus_per_worker(gpus_per_worker=resource_dict.get("gpus_per_core", 0)) diff --git a/executorlib/executor/slurm.py b/executorlib/executor/slurm.py index 3bab3c19..bcb35117 100644 --- a/executorlib/executor/slurm.py +++ b/executorlib/executor/slurm.py @@ -62,6 +62,7 @@ class SlurmClusterExecutor(BaseExecutor): plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Examples: ``` @@ -100,6 +101,7 @@ def __init__( plot_dependency_graph_filename: Optional[str] = None, log_obj_size: bool = False, terminate_tasks_on_shutdown: bool = True, + write_error_file: bool = False, ): """ The executorlib.SlurmClusterExecutor leverages either the message passing interface (MPI), the SLURM workload @@ -143,6 +145,7 @@ def __init__( plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ default_resource_dict: dict = { @@ -183,6 +186,7 @@ def __init__( init_function=init_function, disable_dependencies=disable_dependencies, terminate_tasks_on_shutdown=terminate_tasks_on_shutdown, + write_error_file=write_error_file, ) ) else: @@ -196,6 +200,7 @@ def __init__( hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, + write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -250,6 +255,7 @@ class SlurmJobExecutor(BaseExecutor): plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Examples: ``` @@ -287,6 +293,7 @@ def __init__( plot_dependency_graph_filename: Optional[str] = None, log_obj_size: bool = False, terminate_tasks_on_shutdown: bool = True, + write_error_file: bool = False, ): """ The executorlib.SlurmJobExecutor leverages either the message passing interface (MPI), the SLURM workload @@ -333,6 +340,7 @@ def __init__( plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. terminate_tasks_on_shutdown (bool): Shutdown all tasks when the Executor is shutdown, this is the default. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ default_resource_dict: dict = { @@ -363,6 +371,7 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, + write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -383,6 +392,7 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, + write_error_file=write_error_file, ) ) @@ -396,6 +406,7 @@ def create_slurm_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, log_obj_size: bool = False, + write_error_file: bool = False, ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: """ Create a SLURM executor @@ -432,6 +443,7 @@ def create_slurm_executor( of the individual function. init_function (None): optional function to preset arguments for functions which are submitted later log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: InteractiveStepExecutor/ InteractiveExecutor @@ -442,6 +454,7 @@ def create_slurm_executor( resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost resource_dict["log_obj_size"] = log_obj_size + resource_dict["write_error_file"] = write_error_file check_init_function(block_allocation=block_allocation, init_function=init_function) if block_allocation: resource_dict["init_function"] = init_function diff --git a/executorlib/standalone/cache.py b/executorlib/standalone/cache.py index c3d89483..26a375e4 100644 --- a/executorlib/standalone/cache.py +++ b/executorlib/standalone/cache.py @@ -10,6 +10,7 @@ "error": "error", "runtime": "runtime", "queue_id": "queue_id", + "write_error_file": "write_error_file" } diff --git a/executorlib/standalone/error.py b/executorlib/standalone/error.py index 3655f141..68a945d3 100644 --- a/executorlib/standalone/error.py +++ b/executorlib/standalone/error.py @@ -1,3 +1,6 @@ +import traceback + + def backend_write_error_file(error: Exception, apply_dict: dict) -> None: """ Write an error to a file if specified in the apply_dict. @@ -11,7 +14,4 @@ def backend_write_error_file(error: Exception, apply_dict: dict) -> None: """ if apply_dict.get("write_error_file", False): with open(apply_dict.get("error_file_name", "error.out"), "a") as f: - if hasattr(error, "output"): - f.write(error.output) - else: - f.write(str(error)) + traceback.print_exception(error, file=f) diff --git a/executorlib/task_scheduler/file/hdf.py b/executorlib/task_scheduler/file/hdf.py index 903d7f94..10f26591 100644 --- a/executorlib/task_scheduler/file/hdf.py +++ b/executorlib/task_scheduler/file/hdf.py @@ -52,6 +52,8 @@ def load(file_name: str) -> dict: data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"])) else: data_dict["kwargs"] = {} + if "write_error_file" in hdf: + data_dict["write_error_file"] = cloudpickle.loads(np.void(hdf["/write_error_file"])) return data_dict diff --git a/executorlib/task_scheduler/file/shared.py b/executorlib/task_scheduler/file/shared.py index 42eeee61..3f884f99 100644 --- a/executorlib/task_scheduler/file/shared.py +++ b/executorlib/task_scheduler/file/shared.py @@ -59,6 +59,7 @@ def execute_tasks_h5( pysqa_config_directory: Optional[str] = None, backend: Optional[str] = None, disable_dependencies: bool = False, + write_error_file: bool = False, ) -> None: """ Execute tasks stored in a queue using HDF5 files. @@ -73,6 +74,7 @@ def execute_tasks_h5( pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). backend (str, optional): name of the backend used to spawn tasks. disable_dependencies (boolean): Disable resolving future objects during the submission. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: None @@ -139,6 +141,7 @@ def execute_tasks_h5( file_name = os.path.join(cache_directory, task_key + "_i.h5") if os.path.exists(file_name): os.remove(file_name) + data_dict["write_error_file"] = write_error_file dump(file_name=file_name, data_dict=data_dict) if not disable_dependencies: task_dependent_lst = [ diff --git a/executorlib/task_scheduler/file/task_scheduler.py b/executorlib/task_scheduler/file/task_scheduler.py index 22cb3a48..1c566fd6 100644 --- a/executorlib/task_scheduler/file/task_scheduler.py +++ b/executorlib/task_scheduler/file/task_scheduler.py @@ -36,6 +36,7 @@ def __init__( pysqa_config_directory: Optional[str] = None, backend: Optional[str] = None, disable_dependencies: bool = False, + write_error_file: bool = False, ): """ Initialize the FileExecutor. @@ -50,6 +51,7 @@ def __init__( pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). backend (str, optional): name of the backend used to spawn tasks. disable_dependencies (boolean): Disable resolving future objects during the submission. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ super().__init__(max_cores=None) default_resource_dict = { @@ -70,6 +72,7 @@ def __init__( "pysqa_config_directory": pysqa_config_directory, "backend": backend, "disable_dependencies": disable_dependencies, + "write_error_file": write_error_file, } self._set_process( Thread( @@ -96,6 +99,7 @@ def create_file_executor( disable_dependencies: bool = False, execute_function: Callable = execute_with_pysqa, terminate_tasks_on_shutdown: bool = True, + write_error_file: bool = False, ): if block_allocation: raise ValueError( @@ -126,4 +130,5 @@ def create_file_executor( disable_dependencies=disable_dependencies, execute_function=execute_function, terminate_function=terminate_function, + write_error_file=write_error_file, ) diff --git a/executorlib/task_scheduler/interactive/shared.py b/executorlib/task_scheduler/interactive/shared.py index 7c27e043..e613eea7 100644 --- a/executorlib/task_scheduler/interactive/shared.py +++ b/executorlib/task_scheduler/interactive/shared.py @@ -26,6 +26,7 @@ def execute_tasks( cache_key: Optional[str] = None, queue_join_on_shutdown: bool = True, log_obj_size: bool = False, + write_error_file: bool = False, **kwargs, ) -> None: """ @@ -48,6 +49,7 @@ def execute_tasks( overwritten by setting the cache_key. queue_join_on_shutdown (bool): Join communication queue when thread is closed. Defaults to True. log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails """ interface = interface_bootup( command_lst=_get_backend_path( @@ -70,6 +72,7 @@ def execute_tasks( future_queue.join() break elif "fn" in task_dict and "future" in task_dict: + task_dict["write_error_file"] = write_error_file if cache_directory is None: _execute_task_without_cache( interface=interface, task_dict=task_dict, future_queue=future_queue diff --git a/tests/test_cache_fileexecutor_serial.py b/tests/test_cache_fileexecutor_serial.py index eb62c166..0395a0cf 100644 --- a/tests/test_cache_fileexecutor_serial.py +++ b/tests/test_cache_fileexecutor_serial.py @@ -84,11 +84,25 @@ def test_executor_working_directory(self): def test_executor_error(self): cwd = os.path.join(os.path.dirname(__file__), "executables") with FileTaskScheduler( - resource_dict={"cwd": cwd}, execute_function=execute_in_subprocess + resource_dict={"cwd": cwd}, execute_function=execute_in_subprocess, write_error_file=False, + ) as exe: + fs1 = exe.submit(get_error, a=1) + with self.assertRaises(ValueError): + fs1.result() + self.assertEqual(len(os.listdir(cwd)), 1) + + def test_executor_error_file(self): + cwd = os.path.join(os.path.dirname(__file__), "executables") + with FileTaskScheduler( + resource_dict={"cwd": cwd}, execute_function=execute_in_subprocess, write_error_file=True, ) as exe: fs1 = exe.submit(get_error, a=1) with self.assertRaises(ValueError): fs1.result() + working_directory_file_lst = os.listdir(cwd) + self.assertEqual(len(working_directory_file_lst), 2) + self.assertTrue("error.out" in working_directory_file_lst) + os.remove(os.path.join(cwd, "error.out")) def test_executor_function(self): fs1 = Future() @@ -208,5 +222,5 @@ def test_execute_in_subprocess_errors(self): with self.assertRaises(ValueError): execute_in_subprocess(file_name=__file__, command=[], backend="flux") - def tearDown(self): - shutil.rmtree("executorlib_cache", ignore_errors=True) + # def tearDown(self): + # shutil.rmtree("executorlib_cache", ignore_errors=True) diff --git a/tests/test_singlenodeexecutor_cache.py b/tests/test_singlenodeexecutor_cache.py index 531d369d..a7ed7367 100644 --- a/tests/test_singlenodeexecutor_cache.py +++ b/tests/test_singlenodeexecutor_cache.py @@ -51,12 +51,24 @@ def test_cache_key(self): def test_cache_error(self): cache_directory = os.path.abspath("cache_error") - with SingleNodeExecutor(cache_directory=cache_directory) as exe: + with SingleNodeExecutor(cache_directory=cache_directory, write_error_file=False) as exe: + self.assertTrue(exe) + cloudpickle_register(ind=1) + f = exe.submit(get_error, a=1) + with self.assertRaises(ValueError): + print(f.result()) + + def test_cache_error_file(self): + cache_directory = os.path.abspath("cache_error") + with SingleNodeExecutor(cache_directory=cache_directory, write_error_file=True) as exe: self.assertTrue(exe) cloudpickle_register(ind=1) f = exe.submit(get_error, a=1) with self.assertRaises(ValueError): print(f.result()) + error_out = os.path.join(os.path.dirname(__file__), "error.out") + self.assertTrue(os.path.exists(error_out)) + os.remove(error_out) def tearDown(self): shutil.rmtree("executorlib_cache", ignore_errors=True) From 481ba85cd8f2b211ab122ed7525394f5479edb02 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 13:13:32 +0000 Subject: [PATCH 15/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/standalone/cache.py | 2 +- executorlib/task_scheduler/file/hdf.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/executorlib/standalone/cache.py b/executorlib/standalone/cache.py index 26a375e4..ec0ddcd2 100644 --- a/executorlib/standalone/cache.py +++ b/executorlib/standalone/cache.py @@ -10,7 +10,7 @@ "error": "error", "runtime": "runtime", "queue_id": "queue_id", - "write_error_file": "write_error_file" + "write_error_file": "write_error_file", } diff --git a/executorlib/task_scheduler/file/hdf.py b/executorlib/task_scheduler/file/hdf.py index 10f26591..5eb816f7 100644 --- a/executorlib/task_scheduler/file/hdf.py +++ b/executorlib/task_scheduler/file/hdf.py @@ -53,7 +53,9 @@ def load(file_name: str) -> dict: else: data_dict["kwargs"] = {} if "write_error_file" in hdf: - data_dict["write_error_file"] = cloudpickle.loads(np.void(hdf["/write_error_file"])) + data_dict["write_error_file"] = cloudpickle.loads( + np.void(hdf["/write_error_file"]) + ) return data_dict From f6e90d9cea732b16530505929aec12fd44667656 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 15:15:04 +0200 Subject: [PATCH 16/25] clean up --- executorlib/backend/cache_parallel.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/executorlib/backend/cache_parallel.py b/executorlib/backend/cache_parallel.py index 2cc9ae82..49ad6d6e 100644 --- a/executorlib/backend/cache_parallel.py +++ b/executorlib/backend/cache_parallel.py @@ -10,8 +10,6 @@ backend_write_file, ) -write_error_file = False - def main() -> None: """ From d67055a724b46abac57906178632e6cacb7ceb5d Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 15:20:11 +0200 Subject: [PATCH 17/25] Add more details in log --- executorlib/standalone/error.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/executorlib/standalone/error.py b/executorlib/standalone/error.py index 68a945d3..ccbb5ac1 100644 --- a/executorlib/standalone/error.py +++ b/executorlib/standalone/error.py @@ -14,4 +14,7 @@ def backend_write_error_file(error: Exception, apply_dict: dict) -> None: """ if apply_dict.get("write_error_file", False): with open(apply_dict.get("error_file_name", "error.out"), "a") as f: + f.write("function: " + str(apply_dict["fn"]) + "\n") + f.write("args: " + str(apply_dict["args"])+ "\n") + f.write("kwargs: " + str(apply_dict["kwargs"])+ "\n") traceback.print_exception(error, file=f) From ac9ceea46efb4817f27c9f4a3d2b1e696dc00e7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 13:20:19 +0000 Subject: [PATCH 18/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/standalone/error.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/executorlib/standalone/error.py b/executorlib/standalone/error.py index ccbb5ac1..2dd31ca7 100644 --- a/executorlib/standalone/error.py +++ b/executorlib/standalone/error.py @@ -15,6 +15,6 @@ def backend_write_error_file(error: Exception, apply_dict: dict) -> None: if apply_dict.get("write_error_file", False): with open(apply_dict.get("error_file_name", "error.out"), "a") as f: f.write("function: " + str(apply_dict["fn"]) + "\n") - f.write("args: " + str(apply_dict["args"])+ "\n") - f.write("kwargs: " + str(apply_dict["kwargs"])+ "\n") + f.write("args: " + str(apply_dict["args"]) + "\n") + f.write("kwargs: " + str(apply_dict["kwargs"]) + "\n") traceback.print_exception(error, file=f) From db577260a8fe2b6d68091ca4ebb02a2f358eafab Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 15:24:01 +0200 Subject: [PATCH 19/25] fix formatting --- executorlib/backend/interactive_parallel.py | 5 +---- executorlib/backend/interactive_serial.py | 7 ++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/executorlib/backend/interactive_parallel.py b/executorlib/backend/interactive_parallel.py index 6e50ecf0..3d5aadcc 100644 --- a/executorlib/backend/interactive_parallel.py +++ b/executorlib/backend/interactive_parallel.py @@ -7,10 +7,7 @@ import zmq from executorlib.standalone.error import backend_write_error_file -from executorlib.standalone.interactive.backend import ( - call_funct, - parse_arguments, -) +from executorlib.standalone.interactive.backend import call_funct, parse_arguments from executorlib.standalone.interactive.communication import ( interface_connect, interface_receive, diff --git a/executorlib/backend/interactive_serial.py b/executorlib/backend/interactive_serial.py index 8b2f9187..8f9088cd 100644 --- a/executorlib/backend/interactive_serial.py +++ b/executorlib/backend/interactive_serial.py @@ -3,10 +3,7 @@ from typing import Optional from executorlib.standalone.error import backend_write_error_file -from executorlib.standalone.interactive.backend import ( - call_funct, - parse_arguments, -) +from executorlib.standalone.interactive.backend import call_funct, parse_arguments from executorlib.standalone.interactive.communication import ( interface_connect, interface_receive, @@ -21,7 +18,7 @@ def main(argument_lst: Optional[list[str]] = None): Args: argument_lst (Optional[List[str]]): List of command line arguments. If None, sys.argv will be used. - write_error_file (bool): + write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: None From f37eea1c36b0f75b593fe17b1684fad9eba0cd59 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 15:38:26 +0200 Subject: [PATCH 20/25] fix info tests --- tests/test_singlenodeexecutor_dependencies.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_singlenodeexecutor_dependencies.py b/tests/test_singlenodeexecutor_dependencies.py index cc8ea515..9568d09b 100644 --- a/tests/test_singlenodeexecutor_dependencies.py +++ b/tests/test_singlenodeexecutor_dependencies.py @@ -345,6 +345,7 @@ def setUp(self): 'spawner': MpiExecSpawner, 'max_cores': None, 'max_workers': None, + 'write_error_file': False } def test_info_disable_dependencies_true(self): From e9ed69286a239b710705bca3a9895a1ea5f46996 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 15:56:57 +0200 Subject: [PATCH 21/25] set write_error_file in resource_dict --- executorlib/executor/flux.py | 9 ++------- executorlib/executor/single.py | 9 ++------- executorlib/executor/slurm.py | 9 ++------- executorlib/task_scheduler/file/shared.py | 5 ++--- executorlib/task_scheduler/file/task_scheduler.py | 2 +- 5 files changed, 9 insertions(+), 25 deletions(-) diff --git a/executorlib/executor/flux.py b/executorlib/executor/flux.py index 9ab2882d..b8c27c9d 100644 --- a/executorlib/executor/flux.py +++ b/executorlib/executor/flux.py @@ -164,6 +164,7 @@ def __init__( "cwd": None, "openmpi_oversubscribe": False, "slurm_cmd_args": [], + "write_error_file": write_error_file, } if resource_dict is None: resource_dict = {} @@ -189,7 +190,6 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, - write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -214,7 +214,6 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, - write_error_file=write_error_file, ) ) @@ -354,6 +353,7 @@ def __init__( "cwd": None, "openmpi_oversubscribe": False, "slurm_cmd_args": [], + "write_error_file": write_error_file, } if resource_dict is None: resource_dict = {} @@ -385,7 +385,6 @@ def __init__( init_function=init_function, disable_dependencies=disable_dependencies, terminate_tasks_on_shutdown=terminate_tasks_on_shutdown, - write_error_file=write_error_file, ) ) else: @@ -403,7 +402,6 @@ def __init__( hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, - write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -426,7 +424,6 @@ def create_flux_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, log_obj_size: bool = False, - write_error_file: bool = False, ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: """ Create a flux executor @@ -463,7 +460,6 @@ def create_flux_executor( of the individual function. init_function (None): optional function to preset arguments for functions which are submitted later log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. - write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: InteractiveStepExecutor/ InteractiveExecutor @@ -479,7 +475,6 @@ def create_flux_executor( resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost resource_dict["log_obj_size"] = log_obj_size - resource_dict["write_error_file"] = write_error_file check_init_function(block_allocation=block_allocation, init_function=init_function) check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode) check_oversubscribe(oversubscribe=resource_dict.get("openmpi_oversubscribe", False)) diff --git a/executorlib/executor/single.py b/executorlib/executor/single.py index 3ff6f57d..018aa1fc 100644 --- a/executorlib/executor/single.py +++ b/executorlib/executor/single.py @@ -146,6 +146,7 @@ def __init__( "cwd": None, "openmpi_oversubscribe": False, "slurm_cmd_args": [], + "write_error_file": write_error_file, } if resource_dict is None: resource_dict = {} @@ -164,7 +165,6 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, - write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -185,7 +185,6 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, - write_error_file=write_error_file, ) ) @@ -307,6 +306,7 @@ def __init__( "gpus_per_core": 0, "cwd": None, "openmpi_oversubscribe": False, + "write_error_file": write_error_file, } if resource_dict is None: resource_dict = {} @@ -338,7 +338,6 @@ def __init__( init_function=init_function, disable_dependencies=disable_dependencies, execute_function=execute_in_subprocess, - write_error_file=write_error_file, ) ) else: @@ -353,7 +352,6 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, - write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -372,7 +370,6 @@ def create_single_node_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, log_obj_size: bool = False, - write_error_file: bool = False, ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: """ Create a single node executor @@ -405,7 +402,6 @@ def create_single_node_executor( of the individual function. init_function (None): optional function to preset arguments for functions which are submitted later log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. - write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: InteractiveStepExecutor/ InteractiveExecutor @@ -416,7 +412,6 @@ def create_single_node_executor( resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost resource_dict["log_obj_size"] = log_obj_size - resource_dict["write_error_file"] = write_error_file check_init_function(block_allocation=block_allocation, init_function=init_function) check_gpus_per_worker(gpus_per_worker=resource_dict.get("gpus_per_core", 0)) diff --git a/executorlib/executor/slurm.py b/executorlib/executor/slurm.py index bcb35117..da507f2b 100644 --- a/executorlib/executor/slurm.py +++ b/executorlib/executor/slurm.py @@ -155,6 +155,7 @@ def __init__( "cwd": None, "openmpi_oversubscribe": False, "slurm_cmd_args": [], + "write_error_file": write_error_file, } if resource_dict is None: resource_dict = {} @@ -186,7 +187,6 @@ def __init__( init_function=init_function, disable_dependencies=disable_dependencies, terminate_tasks_on_shutdown=terminate_tasks_on_shutdown, - write_error_file=write_error_file, ) ) else: @@ -200,7 +200,6 @@ def __init__( hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, - write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -350,6 +349,7 @@ def __init__( "cwd": None, "openmpi_oversubscribe": False, "slurm_cmd_args": [], + "write_error_file": write_error_file, } if resource_dict is None: resource_dict = {} @@ -371,7 +371,6 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, - write_error_file=write_error_file, ), max_cores=max_cores, refresh_rate=refresh_rate, @@ -392,7 +391,6 @@ def __init__( block_allocation=block_allocation, init_function=init_function, log_obj_size=log_obj_size, - write_error_file=write_error_file, ) ) @@ -406,7 +404,6 @@ def create_slurm_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, log_obj_size: bool = False, - write_error_file: bool = False, ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: """ Create a SLURM executor @@ -443,7 +440,6 @@ def create_slurm_executor( of the individual function. init_function (None): optional function to preset arguments for functions which are submitted later log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. - write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: InteractiveStepExecutor/ InteractiveExecutor @@ -454,7 +450,6 @@ def create_slurm_executor( resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost resource_dict["log_obj_size"] = log_obj_size - resource_dict["write_error_file"] = write_error_file check_init_function(block_allocation=block_allocation, init_function=init_function) if block_allocation: resource_dict["init_function"] = init_function diff --git a/executorlib/task_scheduler/file/shared.py b/executorlib/task_scheduler/file/shared.py index 3f884f99..288affd4 100644 --- a/executorlib/task_scheduler/file/shared.py +++ b/executorlib/task_scheduler/file/shared.py @@ -59,7 +59,6 @@ def execute_tasks_h5( pysqa_config_directory: Optional[str] = None, backend: Optional[str] = None, disable_dependencies: bool = False, - write_error_file: bool = False, ) -> None: """ Execute tasks stored in a queue using HDF5 files. @@ -74,7 +73,6 @@ def execute_tasks_h5( pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). backend (str, optional): name of the backend used to spawn tasks. disable_dependencies (boolean): Disable resolving future objects during the submission. - write_error_file (boolean): Enable writing error.out files when the computation of a Python function fails Returns: None @@ -127,6 +125,7 @@ def execute_tasks_h5( ) cache_key = task_resource_dict.pop("cache_key", None) cache_directory = os.path.abspath(task_resource_dict.pop("cache_directory")) + write_error_file = task_resource_dict.pop("write_error_file", False) task_key, data_dict = serialize_funct_h5( fn=task_dict["fn"], fn_args=task_args, @@ -134,6 +133,7 @@ def execute_tasks_h5( resource_dict=task_resource_dict, cache_key=cache_key, ) + data_dict["write_error_file"] = write_error_file if task_key not in memory_dict: if os.path.join( cache_directory, task_key + "_o.h5" @@ -141,7 +141,6 @@ def execute_tasks_h5( file_name = os.path.join(cache_directory, task_key + "_i.h5") if os.path.exists(file_name): os.remove(file_name) - data_dict["write_error_file"] = write_error_file dump(file_name=file_name, data_dict=data_dict) if not disable_dependencies: task_dependent_lst = [ diff --git a/executorlib/task_scheduler/file/task_scheduler.py b/executorlib/task_scheduler/file/task_scheduler.py index 1c566fd6..fa491085 100644 --- a/executorlib/task_scheduler/file/task_scheduler.py +++ b/executorlib/task_scheduler/file/task_scheduler.py @@ -58,6 +58,7 @@ def __init__( "cores": 1, "cwd": None, "cache_directory": "executorlib_cache", + "write_error_file": write_error_file, } if resource_dict is None: resource_dict = {} @@ -72,7 +73,6 @@ def __init__( "pysqa_config_directory": pysqa_config_directory, "backend": backend, "disable_dependencies": disable_dependencies, - "write_error_file": write_error_file, } self._set_process( Thread( From da0f1d867d8836e7241ae7079deea4a0fa8bf21e Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 17:04:20 +0200 Subject: [PATCH 22/25] fix test --- tests/test_cache_fileexecutor_serial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cache_fileexecutor_serial.py b/tests/test_cache_fileexecutor_serial.py index b9419dd3..617956fd 100644 --- a/tests/test_cache_fileexecutor_serial.py +++ b/tests/test_cache_fileexecutor_serial.py @@ -233,5 +233,5 @@ def test_execute_in_subprocess_errors(self): backend="flux", ) - # def tearDown(self): - # shutil.rmtree("executorlib_cache", ignore_errors=True) + def tearDown(self): + shutil.rmtree("executorlib_cache", ignore_errors=True) From db24c95d19c53cf62455994d7f113b133e0feddb Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 18:03:52 +0200 Subject: [PATCH 23/25] fix file location --- tests/test_singlenodeexecutor_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_singlenodeexecutor_cache.py b/tests/test_singlenodeexecutor_cache.py index a7ed7367..5d8a810f 100644 --- a/tests/test_singlenodeexecutor_cache.py +++ b/tests/test_singlenodeexecutor_cache.py @@ -66,7 +66,7 @@ def test_cache_error_file(self): f = exe.submit(get_error, a=1) with self.assertRaises(ValueError): print(f.result()) - error_out = os.path.join(os.path.dirname(__file__), "error.out") + error_out = "error.out" self.assertTrue(os.path.exists(error_out)) os.remove(error_out) From c8500a370fa22ffb87fdda3d67f428089739d5fb Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 18:04:46 +0200 Subject: [PATCH 24/25] Drop Python 3.9 --- .github/workflows/pipeline.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 5b1e02a7..c51853d0 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -402,7 +402,7 @@ jobs: run: echo -e "channels:\n - conda-forge\n" > .condarc - uses: conda-incubator/setup-miniconda@v3 with: - python-version: '3.9' + python-version: '3.10' miniforge-version: latest condarc-file: .condarc environment-file: .ci_support/environment-old.yml diff --git a/pyproject.toml b/pyproject.toml index d51a8076..ee8dd253 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Intended Audience :: Science/Research", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", From eb85ed1c0d2def7067d466f71eb7db0e1f06b524 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 16 Jul 2025 18:10:22 +0200 Subject: [PATCH 25/25] extend tests --- tests/test_standalone_hdf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_standalone_hdf.py b/tests/test_standalone_hdf.py index addcce55..1e46378b 100644 --- a/tests/test_standalone_hdf.py +++ b/tests/test_standalone_hdf.py @@ -75,10 +75,12 @@ def test_hdf_kwargs(self): "args": (), "kwargs": {"a": a, "b": b}, "queue_id": 123, + "write_error_file": True, }, ) data_dict = load(file_name=file_name) self.assertTrue("fn" in data_dict.keys()) + self.assertTrue(data_dict.get("write_error_file", False)) self.assertEqual(data_dict["args"], ()) self.assertEqual(data_dict["kwargs"], {"a": a, "b": b}) self.assertEqual(get_queue_id(file_name=file_name), 123)