diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 8cf6b7b2..d1fd319f 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -29,7 +29,7 @@ import os import sys import time -import traceback + from datetime import datetime, timedelta from importlib.metadata import version from multiprocessing import Process @@ -51,7 +51,7 @@ from itzi.providers.grass_interface import GrassInterface -def main(argv: list[str] | None = None) -> int | None: +def main(argv: list[str] | None = None) -> int: """argv: alternative CLI arguments, used for testing (default to sys.argv)""" args = build_parser().parse_args(argv) @@ -65,6 +65,7 @@ def main(argv: list[str] | None = None) -> int | None: command_mapper[args.command](args) except msgr.FatalError: return 1 + return 0 class SimulationRunner: @@ -197,7 +198,7 @@ def __del__(self): self.g_interface.cleanup() -def sim_runner_worker(conf_file: str, hotstart_file: str | None): +def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: """Run one simulation""" msgr.raise_on_error = True msgr._itzi_logger.set_verbosity(msgr.verbosity()) @@ -215,20 +216,30 @@ def sim_runner_worker(conf_file: str, hotstart_file: str | None): ) sim_runner.run().finalize() except msgr.FatalError: - return - except Exception: - msgr.warning("Error during execution: {}".format(traceback.format_exc())) + raise SystemExit(1) from None + except SystemExit as error: + detail = error.code if isinstance(error.code, str) else f"exit status {error.code}" + msgr.warning(f"Simulation terminated with {detail}") + raise SystemExit(1) from None + except Exception as error: + msgr.warning(f"Error during execution: {type(error).__name__}: {error}") + raise SystemExit(1) from None -def itzi_run_one(conf_file: str, hotstart_file: str | None): +def itzi_run_one(conf_file: str, hotstart_file: str | None) -> bool: """Run a simulation in a subprocess""" worker_args = (conf_file, hotstart_file) p = Process(target=sim_runner_worker, args=worker_args) p.start() p.join() - if p.exitcode != 0: - msgr.warning(("Execution of {} ended with an error").format(conf_file)) + exitcode = p.exitcode p.close() + if exitcode == 0: + return True + + reason = f"signal {-exitcode}" if exitcode < 0 else f"exit status {exitcode}" + msgr.warning(f"Execution of {conf_file} ended with an error ({reason})") + return False def reconcile_hotstart_commands( @@ -325,6 +336,7 @@ def itzi_run(cli_args): total_sim_start = time.time() # dictionary to store computation times times_list = [] + failed_files = [] run_commands = reconcile_hotstart_commands( cli_args.config_file, getattr(cli_args, "resume_from", []), @@ -332,7 +344,8 @@ def itzi_run(cli_args): for conf_file, hotstart_file in run_commands: sim_start = time.time() # Run the simulation - itzi_run_one(conf_file, hotstart_file) + if not itzi_run_one(conf_file, hotstart_file): + failed_files.append(conf_file) # store computational time comp_time = timedelta(seconds=int(time.time() - sim_start)) list_elem = (os.path.basename(conf_file), comp_time) @@ -341,12 +354,17 @@ def itzi_run(cli_args): # stop total time counter total_elapsed_time = timedelta(seconds=int(time.time() - total_sim_start)) # display total computation duration - msgr.message("Simulation(s) complete. Elapsed times:") + if failed_files: + msgr.message("Simulation run(s) finished with errors. Elapsed times:") + else: + msgr.message("Simulation(s) complete. Elapsed times:") for f, t in times_list: msgr.message("{}: {}".format(f, t)) msgr.message("Total: {}".format(total_elapsed_time)) avg_time_s = int(total_elapsed_time.total_seconds() / len(times_list)) msgr.message("Average: {}".format(timedelta(seconds=avg_time_s))) + if failed_files: + msgr.fatal(f"{len(failed_files)} simulation(s) failed") def itzi_version(cli_args): diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index eef5466d..cb37a2c9 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -52,6 +52,15 @@ "vdir": str(RULE_VDIR), "froude": str(RULE_FR), } + + +def _init_temporal() -> None: + """Initialize GRASS temporal APIs without allowing them to exit Itzi.""" + gscript.set_raise_on_error(True) + tgis.init(raise_fatal_error=True) + tgis.set_raise_on_error(True) + + # Check if color rule paths are OK for f in colors_rules_dict.values(): assert Path(f).is_file() @@ -194,7 +203,7 @@ def __init__( self.set_temp_mask() self.overwrite = gscript.overwrite() # init temporal module - tgis.init() + _init_temporal() # Create thread and queue for writing raster maps if self.non_blocking_write: self.raster_lock = Lock() @@ -341,7 +350,7 @@ def name_is_stds(name: str) -> bool: False if not """ # make sure temporal module is initialized - tgis.init() + _init_temporal() return bool(tgis.SpaceTimeRasterDataset(name).is_in_db()) @staticmethod diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index ef2167c4..ccf1c4b2 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -10,6 +10,7 @@ from itzi.itzi import ( VerbosityLevel, itzi_run, + itzi_run_one, main, reconcile_hotstart_commands, sim_runner_worker, @@ -46,7 +47,7 @@ def test_run_parser_rejects_v_and_q_together(): def test_prints_version(monkeypatch, capsys): monkeypatch.setattr("itzi.itzi.version", lambda _: "22.2") - assert main(["version"]) is None + assert main(["version"]) == 0 assert capsys.readouterr().out.strip() == "22.2" @@ -79,14 +80,82 @@ def fail(_): monkeypatch.setenv("ITZI_VERBOSE", str(VerbosityLevel.QUIET)) monkeypatch.setattr("itzi.itzi.ConfigReader", fail) - sim_runner_worker("a.ini", None) + with pytest.raises(SystemExit) as error: + sim_runner_worker("a.ini", None) + assert error.value.code == 1 stderr = itzi_stderr.getvalue() assert stderr.count("ERROR: expected worker failure") == 1 assert "Traceback" not in stderr assert "WARNING: Error during execution" not in stderr +def test_worker_reports_unexpected_error_without_traceback(monkeypatch, itzi_stderr): + def fail(_): + raise ValueError("unexpected worker failure") + + monkeypatch.setenv("ITZI_VERBOSE", str(VerbosityLevel.QUIET)) + monkeypatch.setattr("itzi.itzi.ConfigReader", fail) + + with pytest.raises(SystemExit) as error: + sim_runner_worker("a.ini", None) + + assert error.value.code == 1 + stderr = itzi_stderr.getvalue() + assert "WARNING: Error during execution: ValueError: unexpected worker failure" in stderr + assert "Traceback" not in stderr + + +def test_worker_reports_system_exit_without_traceback(monkeypatch, itzi_stderr): + def fail(_): + raise SystemExit("GRASS failure") + + monkeypatch.setenv("ITZI_VERBOSE", str(VerbosityLevel.QUIET)) + monkeypatch.setattr("itzi.itzi.ConfigReader", fail) + + with pytest.raises(SystemExit) as error: + sim_runner_worker("a.ini", None) + + assert error.value.code == 1 + stderr = itzi_stderr.getvalue() + assert "WARNING: Simulation terminated with GRASS failure" in stderr + assert "Traceback" not in stderr + + +def test_worker_formats_numeric_system_exit_as_status(monkeypatch, itzi_stderr): + def fail(_): + raise SystemExit(1) + + monkeypatch.setenv("ITZI_VERBOSE", str(VerbosityLevel.QUIET)) + monkeypatch.setattr("itzi.itzi.ConfigReader", fail) + + with pytest.raises(SystemExit): + sim_runner_worker("a.ini", None) + + assert "WARNING: Simulation terminated with exit status 1" in itzi_stderr.getvalue() + + +def test_run_one_reports_worker_signal(monkeypatch, itzi_stderr): + class FailedProcess: + exitcode = -11 + + def start(self): + pass + + def join(self): + pass + + def close(self): + pass + + monkeypatch.setattr("itzi.itzi.Process", lambda **_: FailedProcess()) + + assert itzi_run_one("a.ini", None) is False + assert "WARNING: Execution of a.ini ended with an error (signal 11)" in ( + itzi_stderr.getvalue() + ) + + def test_reconcile_hotstart_commands_accepts_single_resume_for_single_config(): assert reconcile_hotstart_commands(["/tmp/a.ini"], [(None, "restart_a.zip")]) == [ ("/tmp/a.ini", "restart_a.zip"), @@ -152,9 +221,11 @@ def test_itzi_run_sets_env_and_dispatches(monkeypatch): calls = [] messages = [] - monkeypatch.setattr( - "itzi.itzi.itzi_run_one", lambda conf, hotstart: calls.append((conf, hotstart)) - ) + def record_run(conf_file, hotstart_file): + calls.append((conf_file, hotstart_file)) + return True + + monkeypatch.setattr("itzi.itzi.itzi_run_one", record_run) monkeypatch.setattr("itzi.itzi.msgr.message", messages.append) args = argparse.Namespace( @@ -172,3 +243,14 @@ def test_itzi_run_sets_env_and_dispatches(monkeypatch): assert os.environ["ITZI_VERBOSE"] == str(VerbosityLevel.VERBOSE) assert os.environ["GRASS_VERBOSE"] == "2" assert any("Simulation(s) complete" in m for m in messages) + + +def test_main_returns_error_status_when_simulation_fails(monkeypatch, itzi_stderr): + monkeypatch.setattr("itzi.itzi.itzi_run_one", lambda *_: False) + msgr._itzi_logger.set_verbosity(VerbosityLevel.MESSAGE) + + assert main(["run", "a.ini"]) == 1 + stderr = itzi_stderr.getvalue() + assert "Simulation run(s) finished with errors" in stderr + assert "ERROR: 1 simulation(s) failed" in stderr + assert "Traceback" not in stderr diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py index 2284624d..3dd33077 100644 --- a/tests/grass/test_temporal_overwrite.py +++ b/tests/grass/test_temporal_overwrite.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Iterator from configparser import ConfigParser import os from uuid import uuid4 @@ -12,6 +13,15 @@ from itzi_core.const import TemporalType +@pytest.fixture(autouse=True) +def stop_temporal_subprocesses() -> Iterator[None]: + yield + + import grass.temporal as tgis + + tgis.stop_subprocesses() + + def _build_runner( test_data_temp_path: str, prefix: str, @@ -57,6 +67,23 @@ def _build_runner( return SimulationRunner(conf_data.get_sim_params(), conf_data.get_grass_params()) +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_temporal_fatal_errors_are_raised_as_exceptions() -> None: + import grass.temporal as tgis + from grass.exceptions import FatalError + + from itzi.providers.grass_interface import GrassInterface + + tgis.set_raise_on_error(False) + + with pytest.raises(FatalError, match="mapset is missing"): + GrassInterface.name_is_stds("missing_strds") + + assert gscript.get_raise_on_error() is True + assert tgis.get_raise_on_error() is True + + @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") @pytest.mark.parametrize(