diff --git a/test/test_build.py b/test/test_build.py index 887140a..18dab94 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -42,6 +42,7 @@ def check_artifacts(mocker): # pretty slow. @pytest.fixture(autouse=True) def collect_metadata(mocker): + mocker.patch("tuxmake.build.Build.collect_metadata_early") return mocker.patch("tuxmake.build.Build.collect_metadata") diff --git a/test/test_metadata.py b/test/test_metadata.py index 2b78e3a..d9225d1 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -1,3 +1,4 @@ +import json import os import re import subprocess @@ -86,6 +87,28 @@ def test_order_all(self): assert cls.index(source) < cls.index(git) +class TestEarlyMetadataWrite: + def test_metadata_written_before_build(self, build, linux, mocker): + b = Build(linux, target_arch="arm64") + captured = {} + original = b.build_all_targets + + def capture(*args, **kwargs): + path = b.output_dir / "metadata.json" + captured["data"] = json.loads(path.read_text()) + return original(*args, **kwargs) + + mocker.patch.object(b, "build_all_targets", side_effect=capture) + b.run() + + data = captured["data"] + assert "compiler" in data + assert "tuxmake" in data + assert "git" in data + # results need the finished build, so they must not be there yet + assert "results" not in data + + class TestKernelVersion: def test_happy_path(self, build): assert type(build.metadata["source"]["kernelversion"]) is str diff --git a/tuxmake/build.py b/tuxmake/build.py index cda3a1d..20bcd32 100644 --- a/tuxmake/build.py +++ b/tuxmake/build.py @@ -674,7 +674,11 @@ def failed(self): s = [info.failed for info in self.status.values()] return s and True in set(s) - def collect_metadata(self): + def collect_metadata_early(self): + """ + Collect the metadata we already know after prepare(). We write this + before the build starts, so it survives a hard kill during the build. + """ self.metadata["build"] = { "targets": [t.name for t in self.targets], "target_arch": self.target_arch.name, @@ -688,6 +692,11 @@ def collect_metadata(self): "verbose": self.verbose, "reproducer_cmdline": self.cmdline.reproduce(self), } + self.metadata["tuxmake"] = {"version": __version__} + self.metadata["runtime"] = self.runtime.get_metadata() + self.metadata.update(self.metadata_collector.collect_early()) + + def collect_metadata(self): errors, warnings = self.parse_log() self.metadata["results"] = { "status": ( @@ -708,16 +717,14 @@ def collect_metadata(self): "warnings": warnings, "duration": self.__durations__, } - self.metadata["tuxmake"] = {"version": __version__} - self.metadata["runtime"] = self.runtime.get_metadata() - - extracted = self.metadata_collector.collect() - self.metadata.update(extracted) + self.metadata.update(self.metadata_collector.collect_late()) def save_metadata(self): - with (self.output_dir / "metadata.json").open("w") as f: - f.write(json.dumps(self.metadata, indent=4, sort_keys=True)) - f.write("\n") + # Atomic rename, so a hard kill mid-write can't leave a half-written file. + path = self.output_dir / "metadata.json" + tmp = self.output_dir / "metadata.json.tmp" + tmp.write_text(json.dumps(self.metadata, indent=4, sort_keys=True) + "\n") + os.replace(tmp, path) def parse_log(self): parser = LogParser() @@ -838,6 +845,10 @@ def run(self): prepared = True self.log(quote_command_line(self.cmdline.reproduce(self))) + with self.measure_duration("Early Metadata Extraction"): + self.collect_metadata_early() + self.save_metadata() + with self.go_offline(): with self.measure_duration("Build", metadata="build"): self.build_all_targets() diff --git a/tuxmake/metadata.py b/tuxmake/metadata.py index 0ec5cc6..11030f2 100644 --- a/tuxmake/metadata.py +++ b/tuxmake/metadata.py @@ -7,6 +7,10 @@ from tuxmake.exceptions import UnsupportedMetadata from tuxmake.exceptions import UnsupportedMetadataType +# These handlers only need the prepared build, not the build result. We collect +# them early so the data survives a hard kill like SIGKILL or OOM. +EARLY_METADATA_HANDLERS = ["compiler", "git", "hardware", "os", "tools", "uname"] + class MetadataItemExtactor(ABC): def __init__(self, build): @@ -57,7 +61,17 @@ def before_build(self): for _, _, extractor in self.each_extractor(): extractor.before_build() - def collect(self): + def collect_early(self): + handlers = [h for h in self.handlers if h.name in EARLY_METADATA_HANDLERS] + return self.collect(handlers) + + def collect_late(self): + handlers = [h for h in self.handlers if h.name not in EARLY_METADATA_HANDLERS] + return self.collect(handlers) + + def collect(self, handlers=None): + if handlers is None: + handlers = self.handlers build = self.build compiler = build.toolchain.compiler( build.target_arch, build.makevars.get("CROSS_COMPILE", None) @@ -67,7 +81,7 @@ def collect(self): key: build.format_cmd_part(cmd.replace("{compiler}", compiler)) for key, cmd in handler.commands.items() } - for handler in self.handlers + for handler in handlers } metadata_input = build.build_dir / "metadata.in.json" metadata_input.write_text(json.dumps(metadata_input_data)) @@ -81,11 +95,13 @@ def collect(self): build.run_cmd( ["perl", str(script), str(metadata_input)], echo=False, stdout=f ) - metadata = self.read_json(stdout.read_text()) - self.collect_extra_metadata(metadata) + metadata = self.read_json(stdout.read_text(), handlers) + self.collect_extra_metadata(metadata, handlers) return metadata - def read_json(self, metadata_json): + def read_json(self, metadata_json, handlers=None): + if handlers is None: + handlers = self.handlers if not metadata_json: return {} try: @@ -96,7 +112,7 @@ def read_json(self, metadata_json): return {} result = {} - for handler in self.handlers: + for handler in handlers: for key in handler.commands.keys(): v = metadata[handler.name][key] if v: @@ -107,8 +123,11 @@ def read_json(self, metadata_json): return result - def collect_extra_metadata(self, metadata): + def collect_extra_metadata(self, metadata, handlers): + names = {handler.name for handler in handlers} for handler, item, extractor in self.each_extractor(): + if handler not in names: + continue metadata.setdefault(handler, {}) metadata[handler][item] = extractor.get()