File size: 3,877 Bytes
9422383 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | """Behavior of the command that records evidence around an external benchmark.
The test executes a real child process. Hardware readings remain the local
operating system's responsibility and are asserted only through the recorder's
public JSON output.
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory
from tests.support import read_json, run_module
class BenchmarkRecorderCommandTests(unittest.TestCase):
@staticmethod
def _inputs(directory: Path) -> tuple[Path, Path]:
runtime = directory / "libarm-yolo26n.so.0"
runtime.write_bytes(b"runtime")
manifest = directory / "manifest.json"
manifest.write_text('{"kind":"yolo26n-exported-model","version":1}\n')
return runtime, manifest
def test_recorder_publishes_review_evidence_and_preserves_command_output(self) -> None:
with TemporaryDirectory() as directory:
root = Path(directory)
output = root / "evidence"
runtime, manifest = self._inputs(root)
completed = run_module(
"benchmarks.reporting.record_benchmark",
"--output-dir",
output,
"--runtime-library",
runtime,
"--model-manifest",
manifest,
"--revision",
"0123456789abcdef",
"--",
sys.executable,
"-c",
"print('benchmark output')",
)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertIn("benchmark output", completed.stdout)
self.assertIn("benchmark output", (output / "benchmark.log").read_text())
self.assertEqual(
{path.name for path in output.iterdir()}, {"benchmark.log", "results.json"}
)
report = read_json(output / "results.json")
self.assertEqual(report["kind"], "yolo26n-ultralytics-benchmark-result")
self.assertEqual(report["exit_code"], 0)
self.assertEqual(report["source_revision"], "0123456789abcdef")
self.assertEqual(report["model_manifest"]["path"], str(manifest))
self.assertEqual(
report["model_manifest"]["document"],
{"kind": "yolo26n-exported-model", "version": 1},
)
self.assertEqual(report["runtime_library"]["path"], str(runtime))
self.assertGreaterEqual(report["child_peak_rss_bytes"], 0)
for phase in ("before", "after"):
snapshot = report["host"][phase]
self.assertIn("MemAvailable", snapshot["memory_bytes"])
self.assertIn("Cached", snapshot["memory_bytes"])
self.assertIn("SReclaimable", snapshot["memory_bytes"])
self.assertIn("pgmajfault", snapshot["virtual_memory_events"])
def test_recorder_returns_the_benchmark_failure_after_writing_context(self) -> None:
with TemporaryDirectory() as directory:
root = Path(directory)
output = root / "evidence"
runtime, manifest = self._inputs(root)
completed = run_module(
"benchmarks.reporting.record_benchmark",
"--output-dir",
output,
"--runtime-library",
runtime,
"--model-manifest",
manifest,
"--revision",
"fedcba9876543210",
"--",
sys.executable,
"-c",
"raise SystemExit(7)",
)
self.assertEqual(completed.returncode, 7)
self.assertEqual(read_json(output / "results.json")["exit_code"], 7)
if __name__ == "__main__":
unittest.main()
|