noqt commited on
Commit
d052fcc
·
1 Parent(s): d30a102

Preserve newly terminated run history

Browse files
CHANGELOG.md CHANGED
@@ -16,6 +16,9 @@
16
  - Link a successful autonomous incident to the exact owned workload record
17
  after containment, allowing recovery tooling to identify the selected run
18
  without replacing the authoritative quarantine kill receipt.
 
 
 
19
 
20
  ## 1.0.0 (release candidate)
21
 
 
16
  - Link a successful autonomous incident to the exact owned workload record
17
  after containment, allowing recovery tooling to identify the selected run
18
  without replacing the authoritative quarantine kill receipt.
19
+ - Never prune the terminal run record in the transaction that just persisted
20
+ it, avoiding cross-boot monotonic-clock ordering from erasing the newest kill
21
+ status under a full retained history.
22
 
23
  ## 1.0.0 (release candidate)
24
 
src/lumi_eggcracker/supervisor.py CHANGED
@@ -407,7 +407,7 @@ class Supervisor:
407
  if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
408
  provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
409
  self._clear_stage(record["run_id"])
410
- self._prune_terminal_records()
411
 
412
  @staticmethod
413
  def _clear_stage(run_id: str) -> None:
@@ -428,7 +428,7 @@ class Supervisor:
428
  child.unlink()
429
  root.rmdir()
430
 
431
- def _prune_terminal_records(self) -> None:
432
  records: list[tuple[int, Path]] = []
433
  for path in self.runs.glob("*.json"):
434
  try:
@@ -437,7 +437,13 @@ class Supervisor:
437
  continue
438
  if value["state"] not in ACTIVE_STATES:
439
  records.append((int(value["created_monotonic_ns"]), path))
440
- for _, path in sorted(records, reverse=True)[MAX_TERMINAL_RECORDS:]:
 
 
 
 
 
 
441
  path.unlink(missing_ok=True)
442
 
443
  def _load(self, name: str) -> dict[str, Any]:
 
407
  if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
408
  provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
409
  self._clear_stage(record["run_id"])
410
+ self._prune_terminal_records(protected_run_id=record["run_id"])
411
 
412
  @staticmethod
413
  def _clear_stage(run_id: str) -> None:
 
428
  child.unlink()
429
  root.rmdir()
430
 
431
+ def _prune_terminal_records(self, *, protected_run_id: str | None = None) -> None:
432
  records: list[tuple[int, Path]] = []
433
  for path in self.runs.glob("*.json"):
434
  try:
 
437
  continue
438
  if value["state"] not in ACTIVE_STATES:
439
  records.append((int(value["created_monotonic_ns"]), path))
440
+ remove_count = max(0, len(records) - MAX_TERMINAL_RECORDS)
441
+ removable = [
442
+ value
443
+ for value in sorted(records)
444
+ if value[1].stem != protected_run_id
445
+ ]
446
+ for _, path in removable[:remove_count]:
447
  path.unlink(missing_ok=True)
448
 
449
  def _load(self, name: str) -> dict[str, Any]:
tests/test_supervisor.py CHANGED
@@ -895,6 +895,35 @@ class SupervisorTests(unittest.TestCase):
895
  supervisor._store(replacement)
896
  self.assertFalse((supervisor.runs / ("c" * 24 + ".json")).exists())
897
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
898
  def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
899
  supervisor = self._instance()
900
  with tempfile.TemporaryDirectory() as raw:
 
895
  supervisor._store(replacement)
896
  self.assertFalse((supervisor.runs / ("c" * 24 + ".json")).exists())
897
 
898
+ def test_terminal_pruning_never_removes_the_record_just_persisted(self) -> None:
899
+ supervisor = self._instance()
900
+ with tempfile.TemporaryDirectory() as raw:
901
+ root = Path(raw)
902
+ supervisor.runs = root / "runs"
903
+ supervisor.names = root / "names"
904
+ historical = {
905
+ **record(),
906
+ "cgroup": "/system.slice/lumi-eggcracker-workload-"
907
+ + "b" * 24
908
+ + ".service",
909
+ "created_monotonic_ns": 10_000,
910
+ "name": "historical",
911
+ "run_id": "b" * 24,
912
+ "state": "TERMINATED",
913
+ "unit": "lumi-eggcracker-workload-" + "b" * 24 + ".service",
914
+ }
915
+ current = {
916
+ **record(),
917
+ "created_monotonic_ns": 1,
918
+ "state": "TERMINATED",
919
+ }
920
+ with patch("lumi_eggcracker.supervisor.MAX_TERMINAL_RECORDS", 1):
921
+ supervisor._store(historical)
922
+ supervisor._store(current)
923
+
924
+ self.assertTrue((supervisor.runs / ("a" * 24 + ".json")).is_file())
925
+ self.assertFalse((supervisor.runs / ("b" * 24 + ".json")).exists())
926
+
927
  def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
928
  supervisor = self._instance()
929
  with tempfile.TemporaryDirectory() as raw: