Spaces:
Running
Running
Preserve newly terminated run history
Browse files- CHANGELOG.md +3 -0
- src/lumi_eggcracker/supervisor.py +9 -3
- tests/test_supervisor.py +29 -0
CHANGELOG.md
CHANGED
|
@@ -16,6 +16,9 @@
|
|
| 16 |
- Link a successful autonomous incident to the exact owned workload record
|
| 17 |
after containment, allowing recovery tooling to identify the selected run
|
| 18 |
without replacing the authoritative quarantine kill receipt.
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
## 1.0.0 (release candidate)
|
| 21 |
|
|
|
|
| 16 |
- Link a successful autonomous incident to the exact owned workload record
|
| 17 |
after containment, allowing recovery tooling to identify the selected run
|
| 18 |
without replacing the authoritative quarantine kill receipt.
|
| 19 |
+
- Never prune the terminal run record in the transaction that just persisted
|
| 20 |
+
it, avoiding cross-boot monotonic-clock ordering from erasing the newest kill
|
| 21 |
+
status under a full retained history.
|
| 22 |
|
| 23 |
## 1.0.0 (release candidate)
|
| 24 |
|
src/lumi_eggcracker/supervisor.py
CHANGED
|
@@ -407,7 +407,7 @@ class Supervisor:
|
|
| 407 |
if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
|
| 408 |
provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
|
| 409 |
self._clear_stage(record["run_id"])
|
| 410 |
-
self._prune_terminal_records()
|
| 411 |
|
| 412 |
@staticmethod
|
| 413 |
def _clear_stage(run_id: str) -> None:
|
|
@@ -428,7 +428,7 @@ class Supervisor:
|
|
| 428 |
child.unlink()
|
| 429 |
root.rmdir()
|
| 430 |
|
| 431 |
-
def _prune_terminal_records(self) -> None:
|
| 432 |
records: list[tuple[int, Path]] = []
|
| 433 |
for path in self.runs.glob("*.json"):
|
| 434 |
try:
|
|
@@ -437,7 +437,13 @@ class Supervisor:
|
|
| 437 |
continue
|
| 438 |
if value["state"] not in ACTIVE_STATES:
|
| 439 |
records.append((int(value["created_monotonic_ns"]), path))
|
| 440 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
path.unlink(missing_ok=True)
|
| 442 |
|
| 443 |
def _load(self, name: str) -> dict[str, Any]:
|
|
|
|
| 407 |
if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
|
| 408 |
provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
|
| 409 |
self._clear_stage(record["run_id"])
|
| 410 |
+
self._prune_terminal_records(protected_run_id=record["run_id"])
|
| 411 |
|
| 412 |
@staticmethod
|
| 413 |
def _clear_stage(run_id: str) -> None:
|
|
|
|
| 428 |
child.unlink()
|
| 429 |
root.rmdir()
|
| 430 |
|
| 431 |
+
def _prune_terminal_records(self, *, protected_run_id: str | None = None) -> None:
|
| 432 |
records: list[tuple[int, Path]] = []
|
| 433 |
for path in self.runs.glob("*.json"):
|
| 434 |
try:
|
|
|
|
| 437 |
continue
|
| 438 |
if value["state"] not in ACTIVE_STATES:
|
| 439 |
records.append((int(value["created_monotonic_ns"]), path))
|
| 440 |
+
remove_count = max(0, len(records) - MAX_TERMINAL_RECORDS)
|
| 441 |
+
removable = [
|
| 442 |
+
value
|
| 443 |
+
for value in sorted(records)
|
| 444 |
+
if value[1].stem != protected_run_id
|
| 445 |
+
]
|
| 446 |
+
for _, path in removable[:remove_count]:
|
| 447 |
path.unlink(missing_ok=True)
|
| 448 |
|
| 449 |
def _load(self, name: str) -> dict[str, Any]:
|
tests/test_supervisor.py
CHANGED
|
@@ -895,6 +895,35 @@ class SupervisorTests(unittest.TestCase):
|
|
| 895 |
supervisor._store(replacement)
|
| 896 |
self.assertFalse((supervisor.runs / ("c" * 24 + ".json")).exists())
|
| 897 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
|
| 899 |
supervisor = self._instance()
|
| 900 |
with tempfile.TemporaryDirectory() as raw:
|
|
|
|
| 895 |
supervisor._store(replacement)
|
| 896 |
self.assertFalse((supervisor.runs / ("c" * 24 + ".json")).exists())
|
| 897 |
|
| 898 |
+
def test_terminal_pruning_never_removes_the_record_just_persisted(self) -> None:
|
| 899 |
+
supervisor = self._instance()
|
| 900 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 901 |
+
root = Path(raw)
|
| 902 |
+
supervisor.runs = root / "runs"
|
| 903 |
+
supervisor.names = root / "names"
|
| 904 |
+
historical = {
|
| 905 |
+
**record(),
|
| 906 |
+
"cgroup": "/system.slice/lumi-eggcracker-workload-"
|
| 907 |
+
+ "b" * 24
|
| 908 |
+
+ ".service",
|
| 909 |
+
"created_monotonic_ns": 10_000,
|
| 910 |
+
"name": "historical",
|
| 911 |
+
"run_id": "b" * 24,
|
| 912 |
+
"state": "TERMINATED",
|
| 913 |
+
"unit": "lumi-eggcracker-workload-" + "b" * 24 + ".service",
|
| 914 |
+
}
|
| 915 |
+
current = {
|
| 916 |
+
**record(),
|
| 917 |
+
"created_monotonic_ns": 1,
|
| 918 |
+
"state": "TERMINATED",
|
| 919 |
+
}
|
| 920 |
+
with patch("lumi_eggcracker.supervisor.MAX_TERMINAL_RECORDS", 1):
|
| 921 |
+
supervisor._store(historical)
|
| 922 |
+
supervisor._store(current)
|
| 923 |
+
|
| 924 |
+
self.assertTrue((supervisor.runs / ("a" * 24 + ".json")).is_file())
|
| 925 |
+
self.assertFalse((supervisor.runs / ("b" * 24 + ".json")).exists())
|
| 926 |
+
|
| 927 |
def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
|
| 928 |
supervisor = self._instance()
|
| 929 |
with tempfile.TemporaryDirectory() as raw:
|