Spaces:
Running
Running
fix: reserve workload names after termination
Browse files- LIMITATIONS.md +2 -0
- src/lumi_eggcracker/supervisor.py +64 -22
- tests/test_supervisor.py +57 -3
LIMITATIONS.md
CHANGED
|
@@ -46,4 +46,6 @@ behavioural modelling, telemetry upload, malware prevention or EDR.
|
|
| 46 |
|
| 47 |
Approvals are exact and deliberately narrow: only a qualified native llama runtime or root-owned CPython executing one absolute regular script is supported. Workload UID, resolved executable identity and complete argv digest must match at the protected pre-exec `start` gate. Python `-c`, `-m`, stdin/interactive, relative-script, package-directory, symlink and unsupported-interpreter forms are rejected. The Python script is identity/digest bound and launched from an immutable per-run root-owned stage; changing it aborts `start`. This does not claim a general dependency-closure analyser or make approved workloads a sandbox. A direct or third-party launch cannot consume an approval, and descendants or siblings do not inherit one. Revoking an approval affects future protected launches, not a workload already running under that approval.
|
| 48 |
|
|
|
|
|
|
|
| 49 |
The explicit `start` command remains non-interactive. It has no terminal attachment, current-working-directory forwarding, timeout or output streaming. It supports one active selected workload globally; PID, memory and CPU limits are explicit. Detection receipts are bounded to 1000 entries.
|
|
|
|
| 46 |
|
| 47 |
Approvals are exact and deliberately narrow: only a qualified native llama runtime or root-owned CPython executing one absolute regular script is supported. Workload UID, resolved executable identity and complete argv digest must match at the protected pre-exec `start` gate. Python `-c`, `-m`, stdin/interactive, relative-script, package-directory, symlink and unsupported-interpreter forms are rejected. The Python script is identity/digest bound and launched from an immutable per-run root-owned stage; changing it aborts `start`. This does not claim a general dependency-closure analyser or make approved workloads a sandbox. A direct or third-party launch cannot consume an approval, and descendants or siblings do not inherit one. Revoking an approval affects future protected launches, not a workload already running under that approval.
|
| 48 |
|
| 49 |
+
Operator-supplied workload names are single-use for the lifetime of an installation. A name remains reserved after completion or containment so a stale status or kill command cannot resolve to a replacement workload. Clean uninstallation removes those reservations with the rest of Eggcracker's local state.
|
| 50 |
+
|
| 51 |
The explicit `start` command remains non-interactive. It has no terminal attachment, current-working-directory forwarding, timeout or output streaming. It supports one active selected workload globally; PID, memory and CPU limits are explicit. Detection receipts are bounded to 1000 entries.
|
src/lumi_eggcracker/supervisor.py
CHANGED
|
@@ -383,22 +383,26 @@ class Supervisor:
|
|
| 383 |
record = validate_run(record)
|
| 384 |
if not hasattr(self, "active_cgroups"):
|
| 385 |
self.active_cgroups = set()
|
| 386 |
-
if record["state"] in ACTIVE_STATES:
|
| 387 |
-
self.active_cgroups.add(record["cgroup"])
|
| 388 |
-
else:
|
| 389 |
-
self.active_cgroups.discard(record["cgroup"])
|
| 390 |
-
self.operations.append("durable-state")
|
| 391 |
-
write_atomic(run_path(self.runs, record["run_id"]), record)
|
| 392 |
pointer = name_path(self.names, record["name"])
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
try:
|
| 397 |
current = load_regular_json(pointer)
|
| 398 |
-
if current == {"run_id": record["run_id"]}:
|
| 399 |
-
pointer.unlink()
|
| 400 |
except JsonInputError:
|
| 401 |
raise JsonInputError("workload name index is invalid")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 402 |
if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
|
| 403 |
provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
|
| 404 |
self._clear_stage(record["run_id"])
|
|
@@ -445,7 +449,7 @@ class Supervisor:
|
|
| 445 |
return record
|
| 446 |
|
| 447 |
def _latest_by_name(self, name: str) -> dict[str, Any]:
|
| 448 |
-
"""Resolve
|
| 449 |
candidates: list[dict[str, Any]] = []
|
| 450 |
for path in self.runs.glob("*.json"):
|
| 451 |
try:
|
|
@@ -458,6 +462,33 @@ class Supervisor:
|
|
| 458 |
raise JsonInputError("workload name is unavailable")
|
| 459 |
return max(candidates, key=lambda item: int(item["created_monotonic_ns"]))
|
| 460 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 461 |
def _receipt_path(self, event_id: str) -> Path:
|
| 462 |
return self.receipts / f"{event_id}.json"
|
| 463 |
|
|
@@ -531,6 +562,7 @@ class Supervisor:
|
|
| 531 |
incident_store.compact(self.incidents)
|
| 532 |
except (JsonInputError, OSError):
|
| 533 |
self.incident_health_healthy = False
|
|
|
|
| 534 |
self._recover()
|
| 535 |
self.quarantine_root = self._prepare_quarantine()
|
| 536 |
self._scan_synchronously()
|
|
@@ -2532,7 +2564,12 @@ class Supervisor:
|
|
| 2532 |
if not self._incident_status()["healthy"]:
|
| 2533 |
raise JsonInputError("local lockdown state is unavailable; root recovery is required")
|
| 2534 |
with self.start_lock, self.approval_lock:
|
| 2535 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2536 |
raise JsonInputError(
|
| 2537 |
"one protected workload is already active or name is unavailable"
|
| 2538 |
)
|
|
@@ -2980,11 +3017,7 @@ class Supervisor:
|
|
| 2980 |
}
|
| 2981 |
if action == "start":
|
| 2982 |
return self._start(args)
|
| 2983 |
-
|
| 2984 |
-
try:
|
| 2985 |
-
record = self._load(args["name"])
|
| 2986 |
-
except JsonInputError:
|
| 2987 |
-
record = self._latest_by_name(args["name"])
|
| 2988 |
return {
|
| 2989 |
"name": record["name"],
|
| 2990 |
"run_id": record["run_id"],
|
|
@@ -3000,11 +3033,20 @@ class Supervisor:
|
|
| 3000 |
"incident_lockdown": self._incident_status()["lockdown"],
|
| 3001 |
"workload_uid": record["workload_uid"],
|
| 3002 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3003 |
if action == "list" and not args:
|
| 3004 |
-
runs = [
|
| 3005 |
-
|
| 3006 |
-
|
| 3007 |
-
|
|
|
|
|
|
|
| 3008 |
return {"runs": runs}
|
| 3009 |
if action in {"approve", "revoke", "exec_policy_create", "exec_policy_revoke"} and not self._incident_status()["healthy"]:
|
| 3010 |
raise JsonInputError("local lockdown state is unavailable; root recovery is required")
|
|
|
|
| 383 |
record = validate_run(record)
|
| 384 |
if not hasattr(self, "active_cgroups"):
|
| 385 |
self.active_cgroups = set()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 386 |
pointer = name_path(self.names, record["name"])
|
| 387 |
+
expected_pointer = {"run_id": record["run_id"]}
|
| 388 |
+
pointer_missing = not pointer.exists() and not pointer.is_symlink()
|
| 389 |
+
if not pointer_missing:
|
| 390 |
try:
|
| 391 |
current = load_regular_json(pointer)
|
|
|
|
|
|
|
| 392 |
except JsonInputError:
|
| 393 |
raise JsonInputError("workload name index is invalid")
|
| 394 |
+
if current != expected_pointer:
|
| 395 |
+
raise JsonInputError("workload name index collision")
|
| 396 |
+
if pointer_missing:
|
| 397 |
+
# A reserved name is safe even if the following record write
|
| 398 |
+
# fails; allowing reuse after a partial start is not.
|
| 399 |
+
write_atomic(pointer, expected_pointer)
|
| 400 |
+
self.operations.append("durable-state")
|
| 401 |
+
write_atomic(run_path(self.runs, record["run_id"]), record)
|
| 402 |
+
if record["state"] in ACTIVE_STATES:
|
| 403 |
+
self.active_cgroups.add(record["cgroup"])
|
| 404 |
+
else:
|
| 405 |
+
self.active_cgroups.discard(record["cgroup"])
|
| 406 |
if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
|
| 407 |
provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
|
| 408 |
self._clear_stage(record["run_id"])
|
|
|
|
| 449 |
return record
|
| 450 |
|
| 451 |
def _latest_by_name(self, name: str) -> dict[str, Any]:
|
| 452 |
+
"""Resolve retained history for an unavailable or completed name."""
|
| 453 |
candidates: list[dict[str, Any]] = []
|
| 454 |
for path in self.runs.glob("*.json"):
|
| 455 |
try:
|
|
|
|
| 462 |
raise JsonInputError("workload name is unavailable")
|
| 463 |
return max(candidates, key=lambda item: int(item["created_monotonic_ns"]))
|
| 464 |
|
| 465 |
+
def _backfill_name_tombstones(self) -> None:
|
| 466 |
+
"""Reserve names created by releases that removed terminal pointers."""
|
| 467 |
+
latest: dict[str, dict[str, Any]] = {}
|
| 468 |
+
for path in self.runs.glob("*.json"):
|
| 469 |
+
try:
|
| 470 |
+
record = load_run(self.runs, path.stem)
|
| 471 |
+
except JsonInputError:
|
| 472 |
+
continue
|
| 473 |
+
prior = latest.get(record["name"])
|
| 474 |
+
if prior is None or int(record["created_monotonic_ns"]) > int(
|
| 475 |
+
prior["created_monotonic_ns"]
|
| 476 |
+
):
|
| 477 |
+
latest[record["name"]] = record
|
| 478 |
+
for name, record in latest.items():
|
| 479 |
+
pointer = name_path(self.names, name)
|
| 480 |
+
if pointer.exists() or pointer.is_symlink():
|
| 481 |
+
value = load_regular_json(pointer)
|
| 482 |
+
run_id = value.get("run_id")
|
| 483 |
+
if (
|
| 484 |
+
set(value) != {"run_id"}
|
| 485 |
+
or not isinstance(run_id, str)
|
| 486 |
+
or not RUN_ID.fullmatch(run_id)
|
| 487 |
+
):
|
| 488 |
+
raise JsonInputError("workload name index is invalid")
|
| 489 |
+
continue
|
| 490 |
+
write_atomic(pointer, {"run_id": record["run_id"]})
|
| 491 |
+
|
| 492 |
def _receipt_path(self, event_id: str) -> Path:
|
| 493 |
return self.receipts / f"{event_id}.json"
|
| 494 |
|
|
|
|
| 562 |
incident_store.compact(self.incidents)
|
| 563 |
except (JsonInputError, OSError):
|
| 564 |
self.incident_health_healthy = False
|
| 565 |
+
self._backfill_name_tombstones()
|
| 566 |
self._recover()
|
| 567 |
self.quarantine_root = self._prepare_quarantine()
|
| 568 |
self._scan_synchronously()
|
|
|
|
| 2564 |
if not self._incident_status()["healthy"]:
|
| 2565 |
raise JsonInputError("local lockdown state is unavailable; root recovery is required")
|
| 2566 |
with self.start_lock, self.approval_lock:
|
| 2567 |
+
reserved_name = name_path(self.names, name)
|
| 2568 |
+
if (
|
| 2569 |
+
reserved_name.exists()
|
| 2570 |
+
or reserved_name.is_symlink()
|
| 2571 |
+
or self._active_exists()
|
| 2572 |
+
):
|
| 2573 |
raise JsonInputError(
|
| 2574 |
"one protected workload is already active or name is unavailable"
|
| 2575 |
)
|
|
|
|
| 3017 |
}
|
| 3018 |
if action == "start":
|
| 3019 |
return self._start(args)
|
| 3020 |
+
def public_run(record: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3021 |
return {
|
| 3022 |
"name": record["name"],
|
| 3023 |
"run_id": record["run_id"],
|
|
|
|
| 3033 |
"incident_lockdown": self._incident_status()["lockdown"],
|
| 3034 |
"workload_uid": record["workload_uid"],
|
| 3035 |
}
|
| 3036 |
+
|
| 3037 |
+
if action == "status" and set(args) == {"name"}:
|
| 3038 |
+
try:
|
| 3039 |
+
record = self._load(args["name"])
|
| 3040 |
+
except JsonInputError:
|
| 3041 |
+
record = self._latest_by_name(args["name"])
|
| 3042 |
+
return public_run(record)
|
| 3043 |
if action == "list" and not args:
|
| 3044 |
+
runs = []
|
| 3045 |
+
for path in sorted(self.names.glob("*.json")):
|
| 3046 |
+
try:
|
| 3047 |
+
runs.append(public_run(self._load(path.stem)))
|
| 3048 |
+
except JsonInputError:
|
| 3049 |
+
continue
|
| 3050 |
return {"runs": runs}
|
| 3051 |
if action in {"approve", "revoke", "exec_policy_create", "exec_policy_revoke"} and not self._incident_status()["healthy"]:
|
| 3052 |
raise JsonInputError("local lockdown state is unavailable; root recovery is required")
|
tests/test_supervisor.py
CHANGED
|
@@ -817,7 +817,7 @@ class SupervisorTests(unittest.TestCase):
|
|
| 817 |
self.assertEqual("RUNNING", value["state"])
|
| 818 |
stored.assert_not_called()
|
| 819 |
|
| 820 |
-
def
|
| 821 |
supervisor = self._instance()
|
| 822 |
item = record()
|
| 823 |
item["state"] = "COMPLETED_ALLOWED"
|
|
@@ -826,7 +826,7 @@ class SupervisorTests(unittest.TestCase):
|
|
| 826 |
self.assertEqual("COMPLETED_ALLOWED", value["state"])
|
| 827 |
stored.assert_not_called()
|
| 828 |
|
| 829 |
-
def
|
| 830 |
supervisor = self._instance()
|
| 831 |
with tempfile.TemporaryDirectory() as raw:
|
| 832 |
root = Path(raw)
|
|
@@ -838,7 +838,61 @@ class SupervisorTests(unittest.TestCase):
|
|
| 838 |
self.assertTrue((supervisor.names / "demo.json").is_file())
|
| 839 |
item["state"] = "TERMINATED"
|
| 840 |
supervisor._store(item)
|
| 841 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 842 |
|
| 843 |
def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
|
| 844 |
supervisor = self._instance()
|
|
|
|
| 817 |
self.assertEqual("RUNNING", value["state"])
|
| 818 |
stored.assert_not_called()
|
| 819 |
|
| 820 |
+
def test_status_resolves_retained_terminal_history(self) -> None:
|
| 821 |
supervisor = self._instance()
|
| 822 |
item = record()
|
| 823 |
item["state"] = "COMPLETED_ALLOWED"
|
|
|
|
| 826 |
self.assertEqual("COMPLETED_ALLOWED", value["state"])
|
| 827 |
stored.assert_not_called()
|
| 828 |
|
| 829 |
+
def test_terminal_run_keeps_name_tombstone(self) -> None:
|
| 830 |
supervisor = self._instance()
|
| 831 |
with tempfile.TemporaryDirectory() as raw:
|
| 832 |
root = Path(raw)
|
|
|
|
| 838 |
self.assertTrue((supervisor.names / "demo.json").is_file())
|
| 839 |
item["state"] = "TERMINATED"
|
| 840 |
supervisor._store(item)
|
| 841 |
+
self.assertEqual(
|
| 842 |
+
{"run_id": "a" * 24},
|
| 843 |
+
__import__("json").loads(
|
| 844 |
+
(supervisor.names / "demo.json").read_text(encoding="utf-8")
|
| 845 |
+
),
|
| 846 |
+
)
|
| 847 |
+
|
| 848 |
+
def test_terminal_name_tombstone_is_not_listed_as_active(self) -> None:
|
| 849 |
+
supervisor = self._instance()
|
| 850 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 851 |
+
root = Path(raw)
|
| 852 |
+
supervisor.runs = root / "runs"
|
| 853 |
+
supervisor.names = root / "names"
|
| 854 |
+
item = record()
|
| 855 |
+
supervisor._store(item)
|
| 856 |
+
item["state"] = "TERMINATED"
|
| 857 |
+
supervisor._store(item)
|
| 858 |
+
self.assertEqual({"runs": []}, supervisor.handle({"action": "list", "args": {}}))
|
| 859 |
+
|
| 860 |
+
def test_backfill_reserves_names_from_legacy_terminal_records(self) -> None:
|
| 861 |
+
supervisor = self._instance()
|
| 862 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 863 |
+
root = Path(raw)
|
| 864 |
+
supervisor.runs = root / "runs"
|
| 865 |
+
supervisor.names = root / "names"
|
| 866 |
+
supervisor.runs.mkdir()
|
| 867 |
+
supervisor.names.mkdir()
|
| 868 |
+
item = record()
|
| 869 |
+
item["state"] = "TERMINATED"
|
| 870 |
+
(supervisor.runs / ("a" * 24 + ".json")).write_text(
|
| 871 |
+
__import__("json").dumps(item), encoding="utf-8"
|
| 872 |
+
)
|
| 873 |
+
supervisor._backfill_name_tombstones()
|
| 874 |
+
self.assertTrue((supervisor.names / "demo.json").is_file())
|
| 875 |
+
|
| 876 |
+
def test_store_rejects_a_second_run_using_a_reserved_name(self) -> None:
|
| 877 |
+
supervisor = self._instance()
|
| 878 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 879 |
+
root = Path(raw)
|
| 880 |
+
supervisor.runs = root / "runs"
|
| 881 |
+
supervisor.names = root / "names"
|
| 882 |
+
first = record()
|
| 883 |
+
supervisor._store(first)
|
| 884 |
+
first["state"] = "TERMINATED"
|
| 885 |
+
supervisor._store(first)
|
| 886 |
+
replacement_id = "c" * 24
|
| 887 |
+
replacement = {
|
| 888 |
+
**record(),
|
| 889 |
+
"cgroup": f"/system.slice/lumi-eggcracker-workload-{replacement_id}.service",
|
| 890 |
+
"run_id": replacement_id,
|
| 891 |
+
"unit": f"lumi-eggcracker-workload-{replacement_id}.service",
|
| 892 |
+
}
|
| 893 |
+
with self.assertRaisesRegex(JsonInputError, "name index collision"):
|
| 894 |
+
supervisor._store(replacement)
|
| 895 |
+
self.assertFalse((supervisor.runs / ("c" * 24 + ".json")).exists())
|
| 896 |
|
| 897 |
def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
|
| 898 |
supervisor = self._instance()
|