noqt commited on
Commit
ee6b2e4
·
1 Parent(s): 4671695

fix: reserve workload names after termination

Browse files
LIMITATIONS.md CHANGED
@@ -46,4 +46,6 @@ behavioural modelling, telemetry upload, malware prevention or EDR.
46
 
47
  Approvals are exact and deliberately narrow: only a qualified native llama runtime or root-owned CPython executing one absolute regular script is supported. Workload UID, resolved executable identity and complete argv digest must match at the protected pre-exec `start` gate. Python `-c`, `-m`, stdin/interactive, relative-script, package-directory, symlink and unsupported-interpreter forms are rejected. The Python script is identity/digest bound and launched from an immutable per-run root-owned stage; changing it aborts `start`. This does not claim a general dependency-closure analyser or make approved workloads a sandbox. A direct or third-party launch cannot consume an approval, and descendants or siblings do not inherit one. Revoking an approval affects future protected launches, not a workload already running under that approval.
48
 
 
 
49
  The explicit `start` command remains non-interactive. It has no terminal attachment, current-working-directory forwarding, timeout or output streaming. It supports one active selected workload globally; PID, memory and CPU limits are explicit. Detection receipts are bounded to 1000 entries.
 
46
 
47
  Approvals are exact and deliberately narrow: only a qualified native llama runtime or root-owned CPython executing one absolute regular script is supported. Workload UID, resolved executable identity and complete argv digest must match at the protected pre-exec `start` gate. Python `-c`, `-m`, stdin/interactive, relative-script, package-directory, symlink and unsupported-interpreter forms are rejected. The Python script is identity/digest bound and launched from an immutable per-run root-owned stage; changing it aborts `start`. This does not claim a general dependency-closure analyser or make approved workloads a sandbox. A direct or third-party launch cannot consume an approval, and descendants or siblings do not inherit one. Revoking an approval affects future protected launches, not a workload already running under that approval.
48
 
49
+ Operator-supplied workload names are single-use for the lifetime of an installation. A name remains reserved after completion or containment so a stale status or kill command cannot resolve to a replacement workload. Clean uninstallation removes those reservations with the rest of Eggcracker's local state.
50
+
51
  The explicit `start` command remains non-interactive. It has no terminal attachment, current-working-directory forwarding, timeout or output streaming. It supports one active selected workload globally; PID, memory and CPU limits are explicit. Detection receipts are bounded to 1000 entries.
src/lumi_eggcracker/supervisor.py CHANGED
@@ -383,22 +383,26 @@ class Supervisor:
383
  record = validate_run(record)
384
  if not hasattr(self, "active_cgroups"):
385
  self.active_cgroups = set()
386
- if record["state"] in ACTIVE_STATES:
387
- self.active_cgroups.add(record["cgroup"])
388
- else:
389
- self.active_cgroups.discard(record["cgroup"])
390
- self.operations.append("durable-state")
391
- write_atomic(run_path(self.runs, record["run_id"]), record)
392
  pointer = name_path(self.names, record["name"])
393
- if record["state"] in ACTIVE_STATES:
394
- write_atomic(pointer, {"run_id": record["run_id"]})
395
- elif pointer.exists():
396
  try:
397
  current = load_regular_json(pointer)
398
- if current == {"run_id": record["run_id"]}:
399
- pointer.unlink()
400
  except JsonInputError:
401
  raise JsonInputError("workload name index is invalid")
 
 
 
 
 
 
 
 
 
 
 
 
402
  if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
403
  provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
404
  self._clear_stage(record["run_id"])
@@ -445,7 +449,7 @@ class Supervisor:
445
  return record
446
 
447
  def _latest_by_name(self, name: str) -> dict[str, Any]:
448
- """Resolve a completed run only when its name is no longer active."""
449
  candidates: list[dict[str, Any]] = []
450
  for path in self.runs.glob("*.json"):
451
  try:
@@ -458,6 +462,33 @@ class Supervisor:
458
  raise JsonInputError("workload name is unavailable")
459
  return max(candidates, key=lambda item: int(item["created_monotonic_ns"]))
460
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
461
  def _receipt_path(self, event_id: str) -> Path:
462
  return self.receipts / f"{event_id}.json"
463
 
@@ -531,6 +562,7 @@ class Supervisor:
531
  incident_store.compact(self.incidents)
532
  except (JsonInputError, OSError):
533
  self.incident_health_healthy = False
 
534
  self._recover()
535
  self.quarantine_root = self._prepare_quarantine()
536
  self._scan_synchronously()
@@ -2532,7 +2564,12 @@ class Supervisor:
2532
  if not self._incident_status()["healthy"]:
2533
  raise JsonInputError("local lockdown state is unavailable; root recovery is required")
2534
  with self.start_lock, self.approval_lock:
2535
- if name_path(self.names, name).exists() or self._active_exists():
 
 
 
 
 
2536
  raise JsonInputError(
2537
  "one protected workload is already active or name is unavailable"
2538
  )
@@ -2980,11 +3017,7 @@ class Supervisor:
2980
  }
2981
  if action == "start":
2982
  return self._start(args)
2983
- if action == "status" and set(args) == {"name"}:
2984
- try:
2985
- record = self._load(args["name"])
2986
- except JsonInputError:
2987
- record = self._latest_by_name(args["name"])
2988
  return {
2989
  "name": record["name"],
2990
  "run_id": record["run_id"],
@@ -3000,11 +3033,20 @@ class Supervisor:
3000
  "incident_lockdown": self._incident_status()["lockdown"],
3001
  "workload_uid": record["workload_uid"],
3002
  }
 
 
 
 
 
 
 
3003
  if action == "list" and not args:
3004
- runs = [
3005
- self.handle({"action": "status", "args": {"name": path.stem}})
3006
- for path in sorted(self.names.glob("*.json"))
3007
- ]
 
 
3008
  return {"runs": runs}
3009
  if action in {"approve", "revoke", "exec_policy_create", "exec_policy_revoke"} and not self._incident_status()["healthy"]:
3010
  raise JsonInputError("local lockdown state is unavailable; root recovery is required")
 
383
  record = validate_run(record)
384
  if not hasattr(self, "active_cgroups"):
385
  self.active_cgroups = set()
 
 
 
 
 
 
386
  pointer = name_path(self.names, record["name"])
387
+ expected_pointer = {"run_id": record["run_id"]}
388
+ pointer_missing = not pointer.exists() and not pointer.is_symlink()
389
+ if not pointer_missing:
390
  try:
391
  current = load_regular_json(pointer)
 
 
392
  except JsonInputError:
393
  raise JsonInputError("workload name index is invalid")
394
+ if current != expected_pointer:
395
+ raise JsonInputError("workload name index collision")
396
+ if pointer_missing:
397
+ # A reserved name is safe even if the following record write
398
+ # fails; allowing reuse after a partial start is not.
399
+ write_atomic(pointer, expected_pointer)
400
+ self.operations.append("durable-state")
401
+ write_atomic(run_path(self.runs, record["run_id"]), record)
402
+ if record["state"] in ACTIVE_STATES:
403
+ self.active_cgroups.add(record["cgroup"])
404
+ else:
405
+ self.active_cgroups.discard(record["cgroup"])
406
  if record["state"] not in ACTIVE_STATES and hasattr(self, "launches"):
407
  provenance_path(self.launches, record["run_id"]).unlink(missing_ok=True)
408
  self._clear_stage(record["run_id"])
 
449
  return record
450
 
451
  def _latest_by_name(self, name: str) -> dict[str, Any]:
452
+ """Resolve retained history for an unavailable or completed name."""
453
  candidates: list[dict[str, Any]] = []
454
  for path in self.runs.glob("*.json"):
455
  try:
 
462
  raise JsonInputError("workload name is unavailable")
463
  return max(candidates, key=lambda item: int(item["created_monotonic_ns"]))
464
 
465
+ def _backfill_name_tombstones(self) -> None:
466
+ """Reserve names created by releases that removed terminal pointers."""
467
+ latest: dict[str, dict[str, Any]] = {}
468
+ for path in self.runs.glob("*.json"):
469
+ try:
470
+ record = load_run(self.runs, path.stem)
471
+ except JsonInputError:
472
+ continue
473
+ prior = latest.get(record["name"])
474
+ if prior is None or int(record["created_monotonic_ns"]) > int(
475
+ prior["created_monotonic_ns"]
476
+ ):
477
+ latest[record["name"]] = record
478
+ for name, record in latest.items():
479
+ pointer = name_path(self.names, name)
480
+ if pointer.exists() or pointer.is_symlink():
481
+ value = load_regular_json(pointer)
482
+ run_id = value.get("run_id")
483
+ if (
484
+ set(value) != {"run_id"}
485
+ or not isinstance(run_id, str)
486
+ or not RUN_ID.fullmatch(run_id)
487
+ ):
488
+ raise JsonInputError("workload name index is invalid")
489
+ continue
490
+ write_atomic(pointer, {"run_id": record["run_id"]})
491
+
492
  def _receipt_path(self, event_id: str) -> Path:
493
  return self.receipts / f"{event_id}.json"
494
 
 
562
  incident_store.compact(self.incidents)
563
  except (JsonInputError, OSError):
564
  self.incident_health_healthy = False
565
+ self._backfill_name_tombstones()
566
  self._recover()
567
  self.quarantine_root = self._prepare_quarantine()
568
  self._scan_synchronously()
 
2564
  if not self._incident_status()["healthy"]:
2565
  raise JsonInputError("local lockdown state is unavailable; root recovery is required")
2566
  with self.start_lock, self.approval_lock:
2567
+ reserved_name = name_path(self.names, name)
2568
+ if (
2569
+ reserved_name.exists()
2570
+ or reserved_name.is_symlink()
2571
+ or self._active_exists()
2572
+ ):
2573
  raise JsonInputError(
2574
  "one protected workload is already active or name is unavailable"
2575
  )
 
3017
  }
3018
  if action == "start":
3019
  return self._start(args)
3020
+ def public_run(record: dict[str, Any]) -> dict[str, Any]:
 
 
 
 
3021
  return {
3022
  "name": record["name"],
3023
  "run_id": record["run_id"],
 
3033
  "incident_lockdown": self._incident_status()["lockdown"],
3034
  "workload_uid": record["workload_uid"],
3035
  }
3036
+
3037
+ if action == "status" and set(args) == {"name"}:
3038
+ try:
3039
+ record = self._load(args["name"])
3040
+ except JsonInputError:
3041
+ record = self._latest_by_name(args["name"])
3042
+ return public_run(record)
3043
  if action == "list" and not args:
3044
+ runs = []
3045
+ for path in sorted(self.names.glob("*.json")):
3046
+ try:
3047
+ runs.append(public_run(self._load(path.stem)))
3048
+ except JsonInputError:
3049
+ continue
3050
  return {"runs": runs}
3051
  if action in {"approve", "revoke", "exec_policy_create", "exec_policy_revoke"} and not self._incident_status()["healthy"]:
3052
  raise JsonInputError("local lockdown state is unavailable; root recovery is required")
tests/test_supervisor.py CHANGED
@@ -817,7 +817,7 @@ class SupervisorTests(unittest.TestCase):
817
  self.assertEqual("RUNNING", value["state"])
818
  stored.assert_not_called()
819
 
820
- def test_status_resolves_the_latest_terminal_run_after_name_reuse_is_enabled(self) -> None:
821
  supervisor = self._instance()
822
  item = record()
823
  item["state"] = "COMPLETED_ALLOWED"
@@ -826,7 +826,7 @@ class SupervisorTests(unittest.TestCase):
826
  self.assertEqual("COMPLETED_ALLOWED", value["state"])
827
  stored.assert_not_called()
828
 
829
- def test_run_records_are_keyed_by_run_id_and_name_pointer_is_removed_when_terminal(self) -> None:
830
  supervisor = self._instance()
831
  with tempfile.TemporaryDirectory() as raw:
832
  root = Path(raw)
@@ -838,7 +838,61 @@ class SupervisorTests(unittest.TestCase):
838
  self.assertTrue((supervisor.names / "demo.json").is_file())
839
  item["state"] = "TERMINATED"
840
  supervisor._store(item)
841
- self.assertFalse((supervisor.names / "demo.json").exists())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
842
 
843
  def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
844
  supervisor = self._instance()
 
817
  self.assertEqual("RUNNING", value["state"])
818
  stored.assert_not_called()
819
 
820
+ def test_status_resolves_retained_terminal_history(self) -> None:
821
  supervisor = self._instance()
822
  item = record()
823
  item["state"] = "COMPLETED_ALLOWED"
 
826
  self.assertEqual("COMPLETED_ALLOWED", value["state"])
827
  stored.assert_not_called()
828
 
829
+ def test_terminal_run_keeps_name_tombstone(self) -> None:
830
  supervisor = self._instance()
831
  with tempfile.TemporaryDirectory() as raw:
832
  root = Path(raw)
 
838
  self.assertTrue((supervisor.names / "demo.json").is_file())
839
  item["state"] = "TERMINATED"
840
  supervisor._store(item)
841
+ self.assertEqual(
842
+ {"run_id": "a" * 24},
843
+ __import__("json").loads(
844
+ (supervisor.names / "demo.json").read_text(encoding="utf-8")
845
+ ),
846
+ )
847
+
848
+ def test_terminal_name_tombstone_is_not_listed_as_active(self) -> None:
849
+ supervisor = self._instance()
850
+ with tempfile.TemporaryDirectory() as raw:
851
+ root = Path(raw)
852
+ supervisor.runs = root / "runs"
853
+ supervisor.names = root / "names"
854
+ item = record()
855
+ supervisor._store(item)
856
+ item["state"] = "TERMINATED"
857
+ supervisor._store(item)
858
+ self.assertEqual({"runs": []}, supervisor.handle({"action": "list", "args": {}}))
859
+
860
+ def test_backfill_reserves_names_from_legacy_terminal_records(self) -> None:
861
+ supervisor = self._instance()
862
+ with tempfile.TemporaryDirectory() as raw:
863
+ root = Path(raw)
864
+ supervisor.runs = root / "runs"
865
+ supervisor.names = root / "names"
866
+ supervisor.runs.mkdir()
867
+ supervisor.names.mkdir()
868
+ item = record()
869
+ item["state"] = "TERMINATED"
870
+ (supervisor.runs / ("a" * 24 + ".json")).write_text(
871
+ __import__("json").dumps(item), encoding="utf-8"
872
+ )
873
+ supervisor._backfill_name_tombstones()
874
+ self.assertTrue((supervisor.names / "demo.json").is_file())
875
+
876
+ def test_store_rejects_a_second_run_using_a_reserved_name(self) -> None:
877
+ supervisor = self._instance()
878
+ with tempfile.TemporaryDirectory() as raw:
879
+ root = Path(raw)
880
+ supervisor.runs = root / "runs"
881
+ supervisor.names = root / "names"
882
+ first = record()
883
+ supervisor._store(first)
884
+ first["state"] = "TERMINATED"
885
+ supervisor._store(first)
886
+ replacement_id = "c" * 24
887
+ replacement = {
888
+ **record(),
889
+ "cgroup": f"/system.slice/lumi-eggcracker-workload-{replacement_id}.service",
890
+ "run_id": replacement_id,
891
+ "unit": f"lumi-eggcracker-workload-{replacement_id}.service",
892
+ }
893
+ with self.assertRaisesRegex(JsonInputError, "name index collision"):
894
+ supervisor._store(replacement)
895
+ self.assertFalse((supervisor.runs / ("c" * 24 + ".json")).exists())
896
 
897
  def test_autonomous_kill_terminal_state_wins_completion_race(self) -> None:
898
  supervisor = self._instance()