Spaces:
Running
Running
Repair Daybreak adversarial discovery invariants
Browse files- src/lumi_eggcracker/approvals.py +2 -0
- src/lumi_eggcracker/artifacts.py +41 -6
- src/lumi_eggcracker/discovery.py +57 -17
- src/lumi_eggcracker/elfmarkers.py +155 -48
- src/lumi_eggcracker/observation.py +10 -0
- src/lumi_eggcracker/supervisor.py +192 -20
- src/lumi_eggcracker/watchdog.py +36 -3
- tests/test_approvals.py +8 -0
- tests/test_artifacts.py +19 -1
- tests/test_discovery.py +49 -0
- tests/test_elfmarkers.py +49 -0
- tests/test_supervisor.py +59 -0
- tests/test_watchdog.py +22 -0
src/lumi_eggcracker/approvals.py
CHANGED
|
@@ -86,6 +86,8 @@ def approved(
|
|
| 86 |
*,
|
| 87 |
executable_metadata: os.stat_result | tuple[int, int] | None = None,
|
| 88 |
) -> bool:
|
|
|
|
|
|
|
| 89 |
command_hash = argv_digest(snapshot.argv)
|
| 90 |
if executable_metadata is None:
|
| 91 |
try:
|
|
|
|
| 86 |
*,
|
| 87 |
executable_metadata: os.stat_result | tuple[int, int] | None = None,
|
| 88 |
) -> bool:
|
| 89 |
+
if not getattr(snapshot, "argv_complete", True):
|
| 90 |
+
return False
|
| 91 |
command_hash = argv_digest(snapshot.argv)
|
| 92 |
if executable_metadata is None:
|
| 93 |
try:
|
src/lumi_eggcracker/artifacts.py
CHANGED
|
@@ -16,6 +16,7 @@ from .jsonio import JsonInputError
|
|
| 16 |
MAX_ARTIFACT_BYTES = 1 * 1024 * 1024
|
| 17 |
MAX_ARTIFACTS = 16
|
| 18 |
MAX_TOTAL_BYTES = 4 * 1024 * 1024
|
|
|
|
| 19 |
MAX_GGUF_ITEMS = 10_000_000
|
| 20 |
MAX_SAFETENSORS_TENSORS = 100_000
|
| 21 |
MAX_SAFETENSORS_DIMS = 64
|
|
@@ -292,19 +293,43 @@ def from_process_fds(
|
|
| 292 |
*,
|
| 293 |
proc: Path = Path("/proc"),
|
| 294 |
cache: ArtifactCache | None = None,
|
|
|
|
|
|
|
| 295 |
) -> tuple[ArtifactEvidence, ...]:
|
| 296 |
-
"""Inspect
|
| 297 |
|
| 298 |
The descriptor is opened through ``/proc/<pid>/fd/<n>``. The link name is
|
| 299 |
used only to find a descriptor number; it is never accepted as evidence.
|
|
|
|
|
|
|
| 300 |
"""
|
| 301 |
identity = snapshot.identity
|
| 302 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
result: list[ArtifactEvidence] = []
|
| 304 |
consumed = 0
|
| 305 |
-
for number
|
| 306 |
-
if not isinstance(number, int) or number < 0:
|
| 307 |
-
continue
|
| 308 |
# /proc/<pid>/fd/<n> is necessarily a procfs symlink. We open that
|
| 309 |
# exact descriptor target, then validate the opened inode; O_NOFOLLOW
|
| 310 |
# would reject every legitimate procfs descriptor.
|
|
@@ -349,9 +374,19 @@ def from_snapshot(
|
|
| 349 |
*,
|
| 350 |
proc: Path = Path("/proc"),
|
| 351 |
cache: ArtifactCache | None = None,
|
|
|
|
|
|
|
| 352 |
) -> tuple[ArtifactEvidence, ...]:
|
| 353 |
"""Collect bounded GGUF evidence from open descriptors and mapped files."""
|
| 354 |
-
result = list(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 355 |
for raw in tuple(getattr(snapshot, "map_paths", ())):
|
| 356 |
if not isinstance(raw, str) or not raw.startswith("/"):
|
| 357 |
continue
|
|
|
|
| 16 |
MAX_ARTIFACT_BYTES = 1 * 1024 * 1024
|
| 17 |
MAX_ARTIFACTS = 16
|
| 18 |
MAX_TOTAL_BYTES = 4 * 1024 * 1024
|
| 19 |
+
MAX_FD_PROBES_PER_SCAN = 256
|
| 20 |
MAX_GGUF_ITEMS = 10_000_000
|
| 21 |
MAX_SAFETENSORS_TENSORS = 100_000
|
| 22 |
MAX_SAFETENSORS_DIMS = 64
|
|
|
|
| 293 |
*,
|
| 294 |
proc: Path = Path("/proc"),
|
| 295 |
cache: ArtifactCache | None = None,
|
| 296 |
+
start_index: int = 0,
|
| 297 |
+
max_probes: int = MAX_FD_PROBES_PER_SCAN,
|
| 298 |
) -> tuple[ArtifactEvidence, ...]:
|
| 299 |
+
"""Inspect one fair bounded window of currently open target descriptors.
|
| 300 |
|
| 301 |
The descriptor is opened through ``/proc/<pid>/fd/<n>``. The link name is
|
| 302 |
used only to find a descriptor number; it is never accepted as evidence.
|
| 303 |
+
Callers advance ``start_index`` between scans so a held-open artifact
|
| 304 |
+
cannot be hidden permanently behind harmless lower-numbered descriptors.
|
| 305 |
"""
|
| 306 |
identity = snapshot.identity
|
| 307 |
+
if isinstance(start_index, bool) or not isinstance(start_index, int) or start_index < 0:
|
| 308 |
+
raise JsonInputError("artifact descriptor cursor is invalid")
|
| 309 |
+
if isinstance(max_probes, bool) or not isinstance(max_probes, int) or max_probes < 1:
|
| 310 |
+
raise JsonInputError("artifact descriptor probe limit is invalid")
|
| 311 |
+
directory = proc / str(identity.pid) / "fd"
|
| 312 |
+
try:
|
| 313 |
+
numbers = sorted(
|
| 314 |
+
int(item.name) for item in directory.iterdir() if item.name.isdigit()
|
| 315 |
+
)
|
| 316 |
+
except (AttributeError, OSError):
|
| 317 |
+
numbers = sorted(
|
| 318 |
+
{
|
| 319 |
+
number
|
| 320 |
+
for number, _target in tuple(getattr(snapshot, "fd_entries", ()))
|
| 321 |
+
if isinstance(number, int) and not isinstance(number, bool) and number >= 0
|
| 322 |
+
}
|
| 323 |
+
)
|
| 324 |
+
if numbers:
|
| 325 |
+
offset = start_index % len(numbers)
|
| 326 |
+
probe_count = min(max_probes, len(numbers))
|
| 327 |
+
selected = tuple(numbers[(offset + index) % len(numbers)] for index in range(probe_count))
|
| 328 |
+
else:
|
| 329 |
+
selected = ()
|
| 330 |
result: list[ArtifactEvidence] = []
|
| 331 |
consumed = 0
|
| 332 |
+
for number in selected:
|
|
|
|
|
|
|
| 333 |
# /proc/<pid>/fd/<n> is necessarily a procfs symlink. We open that
|
| 334 |
# exact descriptor target, then validate the opened inode; O_NOFOLLOW
|
| 335 |
# would reject every legitimate procfs descriptor.
|
|
|
|
| 374 |
*,
|
| 375 |
proc: Path = Path("/proc"),
|
| 376 |
cache: ArtifactCache | None = None,
|
| 377 |
+
fd_start_index: int = 0,
|
| 378 |
+
fd_max_probes: int = MAX_FD_PROBES_PER_SCAN,
|
| 379 |
) -> tuple[ArtifactEvidence, ...]:
|
| 380 |
"""Collect bounded GGUF evidence from open descriptors and mapped files."""
|
| 381 |
+
result = list(
|
| 382 |
+
from_process_fds(
|
| 383 |
+
snapshot,
|
| 384 |
+
proc=proc,
|
| 385 |
+
cache=cache,
|
| 386 |
+
start_index=fd_start_index,
|
| 387 |
+
max_probes=fd_max_probes,
|
| 388 |
+
)
|
| 389 |
+
)
|
| 390 |
for raw in tuple(getattr(snapshot, "map_paths", ())):
|
| 391 |
if not isinstance(raw, str) or not raw.startswith("/"):
|
| 392 |
continue
|
src/lumi_eggcracker/discovery.py
CHANGED
|
@@ -13,7 +13,8 @@ from .jsonio import JsonInputError
|
|
| 13 |
|
| 14 |
PROC = Path("/proc")
|
| 15 |
MAX_READ = 64 * 1024
|
| 16 |
-
|
|
|
|
| 17 |
MAX_FDS = 256
|
| 18 |
MAX_MAPS = 512
|
| 19 |
|
|
@@ -37,6 +38,7 @@ class ProcessSnapshot:
|
|
| 37 |
fd_entries: tuple[tuple[int, str], ...] = ()
|
| 38 |
map_paths: tuple[str, ...] = ()
|
| 39 |
parent: ProcessIdentity | None = None
|
|
|
|
| 40 |
|
| 41 |
|
| 42 |
def _read(path: Path, limit: int = MAX_READ) -> bytes:
|
|
@@ -79,7 +81,9 @@ def identity(pid: int, *, proc: Path = PROC) -> ProcessIdentity | None:
|
|
| 79 |
|
| 80 |
|
| 81 |
def argv_digest(argv: tuple[str, ...] | list[str]) -> str:
|
| 82 |
-
return hashlib.sha256(
|
|
|
|
|
|
|
| 83 |
|
| 84 |
|
| 85 |
def executable_digest(path: Path) -> str:
|
|
@@ -214,9 +218,11 @@ def _links(directory: Path, *, limit: int) -> tuple[str, ...]:
|
|
| 214 |
|
| 215 |
def _fd_entries(directory: Path, *, limit: int) -> tuple[tuple[int, str], ...]:
|
| 216 |
values: list[tuple[int, str]] = []
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
|
|
|
|
|
|
| 220 |
try:
|
| 221 |
target = os.readlink(item)
|
| 222 |
except OSError:
|
|
@@ -229,7 +235,7 @@ def _fd_entries(directory: Path, *, limit: int) -> tuple[tuple[int, str], ...]:
|
|
| 229 |
def _maps(path: Path) -> tuple[str, ...]:
|
| 230 |
try:
|
| 231 |
raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
|
| 232 |
-
except OSError:
|
| 233 |
return ()
|
| 234 |
values: list[str] = []
|
| 235 |
for line in raw.splitlines()[:MAX_MAPS]:
|
|
@@ -242,7 +248,7 @@ def _maps(path: Path) -> tuple[str, ...]:
|
|
| 242 |
def _map_paths(path: Path) -> tuple[str, ...]:
|
| 243 |
try:
|
| 244 |
raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
|
| 245 |
-
except OSError:
|
| 246 |
return ()
|
| 247 |
values: list[str] = []
|
| 248 |
for line in raw.splitlines()[:MAX_MAPS]:
|
|
@@ -252,7 +258,12 @@ def _map_paths(path: Path) -> tuple[str, ...]:
|
|
| 252 |
return tuple(values)
|
| 253 |
|
| 254 |
|
| 255 |
-
def snapshot(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
current = identity(value.pid, proc=proc)
|
| 257 |
if current != value:
|
| 258 |
return None
|
|
@@ -260,20 +271,45 @@ def snapshot(value: ProcessIdentity, *, proc: Path = PROC) -> ProcessSnapshot |
|
|
| 260 |
try:
|
| 261 |
parent_pid, _start = parse_stat(_read(root / "stat", 4096).decode("utf-8"))
|
| 262 |
exe = os.readlink(root / "exe")
|
| 263 |
-
command = tuple(
|
| 264 |
-
item for item in _read(root / "cmdline").decode("utf-8").split("\0") if item
|
| 265 |
-
)
|
| 266 |
cgroups = tuple(
|
| 267 |
line for line in _read(root / "cgroup", 8192).decode("utf-8").splitlines() if line
|
| 268 |
)
|
| 269 |
uid = _uid(_read(root / "status", 8192).decode("utf-8"))
|
| 270 |
-
fd_entries = _fd_entries(root / "fd", limit=MAX_FDS)
|
| 271 |
-
fd_paths = tuple(item[1] for item in fd_entries)
|
| 272 |
except (OSError, UnicodeDecodeError, JsonInputError):
|
| 273 |
return None
|
| 274 |
-
if not
|
| 275 |
return None
|
| 276 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
parent = identity(parent_pid, proc=proc) if parent_pid > 0 else None
|
| 278 |
return ProcessSnapshot(
|
| 279 |
value,
|
|
@@ -287,11 +323,15 @@ def snapshot(value: ProcessIdentity, *, proc: Path = PROC) -> ProcessSnapshot |
|
|
| 287 |
fd_entries,
|
| 288 |
maps,
|
| 289 |
parent,
|
|
|
|
| 290 |
)
|
| 291 |
|
| 292 |
|
| 293 |
def scan(
|
| 294 |
-
*,
|
|
|
|
|
|
|
|
|
|
| 295 |
) -> list[ProcessSnapshot]:
|
| 296 |
result: list[ProcessSnapshot] = []
|
| 297 |
try:
|
|
@@ -302,7 +342,7 @@ def scan(
|
|
| 302 |
item = identity(pid, proc=proc)
|
| 303 |
if item is None:
|
| 304 |
continue
|
| 305 |
-
current = snapshot(item, proc=proc)
|
| 306 |
if current is not None and not (exclude and exclude(current)):
|
| 307 |
result.append(current)
|
| 308 |
return result
|
|
|
|
| 13 |
|
| 14 |
PROC = Path("/proc")
|
| 15 |
MAX_READ = 64 * 1024
|
| 16 |
+
MAX_CMDLINE_BYTES = 8 * 1024 * 1024
|
| 17 |
+
MAX_MAP_BYTES = 8 * 1024 * 1024
|
| 18 |
MAX_FDS = 256
|
| 19 |
MAX_MAPS = 512
|
| 20 |
|
|
|
|
| 38 |
fd_entries: tuple[tuple[int, str], ...] = ()
|
| 39 |
map_paths: tuple[str, ...] = ()
|
| 40 |
parent: ProcessIdentity | None = None
|
| 41 |
+
argv_complete: bool = True
|
| 42 |
|
| 43 |
|
| 44 |
def _read(path: Path, limit: int = MAX_READ) -> bytes:
|
|
|
|
| 81 |
|
| 82 |
|
| 83 |
def argv_digest(argv: tuple[str, ...] | list[str]) -> str:
|
| 84 |
+
return hashlib.sha256(
|
| 85 |
+
"\0".join(argv).encode("utf-8", errors="surrogateescape")
|
| 86 |
+
).hexdigest()
|
| 87 |
|
| 88 |
|
| 89 |
def executable_digest(path: Path) -> str:
|
|
|
|
| 218 |
|
| 219 |
def _fd_entries(directory: Path, *, limit: int) -> tuple[tuple[int, str], ...]:
|
| 220 |
values: list[tuple[int, str]] = []
|
| 221 |
+
entries = sorted(
|
| 222 |
+
(item for item in directory.iterdir() if item.name.isdigit()),
|
| 223 |
+
key=lambda value: int(value.name),
|
| 224 |
+
)
|
| 225 |
+
for item in entries[:limit]:
|
| 226 |
try:
|
| 227 |
target = os.readlink(item)
|
| 228 |
except OSError:
|
|
|
|
| 235 |
def _maps(path: Path) -> tuple[str, ...]:
|
| 236 |
try:
|
| 237 |
raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
|
| 238 |
+
except (OSError, JsonInputError):
|
| 239 |
return ()
|
| 240 |
values: list[str] = []
|
| 241 |
for line in raw.splitlines()[:MAX_MAPS]:
|
|
|
|
| 248 |
def _map_paths(path: Path) -> tuple[str, ...]:
|
| 249 |
try:
|
| 250 |
raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
|
| 251 |
+
except (OSError, JsonInputError):
|
| 252 |
return ()
|
| 253 |
values: list[str] = []
|
| 254 |
for line in raw.splitlines()[:MAX_MAPS]:
|
|
|
|
| 258 |
return tuple(values)
|
| 259 |
|
| 260 |
|
| 261 |
+
def snapshot(
|
| 262 |
+
value: ProcessIdentity,
|
| 263 |
+
*,
|
| 264 |
+
proc: Path = PROC,
|
| 265 |
+
include_evidence: bool = True,
|
| 266 |
+
) -> ProcessSnapshot | None:
|
| 267 |
current = identity(value.pid, proc=proc)
|
| 268 |
if current != value:
|
| 269 |
return None
|
|
|
|
| 271 |
try:
|
| 272 |
parent_pid, _start = parse_stat(_read(root / "stat", 4096).decode("utf-8"))
|
| 273 |
exe = os.readlink(root / "exe")
|
|
|
|
|
|
|
|
|
|
| 274 |
cgroups = tuple(
|
| 275 |
line for line in _read(root / "cgroup", 8192).decode("utf-8").splitlines() if line
|
| 276 |
)
|
| 277 |
uid = _uid(_read(root / "status", 8192).decode("utf-8"))
|
|
|
|
|
|
|
| 278 |
except (OSError, UnicodeDecodeError, JsonInputError):
|
| 279 |
return None
|
| 280 |
+
if not exe or len(exe) > 4096:
|
| 281 |
return None
|
| 282 |
+
|
| 283 |
+
# Linux argv is a byte sequence. Invalid UTF-8 must not erase the whole
|
| 284 |
+
# process from discovery, and a bounded-read failure must make approval
|
| 285 |
+
# unavailable rather than making the process invisible.
|
| 286 |
+
try:
|
| 287 |
+
command = tuple(
|
| 288 |
+
item
|
| 289 |
+
for item in _read(root / "cmdline", MAX_CMDLINE_BYTES)
|
| 290 |
+
.decode("utf-8", errors="surrogateescape")
|
| 291 |
+
.split("\0")
|
| 292 |
+
if item
|
| 293 |
+
)
|
| 294 |
+
argv_complete = True
|
| 295 |
+
except (OSError, JsonInputError):
|
| 296 |
+
command = ()
|
| 297 |
+
argv_complete = False
|
| 298 |
+
|
| 299 |
+
fd_entries: tuple[tuple[int, str], ...] = ()
|
| 300 |
+
maps: tuple[str, ...] = ()
|
| 301 |
+
if include_evidence:
|
| 302 |
+
try:
|
| 303 |
+
fd_entries = _fd_entries(root / "fd", limit=MAX_FDS)
|
| 304 |
+
except OSError:
|
| 305 |
+
fd_entries = ()
|
| 306 |
+
# Native runtime discovery uses stable /proc/<pid>/map_files handles.
|
| 307 |
+
# This fallback is retained for kernels/test fixtures without it and
|
| 308 |
+
# is deliberately isolated so one oversized maps file cannot abort a
|
| 309 |
+
# global discovery pass.
|
| 310 |
+
if not (root / "map_files").is_dir():
|
| 311 |
+
maps = _map_paths(root / "maps")
|
| 312 |
+
fd_paths = tuple(item[1] for item in fd_entries)
|
| 313 |
parent = identity(parent_pid, proc=proc) if parent_pid > 0 else None
|
| 314 |
return ProcessSnapshot(
|
| 315 |
value,
|
|
|
|
| 323 |
fd_entries,
|
| 324 |
maps,
|
| 325 |
parent,
|
| 326 |
+
argv_complete,
|
| 327 |
)
|
| 328 |
|
| 329 |
|
| 330 |
def scan(
|
| 331 |
+
*,
|
| 332 |
+
proc: Path = PROC,
|
| 333 |
+
exclude: Callable[[ProcessSnapshot], bool] | None = None,
|
| 334 |
+
include_evidence: bool = True,
|
| 335 |
) -> list[ProcessSnapshot]:
|
| 336 |
result: list[ProcessSnapshot] = []
|
| 337 |
try:
|
|
|
|
| 342 |
item = identity(pid, proc=proc)
|
| 343 |
if item is None:
|
| 344 |
continue
|
| 345 |
+
current = snapshot(item, proc=proc, include_evidence=include_evidence)
|
| 346 |
if current is not None and not (exclude and exclude(current)):
|
| 347 |
result.append(current)
|
| 348 |
return result
|
src/lumi_eggcracker/elfmarkers.py
CHANGED
|
@@ -30,7 +30,7 @@ PINNED_PYTORCH_ATEN_BUILD_IDS = frozenset({"ad9ab6eeec3b28a0ec3f12f266627610de90
|
|
| 30 |
PYTORCH_BRIDGE_EVIDENCE_ID = "pytorch-bridge-build-id-pinned-cpu"
|
| 31 |
PYTORCH_ATEN_EVIDENCE_ID = "pytorch-aten-build-id-pinned-cpu"
|
| 32 |
PYTORCH_PAIR_EVIDENCE_ID = "pytorch-bridge-aten-pair-pinned-cpu"
|
| 33 |
-
MAX_RUNTIME_CANDIDATES =
|
| 34 |
|
| 35 |
|
| 36 |
@dataclass(frozen=True)
|
|
@@ -182,13 +182,8 @@ def _build_id(descriptor: int, size: int) -> str | None:
|
|
| 182 |
return None
|
| 183 |
|
| 184 |
|
| 185 |
-
def
|
| 186 |
-
"""Recognise
|
| 187 |
-
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
| 188 |
-
try:
|
| 189 |
-
descriptor = os.open(path, flags)
|
| 190 |
-
except OSError:
|
| 191 |
-
return None
|
| 192 |
try:
|
| 193 |
before = os.fstat(descriptor)
|
| 194 |
if not stat.S_ISREG(before.st_mode) or before.st_size < 64 or before.st_size > (1 << 40):
|
|
@@ -214,17 +209,10 @@ def inspect_path(path: Path) -> RuntimeEvidence | None:
|
|
| 214 |
return None
|
| 215 |
except (JsonInputError, OSError, struct.error):
|
| 216 |
return None
|
| 217 |
-
finally:
|
| 218 |
-
os.close(descriptor)
|
| 219 |
|
| 220 |
|
| 221 |
-
def
|
| 222 |
-
"""Recognise one exact pinned PyTorch bridge/ATen
|
| 223 |
-
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
| 224 |
-
try:
|
| 225 |
-
descriptor = os.open(path, flags)
|
| 226 |
-
except OSError:
|
| 227 |
-
return None
|
| 228 |
try:
|
| 229 |
before = os.fstat(descriptor)
|
| 230 |
if not stat.S_ISREG(before.st_mode) or before.st_size < 64:
|
|
@@ -248,15 +236,50 @@ def inspect_pytorch_path(path: Path) -> RuntimeEvidence | None:
|
|
| 248 |
return None
|
| 249 |
except (JsonInputError, OSError, struct.error):
|
| 250 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
finally:
|
| 252 |
os.close(descriptor)
|
| 253 |
|
| 254 |
|
| 255 |
def _inspect_candidate(path: Path) -> tuple[RuntimeEvidence, ...]:
|
| 256 |
values: list[RuntimeEvidence] = []
|
| 257 |
-
|
| 258 |
-
if
|
| 259 |
-
values.append(
|
| 260 |
pytorch = inspect_pytorch_path(path)
|
| 261 |
if pytorch is not None and pytorch not in values:
|
| 262 |
values.append(pytorch)
|
|
@@ -276,38 +299,122 @@ def with_pytorch_pair(values: Iterable[RuntimeEvidence]) -> tuple[RuntimeEvidenc
|
|
| 276 |
return tuple(result)
|
| 277 |
|
| 278 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
def from_snapshot(
|
| 280 |
-
snapshot: object,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
) -> tuple[RuntimeEvidence, ...]:
|
| 282 |
-
"""Inspect
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
|
|
|
| 293 |
result: list[RuntimeEvidence] = []
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
continue
|
| 297 |
-
path = Path(raw)
|
| 298 |
-
if cache is None:
|
| 299 |
-
values = _inspect_candidate(path)
|
| 300 |
-
else:
|
| 301 |
-
try:
|
| 302 |
-
key = _cache_key(path.stat())
|
| 303 |
-
except OSError:
|
| 304 |
-
continue
|
| 305 |
-
if key in cache:
|
| 306 |
-
values = cache[key]
|
| 307 |
-
else:
|
| 308 |
-
values = _inspect_candidate(path)
|
| 309 |
-
cache[key] = values
|
| 310 |
for evidence in values:
|
| 311 |
if evidence not in result:
|
| 312 |
result.append(evidence)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
return with_pytorch_pair(result)
|
|
|
|
| 30 |
PYTORCH_BRIDGE_EVIDENCE_ID = "pytorch-bridge-build-id-pinned-cpu"
|
| 31 |
PYTORCH_ATEN_EVIDENCE_ID = "pytorch-aten-build-id-pinned-cpu"
|
| 32 |
PYTORCH_PAIR_EVIDENCE_ID = "pytorch-bridge-aten-pair-pinned-cpu"
|
| 33 |
+
MAX_RUNTIME_CANDIDATES = 256
|
| 34 |
|
| 35 |
|
| 36 |
@dataclass(frozen=True)
|
|
|
|
| 182 |
return None
|
| 183 |
|
| 184 |
|
| 185 |
+
def _inspect_llama_descriptor(descriptor: int) -> RuntimeEvidence | None:
|
| 186 |
+
"""Recognise llama/GGML through one already-open, stable descriptor."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
try:
|
| 188 |
before = os.fstat(descriptor)
|
| 189 |
if not stat.S_ISREG(before.st_mode) or before.st_size < 64 or before.st_size > (1 << 40):
|
|
|
|
| 209 |
return None
|
| 210 |
except (JsonInputError, OSError, struct.error):
|
| 211 |
return None
|
|
|
|
|
|
|
| 212 |
|
| 213 |
|
| 214 |
+
def _inspect_pytorch_descriptor(descriptor: int) -> RuntimeEvidence | None:
|
| 215 |
+
"""Recognise one exact pinned PyTorch bridge/ATen descriptor identity."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
try:
|
| 217 |
before = os.fstat(descriptor)
|
| 218 |
if not stat.S_ISREG(before.st_mode) or before.st_size < 64:
|
|
|
|
| 236 |
return None
|
| 237 |
except (JsonInputError, OSError, struct.error):
|
| 238 |
return None
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _inspect_descriptor(descriptor: int) -> tuple[RuntimeEvidence, ...]:
|
| 242 |
+
values: list[RuntimeEvidence] = []
|
| 243 |
+
llama = _inspect_llama_descriptor(descriptor)
|
| 244 |
+
if llama is not None:
|
| 245 |
+
values.append(llama)
|
| 246 |
+
pytorch = _inspect_pytorch_descriptor(descriptor)
|
| 247 |
+
if pytorch is not None and pytorch not in values:
|
| 248 |
+
values.append(pytorch)
|
| 249 |
+
return tuple(values)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def inspect_path(path: Path) -> RuntimeEvidence | None:
|
| 253 |
+
"""Recognise a llama/GGML ELF by two valid symbol-table markers, not name."""
|
| 254 |
+
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
| 255 |
+
try:
|
| 256 |
+
descriptor = os.open(path, flags)
|
| 257 |
+
except OSError:
|
| 258 |
+
return None
|
| 259 |
+
try:
|
| 260 |
+
return _inspect_llama_descriptor(descriptor)
|
| 261 |
+
finally:
|
| 262 |
+
os.close(descriptor)
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def inspect_pytorch_path(path: Path) -> RuntimeEvidence | None:
|
| 266 |
+
"""Recognise one exact pinned PyTorch bridge/ATen ELF identity."""
|
| 267 |
+
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
| 268 |
+
try:
|
| 269 |
+
descriptor = os.open(path, flags)
|
| 270 |
+
except OSError:
|
| 271 |
+
return None
|
| 272 |
+
try:
|
| 273 |
+
return _inspect_pytorch_descriptor(descriptor)
|
| 274 |
finally:
|
| 275 |
os.close(descriptor)
|
| 276 |
|
| 277 |
|
| 278 |
def _inspect_candidate(path: Path) -> tuple[RuntimeEvidence, ...]:
|
| 279 |
values: list[RuntimeEvidence] = []
|
| 280 |
+
llama = inspect_path(path)
|
| 281 |
+
if llama is not None:
|
| 282 |
+
values.append(llama)
|
| 283 |
pytorch = inspect_pytorch_path(path)
|
| 284 |
if pytorch is not None and pytorch not in values:
|
| 285 |
values.append(pytorch)
|
|
|
|
| 299 |
return tuple(result)
|
| 300 |
|
| 301 |
|
| 302 |
+
def _cached_descriptor(
|
| 303 |
+
descriptor: int, cache: RuntimeCache | None
|
| 304 |
+
) -> tuple[RuntimeEvidence, ...]:
|
| 305 |
+
try:
|
| 306 |
+
metadata = os.fstat(descriptor)
|
| 307 |
+
key = _cache_key(metadata)
|
| 308 |
+
except OSError:
|
| 309 |
+
return ()
|
| 310 |
+
if cache is not None and key in cache:
|
| 311 |
+
return cache[key]
|
| 312 |
+
values = _inspect_descriptor(descriptor)
|
| 313 |
+
if cache is not None:
|
| 314 |
+
cache[key] = values
|
| 315 |
+
return values
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def _mapping_references(snapshot: object, proc: Path) -> tuple[str, ...]:
|
| 319 |
+
try:
|
| 320 |
+
pid = snapshot.identity.pid
|
| 321 |
+
directory = proc / str(pid) / "map_files"
|
| 322 |
+
values = [
|
| 323 |
+
item.name
|
| 324 |
+
for item in directory.iterdir()
|
| 325 |
+
if len(item.name.split("-", 1)) == 2
|
| 326 |
+
and all(
|
| 327 |
+
part and all(character in "0123456789abcdefABCDEF" for character in part)
|
| 328 |
+
for part in item.name.split("-", 1)
|
| 329 |
+
)
|
| 330 |
+
]
|
| 331 |
+
except (AttributeError, OSError):
|
| 332 |
+
return ()
|
| 333 |
+
return tuple(sorted(values, key=lambda value: int(value.split("-", 1)[0], 16)))
|
| 334 |
+
|
| 335 |
+
|
| 336 |
def from_snapshot(
|
| 337 |
+
snapshot: object,
|
| 338 |
+
*,
|
| 339 |
+
cache: RuntimeCache | None = None,
|
| 340 |
+
proc: Path = Path("/proc"),
|
| 341 |
+
start_index: int = 0,
|
| 342 |
+
max_candidates: int = MAX_RUNTIME_CANDIDATES,
|
| 343 |
) -> tuple[RuntimeEvidence, ...]:
|
| 344 |
+
"""Inspect one fair window of live executable/mapping descriptors.
|
| 345 |
+
|
| 346 |
+
Native Linux inspection opens ``/proc/<pid>/map_files/<range>`` directly.
|
| 347 |
+
The mapped inode therefore remains inspectable after its pathname is
|
| 348 |
+
renamed or deleted. Path reopening is only a compatibility fallback for
|
| 349 |
+
fixtures or kernels where ``map_files`` is unavailable.
|
| 350 |
+
"""
|
| 351 |
+
if isinstance(start_index, bool) or not isinstance(start_index, int) or start_index < 0:
|
| 352 |
+
raise JsonInputError("runtime mapping cursor is invalid")
|
| 353 |
+
if isinstance(max_candidates, bool) or not isinstance(max_candidates, int) or max_candidates < 1:
|
| 354 |
+
raise JsonInputError("runtime candidate limit is invalid")
|
| 355 |
+
|
| 356 |
result: list[RuntimeEvidence] = []
|
| 357 |
+
|
| 358 |
+
def add(values: tuple[RuntimeEvidence, ...]) -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
for evidence in values:
|
| 360 |
if evidence not in result:
|
| 361 |
result.append(evidence)
|
| 362 |
+
|
| 363 |
+
# Bind the executable through procfs as well; a deleted executable path is
|
| 364 |
+
# not an excuse to lose an otherwise exact runtime identity.
|
| 365 |
+
try:
|
| 366 |
+
executable = proc / str(snapshot.identity.pid) / "exe"
|
| 367 |
+
descriptor = os.open(executable, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
|
| 368 |
+
except (AttributeError, OSError):
|
| 369 |
+
descriptor = -1
|
| 370 |
+
if descriptor >= 0:
|
| 371 |
+
try:
|
| 372 |
+
add(_cached_descriptor(descriptor, cache))
|
| 373 |
+
finally:
|
| 374 |
+
os.close(descriptor)
|
| 375 |
+
|
| 376 |
+
references = _mapping_references(snapshot, proc)
|
| 377 |
+
if references:
|
| 378 |
+
offset = start_index % len(references)
|
| 379 |
+
count = min(max_candidates, len(references))
|
| 380 |
+
selected = tuple(
|
| 381 |
+
references[(offset + index) % len(references)] for index in range(count)
|
| 382 |
+
)
|
| 383 |
+
for reference in selected:
|
| 384 |
+
try:
|
| 385 |
+
descriptor = os.open(
|
| 386 |
+
proc / str(snapshot.identity.pid) / "map_files" / reference,
|
| 387 |
+
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0),
|
| 388 |
+
)
|
| 389 |
+
except (AttributeError, OSError):
|
| 390 |
+
continue
|
| 391 |
+
try:
|
| 392 |
+
add(_cached_descriptor(descriptor, cache))
|
| 393 |
+
finally:
|
| 394 |
+
os.close(descriptor)
|
| 395 |
+
else:
|
| 396 |
+
candidates: list[str] = []
|
| 397 |
+
seen: set[str] = set()
|
| 398 |
+
for raw in (getattr(snapshot, "exe_path", ""), *getattr(snapshot, "map_paths", ())):
|
| 399 |
+
if isinstance(raw, str) and raw.startswith("/") and raw not in seen:
|
| 400 |
+
seen.add(raw)
|
| 401 |
+
candidates.append(raw)
|
| 402 |
+
if candidates:
|
| 403 |
+
offset = start_index % len(candidates)
|
| 404 |
+
count = min(max_candidates, len(candidates))
|
| 405 |
+
for index in range(count):
|
| 406 |
+
path = Path(candidates[(offset + index) % len(candidates)])
|
| 407 |
+
if cache is None:
|
| 408 |
+
values = _inspect_candidate(path)
|
| 409 |
+
else:
|
| 410 |
+
try:
|
| 411 |
+
key = _cache_key(path.stat())
|
| 412 |
+
except OSError:
|
| 413 |
+
continue
|
| 414 |
+
if key in cache:
|
| 415 |
+
values = cache[key]
|
| 416 |
+
else:
|
| 417 |
+
values = _inspect_candidate(path)
|
| 418 |
+
cache[key] = values
|
| 419 |
+
add(values)
|
| 420 |
return with_pytorch_pair(result)
|
src/lumi_eggcracker/observation.py
CHANGED
|
@@ -39,6 +39,16 @@ class ObservationStore:
|
|
| 39 |
self._values.pop(oldest, None)
|
| 40 |
return value
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
def expire(self, *, now_ns: int | None = None) -> None:
|
| 43 |
now = time.monotonic_ns() if now_ns is None else now_ns
|
| 44 |
for key, value in tuple(self._values.items()):
|
|
|
|
| 39 |
self._values.pop(oldest, None)
|
| 40 |
return value
|
| 41 |
|
| 42 |
+
def get(
|
| 43 |
+
self, identity: Hashable, *, now_ns: int | None = None
|
| 44 |
+
) -> Observation | None:
|
| 45 |
+
self.expire(now_ns=now_ns)
|
| 46 |
+
return self._values.get(identity)
|
| 47 |
+
|
| 48 |
+
def identities(self, *, now_ns: int | None = None) -> frozenset[Hashable]:
|
| 49 |
+
self.expire(now_ns=now_ns)
|
| 50 |
+
return frozenset(self._values)
|
| 51 |
+
|
| 52 |
def expire(self, *, now_ns: int | None = None) -> None:
|
| 53 |
now = time.monotonic_ns() if now_ns is None else now_ns
|
| 54 |
for key, value in tuple(self._values.items()):
|
src/lumi_eggcracker/supervisor.py
CHANGED
|
@@ -5,7 +5,6 @@ from __future__ import annotations
|
|
| 5 |
import argparse
|
| 6 |
import datetime as dt
|
| 7 |
import errno
|
| 8 |
-
import hashlib
|
| 9 |
import json
|
| 10 |
import os
|
| 11 |
import re
|
|
@@ -27,7 +26,7 @@ from .approvals import approved, revoke
|
|
| 27 |
from .approvals import create as create_approval
|
| 28 |
from .approvals import load_all as load_approvals
|
| 29 |
from .approvals import public as public_approval
|
| 30 |
-
from .artifacts import ArtifactEvidence
|
| 31 |
from .artifacts import from_snapshot as artifacts_from_snapshot
|
| 32 |
from .containment import (
|
| 33 |
capture_identity,
|
|
@@ -40,11 +39,12 @@ from .detectors import Catalogue, DetectionMatch, load_catalogue, match, public_
|
|
| 40 |
from .discovery import (
|
| 41 |
ProcessIdentity,
|
| 42 |
ProcessSnapshot,
|
|
|
|
| 43 |
executable_digest_for_identity,
|
| 44 |
executable_metadata_for_identity,
|
| 45 |
scan,
|
| 46 |
)
|
| 47 |
-
from .elfmarkers import RuntimeEvidence, with_pytorch_pair
|
| 48 |
from .elfmarkers import from_snapshot as runtime_from_snapshot
|
| 49 |
from .jsonio import JsonInputError, load_regular_json
|
| 50 |
from .observation import ObservationStore
|
|
@@ -217,6 +217,10 @@ class Supervisor:
|
|
| 217 |
] = {}
|
| 218 |
self.enforcement_slots = threading.BoundedSemaphore(MAX_ENFORCEMENT_TASKS)
|
| 219 |
self.observations = ObservationStore()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
self.content_scan_tick = 0
|
| 221 |
self.last_scan_completed_ns = 0
|
| 222 |
self.last_scan_duration_ns = 0
|
|
@@ -423,7 +427,11 @@ class Supervisor:
|
|
| 423 |
candidates: list[_EvidenceCandidate],
|
| 424 |
snapshots: dict[ProcessIdentity, ProcessSnapshot],
|
| 425 |
) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
|
| 426 |
-
"""Union
|
|
|
|
|
|
|
|
|
|
|
|
|
| 427 |
if not candidates:
|
| 428 |
return []
|
| 429 |
parent = list(range(len(candidates)))
|
|
@@ -452,8 +460,6 @@ class Supervisor:
|
|
| 452 |
groups.setdefault(find(index), []).append(index)
|
| 453 |
result: list[tuple[tuple[_EvidenceCandidate, ...], str]] = []
|
| 454 |
for indexes in groups.values():
|
| 455 |
-
if len(indexes) > MAX_CORRELATED_PROCESSES:
|
| 456 |
-
continue
|
| 457 |
values = tuple(candidates[index] for index in indexes)
|
| 458 |
boundary_types = {
|
| 459 |
value
|
|
@@ -464,6 +470,97 @@ class Supervisor:
|
|
| 464 |
result.append((values, boundary))
|
| 465 |
return result
|
| 466 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 467 |
def _cached_executable_digest(self, snapshot: ProcessSnapshot) -> str:
|
| 468 |
metadata = executable_metadata_for_identity(snapshot.identity)
|
| 469 |
key = (
|
|
@@ -571,7 +668,8 @@ class Supervisor:
|
|
| 571 |
"executable": executable_record,
|
| 572 |
"observed": {
|
| 573 |
"argv_count": len(snapshot.argv),
|
| 574 |
-
"argv_sha256":
|
|
|
|
| 575 |
"pid": snapshot.identity.pid,
|
| 576 |
"start_time": snapshot.identity.start_time,
|
| 577 |
"uid": snapshot.uid,
|
|
@@ -739,9 +837,20 @@ class Supervisor:
|
|
| 739 |
except JsonInputError:
|
| 740 |
approvals = [] # A corrupt approval never authorizes a matching workload.
|
| 741 |
snapshots = scan(
|
| 742 |
-
exclude=
|
|
|
|
| 743 |
)
|
| 744 |
snapshot_map = {item.identity: item for item in snapshots}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 745 |
candidates: list[_EvidenceCandidate] = []
|
| 746 |
for snapshot in snapshots:
|
| 747 |
fast_match = match(self.catalogue, snapshot)
|
|
@@ -749,33 +858,82 @@ class Supervisor:
|
|
| 749 |
runtimes: tuple[RuntimeEvidence, ...] = ()
|
| 750 |
first_seen_ns = time.monotonic_ns()
|
| 751 |
if content_due:
|
| 752 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 753 |
# Runtime evidence is intentionally collected independently so
|
| 754 |
# it can be correlated with content held by a bounded peer.
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 759 |
}
|
| 760 |
-
if
|
| 761 |
observation = self.observations.observe(
|
| 762 |
-
snapshot.identity,
|
| 763 |
)
|
| 764 |
first_seen_ns = observation.first_seen_ns
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 765 |
if fast_match is None and not content and not runtimes:
|
| 766 |
continue
|
| 767 |
candidates.append(
|
| 768 |
_EvidenceCandidate(snapshot, content, runtimes, first_seen_ns, fast_match)
|
| 769 |
)
|
| 770 |
|
| 771 |
-
content_groups = self.
|
| 772 |
-
[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 773 |
)
|
| 774 |
-
groups =
|
| 775 |
((item,), "same-process")
|
| 776 |
for item in candidates
|
| 777 |
if item.fast_match is not None
|
| 778 |
-
]
|
| 779 |
for group, boundary_type in groups:
|
| 780 |
trigger_candidate = group[0]
|
| 781 |
aggregate_content: list[ArtifactEvidence] = []
|
|
@@ -1313,7 +1471,20 @@ class Supervisor:
|
|
| 1313 |
and "pids"
|
| 1314 |
in Path("/sys/fs/cgroup/cgroup.controllers").read_text(encoding="ascii").split()
|
| 1315 |
)
|
| 1316 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1317 |
return {
|
| 1318 |
"autonomous_discovery": ready,
|
| 1319 |
"backend": "root-supervisor",
|
|
@@ -1323,6 +1494,7 @@ class Supervisor:
|
|
| 1323 |
"consecutive_failures": self.discovery_failures,
|
| 1324 |
"last_scan_duration_ms": self.last_scan_duration_ns / 1_000_000,
|
| 1325 |
"last_scan_completed": self.last_scan_completed_ns > 0,
|
|
|
|
| 1326 |
"scan_health_timeout_ms": SCAN_HEALTH_TIMEOUT_NS / 1_000_000,
|
| 1327 |
},
|
| 1328 |
"pidfd": pidfd_available(),
|
|
|
|
| 5 |
import argparse
|
| 6 |
import datetime as dt
|
| 7 |
import errno
|
|
|
|
| 8 |
import json
|
| 9 |
import os
|
| 10 |
import re
|
|
|
|
| 26 |
from .approvals import create as create_approval
|
| 27 |
from .approvals import load_all as load_approvals
|
| 28 |
from .approvals import public as public_approval
|
| 29 |
+
from .artifacts import MAX_FD_PROBES_PER_SCAN, ArtifactEvidence
|
| 30 |
from .artifacts import from_snapshot as artifacts_from_snapshot
|
| 31 |
from .containment import (
|
| 32 |
capture_identity,
|
|
|
|
| 39 |
from .discovery import (
|
| 40 |
ProcessIdentity,
|
| 41 |
ProcessSnapshot,
|
| 42 |
+
argv_digest,
|
| 43 |
executable_digest_for_identity,
|
| 44 |
executable_metadata_for_identity,
|
| 45 |
scan,
|
| 46 |
)
|
| 47 |
+
from .elfmarkers import MAX_RUNTIME_CANDIDATES, RuntimeEvidence, with_pytorch_pair
|
| 48 |
from .elfmarkers import from_snapshot as runtime_from_snapshot
|
| 49 |
from .jsonio import JsonInputError, load_regular_json
|
| 50 |
from .observation import ObservationStore
|
|
|
|
| 217 |
] = {}
|
| 218 |
self.enforcement_slots = threading.BoundedSemaphore(MAX_ENFORCEMENT_TASKS)
|
| 219 |
self.observations = ObservationStore()
|
| 220 |
+
self.artifact_fd_offsets: dict[ProcessIdentity, int] = {}
|
| 221 |
+
self.runtime_map_offsets: dict[ProcessIdentity, int] = {}
|
| 222 |
+
self.observed_content: dict[ProcessIdentity, dict[str, ArtifactEvidence]] = {}
|
| 223 |
+
self.observed_runtimes: dict[ProcessIdentity, dict[str, RuntimeEvidence]] = {}
|
| 224 |
self.content_scan_tick = 0
|
| 225 |
self.last_scan_completed_ns = 0
|
| 226 |
self.last_scan_duration_ns = 0
|
|
|
|
| 427 |
candidates: list[_EvidenceCandidate],
|
| 428 |
snapshots: dict[ProcessIdentity, ProcessSnapshot],
|
| 429 |
) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
|
| 430 |
+
"""Union evidence sets joined by an exact live relation.
|
| 431 |
+
|
| 432 |
+
Component size is never interpreted as absence. Callers reduce a
|
| 433 |
+
component to a minimal complete evidence set before enforcement.
|
| 434 |
+
"""
|
| 435 |
if not candidates:
|
| 436 |
return []
|
| 437 |
parent = list(range(len(candidates)))
|
|
|
|
| 460 |
groups.setdefault(find(index), []).append(index)
|
| 461 |
result: list[tuple[tuple[_EvidenceCandidate, ...], str]] = []
|
| 462 |
for indexes in groups.values():
|
|
|
|
|
|
|
| 463 |
values = tuple(candidates[index] for index in indexes)
|
| 464 |
boundary_types = {
|
| 465 |
value
|
|
|
|
| 470 |
result.append((values, boundary))
|
| 471 |
return result
|
| 472 |
|
| 473 |
+
def _minimal_content_groups(
|
| 474 |
+
self,
|
| 475 |
+
candidates: list[_EvidenceCandidate],
|
| 476 |
+
snapshots: dict[ProcessIdentity, ProcessSnapshot],
|
| 477 |
+
) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
|
| 478 |
+
"""Return one minimal complete evidence set per related component.
|
| 479 |
+
|
| 480 |
+
Repeated partial evidence does not revoke an independently complete
|
| 481 |
+
approval, and a component larger than the historical containment cap
|
| 482 |
+
cannot suppress a complete profile. Only evidence identities that
|
| 483 |
+
contribute a new predicate are selected.
|
| 484 |
+
"""
|
| 485 |
+
result: list[tuple[tuple[_EvidenceCandidate, ...], str]] = []
|
| 486 |
+
for component, boundary in self._correlate(candidates, snapshots):
|
| 487 |
+
selected: list[_EvidenceCandidate] = []
|
| 488 |
+
seen_content: set[str] = set()
|
| 489 |
+
seen_runtime: set[str] = set()
|
| 490 |
+
detected: DetectionMatch | None = None
|
| 491 |
+
for candidate in sorted(component, key=lambda item: item.snapshot.identity):
|
| 492 |
+
content_ids = {item.evidence_id for item in candidate.content}
|
| 493 |
+
runtime_ids = {item.evidence_id for item in candidate.runtimes}
|
| 494 |
+
if content_ids <= seen_content and runtime_ids <= seen_runtime:
|
| 495 |
+
continue
|
| 496 |
+
selected.append(candidate)
|
| 497 |
+
seen_content.update(content_ids)
|
| 498 |
+
seen_runtime.update(runtime_ids)
|
| 499 |
+
paired = with_pytorch_pair(
|
| 500 |
+
item for value in selected for item in value.runtimes
|
| 501 |
+
)
|
| 502 |
+
detected = match(
|
| 503 |
+
self.catalogue,
|
| 504 |
+
selected[0].snapshot,
|
| 505 |
+
evidence={
|
| 506 |
+
"MODEL_CONTENT": {
|
| 507 |
+
item.evidence_id for value in selected for item in value.content
|
| 508 |
+
},
|
| 509 |
+
"MODEL_RUNTIME": {item.evidence_id for item in paired},
|
| 510 |
+
},
|
| 511 |
+
)
|
| 512 |
+
if detected is not None:
|
| 513 |
+
break
|
| 514 |
+
if len(selected) >= MAX_CORRELATED_PROCESSES:
|
| 515 |
+
# There are currently only two independent evidence groups
|
| 516 |
+
# and at most three raw identities are required for the
|
| 517 |
+
# pinned profiles. Reaching this guard means no bounded
|
| 518 |
+
# minimal set was established; do not reinterpret partial
|
| 519 |
+
# evidence as a destructive match.
|
| 520 |
+
break
|
| 521 |
+
if detected is not None:
|
| 522 |
+
result.append((tuple(selected), boundary))
|
| 523 |
+
return result
|
| 524 |
+
|
| 525 |
+
def _content_groups(
|
| 526 |
+
self,
|
| 527 |
+
candidates: list[_EvidenceCandidate],
|
| 528 |
+
snapshots: dict[ProcessIdentity, ProcessSnapshot],
|
| 529 |
+
) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
|
| 530 |
+
"""Prefer exact same-process matches before any relation aggregate."""
|
| 531 |
+
complete: list[_EvidenceCandidate] = []
|
| 532 |
+
partial: list[_EvidenceCandidate] = []
|
| 533 |
+
for item in candidates:
|
| 534 |
+
own_runtime = with_pytorch_pair(item.runtimes)
|
| 535 |
+
own_match = match(
|
| 536 |
+
self.catalogue,
|
| 537 |
+
item.snapshot,
|
| 538 |
+
evidence={
|
| 539 |
+
"MODEL_CONTENT": {value.evidence_id for value in item.content},
|
| 540 |
+
"MODEL_RUNTIME": {value.evidence_id for value in own_runtime},
|
| 541 |
+
},
|
| 542 |
+
)
|
| 543 |
+
if own_match is None:
|
| 544 |
+
partial.append(item)
|
| 545 |
+
else:
|
| 546 |
+
complete.append(
|
| 547 |
+
_EvidenceCandidate(
|
| 548 |
+
item.snapshot,
|
| 549 |
+
item.content,
|
| 550 |
+
tuple(own_runtime),
|
| 551 |
+
item.first_seen_ns,
|
| 552 |
+
own_match,
|
| 553 |
+
)
|
| 554 |
+
)
|
| 555 |
+
return [((item,), "same-process") for item in complete] + self._minimal_content_groups(
|
| 556 |
+
partial, snapshots
|
| 557 |
+
)
|
| 558 |
+
|
| 559 |
+
@staticmethod
|
| 560 |
+
def _discovery_excluded(snapshot: ProcessSnapshot) -> bool:
|
| 561 |
+
"""Exclude only the supervisor itself; a run record is not approval."""
|
| 562 |
+
return snapshot.identity.pid == os.getpid()
|
| 563 |
+
|
| 564 |
def _cached_executable_digest(self, snapshot: ProcessSnapshot) -> str:
|
| 565 |
metadata = executable_metadata_for_identity(snapshot.identity)
|
| 566 |
key = (
|
|
|
|
| 668 |
"executable": executable_record,
|
| 669 |
"observed": {
|
| 670 |
"argv_count": len(snapshot.argv),
|
| 671 |
+
"argv_sha256": argv_digest(snapshot.argv),
|
| 672 |
+
"argv_complete": snapshot.argv_complete,
|
| 673 |
"pid": snapshot.identity.pid,
|
| 674 |
"start_time": snapshot.identity.start_time,
|
| 675 |
"uid": snapshot.uid,
|
|
|
|
| 837 |
except JsonInputError:
|
| 838 |
approvals = [] # A corrupt approval never authorizes a matching workload.
|
| 839 |
snapshots = scan(
|
| 840 |
+
exclude=self._discovery_excluded,
|
| 841 |
+
include_evidence=content_due,
|
| 842 |
)
|
| 843 |
snapshot_map = {item.identity: item for item in snapshots}
|
| 844 |
+
live_identities = set(snapshot_map)
|
| 845 |
+
for cache in (
|
| 846 |
+
getattr(self, "artifact_fd_offsets", {}),
|
| 847 |
+
getattr(self, "runtime_map_offsets", {}),
|
| 848 |
+
getattr(self, "observed_content", {}),
|
| 849 |
+
getattr(self, "observed_runtimes", {}),
|
| 850 |
+
):
|
| 851 |
+
for identity in tuple(cache):
|
| 852 |
+
if identity not in live_identities:
|
| 853 |
+
cache.pop(identity, None)
|
| 854 |
candidates: list[_EvidenceCandidate] = []
|
| 855 |
for snapshot in snapshots:
|
| 856 |
fast_match = match(self.catalogue, snapshot)
|
|
|
|
| 858 |
runtimes: tuple[RuntimeEvidence, ...] = ()
|
| 859 |
first_seen_ns = time.monotonic_ns()
|
| 860 |
if content_due:
|
| 861 |
+
artifact_offsets = getattr(self, "artifact_fd_offsets", None)
|
| 862 |
+
if artifact_offsets is None:
|
| 863 |
+
self.artifact_fd_offsets = {}
|
| 864 |
+
artifact_offsets = self.artifact_fd_offsets
|
| 865 |
+
runtime_offsets = getattr(self, "runtime_map_offsets", None)
|
| 866 |
+
if runtime_offsets is None:
|
| 867 |
+
self.runtime_map_offsets = {}
|
| 868 |
+
runtime_offsets = self.runtime_map_offsets
|
| 869 |
+
content_now = artifacts_from_snapshot(
|
| 870 |
+
snapshot,
|
| 871 |
+
cache=self.artifact_cache,
|
| 872 |
+
fd_start_index=artifact_offsets.get(snapshot.identity, 0),
|
| 873 |
+
)
|
| 874 |
+
artifact_offsets[snapshot.identity] = (
|
| 875 |
+
artifact_offsets.get(snapshot.identity, 0) + MAX_FD_PROBES_PER_SCAN
|
| 876 |
+
)
|
| 877 |
# Runtime evidence is intentionally collected independently so
|
| 878 |
# it can be correlated with content held by a bounded peer.
|
| 879 |
+
runtimes_now = runtime_from_snapshot(
|
| 880 |
+
snapshot,
|
| 881 |
+
cache=self.runtime_cache,
|
| 882 |
+
start_index=runtime_offsets.get(snapshot.identity, 0),
|
| 883 |
+
)
|
| 884 |
+
runtime_offsets[snapshot.identity] = (
|
| 885 |
+
runtime_offsets.get(snapshot.identity, 0) + MAX_RUNTIME_CANDIDATES
|
| 886 |
+
)
|
| 887 |
+
observed_content = getattr(self, "observed_content", None)
|
| 888 |
+
if observed_content is None:
|
| 889 |
+
self.observed_content = {}
|
| 890 |
+
observed_content = self.observed_content
|
| 891 |
+
observed_runtimes = getattr(self, "observed_runtimes", None)
|
| 892 |
+
if observed_runtimes is None:
|
| 893 |
+
self.observed_runtimes = {}
|
| 894 |
+
observed_runtimes = self.observed_runtimes
|
| 895 |
+
content_values = observed_content.setdefault(snapshot.identity, {})
|
| 896 |
+
runtime_values = observed_runtimes.setdefault(snapshot.identity, {})
|
| 897 |
+
content_values.update({item.evidence_id: item for item in content_now})
|
| 898 |
+
runtime_values.update({item.evidence_id: item for item in runtimes_now})
|
| 899 |
+
current_ids = {
|
| 900 |
+
*(item.evidence_id for item in content_now),
|
| 901 |
+
*(item.evidence_id for item in runtimes_now),
|
| 902 |
}
|
| 903 |
+
if current_ids:
|
| 904 |
observation = self.observations.observe(
|
| 905 |
+
snapshot.identity, current_ids
|
| 906 |
)
|
| 907 |
first_seen_ns = observation.first_seen_ns
|
| 908 |
+
else:
|
| 909 |
+
observation = self.observations.get(snapshot.identity)
|
| 910 |
+
if observation is not None:
|
| 911 |
+
active = observation.evidence
|
| 912 |
+
content = tuple(
|
| 913 |
+
item for key, item in content_values.items() if key in active
|
| 914 |
+
)
|
| 915 |
+
runtimes = with_pytorch_pair(
|
| 916 |
+
item for key, item in runtime_values.items() if key in active
|
| 917 |
+
)
|
| 918 |
if fast_match is None and not content and not runtimes:
|
| 919 |
continue
|
| 920 |
candidates.append(
|
| 921 |
_EvidenceCandidate(snapshot, content, runtimes, first_seen_ns, fast_match)
|
| 922 |
)
|
| 923 |
|
| 924 |
+
content_groups = self._content_groups(
|
| 925 |
+
[
|
| 926 |
+
item
|
| 927 |
+
for item in candidates
|
| 928 |
+
if (item.content or item.runtimes) and item.fast_match is None
|
| 929 |
+
],
|
| 930 |
+
snapshot_map,
|
| 931 |
)
|
| 932 |
+
groups = [
|
| 933 |
((item,), "same-process")
|
| 934 |
for item in candidates
|
| 935 |
if item.fast_match is not None
|
| 936 |
+
] + content_groups
|
| 937 |
for group, boundary_type in groups:
|
| 938 |
trigger_candidate = group[0]
|
| 939 |
aggregate_content: list[ArtifactEvidence] = []
|
|
|
|
| 1471 |
and "pids"
|
| 1472 |
in Path("/sys/fs/cgroup/cgroup.controllers").read_text(encoding="ascii").split()
|
| 1473 |
)
|
| 1474 |
+
now_ns = time.monotonic_ns()
|
| 1475 |
+
scan_healthy = (
|
| 1476 |
+
self.last_scan_completed_ns > 0
|
| 1477 |
+
and now_ns - self.last_scan_completed_ns <= SCAN_HEALTH_TIMEOUT_NS
|
| 1478 |
+
and self.discovery_failures < MAX_DISCOVERY_FAILURES
|
| 1479 |
+
and self.discovery_thread is not None
|
| 1480 |
+
and self.discovery_thread.is_alive()
|
| 1481 |
+
)
|
| 1482 |
+
ready = (
|
| 1483 |
+
available
|
| 1484 |
+
and pidfd_available()
|
| 1485 |
+
and self.quarantine_root is not None
|
| 1486 |
+
and scan_healthy
|
| 1487 |
+
)
|
| 1488 |
return {
|
| 1489 |
"autonomous_discovery": ready,
|
| 1490 |
"backend": "root-supervisor",
|
|
|
|
| 1494 |
"consecutive_failures": self.discovery_failures,
|
| 1495 |
"last_scan_duration_ms": self.last_scan_duration_ns / 1_000_000,
|
| 1496 |
"last_scan_completed": self.last_scan_completed_ns > 0,
|
| 1497 |
+
"healthy": scan_healthy,
|
| 1498 |
"scan_health_timeout_ms": SCAN_HEALTH_TIMEOUT_NS / 1_000_000,
|
| 1499 |
},
|
| 1500 |
"pidfd": pidfd_available(),
|
src/lumi_eggcracker/watchdog.py
CHANGED
|
@@ -90,12 +90,45 @@ def _kill_target(path: Path) -> dict[str, object]:
|
|
| 90 |
_write(path, "cgroup.kill", b"1\n")
|
| 91 |
deadline = time.monotonic() + 2.0
|
| 92 |
while time.monotonic() < deadline:
|
| 93 |
-
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
time.sleep(0.005)
|
| 96 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
return {
|
| 98 |
"cgroup": str(path),
|
|
|
|
| 99 |
"frozen": frozen,
|
| 100 |
"populated": int(value.get("populated", "1")),
|
| 101 |
"started_ns": before,
|
|
|
|
| 90 |
_write(path, "cgroup.kill", b"1\n")
|
| 91 |
deadline = time.monotonic() + 2.0
|
| 92 |
while time.monotonic() < deadline:
|
| 93 |
+
try:
|
| 94 |
+
empty = _events(path).get("populated") == "0"
|
| 95 |
+
except JsonInputError:
|
| 96 |
+
# systemd may collect an empty transient cgroup immediately after
|
| 97 |
+
# the authoritative kill write. Collection after that write is
|
| 98 |
+
# equivalent to an empty proof, not a containment failure.
|
| 99 |
+
if not path.exists():
|
| 100 |
+
return {
|
| 101 |
+
"cgroup": str(path),
|
| 102 |
+
"collected": True,
|
| 103 |
+
"frozen": frozen,
|
| 104 |
+
"populated": 0,
|
| 105 |
+
"started_ns": before,
|
| 106 |
+
}
|
| 107 |
+
raise
|
| 108 |
+
if empty:
|
| 109 |
+
return {
|
| 110 |
+
"cgroup": str(path),
|
| 111 |
+
"collected": False,
|
| 112 |
+
"frozen": frozen,
|
| 113 |
+
"populated": 0,
|
| 114 |
+
"started_ns": before,
|
| 115 |
+
}
|
| 116 |
time.sleep(0.005)
|
| 117 |
+
try:
|
| 118 |
+
value = _events(path)
|
| 119 |
+
except JsonInputError:
|
| 120 |
+
if not path.exists():
|
| 121 |
+
return {
|
| 122 |
+
"cgroup": str(path),
|
| 123 |
+
"collected": True,
|
| 124 |
+
"frozen": frozen,
|
| 125 |
+
"populated": 0,
|
| 126 |
+
"started_ns": before,
|
| 127 |
+
}
|
| 128 |
+
raise
|
| 129 |
return {
|
| 130 |
"cgroup": str(path),
|
| 131 |
+
"collected": False,
|
| 132 |
"frozen": frozen,
|
| 133 |
"populated": int(value.get("populated", "1")),
|
| 134 |
"started_ns": before,
|
tests/test_approvals.py
CHANGED
|
@@ -14,6 +14,7 @@ class Sample:
|
|
| 14 |
uid: int
|
| 15 |
exe_path: str
|
| 16 |
argv: tuple[str, ...]
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
class ApprovalTests(unittest.TestCase):
|
|
@@ -28,6 +29,13 @@ class ApprovalTests(unittest.TestCase):
|
|
| 28 |
self.assertTrue(approved(sample, executable_digest(binary), values))
|
| 29 |
self.assertFalse(approved(Sample(1002, str(binary), sample.argv), executable_digest(binary), values))
|
| 30 |
self.assertFalse(approved(Sample(1001, str(binary), (str(binary), "-m", "/models/other.gguf")), executable_digest(binary), values))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
self.assertNotIn("/models/qwen.gguf", record.values())
|
| 32 |
self.assertEqual(argv_digest(sample.argv), record["argv_sha256"])
|
| 33 |
|
|
|
|
| 14 |
uid: int
|
| 15 |
exe_path: str
|
| 16 |
argv: tuple[str, ...]
|
| 17 |
+
argv_complete: bool = True
|
| 18 |
|
| 19 |
|
| 20 |
class ApprovalTests(unittest.TestCase):
|
|
|
|
| 29 |
self.assertTrue(approved(sample, executable_digest(binary), values))
|
| 30 |
self.assertFalse(approved(Sample(1002, str(binary), sample.argv), executable_digest(binary), values))
|
| 31 |
self.assertFalse(approved(Sample(1001, str(binary), (str(binary), "-m", "/models/other.gguf")), executable_digest(binary), values))
|
| 32 |
+
self.assertFalse(
|
| 33 |
+
approved(
|
| 34 |
+
Sample(1001, str(binary), sample.argv, argv_complete=False),
|
| 35 |
+
executable_digest(binary),
|
| 36 |
+
values,
|
| 37 |
+
)
|
| 38 |
+
)
|
| 39 |
self.assertNotIn("/models/qwen.gguf", record.values())
|
| 40 |
self.assertEqual(argv_digest(sample.argv), record["argv_sha256"])
|
| 41 |
|
tests/test_artifacts.py
CHANGED
|
@@ -7,7 +7,8 @@ import tempfile
|
|
| 7 |
import unittest
|
| 8 |
from pathlib import Path
|
| 9 |
|
| 10 |
-
from lumi_eggcracker.artifacts import from_snapshot, validate_path
|
|
|
|
| 11 |
|
| 12 |
|
| 13 |
def gguf(*, version: int = 3, tensors: int = 1, metadata: int = 0, padding: int = 64) -> bytes:
|
|
@@ -26,6 +27,23 @@ def safetensors(*, dtype: str = "F32", shape: list[int] | None = None, offsets:
|
|
| 26 |
|
| 27 |
|
| 28 |
class ArtifactTests(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
def test_valid_gguf_content_does_not_depend_on_filename(self) -> None:
|
| 30 |
with tempfile.TemporaryDirectory() as raw:
|
| 31 |
path = Path(raw) / "opaque-input"
|
|
|
|
| 7 |
import unittest
|
| 8 |
from pathlib import Path
|
| 9 |
|
| 10 |
+
from lumi_eggcracker.artifacts import from_process_fds, from_snapshot, validate_path
|
| 11 |
+
from lumi_eggcracker.discovery import ProcessIdentity
|
| 12 |
|
| 13 |
|
| 14 |
def gguf(*, version: int = 3, tensors: int = 1, metadata: int = 0, padding: int = 64) -> bytes:
|
|
|
|
| 27 |
|
| 28 |
|
| 29 |
class ArtifactTests(unittest.TestCase):
|
| 30 |
+
def test_descriptor_windows_eventually_cover_late_model_fd(self) -> None:
|
| 31 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 32 |
+
proc = Path(raw)
|
| 33 |
+
directory = proc / "42" / "fd"
|
| 34 |
+
directory.mkdir(parents=True)
|
| 35 |
+
for number in range(20):
|
| 36 |
+
(directory / str(number)).write_bytes(b"harmless")
|
| 37 |
+
(directory / "20").write_bytes(gguf())
|
| 38 |
+
sample = type("Snapshot", (), {"identity": ProcessIdentity(42, 100), "fd_entries": ()})()
|
| 39 |
+
self.assertEqual(
|
| 40 |
+
(), from_process_fds(sample, proc=proc, start_index=0, max_probes=16)
|
| 41 |
+
)
|
| 42 |
+
evidence = from_process_fds(
|
| 43 |
+
sample, proc=proc, start_index=16, max_probes=16
|
| 44 |
+
)
|
| 45 |
+
self.assertEqual({"gguf-v3"}, {item.evidence_id for item in evidence})
|
| 46 |
+
|
| 47 |
def test_valid_gguf_content_does_not_depend_on_filename(self) -> None:
|
| 48 |
with tempfile.TemporaryDirectory() as raw:
|
| 49 |
path = Path(raw) / "opaque-input"
|
tests/test_discovery.py
CHANGED
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
| 3 |
import tempfile
|
| 4 |
import unittest
|
| 5 |
from pathlib import Path
|
|
|
|
| 6 |
|
| 7 |
from lumi_eggcracker.discovery import (
|
| 8 |
ProcessIdentity,
|
|
@@ -11,11 +12,25 @@ from lumi_eggcracker.discovery import (
|
|
| 11 |
executable_digest_for_identity,
|
| 12 |
identity,
|
| 13 |
parse_stat,
|
|
|
|
| 14 |
)
|
| 15 |
from lumi_eggcracker.jsonio import JsonInputError
|
| 16 |
|
| 17 |
|
| 18 |
class DiscoveryTests(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
def test_stat_parser_handles_spaces_in_command_name(self) -> None:
|
| 20 |
raw = "77 (worker with spaces) S 4 " + "0 " * 17 + "901\n"
|
| 21 |
self.assertEqual((4, 901), parse_stat(raw))
|
|
@@ -52,6 +67,40 @@ class DiscoveryTests(unittest.TestCase):
|
|
| 52 |
self.assertEqual(512, len(values))
|
| 53 |
self.assertTrue(values[0].startswith("/opt/runtime/"))
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
def test_process_executable_is_hashed_from_bound_proc_identity(self) -> None:
|
| 56 |
with tempfile.TemporaryDirectory() as raw:
|
| 57 |
proc = Path(raw)
|
|
|
|
| 3 |
import tempfile
|
| 4 |
import unittest
|
| 5 |
from pathlib import Path
|
| 6 |
+
from unittest.mock import patch
|
| 7 |
|
| 8 |
from lumi_eggcracker.discovery import (
|
| 9 |
ProcessIdentity,
|
|
|
|
| 12 |
executable_digest_for_identity,
|
| 13 |
identity,
|
| 14 |
parse_stat,
|
| 15 |
+
snapshot,
|
| 16 |
)
|
| 17 |
from lumi_eggcracker.jsonio import JsonInputError
|
| 18 |
|
| 19 |
|
| 20 |
class DiscoveryTests(unittest.TestCase):
|
| 21 |
+
def _proc_entry(self, proc: Path, command: bytes) -> ProcessIdentity:
|
| 22 |
+
entry = proc / "42"
|
| 23 |
+
entry.mkdir()
|
| 24 |
+
(entry / "stat").write_text(
|
| 25 |
+
"42 (runner) S 1 " + "0 " * 17 + "100\n", encoding="utf-8"
|
| 26 |
+
)
|
| 27 |
+
(entry / "cgroup").write_text("0::/user.slice/test\n", encoding="utf-8")
|
| 28 |
+
(entry / "status").write_text("Uid:\t1000\t1000\t1000\t1000\n", encoding="utf-8")
|
| 29 |
+
(entry / "cmdline").write_bytes(command)
|
| 30 |
+
(entry / "fd").mkdir()
|
| 31 |
+
(entry / "map_files").mkdir()
|
| 32 |
+
return ProcessIdentity(42, 100)
|
| 33 |
+
|
| 34 |
def test_stat_parser_handles_spaces_in_command_name(self) -> None:
|
| 35 |
raw = "77 (worker with spaces) S 4 " + "0 " * 17 + "901\n"
|
| 36 |
self.assertEqual((4, 901), parse_stat(raw))
|
|
|
|
| 67 |
self.assertEqual(512, len(values))
|
| 68 |
self.assertTrue(values[0].startswith("/opt/runtime/"))
|
| 69 |
|
| 70 |
+
def test_oversized_maps_file_is_localized_to_one_empty_fallback(self) -> None:
|
| 71 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 72 |
+
path = Path(raw) / "maps"
|
| 73 |
+
path.write_bytes(b"x" * 64)
|
| 74 |
+
with patch("lumi_eggcracker.discovery.MAX_MAP_BYTES", 16):
|
| 75 |
+
self.assertEqual((), _map_paths(path))
|
| 76 |
+
|
| 77 |
+
def test_invalid_utf8_argv_does_not_erase_process_snapshot(self) -> None:
|
| 78 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 79 |
+
proc = Path(raw)
|
| 80 |
+
value = self._proc_entry(proc, b"/usr/bin/python3\0bad\xffarg\0")
|
| 81 |
+
with patch(
|
| 82 |
+
"lumi_eggcracker.discovery.os.readlink", return_value="/usr/bin/python3"
|
| 83 |
+
):
|
| 84 |
+
current = snapshot(value, proc=proc)
|
| 85 |
+
self.assertIsNotNone(current)
|
| 86 |
+
assert current is not None
|
| 87 |
+
self.assertTrue(current.argv_complete)
|
| 88 |
+
self.assertEqual(2, len(current.argv))
|
| 89 |
+
self.assertEqual(64, len(argv_digest(current.argv)))
|
| 90 |
+
|
| 91 |
+
def test_oversized_argv_is_unapprovable_not_invisible(self) -> None:
|
| 92 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 93 |
+
proc = Path(raw)
|
| 94 |
+
value = self._proc_entry(proc, b"/usr/bin/python3\0" + b"x" * 64)
|
| 95 |
+
with patch("lumi_eggcracker.discovery.MAX_CMDLINE_BYTES", 16), patch(
|
| 96 |
+
"lumi_eggcracker.discovery.os.readlink", return_value="/usr/bin/python3"
|
| 97 |
+
):
|
| 98 |
+
current = snapshot(value, proc=proc)
|
| 99 |
+
self.assertIsNotNone(current)
|
| 100 |
+
assert current is not None
|
| 101 |
+
self.assertFalse(current.argv_complete)
|
| 102 |
+
self.assertEqual((), current.argv)
|
| 103 |
+
|
| 104 |
def test_process_executable_is_hashed_from_bound_proc_identity(self) -> None:
|
| 105 |
with tempfile.TemporaryDirectory() as raw:
|
| 106 |
proc = Path(raw)
|
tests/test_elfmarkers.py
CHANGED
|
@@ -7,6 +7,7 @@ import unittest
|
|
| 7 |
from pathlib import Path
|
| 8 |
from unittest.mock import patch
|
| 9 |
|
|
|
|
| 10 |
from lumi_eggcracker.elfmarkers import (
|
| 11 |
PINNED_LLAMA_BUILD_IDS,
|
| 12 |
PYTORCH_ATEN_EVIDENCE_ID,
|
|
@@ -44,7 +45,55 @@ def elf(*, markers: tuple[str, ...], append_decoys: bool = False) -> bytes:
|
|
| 44 |
return bytes(body)
|
| 45 |
|
| 46 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
class ElfMarkerTests(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
def test_two_valid_symbol_markers_qualify_regardless_of_filename(self) -> None:
|
| 49 |
with tempfile.TemporaryDirectory() as raw:
|
| 50 |
path = Path(raw) / "unrelated-name"
|
|
|
|
| 7 |
from pathlib import Path
|
| 8 |
from unittest.mock import patch
|
| 9 |
|
| 10 |
+
from lumi_eggcracker.discovery import ProcessIdentity
|
| 11 |
from lumi_eggcracker.elfmarkers import (
|
| 12 |
PINNED_LLAMA_BUILD_IDS,
|
| 13 |
PYTORCH_ATEN_EVIDENCE_ID,
|
|
|
|
| 45 |
return bytes(body)
|
| 46 |
|
| 47 |
|
| 48 |
+
def build_id_elf(identifier: bytes) -> bytes:
|
| 49 |
+
note = struct.pack("<III", 4, len(identifier), 3) + b"GNU\0" + identifier
|
| 50 |
+
note += b"\0" * ((4 - len(identifier) % 4) % 4)
|
| 51 |
+
body = bytearray(128 + len(note))
|
| 52 |
+
body[:16] = b"\x7fELF\x02\x01\x01" + b"\0" * 9
|
| 53 |
+
body[16:64] = struct.pack(
|
| 54 |
+
"<HHIQQQIHHHHHH", 3, 62, 1, 0, 64, 0, 0, 64, 56, 1, 0, 0, 0
|
| 55 |
+
)
|
| 56 |
+
body[64:120] = struct.pack("<IIQQQQQQ", 4, 0, 128, 0, 0, len(note), 0, 4)
|
| 57 |
+
body[128:] = note
|
| 58 |
+
return bytes(body)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
class ElfMarkerTests(unittest.TestCase):
|
| 62 |
+
def test_proc_mapping_descriptors_are_inspected_without_pathnames(self) -> None:
|
| 63 |
+
bridge_id = bytes.fromhex("44" * 20)
|
| 64 |
+
aten_id = bytes.fromhex("55" * 20)
|
| 65 |
+
with tempfile.TemporaryDirectory() as raw:
|
| 66 |
+
proc = Path(raw)
|
| 67 |
+
mappings = proc / "42" / "map_files"
|
| 68 |
+
mappings.mkdir(parents=True)
|
| 69 |
+
(mappings / "1000-2000").write_bytes(build_id_elf(bridge_id))
|
| 70 |
+
(mappings / "3000-4000").write_bytes(build_id_elf(aten_id))
|
| 71 |
+
sample = type(
|
| 72 |
+
"Snapshot",
|
| 73 |
+
(),
|
| 74 |
+
{
|
| 75 |
+
"identity": ProcessIdentity(42, 100),
|
| 76 |
+
"exe_path": "/deleted-runtime",
|
| 77 |
+
"map_paths": (),
|
| 78 |
+
},
|
| 79 |
+
)()
|
| 80 |
+
with patch(
|
| 81 |
+
"lumi_eggcracker.elfmarkers.PINNED_PYTORCH_BRIDGE_BUILD_IDS",
|
| 82 |
+
{bridge_id.hex()},
|
| 83 |
+
), patch(
|
| 84 |
+
"lumi_eggcracker.elfmarkers.PINNED_PYTORCH_ATEN_BUILD_IDS",
|
| 85 |
+
{aten_id.hex()},
|
| 86 |
+
):
|
| 87 |
+
evidence = from_snapshot(sample, proc=proc, max_candidates=2)
|
| 88 |
+
self.assertEqual(
|
| 89 |
+
{
|
| 90 |
+
PYTORCH_BRIDGE_EVIDENCE_ID,
|
| 91 |
+
PYTORCH_ATEN_EVIDENCE_ID,
|
| 92 |
+
PYTORCH_PAIR_EVIDENCE_ID,
|
| 93 |
+
},
|
| 94 |
+
{item.evidence_id for item in evidence},
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
def test_two_valid_symbol_markers_qualify_regardless_of_filename(self) -> None:
|
| 98 |
with tempfile.TemporaryDirectory() as raw:
|
| 99 |
path = Path(raw) / "unrelated-name"
|
tests/test_supervisor.py
CHANGED
|
@@ -67,6 +67,65 @@ class SupervisorTests(unittest.TestCase):
|
|
| 67 |
(),
|
| 68 |
)
|
| 69 |
self.assertTrue(supervisor._managed(snapshot))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
|
| 71 |
def test_heartbeat_stops_when_no_scan_completed_within_health_bound(self) -> None:
|
| 72 |
supervisor = self._instance()
|
|
|
|
| 67 |
(),
|
| 68 |
)
|
| 69 |
self.assertTrue(supervisor._managed(snapshot))
|
| 70 |
+
self.assertFalse(supervisor._discovery_excluded(snapshot))
|
| 71 |
+
|
| 72 |
+
def test_complete_identity_is_not_suppressed_or_broadened_by_64_partials(self) -> None:
|
| 73 |
+
supervisor = self._instance()
|
| 74 |
+
supervisor.catalogue = load_bundled()
|
| 75 |
+
parent = ProcessIdentity(10, 100)
|
| 76 |
+
parent_snapshot = ProcessSnapshot(
|
| 77 |
+
parent, 2001, "/usr/bin/worker", "worker", ("worker",), (), (), ()
|
| 78 |
+
)
|
| 79 |
+
content = ArtifactEvidence(
|
| 80 |
+
"safetensors-v1", "SAFETENSORS", 1, 2, 4096, "a" * 64
|
| 81 |
+
)
|
| 82 |
+
runtime = RuntimeEvidence(
|
| 83 |
+
"pytorch-bridge-aten-pair-pinned-cpu",
|
| 84 |
+
"PyTorch/ATen",
|
| 85 |
+
"BUILD_ID_PAIR",
|
| 86 |
+
(),
|
| 87 |
+
)
|
| 88 |
+
snapshots = {parent: parent_snapshot}
|
| 89 |
+
candidates: list[_EvidenceCandidate] = []
|
| 90 |
+
for offset in range(64):
|
| 91 |
+
identity = ProcessIdentity(100 + offset, 1000 + offset)
|
| 92 |
+
current = ProcessSnapshot(
|
| 93 |
+
identity,
|
| 94 |
+
2001,
|
| 95 |
+
"/usr/bin/python3",
|
| 96 |
+
"python3",
|
| 97 |
+
("python3",),
|
| 98 |
+
(),
|
| 99 |
+
(),
|
| 100 |
+
(),
|
| 101 |
+
parent=parent,
|
| 102 |
+
)
|
| 103 |
+
snapshots[identity] = current
|
| 104 |
+
candidates.append(_EvidenceCandidate(current, (content,), (), 1))
|
| 105 |
+
complete_identity = ProcessIdentity(999, 1999)
|
| 106 |
+
complete_snapshot = ProcessSnapshot(
|
| 107 |
+
complete_identity,
|
| 108 |
+
2001,
|
| 109 |
+
"/usr/bin/python3",
|
| 110 |
+
"python3",
|
| 111 |
+
("python3",),
|
| 112 |
+
(),
|
| 113 |
+
(),
|
| 114 |
+
(),
|
| 115 |
+
parent=parent,
|
| 116 |
+
)
|
| 117 |
+
snapshots[complete_identity] = complete_snapshot
|
| 118 |
+
candidates.append(
|
| 119 |
+
_EvidenceCandidate(complete_snapshot, (content,), (runtime,), 2)
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
correlated = supervisor._correlate(candidates, snapshots)
|
| 123 |
+
self.assertEqual(1, len(correlated))
|
| 124 |
+
self.assertEqual(65, len(correlated[0][0]))
|
| 125 |
+
|
| 126 |
+
groups = supervisor._content_groups(candidates, snapshots)
|
| 127 |
+
self.assertEqual(1, len(groups))
|
| 128 |
+
self.assertEqual((complete_identity,), tuple(item.snapshot.identity for item in groups[0][0]))
|
| 129 |
|
| 130 |
def test_heartbeat_stops_when_no_scan_completed_within_health_bound(self) -> None:
|
| 131 |
supervisor = self._instance()
|
tests/test_watchdog.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import unittest
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from unittest.mock import patch
|
| 6 |
+
|
| 7 |
+
from lumi_eggcracker.jsonio import JsonInputError
|
| 8 |
+
from lumi_eggcracker.watchdog import _kill_target
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class WatchdogTests(unittest.TestCase):
|
| 12 |
+
def test_collection_after_kill_write_is_successful_empty_containment(self) -> None:
|
| 13 |
+
path = Path("/definitely-absent-eggcracker-cgroup")
|
| 14 |
+
with patch("lumi_eggcracker.watchdog._write") as write, patch(
|
| 15 |
+
"lumi_eggcracker.watchdog._events",
|
| 16 |
+
side_effect=JsonInputError("cannot inspect watchdog cgroup"),
|
| 17 |
+
):
|
| 18 |
+
result = _kill_target(path)
|
| 19 |
+
write.assert_called_once_with(path, "cgroup.kill", b"1\n")
|
| 20 |
+
self.assertEqual(0, result["populated"])
|
| 21 |
+
self.assertTrue(result["collected"])
|
| 22 |
+
|