noqt commited on
Commit
1508f83
·
1 Parent(s): ff3119d

Repair Daybreak adversarial discovery invariants

Browse files
src/lumi_eggcracker/approvals.py CHANGED
@@ -86,6 +86,8 @@ def approved(
86
  *,
87
  executable_metadata: os.stat_result | tuple[int, int] | None = None,
88
  ) -> bool:
 
 
89
  command_hash = argv_digest(snapshot.argv)
90
  if executable_metadata is None:
91
  try:
 
86
  *,
87
  executable_metadata: os.stat_result | tuple[int, int] | None = None,
88
  ) -> bool:
89
+ if not getattr(snapshot, "argv_complete", True):
90
+ return False
91
  command_hash = argv_digest(snapshot.argv)
92
  if executable_metadata is None:
93
  try:
src/lumi_eggcracker/artifacts.py CHANGED
@@ -16,6 +16,7 @@ from .jsonio import JsonInputError
16
  MAX_ARTIFACT_BYTES = 1 * 1024 * 1024
17
  MAX_ARTIFACTS = 16
18
  MAX_TOTAL_BYTES = 4 * 1024 * 1024
 
19
  MAX_GGUF_ITEMS = 10_000_000
20
  MAX_SAFETENSORS_TENSORS = 100_000
21
  MAX_SAFETENSORS_DIMS = 64
@@ -292,19 +293,43 @@ def from_process_fds(
292
  *,
293
  proc: Path = Path("/proc"),
294
  cache: ArtifactCache | None = None,
 
 
295
  ) -> tuple[ArtifactEvidence, ...]:
296
- """Inspect a bounded set of currently open target descriptors.
297
 
298
  The descriptor is opened through ``/proc/<pid>/fd/<n>``. The link name is
299
  used only to find a descriptor number; it is never accepted as evidence.
 
 
300
  """
301
  identity = snapshot.identity
302
- entries = tuple(getattr(snapshot, "fd_entries", ()))[:MAX_ARTIFACTS]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
  result: list[ArtifactEvidence] = []
304
  consumed = 0
305
- for number, _target in entries:
306
- if not isinstance(number, int) or number < 0:
307
- continue
308
  # /proc/<pid>/fd/<n> is necessarily a procfs symlink. We open that
309
  # exact descriptor target, then validate the opened inode; O_NOFOLLOW
310
  # would reject every legitimate procfs descriptor.
@@ -349,9 +374,19 @@ def from_snapshot(
349
  *,
350
  proc: Path = Path("/proc"),
351
  cache: ArtifactCache | None = None,
 
 
352
  ) -> tuple[ArtifactEvidence, ...]:
353
  """Collect bounded GGUF evidence from open descriptors and mapped files."""
354
- result = list(from_process_fds(snapshot, proc=proc, cache=cache))
 
 
 
 
 
 
 
 
355
  for raw in tuple(getattr(snapshot, "map_paths", ())):
356
  if not isinstance(raw, str) or not raw.startswith("/"):
357
  continue
 
16
  MAX_ARTIFACT_BYTES = 1 * 1024 * 1024
17
  MAX_ARTIFACTS = 16
18
  MAX_TOTAL_BYTES = 4 * 1024 * 1024
19
+ MAX_FD_PROBES_PER_SCAN = 256
20
  MAX_GGUF_ITEMS = 10_000_000
21
  MAX_SAFETENSORS_TENSORS = 100_000
22
  MAX_SAFETENSORS_DIMS = 64
 
293
  *,
294
  proc: Path = Path("/proc"),
295
  cache: ArtifactCache | None = None,
296
+ start_index: int = 0,
297
+ max_probes: int = MAX_FD_PROBES_PER_SCAN,
298
  ) -> tuple[ArtifactEvidence, ...]:
299
+ """Inspect one fair bounded window of currently open target descriptors.
300
 
301
  The descriptor is opened through ``/proc/<pid>/fd/<n>``. The link name is
302
  used only to find a descriptor number; it is never accepted as evidence.
303
+ Callers advance ``start_index`` between scans so a held-open artifact
304
+ cannot be hidden permanently behind harmless lower-numbered descriptors.
305
  """
306
  identity = snapshot.identity
307
+ if isinstance(start_index, bool) or not isinstance(start_index, int) or start_index < 0:
308
+ raise JsonInputError("artifact descriptor cursor is invalid")
309
+ if isinstance(max_probes, bool) or not isinstance(max_probes, int) or max_probes < 1:
310
+ raise JsonInputError("artifact descriptor probe limit is invalid")
311
+ directory = proc / str(identity.pid) / "fd"
312
+ try:
313
+ numbers = sorted(
314
+ int(item.name) for item in directory.iterdir() if item.name.isdigit()
315
+ )
316
+ except (AttributeError, OSError):
317
+ numbers = sorted(
318
+ {
319
+ number
320
+ for number, _target in tuple(getattr(snapshot, "fd_entries", ()))
321
+ if isinstance(number, int) and not isinstance(number, bool) and number >= 0
322
+ }
323
+ )
324
+ if numbers:
325
+ offset = start_index % len(numbers)
326
+ probe_count = min(max_probes, len(numbers))
327
+ selected = tuple(numbers[(offset + index) % len(numbers)] for index in range(probe_count))
328
+ else:
329
+ selected = ()
330
  result: list[ArtifactEvidence] = []
331
  consumed = 0
332
+ for number in selected:
 
 
333
  # /proc/<pid>/fd/<n> is necessarily a procfs symlink. We open that
334
  # exact descriptor target, then validate the opened inode; O_NOFOLLOW
335
  # would reject every legitimate procfs descriptor.
 
374
  *,
375
  proc: Path = Path("/proc"),
376
  cache: ArtifactCache | None = None,
377
+ fd_start_index: int = 0,
378
+ fd_max_probes: int = MAX_FD_PROBES_PER_SCAN,
379
  ) -> tuple[ArtifactEvidence, ...]:
380
  """Collect bounded GGUF evidence from open descriptors and mapped files."""
381
+ result = list(
382
+ from_process_fds(
383
+ snapshot,
384
+ proc=proc,
385
+ cache=cache,
386
+ start_index=fd_start_index,
387
+ max_probes=fd_max_probes,
388
+ )
389
+ )
390
  for raw in tuple(getattr(snapshot, "map_paths", ())):
391
  if not isinstance(raw, str) or not raw.startswith("/"):
392
  continue
src/lumi_eggcracker/discovery.py CHANGED
@@ -13,7 +13,8 @@ from .jsonio import JsonInputError
13
 
14
  PROC = Path("/proc")
15
  MAX_READ = 64 * 1024
16
- MAX_MAP_BYTES = 256 * 1024
 
17
  MAX_FDS = 256
18
  MAX_MAPS = 512
19
 
@@ -37,6 +38,7 @@ class ProcessSnapshot:
37
  fd_entries: tuple[tuple[int, str], ...] = ()
38
  map_paths: tuple[str, ...] = ()
39
  parent: ProcessIdentity | None = None
 
40
 
41
 
42
  def _read(path: Path, limit: int = MAX_READ) -> bytes:
@@ -79,7 +81,9 @@ def identity(pid: int, *, proc: Path = PROC) -> ProcessIdentity | None:
79
 
80
 
81
  def argv_digest(argv: tuple[str, ...] | list[str]) -> str:
82
- return hashlib.sha256("\0".join(argv).encode("utf-8")).hexdigest()
 
 
83
 
84
 
85
  def executable_digest(path: Path) -> str:
@@ -214,9 +218,11 @@ def _links(directory: Path, *, limit: int) -> tuple[str, ...]:
214
 
215
  def _fd_entries(directory: Path, *, limit: int) -> tuple[tuple[int, str], ...]:
216
  values: list[tuple[int, str]] = []
217
- for item in sorted(directory.iterdir(), key=lambda value: value.name)[:limit]:
218
- if not item.name.isdigit():
219
- continue
 
 
220
  try:
221
  target = os.readlink(item)
222
  except OSError:
@@ -229,7 +235,7 @@ def _fd_entries(directory: Path, *, limit: int) -> tuple[tuple[int, str], ...]:
229
  def _maps(path: Path) -> tuple[str, ...]:
230
  try:
231
  raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
232
- except OSError:
233
  return ()
234
  values: list[str] = []
235
  for line in raw.splitlines()[:MAX_MAPS]:
@@ -242,7 +248,7 @@ def _maps(path: Path) -> tuple[str, ...]:
242
  def _map_paths(path: Path) -> tuple[str, ...]:
243
  try:
244
  raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
245
- except OSError:
246
  return ()
247
  values: list[str] = []
248
  for line in raw.splitlines()[:MAX_MAPS]:
@@ -252,7 +258,12 @@ def _map_paths(path: Path) -> tuple[str, ...]:
252
  return tuple(values)
253
 
254
 
255
- def snapshot(value: ProcessIdentity, *, proc: Path = PROC) -> ProcessSnapshot | None:
 
 
 
 
 
256
  current = identity(value.pid, proc=proc)
257
  if current != value:
258
  return None
@@ -260,20 +271,45 @@ def snapshot(value: ProcessIdentity, *, proc: Path = PROC) -> ProcessSnapshot |
260
  try:
261
  parent_pid, _start = parse_stat(_read(root / "stat", 4096).decode("utf-8"))
262
  exe = os.readlink(root / "exe")
263
- command = tuple(
264
- item for item in _read(root / "cmdline").decode("utf-8").split("\0") if item
265
- )
266
  cgroups = tuple(
267
  line for line in _read(root / "cgroup", 8192).decode("utf-8").splitlines() if line
268
  )
269
  uid = _uid(_read(root / "status", 8192).decode("utf-8"))
270
- fd_entries = _fd_entries(root / "fd", limit=MAX_FDS)
271
- fd_paths = tuple(item[1] for item in fd_entries)
272
  except (OSError, UnicodeDecodeError, JsonInputError):
273
  return None
274
- if not command or not exe or len(exe) > 4096:
275
  return None
276
- maps = _map_paths(root / "maps")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
277
  parent = identity(parent_pid, proc=proc) if parent_pid > 0 else None
278
  return ProcessSnapshot(
279
  value,
@@ -287,11 +323,15 @@ def snapshot(value: ProcessIdentity, *, proc: Path = PROC) -> ProcessSnapshot |
287
  fd_entries,
288
  maps,
289
  parent,
 
290
  )
291
 
292
 
293
  def scan(
294
- *, proc: Path = PROC, exclude: Callable[[ProcessSnapshot], bool] | None = None
 
 
 
295
  ) -> list[ProcessSnapshot]:
296
  result: list[ProcessSnapshot] = []
297
  try:
@@ -302,7 +342,7 @@ def scan(
302
  item = identity(pid, proc=proc)
303
  if item is None:
304
  continue
305
- current = snapshot(item, proc=proc)
306
  if current is not None and not (exclude and exclude(current)):
307
  result.append(current)
308
  return result
 
13
 
14
  PROC = Path("/proc")
15
  MAX_READ = 64 * 1024
16
+ MAX_CMDLINE_BYTES = 8 * 1024 * 1024
17
+ MAX_MAP_BYTES = 8 * 1024 * 1024
18
  MAX_FDS = 256
19
  MAX_MAPS = 512
20
 
 
38
  fd_entries: tuple[tuple[int, str], ...] = ()
39
  map_paths: tuple[str, ...] = ()
40
  parent: ProcessIdentity | None = None
41
+ argv_complete: bool = True
42
 
43
 
44
  def _read(path: Path, limit: int = MAX_READ) -> bytes:
 
81
 
82
 
83
  def argv_digest(argv: tuple[str, ...] | list[str]) -> str:
84
+ return hashlib.sha256(
85
+ "\0".join(argv).encode("utf-8", errors="surrogateescape")
86
+ ).hexdigest()
87
 
88
 
89
  def executable_digest(path: Path) -> str:
 
218
 
219
  def _fd_entries(directory: Path, *, limit: int) -> tuple[tuple[int, str], ...]:
220
  values: list[tuple[int, str]] = []
221
+ entries = sorted(
222
+ (item for item in directory.iterdir() if item.name.isdigit()),
223
+ key=lambda value: int(value.name),
224
+ )
225
+ for item in entries[:limit]:
226
  try:
227
  target = os.readlink(item)
228
  except OSError:
 
235
  def _maps(path: Path) -> tuple[str, ...]:
236
  try:
237
  raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
238
+ except (OSError, JsonInputError):
239
  return ()
240
  values: list[str] = []
241
  for line in raw.splitlines()[:MAX_MAPS]:
 
248
  def _map_paths(path: Path) -> tuple[str, ...]:
249
  try:
250
  raw = _read(path, MAX_MAP_BYTES).decode("utf-8", errors="replace")
251
+ except (OSError, JsonInputError):
252
  return ()
253
  values: list[str] = []
254
  for line in raw.splitlines()[:MAX_MAPS]:
 
258
  return tuple(values)
259
 
260
 
261
+ def snapshot(
262
+ value: ProcessIdentity,
263
+ *,
264
+ proc: Path = PROC,
265
+ include_evidence: bool = True,
266
+ ) -> ProcessSnapshot | None:
267
  current = identity(value.pid, proc=proc)
268
  if current != value:
269
  return None
 
271
  try:
272
  parent_pid, _start = parse_stat(_read(root / "stat", 4096).decode("utf-8"))
273
  exe = os.readlink(root / "exe")
 
 
 
274
  cgroups = tuple(
275
  line for line in _read(root / "cgroup", 8192).decode("utf-8").splitlines() if line
276
  )
277
  uid = _uid(_read(root / "status", 8192).decode("utf-8"))
 
 
278
  except (OSError, UnicodeDecodeError, JsonInputError):
279
  return None
280
+ if not exe or len(exe) > 4096:
281
  return None
282
+
283
+ # Linux argv is a byte sequence. Invalid UTF-8 must not erase the whole
284
+ # process from discovery, and a bounded-read failure must make approval
285
+ # unavailable rather than making the process invisible.
286
+ try:
287
+ command = tuple(
288
+ item
289
+ for item in _read(root / "cmdline", MAX_CMDLINE_BYTES)
290
+ .decode("utf-8", errors="surrogateescape")
291
+ .split("\0")
292
+ if item
293
+ )
294
+ argv_complete = True
295
+ except (OSError, JsonInputError):
296
+ command = ()
297
+ argv_complete = False
298
+
299
+ fd_entries: tuple[tuple[int, str], ...] = ()
300
+ maps: tuple[str, ...] = ()
301
+ if include_evidence:
302
+ try:
303
+ fd_entries = _fd_entries(root / "fd", limit=MAX_FDS)
304
+ except OSError:
305
+ fd_entries = ()
306
+ # Native runtime discovery uses stable /proc/<pid>/map_files handles.
307
+ # This fallback is retained for kernels/test fixtures without it and
308
+ # is deliberately isolated so one oversized maps file cannot abort a
309
+ # global discovery pass.
310
+ if not (root / "map_files").is_dir():
311
+ maps = _map_paths(root / "maps")
312
+ fd_paths = tuple(item[1] for item in fd_entries)
313
  parent = identity(parent_pid, proc=proc) if parent_pid > 0 else None
314
  return ProcessSnapshot(
315
  value,
 
323
  fd_entries,
324
  maps,
325
  parent,
326
+ argv_complete,
327
  )
328
 
329
 
330
  def scan(
331
+ *,
332
+ proc: Path = PROC,
333
+ exclude: Callable[[ProcessSnapshot], bool] | None = None,
334
+ include_evidence: bool = True,
335
  ) -> list[ProcessSnapshot]:
336
  result: list[ProcessSnapshot] = []
337
  try:
 
342
  item = identity(pid, proc=proc)
343
  if item is None:
344
  continue
345
+ current = snapshot(item, proc=proc, include_evidence=include_evidence)
346
  if current is not None and not (exclude and exclude(current)):
347
  result.append(current)
348
  return result
src/lumi_eggcracker/elfmarkers.py CHANGED
@@ -30,7 +30,7 @@ PINNED_PYTORCH_ATEN_BUILD_IDS = frozenset({"ad9ab6eeec3b28a0ec3f12f266627610de90
30
  PYTORCH_BRIDGE_EVIDENCE_ID = "pytorch-bridge-build-id-pinned-cpu"
31
  PYTORCH_ATEN_EVIDENCE_ID = "pytorch-aten-build-id-pinned-cpu"
32
  PYTORCH_PAIR_EVIDENCE_ID = "pytorch-bridge-aten-pair-pinned-cpu"
33
- MAX_RUNTIME_CANDIDATES = 128
34
 
35
 
36
  @dataclass(frozen=True)
@@ -182,13 +182,8 @@ def _build_id(descriptor: int, size: int) -> str | None:
182
  return None
183
 
184
 
185
- def inspect_path(path: Path) -> RuntimeEvidence | None:
186
- """Recognise a llama/GGML ELF by two valid symbol-table markers, not name."""
187
- flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
188
- try:
189
- descriptor = os.open(path, flags)
190
- except OSError:
191
- return None
192
  try:
193
  before = os.fstat(descriptor)
194
  if not stat.S_ISREG(before.st_mode) or before.st_size < 64 or before.st_size > (1 << 40):
@@ -214,17 +209,10 @@ def inspect_path(path: Path) -> RuntimeEvidence | None:
214
  return None
215
  except (JsonInputError, OSError, struct.error):
216
  return None
217
- finally:
218
- os.close(descriptor)
219
 
220
 
221
- def inspect_pytorch_path(path: Path) -> RuntimeEvidence | None:
222
- """Recognise one exact pinned PyTorch bridge/ATen ELF identity."""
223
- flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
224
- try:
225
- descriptor = os.open(path, flags)
226
- except OSError:
227
- return None
228
  try:
229
  before = os.fstat(descriptor)
230
  if not stat.S_ISREG(before.st_mode) or before.st_size < 64:
@@ -248,15 +236,50 @@ def inspect_pytorch_path(path: Path) -> RuntimeEvidence | None:
248
  return None
249
  except (JsonInputError, OSError, struct.error):
250
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
251
  finally:
252
  os.close(descriptor)
253
 
254
 
255
  def _inspect_candidate(path: Path) -> tuple[RuntimeEvidence, ...]:
256
  values: list[RuntimeEvidence] = []
257
- evidence = inspect_path(path)
258
- if evidence is not None:
259
- values.append(evidence)
260
  pytorch = inspect_pytorch_path(path)
261
  if pytorch is not None and pytorch not in values:
262
  values.append(pytorch)
@@ -276,38 +299,122 @@ def with_pytorch_pair(values: Iterable[RuntimeEvidence]) -> tuple[RuntimeEvidenc
276
  return tuple(result)
277
 
278
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
  def from_snapshot(
280
- snapshot: object, *, cache: RuntimeCache | None = None
 
 
 
 
 
281
  ) -> tuple[RuntimeEvidence, ...]:
282
- """Inspect executable and mapped regular files; their names are ignored."""
283
- # ``/proc/<pid>/maps`` repeats each shared object once per mapped segment.
284
- # Deduplicate before applying the bounded candidate cap; otherwise a busy
285
- # Python/AI process can exhaust the cap on unrelated NumPy/locale segments
286
- # and hide the exact PyTorch bridge/ATen objects that are mapped later.
287
- candidates: list[str] = []
288
- seen: set[str] = set()
289
- for raw in (getattr(snapshot, "exe_path", ""), *getattr(snapshot, "map_paths", ())):
290
- if isinstance(raw, str) and raw not in seen:
291
- seen.add(raw)
292
- candidates.append(raw)
 
293
  result: list[RuntimeEvidence] = []
294
- for raw in candidates[:MAX_RUNTIME_CANDIDATES]:
295
- if not isinstance(raw, str) or not raw.startswith("/"):
296
- continue
297
- path = Path(raw)
298
- if cache is None:
299
- values = _inspect_candidate(path)
300
- else:
301
- try:
302
- key = _cache_key(path.stat())
303
- except OSError:
304
- continue
305
- if key in cache:
306
- values = cache[key]
307
- else:
308
- values = _inspect_candidate(path)
309
- cache[key] = values
310
  for evidence in values:
311
  if evidence not in result:
312
  result.append(evidence)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
313
  return with_pytorch_pair(result)
 
30
  PYTORCH_BRIDGE_EVIDENCE_ID = "pytorch-bridge-build-id-pinned-cpu"
31
  PYTORCH_ATEN_EVIDENCE_ID = "pytorch-aten-build-id-pinned-cpu"
32
  PYTORCH_PAIR_EVIDENCE_ID = "pytorch-bridge-aten-pair-pinned-cpu"
33
+ MAX_RUNTIME_CANDIDATES = 256
34
 
35
 
36
  @dataclass(frozen=True)
 
182
  return None
183
 
184
 
185
+ def _inspect_llama_descriptor(descriptor: int) -> RuntimeEvidence | None:
186
+ """Recognise llama/GGML through one already-open, stable descriptor."""
 
 
 
 
 
187
  try:
188
  before = os.fstat(descriptor)
189
  if not stat.S_ISREG(before.st_mode) or before.st_size < 64 or before.st_size > (1 << 40):
 
209
  return None
210
  except (JsonInputError, OSError, struct.error):
211
  return None
 
 
212
 
213
 
214
+ def _inspect_pytorch_descriptor(descriptor: int) -> RuntimeEvidence | None:
215
+ """Recognise one exact pinned PyTorch bridge/ATen descriptor identity."""
 
 
 
 
 
216
  try:
217
  before = os.fstat(descriptor)
218
  if not stat.S_ISREG(before.st_mode) or before.st_size < 64:
 
236
  return None
237
  except (JsonInputError, OSError, struct.error):
238
  return None
239
+
240
+
241
+ def _inspect_descriptor(descriptor: int) -> tuple[RuntimeEvidence, ...]:
242
+ values: list[RuntimeEvidence] = []
243
+ llama = _inspect_llama_descriptor(descriptor)
244
+ if llama is not None:
245
+ values.append(llama)
246
+ pytorch = _inspect_pytorch_descriptor(descriptor)
247
+ if pytorch is not None and pytorch not in values:
248
+ values.append(pytorch)
249
+ return tuple(values)
250
+
251
+
252
+ def inspect_path(path: Path) -> RuntimeEvidence | None:
253
+ """Recognise a llama/GGML ELF by two valid symbol-table markers, not name."""
254
+ flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
255
+ try:
256
+ descriptor = os.open(path, flags)
257
+ except OSError:
258
+ return None
259
+ try:
260
+ return _inspect_llama_descriptor(descriptor)
261
+ finally:
262
+ os.close(descriptor)
263
+
264
+
265
+ def inspect_pytorch_path(path: Path) -> RuntimeEvidence | None:
266
+ """Recognise one exact pinned PyTorch bridge/ATen ELF identity."""
267
+ flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
268
+ try:
269
+ descriptor = os.open(path, flags)
270
+ except OSError:
271
+ return None
272
+ try:
273
+ return _inspect_pytorch_descriptor(descriptor)
274
  finally:
275
  os.close(descriptor)
276
 
277
 
278
  def _inspect_candidate(path: Path) -> tuple[RuntimeEvidence, ...]:
279
  values: list[RuntimeEvidence] = []
280
+ llama = inspect_path(path)
281
+ if llama is not None:
282
+ values.append(llama)
283
  pytorch = inspect_pytorch_path(path)
284
  if pytorch is not None and pytorch not in values:
285
  values.append(pytorch)
 
299
  return tuple(result)
300
 
301
 
302
+ def _cached_descriptor(
303
+ descriptor: int, cache: RuntimeCache | None
304
+ ) -> tuple[RuntimeEvidence, ...]:
305
+ try:
306
+ metadata = os.fstat(descriptor)
307
+ key = _cache_key(metadata)
308
+ except OSError:
309
+ return ()
310
+ if cache is not None and key in cache:
311
+ return cache[key]
312
+ values = _inspect_descriptor(descriptor)
313
+ if cache is not None:
314
+ cache[key] = values
315
+ return values
316
+
317
+
318
+ def _mapping_references(snapshot: object, proc: Path) -> tuple[str, ...]:
319
+ try:
320
+ pid = snapshot.identity.pid
321
+ directory = proc / str(pid) / "map_files"
322
+ values = [
323
+ item.name
324
+ for item in directory.iterdir()
325
+ if len(item.name.split("-", 1)) == 2
326
+ and all(
327
+ part and all(character in "0123456789abcdefABCDEF" for character in part)
328
+ for part in item.name.split("-", 1)
329
+ )
330
+ ]
331
+ except (AttributeError, OSError):
332
+ return ()
333
+ return tuple(sorted(values, key=lambda value: int(value.split("-", 1)[0], 16)))
334
+
335
+
336
  def from_snapshot(
337
+ snapshot: object,
338
+ *,
339
+ cache: RuntimeCache | None = None,
340
+ proc: Path = Path("/proc"),
341
+ start_index: int = 0,
342
+ max_candidates: int = MAX_RUNTIME_CANDIDATES,
343
  ) -> tuple[RuntimeEvidence, ...]:
344
+ """Inspect one fair window of live executable/mapping descriptors.
345
+
346
+ Native Linux inspection opens ``/proc/<pid>/map_files/<range>`` directly.
347
+ The mapped inode therefore remains inspectable after its pathname is
348
+ renamed or deleted. Path reopening is only a compatibility fallback for
349
+ fixtures or kernels where ``map_files`` is unavailable.
350
+ """
351
+ if isinstance(start_index, bool) or not isinstance(start_index, int) or start_index < 0:
352
+ raise JsonInputError("runtime mapping cursor is invalid")
353
+ if isinstance(max_candidates, bool) or not isinstance(max_candidates, int) or max_candidates < 1:
354
+ raise JsonInputError("runtime candidate limit is invalid")
355
+
356
  result: list[RuntimeEvidence] = []
357
+
358
+ def add(values: tuple[RuntimeEvidence, ...]) -> None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
359
  for evidence in values:
360
  if evidence not in result:
361
  result.append(evidence)
362
+
363
+ # Bind the executable through procfs as well; a deleted executable path is
364
+ # not an excuse to lose an otherwise exact runtime identity.
365
+ try:
366
+ executable = proc / str(snapshot.identity.pid) / "exe"
367
+ descriptor = os.open(executable, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
368
+ except (AttributeError, OSError):
369
+ descriptor = -1
370
+ if descriptor >= 0:
371
+ try:
372
+ add(_cached_descriptor(descriptor, cache))
373
+ finally:
374
+ os.close(descriptor)
375
+
376
+ references = _mapping_references(snapshot, proc)
377
+ if references:
378
+ offset = start_index % len(references)
379
+ count = min(max_candidates, len(references))
380
+ selected = tuple(
381
+ references[(offset + index) % len(references)] for index in range(count)
382
+ )
383
+ for reference in selected:
384
+ try:
385
+ descriptor = os.open(
386
+ proc / str(snapshot.identity.pid) / "map_files" / reference,
387
+ os.O_RDONLY | getattr(os, "O_CLOEXEC", 0),
388
+ )
389
+ except (AttributeError, OSError):
390
+ continue
391
+ try:
392
+ add(_cached_descriptor(descriptor, cache))
393
+ finally:
394
+ os.close(descriptor)
395
+ else:
396
+ candidates: list[str] = []
397
+ seen: set[str] = set()
398
+ for raw in (getattr(snapshot, "exe_path", ""), *getattr(snapshot, "map_paths", ())):
399
+ if isinstance(raw, str) and raw.startswith("/") and raw not in seen:
400
+ seen.add(raw)
401
+ candidates.append(raw)
402
+ if candidates:
403
+ offset = start_index % len(candidates)
404
+ count = min(max_candidates, len(candidates))
405
+ for index in range(count):
406
+ path = Path(candidates[(offset + index) % len(candidates)])
407
+ if cache is None:
408
+ values = _inspect_candidate(path)
409
+ else:
410
+ try:
411
+ key = _cache_key(path.stat())
412
+ except OSError:
413
+ continue
414
+ if key in cache:
415
+ values = cache[key]
416
+ else:
417
+ values = _inspect_candidate(path)
418
+ cache[key] = values
419
+ add(values)
420
  return with_pytorch_pair(result)
src/lumi_eggcracker/observation.py CHANGED
@@ -39,6 +39,16 @@ class ObservationStore:
39
  self._values.pop(oldest, None)
40
  return value
41
 
 
 
 
 
 
 
 
 
 
 
42
  def expire(self, *, now_ns: int | None = None) -> None:
43
  now = time.monotonic_ns() if now_ns is None else now_ns
44
  for key, value in tuple(self._values.items()):
 
39
  self._values.pop(oldest, None)
40
  return value
41
 
42
+ def get(
43
+ self, identity: Hashable, *, now_ns: int | None = None
44
+ ) -> Observation | None:
45
+ self.expire(now_ns=now_ns)
46
+ return self._values.get(identity)
47
+
48
+ def identities(self, *, now_ns: int | None = None) -> frozenset[Hashable]:
49
+ self.expire(now_ns=now_ns)
50
+ return frozenset(self._values)
51
+
52
  def expire(self, *, now_ns: int | None = None) -> None:
53
  now = time.monotonic_ns() if now_ns is None else now_ns
54
  for key, value in tuple(self._values.items()):
src/lumi_eggcracker/supervisor.py CHANGED
@@ -5,7 +5,6 @@ from __future__ import annotations
5
  import argparse
6
  import datetime as dt
7
  import errno
8
- import hashlib
9
  import json
10
  import os
11
  import re
@@ -27,7 +26,7 @@ from .approvals import approved, revoke
27
  from .approvals import create as create_approval
28
  from .approvals import load_all as load_approvals
29
  from .approvals import public as public_approval
30
- from .artifacts import ArtifactEvidence
31
  from .artifacts import from_snapshot as artifacts_from_snapshot
32
  from .containment import (
33
  capture_identity,
@@ -40,11 +39,12 @@ from .detectors import Catalogue, DetectionMatch, load_catalogue, match, public_
40
  from .discovery import (
41
  ProcessIdentity,
42
  ProcessSnapshot,
 
43
  executable_digest_for_identity,
44
  executable_metadata_for_identity,
45
  scan,
46
  )
47
- from .elfmarkers import RuntimeEvidence, with_pytorch_pair
48
  from .elfmarkers import from_snapshot as runtime_from_snapshot
49
  from .jsonio import JsonInputError, load_regular_json
50
  from .observation import ObservationStore
@@ -217,6 +217,10 @@ class Supervisor:
217
  ] = {}
218
  self.enforcement_slots = threading.BoundedSemaphore(MAX_ENFORCEMENT_TASKS)
219
  self.observations = ObservationStore()
 
 
 
 
220
  self.content_scan_tick = 0
221
  self.last_scan_completed_ns = 0
222
  self.last_scan_duration_ns = 0
@@ -423,7 +427,11 @@ class Supervisor:
423
  candidates: list[_EvidenceCandidate],
424
  snapshots: dict[ProcessIdentity, ProcessSnapshot],
425
  ) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
426
- """Union only small evidence sets joined by an exact live relation."""
 
 
 
 
427
  if not candidates:
428
  return []
429
  parent = list(range(len(candidates)))
@@ -452,8 +460,6 @@ class Supervisor:
452
  groups.setdefault(find(index), []).append(index)
453
  result: list[tuple[tuple[_EvidenceCandidate, ...], str]] = []
454
  for indexes in groups.values():
455
- if len(indexes) > MAX_CORRELATED_PROCESSES:
456
- continue
457
  values = tuple(candidates[index] for index in indexes)
458
  boundary_types = {
459
  value
@@ -464,6 +470,97 @@ class Supervisor:
464
  result.append((values, boundary))
465
  return result
466
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
467
  def _cached_executable_digest(self, snapshot: ProcessSnapshot) -> str:
468
  metadata = executable_metadata_for_identity(snapshot.identity)
469
  key = (
@@ -571,7 +668,8 @@ class Supervisor:
571
  "executable": executable_record,
572
  "observed": {
573
  "argv_count": len(snapshot.argv),
574
- "argv_sha256": hashlib.sha256("\0".join(snapshot.argv).encode("utf-8")).hexdigest(),
 
575
  "pid": snapshot.identity.pid,
576
  "start_time": snapshot.identity.start_time,
577
  "uid": snapshot.uid,
@@ -739,9 +837,20 @@ class Supervisor:
739
  except JsonInputError:
740
  approvals = [] # A corrupt approval never authorizes a matching workload.
741
  snapshots = scan(
742
- exclude=lambda item: item.identity.pid == os.getpid() or self._managed(item)
 
743
  )
744
  snapshot_map = {item.identity: item for item in snapshots}
 
 
 
 
 
 
 
 
 
 
745
  candidates: list[_EvidenceCandidate] = []
746
  for snapshot in snapshots:
747
  fast_match = match(self.catalogue, snapshot)
@@ -749,33 +858,82 @@ class Supervisor:
749
  runtimes: tuple[RuntimeEvidence, ...] = ()
750
  first_seen_ns = time.monotonic_ns()
751
  if content_due:
752
- content = artifacts_from_snapshot(snapshot, cache=self.artifact_cache)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
753
  # Runtime evidence is intentionally collected independently so
754
  # it can be correlated with content held by a bounded peer.
755
- runtimes = runtime_from_snapshot(snapshot, cache=self.runtime_cache)
756
- supplied = {
757
- "MODEL_CONTENT": {item.evidence_id for item in content},
758
- "MODEL_RUNTIME": {item.evidence_id for item in runtimes},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
759
  }
760
- if content or runtimes:
761
  observation = self.observations.observe(
762
- snapshot.identity, set().union(*supplied.values())
763
  )
764
  first_seen_ns = observation.first_seen_ns
 
 
 
 
 
 
 
 
 
 
765
  if fast_match is None and not content and not runtimes:
766
  continue
767
  candidates.append(
768
  _EvidenceCandidate(snapshot, content, runtimes, first_seen_ns, fast_match)
769
  )
770
 
771
- content_groups = self._correlate(
772
- [item for item in candidates if (item.content or item.runtimes) and item.fast_match is None], snapshot_map
 
 
 
 
 
773
  )
774
- groups = content_groups + [
775
  ((item,), "same-process")
776
  for item in candidates
777
  if item.fast_match is not None
778
- ]
779
  for group, boundary_type in groups:
780
  trigger_candidate = group[0]
781
  aggregate_content: list[ArtifactEvidence] = []
@@ -1313,7 +1471,20 @@ class Supervisor:
1313
  and "pids"
1314
  in Path("/sys/fs/cgroup/cgroup.controllers").read_text(encoding="ascii").split()
1315
  )
1316
- ready = available and pidfd_available() and self.quarantine_root is not None
 
 
 
 
 
 
 
 
 
 
 
 
 
1317
  return {
1318
  "autonomous_discovery": ready,
1319
  "backend": "root-supervisor",
@@ -1323,6 +1494,7 @@ class Supervisor:
1323
  "consecutive_failures": self.discovery_failures,
1324
  "last_scan_duration_ms": self.last_scan_duration_ns / 1_000_000,
1325
  "last_scan_completed": self.last_scan_completed_ns > 0,
 
1326
  "scan_health_timeout_ms": SCAN_HEALTH_TIMEOUT_NS / 1_000_000,
1327
  },
1328
  "pidfd": pidfd_available(),
 
5
  import argparse
6
  import datetime as dt
7
  import errno
 
8
  import json
9
  import os
10
  import re
 
26
  from .approvals import create as create_approval
27
  from .approvals import load_all as load_approvals
28
  from .approvals import public as public_approval
29
+ from .artifacts import MAX_FD_PROBES_PER_SCAN, ArtifactEvidence
30
  from .artifacts import from_snapshot as artifacts_from_snapshot
31
  from .containment import (
32
  capture_identity,
 
39
  from .discovery import (
40
  ProcessIdentity,
41
  ProcessSnapshot,
42
+ argv_digest,
43
  executable_digest_for_identity,
44
  executable_metadata_for_identity,
45
  scan,
46
  )
47
+ from .elfmarkers import MAX_RUNTIME_CANDIDATES, RuntimeEvidence, with_pytorch_pair
48
  from .elfmarkers import from_snapshot as runtime_from_snapshot
49
  from .jsonio import JsonInputError, load_regular_json
50
  from .observation import ObservationStore
 
217
  ] = {}
218
  self.enforcement_slots = threading.BoundedSemaphore(MAX_ENFORCEMENT_TASKS)
219
  self.observations = ObservationStore()
220
+ self.artifact_fd_offsets: dict[ProcessIdentity, int] = {}
221
+ self.runtime_map_offsets: dict[ProcessIdentity, int] = {}
222
+ self.observed_content: dict[ProcessIdentity, dict[str, ArtifactEvidence]] = {}
223
+ self.observed_runtimes: dict[ProcessIdentity, dict[str, RuntimeEvidence]] = {}
224
  self.content_scan_tick = 0
225
  self.last_scan_completed_ns = 0
226
  self.last_scan_duration_ns = 0
 
427
  candidates: list[_EvidenceCandidate],
428
  snapshots: dict[ProcessIdentity, ProcessSnapshot],
429
  ) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
430
+ """Union evidence sets joined by an exact live relation.
431
+
432
+ Component size is never interpreted as absence. Callers reduce a
433
+ component to a minimal complete evidence set before enforcement.
434
+ """
435
  if not candidates:
436
  return []
437
  parent = list(range(len(candidates)))
 
460
  groups.setdefault(find(index), []).append(index)
461
  result: list[tuple[tuple[_EvidenceCandidate, ...], str]] = []
462
  for indexes in groups.values():
 
 
463
  values = tuple(candidates[index] for index in indexes)
464
  boundary_types = {
465
  value
 
470
  result.append((values, boundary))
471
  return result
472
 
473
+ def _minimal_content_groups(
474
+ self,
475
+ candidates: list[_EvidenceCandidate],
476
+ snapshots: dict[ProcessIdentity, ProcessSnapshot],
477
+ ) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
478
+ """Return one minimal complete evidence set per related component.
479
+
480
+ Repeated partial evidence does not revoke an independently complete
481
+ approval, and a component larger than the historical containment cap
482
+ cannot suppress a complete profile. Only evidence identities that
483
+ contribute a new predicate are selected.
484
+ """
485
+ result: list[tuple[tuple[_EvidenceCandidate, ...], str]] = []
486
+ for component, boundary in self._correlate(candidates, snapshots):
487
+ selected: list[_EvidenceCandidate] = []
488
+ seen_content: set[str] = set()
489
+ seen_runtime: set[str] = set()
490
+ detected: DetectionMatch | None = None
491
+ for candidate in sorted(component, key=lambda item: item.snapshot.identity):
492
+ content_ids = {item.evidence_id for item in candidate.content}
493
+ runtime_ids = {item.evidence_id for item in candidate.runtimes}
494
+ if content_ids <= seen_content and runtime_ids <= seen_runtime:
495
+ continue
496
+ selected.append(candidate)
497
+ seen_content.update(content_ids)
498
+ seen_runtime.update(runtime_ids)
499
+ paired = with_pytorch_pair(
500
+ item for value in selected for item in value.runtimes
501
+ )
502
+ detected = match(
503
+ self.catalogue,
504
+ selected[0].snapshot,
505
+ evidence={
506
+ "MODEL_CONTENT": {
507
+ item.evidence_id for value in selected for item in value.content
508
+ },
509
+ "MODEL_RUNTIME": {item.evidence_id for item in paired},
510
+ },
511
+ )
512
+ if detected is not None:
513
+ break
514
+ if len(selected) >= MAX_CORRELATED_PROCESSES:
515
+ # There are currently only two independent evidence groups
516
+ # and at most three raw identities are required for the
517
+ # pinned profiles. Reaching this guard means no bounded
518
+ # minimal set was established; do not reinterpret partial
519
+ # evidence as a destructive match.
520
+ break
521
+ if detected is not None:
522
+ result.append((tuple(selected), boundary))
523
+ return result
524
+
525
+ def _content_groups(
526
+ self,
527
+ candidates: list[_EvidenceCandidate],
528
+ snapshots: dict[ProcessIdentity, ProcessSnapshot],
529
+ ) -> list[tuple[tuple[_EvidenceCandidate, ...], str]]:
530
+ """Prefer exact same-process matches before any relation aggregate."""
531
+ complete: list[_EvidenceCandidate] = []
532
+ partial: list[_EvidenceCandidate] = []
533
+ for item in candidates:
534
+ own_runtime = with_pytorch_pair(item.runtimes)
535
+ own_match = match(
536
+ self.catalogue,
537
+ item.snapshot,
538
+ evidence={
539
+ "MODEL_CONTENT": {value.evidence_id for value in item.content},
540
+ "MODEL_RUNTIME": {value.evidence_id for value in own_runtime},
541
+ },
542
+ )
543
+ if own_match is None:
544
+ partial.append(item)
545
+ else:
546
+ complete.append(
547
+ _EvidenceCandidate(
548
+ item.snapshot,
549
+ item.content,
550
+ tuple(own_runtime),
551
+ item.first_seen_ns,
552
+ own_match,
553
+ )
554
+ )
555
+ return [((item,), "same-process") for item in complete] + self._minimal_content_groups(
556
+ partial, snapshots
557
+ )
558
+
559
+ @staticmethod
560
+ def _discovery_excluded(snapshot: ProcessSnapshot) -> bool:
561
+ """Exclude only the supervisor itself; a run record is not approval."""
562
+ return snapshot.identity.pid == os.getpid()
563
+
564
  def _cached_executable_digest(self, snapshot: ProcessSnapshot) -> str:
565
  metadata = executable_metadata_for_identity(snapshot.identity)
566
  key = (
 
668
  "executable": executable_record,
669
  "observed": {
670
  "argv_count": len(snapshot.argv),
671
+ "argv_sha256": argv_digest(snapshot.argv),
672
+ "argv_complete": snapshot.argv_complete,
673
  "pid": snapshot.identity.pid,
674
  "start_time": snapshot.identity.start_time,
675
  "uid": snapshot.uid,
 
837
  except JsonInputError:
838
  approvals = [] # A corrupt approval never authorizes a matching workload.
839
  snapshots = scan(
840
+ exclude=self._discovery_excluded,
841
+ include_evidence=content_due,
842
  )
843
  snapshot_map = {item.identity: item for item in snapshots}
844
+ live_identities = set(snapshot_map)
845
+ for cache in (
846
+ getattr(self, "artifact_fd_offsets", {}),
847
+ getattr(self, "runtime_map_offsets", {}),
848
+ getattr(self, "observed_content", {}),
849
+ getattr(self, "observed_runtimes", {}),
850
+ ):
851
+ for identity in tuple(cache):
852
+ if identity not in live_identities:
853
+ cache.pop(identity, None)
854
  candidates: list[_EvidenceCandidate] = []
855
  for snapshot in snapshots:
856
  fast_match = match(self.catalogue, snapshot)
 
858
  runtimes: tuple[RuntimeEvidence, ...] = ()
859
  first_seen_ns = time.monotonic_ns()
860
  if content_due:
861
+ artifact_offsets = getattr(self, "artifact_fd_offsets", None)
862
+ if artifact_offsets is None:
863
+ self.artifact_fd_offsets = {}
864
+ artifact_offsets = self.artifact_fd_offsets
865
+ runtime_offsets = getattr(self, "runtime_map_offsets", None)
866
+ if runtime_offsets is None:
867
+ self.runtime_map_offsets = {}
868
+ runtime_offsets = self.runtime_map_offsets
869
+ content_now = artifacts_from_snapshot(
870
+ snapshot,
871
+ cache=self.artifact_cache,
872
+ fd_start_index=artifact_offsets.get(snapshot.identity, 0),
873
+ )
874
+ artifact_offsets[snapshot.identity] = (
875
+ artifact_offsets.get(snapshot.identity, 0) + MAX_FD_PROBES_PER_SCAN
876
+ )
877
  # Runtime evidence is intentionally collected independently so
878
  # it can be correlated with content held by a bounded peer.
879
+ runtimes_now = runtime_from_snapshot(
880
+ snapshot,
881
+ cache=self.runtime_cache,
882
+ start_index=runtime_offsets.get(snapshot.identity, 0),
883
+ )
884
+ runtime_offsets[snapshot.identity] = (
885
+ runtime_offsets.get(snapshot.identity, 0) + MAX_RUNTIME_CANDIDATES
886
+ )
887
+ observed_content = getattr(self, "observed_content", None)
888
+ if observed_content is None:
889
+ self.observed_content = {}
890
+ observed_content = self.observed_content
891
+ observed_runtimes = getattr(self, "observed_runtimes", None)
892
+ if observed_runtimes is None:
893
+ self.observed_runtimes = {}
894
+ observed_runtimes = self.observed_runtimes
895
+ content_values = observed_content.setdefault(snapshot.identity, {})
896
+ runtime_values = observed_runtimes.setdefault(snapshot.identity, {})
897
+ content_values.update({item.evidence_id: item for item in content_now})
898
+ runtime_values.update({item.evidence_id: item for item in runtimes_now})
899
+ current_ids = {
900
+ *(item.evidence_id for item in content_now),
901
+ *(item.evidence_id for item in runtimes_now),
902
  }
903
+ if current_ids:
904
  observation = self.observations.observe(
905
+ snapshot.identity, current_ids
906
  )
907
  first_seen_ns = observation.first_seen_ns
908
+ else:
909
+ observation = self.observations.get(snapshot.identity)
910
+ if observation is not None:
911
+ active = observation.evidence
912
+ content = tuple(
913
+ item for key, item in content_values.items() if key in active
914
+ )
915
+ runtimes = with_pytorch_pair(
916
+ item for key, item in runtime_values.items() if key in active
917
+ )
918
  if fast_match is None and not content and not runtimes:
919
  continue
920
  candidates.append(
921
  _EvidenceCandidate(snapshot, content, runtimes, first_seen_ns, fast_match)
922
  )
923
 
924
+ content_groups = self._content_groups(
925
+ [
926
+ item
927
+ for item in candidates
928
+ if (item.content or item.runtimes) and item.fast_match is None
929
+ ],
930
+ snapshot_map,
931
  )
932
+ groups = [
933
  ((item,), "same-process")
934
  for item in candidates
935
  if item.fast_match is not None
936
+ ] + content_groups
937
  for group, boundary_type in groups:
938
  trigger_candidate = group[0]
939
  aggregate_content: list[ArtifactEvidence] = []
 
1471
  and "pids"
1472
  in Path("/sys/fs/cgroup/cgroup.controllers").read_text(encoding="ascii").split()
1473
  )
1474
+ now_ns = time.monotonic_ns()
1475
+ scan_healthy = (
1476
+ self.last_scan_completed_ns > 0
1477
+ and now_ns - self.last_scan_completed_ns <= SCAN_HEALTH_TIMEOUT_NS
1478
+ and self.discovery_failures < MAX_DISCOVERY_FAILURES
1479
+ and self.discovery_thread is not None
1480
+ and self.discovery_thread.is_alive()
1481
+ )
1482
+ ready = (
1483
+ available
1484
+ and pidfd_available()
1485
+ and self.quarantine_root is not None
1486
+ and scan_healthy
1487
+ )
1488
  return {
1489
  "autonomous_discovery": ready,
1490
  "backend": "root-supervisor",
 
1494
  "consecutive_failures": self.discovery_failures,
1495
  "last_scan_duration_ms": self.last_scan_duration_ns / 1_000_000,
1496
  "last_scan_completed": self.last_scan_completed_ns > 0,
1497
+ "healthy": scan_healthy,
1498
  "scan_health_timeout_ms": SCAN_HEALTH_TIMEOUT_NS / 1_000_000,
1499
  },
1500
  "pidfd": pidfd_available(),
src/lumi_eggcracker/watchdog.py CHANGED
@@ -90,12 +90,45 @@ def _kill_target(path: Path) -> dict[str, object]:
90
  _write(path, "cgroup.kill", b"1\n")
91
  deadline = time.monotonic() + 2.0
92
  while time.monotonic() < deadline:
93
- if _events(path).get("populated") == "0":
94
- return {"cgroup": str(path), "frozen": frozen, "populated": 0, "started_ns": before}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  time.sleep(0.005)
96
- value = _events(path)
 
 
 
 
 
 
 
 
 
 
 
97
  return {
98
  "cgroup": str(path),
 
99
  "frozen": frozen,
100
  "populated": int(value.get("populated", "1")),
101
  "started_ns": before,
 
90
  _write(path, "cgroup.kill", b"1\n")
91
  deadline = time.monotonic() + 2.0
92
  while time.monotonic() < deadline:
93
+ try:
94
+ empty = _events(path).get("populated") == "0"
95
+ except JsonInputError:
96
+ # systemd may collect an empty transient cgroup immediately after
97
+ # the authoritative kill write. Collection after that write is
98
+ # equivalent to an empty proof, not a containment failure.
99
+ if not path.exists():
100
+ return {
101
+ "cgroup": str(path),
102
+ "collected": True,
103
+ "frozen": frozen,
104
+ "populated": 0,
105
+ "started_ns": before,
106
+ }
107
+ raise
108
+ if empty:
109
+ return {
110
+ "cgroup": str(path),
111
+ "collected": False,
112
+ "frozen": frozen,
113
+ "populated": 0,
114
+ "started_ns": before,
115
+ }
116
  time.sleep(0.005)
117
+ try:
118
+ value = _events(path)
119
+ except JsonInputError:
120
+ if not path.exists():
121
+ return {
122
+ "cgroup": str(path),
123
+ "collected": True,
124
+ "frozen": frozen,
125
+ "populated": 0,
126
+ "started_ns": before,
127
+ }
128
+ raise
129
  return {
130
  "cgroup": str(path),
131
+ "collected": False,
132
  "frozen": frozen,
133
  "populated": int(value.get("populated", "1")),
134
  "started_ns": before,
tests/test_approvals.py CHANGED
@@ -14,6 +14,7 @@ class Sample:
14
  uid: int
15
  exe_path: str
16
  argv: tuple[str, ...]
 
17
 
18
 
19
  class ApprovalTests(unittest.TestCase):
@@ -28,6 +29,13 @@ class ApprovalTests(unittest.TestCase):
28
  self.assertTrue(approved(sample, executable_digest(binary), values))
29
  self.assertFalse(approved(Sample(1002, str(binary), sample.argv), executable_digest(binary), values))
30
  self.assertFalse(approved(Sample(1001, str(binary), (str(binary), "-m", "/models/other.gguf")), executable_digest(binary), values))
 
 
 
 
 
 
 
31
  self.assertNotIn("/models/qwen.gguf", record.values())
32
  self.assertEqual(argv_digest(sample.argv), record["argv_sha256"])
33
 
 
14
  uid: int
15
  exe_path: str
16
  argv: tuple[str, ...]
17
+ argv_complete: bool = True
18
 
19
 
20
  class ApprovalTests(unittest.TestCase):
 
29
  self.assertTrue(approved(sample, executable_digest(binary), values))
30
  self.assertFalse(approved(Sample(1002, str(binary), sample.argv), executable_digest(binary), values))
31
  self.assertFalse(approved(Sample(1001, str(binary), (str(binary), "-m", "/models/other.gguf")), executable_digest(binary), values))
32
+ self.assertFalse(
33
+ approved(
34
+ Sample(1001, str(binary), sample.argv, argv_complete=False),
35
+ executable_digest(binary),
36
+ values,
37
+ )
38
+ )
39
  self.assertNotIn("/models/qwen.gguf", record.values())
40
  self.assertEqual(argv_digest(sample.argv), record["argv_sha256"])
41
 
tests/test_artifacts.py CHANGED
@@ -7,7 +7,8 @@ import tempfile
7
  import unittest
8
  from pathlib import Path
9
 
10
- from lumi_eggcracker.artifacts import from_snapshot, validate_path
 
11
 
12
 
13
  def gguf(*, version: int = 3, tensors: int = 1, metadata: int = 0, padding: int = 64) -> bytes:
@@ -26,6 +27,23 @@ def safetensors(*, dtype: str = "F32", shape: list[int] | None = None, offsets:
26
 
27
 
28
  class ArtifactTests(unittest.TestCase):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  def test_valid_gguf_content_does_not_depend_on_filename(self) -> None:
30
  with tempfile.TemporaryDirectory() as raw:
31
  path = Path(raw) / "opaque-input"
 
7
  import unittest
8
  from pathlib import Path
9
 
10
+ from lumi_eggcracker.artifacts import from_process_fds, from_snapshot, validate_path
11
+ from lumi_eggcracker.discovery import ProcessIdentity
12
 
13
 
14
  def gguf(*, version: int = 3, tensors: int = 1, metadata: int = 0, padding: int = 64) -> bytes:
 
27
 
28
 
29
  class ArtifactTests(unittest.TestCase):
30
+ def test_descriptor_windows_eventually_cover_late_model_fd(self) -> None:
31
+ with tempfile.TemporaryDirectory() as raw:
32
+ proc = Path(raw)
33
+ directory = proc / "42" / "fd"
34
+ directory.mkdir(parents=True)
35
+ for number in range(20):
36
+ (directory / str(number)).write_bytes(b"harmless")
37
+ (directory / "20").write_bytes(gguf())
38
+ sample = type("Snapshot", (), {"identity": ProcessIdentity(42, 100), "fd_entries": ()})()
39
+ self.assertEqual(
40
+ (), from_process_fds(sample, proc=proc, start_index=0, max_probes=16)
41
+ )
42
+ evidence = from_process_fds(
43
+ sample, proc=proc, start_index=16, max_probes=16
44
+ )
45
+ self.assertEqual({"gguf-v3"}, {item.evidence_id for item in evidence})
46
+
47
  def test_valid_gguf_content_does_not_depend_on_filename(self) -> None:
48
  with tempfile.TemporaryDirectory() as raw:
49
  path = Path(raw) / "opaque-input"
tests/test_discovery.py CHANGED
@@ -3,6 +3,7 @@ from __future__ import annotations
3
  import tempfile
4
  import unittest
5
  from pathlib import Path
 
6
 
7
  from lumi_eggcracker.discovery import (
8
  ProcessIdentity,
@@ -11,11 +12,25 @@ from lumi_eggcracker.discovery import (
11
  executable_digest_for_identity,
12
  identity,
13
  parse_stat,
 
14
  )
15
  from lumi_eggcracker.jsonio import JsonInputError
16
 
17
 
18
  class DiscoveryTests(unittest.TestCase):
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  def test_stat_parser_handles_spaces_in_command_name(self) -> None:
20
  raw = "77 (worker with spaces) S 4 " + "0 " * 17 + "901\n"
21
  self.assertEqual((4, 901), parse_stat(raw))
@@ -52,6 +67,40 @@ class DiscoveryTests(unittest.TestCase):
52
  self.assertEqual(512, len(values))
53
  self.assertTrue(values[0].startswith("/opt/runtime/"))
54
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  def test_process_executable_is_hashed_from_bound_proc_identity(self) -> None:
56
  with tempfile.TemporaryDirectory() as raw:
57
  proc = Path(raw)
 
3
  import tempfile
4
  import unittest
5
  from pathlib import Path
6
+ from unittest.mock import patch
7
 
8
  from lumi_eggcracker.discovery import (
9
  ProcessIdentity,
 
12
  executable_digest_for_identity,
13
  identity,
14
  parse_stat,
15
+ snapshot,
16
  )
17
  from lumi_eggcracker.jsonio import JsonInputError
18
 
19
 
20
  class DiscoveryTests(unittest.TestCase):
21
+ def _proc_entry(self, proc: Path, command: bytes) -> ProcessIdentity:
22
+ entry = proc / "42"
23
+ entry.mkdir()
24
+ (entry / "stat").write_text(
25
+ "42 (runner) S 1 " + "0 " * 17 + "100\n", encoding="utf-8"
26
+ )
27
+ (entry / "cgroup").write_text("0::/user.slice/test\n", encoding="utf-8")
28
+ (entry / "status").write_text("Uid:\t1000\t1000\t1000\t1000\n", encoding="utf-8")
29
+ (entry / "cmdline").write_bytes(command)
30
+ (entry / "fd").mkdir()
31
+ (entry / "map_files").mkdir()
32
+ return ProcessIdentity(42, 100)
33
+
34
  def test_stat_parser_handles_spaces_in_command_name(self) -> None:
35
  raw = "77 (worker with spaces) S 4 " + "0 " * 17 + "901\n"
36
  self.assertEqual((4, 901), parse_stat(raw))
 
67
  self.assertEqual(512, len(values))
68
  self.assertTrue(values[0].startswith("/opt/runtime/"))
69
 
70
+ def test_oversized_maps_file_is_localized_to_one_empty_fallback(self) -> None:
71
+ with tempfile.TemporaryDirectory() as raw:
72
+ path = Path(raw) / "maps"
73
+ path.write_bytes(b"x" * 64)
74
+ with patch("lumi_eggcracker.discovery.MAX_MAP_BYTES", 16):
75
+ self.assertEqual((), _map_paths(path))
76
+
77
+ def test_invalid_utf8_argv_does_not_erase_process_snapshot(self) -> None:
78
+ with tempfile.TemporaryDirectory() as raw:
79
+ proc = Path(raw)
80
+ value = self._proc_entry(proc, b"/usr/bin/python3\0bad\xffarg\0")
81
+ with patch(
82
+ "lumi_eggcracker.discovery.os.readlink", return_value="/usr/bin/python3"
83
+ ):
84
+ current = snapshot(value, proc=proc)
85
+ self.assertIsNotNone(current)
86
+ assert current is not None
87
+ self.assertTrue(current.argv_complete)
88
+ self.assertEqual(2, len(current.argv))
89
+ self.assertEqual(64, len(argv_digest(current.argv)))
90
+
91
+ def test_oversized_argv_is_unapprovable_not_invisible(self) -> None:
92
+ with tempfile.TemporaryDirectory() as raw:
93
+ proc = Path(raw)
94
+ value = self._proc_entry(proc, b"/usr/bin/python3\0" + b"x" * 64)
95
+ with patch("lumi_eggcracker.discovery.MAX_CMDLINE_BYTES", 16), patch(
96
+ "lumi_eggcracker.discovery.os.readlink", return_value="/usr/bin/python3"
97
+ ):
98
+ current = snapshot(value, proc=proc)
99
+ self.assertIsNotNone(current)
100
+ assert current is not None
101
+ self.assertFalse(current.argv_complete)
102
+ self.assertEqual((), current.argv)
103
+
104
  def test_process_executable_is_hashed_from_bound_proc_identity(self) -> None:
105
  with tempfile.TemporaryDirectory() as raw:
106
  proc = Path(raw)
tests/test_elfmarkers.py CHANGED
@@ -7,6 +7,7 @@ import unittest
7
  from pathlib import Path
8
  from unittest.mock import patch
9
 
 
10
  from lumi_eggcracker.elfmarkers import (
11
  PINNED_LLAMA_BUILD_IDS,
12
  PYTORCH_ATEN_EVIDENCE_ID,
@@ -44,7 +45,55 @@ def elf(*, markers: tuple[str, ...], append_decoys: bool = False) -> bytes:
44
  return bytes(body)
45
 
46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  class ElfMarkerTests(unittest.TestCase):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  def test_two_valid_symbol_markers_qualify_regardless_of_filename(self) -> None:
49
  with tempfile.TemporaryDirectory() as raw:
50
  path = Path(raw) / "unrelated-name"
 
7
  from pathlib import Path
8
  from unittest.mock import patch
9
 
10
+ from lumi_eggcracker.discovery import ProcessIdentity
11
  from lumi_eggcracker.elfmarkers import (
12
  PINNED_LLAMA_BUILD_IDS,
13
  PYTORCH_ATEN_EVIDENCE_ID,
 
45
  return bytes(body)
46
 
47
 
48
+ def build_id_elf(identifier: bytes) -> bytes:
49
+ note = struct.pack("<III", 4, len(identifier), 3) + b"GNU\0" + identifier
50
+ note += b"\0" * ((4 - len(identifier) % 4) % 4)
51
+ body = bytearray(128 + len(note))
52
+ body[:16] = b"\x7fELF\x02\x01\x01" + b"\0" * 9
53
+ body[16:64] = struct.pack(
54
+ "<HHIQQQIHHHHHH", 3, 62, 1, 0, 64, 0, 0, 64, 56, 1, 0, 0, 0
55
+ )
56
+ body[64:120] = struct.pack("<IIQQQQQQ", 4, 0, 128, 0, 0, len(note), 0, 4)
57
+ body[128:] = note
58
+ return bytes(body)
59
+
60
+
61
  class ElfMarkerTests(unittest.TestCase):
62
+ def test_proc_mapping_descriptors_are_inspected_without_pathnames(self) -> None:
63
+ bridge_id = bytes.fromhex("44" * 20)
64
+ aten_id = bytes.fromhex("55" * 20)
65
+ with tempfile.TemporaryDirectory() as raw:
66
+ proc = Path(raw)
67
+ mappings = proc / "42" / "map_files"
68
+ mappings.mkdir(parents=True)
69
+ (mappings / "1000-2000").write_bytes(build_id_elf(bridge_id))
70
+ (mappings / "3000-4000").write_bytes(build_id_elf(aten_id))
71
+ sample = type(
72
+ "Snapshot",
73
+ (),
74
+ {
75
+ "identity": ProcessIdentity(42, 100),
76
+ "exe_path": "/deleted-runtime",
77
+ "map_paths": (),
78
+ },
79
+ )()
80
+ with patch(
81
+ "lumi_eggcracker.elfmarkers.PINNED_PYTORCH_BRIDGE_BUILD_IDS",
82
+ {bridge_id.hex()},
83
+ ), patch(
84
+ "lumi_eggcracker.elfmarkers.PINNED_PYTORCH_ATEN_BUILD_IDS",
85
+ {aten_id.hex()},
86
+ ):
87
+ evidence = from_snapshot(sample, proc=proc, max_candidates=2)
88
+ self.assertEqual(
89
+ {
90
+ PYTORCH_BRIDGE_EVIDENCE_ID,
91
+ PYTORCH_ATEN_EVIDENCE_ID,
92
+ PYTORCH_PAIR_EVIDENCE_ID,
93
+ },
94
+ {item.evidence_id for item in evidence},
95
+ )
96
+
97
  def test_two_valid_symbol_markers_qualify_regardless_of_filename(self) -> None:
98
  with tempfile.TemporaryDirectory() as raw:
99
  path = Path(raw) / "unrelated-name"
tests/test_supervisor.py CHANGED
@@ -67,6 +67,65 @@ class SupervisorTests(unittest.TestCase):
67
  (),
68
  )
69
  self.assertTrue(supervisor._managed(snapshot))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
 
71
  def test_heartbeat_stops_when_no_scan_completed_within_health_bound(self) -> None:
72
  supervisor = self._instance()
 
67
  (),
68
  )
69
  self.assertTrue(supervisor._managed(snapshot))
70
+ self.assertFalse(supervisor._discovery_excluded(snapshot))
71
+
72
+ def test_complete_identity_is_not_suppressed_or_broadened_by_64_partials(self) -> None:
73
+ supervisor = self._instance()
74
+ supervisor.catalogue = load_bundled()
75
+ parent = ProcessIdentity(10, 100)
76
+ parent_snapshot = ProcessSnapshot(
77
+ parent, 2001, "/usr/bin/worker", "worker", ("worker",), (), (), ()
78
+ )
79
+ content = ArtifactEvidence(
80
+ "safetensors-v1", "SAFETENSORS", 1, 2, 4096, "a" * 64
81
+ )
82
+ runtime = RuntimeEvidence(
83
+ "pytorch-bridge-aten-pair-pinned-cpu",
84
+ "PyTorch/ATen",
85
+ "BUILD_ID_PAIR",
86
+ (),
87
+ )
88
+ snapshots = {parent: parent_snapshot}
89
+ candidates: list[_EvidenceCandidate] = []
90
+ for offset in range(64):
91
+ identity = ProcessIdentity(100 + offset, 1000 + offset)
92
+ current = ProcessSnapshot(
93
+ identity,
94
+ 2001,
95
+ "/usr/bin/python3",
96
+ "python3",
97
+ ("python3",),
98
+ (),
99
+ (),
100
+ (),
101
+ parent=parent,
102
+ )
103
+ snapshots[identity] = current
104
+ candidates.append(_EvidenceCandidate(current, (content,), (), 1))
105
+ complete_identity = ProcessIdentity(999, 1999)
106
+ complete_snapshot = ProcessSnapshot(
107
+ complete_identity,
108
+ 2001,
109
+ "/usr/bin/python3",
110
+ "python3",
111
+ ("python3",),
112
+ (),
113
+ (),
114
+ (),
115
+ parent=parent,
116
+ )
117
+ snapshots[complete_identity] = complete_snapshot
118
+ candidates.append(
119
+ _EvidenceCandidate(complete_snapshot, (content,), (runtime,), 2)
120
+ )
121
+
122
+ correlated = supervisor._correlate(candidates, snapshots)
123
+ self.assertEqual(1, len(correlated))
124
+ self.assertEqual(65, len(correlated[0][0]))
125
+
126
+ groups = supervisor._content_groups(candidates, snapshots)
127
+ self.assertEqual(1, len(groups))
128
+ self.assertEqual((complete_identity,), tuple(item.snapshot.identity for item in groups[0][0]))
129
 
130
  def test_heartbeat_stops_when_no_scan_completed_within_health_bound(self) -> None:
131
  supervisor = self._instance()
tests/test_watchdog.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import unittest
4
+ from pathlib import Path
5
+ from unittest.mock import patch
6
+
7
+ from lumi_eggcracker.jsonio import JsonInputError
8
+ from lumi_eggcracker.watchdog import _kill_target
9
+
10
+
11
+ class WatchdogTests(unittest.TestCase):
12
+ def test_collection_after_kill_write_is_successful_empty_containment(self) -> None:
13
+ path = Path("/definitely-absent-eggcracker-cgroup")
14
+ with patch("lumi_eggcracker.watchdog._write") as write, patch(
15
+ "lumi_eggcracker.watchdog._events",
16
+ side_effect=JsonInputError("cannot inspect watchdog cgroup"),
17
+ ):
18
+ result = _kill_target(path)
19
+ write.assert_called_once_with(path, "cgroup.kill", b"1\n")
20
+ self.assertEqual(0, result["populated"])
21
+ self.assertTrue(result["collected"])
22
+