scripts/render_manifest.py

#!/usr/bin/env python3
from __future__ import annotations

import hashlib
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
EXCLUDED_PARTS = {".env", ".cache", "outputs", "secrets", "images", "__pycache__"}
EXCLUDED_FILES = {Path("configs/stage2_covt.yaml")}


def digest(path: Path) -> str:
    # Image tar shards are hashed while they are built. Reuse the sidecar so
    # rendering the global manifest does not reread roughly 288 GB.
    if path.suffix == ".tar":
        sidecar = Path(f"{path}.sha256")
        if sidecar.is_file():
            value = sidecar.read_text(encoding="utf-8").split()[0]
            if len(value) == 64:
                return value
    hasher = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(16 * 1024 * 1024), b""):
            hasher.update(chunk)
    return hasher.hexdigest()


def main() -> None:
    output = ROOT / "MANIFEST.sha256"
    files = []
    for path in ROOT.rglob("*"):
        if not path.is_file() or path == output:
            continue
        rel = path.relative_to(ROOT)
        if any(part in EXCLUDED_PARTS or part.endswith(".egg-info") for part in rel.parts):
            continue
        if rel in EXCLUDED_FILES:
            continue
        if rel.suffix == ".pyc":
            continue
        files.append((rel, path))
    with output.open("w", encoding="utf-8") as handle:
        for rel, path in sorted(files):
            handle.write(f"{digest(path)}  {rel.as_posix()}\n")
    print(f"manifest_files={len(files)} output={output}")


if __name__ == "__main__":
    main()