File size: 1,615 Bytes
20b4034 70e1bae 20b4034 70e1bae 20b4034 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | #!/usr/bin/env python3
from __future__ import annotations
import hashlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EXCLUDED_PARTS = {".env", ".cache", "outputs", "secrets", "images", "__pycache__"}
EXCLUDED_FILES = {Path("configs/stage2_covt.yaml")}
def digest(path: Path) -> str:
# Image tar shards are hashed while they are built. Reuse the sidecar so
# rendering the global manifest does not reread roughly 288 GB.
if path.suffix == ".tar":
sidecar = Path(f"{path}.sha256")
if sidecar.is_file():
value = sidecar.read_text(encoding="utf-8").split()[0]
if len(value) == 64:
return value
hasher = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(16 * 1024 * 1024), b""):
hasher.update(chunk)
return hasher.hexdigest()
def main() -> None:
output = ROOT / "MANIFEST.sha256"
files = []
for path in ROOT.rglob("*"):
if not path.is_file() or path == output:
continue
rel = path.relative_to(ROOT)
if any(part in EXCLUDED_PARTS or part.endswith(".egg-info") for part in rel.parts):
continue
if rel in EXCLUDED_FILES:
continue
if rel.suffix == ".pyc":
continue
files.append((rel, path))
with output.open("w", encoding="utf-8") as handle:
for rel, path in sorted(files):
handle.write(f"{digest(path)} {rel.as_posix()}\n")
print(f"manifest_files={len(files)} output={output}")
if __name__ == "__main__":
main()
|