"""Create a reviewed environment-only Docker context from a fixed Conda prefix.""" import argparse import hashlib import importlib.metadata import io import json import os import shutil import tarfile from pathlib import Path WEIGHT_SUFFIXES = {".pt", ".ckpt", ".safetensors", ".onnx", ".model"} SKIP_DIRS = {"__pycache__", ".git", ".cache", "conda-meta", "tests", "test"} SKIP_NAMES = {"direct_url.json", "pip.conf", ".netrc", ".condarc", "hfkey.txt", "token", "stored_tokens", "credentials", "id_rsa", "id_ed25519", ".npmrc", ".pypirc"} REVIEWED_BINARY_RESOURCES = { "matplotlib/mpl-data/sample_data/goog.npz", "matplotlib/mpl-data/sample_data/jacksboro_fault_dem.npz", "matplotlib/mpl-data/sample_data/topobathy.npz", "scipy/stats/_sobol_direction_numbers.npz", "sentencepiece/package_data/nfkc.bin", "sentencepiece/package_data/nfkc_cf.bin", "sentencepiece/package_data/nmt_nfkc.bin", "sentencepiece/package_data/nmt_nfkc_cf.bin", "text_unidecode/data.bin", "whisper/assets/mel_filters.npz", } def main(): parser = argparse.ArgumentParser() parser.add_argument("--prefix", type=Path, default=Path("/opt/conda/envs/hybrid")) parser.add_argument("--out", type=Path, required=True) args = parser.parse_args() prefix = args.prefix.resolve() assert prefix == Path("/opt/conda/envs/hybrid"), "Snapshot preserves its original prefix" args.out.mkdir(parents=True, exist_ok=False) files, excluded = [], [] for directory, dirs, names in os.walk(prefix, followlinks=False): dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) for d in dirs: path = Path(directory) / d if path.is_symlink(): assert path.resolve().is_relative_to(prefix), f"External directory link: {path}" files.append(path) for name in sorted(names): path = Path(directory) / name rel = path.relative_to(prefix) if name in SKIP_NAMES or name.startswith(".env") or path.suffix in WEIGHT_SUFFIXES or path.suffix == ".pyc": excluded.append(str(rel)) continue if path.suffix == ".pth" and path.parent.name != "site-packages": excluded.append(str(rel)) continue if path.suffix.lower() in {".bin", ".pkl", ".npz", ".h5", ".hdf5"}: resource = str(rel).removeprefix("lib/python3.12/site-packages/") if resource not in REVIEWED_BINARY_RESOURCES: raise ValueError(f"Review potential model/data before including: {rel}") if path.is_symlink(): assert path.resolve().is_relative_to(prefix), f"External file link: {path}" files.append(path) manifest = [] patch_prefix = b"import inspect\nif not hasattr(inspect, 'getargspec'):\n inspect.getargspec = inspect.getfullargspec\n" with tarfile.open(args.out / "environment.tar", "x", dereference=False) as archive: for path in sorted(set(files)): rel = path.relative_to(prefix) info = archive.gettarinfo(str(path), arcname=str(rel)) info.uid = info.gid = 0 info.uname = info.gname = "root" if info.issym(): archive.addfile(info) manifest.append({"file": str(rel), "link": info.linkname}) continue if not info.isfile(): raise ValueError(f"Unsupported file: {rel}") modified = str(rel) == "lib/python3.12/site-packages/chumpy/__init__.py" if modified: original = path.read_bytes() data = patch_prefix + original info.size = len(data) archive.addfile(info, io.BytesIO(data)) digest = hashlib.sha256(data).hexdigest() else: h = hashlib.sha256() with path.open("rb") as source: for block in iter(lambda: source.read(1024 * 1024), b""): h.update(block) digest = h.hexdigest() with path.open("rb") as source: archive.addfile(info, source) entry = {"file": str(rel), "bytes": info.size, "sha256": digest} if modified: entry["patch"] = "chumpy-local inspect.getargspec fallback for Python 3.12" entry["original_sha256"] = hashlib.sha256(original).hexdigest() manifest.append(entry) versions = sorted({f"{dist.metadata['Name']}=={dist.version}" for dist in importlib.metadata.distributions() if dist.metadata.get("Name")}) (args.out / "requirements.snapshot.txt").write_text("\n".join(versions) + "\n") report = { "prefix": str(prefix), "snapshot_kind": "installed dependencies, not a pip-rebuild lock", "excluded": excluded, "skipped_directories": sorted(SKIP_DIRS), "files": manifest, "scope": "No project, user home, HF cache, datasets, pretrained weights or credentials; optional package test/LPIPS weights removed", } (args.out / "environment-inventory.json").write_text(json.dumps(report, indent=2)) source = Path(__file__).resolve().parent for name in ("Dockerfile", "smoke_test.py"): shutil.copy2(source / name, args.out / name) print(json.dumps({"context": str(args.out), "tar_bytes": (args.out / "environment.tar").stat().st_size, "entries": len(manifest), "excluded_files": len(excluded)}), flush=True) if __name__ == "__main__": main()