Spaces:
Running
Running
Download image/chunk.py from ereniko/LocalWebAgent: direct link, hf CLI and curl.
- Browser
- Download file 2.2 kB
-
https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/image/chunk.py
- Command line
-
hf download hf://spaces/ereniko/LocalWebAgent/image/chunk.py
-
curl -L -o chunk.py https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/image/chunk.py
2.2 kB
| #!/usr/bin/env python3 | |
| """Splits a disk image into the format LocalWebAgent downloads. | |
| python3 image/chunk.py disk.ext2 disk/ | |
| Writes disk/chunk-NNNNN.gz (one gzipped 4 MiB chunk each) and disk/manifest.json: | |
| {"format": 1, "size": <image bytes>, "chunkSize": 4194304, | |
| "chunks": [<gz size or 0>...], "downloadBytes": <sum>, "id": <content hash>} | |
| All-zero chunks are skipped (size 0) and served as zeros, so free space costs nothing to | |
| download. "id" changes whenever the content does; the app keys the VM's writable layer | |
| by it, so a new image always starts with a fresh machine. | |
| """ | |
| import gzip | |
| import hashlib | |
| import json | |
| import os | |
| import sys | |
| CHUNK = 4 * 1024 * 1024 | |
| def main(image, out): | |
| os.makedirs(out, exist_ok=True) | |
| for name in os.listdir(out): | |
| if name.startswith("chunk-") and name.endswith(".gz"): | |
| os.remove(os.path.join(out, name)) | |
| size = os.path.getsize(image) | |
| chunks, digest, total = [], hashlib.sha256(), 0 | |
| zero = bytes(CHUNK) | |
| with open(image, "rb") as f: | |
| i = 0 | |
| while True: | |
| data = f.read(CHUNK) | |
| if not data: | |
| break | |
| digest.update(data) | |
| if data == zero[: len(data)]: | |
| chunks.append(0) | |
| else: | |
| # mtime=0: identical content gives byte-identical chunks (stable diffs). | |
| gz = gzip.compress(data, compresslevel=9, mtime=0) | |
| with open(os.path.join(out, f"chunk-{i:05d}.gz"), "wb") as g: | |
| g.write(gz) | |
| chunks.append(len(gz)) | |
| total += len(gz) | |
| i += 1 | |
| manifest = { | |
| "format": 1, | |
| "size": size, | |
| "chunkSize": CHUNK, | |
| "chunks": chunks, | |
| "downloadBytes": total, | |
| "id": digest.hexdigest()[:16], | |
| } | |
| with open(os.path.join(out, "manifest.json"), "w") as f: | |
| json.dump(manifest, f, separators=(",", ":")) | |
| stored = sum(1 for c in chunks if c) | |
| print(f"{size / 1e6:.0f} MB image -> {stored} chunks, {total / 1e6:.0f} MB to download, id {manifest['id']}") | |
| if __name__ == "__main__": | |
| if len(sys.argv) != 3: | |
| sys.exit(__doc__) | |
| main(sys.argv[1], sys.argv[2]) | |