LocalWebAgent / image /chunk.py
ereniko's picture
v0.9.0: Debian 12 Linux image with a much bigger toolset
0d56bc3 unverified
Raw History Blame Contribute Delete
2.2 kB
#!/usr/bin/env python3
"""Splits a disk image into the format LocalWebAgent downloads.
python3 image/chunk.py disk.ext2 disk/
Writes disk/chunk-NNNNN.gz (one gzipped 4 MiB chunk each) and disk/manifest.json:
{"format": 1, "size": <image bytes>, "chunkSize": 4194304,
"chunks": [<gz size or 0>...], "downloadBytes": <sum>, "id": <content hash>}
All-zero chunks are skipped (size 0) and served as zeros, so free space costs nothing to
download. "id" changes whenever the content does; the app keys the VM's writable layer
by it, so a new image always starts with a fresh machine.
"""
import gzip
import hashlib
import json
import os
import sys
CHUNK = 4 * 1024 * 1024
def main(image, out):
os.makedirs(out, exist_ok=True)
for name in os.listdir(out):
if name.startswith("chunk-") and name.endswith(".gz"):
os.remove(os.path.join(out, name))
size = os.path.getsize(image)
chunks, digest, total = [], hashlib.sha256(), 0
zero = bytes(CHUNK)
with open(image, "rb") as f:
i = 0
while True:
data = f.read(CHUNK)
if not data:
break
digest.update(data)
if data == zero[: len(data)]:
chunks.append(0)
else:
# mtime=0: identical content gives byte-identical chunks (stable diffs).
gz = gzip.compress(data, compresslevel=9, mtime=0)
with open(os.path.join(out, f"chunk-{i:05d}.gz"), "wb") as g:
g.write(gz)
chunks.append(len(gz))
total += len(gz)
i += 1
manifest = {
"format": 1,
"size": size,
"chunkSize": CHUNK,
"chunks": chunks,
"downloadBytes": total,
"id": digest.hexdigest()[:16],
}
with open(os.path.join(out, "manifest.json"), "w") as f:
json.dump(manifest, f, separators=(",", ":"))
stored = sum(1 for c in chunks if c)
print(f"{size / 1e6:.0f} MB image -> {stored} chunks, {total / 1e6:.0f} MB to download, id {manifest['id']}")
if __name__ == "__main__":
if len(sys.argv) != 3:
sys.exit(__doc__)
main(sys.argv[1], sys.argv[2])