File size: 2,931 Bytes
bdfb884 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | """
Step 1: verified GF(256) multiply (RS keystone) + content-addressed chunk store.
(a) LOG / EXP verified N/N (255/255 each)
(b) COMPOSED GF(256) multiply verified over all 65,536 (a,b) pairs
(c) chunk store reconstructs a 'drive image' bit-exact
(d) dedup removes redundancy (real savings), and is bit-verified by SHA-256
(e) honesty: incompressible data yields ~no savings (no beating entropy)
"""
import os
import torch
from storage import gf256
from storage.common import verify
from storage.chunkstore import ChunkStore
torch.manual_seed(0)
print("=" * 62)
print("STEP 1 -- verified GF(256) + content-addressed chunk store")
print("=" * 62)
# ---- verified coding keystone ----
net_log, net_exp = gf256.train_units()
lok, ltot = verify(net_log, *gf256.log_domain())
eok, etot = verify(net_exp, *gf256.exp_domain())
mok, mtot = gf256.verify_mul(net_log, net_exp)
print("-" * 62)
print(f"(a) LOG verified {lok}/{ltot} | EXP verified {eok}/{etot} -> "
f"{'PASS' if lok==ltot and eok==etot else 'FAIL'}")
print(f"(b) COMPOSED GF(256) multiply verified {mok}/{mtot} -> "
f"{'PASS' if mok==mtot else 'FAIL'}")
# ---- content-addressed chunk store on a synthetic 'drive image' ----
fileA = os.urandom(64 * 1024)
fileB = os.urandom(64 * 1024)
fileC = os.urandom(64 * 1024)
tail = os.urandom(64 * 1024)
# a drive with duplicated files (A and B appear twice) + unique C + unique tail
image = fileA + fileB + fileA + fileC + fileB + tail
cs = ChunkStore()
manifest = cs.put_stream(image)
rebuilt = cs.reconstruct(manifest)
exact = rebuilt == image
print("-" * 62)
print(f"(c) reconstruct drive image: {'bit-exact PASS' if exact else 'FAIL'} "
f"({len(image)} bytes)")
print(f"(d) integrity (all chunks hash-verified): "
f"{'PASS' if cs.integrity_ok() else 'FAIL'}")
st = cs.stats()
print(f" logical {st['logical_bytes']//1024} KB -> stored {st['stored_bytes']//1024} KB "
f"dedup {st['dedup_ratio']:.2f}x "
f"({st['unique_chunks']}/{st['total_chunks']} chunks unique)")
# ---- honesty check: incompressible random data must NOT shrink ----
rnd = os.urandom(384 * 1024)
cs2 = ChunkStore()
cs2.reconstruct(cs2.put_stream(rnd)) # store it
st2 = cs2.stats()
print(f"(e) incompressible 384 KB: dedup {st2['dedup_ratio']:.2f}x "
f"-> {'PASS (no magic)' if st2['dedup_ratio'] < 1.05 else 'unexpected'}")
allpass = (lok == ltot and eok == etot and mok == mtot and exact
and cs.integrity_ok() and st['dedup_ratio'] > 1.2 and st2['dedup_ratio'] < 1.05)
print("=" * 62)
print(f"OVERALL: {'ALL PASS' if allpass else 'some checks failed'}")
if lok == ltot and eok == etot and mok == mtot:
torch.save({"log": net_log.state_dict(), "exp": net_exp.state_dict(),
"meta": {"unit": "GF256_MUL(log/exp)", "poly": "0x11D",
"multiply_verified": f"{mok}/{mtot}"}}, "GF256.pt")
print("saved verified units -> GF256.pt")
|