ProCreations's picture
Publish validated ICML reproduction
82524c9 verified
Raw
History Blame Contribute Delete
3.77 kB
#!/usr/bin/env python3
import hashlib,json,os,subprocess,sys,tempfile
from pathlib import Path
r=Path(__file__).resolve().parent
c=json.loads((r/'official_claims.json').read_text());a=json.loads((r/'outputs/results.json').read_text());b=json.loads((r/'replay_a/results.json').read_text());m=json.loads((r/'EVIDENCE_MATRIX.json').read_text());f=json.loads((r/'final_assessment.json').read_text())
assert c==['The paper reformulates mechanisms as returning pairs (y, ε) rather than separate output and privacy-loss functions, resolving prior notational obstacles to defining post-processing immunity for accuracy-first privacy (Section 3.1).', 'Pure ex-post privacy (δ=0) satisfies post-processing immunity, but δ-probabilistic ex-post privacy with δ>0 does not, even though the latter is shown equivalent to (ε,δ)-probabilistic differential privacy for constant ε (Theorem 3.2, Section 3.2).', 'The paper introduces α-ex-post Rényi differential privacy (Section 4), proves it satisfies post-processing immunity (Theorem 4), and proves it composes adaptively with total privacy loss ε* = Σε_i (Theorem 6).', 'The sequential precision-weighted Gaussian mechanism (Algorithm 2, Appendix C.4) is shown to achieve α-ex-post RDP equivalent to the original Brownian mechanism (Theorem 5, Section 5).', 'Table 1 summarizes that pure ex-post privacy has post-processing immunity but the Brownian mechanism does not satisfy it, whereas α-ex-post RDP satisfies both post-processing immunity and compatibility with the Brownian mechanism (Table 1, Section 3).', 'On the Adult dataset, a data-dependent stopping rule (Algorithm 1) using a private validation set generates synthetic data while minimizing privacy budget over ε ∈ [0.01, 1] subject to maintaining classifier accuracy thresholds (Figure 1, Section 6).'] and len(c)==6 and a==b and a['summary']=={'all_passed':True,'passed':20,'total':20}
assert all(g['passed'] for g in a['gates']) and m['release_quality_gate']['expected_verified_points']==12 and [x['assessment'] for x in m['claims']]==['verified']*6 and f['conservative_points']==12
pins={'source/paper.pdf':'85b0e1c0bff38e57229697e393a3036967edc048e9abb84eff893c0df2cc2edc', 'source/paper-source.tar':'e44b3bdff38f92d70956ea0861b7ee0c4dfe868da2cbd212f1d126b793f7f919', 'source/adult.zip':'7537312dd56c2b98035880805ce99e68183a30ee468aa5329d6df0fbb3cc21bb', 'source/adult-preprocessed.csv':'2945341e7469e49dab2f0f3381d26c446d537acf95fd52040037c351c63a0831', 'source/official-code.tar.gz':'67f54df9c48b68699608696cb0b87df4fee244a5db04879fc24e0ec49fd60e11'}
for name,digest in pins.items():assert hashlib.sha256((r/name).read_bytes()).hexdigest()==digest,name
for name in ['results.json']+[f'claim{i}.csv' for i in range(1,7)]:assert (r/'outputs'/name).read_bytes()==(r/'replay_a'/name).read_bytes(),name
text='\n'.join(p.read_text(errors='ignore') for p in r.rglob('*') if p.is_file() and p.suffix in {'.md','.json','.txt','.html','.tex','.py','.csv'})
assert all(claim in text for claim in c) and 'paper-CVDEc0doW8' in text
with tempfile.TemporaryDirectory() as t:
fresh=Path(t)/'fresh';env={**os.environ,'PYTHONDONTWRITEBYTECODE':'1','PYTHONHASHSEED':'0','PYTHONWARNINGS':'error'}
subprocess.run([sys.executable,'-W','error',str(r/'reproduce.py'),'--paper-pdf',str(r/'source/paper.pdf'),'--paper-source',str(r/'source/paper-source.tar'),'--adult-zip',str(r/'source/adult.zip'),'--adult-csv',str(r/'source/adult-preprocessed.csv'),'--out',str(fresh)],check=True,env=env,stdout=subprocess.DEVNULL)
for name in ['results.json']+[f'claim{i}.csv' for i in range(1,7)]:assert (fresh/name).read_bytes()==(r/'outputs'/name).read_bytes(),name
print('evidence validated: six exact claims, 20/20 gates, paired byte-identical privacy and full-Adult replay')