betterwithage commited on
Commit
e33458f
·
verified ·
1 Parent(s): a555cb6

fix: make receipt evaluation portable

Browse files

Copy the complete repository into the isolated re-forge directory so packaged kernels and doctrine inputs resolve on fresh checkouts. Avoid privileged symlinks and non-ASCII status output on Windows.

Files changed (1) hide show
  1. scripts/eval.py +5 -4
scripts/eval.py CHANGED
@@ -12,13 +12,14 @@ print(f"model.safetensors sha256 {'MATCHES receipt' if got==want else 'MISMATCH
12
  if got != want:
13
  sys.exit(1)
14
  with tempfile.TemporaryDirectory() as td:
15
- shutil.copy(f"{root}/scripts/forge.py", f"{td}/forge.py")
16
- out = subprocess.run([sys.executable, f"{td}/forge.py"], capture_output=True, text=True, cwd=td)
 
17
  print(out.stdout[-400:] if out.returncode == 0 else out.stderr[-600:])
18
  if out.returncode:
19
  sys.exit(1)
20
- re_receipt = json.load(open(f"{td}/TRAINING_RECEIPT.json"))
21
  d = abs(re_receipt["metrics_MEASURED"]["fidelity_vs_kernel_heldout"]
22
  - receipt["metrics_MEASURED"]["fidelity_vs_kernel_heldout"])
23
- print(f"re-measured fidelity delta vs receipt: {d:.4f} ({'OK 0.02' if d<=0.02 else 'FAIL'})")
24
  sys.exit(0 if d <= 0.02 else 1)
 
12
  if got != want:
13
  sys.exit(1)
14
  with tempfile.TemporaryDirectory() as td:
15
+ copy = f"{td}/repo"
16
+ shutil.copytree(root, copy, ignore=shutil.ignore_patterns(".cache"))
17
+ out = subprocess.run([sys.executable, f"{copy}/scripts/forge.py"], capture_output=True, text=True, cwd=copy)
18
  print(out.stdout[-400:] if out.returncode == 0 else out.stderr[-600:])
19
  if out.returncode:
20
  sys.exit(1)
21
+ re_receipt = json.load(open(f"{copy}/scripts/TRAINING_RECEIPT.json"))
22
  d = abs(re_receipt["metrics_MEASURED"]["fidelity_vs_kernel_heldout"]
23
  - receipt["metrics_MEASURED"]["fidelity_vs_kernel_heldout"])
24
+ print(f"re-measured fidelity delta vs receipt: {d:.4f} ({'OK <=0.02' if d<=0.02 else 'FAIL'})")
25
  sys.exit(0 if d <= 0.02 else 1)