| |
| """Run the native Swift host and compare its EOS-bound output with a BF16 receipt.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import subprocess |
| import time |
| from pathlib import Path |
|
|
|
|
| EOS_TOKEN_ID = 2 |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--executable", type=Path, required=True) |
| parser.add_argument("--model-dir", type=Path, required=True) |
| parser.add_argument("--pixel-values", type=Path, required=True) |
| parser.add_argument("--expected-receipt", type=Path, required=True) |
| parser.add_argument("--expected-row", type=int, default=0) |
| parser.add_argument("--max-new-tokens", type=int, default=64) |
| parser.add_argument("--output", type=Path, required=True) |
| args = parser.parse_args() |
|
|
| expected_receipt = json.loads(args.expected_receipt.read_text()) |
| if "rows" in expected_receipt: |
| row = expected_receipt["rows"][args.expected_row] |
| expected_tokens = row["reference_tokens"] |
| expected_text = row["reference_text"] |
| source = row.get("image") |
| else: |
| expected_tokens = expected_receipt.get("native_tokens", expected_receipt["coreml_tokens"]) |
| expected_tokens = expected_tokens[: expected_tokens.index(EOS_TOKEN_ID) + 1] |
| expected_text = None |
| source = "synthetic-canary" |
|
|
| command = [ |
| str(args.executable.expanduser().resolve()), |
| "--model-dir", |
| str(args.model_dir.expanduser().resolve()), |
| "--pixel-values-fp32", |
| str(args.pixel_values.expanduser().resolve()), |
| "--max-tokens", |
| str(args.max_new_tokens), |
| ] |
| started = time.perf_counter() |
| completed = subprocess.run(command, check=False, capture_output=True, text=True) |
| seconds = time.perf_counter() - started |
| stdout_lines = completed.stdout.splitlines() |
| if not stdout_lines: |
| raise RuntimeError(f"Swift runtime produced no stdout: {completed.stderr}") |
| tokens = [int(value) for value in stdout_lines[0].split()] |
| text = "\n".join(stdout_lines[1:]) |
| receipt = { |
| "source": source, |
| "command": [ |
| args.executable.name, |
| "--model-dir", |
| ".", |
| "--pixel-values-fp32", |
| args.pixel_values.name, |
| "--max-tokens", |
| str(args.max_new_tokens), |
| ], |
| "exit_code": completed.returncode, |
| "seconds": seconds, |
| "tokens": tokens, |
| "expected_tokens": expected_tokens, |
| "token_exact": tokens == expected_tokens, |
| "stop_reason": "eos" if tokens and tokens[-1] == EOS_TOKEN_ID else "max_new_tokens", |
| "text": text, |
| "expected_text": expected_text, |
| "text_exact": expected_text is None or text == expected_text, |
| "stderr": completed.stderr, |
| } |
| args.output.parent.mkdir(parents=True, exist_ok=True) |
| args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n") |
| print(json.dumps(receipt, ensure_ascii=False, indent=2)) |
| if ( |
| completed.returncode != 0 |
| or not receipt["token_exact"] |
| or not receipt["text_exact"] |
| or receipt["stop_reason"] != "eos" |
| ): |
| raise SystemExit(1) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
