| { |
| "schema_version": "0.1", |
| "kind": "llm", |
| "imported_at": "2026-07-10T11:35:30.055697+00:00", |
| "source": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/eval-inputs/eval_pairwise_path-questions-heldout.jsonl", |
| "suite": "compare_vs_baseline", |
| "evaluator": "smelt_compare_regex_match", |
| "sample_count": 35, |
| "exact_match": 0.0, |
| "compare_vs_baseline": { |
| "key_metric": "regex_match", |
| "regex_pattern": "67", |
| "winner": "student", |
| "verdict": "Student wins: regex_match 1.000 vs baseline 0.000 (\u0394 +1.000) on the same 35 held-out rows.", |
| "n_eval": 35, |
| "baseline": { |
| "provider": "local_transformers", |
| "model": "HuggingFaceTB/SmolLM2-135M-Instruct", |
| "endpoint": "local/open-weights", |
| "tier": "green", |
| "tos_ref": "smollm", |
| "registry_version": "0.0.1-scaffold", |
| "device": "cuda", |
| "metrics": { |
| "exact_match": 0.0, |
| "token_f1": 0.0, |
| "rouge_l": 0.0, |
| "regex_match": 0.0 |
| } |
| }, |
| "student": { |
| "backend": "adapter", |
| "simulated": false, |
| "adapter": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/output", |
| "base": "HuggingFaceTB/SmolLM2-135M-Instruct", |
| "device": "cuda", |
| "metrics": { |
| "exact_match": 0.0, |
| "token_f1": 0.0309, |
| "rouge_l": 0.0309, |
| "regex_match": 1.0 |
| } |
| }, |
| "delta": { |
| "exact_match": 0.0, |
| "token_f1": 0.0309, |
| "rouge_l": 0.0309, |
| "regex_match": 1.0 |
| }, |
| "leakage": { |
| "overlap": 0, |
| "method": "exact-string", |
| "split": "in-distribution", |
| "eval_n": 35, |
| "train_n": 400, |
| "train_path": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/datasets/guard.jsonl" |
| } |
| }, |
| "artifact_path": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/output/external_eval/20260710T113530.055697Z0000-compare_vs_baseline.eval.json", |
| "artifact_sha256": "ec45c4a3588251d14175baf98fabcac7e304383d2abbb9eb66a2194306b75774" |
| } |
|
|
