File size: 2,096 Bytes
6e97fd3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | {
"schema_version": "0.1",
"kind": "llm",
"imported_at": "2026-07-10T11:35:30.055697+00:00",
"source": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/eval-inputs/eval_pairwise_path-questions-heldout.jsonl",
"suite": "compare_vs_baseline",
"evaluator": "smelt_compare_regex_match",
"sample_count": 35,
"exact_match": 0.0,
"compare_vs_baseline": {
"key_metric": "regex_match",
"regex_pattern": "67",
"winner": "student",
"verdict": "Student wins: regex_match 1.000 vs baseline 0.000 (\u0394 +1.000) on the same 35 held-out rows.",
"n_eval": 35,
"baseline": {
"provider": "local_transformers",
"model": "HuggingFaceTB/SmolLM2-135M-Instruct",
"endpoint": "local/open-weights",
"tier": "green",
"tos_ref": "smollm",
"registry_version": "0.0.1-scaffold",
"device": "cuda",
"metrics": {
"exact_match": 0.0,
"token_f1": 0.0,
"rouge_l": 0.0,
"regex_match": 0.0
}
},
"student": {
"backend": "adapter",
"simulated": false,
"adapter": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/output",
"base": "HuggingFaceTB/SmolLM2-135M-Instruct",
"device": "cuda",
"metrics": {
"exact_match": 0.0,
"token_f1": 0.0309,
"rouge_l": 0.0309,
"regex_match": 1.0
}
},
"delta": {
"exact_match": 0.0,
"token_f1": 0.0309,
"rouge_l": 0.0309,
"regex_match": 1.0
},
"leakage": {
"overlap": 0,
"method": "exact-string",
"split": "in-distribution",
"eval_n": 35,
"train_n": 400,
"train_path": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/datasets/guard.jsonl"
}
},
"artifact_path": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/output/external_eval/20260710T113530.055697Z0000-compare_vs_baseline.eval.json",
"artifact_sha256": "ec45c4a3588251d14175baf98fabcac7e304383d2abbb9eb66a2194306b75774"
}
|