{
  "schema_version": "0.1",
  "kind": "llm",
  "imported_at": "2026-07-10T11:35:30.055697+00:00",
  "source": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/eval-inputs/eval_pairwise_path-questions-heldout.jsonl",
  "suite": "compare_vs_baseline",
  "evaluator": "smelt_compare_regex_match",
  "sample_count": 35,
  "exact_match": 0.0,
  "compare_vs_baseline": {
    "key_metric": "regex_match",
    "regex_pattern": "67",
    "winner": "student",
    "verdict": "Student wins: regex_match 1.000 vs baseline 0.000 (\u0394 +1.000) on the same 35 held-out rows.",
    "n_eval": 35,
    "baseline": {
      "provider": "local_transformers",
      "model": "HuggingFaceTB/SmolLM2-135M-Instruct",
      "endpoint": "local/open-weights",
      "tier": "green",
      "tos_ref": "smollm",
      "registry_version": "0.0.1-scaffold",
      "device": "cuda",
      "metrics": {
        "exact_match": 0.0,
        "token_f1": 0.0,
        "rouge_l": 0.0,
        "regex_match": 0.0
      }
    },
    "student": {
      "backend": "adapter",
      "simulated": false,
      "adapter": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/output",
      "base": "HuggingFaceTB/SmolLM2-135M-Instruct",
      "device": "cuda",
      "metrics": {
        "exact_match": 0.0,
        "token_f1": 0.0309,
        "rouge_l": 0.0309,
        "regex_match": 1.0
      }
    },
    "delta": {
      "exact_match": 0.0,
      "token_f1": 0.0309,
      "rouge_l": 0.0309,
      "regex_match": 1.0
    },
    "leakage": {
      "overlap": 0,
      "method": "exact-string",
      "split": "in-distribution",
      "eval_n": 35,
      "train_n": 400,
      "train_path": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/datasets/guard.jsonl"
    }
  },
  "artifact_path": "/home/sky/.sky/file_mounts/gpt67v2-run/.smelt/runs/20260710T112735663356Z/student/output/external_eval/20260710T113530.055697Z0000-compare_vs_baseline.eval.json",
  "artifact_sha256": "ec45c4a3588251d14175baf98fabcac7e304383d2abbb9eb66a2194306b75774"
}