continuation_benchmark/manifest.json

{
  "version": "real_sft_v2026-05-02",
  "description": "Continuation + surprisal benchmarks computed against UCL-CSSB/PlasmidGPT-SFT after the May 2026 HF cleanup that promoted the canonical fine-tuned weights to model.safetensors. SFT now actually fine-tuned (148 of 148 layers differ from Base).",
  "generated_utc": "2026-05-02T00:27:25.932676Z",
  "method_completion": "teacher-forced log-prob of next 100 bp given 400 bp prefix; sliding window stride 300 across 11 held-out plasmids.",
  "method_surprisal": "100-bp window centered on Prodigal-called CDS start; mean per-token log-prob.",
  "models": {
    "Base": "UCL-CSSB/PlasmidGPT",
    "SFT": "UCL-CSSB/PlasmidGPT-SFT  (commit daeaabf, sha daeaabf05bdc71d4d4d03d5ebec34947daf5ea0b)",
    "RL": "UCL-CSSB/PlasmidGPT-GRPO"
  },
  "completion_benchmark": {
    "file": "real_sft/completion_benchmark.csv",
    "sha256": "13fa3de0fa9523b466ac9fb88bfa0870976de0fb383f4023691a35b5cf253224",
    "summary": {
      "Base": {
        "mean": -12.449167,
        "std": 6.144014,
        "n": 85
      },
      "SFT": {
        "mean": -6.121558,
        "std": 3.464686,
        "n": 85
      },
      "RL": {
        "mean": -10.966027,
        "std": 2.741454,
        "n": 85
      }
    }
  },
  "surprisal_benchmark": {
    "file": "real_sft/surprisal_benchmark.csv",
    "sha256": "6eff64f8a637bb8cb8c286245fc9509374f7d8297b4c7944018e76d8e73a1e24",
    "summary": {
      "Base": {
        "mean": -13.838381,
        "std": 2.511,
        "n": 28
      },
      "SFT": {
        "mean": -6.441413,
        "std": 1.972105,
        "n": 28
      },
      "RL": {
        "mean": -11.663668,
        "std": 1.821043,
        "n": 28
      }
    }
  },
  "delta_vs_broken_sft": {
    "completion_SFT_mean_old": -12.4492,
    "completion_SFT_mean_new": -6.1216,
    "surprisal_SFT_mean_old": -13.8384,
    "surprisal_SFT_mean_new": -6.4414,
    "explanation": "Old SFT log-probs equalled Base because AutoModelForCausalLM was loading the broken model.safetensors (a Base-clone duplicate); the canonical fine-tuned weights were sitting under the non-standard filename PlasmidGPT Finetuned Model.safetensors. Cleanup commit daeaabf swapped the two files."
  },
  "caveats": [
    "Base \u2260 SFT bytewise (max diff in log-prob windows ~28; mean diff ~6.7).",
    "Held-out set is 11 Addgene-style E. coli vectors; surprisal uses 10 (GFP_cassette has no pLann.csv).",
    "MFE under DNA params for SFT_real is being computed and will be added to this folder when ready."
  ]
}