rejection_topK/summary.json

{
  "version": "rejection_topK_v1",
  "M_attempts_per_cell": 50,
  "Ks": [
    1,
    4,
    16,
    64
  ],
  "models": [
    "Base",
    "SFT_real",
    "GRPO"
  ],
  "n_prompts": 8,
  "total_attempts": 4800,
  "total_kept_samples": 2817,
  "success_rate_pct": {
    "1": {
      "Base": 4.25,
      "GRPO": 76.75,
      "SFT_real": 9.75
    },
    "4": {
      "Base": 14.5,
      "GRPO": 95.0,
      "SFT_real": 36.25
    },
    "16": {
      "Base": 38.75,
      "GRPO": 99.0,
      "SFT_real": 76.25
    },
    "64": {
      "Base": 54.5,
      "GRPO": 100.0,
      "SFT_real": 99.25
    }
  },
  "diversity": [
    {
      "model": "Base",
      "K": 1,
      "n_kept": 17,
      "n_pairs": 136,
      "mean_jaccard": 0.221,
      "median_jaccard": 0.1906,
      "min_jaccard": 0.0692
    },
    {
      "model": "Base",
      "K": 4,
      "n_kept": 58,
      "n_pairs": 1653,
      "mean_jaccard": 0.2364,
      "median_jaccard": 0.2166,
      "min_jaccard": 0.0515
    },
    {
      "model": "Base",
      "K": 16,
      "n_kept": 155,
      "n_pairs": 11935,
      "mean_jaccard": 0.2119,
      "median_jaccard": 0.1985,
      "min_jaccard": 0.0
    },
    {
      "model": "Base",
      "K": 64,
      "n_kept": 218,
      "n_pairs": 23653,
      "mean_jaccard": 0.2165,
      "median_jaccard": 0.2012,
      "min_jaccard": 0.0
    },
    {
      "model": "GRPO",
      "K": 1,
      "n_kept": 307,
      "n_pairs": 46971,
      "mean_jaccard": 0.4263,
      "median_jaccard": 0.5696,
      "min_jaccard": 0.08
    },
    {
      "model": "GRPO",
      "K": 4,
      "n_kept": 380,
      "n_pairs": 72010,
      "mean_jaccard": 0.4132,
      "median_jaccard": 0.5597,
      "min_jaccard": 0.0781
    },
    {
      "model": "GRPO",
      "K": 16,
      "n_kept": 396,
      "n_pairs": 78210,
      "mean_jaccard": 0.3993,
      "median_jaccard": 0.5565,
      "min_jaccard": 0.0724
    },
    {
      "model": "GRPO",
      "K": 64,
      "n_kept": 400,
      "n_pairs": 79800,
      "mean_jaccard": 0.3962,
      "median_jaccard": 0.5542,
      "min_jaccard": 0.0808
    },
    {
      "model": "SFT_real",
      "K": 1,
      "n_kept": 39,
      "n_pairs": 741,
      "mean_jaccard": 0.2245,
      "median_jaccard": 0.2067,
      "min_jaccard": 0.0
    },
    {
      "model": "SFT_real",
      "K": 4,
      "n_kept": 145,
      "n_pairs": 10440,
      "mean_jaccard": 0.241,
      "median_jaccard": 0.2376,
      "min_jaccard": 0.0
    },
    {
      "model": "SFT_real",
      "K": 16,
      "n_kept": 305,
      "n_pairs": 46360,
      "mean_jaccard": 0.2387,
      "median_jaccard": 0.2348,
      "min_jaccard": 0.0
    },
    {
      "model": "SFT_real",
      "K": 64,
      "n_kept": 397,
      "n_pairs": 78606,
      "mean_jaccard": 0.2271,
      "median_jaccard": 0.2163,
      "min_jaccard": 0.0
    }
  ]
}