scripts/build_ensemble.py

#!/usr/bin/env python3
"""Ensemble builder — reproduces go.csv from 19 per-cand prediction CSVs.

Reads `candidate_csvs/<descriptive_name>.csv` and computes V2 medoid_ngram per row.
No GPU needed. Pure CPU + ROUGE scoring (rouge-score==0.1.2).

Run from /mnt/msrh/Magic_submission/ folder:
  python /mnt/msrh/Magic_submission/scripts/build_ensemble.py

Output:
  - A regenerated submission CSV written under /mnt/msrh/Magic_submission/.
  - The script also prints its md5 alongside the shipped go.csv (a2ecca4a8e1aa01acf9a8b9a1d56ebf2)
    so you can compare. md5 byte-equality is NOT guaranteed across machines (see README §3
    "Note on byte-identity vs functional reproducibility"); functional LB equivalence is.
"""
import csv, pathlib, json, sys
from rouge_score import rouge_scorer

# Resolve paths relative to this script
ROOT = pathlib.Path("/mnt/msrh/Magic_submission")
CAND_DIR = ROOT / "candidate_csvs"
OUT_DIR = ROOT
DATA_DIR = ROOT / "data"
if not (DATA_DIR / "Test.csv").exists():
    print(f"ERROR: Test.csv not found at {DATA_DIR/'Test.csv'}")
    print("Place Test.csv + SampleSubmission.csv in /mnt/msrh/Magic_submission/data/ then re-run.")
    sys.exit(1)
TEST_CSV = DATA_DIR / "Test.csv"
SAMPLE_CSV = DATA_DIR / "SampleSubmission.csv"

# 19 candidates in EXACT ORDER — order matters because medoid ties resolve to
# the first-encountered cand. Reordering changes ties → different output bytes.
CAND_ORDER = [
    # ─── 12 fewshot LoRA cands + 3 no-fewshot mediators ───
    # 3 Q3.5-27B K=3 variants (3ep ck-1200/1100, 5ep ck-1200)
    "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200.csv",
    "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100.csv",
    "Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200.csv",
    # Q3.5-27B K=4, K=7 peak ckpts (v1 prompt)
    "Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600.csv",
    "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600.csv",
    # Q3.6-27B family (K=3 RecA, K=4, K=5 peak, K=7)
    "Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600.csv",
    "Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400.csv",
    "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200.csv",
    "Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600.csv",
    # Q3-32B family (K=3, K=5, K=7)
    "Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400.csv",
    "Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700.csv",
    "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600.csv",
    # 3 no-fewshot baselines (mediators) — long-trained without demos for decorrelation
    "Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800.csv",
    "Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600.csv",
    "Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500.csv",
    # ─── v8 anchored-extraction prompt (Q3.5 K=5) ───
    "Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500.csv",  # ⭐ best standalone 0.72325 LB
    # ─── 3 cross-arch EARLY ckpts (anti-overfit) ───
    "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200.csv",   # ⭐ early-tap (vs peak ck-1600)
    "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200.csv",     # ⭐ early-tap
    "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000.csv",   # ⭐ very early-tap (vs peak ck-1200)
]

# Standalone LB for each cand (verified on Zindi public LB, for documentation only)
CAND_LB = {
    "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200.csv":           0.7148,
    "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100.csv":           0.7124,
    "Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200.csv":           0.7102,
    "Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600.csv":           0.7162,
    "Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500.csv":  0.72325,  
    "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600.csv":           0.7150,
    "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200.csv":           0.7084,  
    "Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600.csv":           0.7060,
    "Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400.csv":           0.7091,
    "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200.csv":           0.7136,
    "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000.csv":           0.7086, 
    "Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600.csv":           0.7194,
    "Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400.csv":             0.7011,
    "Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700.csv":             0.7081,
    "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600.csv":             0.71383,
    "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200.csv":             0.7111,   
    "Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800.csv":          0.6948,  
    "Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600.csv":          0.6933,   
    "Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500.csv":            0.6884,
}

scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2"], use_stemmer=False)


def load(path):
    with open(path, newline="") as f:
        r = csv.DictReader(f)
        col = "TargetRLF1" if "TargetRLF1" in r.fieldnames else r.fieldnames[1]
        return {row["ID"]: str(row[col]) for row in r}


def medoid(texts):
    """V2 medoid_ngram: pick text with HIGHEST sum of pairwise (R1.F + R2.F) to all others."""
    best, best_s = 0, -1.0
    for i in range(len(texts)):
        s = 0.0
        for j in range(len(texts)):
            if i == j:
                continue
            rr = scorer.score(texts[j], texts[i])
            s += rr["rouge1"].fmeasure + rr["rouge2"].fmeasure
        if s > best_s:
            best_s, best = s, i
    return best


def main():
    print(f"Loading {len(CAND_ORDER)} cand CSVs from {CAND_DIR}...")
    all_preds = {}
    for name in CAND_ORDER:
        path = CAND_DIR / name
        if not path.exists():
            print(f"  MISSING: {name}")
            continue
        all_preds[name] = load(path)
        print(f"  OK: {name} ({len(all_preds[name])} rows)")

    if len(all_preds) != len(CAND_ORDER):
        print(f"ERROR: only {len(all_preds)}/{len(CAND_ORDER)} cand CSVs loaded.")
        sys.exit(1)

    # Load Test IDs in order
    with open(TEST_CSV, newline="") as f:
        ids = [r["ID"] for r in csv.DictReader(f)]
    with open(SAMPLE_CSV, newline="") as f:
        sample_cols = next(csv.reader(f))
    target_cols = [c for c in sample_cols if c.startswith("Target")]
    print(f"\nProcessing {len(ids)} Test rows × {len(CAND_ORDER)} cands "
          f"({len(CAND_ORDER)*(len(CAND_ORDER)-1)//2} pairwise per row)...")

    members = list(CAND_ORDER)   # ORDER critical for medoid tiebreak
    picks, chosen = [], []
    for i, id_ in enumerate(ids):
        if i % 500 == 0:
            print(f"  row {i}/{len(ids)}")
        texts = [all_preds[m].get(id_, "") for m in members]
        p = medoid(texts)
        picks.append(p)
        chosen.append(texts[p])

    out_path = OUT_DIR / "go_reproduced.csv"
    with open(out_path, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=sample_cols, lineterminator="\n")
        w.writeheader()
        for id_, ans in zip(ids, chosen):
            w.writerow({"ID": id_, **{tc: ans for tc in target_cols}})

    print(f"\n=== DONE ===")
    print(f"Output: {out_path}")

    # Pick distribution
    print("\nPick distribution (per cand %):")
    for i, m in enumerate(members):
        n = picks.count(i)
        pct = n / len(picks) * 100
        marker = " ⭐" if "v8prompt" in m or "ckpt-1200" in m or "ckpt-1000" in m else ""
        print(f"  {m:<60} {n:>4} ({pct:>5.1f}%){marker}")

    # Verify match
    import hashlib
    h = hashlib.md5(open(out_path, "rb").read()).hexdigest()
    EXPECTED = "a2ecca4a8e1aa01acf9a8b9a1d56ebf2"
    if h == EXPECTED:
        print(f"\n✅ MD5 MATCH: {h} (byte-identical to go.csv)")
    else:
        print(f"\n❌ MD5 MISMATCH:")
        print(f"   Reproduced: {h}")
        print(f"   Expected  : {EXPECTED}")


if __name__ == "__main__":
    main()