#!/usr/bin/env python3 """Ensemble builder — reproduces go.csv from 19 per-cand prediction CSVs. Reads `candidate_csvs/.csv` and computes V2 medoid_ngram per row. No GPU needed. Pure CPU + ROUGE scoring (rouge-score==0.1.2). Run from /mnt/msrh/Magic_submission/ folder: python /mnt/msrh/Magic_submission/scripts/build_ensemble.py Output: - A regenerated submission CSV written under /mnt/msrh/Magic_submission/. - The script also prints its md5 alongside the shipped go.csv (a2ecca4a8e1aa01acf9a8b9a1d56ebf2) so you can compare. md5 byte-equality is NOT guaranteed across machines (see README §3 "Note on byte-identity vs functional reproducibility"); functional LB equivalence is. """ import csv, pathlib, json, sys from rouge_score import rouge_scorer # Resolve paths relative to this script ROOT = pathlib.Path("/mnt/msrh/Magic_submission") CAND_DIR = ROOT / "candidate_csvs" OUT_DIR = ROOT DATA_DIR = ROOT / "data" if not (DATA_DIR / "Test.csv").exists(): print(f"ERROR: Test.csv not found at {DATA_DIR/'Test.csv'}") print("Place Test.csv + SampleSubmission.csv in /mnt/msrh/Magic_submission/data/ then re-run.") sys.exit(1) TEST_CSV = DATA_DIR / "Test.csv" SAMPLE_CSV = DATA_DIR / "SampleSubmission.csv" # 19 candidates in EXACT ORDER — order matters because medoid ties resolve to # the first-encountered cand. Reordering changes ties → different output bytes. CAND_ORDER = [ # ─── 12 fewshot LoRA cands + 3 no-fewshot mediators ─── # 3 Q3.5-27B K=3 variants (3ep ck-1200/1100, 5ep ck-1200) "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200.csv", "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100.csv", "Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200.csv", # Q3.5-27B K=4, K=7 peak ckpts (v1 prompt) "Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600.csv", "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600.csv", # Q3.6-27B family (K=3 RecA, K=4, K=5 peak, K=7) "Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600.csv", "Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400.csv", "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200.csv", "Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600.csv", # Q3-32B family (K=3, K=5, K=7) "Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400.csv", "Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700.csv", "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600.csv", # 3 no-fewshot baselines (mediators) — long-trained without demos for decorrelation "Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800.csv", "Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600.csv", "Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500.csv", # ─── v8 anchored-extraction prompt (Q3.5 K=5) ─── "Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500.csv", # ⭐ best standalone 0.72325 LB # ─── 3 cross-arch EARLY ckpts (anti-overfit) ─── "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200.csv", # ⭐ early-tap (vs peak ck-1600) "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200.csv", # ⭐ early-tap "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000.csv", # ⭐ very early-tap (vs peak ck-1200) ] # Standalone LB for each cand (verified on Zindi public LB, for documentation only) CAND_LB = { "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200.csv": 0.7148, "Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100.csv": 0.7124, "Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200.csv": 0.7102, "Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600.csv": 0.7162, "Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500.csv": 0.72325, "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600.csv": 0.7150, "Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200.csv": 0.7084, "Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600.csv": 0.7060, "Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400.csv": 0.7091, "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200.csv": 0.7136, "Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000.csv": 0.7086, "Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600.csv": 0.7194, "Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400.csv": 0.7011, "Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700.csv": 0.7081, "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600.csv": 0.71383, "Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200.csv": 0.7111, "Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800.csv": 0.6948, "Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600.csv": 0.6933, "Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500.csv": 0.6884, } scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2"], use_stemmer=False) def load(path): with open(path, newline="") as f: r = csv.DictReader(f) col = "TargetRLF1" if "TargetRLF1" in r.fieldnames else r.fieldnames[1] return {row["ID"]: str(row[col]) for row in r} def medoid(texts): """V2 medoid_ngram: pick text with HIGHEST sum of pairwise (R1.F + R2.F) to all others.""" best, best_s = 0, -1.0 for i in range(len(texts)): s = 0.0 for j in range(len(texts)): if i == j: continue rr = scorer.score(texts[j], texts[i]) s += rr["rouge1"].fmeasure + rr["rouge2"].fmeasure if s > best_s: best_s, best = s, i return best def main(): print(f"Loading {len(CAND_ORDER)} cand CSVs from {CAND_DIR}...") all_preds = {} for name in CAND_ORDER: path = CAND_DIR / name if not path.exists(): print(f" MISSING: {name}") continue all_preds[name] = load(path) print(f" OK: {name} ({len(all_preds[name])} rows)") if len(all_preds) != len(CAND_ORDER): print(f"ERROR: only {len(all_preds)}/{len(CAND_ORDER)} cand CSVs loaded.") sys.exit(1) # Load Test IDs in order with open(TEST_CSV, newline="") as f: ids = [r["ID"] for r in csv.DictReader(f)] with open(SAMPLE_CSV, newline="") as f: sample_cols = next(csv.reader(f)) target_cols = [c for c in sample_cols if c.startswith("Target")] print(f"\nProcessing {len(ids)} Test rows × {len(CAND_ORDER)} cands " f"({len(CAND_ORDER)*(len(CAND_ORDER)-1)//2} pairwise per row)...") members = list(CAND_ORDER) # ORDER critical for medoid tiebreak picks, chosen = [], [] for i, id_ in enumerate(ids): if i % 500 == 0: print(f" row {i}/{len(ids)}") texts = [all_preds[m].get(id_, "") for m in members] p = medoid(texts) picks.append(p) chosen.append(texts[p]) out_path = OUT_DIR / "go_reproduced.csv" with open(out_path, "w", newline="") as f: w = csv.DictWriter(f, fieldnames=sample_cols, lineterminator="\n") w.writeheader() for id_, ans in zip(ids, chosen): w.writerow({"ID": id_, **{tc: ans for tc in target_cols}}) print(f"\n=== DONE ===") print(f"Output: {out_path}") # Pick distribution print("\nPick distribution (per cand %):") for i, m in enumerate(members): n = picks.count(i) pct = n / len(picks) * 100 marker = " ⭐" if "v8prompt" in m or "ckpt-1200" in m or "ckpt-1000" in m else "" print(f" {m:<60} {n:>4} ({pct:>5.1f}%){marker}") # Verify match import hashlib h = hashlib.md5(open(out_path, "rb").read()).hexdigest() EXPECTED = "a2ecca4a8e1aa01acf9a8b9a1d56ebf2" if h == EXPECTED: print(f"\n✅ MD5 MATCH: {h} (byte-identical to go.csv)") else: print(f"\n❌ MD5 MISMATCH:") print(f" Reproduced: {h}") print(f" Expected : {EXPECTED}") if __name__ == "__main__": main()