| |
| """Mutasyon skorunun örneklem büyüklüğüne duyarlılığı (referans paketlerle, LLM gerekmez). |
| |
| Her öğede üretilebilen TÜM mutantlar bir kez değerlendirilir; örnekleme deterministik ve iç içe olduğundan |
| (N mutantlık örneklem, M > N örnekleminin ön ekidir) N = 10, 20, 30, 50 için skorlar aynı sonuçlardan |
| hesaplanır. Çıktı: reports/MUTASYON_DUYARLILIK.md (+ .json) |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import sys |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT / "backend")) |
|
|
| from app.eval.benchmark import load_manifest |
| from app.mutation.mutator import MutationContext, generate_mutants, run_mutation |
| from app.sandbox.registry import get_runner |
|
|
| SIZES = [10, 20, 30, 50] |
|
|
|
|
| def score(statuses: list[str]): |
| valid = [s for s in statuses if s != "invalid"] |
| return round(100.0 * sum(s in ("killed", "timeout") for s in valid) / len(valid), 1) if valid else None |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--out", default=str(ROOT / "reports" / "MUTASYON_DUYARLILIK.md")) |
| args = ap.parse_args() |
| junit4 = get_runner("java").framework() == "junit4" |
| rows = [] |
| for it in load_manifest(): |
| if not it.get("reference_tests") or it.get("kind") == "ui": |
| continue |
| key = "reference_tests_junit4" if junit4 and it.get("reference_tests_junit4") else "reference_tests" |
| tests = (ROOT / "benchmarks" / it[key]).read_text(encoding="utf-8") |
| runner = get_runner(it["language"], it.get("kind", "unit")) |
| base = runner.run(it["code"], tests, it.get("kind", "unit"), None, with_coverage=True) |
| lang = it["language"] |
| rep = run_mutation(MutationContext( |
| it["code"], lang, base, |
| lambda c: runner.run(c, tests, it.get("kind", "unit"), None, with_coverage=False, |
| timeout=max(20, int(base.duration_s * 3) + 10))), 100_000, |
| workers=2 if lang == "java" else 3) |
| status = {(m.line, m.operator, m.mutated): m.status for m in rep.mutants} |
| row = {"id": it["id"], "split": it.get("split", "dev"), "language": lang, "all_n": len(rep.mutants), |
| "all_valid": sum(m.status != "invalid" for m in rep.mutants), |
| "all": score([m.status for m in rep.mutants])} |
| for n in SIZES: |
| sample = generate_mutants(it["code"], lang, n) |
| row[str(n)] = score([status[(m.line, m.operator, m.mutated)] for m in sample]) |
| rows.append(row) |
| print(f" {it['id']:<16} tümü={row['all']} ({row['all_n']} mutant) " |
| + " ".join(f"N{n}={row[str(n)]}" for n in SIZES), flush=True) |
| def mean(k): |
| v = [r[k] for r in rows if r[k] is not None] |
| return round(sum(v) / len(v), 1) if v else None |
| def mad(k): |
| v = [abs(r[k] - r["all"]) for r in rows if r[k] is not None and r["all"] is not None] |
| return round(sum(v) / len(v), 1) if v else None |
| lines = ["# Mutasyon skorunun örneklem büyüklüğüne duyarlılığı", "", |
| "İnsan yazımı referans paketler; örnekleme operatör aileleri arasında dengeli, sabit tohumlu ve iç içe.", "", |
| "| Öğe | Bölüm | Dil | Mutant (tümü / geçerli) | " + " | ".join(f"N={n}" for n in SIZES) + " | Tümü |", |
| "|---|---|---|---|" + "---|" * len(SIZES) + "---|"] |
| for r in rows: |
| lines.append(f"| {r['id']} | {r['split']} | {r['language']} | {r['all_n']} / {r['all_valid']} | " |
| + " | ".join(f"{r[str(n)]}" for n in SIZES) + f" | {r['all']} |") |
| lines.append("| **Ortalama** | | | | " + " | ".join(f"**{mean(str(n))}**" for n in SIZES) + f" | **{mean('all')}** |") |
| lines.append("| Tümüne göre ort. mutlak sapma | | | | " + " | ".join(f"{mad(str(n))}" for n in SIZES) + " | 0 |") |
| out = Path(args.out) |
| out.parent.mkdir(parents=True, exist_ok=True) |
| out.write_text("\n".join(lines) + "\n", encoding="utf-8") |
| out.with_suffix(".json").write_text(json.dumps(rows, indent=1)) |
| print("\n".join(lines[-2:]), f"\n→ {out}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
