File size: 5,306 Bytes
9438a57 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | #!/usr/bin/env python3
"""Klasik (arama tabanlı) taban çizgisi: Pynguin ile Python öğeleri için test üretip ajanla AYNI ölçüm
hattından (kapsama, mutasyon, yerleştirilmiş hatalar) geçirir.
PYNGUIN=/yol/pynguin python scripts/pynguin_baseline.py --seeds 1 2 3 --budget 120
python scripts/pynguin_baseline.py --measure-only # üretilmiş testleri yeniden ölç
Pynguin Python 3.10 ister ve PYNGUIN_DANGER_AWARE ortam değişkeniyle çalışır (kodu çalıştırır).
Üretilen testler benchmarks/baselines/pynguin/<öğe>_s<tohum>.py olarak saklanır.
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
from app.eval import benchmark # noqa: E402
from app.eval.report import render # noqa: E402
from app.llm.scripted import ScriptedLLM, text, tool # noqa: E402
OUT = ROOT / "benchmarks" / "baselines" / "pynguin"
def generate(item: dict, seed: int, budget: int, pynguin: str) -> dict:
with tempfile.TemporaryDirectory() as tmp:
proj, out = Path(tmp) / "proj", Path(tmp) / "out"
proj.mkdir()
(proj / "solution.py").write_text(item["code"], encoding="utf-8")
t0 = time.time()
p = subprocess.run([pynguin, "--project-path", str(proj), "--module-name", "solution",
"--output-path", str(out), "--maximum-search-time", str(budget), "--seed", str(seed)],
capture_output=True, text=True, env=os.environ | {"PYNGUIN_DANGER_AWARE": "1"},
timeout=budget * 4 + 300)
dur = round(time.time() - t0, 1)
gen = out / "test_solution.py"
if not gen.exists():
return {"item": item["id"], "seed": seed, "ok": False, "duration_s": dur, "log": p.stdout[-1500:]}
dest = OUT / f"{item['id']}_s{seed}.py"
shutil.copy(gen, dest)
return {"item": item["id"], "seed": seed, "ok": True, "duration_s": dur, "file": dest.name}
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--seeds", type=int, nargs="*", default=[1, 2, 3])
ap.add_argument("--budget", type=int, default=120, help="Pynguin arama süresi (sn)")
ap.add_argument("--max-mutants", type=int, default=30)
ap.add_argument("--measure-only", action="store_true")
ap.add_argument("--split", choices=["dev", "test", "bagimsiz", "all"], default="all")
ap.add_argument("--out", default=str(ROOT / "reports" / "PYNGUIN_TABAN_CIZGISI.md"))
args = ap.parse_args()
items = [i for i in benchmark.load_manifest() if i["language"] == "python"
and (args.split == "all" or i.get("split", "dev") == args.split)]
OUT.mkdir(parents=True, exist_ok=True)
meta_path = OUT / "generation.json"
meta = json.loads(meta_path.read_text()) if meta_path.exists() else []
if not args.measure_only:
pynguin = os.environ.get("PYNGUIN", "pynguin")
for seed in args.seeds:
for it in items:
print(f" üretiliyor: {it['id']} tohum={seed} …", flush=True)
r = generate(it, seed, args.budget, pynguin)
meta = [m for m in meta if not (m["item"] == r["item"] and m["seed"] == seed)] + [r]
meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=1))
print(f" {'tamam' if r['ok'] else 'BAŞARISIZ'} ({r['duration_s']} sn)", flush=True)
evals = []
for seed in args.seeds:
files = {it["id"]: OUT / f"{it['id']}_s{seed}.py" for it in items}
ids = [k for k, f in files.items() if f.exists()]
def factory(item, files=files):
return ScriptedLLM([text('{"summary": "Pynguin taban çizgisi"}'),
tool("write_tests", code=files[item["id"]].read_text(encoding="utf-8")),
tool("finish", summary="pynguin")], model="pynguin")
cfg = {"max_iterations": 1, "max_mutants": args.max_mutants, "coverage_target": 85, "run_mutation": True}
ev = benchmark.run_eval(f"Pynguin 0.47 (tohum {seed}, {args.budget} sn)", cfg, ids, None, background=False,
llm_factory=factory, label=f"pynguin-s{seed}")
for it in ev.items:
print(f" s{seed} {it.id:<14} geçen=%{it.final_pass_rate} kapsama={it.final_coverage} "
f"mutasyon={it.mutation_score} yerleştirilmiş={it.seeded_caught}/{it.seeded_total} test={it.tests}")
evals.append(ev)
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(render(evals, title="Pynguin (arama tabanlı) taban çizgisi",
note="Pynguin 0.47.0, Python 3.10; varsayılan DynaMOSA algoritması ve varsayılan "
"doğrulama üretimi. Testler ajanla aynı ölçüm hattında değerlendirilmiştir."),
encoding="utf-8")
out.with_suffix(".json").write_text(json.dumps([e.model_dump() for e in evals], ensure_ascii=False, indent=2))
print(f"Rapor: {out}")
if __name__ == "__main__":
main()
|