Spaces:
Runtime error
Runtime error
| """Metrique DETERMINISTE de qualite du code genere — sans LLM, donc STABLE d'un run | |
| a l'autre (contrairement au jugement LLM, bruite a n=29). | |
| Pour chaque item generate/fix d'un fichier de reponses, extrait le code, passe le | |
| bug-finder, et compte : erreurs lexicales, commandes inventees (hors catalogue), | |
| non-ASCII, lignes > 80, affectations nues. Agrege en taux de code PROPRE. | |
| Usage : python -m eval.code_quality eval/runs/answers_xxx.json [autre.json ...] | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| from pathlib import Path | |
| from pcp import syntax, bugfinder | |
| def analyze_file(path: Path) -> dict: | |
| data = json.loads(path.read_text(encoding="utf-8")) | |
| items = [it for it in data["items"] if it.get("category") in ("generate", "fix")] | |
| rows = [] | |
| for it in items: | |
| code = "\n".join(syntax.extract_code_blocks(it.get("answer", ""))) | |
| f = bugfinder.analyze(code) if code.strip() else [] | |
| errs = sum(1 for x in f if x.severity == "error") | |
| unk = sum(1 for x in f if "non trouve dans le catalogue" in x.message) | |
| non_ascii = sum(1 for ln in code.splitlines() if any(ord(c) > 127 for c in ln)) | |
| rows.append({"id": it["id"], "has_code": bool(code.strip()), | |
| "errors": errs, "unknown": unk, "non_ascii": non_ascii}) | |
| n = len(rows) | |
| with_code = sum(1 for r in rows if r["has_code"]) | |
| clean = sum(1 for r in rows if r["has_code"] and r["errors"] == 0 and r["unknown"] == 0) | |
| return { | |
| "file": path.name, | |
| "provider": data.get("provider"), "model": data.get("model"), | |
| "n_code_items": n, "with_code": with_code, | |
| "clean_code_pct": round(100 * clean / n, 1) if n else 0.0, | |
| "total_errors": sum(r["errors"] for r in rows), | |
| "total_unknown_cmds": sum(r["unknown"] for r in rows), | |
| "total_non_ascii": sum(r["non_ascii"] for r in rows), | |
| "rows": rows, | |
| } | |
| def main(): | |
| paths = [Path(p) for p in sys.argv[1:]] or sorted(Path("eval/runs").glob("answers_*.json")) | |
| print(f"{'file':32} {'model':28} clean% err unk n-ascii") | |
| for p in paths: | |
| if not p.exists(): | |
| continue | |
| r = analyze_file(p) | |
| print(f"{r['file']:32} {str(r['model'])[:28]:28} {r['clean_code_pct']:5.1f} " | |
| f"{r['total_errors']:3d} {r['total_unknown_cmds']:3d} {r['total_non_ascii']:5d}") | |
| if __name__ == "__main__": | |
| main() | |