eval/code_quality.py

"""Metrique DETERMINISTE de qualite du code genere — sans LLM, donc STABLE d'un run
a l'autre (contrairement au jugement LLM, bruite a n=29).

Pour chaque item generate/fix d'un fichier de reponses, extrait le code, passe le
bug-finder, et compte : erreurs lexicales, commandes inventees (hors catalogue),
non-ASCII, lignes > 80, affectations nues. Agrege en taux de code PROPRE.

Usage : python -m eval.code_quality eval/runs/answers_xxx.json [autre.json ...]
"""
from __future__ import annotations

import json
import sys
from pathlib import Path

from pcp import syntax, bugfinder


def analyze_file(path: Path) -> dict:
    data = json.loads(path.read_text(encoding="utf-8"))
    items = [it for it in data["items"] if it.get("category") in ("generate", "fix")]
    rows = []
    for it in items:
        code = "\n".join(syntax.extract_code_blocks(it.get("answer", "")))
        f = bugfinder.analyze(code) if code.strip() else []
        errs = sum(1 for x in f if x.severity == "error")
        unk = sum(1 for x in f if "non trouve dans le catalogue" in x.message)
        non_ascii = sum(1 for ln in code.splitlines() if any(ord(c) > 127 for c in ln))
        rows.append({"id": it["id"], "has_code": bool(code.strip()),
                     "errors": errs, "unknown": unk, "non_ascii": non_ascii})
    n = len(rows)
    with_code = sum(1 for r in rows if r["has_code"])
    clean = sum(1 for r in rows if r["has_code"] and r["errors"] == 0 and r["unknown"] == 0)
    return {
        "file": path.name,
        "provider": data.get("provider"), "model": data.get("model"),
        "n_code_items": n, "with_code": with_code,
        "clean_code_pct": round(100 * clean / n, 1) if n else 0.0,
        "total_errors": sum(r["errors"] for r in rows),
        "total_unknown_cmds": sum(r["unknown"] for r in rows),
        "total_non_ascii": sum(r["non_ascii"] for r in rows),
        "rows": rows,
    }


def main():
    paths = [Path(p) for p in sys.argv[1:]] or sorted(Path("eval/runs").glob("answers_*.json"))
    print(f"{'file':32} {'model':28} clean%  err  unk  n-ascii")
    for p in paths:
        if not p.exists():
            continue
        r = analyze_file(p)
        print(f"{r['file']:32} {str(r['model'])[:28]:28} {r['clean_code_pct']:5.1f}  "
              f"{r['total_errors']:3d}  {r['total_unknown_cmds']:3d}  {r['total_non_ascii']:5d}")


if __name__ == "__main__":
    main()