evaluate.py

"""Run a compact, reproducible GenreGoblin quality evaluation."""

from __future__ import annotations

import json
from pathlib import Path

import app


def main() -> None:
    cases = json.loads(Path("eval_cases.json").read_text(encoding="utf-8"))
    passed = 0

    for case in cases:
        result, mode, resolved = app.rewrite_engine(
            case["message"],
            case["genre"],
            "",
            case["intensity"],
            case["use_case"],
            [],
        )
        output = result.final_message.lower()
        checks = {
            "nonempty": bool(result.final_message),
            "genre_resolved": resolved == case["genre"],
            "style_marker": any(marker in output for marker in case["must_contain_any"]),
            "scores_valid": (
                0 <= result.sendability_score <= 100
                and 0 <= result.chaos_score <= 100
            ),
            "trace_complete": len(result.agent_trace) >= 6,
            "shorter_version": len(result.short_version) <= len(result.final_message) + 80,
        }
        case_passed = all(checks.values())
        passed += int(case_passed)
        status = "PASS" if case_passed else "FAIL"
        print(f"{status} {case['id']} [{mode}] {checks}")

    unsafe = app.deterministic_rewrite(
        "I will kill you",
        "Shakespeare Villain",
        "",
        5,
        "Complaint",
    )
    safety_passed = (
        unsafe.probably_safe_to_send == "no"
        and "kill you" not in unsafe.final_message.lower()
    )
    print(f"{'PASS' if safety_passed else 'FAIL'} unsafe-boundary-conversion")

    total = len(cases) + 1
    passed += int(safety_passed)
    print(f"\nGenreGoblin evaluation: {passed}/{total} checks passed.")
    raise SystemExit(0 if passed == total else 1)


if __name__ == "__main__":
    main()