quality_eval.py

"""Exercise every built-in genre across every product use case."""

from __future__ import annotations

import re

import app


USE_CASE_MESSAGES = {
    "WhatsApp": "I'm running ten minutes late",
    "Email": "Please send the updated file by Friday",
    "Tweet/Post": "The workshop starts at six",
    "Apology": "Sorry I forgot to reply",
    "Invitation": "Join us for dinner at eight",
    "Excuse": "I need one more day to finish the report",
    "Flirt but safe/non-explicit": "Would you like to get coffee with me",
    "Reminder": "Please bring the charger tomorrow",
    "Complaint": "The order arrived damaged, please replace it",
    "Announcement": "The meeting moved to three",
}

STOP_WORDS = {
    "the",
    "and",
    "but",
    "with",
    "please",
    "would",
    "your",
    "you",
    "this",
    "that",
}


def source_terms(message: str) -> set[str]:
    return {
        word
        for word in re.findall(r"[a-z]{4,}", message.lower())
        if word not in STOP_WORDS
    }


def main() -> None:
    failures: list[str] = []
    outputs_by_use_case: dict[str, set[str]] = {
        use_case: set() for use_case in app.USE_CASES
    }

    for genre in app.GENRES:
        profile = app.GENRE_PROFILES[genre]
        markers = [
            item.lower()
            for item in profile.vocabulary + [profile.signature_phrase]
        ]
        for use_case in app.USE_CASES:
            for intensity in range(1, 6):
                message = USE_CASE_MESSAGES[use_case]
                result = app.deterministic_rewrite(
                    message,
                    genre,
                    "",
                    intensity,
                    use_case,
                )
                output = result.final_message.lower()
                case_id = f"{genre} / {use_case} / intensity {intensity}"

                if not any(term in output for term in source_terms(message)):
                    failures.append(f"{case_id}: practical meaning drifted")
                if not any(marker in output for marker in markers):
                    failures.append(f"{case_id}: no genre marker")
                if any(
                    re.search(rf"\b{re.escape(word.lower())}\b", output)
                    for word in profile.forbidden_words
                ):
                    failures.append(f"{case_id}: used forbidden vocabulary")
                if use_case == "Tweet/Post" and len(result.final_message) > 280:
                    failures.append(f"{case_id}: post exceeds 280 characters")
                if use_case == "WhatsApp" and len(result.final_message) > 520:
                    failures.append(f"{case_id}: chat output is too long")
                if result.final_message == result.short_version:
                    failures.append(f"{case_id}: short version is not distinct")
                outputs_by_use_case[use_case].add(result.final_message)

    for use_case, outputs in outputs_by_use_case.items():
        expected = len(app.GENRES) * 5
        if len(outputs) != expected:
            failures.append(f"{use_case}: genres did not produce distinct outputs")

    total = len(app.GENRES) * len(app.USE_CASES) * 5
    if failures:
        print("\n".join(f"FAIL {failure}" for failure in failures))
        raise SystemExit(f"\nGenre/use-case matrix failed: {len(failures)} issues")
    print(f"GenreGoblin quality matrix: {total}/{total} combinations passed.")


if __name__ == "__main__":
    main()