scripts/evaluate_demo_quality.py

import json
from pathlib import Path
from typing import Iterable


def evaluate_text(text: str, required_terms: Iterable[str], banned_terms: Iterable[str]) -> dict:
    lowered = text.lower()
    issues: list[str] = []
    for term in required_terms:
        if term.lower() not in lowered:
            issues.append(f"missing required term: {term}")
    for term in banned_terms:
        if term.lower() in lowered:
            issues.append(f"banned term: {term}")
    if any(marker in lowered for marker in ["you have depression", "this means you will", "certainly predicts"]):
        issues.append("unsafe certainty or diagnosis")
    return {"passes": not issues, "issues": issues}


def load_cases(path: str = "tests/fixtures/demo_eval_cases.json") -> list[dict]:
    return json.loads(Path(path).read_text(encoding="utf-8"))


def case_source_text(case: dict) -> str:
    return " ".join(
        part
        for part in [
            str(case.get("dream_text", "")),
            str(case.get("voice_transcript", "")),
            " ".join(str(clue) for clue in case.get("visual_clues", [])),
        ]
        if part.strip()
    )


def evaluate_case(case: dict) -> dict:
    result = evaluate_text(
        case_source_text(case),
        required_terms=case.get("required_terms", []),
        banned_terms=case.get("banned_terms", []),
    )
    return {"id": case.get("id", ""), **result}


def evaluate_cases(cases: list[dict]) -> dict:
    results = [evaluate_case(case) for case in cases]
    failures = [result for result in results if not result["passes"]]
    return {
        "case_count": len(cases),
        "ids": [case["id"] for case in cases],
        "failures": failures,
        "passes": not failures,
    }


def main() -> int:
    cases = load_cases()
    report = evaluate_cases(cases)
    print(json.dumps(report, indent=2))
    return 0 if report["passes"] else 1


if __name__ == "__main__":
    raise SystemExit(main())