question_quality.py

import json
import re
from typing import Dict, List

from openai_client import llm_text


def _safe_parse_json(text: str) -> Dict:
    s = (text or "").strip()
    first = s.find("{")
    last = s.rfind("}")
    if first == -1 or last == -1 or last <= first:
        return {}
    try:
        return json.loads(s[first : last + 1])
    except Exception:
        return {}


def _heuristic_score(question: str) -> Dict:
    text = (question or "").strip().lower()
    if not text:
        return {"quality_score": 0, "quality_rationale": "Empty question."}

    score = 1
    rationale_bits: List[str] = []

    if any(word in text for word in ["should", "must", "can", "when", "if", "before"]):
        score += 1
        rationale_bits.append("has decision/action framing")
    if any(word in text for word in ["grievance", "discipline", "termination", "compliance", "timeline", "step"]):
        score += 1
        rationale_bits.append("touches grievance/compliance risk")
    if len(re.findall(r"\b\w+\b", text)) >= 12:
        score += 1
        rationale_bits.append("has useful specificity")
    if "supervisor" in text or "manager" in text:
        score += 1
        rationale_bits.append("explicit supervisor orientation")

    score = max(0, min(5, score))
    rationale = ", ".join(rationale_bits) if rationale_bits else "basic lexical quality heuristic"
    return {"quality_score": score, "quality_rationale": rationale}


def score_question_quality(
    question: str,
    domain: str,
    mode: str,
    chunk_citations: List[Dict],
    model: str = "gpt-4.1-mini",
) -> Dict:
    excerpts = []
    for c in (chunk_citations or [])[:4]:
        excerpt = (c.get("text") or c.get("text_excerpt") or "").strip()
        if excerpt:
            excerpts.append(
                {
                    "chunk_id": c.get("chunk_id"),
                    "article": c.get("article"),
                    "section": c.get("section"),
                    "text_excerpt": excerpt[:500],
                }
            )

    prompt = f"""
Score this candidate supervisory question from 0 to 5.

Rubric (integer score):
- realism for a UPS supervisor scenario
- specificity to contract-governed actions
- decision/action orientation for supervisors
- grievance/compliance risk relevance
- clarity and answerability

Return JSON only:
{{
  "quality_score": 0,
  "quality_rationale": "one short sentence"
}}

Candidate:
- Domain: {domain}
- Mode: {mode}
- Question: {question}
- Supporting excerpts: {json.dumps(excerpts, ensure_ascii=False)}
""".strip()

    try:
        raw = llm_text(prompt, model=model)
        parsed = _safe_parse_json(raw)
        score = int(parsed.get("quality_score"))
        rationale = (parsed.get("quality_rationale") or "").strip()
        if score < 0 or score > 5:
            raise ValueError("score out of range")
        if not rationale:
            raise ValueError("missing rationale")
        return {"quality_score": score, "quality_rationale": rationale}
    except Exception:
        return _heuristic_score(question)