import json import re from typing import Dict, List from openai_client import llm_text def _safe_parse_json(text: str) -> Dict: s = (text or "").strip() first = s.find("{") last = s.rfind("}") if first == -1 or last == -1 or last <= first: return {} try: return json.loads(s[first : last + 1]) except Exception: return {} def _heuristic_score(question: str) -> Dict: text = (question or "").strip().lower() if not text: return {"quality_score": 0, "quality_rationale": "Empty question."} score = 1 rationale_bits: List[str] = [] if any(word in text for word in ["should", "must", "can", "when", "if", "before"]): score += 1 rationale_bits.append("has decision/action framing") if any(word in text for word in ["grievance", "discipline", "termination", "compliance", "timeline", "step"]): score += 1 rationale_bits.append("touches grievance/compliance risk") if len(re.findall(r"\b\w+\b", text)) >= 12: score += 1 rationale_bits.append("has useful specificity") if "supervisor" in text or "manager" in text: score += 1 rationale_bits.append("explicit supervisor orientation") score = max(0, min(5, score)) rationale = ", ".join(rationale_bits) if rationale_bits else "basic lexical quality heuristic" return {"quality_score": score, "quality_rationale": rationale} def score_question_quality( question: str, domain: str, mode: str, chunk_citations: List[Dict], model: str = "gpt-4.1-mini", ) -> Dict: excerpts = [] for c in (chunk_citations or [])[:4]: excerpt = (c.get("text") or c.get("text_excerpt") or "").strip() if excerpt: excerpts.append( { "chunk_id": c.get("chunk_id"), "article": c.get("article"), "section": c.get("section"), "text_excerpt": excerpt[:500], } ) prompt = f""" Score this candidate supervisory question from 0 to 5. Rubric (integer score): - realism for a UPS supervisor scenario - specificity to contract-governed actions - decision/action orientation for supervisors - grievance/compliance risk relevance - clarity and answerability Return JSON only: {{ "quality_score": 0, "quality_rationale": "one short sentence" }} Candidate: - Domain: {domain} - Mode: {mode} - Question: {question} - Supporting excerpts: {json.dumps(excerpts, ensure_ascii=False)} """.strip() try: raw = llm_text(prompt, model=model) parsed = _safe_parse_json(raw) score = int(parsed.get("quality_score")) rationale = (parsed.get("quality_rationale") or "").strip() if score < 0 or score > 5: raise ValueError("score out of range") if not rationale: raise ValueError("missing rationale") return {"quality_score": score, "quality_rationale": rationale} except Exception: return _heuristic_score(question)