fix: project evaluator dashboard outcomes

backend/app/routers/calls.py

@@ -279,21 +279,79 @@ def _evaluation_summary(public_id, domain, run=None):
 
     acoustic = presentation.get("acoustic_context") or {}
     action = presentation.get("recommended_action") or {}
+    questions = {
+        item.get("question_id"): item
+        for item in presentation.get("manager_questions") or []
+    }
+
+    def aspect(question_id):
+        item = questions.get(question_id) or {}
+        answer = item.get("answer")
+        if answer == "yes":
+            state = "ok"
+        elif answer in {"partly", "no"}:
+            state = "concern"
+        else:
+            state = "uncertain"
+        return {
+            "state": state,
+            "summary": item.get("summary"),
+            "evidence_ids": item.get("evidence_ids") or [],
+        }
+
+    aspects = {
+        name: aspect(f"call.{name}")
+        for name in ("request", "process", "experience", "outcome")
+    }
+    evaluation_status = (
+        presentation.get("evaluation_status")
+        or decision.get("decision_status")
+        or ("failed" if run_status == "failed" else None)
+    )
+    if available and bool(presentation.get("attention_required")):
+        result = "review"
+    elif available and (
+        evaluation_status != "complete"
+        or any(item["state"] == "uncertain" for item in aspects.values())
+    ):
+        result = "uncertain"
+    elif available:
+        result = "ok"
+    else:
+        result = None
+
+    evidence = [
+        *(presentation.get("evidence") or []),
+        *((presentation.get("details") or {}).get("evidence") or []),
+    ]
     return {
         "analyzed": available,
         "evaluation_available": available,
         "evaluation_supported": evaluator_supported,
         "evaluation_state": state,
-        "evaluation_status": (
-            presentation.get("evaluation_status")
-            or decision.get("decision_status")
-            or ("failed" if run_status == "failed" else None)
-        ),
+        "evaluation_status": evaluation_status,
         "attention_required": (
             bool(presentation.get("attention_required"))
             if available
             else None
         ),
+        "result": result,
+        "aspects": aspects if available else None,
+        "concern_count": (
+            len(presentation.get("primary_reasons") or [])
+            + int(presentation.get("additional_reason_count") or 0)
+            if available
+            else None
+        ),
+        "audio_warning": (
+            any(
+                item.get("kind") == "audio_support"
+                and "reason" in (item.get("purposes") or [])
+                for item in evidence
+            )
+            if available
+            else False
+        ),
         "checklist_counts": checklist_counts,
         "checklist_total": len(checklist),
         "acoustic_status": acoustic.get("status"),

ml-services/evaluation/v2/build_evaluation_data.py

@@ -554,6 +554,7 @@ def _challenge_call(
         facts={
             "request.executes_transaction": True,
             "transfer.executed_during_call": True,
+            "policy.requires_identity_challenge": True,
         },
         acoustic_condition=acoustic,
         segments=segments,

ml-services/evaluation/v2/findings.py

@@ -217,6 +217,26 @@ TEXT_ESCALATION_RULES = (
             r"(?:frustrated|angry|upset)\b",
             r"\bthis\s+is\s+(?:completely\s+|absolutely\s+)?"
             r"unacceptable\b",
+            r"\b(?:this|it)\s+is\s+(?:really\s+|absolutely\s+)?"
+            r"ridiculous\b",
+            r"\bthis\s+is\s+(?:honestly\s+)?(?:a\s+)?nightmare\b",
+            r"\b(?:was|is|has\s+been)\s+stressing\s+me\s+out\s+"
+            r"(?:tremendously|badly)\b",
+        ),
+    ),
+    TextEscalationRule(
+        finding_type="experience.accessibility_barrier",
+        title="Customer reported an access barrier",
+        business_definition=(
+            "The customer explicitly states that a physical-access barrier "
+            "prevents them from using the offered service path."
+        ),
+        severity=FindingSeverity.REVIEW,
+        category=FindingCategory.AGENT_BEHAVIOR,
+        patterns=_patterns(
+            r"\b(?:can't|cannot|unable\s+to)\s+(?:go|visit)\s+"
+            r"(?:to\s+)?(?:the\s+)?(?:bank|branch)\s+physically\b",
+            r"\bi(?:'m|\s+am)\s+(?:currently\s+)?in\s+a\s+wheelchair\b",
         ),
     ),
 )

ml-services/evaluation/v2/presentation.py

@@ -1025,142 +1025,167 @@ def _question_summary(
 
 def _manager_questions(
     decision: CallDecision,
-    acoustic: AcousticContext,
+    _acoustic: AcousticContext,
 ) -> list[ManagerQuestion]:
     all_findings = (
         decision.triggered_findings + decision.positive_findings
     )
-    objective = [
+    request_ids = {
+        "request.intent_confirmed",
+        "loan.purpose_and_stage_confirmed",
+    }
+    request = [
         item
         for item in all_findings
-        if (
-            item.applicability.rule_id == "request.intent_confirmed"
-            or item.category == FindingCategory.OUTCOME
-        )
-    ]
-    objective_negative = [
-        item for item in objective if item.polarity.value == "negative"
-    ]
-    objective_positive = [
-        item for item in objective if item.polarity.value == "positive"
+        if item.applicability.rule_id in request_ids
     ]
-    if objective_negative:
-        objective_answer = ManagerAnswer.PARTLY
-        objective_label = "Partly"
-    elif objective_positive:
-        objective_answer = ManagerAnswer.YES
-        objective_label = "Yes"
-    else:
-        objective_answer = ManagerAnswer.UNCLEAR
-        objective_label = "Unable to determine"
-
-    workflow = [
+    process = [
         item
         for item in all_findings
         if (
             item.detection_rule.detector
             == "structured_requirement_assessment"
-            and item not in objective
+            and item.applicability.rule_id not in request_ids
+            and item.category
+            not in (
+                FindingCategory.OUTCOME,
+                FindingCategory.AGENT_BEHAVIOR,
+            )
         )
     ]
-    workflow_negative = [
-        item for item in workflow if item.polarity.value == "negative"
+    experience = [
+        item
+        for item in all_findings
+        if (
+            item.category == FindingCategory.ESCALATION
+            or item.finding_type.startswith("experience.")
+            or (
+                item.category == FindingCategory.AGENT_BEHAVIOR
+                and item.detection_rule.detector
+                == "structured_requirement_assessment"
+            )
+        )
     ]
-    if any(
-        item.severity.value == "critical"
-        for item in workflow_negative
-    ):
-        workflow_answer = ManagerAnswer.NO
-        workflow_label = "No"
-    elif workflow_negative:
-        workflow_answer = ManagerAnswer.PARTLY
-        workflow_label = "Partly"
-    elif workflow:
-        workflow_answer = ManagerAnswer.YES
-        workflow_label = "Yes"
-    else:
-        workflow_answer = ManagerAnswer.UNCLEAR
-        workflow_label = "Unable to determine"
-
-    friction = [
+    outcome = [
         item
         for item in all_findings
-        if item.category == FindingCategory.ESCALATION
+        if item.category == FindingCategory.OUTCOME
     ]
-    if friction:
-        friction_answer = ManagerAnswer.YES
-        friction_label = "Concern identified"
-    elif acoustic.status == AcousticStatus.AVAILABLE:
-        friction_answer = ManagerAnswer.NO
-        friction_label = "No concern identified"
-    else:
-        friction_answer = ManagerAnswer.UNCLEAR
-        friction_label = "Unable to determine"
 
-    action = decision.recommended_action
-    needs_action = action.action_type != ActionType.NONE
+    def answer(
+        findings: list[Finding],
+        *,
+        clear_when_empty: bool = False,
+    ) -> tuple[ManagerAnswer, str]:
+        negative = [
+            item
+            for item in findings
+            if item.polarity.value == "negative"
+        ]
+        positive = [
+            item
+            for item in findings
+            if item.polarity.value == "positive"
+        ]
+        if any(
+            item.severity.value == "critical"
+            for item in negative
+        ):
+            return ManagerAnswer.NO, "No"
+        if negative:
+            return ManagerAnswer.PARTLY, "Partly"
+        if positive or (
+            clear_when_empty
+            and decision.decision_status == DecisionStatus.COMPLETE
+        ):
+            return ManagerAnswer.YES, "Yes"
+        return ManagerAnswer.UNCLEAR, "Unable to determine"
+
+    request_answer, request_label = answer(request)
+    process_answer, process_label = answer(process)
+    experience_answer, experience_label = answer(
+        experience,
+        clear_when_empty=True,
+    )
+    outcome_answer, outcome_label = answer(outcome)
+
+    request_negative = [
+        item for item in request if item.polarity.value == "negative"
+    ]
+    request_positive = [
+        item for item in request if item.polarity.value == "positive"
+    ]
+    process_negative = [
+        item for item in process if item.polarity.value == "negative"
+    ]
+    experience_negative = [
+        item for item in experience if item.polarity.value == "negative"
+    ]
+    experience_positive = [
+        item for item in experience if item.polarity.value == "positive"
+    ]
+    outcome_negative = [
+        item for item in outcome if item.polarity.value == "negative"
+    ]
+    outcome_positive = [
+        item for item in outcome if item.polarity.value == "positive"
+    ]
     return [
         ManagerQuestion(
-            question_id="call.objective",
-            question="Did the agent understand and address the objective?",
-            answer=objective_answer,
-            answer_label=objective_label,
+            question_id="call.request",
+            question="Was the customer's request understood?",
+            answer=request_answer,
+            answer_label=request_label,
             summary=_question_summary(
-                objective_negative or objective_positive,
-                "The available evidence did not resolve the call objective.",
+                request_negative or request_positive,
+                "The available evidence did not resolve the request.",
             ),
-            evidence_ids=_finding_evidence_ids(objective),
+            evidence_ids=_finding_evidence_ids(request),
         ),
         ManagerQuestion(
-            question_id="call.workflow",
-            question="Did the agent follow the applicable workflow?",
-            answer=workflow_answer,
-            answer_label=workflow_label,
+            question_id="call.process",
+            question="Was the applicable process followed?",
+            answer=process_answer,
+            answer_label=process_label,
             summary=(
                 _question_summary(
-                    workflow_negative,
-                    "No applicable workflow checks were available.",
+                    process_negative,
+                    "No applicable process checks were available.",
                 )
-                if workflow_negative
+                if process_negative
                 else (
-                    f"All {len(workflow)} applicable workflow checks were "
-                    "demonstrated."
-                    if workflow
-                    else "No applicable workflow checks were available."
+                    f"{len(process)} applicable process checks were "
+                    "supported by transcript evidence."
+                    if process
+                    else "No applicable process checks were available."
                 )
             ),
-            evidence_ids=_finding_evidence_ids(workflow),
+            evidence_ids=_finding_evidence_ids(process),
         ),
         ManagerQuestion(
-            question_id="call.friction",
-            question="Did the interaction introduce customer friction?",
-            answer=friction_answer,
-            answer_label=friction_label,
+            question_id="call.experience",
+            question="Was the customer experience handled appropriately?",
+            answer=experience_answer,
+            answer_label=experience_label,
             summary=_question_summary(
-                friction,
-                acoustic.conclusion,
+                experience_negative or experience_positive,
+                (
+                    "No customer-experience concern requiring review was "
+                    "identified in the transcript."
+                ),
             ),
-            evidence_ids=_finding_evidence_ids(friction),
+            evidence_ids=_finding_evidence_ids(experience),
         ),
         ManagerQuestion(
-            question_id="call.follow_up",
-            question="Is follow-up, coaching, or customer action required?",
-            answer=(
-                ManagerAnswer.YES if needs_action else ManagerAnswer.NO
-            ),
-            answer_label="Yes" if needs_action else "No",
-            summary=(
-                action.label
-                if needs_action
-                else "No evidence-backed action was triggered."
-            ),
-            evidence_ids=_finding_evidence_ids(
-                [
-                    item
-                    for item in decision.triggered_findings
-                    if item.finding_id in action.finding_ids
-                ]
+            question_id="call.outcome",
+            question="Was the outcome clear and complete?",
+            answer=outcome_answer,
+            answer_label=outcome_label,
+            summary=_question_summary(
+                outcome_negative or outcome_positive,
+                "The available evidence did not resolve the call outcome.",
             ),
+            evidence_ids=_finding_evidence_ids(outcome),
         ),
     ]
 

ml-services/evaluation/v2/run_shadow_batch.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import argparse
 import json
+import time
 from pathlib import Path
 
 from .runtime import (
@@ -50,7 +51,21 @@ def main() -> int:
         default=[],
         help="Call ID to rerun even when its existing artifact succeeded.",
     )
+    parser.add_argument(
+        "--only-call",
+        action="append",
+        default=[],
+        help="Restrict the batch to one or more call IDs.",
+    )
+    parser.add_argument(
+        "--delay-seconds",
+        type=float,
+        default=0.0,
+        help="Pause between provider calls to respect token rate limits.",
+    )
     args = parser.parse_args()
+    if args.delay_seconds < 0:
+        parser.error("--delay-seconds cannot be negative")
     args.output_dir.mkdir(parents=True, exist_ok=True)
 
     counts: dict[str, int] = {}
@@ -64,7 +79,17 @@ def main() -> int:
     legacy_attention = 0
     v2_attention = 0
     rows = []
-    for transcript_path in sorted(TRANSCRIPT_ROOT.glob("*.json")):
+    transcript_paths = sorted(TRANSCRIPT_ROOT.glob("*.json"))
+    if args.only_call:
+        selected = set(args.only_call)
+        transcript_paths = [
+            path for path in transcript_paths if path.stem in selected
+        ]
+        missing = sorted(selected - {path.stem for path in transcript_paths})
+        if missing:
+            parser.error(f"unknown --only-call values: {missing}")
+
+    for transcript_index, transcript_path in enumerate(transcript_paths):
         call_id = transcript_path.stem
         legacy_path = LEGACY_ROOT / f"{call_id}_graph.json"
         sentiment_path = (
@@ -105,6 +130,11 @@ def main() -> int:
                 json.dumps(run.model_dump(mode="json"), indent=2) + "\n",
                 encoding="utf-8",
             )
+            if (
+                args.delay_seconds
+                and transcript_index < len(transcript_paths) - 1
+            ):
+                time.sleep(args.delay_seconds)
         counts[run.status.value] = counts.get(run.status.value, 0) + 1
         if run.legacy_proxy and run.legacy_proxy.attention_required:
             legacy_attention += 1

ml-services/evaluation/v2/semantic_assessment.py

@@ -24,7 +24,7 @@ from .schemas import (
     SourceProvenance,
 )
 
-SEMANTIC_ASSESSOR_VERSION = "0.2.0"
+SEMANTIC_ASSESSOR_VERSION = "0.3.0"
 _TIERS = ("qa-primary", "qa-fallback", "qa-safety")
 SYSTEM_PROMPT = """You evaluate a customer-service transcript against a fixed
 list of applicable business requirements.
@@ -33,6 +33,25 @@ Rules:
 - Assess every supplied requirement exactly once.
 - Use only the supplied requirement IDs and transcript segment IDs.
 - MET requires direct transcript evidence that demonstrates the requirement.
+- Treat each evidence expectation as a claim that must be supported. A general
+  acknowledgment does not satisfy a requirement whose expectations name
+  specific details.
+- For MET, cite the substantive agent statement and any customer response
+  needed to prove agreement. Do not cite the customer's request as proof that
+  the agent confirmed, explained, authorized, or completed it.
+- A later statement cannot prove that an earlier required authorization
+  occurred. Respect the sequence of the cited segments.
+- If the transcript contains materially conflicting statements about an
+  amount, account, timing, fee, schedule, or outcome, return INCORRECT unless
+  the agent clearly corrects the statement before action and the customer
+  confirms the corrected details.
+- Source and destination accounts, amount, timing or schedule, authorization,
+  and completion are separate requirements. Evidence for one must not be
+  reused as a shortcut for another.
+- Completion requires an explicit statement that the action succeeded or was
+  scheduled. An intention such as "I will do that" is not completion.
+- Outcome and next steps require a closing recap of what happened and any
+  remaining customer action. An earlier explanation alone is insufficient.
 - INCORRECT means the agent directly gave wrong information or performed the
   applicable behavior incorrectly. It requires direct transcript evidence of
   what the agent said or did.