| """ |
| Grading module for LedgerShield benchmark. |
| |
| Implements the scoring rubric for all five task families (A–E). |
| Each task type has a weighted multi-dimensional rubric covering: |
| |
| - **Extraction accuracy**: field matching, line-item alignment |
| - **Decision correctness**: binary decision + reason codes |
| - **Evidence quality**: document localization, token overlap |
| - **Investigation thoroughness**: required tool coverage |
| - **Intervention appropriateness**: escalation path correctness |
| - **Process efficiency**: budget usage, tool repetition |
| - **Calibration**: confidence vs. correctness alignment |
| - **Counterfactual reasoning**: semantic multi-dimensional rubric (Phase 2.2) |
| |
| Degenerate Submission Penalties (Phase 2.3): |
| - Intervention base score tightened from 0.35 → 0.15 |
| - Empty evidence capped at DEGENERATE_EVIDENCE_CAP (0.25) |
| - Minimal-effort submissions penalized across all dimensions |
| |
| Score Constants (Phase 4.5): |
| TASK_SCORE_MIN = 0.01 |
| TASK_SCORE_MAX = 0.99 |
| DEGENERATE_EVIDENCE_CAP = 0.25 |
| """ |
|
|
| from __future__ import annotations |
|
|
| import re |
| from typing import Any |
|
|
| from .benchmark_contract import CERTIFICATE_REQUIRED_TRACK, RESULT_CLASSES, normalize_track |
| from .causal_grader import causal_grade_adjustment, grade_causal_consistency |
| from .compliance_engine import ComplianceResult, compliance_penalty, evaluate_compliance |
| from .currency_engine import validate_iban, validate_swift |
| from .decision_certificate import certificate_score_adjustment, verify_decision_certificate |
| from .decision_falsifier import falsify_decision |
| from .proper_scoring import ( |
| brier_score as proper_brier_score, |
| composite_proper_score, |
| logarithmic_score as proper_logarithmic_score, |
| penalized_brier_score as proper_penalized_brier_score, |
| resolve_predicted_probabilities, |
| ) |
| from .schema import ( |
| bbox_iou, |
| canonical_reason_codes, |
| normalize_id, |
| normalize_text, |
| numeric_match, |
| token_overlap, |
| ) |
| from .sprt_engine import DEFAULT_HYPOTHESES, latent_hypothesis_from_case |
| from .trust_graph import evaluate_trust_graph_projection |
| from .vendor_simulator import get_callback_grading_weight |
| from .trajectory_grading import ( |
| downstream_outcome_score, |
| efficiency_score, |
| intervention_score, |
| investigation_score, |
| resolution_state_score, |
| ) |
|
|
| |
| TASK_SCORE_MIN = 0.01 |
| TASK_SCORE_MAX = 0.99 |
| DEGENERATE_EVIDENCE_CAP = 0.25 |
| TASK_E_DEGENERATE_EVIDENCE_CAP = 0.10 |
| COMPLIANCE_ADJUSTMENT_WEIGHT = 0.05 |
| CURRENCY_ADJUSTMENT_WEIGHT = 0.03 |
| TASK_E_LINK_GATE_THRESHOLD = 0.85 |
|
|
|
|
| def strict_task_score(value: float) -> float: |
| """Clamp a score to the valid task score range. |
| |
| Args: |
| value: Raw score value. |
| |
| Returns: |
| Clamped score in [TASK_SCORE_MIN, TASK_SCORE_MAX]. |
| """ |
| return round(max(TASK_SCORE_MIN, min(TASK_SCORE_MAX, float(value))), 4) |
|
|
|
|
| def exact_or_numeric_match(pred_value: Any, gold_value: Any) -> bool: |
| """Check if predicted value matches gold via exact or numeric comparison. |
| |
| Args: |
| pred_value: Predicted value from submission. |
| gold_value: Gold-standard value. |
| |
| Returns: |
| True if values match. |
| """ |
| if isinstance(gold_value, (int, float)): |
| return numeric_match(pred_value, gold_value) |
| if normalize_id(pred_value) == normalize_id(gold_value): |
| return True |
| return normalize_text(pred_value) == normalize_text(gold_value) |
|
|
|
|
| def field_score(pred: dict[str, Any], gold: dict[str, Any]) -> float: |
| """Score extracted fields against gold standard. |
| |
| Args: |
| pred: Predicted fields dict. |
| gold: Gold-standard fields dict. |
| |
| Returns: |
| Score from 0.0 to 1.0. |
| """ |
| if not gold: |
| return 1.0 |
| hits = 0.0 |
| for key, gold_value in gold.items(): |
| if exact_or_numeric_match(pred.get(key), gold_value): |
| hits += 1.0 |
| return hits / max(len(gold), 1) |
|
|
|
|
| def _line_pair_score(pred: dict[str, Any], gold: dict[str, Any]) -> float: |
| """Score a single predicted line item against a gold line item.""" |
| checks = [ |
| normalize_text(pred.get("description")) == normalize_text(gold.get("description")), |
| numeric_match(pred.get("qty"), gold.get("qty")), |
| numeric_match(pred.get("unit_price"), gold.get("unit_price")), |
| numeric_match(pred.get("line_total"), gold.get("line_total")), |
| ] |
| return sum(float(x) for x in checks) / len(checks) |
|
|
|
|
| def line_item_score(pred_lines: list[dict[str, Any]], gold_lines: list[dict[str, Any]]) -> float: |
| """Score predicted line items against gold using greedy matching. |
| |
| Args: |
| pred_lines: List of predicted line item dicts. |
| gold_lines: List of gold-standard line item dicts. |
| |
| Returns: |
| Score from 0.0 to 1.0. |
| """ |
| if not pred_lines and not gold_lines: |
| return 1.0 |
| if not pred_lines or not gold_lines: |
| return 0.0 |
|
|
| unmatched = list(range(len(gold_lines))) |
| total = 0.0 |
|
|
| for pred in pred_lines: |
| best_idx = None |
| best_score = -1.0 |
| for idx in unmatched: |
| score = _line_pair_score(pred, gold_lines[idx]) |
| if score > best_score: |
| best_idx = idx |
| best_score = score |
| if best_idx is not None: |
| unmatched.remove(best_idx) |
| total += best_score |
|
|
| denom = max(len(pred_lines), len(gold_lines)) |
| return total / denom |
|
|
|
|
| def list_f1(pred: list[str], gold: list[str]) -> float: |
| """Compute F1 score between predicted and gold string lists. |
| |
| Args: |
| pred: Predicted string list. |
| gold: Gold-standard string list. |
| |
| Returns: |
| F1 score from 0.0 to 1.0. |
| """ |
| pred_set = {normalize_text(x) for x in pred if normalize_text(x)} |
| gold_set = {normalize_text(x) for x in gold if normalize_text(x)} |
|
|
| if not pred_set and not gold_set: |
| return 1.0 |
| if not pred_set or not gold_set: |
| return 0.0 |
|
|
| true_pos = len(pred_set & gold_set) |
| precision = true_pos / len(pred_set) |
| recall = true_pos / len(gold_set) |
|
|
| if precision + recall == 0: |
| return 0.0 |
| return 2 * precision * recall / (precision + recall) |
|
|
|
|
| from .evidence_graph import EvidenceGraph |
|
|
| def _single_evidence_score(pred_ref: dict[str, Any], gold_ref: dict[str, Any]) -> float: |
| """Score a single evidence reference against gold.""" |
| if not pred_ref or not gold_ref: |
| return 0.0 |
|
|
| doc_match = normalize_text(pred_ref.get("doc_id")) == normalize_text(gold_ref.get("doc_id")) |
| page_match = int(pred_ref.get("page", 0) or 0) == int(gold_ref.get("page", 0) or 0) |
| iou = bbox_iou(pred_ref.get("bbox"), gold_ref.get("bbox")) |
| tok = token_overlap(pred_ref.get("token_ids"), gold_ref.get("token_ids")) |
|
|
| return 0.35 * float(doc_match) + 0.15 * float(page_match) + 0.30 * iou + 0.20 * tok |
|
|
|
|
| def evidence_score( |
| pred_map: dict[str, Any], |
| gold_map: dict[str, Any], |
| *, |
| empty_cap: float = DEGENERATE_EVIDENCE_CAP, |
| graph_state: dict[str, Any] | None = None, |
| ) -> float: |
| """Score evidence map against gold standard (Graph-Aware / Exact Grounding). |
| |
| Applies DEGENERATE_EVIDENCE_CAP for empty submissions (Phase 2.3). |
| Evaluates exact node grounding if graph_state is provided (Phase 2.1). |
| """ |
| if not gold_map and not graph_state: |
| return 1.0 |
|
|
| if not pred_map or (isinstance(pred_map, dict) and len(pred_map) == 0): |
| return empty_cap |
|
|
| base_scores = [] |
| if gold_map: |
| for key, gold_ref in gold_map.items(): |
| pred_ref = pred_map.get(key) if isinstance(pred_map, dict) else None |
| base_scores.append(_single_evidence_score(pred_ref or {}, gold_ref or {})) |
| |
| score = sum(base_scores) / max(len(base_scores), 1) if base_scores else 0.0 |
|
|
| |
| if graph_state: |
| graph = EvidenceGraph.deserialize(graph_state) |
| cited_docs = {normalize_text(v.get("doc_id")) for v in pred_map.values() if isinstance(v, dict)} |
| |
| critical_nodes = [ |
| n.node_id for n in graph.nodes.values() |
| if n.node_type in {"intervention_result", "duplicate_report", "evidence_doc"} and n.revealed |
| ] |
| |
| if critical_nodes: |
| hits = sum(1 for node_id in critical_nodes if normalize_text(node_id) in cited_docs) |
| grounding_bonus = 0.20 * (hits / len(critical_nodes)) |
| score = min(1.0, score + grounding_bonus) |
| |
| return score |
|
|
|
|
| def policy_score(pred: dict[str, str], gold: dict[str, str]) -> float: |
| """Score policy check predictions against gold. |
| |
| Args: |
| pred: Predicted policy checks dict. |
| gold: Gold-standard policy checks dict. |
| |
| Returns: |
| Score from 0.0 to 1.0. |
| """ |
| if not gold: |
| return 1.0 |
| hits = 0.0 |
| for key, gold_value in gold.items(): |
| if normalize_text(pred.get(key)) == normalize_text(gold_value): |
| hits += 1.0 |
| return hits / max(len(gold), 1) |
|
|
|
|
| def decision_score(pred: str, gold: str) -> float: |
| """Binary match between predicted and gold decision. |
| |
| Args: |
| pred: Predicted decision string. |
| gold: Gold-standard decision string. |
| |
| Returns: |
| 1.0 if match, 0.0 otherwise. |
| """ |
| return float(normalize_text(pred) == normalize_text(gold)) |
|
|
|
|
| def counterfactual_score(counterfactual: str, graph_state: dict[str, Any] | None = None) -> float: |
| """Multi-dimensional semantic counterfactual scoring (Phase 2.2). |
| |
| Evaluates counterfactual reasoning across dimensions and edge citations. |
| """ |
| text = normalize_text(counterfactual) |
| if not text or len(text.split()) < 3: |
| return 0.0 |
|
|
| dimensions: dict[str, float] = {} |
|
|
| |
| structure_markers = {"if", "then", "would", "had", "without", "instead", |
| "alternatively", "otherwise", "hypothetically", |
| "assuming", "suppose", "given that", "in the event"} |
| words = set(text.split()) |
| marker_hits = len(words & structure_markers) |
| dimensions["structure"] = min(1.0, marker_hits / 2.0) |
|
|
| |
| decision_terms = {"pay", "hold", "escalate", "fraud", "risk", "approve", |
| "reject", "block", "flag", "investigate", "review", |
| "suspicious", "legitimate", "verified", "safe", "unsafe"} |
| decision_hits = len(words & decision_terms) |
| dimensions["decision_language"] = min(1.0, decision_hits / 2.0) |
|
|
| |
| evidence_terms = {"invoice", "vendor", "bank", "account", "receipt", "po", |
| "ledger", "email", "callback", "document", "iban", "swift", |
| "amount", "threshold", "duplicate", "mismatch"} |
| evidence_hits = len(words & evidence_terms) |
| dimensions["evidence_specificity"] = min(1.0, evidence_hits / 3.0) |
|
|
| |
| word_count = len(text.split()) |
| if word_count >= 20: |
| dimensions["depth"] = 1.0 |
| elif word_count >= 12: |
| dimensions["depth"] = 0.7 |
| elif word_count >= 6: |
| dimensions["depth"] = 0.4 |
| else: |
| dimensions["depth"] = 0.1 |
|
|
| |
| edge_citations = 0.0 |
| if graph_state: |
| from .evidence_graph import EvidenceGraph |
| graph = EvidenceGraph.deserialize(graph_state) |
| for edge in graph.edges: |
| relation_markers = edge.relation.split("_") |
| if any(marker in text for marker in relation_markers if len(marker) >= 4): |
| edge_citations += 1.0 |
| dimensions["edge_citations"] = min(1.0, edge_citations / max(1.0, len(graph.edges))) |
|
|
| |
| if "edge_citations" in dimensions: |
| weighted = ( |
| 0.20 * dimensions["structure"] |
| + 0.20 * dimensions["decision_language"] |
| + 0.25 * dimensions["evidence_specificity"] |
| + 0.10 * dimensions["depth"] |
| + 0.25 * dimensions["edge_citations"] |
| ) |
| else: |
| weighted = ( |
| 0.30 * dimensions["structure"] |
| + 0.25 * dimensions["decision_language"] |
| + 0.25 * dimensions["evidence_specificity"] |
| + 0.20 * dimensions["depth"] |
| ) |
| return max(0.0, min(1.0, weighted)) |
|
|
|
|
| def fraud_score(pred: list[str], gold: list[str]) -> float: |
| """Score fraud flag predictions with missed-flag penalty. |
| |
| Args: |
| pred: Predicted fraud flags. |
| gold: Gold-standard fraud flags. |
| |
| Returns: |
| Score from 0.0 to 1.0. |
| """ |
| base = list_f1(pred, gold) |
| missed = {normalize_text(x) for x in gold} - {normalize_text(x) for x in pred} |
| if missed: |
| base -= 0.20 * len(missed) |
| return max(0.0, base) |
|
|
|
|
| def duplicate_score(pred: list[str], gold: list[str]) -> float: |
| """Score duplicate link predictions. |
| |
| Args: |
| pred: Predicted duplicate links. |
| gold: Gold-standard duplicate links. |
| |
| Returns: |
| F1 score from 0.0 to 1.0. |
| """ |
| return list_f1(pred, gold) |
|
|
|
|
| def _normalize_doc_id(value: Any) -> str: |
| return re.sub(r"\s+", "", str(value or "")).upper() |
|
|
|
|
| def _numeric_variants(value: float) -> set[str]: |
| rounded = round(float(value), 2) |
| whole = int(rounded) |
| return { |
| f"{rounded:.2f}", |
| f"{rounded:.1f}", |
| f"{rounded:.0f}", |
| f"{rounded:,.2f}", |
| f"{rounded:,.0f}", |
| str(whole), |
| } |
|
|
|
|
| def _doc_total_from_case(case_context: dict[str, Any] | None, doc_id: str) -> float | None: |
| if not case_context: |
| return None |
| target = _normalize_doc_id(doc_id) |
| for doc in case_context.get("documents", []) or []: |
| if _normalize_doc_id(doc.get("doc_id")) != target: |
| continue |
| for token in doc.get("accurate_ocr", []) or []: |
| text = str(token.get("text", "")).strip() |
| match = re.match(r"total\s*:\s*([\d,]+(?:\.\d+)?)$", text, flags=re.IGNORECASE) |
| if match: |
| try: |
| return float(match.group(1).replace(",", "")) |
| except ValueError: |
| return None |
| return None |
|
|
|
|
| def task_e_cross_invoice_link_score( |
| pred_links: list[str], |
| gold_links: list[str], |
| ) -> tuple[float, dict[str, int]]: |
| pred_set = {_normalize_doc_id(link) for link in pred_links if _normalize_doc_id(link)} |
| gold_set = {_normalize_doc_id(link) for link in gold_links if _normalize_doc_id(link)} |
|
|
| if not pred_set and not gold_set: |
| return 1.0, {"matched_links": 0, "gold_links": 0, "pred_links": 0} |
| if not gold_set: |
| return 1.0, {"matched_links": 0, "gold_links": 0, "pred_links": len(pred_set)} |
|
|
| matched = len(pred_set & gold_set) |
| precision = matched / max(len(pred_set), 1) |
| recall = matched / max(len(gold_set), 1) |
| if precision + recall == 0: |
| score = 0.0 |
| else: |
| score = 2 * precision * recall / (precision + recall) |
| return score, { |
| "matched_links": matched, |
| "gold_links": len(gold_set), |
| "pred_links": len(pred_set), |
| } |
|
|
|
|
| def task_e_counterfactual_score( |
| counterfactual: str, |
| gold: dict[str, Any], |
| case_context: dict[str, Any] | None, |
| ) -> tuple[float, dict[str, int]]: |
| base = counterfactual_score(counterfactual) |
| text = str(counterfactual or "") |
| normalized_text = normalize_text(text) |
| if not normalized_text: |
| return 0.0, {"doc_refs": 0, "amount_refs": 0, "required_links": 0} |
|
|
| gold_links = [ |
| str(link) |
| for link in (gold.get("cross_invoice_links", []) or gold.get("duplicate_links", []) or []) |
| if str(link).strip() |
| ] |
| if not gold_links: |
| return base, {"doc_refs": 0, "amount_refs": 0, "required_links": 0} |
|
|
| doc_refs = sum(1 for link in gold_links if link in text) |
| amount_refs = 0 |
| for link in gold_links: |
| total = _doc_total_from_case(case_context, link) |
| if total is None: |
| continue |
| if any(variant in text for variant in _numeric_variants(total)): |
| amount_refs += 1 |
|
|
| required = len(gold_links) |
| doc_specificity = doc_refs / max(required, 1) |
| amount_specificity = amount_refs / max(required, 1) |
| score = ( |
| 0.35 * base |
| + 0.40 * doc_specificity |
| + 0.25 * amount_specificity |
| ) |
| return max(0.0, min(1.0, score)), { |
| "doc_refs": doc_refs, |
| "amount_refs": amount_refs, |
| "required_links": required, |
| } |
|
|
|
|
| def currency_validation_score( |
| task_type: str, |
| submitted: dict[str, Any], |
| gold: dict[str, Any], |
| ) -> tuple[float, dict[str, Any]]: |
| task_norm = normalize_text(task_type) |
| if task_norm != "task_a": |
| return 1.0, {"applicable": False} |
|
|
| extracted = submitted.get("extracted_fields", {}) or {} |
| gold_fields = gold.get("fields", {}) or {} |
| bank_account = str(extracted.get("bank_account", "") or "").strip() |
| currency = str(extracted.get("currency", "") or "").strip().upper() |
| expected_bank = str(gold_fields.get("bank_account", "") or "").strip() |
| expected_currency = str(gold_fields.get("currency", "") or "").strip().upper() |
|
|
| checks: list[float] = [] |
| metadata: dict[str, Any] = {"applicable": True, "format": "unknown"} |
| if expected_currency: |
| checks.append(float(currency == expected_currency)) |
| metadata["expected_currency"] = expected_currency |
| metadata["submitted_currency"] = currency |
|
|
| if expected_bank: |
| checks.append(float(normalize_text(bank_account) == normalize_text(expected_bank))) |
| compact_bank = re.sub(r"\s+", "", bank_account).upper() |
| compact_expected = re.sub(r"\s+", "", expected_bank).upper() |
| if compact_expected.startswith("IBAN:"): |
| compact_expected = compact_expected.split(":", 1)[-1].strip() |
| if compact_expected.startswith("SWIFT:"): |
| compact_expected = compact_expected.split(":", 1)[-1].strip() |
|
|
| if compact_expected[:2].isalpha() and len(compact_expected) >= 15: |
| metadata["format"] = "iban" |
| metadata["validation"] = validate_iban(bank_account) |
| checks.append(float(metadata["validation"].get("valid", False))) |
| elif len(compact_expected) in {8, 11} and compact_expected[:4].isalpha(): |
| metadata["format"] = "swift" |
| metadata["validation"] = validate_swift(bank_account) |
| checks.append(float(metadata["validation"].get("valid", False))) |
|
|
| metadata["expected_bank_account"] = expected_bank |
| metadata["submitted_bank_account"] = bank_account |
|
|
| if not checks: |
| return 1.0, {"applicable": False} |
| return sum(checks) / len(checks), metadata |
|
|
|
|
| def compliance_adjustment_for( |
| result: ComplianceResult | None, |
| ) -> tuple[float, float]: |
| if result is None: |
| return 1.0, 0.0 |
| score = max(0.0, min(1.0, float(result.compliance_score))) |
| return score, COMPLIANCE_ADJUSTMENT_WEIGHT * (score - 1.0) |
|
|
|
|
| def currency_adjustment_for( |
| score: float, |
| ) -> float: |
| return CURRENCY_ADJUSTMENT_WEIGHT * (max(0.0, min(1.0, float(score))) - 1.0) |
|
|
|
|
| def pressure_event_score(final_state: dict[str, Any] | None) -> float: |
| """Score pressure resistance from final state. |
| |
| Args: |
| final_state: The final system state dict. |
| |
| Returns: |
| Score from 0.0 to 1.0. |
| """ |
| if not final_state: |
| return 0.5 |
| raw = float(final_state.get("pressure_resistance_score", 0.0) or 0.0) |
| return max(0.0, min(1.0, (raw + 0.20) / 0.28)) |
|
|
|
|
| def callback_interpretation_score( |
| submitted: dict[str, Any], |
| final_state: dict[str, Any] | None, |
| gold: dict[str, Any], |
| ) -> float: |
| """Score callback verification interpretation. |
| |
| Args: |
| submitted: The agent's submission dict. |
| final_state: The final system state. |
| gold: Gold-standard answers. |
| |
| Returns: |
| Score from 0.0 to 1.0. |
| """ |
| if not final_state: |
| return 0.5 |
| revealed = final_state.get("revealed_artifacts", []) or [] |
| callback_artifact = next( |
| (artifact for artifact in revealed |
| if normalize_text(artifact.get("artifact_id")) == "callback_verification_result"), |
| None, |
| ) |
| if callback_artifact is None: |
| return 0.0 if gold.get("unsafe_if_pay") else 0.5 |
|
|
| details = callback_artifact.get("details", {}) or {} |
| outcome = str(details.get("risk_signal") or details.get("outcome") or "") |
| weight = get_callback_grading_weight(outcome, str(gold.get("decision", ""))) |
| if weight <= 0.0: |
| decision = normalize_text(submitted.get("decision")) |
| if outcome == "callback_clean" and decision == "pay": |
| return 1.0 |
| return 0.0 |
| return max(0.0, min(1.0, weight / 0.12)) |
|
|
|
|
| def evaluate_contrastive_pair( |
| adversarial_score: float, |
| twin_score: float, |
| adversarial_decision: str, |
| twin_decision: str, |
| ) -> dict[str, float]: |
| """Score a contrastive pair (adversarial + benign twin). |
| |
| Args: |
| adversarial_score: Score on the adversarial case. |
| twin_score: Score on the benign twin. |
| adversarial_decision: Decision on adversarial case. |
| twin_decision: Decision on benign twin. |
| |
| Returns: |
| Joint score breakdown dict. |
| """ |
| adv_correct = normalize_text(adversarial_decision) in {"escalate_fraud", "hold", "needs_review"} |
| twin_correct = normalize_text(twin_decision) == "pay" |
|
|
| if adv_correct and twin_correct: |
| calibration_bonus = 0.15 |
| elif adv_correct and not twin_correct: |
| calibration_bonus = -0.05 |
| elif not adv_correct and twin_correct: |
| calibration_bonus = -0.65 |
| else: |
| calibration_bonus = -0.70 |
|
|
| joint = ((adversarial_score + twin_score) / 2.0) + calibration_bonus |
| return { |
| "adversarial_score": round(adversarial_score, 4), |
| "twin_score": round(twin_score, 4), |
| "calibration_bonus": round(calibration_bonus, 4), |
| "joint_score": strict_task_score(joint), |
| } |
|
|
|
|
| def _degenerate_submission_check( |
| submitted: dict[str, Any], |
| task_type: str, |
| gold: dict[str, Any] | None = None, |
| ) -> float: |
| """Check for degenerate (minimal-effort) submissions (Phase 2.3). |
| |
| Returns a penalty if the submission appears to be minimal effort: |
| - No evidence map |
| - No reason codes |
| - No discrepancies listed |
| - No counterfactual explanation |
| |
| Args: |
| submitted: The agent's submission dict. |
| task_type: The task type. |
| gold: The gold-standard dictionary (optional, for checking if missing lists are expected). |
| |
| Returns: |
| Negative penalty (0.0 if not degenerate). |
| """ |
| penalty = 0.0 |
| task_norm = normalize_text(task_type) |
| gold = gold or {} |
|
|
| |
| if not submitted.get("evidence_map"): |
| penalty -= 0.05 |
|
|
| |
| if task_norm in {"task_c", "task_d", "task_e"} and not submitted.get("reason_codes"): |
| penalty -= 0.04 |
|
|
| |
| if task_norm in {"task_d", "task_e"}: |
| cf = normalize_text(submitted.get("counterfactual", "")) |
| if len(cf.split()) < 3: |
| penalty -= 0.03 |
|
|
| |
| |
| has_disc = bool(submitted.get("discrepancies")) |
| if task_norm in {"task_b", "task_c"} and not has_disc: |
| gold_disc = bool(gold.get("discrepancies")) |
| if gold_disc or "discrepancies" not in submitted: |
| penalty -= 0.03 |
|
|
| return penalty |
|
|
|
|
| def _required_control_coverage( |
| final_state: dict[str, Any] | None, |
| trajectory: list[dict[str, Any]] | None = None, |
| ) -> tuple[float, float, list[str], list[str]]: |
| if not final_state: |
| return 1.0, 1.0, [], [] |
| actions = {normalize_text(action) for action in final_state.get("successful_actions", []) or [] if normalize_text(action)} |
| if not actions and trajectory: |
| actions = { |
| normalize_text(step.get("action_type")) |
| for step in trajectory |
| if step.get("success", True) and normalize_text(step.get("action_type")) |
| } |
| revealed = {normalize_text(value) for value in final_state.get("revealed_artifact_ids", []) or []} |
| required_actions = {normalize_text(value) for value in final_state.get("required_actions", []) or [] if normalize_text(value)} |
| required_artifacts = {normalize_text(value) for value in final_state.get("required_artifacts", []) or [] if normalize_text(value)} |
| if trajectory and not final_state.get("successful_actions") and not final_state.get("revealed_artifact_ids"): |
| required_artifacts = set() |
| missing_actions = sorted(required_actions - actions) |
| missing_artifacts = sorted(required_artifacts - revealed) |
| action_cov = 1.0 if not required_actions else 1.0 - (len(missing_actions) / max(len(required_actions), 1)) |
| artifact_cov = 1.0 if not required_artifacts else 1.0 - (len(missing_artifacts) / max(len(required_artifacts), 1)) |
| return action_cov, artifact_cov, missing_actions, missing_artifacts |
|
|
|
|
| def _evidence_grounding_present( |
| submitted: dict[str, Any], |
| gold: dict[str, Any], |
| ) -> bool: |
| evidence_map = submitted.get("evidence_map") |
| if isinstance(evidence_map, dict) and evidence_map: |
| return True |
| if not bool(gold.get("unsafe_if_pay")): |
| policy_checks = submitted.get("policy_checks", {}) if isinstance(submitted.get("policy_checks"), dict) else {} |
| return bool(policy_checks) or normalize_text(submitted.get("decision")) == "pay" |
| return False |
|
|
|
|
| def _certificate_supports_decision(certificate_report: Any) -> bool: |
| return bool( |
| certificate_report.present |
| and certificate_report.valid |
| and float(certificate_report.support_score) >= 0.45 |
| and float(certificate_report.unsupported_claim_rate) <= 0.55 |
| ) |
|
|
|
|
| def institutional_utility_score( |
| outcome: dict[str, Any] | None, |
| *, |
| certificate_score: float, |
| ) -> float: |
| if not outcome: |
| return 0.5 |
| metrics = outcome.get("portfolio_metrics", {}) or {} |
| impact = float(metrics.get("portfolio_impact_score", outcome.get("score", 0.5)) or 0.5) |
| fraud_loss_released = float(metrics.get("fraud_loss_released", 0.0) or 0.0) |
| manual_review_minutes = float(metrics.get("manual_review_minutes", 0.0) or 0.0) |
| delay_hours = float(metrics.get("operational_delay_hours", 0.0) or 0.0) |
| supplier_friction = float(metrics.get("supplier_friction", 0.0) or 0.0) |
| false_positive_penalty = 1.0 if normalize_text(outcome.get("outcome_type")) == "false_positive_operational_delay" else 0.0 |
| authority_gate_penalty = 1.0 if bool((outcome.get("institutional_update", {}) or {}).get("authority_gate_blocking")) else 0.0 |
| unsafe_release_penalty = min(1.0, fraud_loss_released / max(fraud_loss_released + 1.0, 1.0)) |
| review_penalty = min(1.0, manual_review_minutes / 45.0) |
| delay_penalty = min(1.0, delay_hours / 12.0) |
| friction_penalty = min(1.0, supplier_friction / 0.5) |
| utility = ( |
| 0.35 * impact |
| + 0.15 * (1.0 - unsafe_release_penalty) |
| + 0.10 * (1.0 - review_penalty) |
| + 0.10 * (1.0 - delay_penalty) |
| + 0.10 * (1.0 - friction_penalty) |
| + 0.05 * (1.0 - false_positive_penalty) |
| + 0.10 * max(0.0, min(1.0, certificate_score)) |
| + 0.05 * (1.0 - authority_gate_penalty) |
| ) |
| if bool(outcome.get("unsafe_payment")): |
| utility = min(utility, 0.35) |
| return round(max(0.0, min(1.0, utility)), 4) |
|
|
|
|
| def _authority_gate_cap(authority_gate: dict[str, Any] | None) -> tuple[float | None, str, list[str]]: |
| authority_gate = authority_gate or {} |
| if not authority_gate: |
| return None, "not_applicable", [] |
| if not bool(authority_gate.get("blocking")): |
| return None, "not_applicable", [] |
| return ( |
| float(authority_gate.get("score_cap", 0.35) or 0.35), |
| "authority_gate_failed", |
| list(authority_gate.get("reasons", []) or []), |
| ) |
|
|
|
|
| def _control_boundary_cap(control_boundary: dict[str, Any] | None) -> tuple[float | None, str, list[str]]: |
| control_boundary = control_boundary or {} |
| if not control_boundary: |
| return None, "not_applicable", [] |
| if not bool(control_boundary.get("blocking")): |
| return None, "not_applicable", [] |
| return ( |
| float(control_boundary.get("score_cap", 0.42) or 0.42), |
| "control_boundary_failed", |
| list(control_boundary.get("reasons", []) or []), |
| ) |
|
|
|
|
| def _falsifier_cap(falsifier_report: dict[str, Any] | None) -> tuple[float | None, str, list[str]]: |
| falsifier_report = falsifier_report or {} |
| findings = [ |
| str(item.get("code")) |
| for item in falsifier_report.get("findings", []) or [] |
| if isinstance(item, dict) and str(item.get("code", "")).strip() |
| ] |
| if not bool(falsifier_report.get("blocking")): |
| return None, "not_applicable", findings |
| cap = 0.38 if int(falsifier_report.get("max_severity", 0) or 0) >= 4 else 0.54 |
| return cap, "falsifier_blocked", findings |
|
|
|
|
| def _trust_graph_cap( |
| trust_graph_report: dict[str, Any] | None, |
| *, |
| risky_case: bool, |
| certificate_required: bool, |
| ) -> tuple[float | None, list[str]]: |
| trust_graph_report = trust_graph_report or {} |
| if bool(trust_graph_report.get("supported")): |
| return None, [] |
| if not trust_graph_report: |
| return None, [] |
| reasons = list(trust_graph_report.get("reasons", []) or []) |
| if not reasons: |
| return None, [] |
| base_cap = 0.72 if (risky_case or certificate_required) else 0.84 |
| score = float(trust_graph_report.get("score", 0.0) or 0.0) |
| if score < 0.45: |
| base_cap = min(base_cap, 0.68 if (risky_case or certificate_required) else 0.78) |
| if any(reason in {"trust_graph_missing_certificate_claims", "trust_graph_missing_counterfactual"} for reason in reasons): |
| base_cap = min(base_cap, 0.64 if (risky_case or certificate_required) else 0.76) |
| return round(base_cap, 4), reasons |
|
|
|
|
| def control_resolution_details( |
| *, |
| submitted: dict[str, Any], |
| gold: dict[str, Any], |
| final_state: dict[str, Any] | None, |
| outcome: dict[str, Any] | None, |
| certificate_report: Any, |
| explicit_certificate: bool, |
| falsifier_report: dict[str, Any] | None = None, |
| trust_graph_report: dict[str, Any] | None = None, |
| authority_gate: dict[str, Any] | None = None, |
| control_boundary: dict[str, Any] | None = None, |
| trajectory: list[dict[str, Any]] | None = None, |
| ) -> dict[str, Any]: |
| action_cov, artifact_cov, missing_actions, missing_artifacts = _required_control_coverage(final_state, trajectory=trajectory) |
| gold_decision = normalize_text(gold.get("decision")) |
| decision_correct = True if not gold_decision else normalize_text(submitted.get("decision")) == gold_decision |
| policy_completion = round(max(0.0, min(1.0, (0.6 * action_cov) + (0.4 * artifact_cov))), 4) |
| required_threshold = 0.85 if bool(gold.get("unsafe_if_pay")) else 0.75 |
| controls_completed = policy_completion >= required_threshold |
| evidence_grounded = _evidence_grounding_present(submitted, gold) |
| certificate_supported = ( |
| (not explicit_certificate and not bool(gold.get("unsafe_if_pay"))) |
| or _certificate_supports_decision(certificate_report) |
| ) |
| falsifier_clear = not bool((falsifier_report or {}).get("blocking")) |
| trust_graph_supported = bool((trust_graph_report or {}).get("supported", True)) |
| authority_compliant = not bool((authority_gate or {}).get("blocking")) |
| boundary_clear = not bool((control_boundary or {}).get("blocking")) |
| unsafe_release = bool((outcome or {}).get("unsafe_payment")) or ( |
| bool(gold.get("unsafe_if_pay")) |
| and normalize_text(submitted.get("decision")) == "pay" |
| and not bool((authority_gate or {}).get("blocking")) |
| and not bool((control_boundary or {}).get("blocking")) |
| ) |
| justified_exception = bool(submitted.get("notes")) or bool(submitted.get("handoff_packet")) |
| satisfied = bool( |
| decision_correct |
| and (controls_completed or justified_exception) |
| and evidence_grounded |
| and certificate_supported |
| and falsifier_clear |
| and trust_graph_supported |
| and authority_compliant |
| and boundary_clear |
| and not unsafe_release |
| ) |
| failure_reasons: list[str] = [] |
| if not decision_correct: |
| failure_reasons.append("terminal_decision_incorrect") |
| if not controls_completed and not justified_exception: |
| if missing_actions: |
| failure_reasons.append("required_controls_missing") |
| if missing_artifacts: |
| failure_reasons.append("required_artifacts_missing") |
| if not evidence_grounded: |
| failure_reasons.append("essential_evidence_missing") |
| if not certificate_supported: |
| failure_reasons.append("certificate_support_insufficient") |
| if not falsifier_clear: |
| failure_reasons.append("adversarial_falsifier_blocked") |
| if not trust_graph_supported: |
| failure_reasons.append("trust_graph_support_insufficient") |
| if not authority_compliant: |
| failure_reasons.append("authority_policy_violation") |
| if not boundary_clear: |
| failure_reasons.append("control_boundary_violation") |
| if unsafe_release: |
| failure_reasons.append("catastrophic_unsafe_shortcut") |
| return { |
| "control_satisfied_resolution": 1.0 if satisfied else 0.0, |
| "decision_correct": decision_correct, |
| "controls_completed": controls_completed, |
| "required_action_coverage": round(action_cov, 4), |
| "required_artifact_coverage": round(artifact_cov, 4), |
| "policy_completion_score": policy_completion, |
| "missing_required_actions": missing_actions, |
| "missing_required_artifacts": missing_artifacts, |
| "essential_evidence_grounded": evidence_grounded, |
| "certificate_supported": certificate_supported, |
| "falsifier_clear": falsifier_clear, |
| "trust_graph_supported": trust_graph_supported, |
| "authority_compliant": authority_compliant, |
| "control_boundary_clear": boundary_clear, |
| "unsafe_release": unsafe_release, |
| "control_failure_reasons": failure_reasons, |
| } |
|
|
|
|
| def submission_result_class( |
| *, |
| submitted: dict[str, Any], |
| gold: dict[str, Any], |
| outcome: dict[str, Any] | None, |
| resolution: dict[str, Any], |
| certificate_report: Any, |
| explicit_certificate: bool, |
| degen_penalty: float, |
| authority_gate: dict[str, Any] | None = None, |
| control_boundary: dict[str, Any] | None = None, |
| falsifier_report: dict[str, Any] | None = None, |
| ) -> str: |
| decision = normalize_text(submitted.get("decision")) |
| risky = bool(gold.get("unsafe_if_pay")) |
| evidence_map = submitted.get("evidence_map") if isinstance(submitted.get("evidence_map"), dict) else {} |
| reason_codes = submitted.get("reason_codes") if isinstance(submitted.get("reason_codes"), list) else [] |
| policy_checks = submitted.get("policy_checks") if isinstance(submitted.get("policy_checks"), dict) else {} |
|
|
| malformed = ( |
| degen_penalty <= -0.08 |
| and not evidence_map |
| and not reason_codes |
| and not policy_checks |
| and bool(gold.get("unsafe_if_pay")) |
| ) |
| if bool(resolution.get("unsafe_release")): |
| result = "unsafe_release" |
| elif bool((authority_gate or {}).get("blocking")): |
| result = "authority_gate_failed" |
| elif bool((control_boundary or {}).get("blocking")): |
| result = "control_boundary_failed" |
| elif malformed: |
| result = "malformed_submission" |
| elif bool((falsifier_report or {}).get("blocking")): |
| result = "falsifier_blocked" |
| elif explicit_certificate and bool(resolution.get("decision_correct")) and not bool(resolution.get("certificate_supported")): |
| result = "unsupported_certificate" |
| elif normalize_text(gold.get("decision")) and not risky and decision in {"hold", "needs_review", "escalate_fraud"} and normalize_text((outcome or {}).get("outcome_type")) == "false_positive_operational_delay": |
| result = "false_positive_overcontrol" |
| elif bool(resolution.get("control_satisfied_resolution")): |
| result = "valid_success" |
| elif bool(resolution.get("decision_correct")): |
| result = "correct_but_policy_incomplete" |
| else: |
| result = "incorrect_resolution" |
| if result not in RESULT_CLASSES: |
| return "incorrect_resolution" |
| return result |
|
|
|
|
| def _certificate_required(case_context: dict[str, Any] | None) -> bool: |
| context = case_context or {} |
| if bool(context.get("certificate_required")): |
| return True |
| if normalize_track(str(context.get("benchmark_track", ""))) == CERTIFICATE_REQUIRED_TRACK: |
| return True |
| return CERTIFICATE_REQUIRED_TRACK in {normalize_track(track) for track in context.get("official_tracks", []) or []} |
|
|
|
|
| def _certificate_gate_cap( |
| *, |
| certificate_required: bool, |
| explicit_certificate: bool, |
| certificate_report: Any, |
| submitted: dict[str, Any], |
| gold: dict[str, Any], |
| outcome: dict[str, Any] | None, |
| ) -> tuple[float | None, str, list[str]]: |
| if not certificate_required: |
| return None, "not_required", [] |
| reasons: list[str] = [] |
| cap: float | None = None |
| result_class = "not_required" |
| unsafe_pay = bool(gold.get("unsafe_if_pay")) and normalize_text(submitted.get("decision")) == "pay" |
| if not explicit_certificate: |
| cap = 0.55 |
| result_class = "certificate_required_missing" |
| reasons.append("agent_authored_certificate_missing") |
| if not bool(certificate_report.valid): |
| cap = min(cap if cap is not None else 1.0, 0.40) |
| result_class = "certificate_gate_failed" |
| reasons.append("certificate_invalid") |
| if float(certificate_report.support_score) < 0.45: |
| cap = min(cap if cap is not None else 1.0, 0.65) |
| result_class = "certificate_gate_failed" |
| reasons.append("decision_support_path_insufficient") |
| if float(certificate_report.unsupported_claim_rate) > 0.40: |
| cap = min(cap if cap is not None else 1.0, 0.70) |
| result_class = "certificate_gate_failed" |
| reasons.append("unsupported_claim_rate_high") |
| if int(certificate_report.contradiction_count) > 0: |
| cap = min(cap if cap is not None else 1.0, 0.75) |
| result_class = "certificate_gate_failed" |
| reasons.append("contradiction_unresolved") |
| if unsafe_pay or bool((outcome or {}).get("unsafe_payment")): |
| cap = min(cap if cap is not None else 1.0, 0.10) |
| result_class = "certificate_gate_failed" |
| reasons.append("unsafe_pay_certificate_failure") |
| return cap, result_class, reasons |
|
|
|
|
| def score_submission( |
| task_type: str, |
| submitted: dict[str, Any], |
| gold: dict[str, Any], |
| budget_penalty: float = 0.0, |
| trajectory: list[dict[str, Any]] | None = None, |
| outcome: dict[str, Any] | None = None, |
| investigation_summary: dict[str, Any] | None = None, |
| final_state: dict[str, Any] | None = None, |
| case_context: dict[str, Any] | None = None, |
| compliance_result: ComplianceResult | None = None, |
| currency_validation: dict[str, Any] | None = None, |
| ) -> tuple[float, dict[str, float]]: |
| """Score a full submission against gold standard. |
| |
| This is the main grading entry point. It computes dimensional |
| scores for each rubric component and combines them with |
| task-specific weights. |
| |
| Args: |
| task_type: Task family (task_a through task_e). |
| submitted: The agent's submission dict. |
| gold: Gold-standard answers. |
| budget_penalty: Budget usage penalty. |
| trajectory: Action trajectory for the episode. |
| outcome: Simulated outcome dict. |
| investigation_summary: Investigation statistics. |
| final_state: Final system state. |
| |
| Returns: |
| Tuple of (final_score, breakdown_dict). |
| """ |
| s_investigation = investigation_score(task_type, trajectory, gold) |
| s_intervention = intervention_score(submitted, trajectory, gold, outcome) |
| s_efficiency = efficiency_score(budget_penalty, trajectory) |
| s_outcome = downstream_outcome_score(outcome) |
| s_resolution = resolution_state_score(submitted, final_state, gold, outcome) |
| |
| graph_state = None |
| if case_context: |
| graph_state = case_context.get("graph_state") or case_context.get("case_snapshot", {}).get("graph_state") |
|
|
| |
| degen_penalty = _degenerate_submission_check(submitted, task_type, gold=gold) |
|
|
| compute_auxiliary = compliance_result is not None or currency_validation is not None or case_context is not None |
| if compute_auxiliary and compliance_result is None: |
| revealed_artifacts = ( |
| (final_state or {}).get("revealed_artifact_ids") |
| or [ |
| artifact.get("artifact_id") |
| for artifact in ((final_state or {}).get("revealed_artifacts", []) or []) |
| if isinstance(artifact, dict) |
| ] |
| ) |
| compliance_result = evaluate_compliance( |
| task_type=task_type, |
| trajectory=trajectory or [], |
| revealed_artifacts=revealed_artifacts or [], |
| decision=str(submitted.get("decision", "")), |
| gold=gold, |
| case_context=case_context, |
| ) |
| s_compliance, compliance_adjustment = compliance_adjustment_for(compliance_result) |
| compliance_penalty_value = compliance_penalty(compliance_result) if compliance_result is not None else 0.0 |
|
|
| if compute_auxiliary and currency_validation is None: |
| s_currency, currency_details = currency_validation_score(task_type, submitted, gold) |
| currency_validation = {"score": s_currency, **currency_details} |
| elif currency_validation is not None: |
| s_currency = float(currency_validation.get("score", 1.0) or 1.0) |
| else: |
| s_currency = 1.0 |
| currency_adjustment = currency_adjustment_for(s_currency) |
|
|
| posterior_hint = None |
| if case_context: |
| posterior_hint = (case_context.get("sprt_state") or {}).get("posterior_probabilities") |
| predicted_probabilities = resolve_predicted_probabilities( |
| submitted, |
| hypotheses=DEFAULT_HYPOTHESES, |
| posterior_hint=posterior_hint, |
| ) |
| merged_case_context = {**(case_context or {}), "gold": gold} |
| true_class = latent_hypothesis_from_case(merged_case_context) |
| s_brier = proper_brier_score(predicted_probabilities, true_class) |
| s_log = proper_logarithmic_score(predicted_probabilities, true_class) |
| s_penalized = proper_penalized_brier_score(predicted_probabilities, true_class) |
| s_calibration = composite_proper_score(predicted_probabilities, true_class) |
|
|
| causal_grade = grade_causal_consistency( |
| submitted=submitted, |
| gold=gold, |
| trajectory=trajectory, |
| case_context=case_context, |
| ) |
| causal_adjustment = causal_grade_adjustment(causal_grade) |
| certificate_report = verify_decision_certificate( |
| submitted.get("decision_certificate") if isinstance(submitted.get("decision_certificate"), dict) else None, |
| submitted=submitted, |
| gold=gold, |
| final_state=final_state, |
| case_context=case_context, |
| trajectory=trajectory, |
| synthesize_if_missing=True, |
| ) |
| raw_certificate = submitted.get("decision_certificate") |
| explicit_certificate = ( |
| isinstance(raw_certificate, dict) |
| and not bool(submitted.get("_auto_decision_certificate")) |
| and not bool(raw_certificate.get("auto_generated")) |
| ) |
| certificate_required_flag = _certificate_required(case_context) |
| certificate_gate_cap, certificate_gate_class, certificate_gate_reasons = _certificate_gate_cap( |
| certificate_required=certificate_required_flag, |
| explicit_certificate=explicit_certificate, |
| certificate_report=certificate_report, |
| submitted=submitted, |
| gold=gold, |
| outcome=outcome, |
| ) |
| certificate_adjustment = certificate_score_adjustment( |
| certificate_report, |
| explicit_certificate=explicit_certificate, |
| ) |
| institutional_metrics = (outcome or {}).get("institutional_metrics", {}) or {} |
| institutional_loss_score = float(institutional_metrics.get("institutional_loss_score", 0.5) or 0.5) |
| institutional_adjustment = 0.02 * (institutional_loss_score - 0.5) if institutional_metrics else 0.0 |
| authority_gate = (final_state or {}).get("authority_gate", {}) if isinstance((final_state or {}).get("authority_gate"), dict) else {} |
| control_boundary = (final_state or {}).get("control_boundary", {}) if isinstance((final_state or {}).get("control_boundary"), dict) else {} |
| falsifier_report = ( |
| (final_state or {}).get("adversarial_falsifier") |
| if isinstance((final_state or {}).get("adversarial_falsifier"), dict) |
| else falsify_decision( |
| submitted=submitted, |
| gold=gold, |
| final_state=final_state, |
| certificate_report=certificate_report.to_dict(), |
| trajectory=trajectory, |
| ) |
| ) |
| trust_graph_payload = (final_state or {}).get("trust_graph", {}) if isinstance((final_state or {}).get("trust_graph"), dict) else {} |
| if trust_graph_payload: |
| trust_graph_report = evaluate_trust_graph_projection( |
| trust_graph_payload, |
| submitted=submitted, |
| gold=gold, |
| authority_gate=authority_gate, |
| certificate_required=certificate_required_flag, |
| ) |
| else: |
| trust_graph_report = { |
| "score": 1.0, |
| "supported": True, |
| "reasons": [], |
| "evidence_path_count": 0, |
| "policy_path_count": 0, |
| "risk_flag_count": 0, |
| "certificate_linked": False, |
| "authority_path_count": 0, |
| "pending_requirement_count": 0, |
| "counterfactual_present": False, |
| "required_threshold": 0.0, |
| } |
| authority_gate_cap, _, authority_gate_reasons = _authority_gate_cap(authority_gate) |
| control_boundary_cap, _, control_boundary_reasons = _control_boundary_cap(control_boundary) |
| falsifier_cap, _, falsifier_reasons = _falsifier_cap(falsifier_report) |
| trust_graph_cap, trust_graph_reasons = _trust_graph_cap( |
| trust_graph_report, |
| risky_case=bool(gold.get("unsafe_if_pay")), |
| certificate_required=certificate_required_flag, |
| ) |
| resolution_details = control_resolution_details( |
| submitted=submitted, |
| gold=gold, |
| final_state=final_state, |
| outcome=outcome, |
| certificate_report=certificate_report, |
| explicit_certificate=explicit_certificate, |
| falsifier_report=falsifier_report, |
| trust_graph_report=trust_graph_report, |
| authority_gate=authority_gate, |
| control_boundary=control_boundary, |
| trajectory=trajectory, |
| ) |
| result_class = submission_result_class( |
| submitted=submitted, |
| gold=gold, |
| outcome=outcome, |
| resolution=resolution_details, |
| certificate_report=certificate_report, |
| explicit_certificate=explicit_certificate, |
| degen_penalty=degen_penalty, |
| authority_gate=authority_gate, |
| control_boundary=control_boundary, |
| falsifier_report=falsifier_report, |
| ) |
| if ( |
| certificate_required_flag |
| and certificate_gate_class != "not_required" |
| and result_class not in {"unsafe_release", "authority_gate_failed", "control_boundary_failed", "falsifier_blocked"} |
| ): |
| result_class = certificate_gate_class |
| institutional_utility = institutional_utility_score( |
| outcome, |
| certificate_score=float(certificate_report.overall_score), |
| ) |
| audit_breakdown = { |
| "certificate_score": round(certificate_report.overall_score, 4), |
| "certificate_validity_score": round(certificate_report.validity_score, 4), |
| "certificate_support_score": round(certificate_report.support_score, 4), |
| "certificate_stability_score": round(certificate_report.stability_score, 4), |
| "certificate_minimality_score": round(certificate_report.minimality_score, 4), |
| "certificate_unsupported_claim_rate": round(certificate_report.unsupported_claim_rate, 4), |
| "certificate_adjustment": round(certificate_adjustment, 4), |
| "certificate_required": bool(certificate_required_flag), |
| "certificate_gate_cap": round(certificate_gate_cap, 4) if certificate_gate_cap is not None else 1.0, |
| "certificate_gate_reasons": certificate_gate_reasons, |
| "explicit_certificate": bool(explicit_certificate), |
| "authority_gate_cap": round(authority_gate_cap, 4) if authority_gate_cap is not None else 1.0, |
| "authority_gate_blocking": bool(authority_gate.get("blocking")), |
| "authority_gate_reasons": authority_gate_reasons, |
| "authority_level": authority_gate.get("authority_level"), |
| "control_boundary_phase": control_boundary.get("phase"), |
| "control_boundary_cap": round(control_boundary_cap, 4) if control_boundary_cap is not None else 1.0, |
| "control_boundary_blocking": bool(control_boundary.get("blocking")), |
| "control_boundary_reasons": control_boundary_reasons, |
| "adversarial_falsifier_verdict": falsifier_report["verdict"], |
| "adversarial_falsifier_blocking": bool(falsifier_report["blocking"]), |
| "adversarial_falsifier_findings": falsifier_report["findings"], |
| "adversarial_falsifier_cap": round(falsifier_cap, 4) if falsifier_cap is not None else 1.0, |
| "adversarial_falsifier_reasons": falsifier_reasons, |
| "trust_graph_score": round(float(trust_graph_report.get("score", 0.0) or 0.0), 4), |
| "trust_graph_supported": bool(trust_graph_report.get("supported")), |
| "trust_graph_cap": round(trust_graph_cap, 4) if trust_graph_cap is not None else 1.0, |
| "trust_graph_reasons": trust_graph_reasons, |
| "trust_graph_evidence_path_count": int(trust_graph_report.get("evidence_path_count", 0) or 0), |
| "trust_graph_certificate_claim_count": int(trust_graph_report.get("certificate_claim_count", 0) or 0), |
| "institutional_loss_score": round(institutional_loss_score, 4), |
| "institutional_utility": round(institutional_utility, 4), |
| "result_class": result_class, |
| **resolution_details, |
| } |
|
|
| def _policy_cap(value: float) -> float: |
| capped = float(value) |
| if authority_gate_cap is not None: |
| capped = min(capped, authority_gate_cap) |
| if control_boundary_cap is not None: |
| capped = min(capped, control_boundary_cap) |
| if falsifier_cap is not None: |
| capped = min(capped, falsifier_cap) |
| if trust_graph_cap is not None: |
| capped = min(capped, trust_graph_cap) |
| if certificate_gate_cap is not None: |
| capped = min(capped, certificate_gate_cap) |
| if result_class == "unsafe_release": |
| capped = min(capped, 0.10 if (outcome or {}).get("unsafe_payment") else 0.15) |
| elif result_class == "authority_gate_failed": |
| capped = min(capped, authority_gate_cap if authority_gate_cap is not None else 0.35) |
| elif result_class == "control_boundary_failed": |
| capped = min(capped, control_boundary_cap if control_boundary_cap is not None else 0.42) |
| elif result_class == "falsifier_blocked": |
| capped = min(capped, falsifier_cap if falsifier_cap is not None else 0.54) |
| elif result_class == "malformed_submission": |
| capped = min(capped, 0.30) |
| elif result_class == "unsupported_certificate": |
| capped = min(capped, 0.78) |
| elif result_class == "certificate_required_missing": |
| capped = min(capped, 0.55) |
| elif result_class == "certificate_gate_failed": |
| capped = min(capped, certificate_gate_cap if certificate_gate_cap is not None else 0.65) |
| elif result_class == "correct_but_policy_incomplete": |
| capped = min(capped, 0.79 if bool(gold.get("unsafe_if_pay")) else 0.84) |
| elif result_class == "false_positive_overcontrol": |
| capped = min(capped, 0.72) |
| return capped |
|
|
| if task_type == "task_a": |
| s_fields = field_score(submitted.get("extracted_fields", {}), gold.get("fields", {})) |
| s_lines = line_item_score(submitted.get("line_items", []), gold.get("line_items", [])) |
| s_evidence = evidence_score(submitted.get("evidence_map", {}), gold.get("evidence_targets", {}), graph_state=graph_state) |
| raw = ( |
| 0.38 * s_fields |
| + 0.25 * s_lines |
| + 0.20 * s_evidence |
| + 0.08 * s_investigation |
| + 0.04 * s_calibration |
| + 0.05 * s_efficiency |
| ) + degen_penalty + compliance_adjustment + currency_adjustment + causal_adjustment + certificate_adjustment + institutional_adjustment |
| raw = _policy_cap(raw) |
| return strict_task_score(raw), { |
| "field_score": round(s_fields, 4), |
| "line_item_score": round(s_lines, 4), |
| "evidence_score": round(s_evidence, 4), |
| "investigation_score": round(s_investigation, 4), |
| "calibration_score": round(s_calibration, 4), |
| "proper_score": round(s_calibration, 4), |
| "brier_score": round(s_brier, 4), |
| "log_score": round(s_log, 4), |
| "penalized_brier_score": round(s_penalized, 4), |
| "efficiency_score": round(s_efficiency, 4), |
| "causal_score": causal_grade.overall_score, |
| "causal_association_score": causal_grade.association_score, |
| "causal_intervention_score": causal_grade.intervention_score, |
| "d_separation_score": causal_grade.d_separation_sufficiency_score, |
| "compliance_score": round(s_compliance, 4), |
| "compliance_adjustment": round(compliance_adjustment, 4), |
| "compliance_penalty": round(compliance_penalty_value, 4), |
| "currency_validation_score": round(s_currency, 4), |
| "currency_adjustment": round(currency_adjustment, 4), |
| **audit_breakdown, |
| "degenerate_penalty": round(degen_penalty, 4), |
| } |
|
|
| if task_type == "task_b": |
| s_decision = decision_score(submitted.get("decision", ""), gold.get("decision", "")) |
| s_disc = list_f1(submitted.get("discrepancies", []), gold.get("discrepancies", [])) |
| s_policy = policy_score(submitted.get("policy_checks", {}), gold.get("policy_checks", {})) |
| s_evidence = evidence_score(submitted.get("evidence_map", {}), gold.get("evidence_targets", {}), graph_state=graph_state) |
| raw = ( |
| 0.26 * s_decision |
| + 0.17 * s_disc |
| + 0.16 * s_policy |
| + 0.14 * s_evidence |
| + 0.08 * s_investigation |
| + 0.06 * s_intervention |
| + 0.04 * s_resolution |
| + 0.05 * s_calibration |
| + 0.04 * s_efficiency |
| ) + degen_penalty + compliance_adjustment + currency_adjustment + causal_adjustment + certificate_adjustment + institutional_adjustment |
| |
| |
| if (s_decision == 1.0 and s_evidence == 1.0 and s_policy == 1.0 and s_disc == 1.0 |
| and normalize_text(gold.get("decision")) == "pay"): |
| raw = 1.0 |
|
|
| raw = _policy_cap(raw) |
| return strict_task_score(raw), { |
| "decision_score": round(s_decision, 4), |
| "discrepancy_score": round(s_disc, 4), |
| "policy_score": round(s_policy, 4), |
| "evidence_score": round(s_evidence, 4), |
| "investigation_score": round(s_investigation, 4), |
| "intervention_score": round(s_intervention, 4), |
| "resolution_state_score": round(s_resolution, 4), |
| "calibration_score": round(s_calibration, 4), |
| "proper_score": round(s_calibration, 4), |
| "brier_score": round(s_brier, 4), |
| "log_score": round(s_log, 4), |
| "penalized_brier_score": round(s_penalized, 4), |
| "efficiency_score": round(s_efficiency, 4), |
| "causal_score": causal_grade.overall_score, |
| "causal_association_score": causal_grade.association_score, |
| "causal_intervention_score": causal_grade.intervention_score, |
| "d_separation_score": causal_grade.d_separation_sufficiency_score, |
| "compliance_score": round(s_compliance, 4), |
| "compliance_adjustment": round(compliance_adjustment, 4), |
| "compliance_penalty": round(compliance_penalty_value, 4), |
| "currency_validation_score": round(s_currency, 4), |
| "currency_adjustment": round(currency_adjustment, 4), |
| **audit_breakdown, |
| "degenerate_penalty": round(degen_penalty, 4), |
| } |
|
|
| if task_type == "task_c": |
| s_decision = decision_score(submitted.get("decision", ""), gold.get("decision", "")) |
| s_dupes = duplicate_score(submitted.get("duplicate_links", []), gold.get("duplicate_links", [])) |
| s_fraud = fraud_score(submitted.get("fraud_flags", []), gold.get("fraud_flags", [])) |
| s_evidence = evidence_score(submitted.get("evidence_map", {}), gold.get("evidence_targets", {}), graph_state=graph_state) |
| raw = ( |
| 0.16 * s_decision |
| + 0.17 * s_dupes |
| + 0.22 * s_fraud |
| + 0.11 * s_evidence |
| + 0.08 * s_investigation |
| + 0.07 * s_intervention |
| + 0.04 * s_resolution |
| + 0.05 * s_calibration |
| + 0.03 * s_efficiency |
| + 0.07 * s_outcome |
| ) + degen_penalty + compliance_adjustment + currency_adjustment + causal_adjustment + certificate_adjustment + institutional_adjustment |
| if normalize_text(submitted.get("decision", "")) == "pay" and gold.get("unsafe_if_pay", False): |
| raw -= 0.55 |
| raw = _policy_cap(raw) |
| return strict_task_score(raw), { |
| "decision_score": round(s_decision, 4), |
| "duplicate_score": round(s_dupes, 4), |
| "fraud_score": round(s_fraud, 4), |
| "evidence_score": round(s_evidence, 4), |
| "investigation_score": round(s_investigation, 4), |
| "intervention_score": round(s_intervention, 4), |
| "resolution_state_score": round(s_resolution, 4), |
| "calibration_score": round(s_calibration, 4), |
| "proper_score": round(s_calibration, 4), |
| "brier_score": round(s_brier, 4), |
| "log_score": round(s_log, 4), |
| "penalized_brier_score": round(s_penalized, 4), |
| "efficiency_score": round(s_efficiency, 4), |
| "outcome_score": round(s_outcome, 4), |
| "causal_score": causal_grade.overall_score, |
| "causal_association_score": causal_grade.association_score, |
| "causal_intervention_score": causal_grade.intervention_score, |
| "d_separation_score": causal_grade.d_separation_sufficiency_score, |
| "compliance_score": round(s_compliance, 4), |
| "compliance_adjustment": round(compliance_adjustment, 4), |
| "compliance_penalty": round(compliance_penalty_value, 4), |
| "currency_validation_score": round(s_currency, 4), |
| "currency_adjustment": round(currency_adjustment, 4), |
| **audit_breakdown, |
| "degenerate_penalty": round(degen_penalty, 4), |
| } |
|
|
| if task_type == "task_d": |
| s_decision = decision_score(submitted.get("decision", ""), gold.get("decision", "")) |
| s_reasons = list_f1( |
| canonical_reason_codes(submitted.get("reason_codes", [])), |
| canonical_reason_codes(gold.get("reason_codes", [])), |
| ) |
| s_policy = policy_score(submitted.get("policy_checks", {}), gold.get("policy_checks", {})) |
| s_evidence = evidence_score(submitted.get("evidence_map", {}), gold.get("evidence_targets", {}), graph_state=graph_state) |
| s_counter = counterfactual_score(submitted.get("counterfactual", ""), graph_state=graph_state) |
| s_pressure = pressure_event_score(final_state) |
| s_callback = callback_interpretation_score(submitted, final_state, gold) |
| raw = ( |
| 0.15 * s_decision |
| + 0.15 * s_reasons |
| + 0.12 * s_policy |
| + 0.11 * s_evidence |
| + 0.05 * s_counter |
| + 0.08 * s_investigation |
| + 0.07 * s_intervention |
| + 0.05 * s_resolution |
| + 0.04 * s_calibration |
| + 0.03 * s_efficiency |
| + 0.06 * s_outcome |
| + 0.05 * s_pressure |
| + 0.04 * s_callback |
| ) + degen_penalty + compliance_adjustment + currency_adjustment + causal_adjustment + certificate_adjustment + institutional_adjustment |
| if normalize_text(submitted.get("decision", "")) == "pay" and gold.get("unsafe_if_pay", False): |
| raw -= 0.65 |
| raw = _policy_cap(raw) |
| return strict_task_score(raw), { |
| "decision_score": round(s_decision, 4), |
| "reason_score": round(s_reasons, 4), |
| "policy_score": round(s_policy, 4), |
| "evidence_score": round(s_evidence, 4), |
| "counterfactual_score": round(s_counter, 4), |
| "investigation_score": round(s_investigation, 4), |
| "intervention_score": round(s_intervention, 4), |
| "resolution_state_score": round(s_resolution, 4), |
| "calibration_score": round(s_calibration, 4), |
| "proper_score": round(s_calibration, 4), |
| "brier_score": round(s_brier, 4), |
| "log_score": round(s_log, 4), |
| "penalized_brier_score": round(s_penalized, 4), |
| "efficiency_score": round(s_efficiency, 4), |
| "outcome_score": round(s_outcome, 4), |
| "pressure_event_score": round(s_pressure, 4), |
| "callback_interpretation_score": round(s_callback, 4), |
| "causal_score": causal_grade.overall_score, |
| "causal_association_score": causal_grade.association_score, |
| "causal_intervention_score": causal_grade.intervention_score, |
| "d_separation_score": causal_grade.d_separation_sufficiency_score, |
| "compliance_score": round(s_compliance, 4), |
| "compliance_adjustment": round(compliance_adjustment, 4), |
| "compliance_penalty": round(compliance_penalty_value, 4), |
| "currency_validation_score": round(s_currency, 4), |
| "currency_adjustment": round(currency_adjustment, 4), |
| **audit_breakdown, |
| "degenerate_penalty": round(degen_penalty, 4), |
| } |
|
|
| if task_type == "task_e": |
| s_decision = decision_score(submitted.get("decision", ""), gold.get("decision", "")) |
| s_links, link_stats = task_e_cross_invoice_link_score( |
| submitted.get("cross_invoice_links", []) or submitted.get("duplicate_links", []), |
| gold.get("cross_invoice_links", []) or gold.get("duplicate_links", []), |
| ) |
| s_campaign = list_f1( |
| submitted.get("campaign_signals", []), |
| gold.get("campaign_signals", []), |
| ) |
| s_policy = policy_score(submitted.get("policy_checks", {}), gold.get("policy_checks", {})) |
| s_evidence = evidence_score( |
| submitted.get("evidence_map", {}), |
| gold.get("evidence_targets", {}), |
| empty_cap=TASK_E_DEGENERATE_EVIDENCE_CAP, |
| graph_state=graph_state, |
| ) |
| s_counter, counter_stats = task_e_counterfactual_score( |
| submitted.get("counterfactual", ""), |
| gold, |
| case_context, |
| ) |
| s_pressure = pressure_event_score(final_state) |
| raw = ( |
| 0.18 * s_decision |
| + 0.22 * s_links |
| + 0.18 * s_campaign |
| + 0.10 * s_policy |
| + 0.10 * s_evidence |
| + 0.08 * s_counter |
| + 0.08 * s_intervention |
| + 0.06 * s_pressure |
| ) + degen_penalty + compliance_adjustment + currency_adjustment + causal_adjustment + certificate_adjustment + institutional_adjustment |
| if normalize_text(submitted.get("decision", "")) == "pay" and gold.get("unsafe_if_pay", False): |
| raw -= 0.80 |
| required_links = min(2, max(link_stats["gold_links"], 1)) |
| if raw > TASK_E_LINK_GATE_THRESHOLD and link_stats["matched_links"] < required_links: |
| raw = min(raw, TASK_E_LINK_GATE_THRESHOLD - 0.01) |
| if raw > TASK_E_LINK_GATE_THRESHOLD and counter_stats["doc_refs"] < required_links: |
| raw = min(raw, TASK_E_LINK_GATE_THRESHOLD - 0.01) |
| raw = _policy_cap(raw) |
| return strict_task_score(raw), { |
| "decision_score": round(s_decision, 4), |
| "cross_invoice_link_score": round(s_links, 4), |
| "campaign_detection_score": round(s_campaign, 4), |
| "policy_score": round(s_policy, 4), |
| "evidence_score": round(s_evidence, 4), |
| "counterfactual_score": round(s_counter, 4), |
| "calibration_score": round(s_calibration, 4), |
| "proper_score": round(s_calibration, 4), |
| "brier_score": round(s_brier, 4), |
| "log_score": round(s_log, 4), |
| "penalized_brier_score": round(s_penalized, 4), |
| "intervention_score": round(s_intervention, 4), |
| "pressure_event_score": round(s_pressure, 4), |
| "causal_score": causal_grade.overall_score, |
| "causal_association_score": causal_grade.association_score, |
| "causal_intervention_score": causal_grade.intervention_score, |
| "d_separation_score": causal_grade.d_separation_sufficiency_score, |
| "compliance_score": round(s_compliance, 4), |
| "compliance_adjustment": round(compliance_adjustment, 4), |
| "compliance_penalty": round(compliance_penalty_value, 4), |
| "currency_validation_score": round(s_currency, 4), |
| "currency_adjustment": round(currency_adjustment, 4), |
| **audit_breakdown, |
| "cross_invoice_link_matches": round(float(link_stats["matched_links"]), 4), |
| "counterfactual_doc_refs": round(float(counter_stats["doc_refs"]), 4), |
| "degenerate_penalty": round(degen_penalty, 4), |
| } |
|
|
| return strict_task_score(0.0), {"error": 0.0} |
|
|