from __future__ import annotations from copy import deepcopy from typing import Any import random from .attack_library import apply_attack_to_case, list_attack_names from .schema import normalize_text from .evidence_graph import generate_scenario_graph, EvidenceGraph def _ensure_defaults(case: dict[str, Any]) -> dict[str, Any]: cloned = deepcopy(case) cloned.setdefault("budget_total", 15.0) cloned.setdefault("max_steps", 20) cloned.setdefault("difficulty", "medium") cloned.setdefault("documents", []) cloned.setdefault("gold", {}) cloned.setdefault("task_label", cloned.get("task_type", "")) cloned.setdefault( "initial_visible_doc_ids", [doc.get("doc_id") for doc in cloned.get("documents", []) if doc.get("doc_id")], ) return cloned def _derived_variant_id(base_case: dict[str, Any], suffix: str) -> str: base_id = str(base_case.get("case_id", "generated-case")) return f"{base_id}::{suffix}" def _replace_prefixed_token(doc: dict[str, Any], prefix: str, new_text: str) -> None: for token_list_name in ("accurate_ocr", "noisy_ocr"): for token in doc.get(token_list_name, []) or []: text = str(token.get("text", "")) if text.lower().startswith(prefix.lower()): token["text"] = new_text def _clean_email_doc(doc: dict[str, Any], seed: int) -> dict[str, Any]: cleaned = deepcopy(doc) rng = random.Random(seed) sender_domain = cleaned.get("thread_data", {}).get("vendor_domain") or "vendor.example.com" sender = f"billing@{sender_domain}" subject = "Approved invoice copy for scheduled payment run" body = ( "Attaching the approved invoice copy for the scheduled payment run. " "No bank changes requested. Please use the remittance instructions already on file." ) thread_id = cleaned.get("doc_id") vendor_key = cleaned.get("thread_data", {}).get("vendor_key") or cleaned.get("vendor_key") cleaned["thread_data"] = { "thread_id": thread_id, "vendor_key": vendor_key, "sender": sender, "from": sender, "subject": subject, "body": body, "sender_domain": sender_domain, "expected_domain": sender_domain, "vendor_domain": sender_domain, "flags": [], } cleaned["visual_tokens"] = [token for token in cleaned.get("visual_tokens", []) if "urgent" not in str(token)] cleaned["accurate_ocr"] = [ {"token_id": f"{thread_id}-clean-1", "text": f"From: {sender}", "bbox": [10, 10, 260, 20], "page": 1}, {"token_id": f"{thread_id}-clean-2", "text": f"Subject: {subject}", "bbox": [10, 30, 320, 40], "page": 1}, {"token_id": f"{thread_id}-clean-3", "text": body, "bbox": [10, 50, 420, 70], "page": 1}, ] cleaned["noisy_ocr"] = deepcopy(cleaned["accurate_ocr"]) cleaned["crop_text_hint"] = ["No bank change requested; use approved remittance on file."] return cleaned def generate_benign_twin( adversarial_case: dict[str, Any], seed: int, approved_bank_account: str | None = None, ) -> dict[str, Any]: twin = _ensure_defaults(adversarial_case) twin["case_id"] = f"{adversarial_case['case_id']}-TWIN" twin["benchmark_split"] = "contrastive" twin["contrastive_pair_id"] = str(adversarial_case.get("case_id", "contrastive")) twin["contrastive_role"] = "twin" twin["pressure_event"] = None twin["context_overrides"] = { "vendor_history": [], "ledger_index": [], } documents: list[dict[str, Any]] = [] invoice_doc_total = sum(1 for doc in twin.get("documents", []) or [] if normalize_text(doc.get("doc_type")) == "invoice") for index, doc in enumerate(twin.get("documents", []) or []): cloned_doc = deepcopy(doc) doc_type = normalize_text(cloned_doc.get("doc_type")) if doc_type == "email": cloned_doc = _clean_email_doc(cloned_doc, seed + index) if doc_type == "invoice" and approved_bank_account: _replace_prefixed_token(cloned_doc, "bank:", f"Bank: {approved_bank_account}") if invoice_doc_total > 1: _replace_prefixed_token(cloned_doc, "invoice date:", f"Invoice Date: 2026-05-{min(28, 5 + index * 9):02d}") if index == 0 and normalize_text(twin.get("task_type")) == "task_d": _replace_prefixed_token(cloned_doc, "subtotal:", "Subtotal: 2050.00") _replace_prefixed_token(cloned_doc, "tax:", "Tax: 164.00") _replace_prefixed_token(cloned_doc, "total:", "Total: 2214.00") documents.append(cloned_doc) twin["documents"] = documents twin["gold"] = { "decision": "PAY", "reason_codes": [], "policy_checks": { "three_way_match": "pass", "bank_change_verification": "pass", "duplicate_check": "pass", "approval_threshold_check": "pass", }, "unsafe_if_pay": False, "duplicate_links": [], "campaign_signals": [], "evidence_targets": {}, } twin["instruction"] = str(adversarial_case.get("instruction", "")).strip() or ( "Investigate the payment request using the standard AP controls and submit the safest supported decision." ) return twin def generate_case_variant( base_case: dict[str, Any], attack_names: list[str] | None = None, seed: int | None = None, variant_index: int = 0, split: str = "generated", ) -> dict[str, Any]: rng = random.Random(seed) case = _ensure_defaults(base_case) attacks = attack_names[:] if attack_names else [] if not attacks: available = list_attack_names() sample_size = 1 if case.get("task_type") in {"task_a", "task_b"} else 2 attacks = rng.sample(available, k=min(sample_size, len(available))) for idx, attack_name in enumerate(attacks): case = apply_attack_to_case(case, attack_name, seed=(seed or 0) + idx + 1) case["case_id"] = _derived_variant_id(case, f"variant-{variant_index}") case["benchmark_split"] = split case["generator_metadata"] = { **case.get("generator_metadata", {}), "variant_index": variant_index, "seed": seed, "source_case_id": base_case.get("case_id"), } attack_count = len(case.get("generator_metadata", {}).get("applied_attacks", [])) if attack_count >= 2: case["budget_total"] = max(float(case.get("budget_total", 15.0)), 16.0) case["max_steps"] = max(int(case.get("max_steps", 20)), 24) case["difficulty"] = "hard" elif attack_count == 1 and normalize_text(case.get("difficulty")) == "easy": case["difficulty"] = "medium" case["task_label"] = case.get("task_label") or case.get("task_type", "") # Create parameter space and attach Graph State (P3) randomize_case_surface(case, seed) # Enforce Solvability Check (P4) assert_solvability(case) return case def randomize_case_surface(case: dict[str, Any], seed: int) -> None: rng = random.Random(seed) # Parameter spaces bank_prefix = rng.choice(["US", "UK", "DE", "FR"]) bank_number = "".join(rng.choice("0123456789") for _ in range(8)) new_bank = f"{bank_prefix}_BANK_{bank_number}" vendor_names = ["Acme Corp", "Globex", "Initech", "Soylent", "Massive Dynamic"] new_vendor = rng.choice(vendor_names) year = rng.randint(2023, 2026) month = rng.randint(1, 12) day = rng.randint(1, 28) date_str = f"{year}-{month:02d}-{day:02d}" inv_num = f"INV-{rng.randint(1000, 9999)}" scenario_type = case.get("generator_metadata", {}).get("attack_category", "safe") if "applied_attacks" in case.get("generator_metadata", {}): if any("bank" in atk for atk in case["generator_metadata"]["applied_attacks"]): scenario_type = "bank_change_fraud" elif any("duplicate" in atk for atk in case["generator_metadata"]["applied_attacks"]): scenario_type = "duplicate_invoice" graph = generate_scenario_graph(scenario_type, seed) # Mutate actual document surfaces generically for doc in case.get("documents", []): _replace_prefixed_token(doc, "bank:", f"Bank: {new_bank}") _replace_prefixed_token(doc, "invoice date:", f"Invoice Date: {date_str}") _replace_prefixed_token(doc, "invoice number:", f"Invoice Number: {inv_num}") case["graph_state"] = graph.serialize() def assert_solvability(case: dict[str, Any]) -> bool: """Solvability oracle check (P4). Ensures latent graph provides complete path to truth.""" if "graph_state" not in case: return True graph = EvidenceGraph.deserialize(case["graph_state"]) if graph.latent_hypothesis != "safe": if not graph.unlock_rules: raise ValueError(f"Case {case['case_id']} is unsolvable: has hypothesis {graph.latent_hypothesis} but no unlock interventions.") return True def generate_case_batch( base_cases: list[dict[str, Any]], variants_per_case: int = 3, seed: int = 42, split: str = "generated", ) -> list[dict[str, Any]]: rng = random.Random(seed) generated: list[dict[str, Any]] = [] for case_idx, base_case in enumerate(base_cases): for variant_index in range(variants_per_case): variant_seed = rng.randint(1, 10_000_000) generated_case = generate_case_variant( base_case=base_case, attack_names=None, seed=variant_seed, variant_index=variant_index, split=split, ) generated_case["generator_metadata"]["batch_case_index"] = case_idx generated.append(generated_case) return generated def augment_case_library( base_cases: list[dict[str, Any]], variants_per_case: int = 2, seed: int = 42, ) -> list[dict[str, Any]]: original = [_ensure_defaults(case) for case in base_cases] generated = generate_case_batch(base_cases=base_cases, variants_per_case=variants_per_case, seed=seed, split="generated") return original + generated def generate_holdout_suite( base_cases: list[dict[str, Any]], variants_per_case: int = 1, seed: int = 31415, ) -> list[dict[str, Any]]: hard_cases = [ case for case in base_cases if normalize_text(case.get("task_type")) in {"task_c", "task_d", "task_e"} ] or list(base_cases) holdouts = generate_case_batch( base_cases=hard_cases, variants_per_case=variants_per_case, seed=seed, split="holdout", ) for index, case in enumerate(holdouts): case["case_id"] = _derived_variant_id(case, f"holdout-{index}") case.setdefault("generator_metadata", {})["holdout_seed"] = seed return holdouts