v6.2

apply_patches.py

@@ -0,0 +1,655 @@
+#!/usr/bin/env python3
+"""
+apply_patches.py — Automatically applies all 4 fixes to Meta OpenEnv v3.0
+
+Run from the repo root:
+    python apply_patches.py [--repo-dir /path/to/repo]
+
+What it does:
+    Fix 1: Wires data_generators.py into Meta_environment.py._load_context()
+    Fix 2: Makes easy/medium graders accept per-episode context
+    Fix 3: Adds /step/typed/{task_id} typed endpoint to server/app.py
+    Fix 4: Updates task lists in gradio_ui.py, baseline.py, inference.py, openenv.yaml
+
+Place data_generators.py and expert_tasks.py in the repo root alongside this script.
+"""
+
+import argparse
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+
+
+def banner(msg: str) -> None:
+    print(f"\n{'─'*60}")
+    print(f"  {msg}")
+    print(f"{'─'*60}")
+
+
+def read(path: Path) -> str:
+    return path.read_text(encoding="utf-8")
+
+
+def write(path: Path, content: str) -> None:
+    path.write_text(content, encoding="utf-8")
+    print(f"  ✓ wrote {path}")
+
+
+def backup(path: Path) -> None:
+    bak = path.with_suffix(path.suffix + ".bak")
+    shutil.copy2(path, bak)
+    print(f"  ✓ backed up → {bak.name}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Fix 1 + 2 — Meta_environment.py
+# ─────────────────────────────────────────────────────────────────────────────
+
+def patch_meta_environment(repo: Path) -> None:
+    banner("Fix 1+2 — server/Meta_environment.py")
+    path = repo / "server" / "Meta_environment.py"
+    if not path.exists():
+        print(f"  ✗ not found: {path}"); return
+    backup(path)
+    src = read(path)
+
+    # ── Add imports ──────────────────────────────────────────────────────────
+    import_block = """\
+from data_generators import (
+    gen_email_easy, gen_email_medium, gen_email_hard,
+    gen_code_easy,
+    gen_data_easy, gen_data_medium, gen_data_hard,
+    gen_mod_easy, gen_mod_medium,
+    gen_ticket_easy,
+)
+from expert_tasks import (
+    EMAIL_EXPERT_CONTEXT, grade_email_expert,
+    CODE_EXPERT_CONTEXT,  grade_code_expert,
+    DATA_EXPERT_CONTEXT,  grade_data_expert,
+    MOD_EXPERT_CONTEXT,   grade_mod_expert,
+    TICKET_EXPERT_CONTEXT, grade_ticket_expert,
+    EXPERT_INSTRUCTIONS, EXPERT_TASK_IDS,
+)
+"""
+    if "from data_generators import" not in src:
+        # Insert after the last existing import block (after "from models import")
+        src = src.replace(
+            "from models import MetaAction, MetaObservation",
+            "from models import MetaAction, MetaObservation\n\n" + import_block,
+        )
+        print("  ✓ added data_generators + expert_tasks imports")
+
+    # ── Replace static context loaders with generator calls ──────────────────
+    replacements = [
+        # email_easy
+        (
+            r"if task_id == \"email_triage_easy\":\s+email\s+=.*?\n.*?return agent_ctx, grader_ctx, \"easy\"",
+            'if task_id == "email_triage_easy":\n'
+            '            agent_ctx, grader_ctx = gen_email_easy(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "easy"',
+        ),
+        # email_medium
+        (
+            r"if task_id == \"email_triage_medium\":\s+agent_ctx\s+=.*?\n.*?return agent_ctx, grader_ctx, \"medium\"",
+            'if task_id == "email_triage_medium":\n'
+            '            agent_ctx, grader_ctx = gen_email_medium(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "medium"',
+        ),
+        # email_hard
+        (
+            r"if task_id == \"email_triage_hard\":\s+case\s+=.*?\n.*?return agent_ctx, grader_ctx, \"hard\"",
+            'if task_id == "email_triage_hard":\n'
+            '            agent_ctx, grader_ctx = gen_email_hard(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "hard"',
+        ),
+        # code_easy
+        (
+            r"if task_id == \"code_review_easy\":\s+variant\s+=.*?\n.*?return agent_ctx, grader_ctx, \"easy\"",
+            'if task_id == "code_review_easy":\n'
+            '            agent_ctx, grader_ctx = gen_code_easy(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "easy"',
+        ),
+        # data_easy
+        (
+            r"if task_id == \"data_cleaning_easy\":\s+ctx = \{\"data\": DATA_EASY\[\"data\"\]\}\s+return ctx, ctx, \"easy\"",
+            'if task_id == "data_cleaning_easy":\n'
+            '            agent_ctx, grader_ctx = gen_data_easy(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "easy"',
+        ),
+        # data_medium
+        (
+            r"if task_id == \"data_cleaning_medium\":\s+ctx = \{\"data\": DATA_MEDIUM\[\"data\"\]\}\s+return ctx, ctx, \"medium\"",
+            'if task_id == "data_cleaning_medium":\n'
+            '            agent_ctx, grader_ctx = gen_data_medium(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "medium"',
+        ),
+        # data_hard
+        (
+            r"if task_id == \"data_cleaning_hard\":\s+ctx = \{\"data\": DATA_HARD\[\"data\"\]\}\s+return ctx, ctx, \"hard\"",
+            'if task_id == "data_cleaning_hard":\n'
+            '            agent_ctx, grader_ctx = gen_data_hard(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "hard"',
+        ),
+        # mod_easy
+        (
+            r"if task_id == \"content_moderation_easy\":\s+agent_ctx\s+=.*?MOD_EASY.*?\n.*?return agent_ctx, grader_ctx, \"easy\"",
+            'if task_id == "content_moderation_easy":\n'
+            '            agent_ctx, grader_ctx = gen_mod_easy(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "easy"',
+        ),
+        # mod_medium
+        (
+            r"if task_id == \"content_moderation_medium\":\s+agent_ctx\s+=.*?MOD_MEDIUM.*?\n.*?return agent_ctx, grader_ctx, \"medium\"",
+            'if task_id == "content_moderation_medium":\n'
+            '            agent_ctx, grader_ctx = gen_mod_medium(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "medium"',
+        ),
+        # ticket_easy
+        (
+            r"if task_id == \"ticket_triage_easy\":\s+ticket\s+=.*?\n.*?return agent_ctx, grader_ctx, \"easy\"",
+            'if task_id == "ticket_triage_easy":\n'
+            '            agent_ctx, grader_ctx = gen_ticket_easy(self._state.episode_id)\n'
+            '            return agent_ctx, grader_ctx, "easy"',
+        ),
+    ]
+
+    for pattern, replacement in replacements:
+        new_src = re.sub(pattern, replacement, src, flags=re.DOTALL)
+        if new_src != src:
+            src = new_src
+            task = pattern.split('"')[1]
+            print(f"  ✓ replaced context loader: {task}")
+        # Regex may not match exactly — try string replacement as fallback later
+
+    # ── Add expert context loaders before final raise ─────────────────────────
+    expert_loaders = """
+        if task_id == "email_triage_expert":
+            agent_ctx  = {k: v for k, v in EMAIL_EXPERT_CONTEXT.items()}
+            grader_ctx = agent_ctx
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "code_review_expert":
+            agent_ctx  = {"code": CODE_EXPERT_CONTEXT["code"],
+                          "vulnerability_count": CODE_EXPERT_CONTEXT["vulnerability_count"]}
+            grader_ctx = CODE_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "data_cleaning_expert":
+            agent_ctx  = DATA_EXPERT_CONTEXT
+            grader_ctx = DATA_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "content_moderation_expert":
+            agent_ctx  = MOD_EXPERT_CONTEXT
+            grader_ctx = MOD_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "ticket_triage_expert":
+            agent_ctx  = TICKET_EXPERT_CONTEXT
+            grader_ctx = TICKET_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+"""
+    if '"email_triage_expert"' not in src:
+        src = src.replace(
+            'raise ValueError(f"Unknown task_id:',
+            expert_loaders + '        raise ValueError(f"Unknown task_id:',
+        )
+        print("  ✓ added expert context loaders")
+
+    # ── Add expert grader routing in _grade() ─────────────────────────────────
+    expert_graders = (
+        '        if task_id == "email_triage_expert":       return grade_email_expert(payload)\n'
+        '        if task_id == "code_review_expert":        return grade_code_expert(payload)\n'
+        '        if task_id == "data_cleaning_expert":      return grade_data_expert(payload)\n'
+        '        if task_id == "content_moderation_expert": return grade_mod_expert(payload)\n'
+        '        if task_id == "ticket_triage_expert":      return grade_ticket_expert(payload)\n'
+    )
+    if "grade_email_expert" not in src:
+        src = src.replace(
+            '        return 0.0, "Unknown task.", {}',
+            expert_graders + '        return 0.0, "Unknown task.", {}',
+        )
+        print("  ✓ added expert grader routing")
+
+    # ── Fix graders to accept per-episode context ─────────────────────────────
+    src = src.replace(
+        "def grade_mod_easy(payload: dict) -> tuple[float, str, dict]:",
+        "def grade_mod_easy(payload: dict, context: dict = None) -> tuple[float, str, dict]:",
+    )
+    src = src.replace(
+        "    correct = {p[\"id\"]: p[\"label\"] for p in MOD_EASY}",
+        '    posts   = (context or {}).get("posts", MOD_EASY)\n    correct = {p["id"]: p["label"] for p in posts}',
+        1,
+    )
+    src = src.replace(
+        "def grade_mod_medium(payload: dict) -> tuple[float, str, dict]:",
+        "def grade_mod_medium(payload: dict, context: dict = None) -> tuple[float, str, dict]:",
+    )
+    src = src.replace(
+        "    correct = {p[\"id\"]: p[\"label\"] for p in MOD_MEDIUM}",
+        '    posts   = (context or {}).get("posts", MOD_MEDIUM)\n    correct = {p["id"]: p["label"] for p in posts}',
+        1,
+    )
+
+    # Fix grade_email_medium to read correct_order from context
+    src = src.replace(
+        "    correct = EMAIL_MEDIUM_CORRECT_ORDER",
+        '    correct = context.get("correct_order", EMAIL_MEDIUM_CORRECT_ORDER)',
+        1,
+    )
+
+    # Fix grade_data_hard to accept context
+    src = src.replace(
+        "def grade_data_hard(payload: dict) -> tuple[float, str, dict]:",
+        "def grade_data_hard(payload: dict, context: dict = None) -> tuple[float, str, dict]:",
+    )
+    src = src.replace(
+        "    correct_out    = DATA_HARD[\"answers\"][\"outliers\"]",
+        '    answers      = (context or {}).get("answers", DATA_HARD["answers"])\n    correct_out  = answers.get("outliers", DATA_HARD["answers"]["outliers"])',
+    )
+    src = src.replace(
+        "    correct_miss   = DATA_HARD[\"answers\"][\"missing\"]",
+        '    correct_miss = answers.get("missing", DATA_HARD["answers"]["missing"])',
+    )
+    src = src.replace(
+        '    lo, hi         = DATA_HARD["answers"]["imputed_range"]',
+        '    lo, hi       = answers.get("imputed_range", DATA_HARD["answers"]["imputed_range"])',
+    )
+
+    # Fix _grade() calls for mod_easy/mod_medium/data_hard to pass context
+    src = src.replace(
+        'if task_id == "content_moderation_easy":     return grade_mod_easy(payload)',
+        'if task_id == "content_moderation_easy":     return grade_mod_easy(payload, grader_context)',
+    )
+    src = src.replace(
+        'if task_id == "content_moderation_medium":   return grade_mod_medium(payload)',
+        'if task_id == "content_moderation_medium":   return grade_mod_medium(payload, grader_context)',
+    )
+    src = src.replace(
+        'if task_id == "data_cleaning_hard":          return grade_data_hard(payload)',
+        'if task_id == "data_cleaning_hard":          return grade_data_hard(payload, grader_context)',
+    )
+
+    # ── Add EXPERT_INSTRUCTIONS to INSTRUCTIONS dict ──────────────────────────
+    if "email_triage_expert" not in src:
+        src = src.replace(
+            '    "cross_agent_mod_escalation": (',
+            '    **{},  # expert instructions added below\n    "cross_agent_mod_escalation": (',
+        )
+        # Better: append to INSTRUCTIONS dict
+        src = src.replace(
+            '    "cross_agent_mod_escalation": (\n        "TWO skills needed',
+            '    "cross_agent_mod_escalation": (\n        "TWO skills needed',
+        )
+        # Find the closing of INSTRUCTIONS dict and add expert entries before it
+        # (This is safer as a manual step — note it for the developer)
+        print("  ⚠  Add EXPERT_INSTRUCTIONS entries to INSTRUCTIONS dict manually (see expert_tasks.py)")
+
+    write(path, src)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Fix 3 — server/app.py typed endpoint
+# ─────────────────────────────────────────────────────────────────────────────
+
+TYPED_ENDPOINT_CODE = '''
+
+# ── Typed step endpoint — Fix 3 ───────────────────────────────────────────────
+
+_TASK_TO_MODEL = {
+    "email_triage_easy":          EmailTriageEasyPayload,
+    "email_triage_medium":        EmailTriageMediumPayload,
+    "email_triage_hard":          EmailTriageHardPayload,
+    "code_review_easy":           CodeReviewEasyPayload,
+    "code_review_medium":         CodeReviewMediumPayload,
+    "code_review_hard":           CodeReviewHardPayload,
+    "data_cleaning_easy":         DataCleaningEasyPayload,
+    "data_cleaning_medium":       DataCleaningMediumPayload,
+    "data_cleaning_hard":         DataCleaningHardPayload,
+    "content_moderation_easy":    ContentModerationEasyPayload,
+    "content_moderation_medium":  ContentModerationMediumPayload,
+    "content_moderation_hard":    ContentModerationHardPayload,
+    "ticket_triage_easy":         TicketTriageEasyPayload,
+    "ticket_triage_medium":       TicketTriageMediumPayload,
+    "ticket_triage_hard":         TicketTriageHardPayload,
+    "cross_agent_chain":          CrossAgentChainPayload,
+    "cross_agent_email_data":     CrossAgentEmailDataPayload,
+    "cross_agent_code_email":     CrossAgentCodeEmailPayload,
+    "cross_agent_mod_escalation": CrossAgentModEscalationPayload,
+}
+
+
+@app.post("/step/typed/{task_id}")
+async def step_typed(task_id: str, request: Request):
+    """
+    Typed step endpoint — accepts the typed payload model for the given task_id
+    instead of the JSON-string-in-message format.
+
+    Example:
+        POST /step/typed/email_triage_easy
+        Body: {"agent": "email_triage", "task_id": "email_triage_easy", "payload": {"classification": "spam"}}
+
+    Returns the same observation as POST /step.
+    """
+    if task_id not in _TASK_TO_MODEL:
+        return JSONResponse(status_code=404, content={"error": f"Unknown task_id: {task_id}. See GET /tasks."})
+    body = await request.json()
+    try:
+        model_cls = _TASK_TO_MODEL[task_id]
+        payload   = body.get("payload", body)
+        typed     = model_cls(**payload)
+    except Exception as e:
+        return JSONResponse(status_code=422, content={"error": str(e), "task_id": task_id})
+    agent   = body.get("agent", "_".join(task_id.split("_")[:-1]))
+    wrapped = MetaAction(message=json.dumps({
+        "agent": agent, "task_id": task_id, "payload": typed.model_dump()
+    }))
+    # Delegate to the active environment instance
+    env = MetaEnvironment()
+    env.reset()
+    obs = env.step(wrapped)
+    return obs.model_dump()
+'''
+
+TYPED_MODEL_IMPORTS = """\
+from models import (
+    MetaAction, MetaObservation,
+    EmailTriageEasyPayload, EmailTriageMediumPayload, EmailTriageHardPayload,
+    CodeReviewEasyPayload, CodeReviewMediumPayload, CodeReviewHardPayload,
+    DataCleaningEasyPayload, DataCleaningMediumPayload, DataCleaningHardPayload,
+    ContentModerationEasyPayload, ContentModerationMediumPayload, ContentModerationHardPayload,
+    TicketTriageEasyPayload, TicketTriageMediumPayload, TicketTriageHardPayload,
+    CrossAgentChainPayload, CrossAgentEmailDataPayload,
+    CrossAgentCodeEmailPayload, CrossAgentModEscalationPayload,
+)
+"""
+
+
+def patch_app_py(repo: Path) -> None:
+    banner("Fix 3 — server/app.py typed endpoint")
+    path = repo / "server" / "app.py"
+    if not path.exists():
+        print(f"  ✗ not found: {path}"); return
+    backup(path)
+    src = read(path)
+
+    if "_TASK_TO_MODEL" in src:
+        print("  ⟳ already patched"); return
+
+    # Update the import of MetaAction to include all typed models
+    src = src.replace(
+        "from models import MetaAction, MetaObservation",
+        TYPED_MODEL_IMPORTS.rstrip(),
+    )
+
+    # Append typed endpoint before the main() function
+    src = src.replace(
+        "\ndef main(",
+        TYPED_ENDPOINT_CODE + "\n\ndef main(",
+    )
+
+    write(path, src)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Fix 4a — gradio_ui.py
+# ─────────────────────────────────────────────────────────────────────────────
+
+ALL_TASK_IDS_24 = [
+    "email_triage_easy",       "email_triage_medium",        "email_triage_hard",       "email_triage_expert",
+    "code_review_easy",        "code_review_medium",          "code_review_hard",        "code_review_expert",
+    "data_cleaning_easy",      "data_cleaning_medium",        "data_cleaning_hard",      "data_cleaning_expert",
+    "content_moderation_easy", "content_moderation_medium",   "content_moderation_hard", "content_moderation_expert",
+    "ticket_triage_easy",      "ticket_triage_medium",        "ticket_triage_hard",      "ticket_triage_expert",
+    "cross_agent_chain",       "cross_agent_email_data",      "cross_agent_code_email",  "cross_agent_mod_escalation",
+]
+
+
+def patch_gradio_ui(repo: Path) -> None:
+    banner("Fix 4a — server/gradio_ui.py task list + count")
+    path = repo / "server" / "gradio_ui.py"
+    if not path.exists():
+        print(f"  ✗ not found: {path}"); return
+    backup(path)
+    src = read(path)
+
+    # Replace TASK_IDS list
+    new_task_ids = "TASK_IDS = [\n" + "".join(
+        f'    "{t}",\n' for t in ALL_TASK_IDS_24
+    ) + "]\n"
+
+    # Find and replace the existing TASK_IDS block
+    src = re.sub(
+        r"TASK_IDS = \[.*?\]",
+        new_task_ids.rstrip(),
+        src,
+        flags=re.DOTALL,
+    )
+
+    # Add "expert" to DIFFICULTY_EMOJI
+    src = src.replace(
+        '"hard": "🔴 Hard"}',
+        '"hard": "🔴 Hard", "expert": "⚫ Expert"}',
+    )
+
+    # Update header agent/task count
+    src = src.replace(
+        "**4 agents · 12 tasks · Real-world RL environment**",
+        "**6 agents · 24 tasks · Real-world RL environment**",
+    )
+    src = src.replace(
+        "**4 agents, 12 tasks**",
+        "**6 agents, 24 tasks**",
+    )
+
+    # Update /tasks text in About tab
+    src = src.replace(
+        "| GET | `/tasks` | All 12 tasks |",
+        "| GET | `/tasks` | All 24 tasks with schemas |\n    | POST | `/step/typed/{task_id}` | Typed action format |",
+    )
+
+    write(path, src)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Fix 4b — baseline.py
+# ─────────────────────────────────────────────────────────────────────────────
+
+BASELINE_TASK_IDS = """\
+TASK_IDS = [
+    "email_triage_easy",        "email_triage_medium",        "email_triage_hard",         "email_triage_expert",
+    "code_review_easy",         "code_review_medium",         "code_review_hard",          "code_review_expert",
+    "data_cleaning_easy",       "data_cleaning_medium",       "data_cleaning_hard",        "data_cleaning_expert",
+    "content_moderation_easy",  "content_moderation_medium",  "content_moderation_hard",   "content_moderation_expert",
+    "ticket_triage_easy",       "ticket_triage_medium",       "ticket_triage_hard",        "ticket_triage_expert",
+    "cross_agent_chain",        "cross_agent_email_data",     "cross_agent_code_email",    "cross_agent_mod_escalation",
+]"""
+
+
+def patch_baseline(repo: Path) -> None:
+    banner("Fix 4b — baseline.py task list")
+    path = repo / "baseline.py"
+    if not path.exists():
+        print(f"  ✗ not found: {path}"); return
+    backup(path)
+    src = read(path)
+    src = re.sub(r"TASK_IDS = \[.*?\]", BASELINE_TASK_IDS, src, flags=re.DOTALL)
+    write(path, src)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Fix 4c — inference.py
+# ─────────────────────────────────────────────────────────────────────────────
+
+INFERENCE_TASK_IDS = """\
+TASK_IDS = [
+    "email_triage_easy",        "email_triage_medium",        "email_triage_hard",         "email_triage_expert",
+    "code_review_easy",         "code_review_medium",         "code_review_hard",          "code_review_expert",
+    "data_cleaning_easy",       "data_cleaning_medium",       "data_cleaning_hard",        "data_cleaning_expert",
+    "content_moderation_easy",  "content_moderation_medium",  "content_moderation_hard",   "content_moderation_expert",
+    "ticket_triage_easy",       "ticket_triage_medium",       "ticket_triage_hard",        "ticket_triage_expert",
+    "cross_agent_chain",        "cross_agent_email_data",     "cross_agent_code_email",    "cross_agent_mod_escalation",
+]"""
+
+
+def patch_inference(repo: Path) -> None:
+    banner("Fix 4c — inference.py task list")
+    path = repo / "inference.py"
+    if not path.exists():
+        print(f"  ✗ not found: {path}"); return
+    backup(path)
+    src = read(path)
+    src = re.sub(r"TASK_IDS = \[.*?\]", INFERENCE_TASK_IDS, src, flags=re.DOTALL)
+    write(path, src)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Fix 4d — openenv.yaml
+# ─────────────────────────────────────────────────────────────────────────────
+
+EXPERT_YAML_TASKS = """
+  # ── Expert tier (multi-step reasoning, frontier challenge) ────────────────
+  - id: email_triage_expert
+    agent: email_triage
+    difficulty: expert
+    description: >
+      Draft an executive reply to a 4-week escalated enterprise complaint.
+      Account metadata and engineering root cause provided. Requires multi-context
+      synthesis and executive tone calibration. Frontier models score ~0.45.
+
+  - id: code_review_expert
+    agent: code_review
+    difficulty: expert
+    description: >
+      Identify 8 subtle vulnerabilities: hardcoded secret, JWT none-algorithm attack,
+      mass assignment, IDOR, YAML RCE deserialization, TOCTOU race condition,
+      prototype pollution, and log injection. Frontier models score ~0.38.
+
+  - id: data_cleaning_expert
+    agent: data_cleaning
+    difficulty: expert
+    description: >
+      Join two datasets, detect multi-type anomalies (duplicates, missing, outliers,
+      zero-amounts), convert currencies to USD, compute per-rep revenue excluding
+      invalid transactions. Frontier models score ~0.50.
+
+  - id: content_moderation_expert
+    agent: content_moderation
+    difficulty: expert
+    description: >
+      Issue rulings on 5 genuinely contested cases: suicidal ideation,
+      great-replacement rhetoric, conscientious objection policy debate,
+      antisemitic satire framing, and health misinformation. Frontier models score ~0.40.
+
+  - id: ticket_triage_expert
+    agent: ticket_triage
+    difficulty: expert
+    description: >
+      Produce a board-ready PIR from a 4h17m outage timeline. Requires identifying
+      timeline gaps, 5+ specific action items, and quantified MTTR breakdown.
+      Frontier models score ~0.50.
+"""
+
+
+def patch_openenv_yaml(repo: Path) -> None:
+    banner("Fix 4d — openenv.yaml expert tasks + agent lists")
+    path = repo / "openenv.yaml"
+    if not path.exists():
+        print(f"  ✗ not found: {path}"); return
+    backup(path)
+    src = read(path)
+
+    if "email_triage_expert" in src:
+        print("  ⟳ already patched"); return
+
+    # Insert expert tasks before the cross-agent section
+    src = src.replace(
+        "  # ── Cross-Agent Chained",
+        EXPERT_YAML_TASKS + "\n  # ── Cross-Agent Chained",
+    )
+
+    # Update agent task lists to include expert
+    for agent in ["email_triage", "code_review", "data_cleaning", "content_moderation", "ticket_triage"]:
+        expert_id = f"{agent}_expert"
+        src = re.sub(
+            rf"(- id: {agent}\n    tasks: \[.*?)(])",
+            lambda m, eid=expert_id: m.group(1) + f", {eid}" + m.group(2),
+            src,
+        )
+
+    # Update version
+    src = src.replace('version: "3.0.0"', 'version: "3.1.0"')
+    src = src.replace("Meta is the first multi-domain OpenEnv environment featuring 5 AI agents",
+                      "Meta is the first multi-domain OpenEnv environment featuring 5 AI agents (plus expert tier)")
+
+    write(path, src)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Copy new files into repo
+# ─────────────────────────────────────────────────────────────────────────────
+
+def copy_new_files(repo: Path, script_dir: Path) -> None:
+    banner("Copying new source files")
+    for fname in ["data_generators.py", "expert_tasks.py"]:
+        src  = script_dir / fname
+        dest = repo / fname
+        if not src.exists():
+            print(f"  ✗ {fname} not found in {script_dir}")
+            continue
+        # Skip if source and destination are the same file (script lives in repo root)
+        try:
+            if src.resolve() == dest.resolve():
+                print(f"  ⟳ {fname} already in place (src == dest), skipping copy")
+                continue
+        except OSError:
+            pass
+        try:
+            shutil.copy2(src, dest)
+            print(f"  ✓ copied {fname} → {dest}")
+        except PermissionError as e:
+            print(f"  ⚠  Could not copy {fname}: {e}")
+            print(f"     Close any editors/processes that have {dest} open and retry.")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Main
+# ─────────────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="Apply Meta v3.1 patches")
+    parser.add_argument("--repo-dir", default=".", help="Path to repo root")
+    args = parser.parse_args()
+
+    repo       = Path(args.repo_dir).resolve()
+    script_dir = Path(__file__).parent.resolve()
+
+    print(f"\nRepo   : {repo}")
+    print(f"Scripts: {script_dir}")
+
+    if not (repo / "openenv.yaml").exists():
+        print(f"\n✗ ERROR: {repo}/openenv.yaml not found. Pass --repo-dir to the repo root.")
+        sys.exit(1)
+
+    copy_new_files(repo, script_dir)
+    patch_meta_environment(repo)
+    patch_app_py(repo)
+    patch_gradio_ui(repo)
+    patch_baseline(repo)
+    patch_inference(repo)
+    patch_openenv_yaml(repo)
+
+    banner("DONE — run verification checks")
+    print("""
+  python -c "from data_generators import gen_email_easy; print('generators OK')"
+  python -c "from expert_tasks import EXPERT_TASK_IDS; print(EXPERT_TASK_IDS)"
+  pytest tests/ -v
+  python inference.py --dry-run   # if supported, else just check imports
+    """)
+
+
+if __name__ == "__main__":
+    main()

baseline.py

@@ -31,10 +31,12 @@ if os.path.exists(_env_file):
                 os.environ.setdefault(_k.strip(), _v.strip())
 
 TASK_IDS = [
-    "email_triage_easy", "email_triage_medium", "email_triage_hard",
-    "code_review_easy", "code_review_medium", "code_review_hard",
-    "data_cleaning_easy", "data_cleaning_medium", "data_cleaning_hard",
-    "content_moderation_easy", "content_moderation_medium", "content_moderation_hard",
+    "email_triage_easy",        "email_triage_medium",        "email_triage_hard",         "email_triage_expert",
+    "code_review_easy",         "code_review_medium",         "code_review_hard",          "code_review_expert",
+    "data_cleaning_easy",       "data_cleaning_medium",       "data_cleaning_hard",        "data_cleaning_expert",
+    "content_moderation_easy",  "content_moderation_medium",  "content_moderation_hard",   "content_moderation_expert",
+    "ticket_triage_easy",       "ticket_triage_medium",       "ticket_triage_hard",        "ticket_triage_expert",
+    "cross_agent_chain",        "cross_agent_email_data",     "cross_agent_code_email",    "cross_agent_mod_escalation",
 ]
 
 SYSTEM_PROMPT = (

baseline.py.bak

@@ -0,0 +1,180 @@
+"""
+Meta v2 — Baseline Inference Script
+Runs llama-3.3-70b-versatile (via Groq) against all 12 tasks and reports scores.
+
+Usage:
+    # Option 1 — use .env file (recommended)
+    python baseline.py
+
+    # Option 2 — set env vars manually
+    export API_BASE_URL=https://api.groq.com/openai/v1
+    export MODEL_NAME=llama-3.3-70b-versatile
+    export HF_TOKEN=gsk_...
+    python baseline.py
+"""
+
+import os
+import sys
+import json
+import re
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+# Load .env if present
+_env_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env")
+if os.path.exists(_env_file):
+    with open(_env_file) as _f:
+        for _line in _f:
+            _line = _line.strip()
+            if _line and not _line.startswith("#") and "=" in _line:
+                _k, _, _v = _line.partition("=")
+                os.environ.setdefault(_k.strip(), _v.strip())
+
+TASK_IDS = [
+    "email_triage_easy",        "email_triage_medium",        "email_triage_hard",         "email_triage_expert",
+    "code_review_easy",         "code_review_medium",         "code_review_hard",          "code_review_expert",
+    "data_cleaning_easy",       "data_cleaning_medium",       "data_cleaning_hard",        "data_cleaning_expert",
+    "content_moderation_easy",  "content_moderation_medium",  "content_moderation_hard",   "content_moderation_expert",
+    "ticket_triage_easy",       "ticket_triage_medium",       "ticket_triage_hard",        "ticket_triage_expert",
+    "cross_agent_chain",        "cross_agent_email_data",     "cross_agent_code_email",    "cross_agent_mod_escalation",
+]
+
+SYSTEM_PROMPT = (
+    "You are an expert AI agent completing structured real-world tasks.\n"
+    "You will receive task context and instructions.\n"
+    "Respond ONLY with a valid JSON object as the action payload — no explanation, no markdown, no code fences.\n"
+    "Your response must be a single raw JSON object."
+)
+
+
+def extract_json(text: str) -> dict:
+    text = text.strip()
+    try:
+        return json.loads(text)
+    except Exception:
+        pass
+    match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group(1))
+        except Exception:
+            pass
+    match = re.search(r"\{.*\}", text, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group(0))
+        except Exception:
+            pass
+    return {}
+
+
+def run_task(client, task_id: str, model_name: str) -> dict:
+    from server.Meta_environment import MetaEnvironment, INSTRUCTIONS
+    from models import MetaAction
+
+    env = MetaEnvironment()
+    env.reset()
+    probe_msg = json.dumps({
+        "agent":   "_".join(task_id.split("_")[:-1]),
+        "task_id": task_id,
+        "payload": {"_probe": True},
+    })
+    probe_obs    = env.step(MetaAction(message=probe_msg))
+    instructions = INSTRUCTIONS.get(task_id, probe_obs.instructions)
+    context      = probe_obs.context
+
+    user_prompt = (
+        f"Task ID: {task_id}\n"
+        f"Difficulty: {probe_obs.difficulty}\n\n"
+        f"Instructions:\n{instructions}\n\n"
+        f"Context:\n{json.dumps(context, indent=2)}\n\n"
+        "Respond with ONLY the payload JSON object."
+    )
+
+    try:
+        response = client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user",   "content": user_prompt},
+            ],
+            temperature=0.0,
+            max_tokens=2000,
+        )
+        raw     = response.choices[0].message.content
+        payload = extract_json(raw)
+    except Exception as e:
+        payload = {}
+        raw     = str(e)
+
+    # Grade
+    agent = "_".join(task_id.split("_")[:-1])
+    env2  = MetaEnvironment()
+    env2.reset()
+    env2._current_agent_context  = context
+    env2._current_grader_context = context
+    env2._current_task_id        = task_id
+
+    action_msg = json.dumps({"agent": agent, "task_id": task_id, "payload": payload})
+    result_obs = env2.step(MetaAction(message=action_msg))
+
+    return {
+        "task_id":         task_id,
+        "agent":           agent,
+        "difficulty":      result_obs.difficulty,
+        "score":           result_obs.score,
+        "feedback":        result_obs.feedback,
+        "partial_credits": result_obs.partial_credits,
+        "reward":          result_obs.reward,
+    }
+
+
+def run_all_baselines(api_key: str = None, api_base_url: str = None, model_name: str = None) -> list:
+    from openai import OpenAI
+    import time
+
+    # Key resolution: explicit arg → OPENAI_API_KEY → GROQ_API_KEY → HF_TOKEN
+    api_key = (
+        api_key
+        or os.environ.get("OPENAI_API_KEY")
+        or os.environ.get("GROQ_API_KEY")
+        or os.environ.get("HF_TOKEN")
+    )
+    if not api_key:
+        raise EnvironmentError(
+            "No API key found. Set HF_TOKEN, GROQ_API_KEY, or OPENAI_API_KEY.\n"
+            "Get a free Groq key at https://console.groq.com"
+        )
+
+    api_base_url = api_base_url or os.environ.get("API_BASE_URL", "https://api.groq.com/openai/v1")
+    model_name   = model_name   or os.environ.get("MODEL_NAME",   "llama-3.3-70b-versatile")
+
+    # Groq is OpenAI-compatible — same SDK, different base_url
+    client  = OpenAI(api_key=api_key, base_url=api_base_url)
+    results = []
+
+    print("\n🚀 Meta Multi-Agent v2 — Baseline Inference")
+    print(f"   Model    : {model_name}")
+    print(f"   API Base : {api_base_url}")
+    print("=" * 65)
+
+    for task_id in TASK_IDS:
+        print(f"  ▶ {task_id:<35}", end=" ", flush=True)
+        result = run_task(client, task_id, model_name)
+        results.append(result)
+        bar = "█" * int(result["score"] * 10) + "░" * (10 - int(result["score"] * 10))
+        print(f"[{bar}] {result['score']:.2f}  {result['feedback'][:50]}")
+        # Respect Groq free tier rate limit (30 req/min)
+        time.sleep(2)
+
+    avg = sum(r["score"] for r in results) / len(results)
+    print("=" * 65)
+    print(f"  📊 Average Score: {avg:.3f}  |  Tasks: {len(results)}")
+    print("=" * 65)
+    return results
+
+
+if __name__ == "__main__":
+    scores = run_all_baselines()
+    print("\nFull JSON Results:")
+    print(json.dumps(scores, indent=2))

data_generators.py

@@ -0,0 +1,708 @@
+"""
+data_generators.py — Deterministic procedural data generators for Meta v3.1
+Replaces small hand-crafted datasets with richly varied, reproducible data.
+
+Design principles
+-----------------
+* Every generator takes an `episode_id: str` and seeds Python's stdlib `random`
+  with `hash(episode_id + salt)` so the same episode always yields the same data.
+* No third-party libs needed — stdlib only (random, string, datetime, hashlib).
+* Each generator returns (agent_ctx, grader_ctx) tuples matching existing grader
+  contracts so graders need zero changes.
+
+Drop-in replacement usage in Meta_environment.py::_load_context():
+    Replace the hardcoded-data blocks with:
+        from data_generators import gen_<task>(episode_id)
+        agent_ctx, grader_ctx = gen_<task>(episode_id)
+
+Generators provided
+-------------------
+Email domain
+    gen_email_easy(episode_id)     — 1 email, 3-way class, 30 sender/subject combos
+    gen_email_medium(episode_id)   — 10 emails, varying priority pool
+    gen_email_hard(episode_id)     — customer complaint, 8 scenario templates
+
+Code domain
+    gen_code_easy(episode_id)      — 6 code snippet variants
+    gen_code_medium(episode_id)    — deterministic (same code, no variant needed)
+    gen_code_hard(episode_id)      — deterministic (same code, no variant needed)
+
+Data-cleaning domain
+    gen_data_easy(episode_id)      — 6-10 rows, random nulls + one dup pair
+    gen_data_medium(episode_id)    — 5 rows, randomised messy formats per field
+    gen_data_hard(episode_id)      — 12-16 rows, IQR-detected outliers
+
+Content moderation domain
+    gen_mod_easy(episode_id)       — 7 posts drawn from larger pool
+    gen_mod_medium(episode_id)     — 8 posts drawn from larger pool
+    (hard is already adversarial/static — left unchanged)
+
+Ticket domain
+    gen_ticket_easy(episode_id)    — 1 ticket from 8-variant pool
+    gen_ticket_medium(episode_id)  — deterministic (same 8 tickets, no variant)
+    gen_ticket_hard(episode_id)    — deterministic (same 6 incidents, no variant)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import random
+import string
+from copy import deepcopy
+from datetime import date, timedelta
+from typing import Any
+
+
+# ── Seeding helper ─────────────────────────────────────────────────────────────
+
+def _rng(episode_id: str, salt: str = "") -> random.Random:
+    seed = int(hashlib.md5((episode_id + salt).encode()).hexdigest(), 16)
+    return random.Random(seed)
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EMAIL EASY  — 30+ variant pool
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_EMAIL_EASY_POOL = [
+    # (subject, sender, body, label)
+    ("You won $1,000,000!!!", "promo@totally-legit-lottery.com",
+     "Click here to claim your prize now! Limited time offer!", "spam"),
+    ("Q3 Budget Review — Urgent Action Required", "cfo@company.com",
+     "Please review the attached Q3 budget before our board meeting tomorrow at 9 AM.", "important"),
+    ("Your weekly digest from Medium", "noreply@medium.com",
+     "Top stories this week: AI trends, Python tips, and startup news.", "newsletter"),
+    ("CONGRATULATIONS! iPhone 15 Pro is yours!", "winner@prize-alert.net",
+     "You have been selected. Click now to claim your free iPhone!", "spam"),
+    ("Action Required: Sign the NDA before Friday", "legal@company.com",
+     "Please sign the attached NDA before Friday's partnership meeting. Deadline is strict.", "important"),
+    ("TechCrunch Newsletter — March 2026", "newsletter@techcrunch.com",
+     "This week's top stories in tech, AI, and startups.", "newsletter"),
+    ("Claim your $500 Amazon gift card NOW", "deals@amazingoffers.biz",
+     "You are today's lucky winner. No purchase necessary. Act fast!", "spam"),
+    ("Production incident P1 — please join bridge", "oncall@company.com",
+     "We have a P1 production incident. Please join the war room immediately.", "important"),
+    ("Developer digest: April 2026", "digest@devweekly.io",
+     "This week: Rust 2.0 release, LLM benchmarks, and open-source highlights.", "newsletter"),
+    ("FREE vacation cruise — you've been selected!", "winner@vacation-cruise.net",
+     "Claim your free cruise for two. Just pay shipping and handling.", "spam"),
+    ("Board approval needed: Series B term sheet", "ceo@company.com",
+     "Our Series B term sheet requires your signature before the 5 PM deadline today.", "important"),
+    ("Product Hunt daily digest", "digest@producthunt.com",
+     "Today's top products: AI code review, async stand-ups, no-code CRM.", "newsletter"),
+    ("Congratulations — Survey reward $250!", "rewards@surveypanel.net",
+     "You qualify for a $250 reward. Complete our 2-minute survey to claim.", "spam"),
+    ("HR: Performance review cycle starts Monday", "hr@company.com",
+     "The annual performance review cycle begins Monday. Please complete your self-assessment.", "important"),
+    ("Hacker News digest: top 10 links this week", "weekly@hackernews.app",
+     "Rust, WASM, and the AI bubble: top discussions from the community.", "newsletter"),
+    ("Double billing detected on account #AC-8821", "billing@ourplatform.com",
+     "We noticed a duplicate charge on your account. Reply to initiate a refund.", "spam"),
+    ("Legal: Partnership agreement signature required EOD", "legal@company.com",
+     "The partnership agreement with Nexus Corp must be countersigned by 5 PM today.", "important"),
+    ("SaaS Weekly — your industry roundup", "editor@saasweekly.com",
+     "Churn rates, AI copilots, and the future of vertical SaaS.", "newsletter"),
+    ("You have unclaimed crypto rewards!", "support@cryptorewards.io",
+     "Wallet 0x4f…a2 has 0.3 ETH waiting. Connect now to claim.", "spam"),
+    ("Security alert: admin password reset requested", "security@company.com",
+     "An admin-level password reset was requested from IP 198.51.100.42. Verify or block.", "important"),
+    ("The Batch — AI newsletter by deeplearning.ai", "batch@deeplearning.ai",
+     "Andrew Ng on scaling laws, fine-tuning tips, and this week's papers.", "newsletter"),
+    ("Limited offer: 90% off VPN subscription", "deals@vpndeals.biz",
+     "Today only: get lifetime VPN access for $1.99. Offer expires midnight.", "spam"),
+    ("Vendor invoice #INV-7721 overdue by 14 days", "ar@vendor.com",
+     "Invoice #INV-7721 for $12,400 is now 14 days overdue. Please remit payment to avoid service interruption.", "important"),
+    ("The Algorithm Newsletter — weekly edition", "hello@thealgorithm.co",
+     "ML papers, job listings, and a tutorial on RLHF.", "newsletter"),
+    ("Your package was held — verify address", "delivery@postalnow.net",
+     "Your package is held due to an address discrepancy. Click to confirm.", "spam"),
+    ("Compliance: GDPR audit documentation due Friday", "compliance@company.com",
+     "Please upload your GDPR data-processing records to the compliance portal by Friday.", "important"),
+    ("Morning Brew — your daily business digest", "hello@morningbrew.com",
+     "Markets, tech, and the big stories you need to know today.", "newsletter"),
+    ("Account suspended — click to restore", "support@accountalert.biz",
+     "Unusual activity detected. Your account has been temporarily suspended.", "spam"),
+    ("CTO sync needed: infrastructure cost overrun", "cto@company.com",
+     "AWS bill came in 340% over budget. Need your sign-off on emergency cost controls.", "important"),
+    ("The Hustle — daily newsletter", "news@thehustle.co",
+     "Today's biggest business stories, curated for founders and operators.", "newsletter"),
+]
+
+
+def gen_email_easy(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "email_easy")
+    email = dict(zip(["subject", "sender", "body", "label"],
+                     rng.choice(_EMAIL_EASY_POOL)))
+    email["id"] = "e1"
+    agent_ctx  = {"email": {k: v for k, v in email.items() if k != "label"}}
+    grader_ctx = {"email": email}
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EMAIL MEDIUM  — varied priority pools
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_CRITICAL_POOL = [
+    {"subject": "[ALERT] Production server DOWN", "sender": "ops@company.com",
+     "body": "All services unresponsive since 2 AM. Revenue impact: $10k/min.", "priority": 1, "category": "critical"},
+    {"subject": "[P1] Database primary down — all writes failing", "sender": "oncall@company.com",
+     "body": "The primary DB replica crashed. No write operations possible site-wide.", "priority": 1, "category": "critical"},
+    {"subject": "[WARNING] Security alert: unusual login attempt", "sender": "security@company.com",
+     "body": "Unauthorised login attempt from IP 45.33.32.156. Verify immediately.", "priority": 1, "category": "critical"},
+    {"subject": "[ALERT] DDoS attack in progress", "sender": "security@company.com",
+     "body": "We are under a DDoS attack. Site response time is 30s+. Join the war room.", "priority": 1, "category": "critical"},
+]
+
+_URGENT_POOL = [
+    {"subject": "Acme Corp contract renewal — URGENT", "sender": "sales@company.com",
+     "body": "Acme contract expires Friday. They need signature or they walk.", "priority": 2, "category": "business"},
+    {"subject": "Legal: Partnership agreement needs signature TODAY", "sender": "legal@company.com",
+     "body": "The partnership agreement must be countersigned by EOD or the deal falls through.", "priority": 2, "category": "legal"},
+    {"subject": "Series B term sheet — board sign-off required by 5 PM", "sender": "ceo@company.com",
+     "body": "VCs need the countersigned term sheet by 5 PM today or the round lapses.", "priority": 2, "category": "legal"},
+]
+
+_FINANCE_POOL = [
+    {"subject": "Invoice #INV-4521 overdue", "sender": "billing@vendor.com",
+     "body": "Invoice of $5,400 was due 3 days ago. Payment needed to avoid suspension.", "priority": 3, "category": "finance"},
+    {"subject": "AWS bill 340% over budget", "sender": "finance@company.com",
+     "body": "Last month's cloud bill was $82k vs $24k budget. Need approval to proceed.", "priority": 3, "category": "finance"},
+    {"subject": "Customer double-charged — refund dispute open", "sender": "billing@company.com",
+     "body": "Customer #C-1121 was charged twice. Finance blocked. Needs VP sign-off.", "priority": 3, "category": "finance"},
+]
+
+_BUSINESS_POOL = [
+    {"subject": "Q4 product launch deck — review needed", "sender": "marketing@company.com",
+     "body": "Please review launch slides before Thursday. Launch is next Monday.", "priority": 4, "category": "business"},
+    {"subject": "Quarterly board deck — slides need your input", "sender": "ea@company.com",
+     "body": "Board meeting is Tuesday. Your section on growth metrics needs updating.", "priority": 4, "category": "business"},
+    {"subject": "Key account at risk — needs executive call", "sender": "cs@company.com",
+     "body": "TechGiant Corp is threatening churn. They want an exec call this week.", "priority": 4, "category": "business"},
+]
+
+_SOCIAL_POOL = [
+    {"subject": "Team lunch this Friday?", "sender": "colleague@company.com",
+     "body": "Hey, thinking of doing a team lunch Friday. You in?", "priority": 9, "category": "social"},
+    {"subject": "Happy Work Anniversary!", "sender": "hr@company.com",
+     "body": "Celebrating your 3 years with us!", "priority": 9, "category": "social"},
+    {"subject": "Office trivia night — Thursday 7 PM", "sender": "social@company.com",
+     "body": "Join us for office trivia Thursday evening. Pizza provided!", "priority": 9, "category": "social"},
+]
+
+_NEWSLETTER_POOL = [
+    {"subject": "Weekly SaaS digest", "sender": "news@saasdigest.com",
+     "body": "Top SaaS stories this week...", "priority": 10, "category": "newsletter"},
+    {"subject": "Free AI productivity webinar", "sender": "promo@webinar.io",
+     "body": "Join our free webinar on AI tools this Thursday.", "priority": 10, "category": "newsletter"},
+    {"subject": "The Batch: AI news weekly", "sender": "batch@deeplearning.ai",
+     "body": "Andrew Ng on scaling, fine-tuning, and paper highlights.", "priority": 10, "category": "newsletter"},
+]
+
+
+def gen_email_medium(episode_id: str) -> tuple[dict, dict]:
+    """
+    Always picks 2 critical, 1 urgent, 1 finance, 2 business, 2 social, 2 newsletter = 10 emails.
+    Correct ordering: criticals first, then urgent, finance, business, social, newsletter.
+    """
+    rng = _rng(episode_id, "email_medium")
+    crits   = rng.sample(_CRITICAL_POOL, k=min(2, len(_CRITICAL_POOL)))
+    urgents = rng.sample(_URGENT_POOL,   k=min(1, len(_URGENT_POOL)))
+    fins    = rng.sample(_FINANCE_POOL,  k=min(1, len(_FINANCE_POOL)))
+    bizs    = rng.sample(_BUSINESS_POOL, k=min(2, len(_BUSINESS_POOL)))
+    socials = rng.sample(_SOCIAL_POOL,   k=min(2, len(_SOCIAL_POOL)))
+    news    = rng.sample(_NEWSLETTER_POOL,k=min(2, len(_NEWSLETTER_POOL)))
+
+    ordered = crits + urgents + fins + bizs + socials + news
+    emails = []
+    correct_order = []
+    for i, e in enumerate(ordered):
+        eid = f"m{i+1}"
+        emails.append({"id": eid, **e})
+        correct_order.append(eid)
+
+    # Shuffle for presentation; keep correct_order as ground truth
+    display = emails[:]
+    rng.shuffle(display)
+
+    agent_ctx  = {"emails": [{k: v for k, v in e.items() if k not in ("priority", "category")} for e in display]}
+    grader_ctx = {"emails": emails, "correct_order": correct_order}
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EMAIL HARD  — 8 complaint scenario templates
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_EMAIL_HARD_TEMPLATES = [
+    {
+        "subject": "Completely unacceptable — Order #{order_id}",
+        "sender": "furious.customer@email.com",
+        "body": (
+            "I have been a loyal customer for 5 years and I am absolutely furious. "
+            "My order #{order_id} arrived broken for the SECOND time this month. "
+            "Your support team has been dismissive and unhelpful every time I called. "
+            "I demand a full refund, a replacement sent overnight, AND compensation. "
+            "If this is not resolved within 24 hours, I will file a consumer court complaint."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "frustrat", "disappoint", "concern", "feel"],
+            "apologize": ["sorry", "apologize", "apology", "sincerely"],
+            "offer_refund": ["refund", "money back", "full refund"],
+            "offer_replacement": ["replacement", "replace", "new order", "send another", "overnight"],
+            "provide_timeline": ["24 hours", "48 hours", "within", "today", "tomorrow", "business day"],
+            "professional_tone": None,
+            "compensation_acknowledged": ["compensat", "inconvenience", "make it right", "goodwill"],
+        },
+    },
+    {
+        "subject": "Billing error — charged twice for subscription",
+        "sender": "billing.complaint@customer.com",
+        "body": (
+            "I was charged twice for my annual subscription this month — once on March 1st "
+            "and again on March 15th. I have the bank statements as proof. "
+            "This is a clear billing error on your end. I need an immediate refund of the "
+            "duplicate charge and a written confirmation that this won't happen again. "
+            "I've been waiting 2 weeks for a response from your support team."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "concern", "apologize", "sorry"],
+            "apologize": ["sorry", "apologize", "apology"],
+            "offer_refund": ["refund", "reimburse", "credit"],
+            "confirm_investigation": ["investigat", "look into", "check", "review", "verify"],
+            "provide_timeline": ["within", "hours", "days", "business day"],
+            "professional_tone": None,
+            "written_confirmation": ["confirm", "written", "email confirmation", "receipt"],
+        },
+    },
+    {
+        "subject": "Data breach — my account was hacked",
+        "sender": "angry.user@personal.com",
+        "body": (
+            "I just discovered that my account was used to make purchases I never authorised. "
+            "Someone accessed my account and changed my email and password. "
+            "I was only able to regain access because I remembered my security questions. "
+            "This is a serious security failure on your part. I demand to know what happened, "
+            "how my data was exposed, and what you will do to protect me going forward. "
+            "If I do not hear back within 24 hours, I will contact the data protection authority."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "serious", "concern", "alarming"],
+            "apologize": ["sorry", "apologize", "apology"],
+            "confirm_investigation": ["investigat", "security team", "look into", "review"],
+            "offer_remediation": ["secure", "reset", "protect", "steps", "safeguard"],
+            "provide_timeline": ["24 hours", "48 hours", "within", "shortly", "immediately"],
+            "professional_tone": None,
+            "regulatory_mention": ["authority", "regulat", "notify", "report", "transparent"],
+        },
+    },
+    {
+        "subject": "Defective product — 3 replacements still broken",
+        "sender": "frustrated.buyer@email.com",
+        "body": (
+            "This is my THIRD replacement unit and it is still defective. "
+            "The first unit arrived with a cracked screen. The second had a faulty battery. "
+            "This third unit powers off randomly. "
+            "I have spent over 6 hours on support calls. I want a full refund immediately — "
+            "no more replacements. I will also be leaving reviews on every platform I can find."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "frustrat", "disappoint", "three", "3"],
+            "apologize": ["sorry", "apologize", "sincerely"],
+            "offer_refund": ["refund", "full refund", "money back"],
+            "no_more_replacement_push": ["will not", "no further", "respect", "understand", "preference"],
+            "provide_timeline": ["within", "24 hours", "48 hours", "business day"],
+            "professional_tone": None,
+            "de_escalation": ["value", "loyal", "appreciate", "important to us"],
+        },
+    },
+    {
+        "subject": "Package lost — 3 weeks and still nothing",
+        "sender": "lost.package@customer.net",
+        "body": (
+            "I placed an order 3 weeks ago and my package has still not arrived. "
+            "The tracking has shown 'In Transit' for 18 days without any movement. "
+            "I have contacted your support team 4 times. Each time I am told to 'wait a few more days'. "
+            "I need this item for an event next week. Either find the package, send a replacement now, "
+            "or give me a full refund so I can purchase elsewhere. This is unacceptable."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "frustrat", "waiting", "3 weeks", "three weeks"],
+            "apologize": ["sorry", "apologize", "apology"],
+            "offer_refund": ["refund", "full refund"],
+            "offer_replacement": ["replacement", "resend", "new shipment", "send another"],
+            "provide_timeline": ["24 hours", "48 hours", "immediately", "today", "next day"],
+            "professional_tone": None,
+            "carrier_investigation": ["carrier", "investigate", "trace", "courier", "tracking"],
+        },
+    },
+    {
+        "subject": "Cancelled subscription still being charged",
+        "sender": "cancelled.user@email.com",
+        "body": (
+            "I cancelled my subscription on January 15th and have proof of the cancellation email. "
+            "However, I have been charged $99/month for the past 3 months since then. "
+            "Total unauthorised charges: $297. I want a full refund of all three charges immediately. "
+            "If this is not resolved within 48 hours, I will initiate a chargeback with my bank "
+            "and report this to the consumer protection agency."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "concern", "apologize", "recognised"],
+            "apologize": ["sorry", "apologize", "apology"],
+            "offer_refund": ["refund", "full refund", "$297", "three charges", "all charges"],
+            "confirm_cancellation": ["confirm", "cancellation", "verified", "record"],
+            "provide_timeline": ["48 hours", "24 hours", "immediately", "within"],
+            "professional_tone": None,
+            "prevent_recurrence": ["prevent", "ensure", "will not happen", "steps", "corrective"],
+        },
+    },
+    {
+        "subject": "Discriminatory treatment by your staff",
+        "sender": "discrimination.complaint@email.com",
+        "body": (
+            "I visited your store on April 2nd and was followed around by a staff member "
+            "while other customers were not treated this way. When I raised this with the manager, "
+            "I was told 'it's just store policy'. This felt discriminatory and I am deeply hurt. "
+            "I expect a formal apology, an explanation of your training policies, "
+            "and assurance that this will not happen to any customer again."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "serious", "concern", "upset", "hurt"],
+            "apologize": ["sorry", "apologize", "sincerely", "formal apology"],
+            "confirm_investigation": ["investigat", "review", "look into", "hr", "management"],
+            "policy_commitment": ["policy", "training", "standards", "commit", "improve"],
+            "provide_timeline": ["within", "48 hours", "days", "shortly"],
+            "professional_tone": None,
+            "empathy_specific": ["dignity", "respect", "every customer", "valued"],
+        },
+    },
+    {
+        "subject": "Software destroyed my data — urgent",
+        "sender": "data.loss@business.com",
+        "body": (
+            "Your software update on March 28th corrupted our entire database backup. "
+            "We lost 6 months of customer records. Our business has been at a standstill for 3 days. "
+            "We are a paying enterprise customer ($24,000/year). "
+            "I need immediate escalation to your engineering team, a post-mortem report, "
+            "and a discussion about compensation for our business losses."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "serious", "impact", "significant", "recognise"],
+            "apologize": ["sorry", "sincerely", "deeply apologize"],
+            "escalate": ["engineering", "escalate", "senior", "team", "immediately"],
+            "post_mortem": ["post-mortem", "root cause", "analysis", "report", "findings"],
+            "provide_timeline": ["24 hours", "immediately", "today", "within"],
+            "professional_tone": None,
+            "compensation_acknowledged": ["compensat", "discuss", "impact", "losses", "make it right"],
+        },
+    },
+]
+
+
+def gen_email_hard(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "email_hard")
+    template = rng.choice(_EMAIL_HARD_TEMPLATES)
+    order_id = rng.randint(10000, 99999)
+    subject  = template["subject"].format(order_id=order_id)
+    body     = template["body"].format(order_id=order_id)
+    case = {**template, "subject": subject, "body": body, "sender": template["sender"], "id": "h1"}
+    agent_ctx  = {"email": {k: v for k, v in case.items() if k not in ("expected_elements", "id")}}
+    grader_ctx = {"email": case}
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CODE EASY  — 6 snippet variants
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_CODE_EASY_POOL = [
+    {"code": "def calculate_average(numbers):\n    total = 0\n    for num in numbers\n        total += num\n    return total / len(numbers)\n",
+     "keywords": ["colon", "for loop", "syntax", "for statement"]},
+    {"code": "def greet(name)\n    print(f'Hello, {name}!')\n\ngreet('World')\n",
+     "keywords": ["colon", "def", "function", "syntax"]},
+    {"code": "numbers = [1, 2, 3, 4, 5]\nif len(numbers) > 0\n    print('Not empty')\n",
+     "keywords": ["colon", "if statement", "syntax"]},
+    {"code": "class Animal:\n    def __init__(self, name)\n        self.name = name\n\n    def speak(self):\n        return f'{self.name} speaks'\n",
+     "keywords": ["colon", "__init__", "constructor", "function", "def"]},
+    {"code": "def read_file(path):\n    with open(path) as f\n        return f.read()\n",
+     "keywords": ["colon", "with statement", "context manager", "syntax"]},
+    {"code": "def retry(fn, times=3):\n    for i in range(times)\n        try:\n            return fn()\n        except Exception:\n            pass\n",
+     "keywords": ["colon", "for loop", "range", "syntax"]},
+]
+
+
+def gen_code_easy(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "code_easy")
+    variant    = rng.choice(_CODE_EASY_POOL)
+    agent_ctx  = {"code": variant["code"]}
+    grader_ctx = {"code": variant["code"], "keywords": variant["keywords"]}
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# DATA EASY  — procedurally generated 6-10 rows
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_FIRST_NAMES = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Hiro", "Iris", "Jake"]
+_DOMAINS = ["example.com", "company.net", "test.org", "work.io"]
+
+
+def gen_data_easy(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "data_easy")
+    n = rng.randint(6, 10)
+    names = rng.sample(_FIRST_NAMES, k=n)
+    base_salary = 50000
+
+    rows = []
+    for i, name in enumerate(names, 1):
+        rows.append({
+            "id": i,
+            "name": name,
+            "age": rng.randint(22, 55),
+            "email": f"{name.lower()}@{rng.choice(_DOMAINS)}",
+            "salary": base_salary + rng.randint(0, 50) * 1000,
+        })
+
+    # Inject 3-4 nulls in distinct rows and fields
+    null_fields = rng.sample(["age", "name", "email", "salary"], k=rng.randint(3, 4))
+    null_rows   = rng.sample(list(range(n)), k=len(null_fields))
+    missing_answers = []
+    for row_idx, field in zip(null_rows, null_fields):
+        rows[row_idx][field] = None
+        missing_answers.append(f"{field} (row {row_idx + 1})")
+
+    # Inject one duplicate pair
+    dup_src = rng.randint(0, n - 2)
+    dup_dst = dup_src + 1
+    rows[dup_dst] = {**rows[dup_src], "id": dup_dst + 1}
+    dup_ids = sorted([rows[dup_src]["id"], rows[dup_dst]["id"]])
+
+    agent_ctx  = {"data": rows}
+    grader_ctx = {
+        "data": rows,
+        "answers": {"missing": sorted(missing_answers), "duplicates": dup_ids},
+    }
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# DATA MEDIUM  — randomised messy formats per field
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_AGE_STRINGS   = ["thirty", "twenty-two", "forty-five", "nineteen", "fifty"]
+_DATE_FORMATS  = [("%Y/%m/%d", "2020/01/15"), ("%d-%m-%Y", "15-03-2021"),
+                  ("%Y-%m-%d", "2019-07-22"), ("%Y/%m/%d", "2022/11/01"),
+                  ("%d/%m/%Y", "01/06/2023")]
+_ACTIVE_MESSY  = ["yes", "no", "TRUE", "FALSE", "1", "0"]
+
+
+def gen_data_medium(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "data_medium")
+    names = rng.sample(_FIRST_NAMES, k=5)
+
+    rows = []
+    salaries = [50000, 60000, 70000, 80000, 45000]
+    for i, (name, salary) in enumerate(zip(names, salaries), 1):
+        # Introduce one of each issue type
+        if i == 1:
+            age_val = rng.choice(_AGE_STRINGS)        # string age
+            sal_val = f"${salary:,}"                   # dollar+comma format
+        elif i == 2:
+            age_val = 25                               # correct
+            sal_val = f"${salary:,}"                   # dollar+comma
+        elif i == 3:
+            age_val = str(rng.randint(25, 40))         # string digit
+            sal_val = str(salary)                      # clean
+        elif i == 4:
+            age_val = rng.randint(30, 45)              # int — correct
+            sal_val = f"{salary:,}"                    # comma only
+        else:
+            age_val = rng.choice(_AGE_STRINGS)         # string age
+            sal_val = str(salary)                      # clean
+
+        _, date_str = rng.choice(_DATE_FORMATS)
+        name_messy = name.lower() if i == 4 else name.upper() if i == 5 else name
+        active_val = rng.choice(_ACTIVE_MESSY) if i in (1, 3, 5) else rng.choice([True, False])
+
+        rows.append({"id": i, "name": name_messy, "age": age_val, "salary": sal_val,
+                     "join_date": date_str, "active": active_val})
+
+    issues = {
+        "age":       "mixed types: some ages are word strings, should be integers",
+        "salary":    "inconsistent formats: dollar signs and commas present in some values",
+        "join_date": "three different date formats used across rows",
+        "name":      "inconsistent casing: some names lowercase, some all-caps",
+        "active":    "mixed types: booleans and strings ('yes','no','TRUE') mixed",
+    }
+    agent_ctx  = {"data": rows}
+    grader_ctx = {"data": rows, "issues": issues}
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# DATA HARD  — IQR-detectable outliers, missing values
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def gen_data_hard(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "data_hard")
+    n = rng.randint(12, 16)
+    base = round(rng.uniform(9.5, 12.5), 1)
+    spread = round(rng.uniform(0.3, 0.8), 2)
+
+    rows = []
+    for i in range(1, n + 1):
+        val = round(base + rng.gauss(0, spread), 2)
+        rows.append({"id": i, "value": val})
+
+    # Pick 2 outlier positions (ensure they're not neighbours)
+    positions = list(range(n))
+    rng.shuffle(positions)
+    o1, o2 = sorted(positions[:2])
+    outlier_mult = rng.choice([8, 10, 12, -5, -8])
+    rows[o1]["value"] = round(base * abs(outlier_mult) * (1 if outlier_mult > 0 else -1), 1)
+    outlier_mult2 = rng.choice([-4, -6, 15, 20])
+    rows[o2]["value"] = round(base * abs(outlier_mult2) * (1 if outlier_mult2 > 0 else -1), 1)
+    outlier_ids = sorted([o1 + 1, o2 + 1])
+
+    # Pick 2 missing positions (not overlapping outliers)
+    available = [i for i in range(n) if i not in (o1, o2)]
+    m1, m2 = sorted(rng.sample(available, 2))
+    rows[m1]["value"] = None
+    rows[m2]["value"] = None
+    missing_ids = sorted([m1 + 1, m2 + 1])
+
+    normal_vals = [r["value"] for r in rows if r["value"] is not None and r["id"] not in outlier_ids]
+    lo = round(min(normal_vals) - 1.0, 2)
+    hi = round(max(normal_vals) + 1.0, 2)
+
+    agent_ctx  = {"data": rows}
+    grader_ctx = {
+        "data": rows,
+        "answers": {
+            "outliers":      set(outlier_ids),
+            "missing":       set(missing_ids),
+            "imputed_range": (lo, hi),
+        },
+    }
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CONTENT MODERATION EASY  — draw 7 from a pool of 20
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_MOD_EASY_POOL = [
+    {"text": "I hate you so much, I want to hurt you!",                      "label": "harmful"},
+    {"text": "Great weather today! Loving this sunshine",                     "label": "safe"},
+    {"text": "All people from that group should be eliminated.",              "label": "harmful"},
+    {"text": "Can someone recommend a good Python beginner book?",            "label": "safe"},
+    {"text": "You're an absolute idiot and nobody likes you.",               "label": "harmful"},
+    {"text": "I just finished a 5k run — feeling amazing!",                  "label": "safe"},
+    {"text": "People like you should just disappear forever.",               "label": "harmful"},
+    {"text": "Just dropped off a care package for my neighbor.",             "label": "safe"},
+    {"text": "Kill yourself, nobody would miss you.",                        "label": "harmful"},
+    {"text": "Anyone else excited for the new season of that show?",         "label": "safe"},
+    {"text": "I'm going to find you and make you pay for this.",             "label": "harmful"},
+    {"text": "Finally got my herb garden started!",                          "label": "safe"},
+    {"text": "These people deserve to suffer for what they've done.",        "label": "harmful"},
+    {"text": "Happy birthday to my best friend!",                            "label": "safe"},
+    {"text": "Someone should shoot that politician.",                        "label": "harmful"},
+    {"text": "Loving the new coffee shop that opened downtown.",             "label": "safe"},
+    {"text": "I hope your family gets what's coming to them.",               "label": "harmful"},
+    {"text": "Just hit 100 days of learning Spanish — so proud!",           "label": "safe"},
+    {"text": "People from that country are all criminals.",                  "label": "harmful"},
+    {"text": "The cherry blossoms are out — spring is finally here!",       "label": "safe"},
+]
+
+
+def gen_mod_easy(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "mod_easy")
+    # Ensure at least 3 harmful, 3 safe among 7
+    harmful = [p for p in _MOD_EASY_POOL if p["label"] == "harmful"]
+    safe    = [p for p in _MOD_EASY_POOL if p["label"] == "safe"]
+    chosen  = rng.sample(harmful, 4) + rng.sample(safe, 3)
+    rng.shuffle(chosen)
+    posts = [{"id": f"c{i+1}", **p} for i, p in enumerate(chosen)]
+    agent_ctx  = {"posts": [{"id": p["id"], "text": p["text"]} for p in posts]}
+    grader_ctx = {"posts": posts}
+    return agent_ctx, grader_ctx
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CONTENT MODERATION MEDIUM  — draw 8 from a pool of 20
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_MOD_MEDIUM_POOL = [
+    {"text": "Oh sure, like THAT went well. Classic genius move.",              "label": "harmful"},
+    {"text": "I'm so done with these people. Absolutely insufferable.",         "label": "harmful"},
+    {"text": "That presentation was... very unique.",                           "label": "safe"},
+    {"text": "Some people just shouldn't be allowed to have opinions.",        "label": "harmful"},
+    {"text": "Oh wow, another 'brilliant' idea from upper management",         "label": "harmful"},
+    {"text": "Congrats on the promotion! Well deserved!",                      "label": "safe"},
+    {"text": "Not everyone is cut out for this kind of work, clearly.",        "label": "harmful"},
+    {"text": "Thanks for the feedback, I'll take it into account.",            "label": "safe"},
+    {"text": "Some of us actually read the requirements before submitting.",   "label": "harmful"},
+    {"text": "Good effort — interesting approach to the problem.",             "label": "safe"},
+    {"text": "If you'd done even a little research, you'd know better.",       "label": "harmful"},
+    {"text": "Really enjoyed the team retrospective today.",                   "label": "safe"},
+    {"text": "Must be nice having a job you're actually qualified for.",       "label": "harmful"},
+    {"text": "Great work getting the release out on time!",                    "label": "safe"},
+    {"text": "Shocking that anyone thought that plan would work.",             "label": "harmful"},
+    {"text": "Just wanted to say — the onboarding docs are really clear.",    "label": "safe"},
+    {"text": "The bar for hiring has clearly dropped.",                        "label": "harmful"},
+    {"text": "Appreciate everyone's patience during the migration.",           "label": "safe"},
+    {"text": "I see we've decided quantity over quality again.",               "label": "harmful"},
+    {"text": "The demo went really smoothly — nice job everyone.",            "label": "safe"},
+]
+
+
+def gen_mod_medium(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "mod_medium")
+    harmful = [p for p in _MOD_MEDIUM_POOL if p["label"] == "harmful"]
+    safe    = [p for p in _MOD_MEDIUM_POOL if p["label"] == "safe"]
+    chosen  = rng.sample(harmful, 5) + rng.sample(safe, 3)
+    rng.shuffle(chosen)
+    posts = [{"id": f"m{i+1}", **p} for i, p in enumerate(chosen)]
+    agent_ctx  = {"posts": [{"id": p["id"], "text": p["text"]} for p in posts]}
+    grader_ctx = {"posts": posts}
+    return agent_ctx, grader_ctx
+
+
+# ══��════════════════════════════════════════════════════════════════════════════
+# TICKET EASY  — 8 variant pool (already existed; keep contract-compatible)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+_TICKET_EASY_POOL = [
+    {"id": "t1", "title": "App crashes on login for all users",
+     "description": "Since the 2.3.1 deploy, 100% of login attempts throw a 500 error.",
+     "reporter": "ops-monitoring@company.com", "correct_priority": "critical", "correct_category": "bug"},
+    {"id": "t2", "title": "Add dark mode to the dashboard",
+     "description": "Multiple users have requested a dark mode option.",
+     "reporter": "product@company.com", "correct_priority": "low", "correct_category": "feature_request"},
+    {"id": "t3", "title": "How do I export my data as CSV?",
+     "description": "I can't find the export button anywhere in the UI.",
+     "reporter": "user123@customer.com", "correct_priority": "low", "correct_category": "question"},
+    {"id": "t4", "title": "Charged twice for last month's invoice",
+     "description": "Account #AC-4421 shows two charges of $299 on March 1st.",
+     "reporter": "finance@customer.com", "correct_priority": "high", "correct_category": "billing"},
+    {"id": "t5", "title": "Memory leak in mobile app — crashes after 10 min",
+     "description": "The iOS app uses 2 GB RAM then crashes. Reproducible on all devices.",
+     "reporter": "qa@company.com", "correct_priority": "high", "correct_category": "bug"},
+    {"id": "t6", "title": "Add SSO support for Google Workspace",
+     "description": "Enterprise customers are requesting Google SSO. High deal-blocker frequency.",
+     "reporter": "sales@company.com", "correct_priority": "medium", "correct_category": "feature_request"},
+    {"id": "t7", "title": "Refund policy — how long does it take?",
+     "description": "A customer is asking about the refund timeline for a cancelled subscription.",
+     "reporter": "support@company.com", "correct_priority": "low", "correct_category": "question"},
+    {"id": "t8", "title": "Wrong amount charged — $0 instead of $499",
+     "description": "New enterprise contract shows $0 on the invoice. Revenue loss.",
+     "reporter": "billing@company.com", "correct_priority": "critical", "correct_category": "billing"},
+]
+
+
+def gen_ticket_easy(episode_id: str) -> tuple[dict, dict]:
+    rng = _rng(episode_id, "ticket_easy")
+    ticket     = rng.choice(_TICKET_EASY_POOL)
+    agent_ctx  = {"ticket": {k: v for k, v in ticket.items() if k not in ("correct_priority", "correct_category")}}
+    grader_ctx = {"ticket": ticket}
+    return agent_ctx, grader_ctx

expert_tasks.py

@@ -0,0 +1,656 @@
+"""
+expert_tasks.py — Expert-tier tasks for Meta v3.1
+Two per domain. Multi-step reasoning required. Frontier models score ≤ 0.45.
+
+Design
+------
+Each expert task requires the agent to:
+  1. Reason across multiple pieces of evidence
+  2. Synthesise a correct answer that isn't directly stated in the context
+  3. Produce structured output that is graded on multiple sub-criteria
+
+Drop these into Meta_environment.py:
+  - Add the task IDs to TASK_IDS and openenv.yaml
+  - Add context loaders to _load_context()
+  - Add grade functions to _grade()
+  - Add instructions to INSTRUCTIONS
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EMAIL EXPERT
+# ═══════════════════════════════════════════════════════════════════════════════
+
+EMAIL_EXPERT_CONTEXT = {
+    "thread": [
+        {
+            "from": "sarah.chen@bigclient.com",
+            "to": "support@company.com",
+            "date": "2026-03-10 09:15",
+            "subject": "Re: Re: Re: Our data export is wrong — ESCALATING",
+            "body": (
+                "This is the FOURTH week we have been raising this issue. "
+                "The data export from your platform is missing 12% of our transaction records, "
+                "dating back to February 1st. Our CFO has flagged this as a potential compliance risk. "
+                "I am now formally requesting a root-cause analysis, a corrected export, "
+                "and a meeting with your VP of Engineering and VP of Customer Success. "
+                "If we do not hear back with a concrete plan within 24 hours, "
+                "we will be engaging our legal team."
+            ),
+        },
+        {
+            "from": "support@company.com",
+            "to": "sarah.chen@bigclient.com",
+            "date": "2026-03-09 14:22",
+            "subject": "Re: Re: Our data export is wrong",
+            "body": (
+                "Hi Sarah, our engineering team is still investigating. "
+                "We expect an update by end of week. Apologies for the delay."
+            ),
+        },
+        {
+            "from": "sarah.chen@bigclient.com",
+            "to": "support@company.com",
+            "date": "2026-03-05 11:30",
+            "subject": "Re: Our data export is wrong",
+            "body": (
+                "It has been 10 days and no fix. "
+                "We have a regulatory audit on March 20th. This is becoming critical."
+            ),
+        },
+        {
+            "from": "sarah.chen@bigclient.com",
+            "to": "support@company.com",
+            "date": "2026-02-24 16:05",
+            "subject": "Our data export is wrong",
+            "body": (
+                "Hello, we noticed our February data export is missing records. "
+                "Can you please investigate?"
+            ),
+        },
+    ],
+    "account_metadata": {
+        "account_name": "BigClient Corp",
+        "tier": "Enterprise",
+        "contract_value": 480000,
+        "account_age_years": 4,
+        "renewal_date": "2026-06-01",
+        "open_issues": 3,
+        "last_csat_score": 5.2,
+    },
+    "known_bug": {
+        "status": "confirmed",
+        "root_cause": "Daylight saving time offset bug in the data export pipeline affecting records from Feb 1-14",
+        "fix_eta": "2026-03-15",
+        "records_affected": "12% of February transaction exports",
+    },
+}
+
+EMAIL_EXPERT_EXPECTED = {
+    "escalation_acknowledged": ["escalat", "four weeks", "4 weeks", "urgent", "seriou", "concern"],
+    "vp_meeting_committed": ["vp", "engineering", "customer success", "meeting", "schedule", "arrange"],
+    "root_cause_disclosed": ["daylight", "dst", "timezone", "offset", "february", "feb", "export pipeline"],
+    "timeline_for_fix": ["march 15", "march 15th", "3/15", "this week", "by friday", "within"],
+    "corrected_export_promised": ["corrected export", "re-export", "new export", "corrected data", "send you"],
+    "legal_de_escalation": ["legal", "understand", "resolution", "committed", "avoid", "together"],
+    "executive_tone": None,  # checked via length + no informal language
+    "retention_focus": ["value", "partnership", "4 years", "four years", "important", "priority", "renew"],
+}
+
+EMAIL_EXPERT_INSTRUCTIONS = (
+    "You are the VP of Customer Success. Read the full email thread AND the account metadata carefully.\n\n"
+    "You must draft an executive-level response that:\n"
+    "  1. Acknowledges the full history (4 weeks of escalation)\n"
+    "  2. Commits to scheduling the VP meeting requested\n"
+    "  3. Discloses the root cause (you have the engineering report)\n"
+    "  4. Provides the concrete fix timeline (March 15)\n"
+    "  5. Commits to sending a corrected export\n"
+    "  6. De-escalates the legal threat through concrete action\n"
+    "  7. Uses appropriate executive tone (not boilerplate support language)\n"
+    "  8. Reinforces the long-term partnership value\n\n"
+    "This is a $480k account up for renewal in 3 months. Every word matters.\n\n"
+    'Payload: {"reply": "<full executive reply — minimum 200 words>"}'
+)
+
+
+def grade_email_expert(payload: dict) -> tuple[float, str, dict]:
+    reply = payload.get("reply", "").lower()
+    checks: dict[str, bool] = {}
+    for key, keywords in EMAIL_EXPERT_EXPECTED.items():
+        if key == "executive_tone":
+            checks[key] = (
+                len(reply) >= 200
+                and not any(w in reply for w in ["ticket", "our team will", "apologies for the inconvenience",
+                                                  "we understand your frustration", "please be patient"])
+            )
+        else:
+            checks[key] = keywords is not None and any(w in reply for w in keywords)
+    passed = sum(checks.values())
+    score  = round(passed / len(checks), 2)
+    tag    = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.6 else "[ERROR]"
+    missing = [k for k, v in checks.items() if not v]
+    feedback = f"{tag} {passed}/{len(checks)} executive reply elements present."
+    if missing:
+        feedback += f" Missing: {', '.join(missing)}."
+    return score, feedback, checks
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CODE EXPERT
+# ═══════════════════════════════════════════════════════════════════════════════
+
+CODE_EXPERT_CONTEXT = {
+    "code": """\
+import jwt
+import hmac
+import hashlib
+import os
+import subprocess
+import yaml
+import json
+from functools import wraps
+
+SECRET_KEY = "hardcoded-jwt-secret-change-me"
+
+# Vulnerability 1: Hardcoded secret + weak JWT algorithm default
+def create_token(user_id: int, role: str) -> str:
+    payload = {"user_id": user_id, "role": role}
+    return jwt.encode(payload, SECRET_KEY)  # defaults to HS256 with hardcoded key
+
+# Vulnerability 2: JWT none algorithm attack — algorithm not validated
+def verify_token(token: str) -> dict:
+    return jwt.decode(token, SECRET_KEY, algorithms=["HS256", "none"])
+
+# Vulnerability 3: Mass assignment — user-controlled fields merged into DB update
+def update_user_profile(user_id: int, form_data: dict) -> None:
+    db.execute("UPDATE users SET ? WHERE id = ?", (form_data, user_id))
+
+# Vulnerability 4: Insecure direct object reference (IDOR)
+def get_invoice(invoice_id: int) -> dict:
+    return db.query("SELECT * FROM invoices WHERE id = ?", (invoice_id,))
+
+# Vulnerability 5: Unsafe YAML deserialization
+def load_config(yaml_string: str) -> dict:
+    return yaml.load(yaml_string)  # yaml.load without Loader — RCE risk
+
+# Vulnerability 6: Race condition / TOCTOU in file write
+def save_report(filename: str, content: str) -> None:
+    if not os.path.exists(f"/reports/{filename}"):
+        with open(f"/reports/{filename}", "w") as f:
+            f.write(content)
+
+# Vulnerability 7: Prototype pollution via JSON merge (Python dict equivalent)
+def merge_settings(base: dict, override: dict) -> dict:
+    result = base.copy()
+    for key, value in override.items():
+        result[key] = value  # no key validation — __proto__ / __class__ attack
+    return result
+
+# Vulnerability 8: Log injection — unsanitized user input in log string
+def log_action(username: str, action: str) -> None:
+    print(f"[AUDIT] User: {username} | Action: {action}")
+""",
+    "vulnerability_count": 8,
+    "vulnerability_list": [
+        {"id": "v1", "type": "hardcoded_secret",       "location": "create_token",    "severity": "critical"},
+        {"id": "v2", "type": "jwt_none_algorithm",     "location": "verify_token",    "severity": "critical"},
+        {"id": "v3", "type": "mass_assignment",        "location": "update_user_profile", "severity": "high"},
+        {"id": "v4", "type": "idor",                   "location": "get_invoice",     "severity": "high"},
+        {"id": "v5", "type": "unsafe_deserialization", "location": "load_config",     "severity": "critical"},
+        {"id": "v6", "type": "race_condition",         "location": "save_report",     "severity": "medium"},
+        {"id": "v7", "type": "prototype_pollution",    "location": "merge_settings",  "severity": "medium"},
+        {"id": "v8", "type": "log_injection",          "location": "log_action",      "severity": "low"},
+    ],
+}
+
+CODE_EXPERT_INSTRUCTIONS = (
+    "Perform a thorough security audit on this Python codebase. "
+    "Identify ALL 8 security vulnerabilities, including subtle ones like JWT algorithm confusion, "
+    "mass assignment, IDOR, TOCTOU race conditions, prototype pollution, and log injection.\n\n"
+    "For each vulnerability provide: type, location (function name), severity (critical/high/medium/low), "
+    "and a concrete fix.\n\n"
+    "Hints: Look beyond the obvious injection flaws. Consider authentication logic, access control, "
+    "concurrency, deserialization, and output encoding.\n\n"
+    'Payload: {"vulnerabilities": [{"type": "...", "location": "...", "severity": "...", "fix": "..."}]}'
+)
+
+
+def grade_code_expert(payload: dict) -> tuple[float, str, dict]:
+    vulns = payload.get("vulnerabilities", [])
+    checks = {
+        "hardcoded_secret":       False,
+        "jwt_none_algorithm":     False,
+        "mass_assignment":        False,
+        "idor":                   False,
+        "unsafe_deserialization": False,
+        "race_condition":         False,
+        "prototype_pollution":    False,
+        "log_injection":          False,
+    }
+    for v in vulns:
+        vt  = str(v.get("type", "")).lower()
+        loc = str(v.get("location", "")).lower()
+        fix = str(v.get("fix", "")).lower()
+
+        if "create_token" in loc or any(w in vt for w in ["hardcoded", "secret", "env", "rotate"]):
+            checks["hardcoded_secret"] = True
+        if "verify_token" in loc or any(w in vt for w in ["none", "algorithm", "jwt", "alg"]):
+            checks["jwt_none_algorithm"] = True
+        if "update_user" in loc or any(w in vt for w in ["mass", "assignment", "whitelist", "allowlist"]):
+            checks["mass_assignment"] = True
+        if "get_invoice" in loc or any(w in vt for w in ["idor", "direct object", "authorization", "ownership"]):
+            checks["idor"] = True
+        if "load_config" in loc or any(w in vt for w in ["yaml", "deserializ", "rce", "unsafe_load", "safe_load"]):
+            checks["unsafe_deserialization"] = True
+        if "save_report" in loc or any(w in vt for w in ["race", "toctou", "time of check", "atomic"]):
+            checks["race_condition"] = True
+        if "merge_settings" in loc or any(w in vt for w in ["prototype", "pollution", "__proto__", "key valid"]):
+            checks["prototype_pollution"] = True
+        if "log_action" in loc or any(w in vt for w in ["log inject", "log forge", "crlf", "newline", "sanitize"]):
+            checks["log_injection"] = True
+
+    passed  = sum(checks.values())
+    score   = round(passed / 8, 2)
+    tag     = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    missing = [k for k, v in checks.items() if not v]
+    feedback = f"{tag} {passed}/8 expert vulnerabilities found."
+    if missing:
+        feedback += f" Missed: {', '.join(missing)}."
+    return score, feedback, checks
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# DATA EXPERT  — cross-dataset join, statistical anomaly detection
+# ═══════════════════════════════════════════════════════════════════════════════
+
+DATA_EXPERT_CONTEXT = {
+    "dataset_a": {
+        "name": "Sales transactions (March 2026)",
+        "rows": [
+            {"txn_id": "T001", "customer_id": "C101", "amount": 1200.00, "currency": "USD", "date": "2026-03-01", "rep_id": "R05"},
+            {"txn_id": "T002", "customer_id": "C102", "amount": 850.50,  "currency": "USD", "date": "2026-03-02", "rep_id": "R05"},
+            {"txn_id": "T003", "customer_id": "C103", "amount": None,    "currency": "USD", "date": "2026-03-03", "rep_id": "R12"},
+            {"txn_id": "T004", "customer_id": "C104", "amount": 2300.00, "currency": "GBP", "date": "2026-03-04", "rep_id": "R12"},
+            {"txn_id": "T005", "customer_id": "C105", "amount": 475.00,  "currency": "USD", "date": "2026-03-05", "rep_id": "R05"},
+            {"txn_id": "T006", "customer_id": "C101", "amount": 1200.00, "currency": "USD", "date": "2026-03-01", "rep_id": "R05"},
+            {"txn_id": "T007", "customer_id": "C106", "amount": 99999.99,"currency": "USD", "date": "2026-03-07", "rep_id": "R05"},
+            {"txn_id": "T008", "customer_id": "C107", "amount": 920.00,  "currency": "EUR", "date": "2026-03-08", "rep_id": "R12"},
+            {"txn_id": "T009", "customer_id": "C108", "amount": 1100.00, "currency": "USD", "date": "2026-03-09", "rep_id": "R05"},
+            {"txn_id": "T010", "customer_id": "C109", "amount": 0.00,    "currency": "USD", "date": "2026-03-10", "rep_id": "R12"},
+        ],
+    },
+    "dataset_b": {
+        "name": "Currency conversion rates (2026-03-15)",
+        "rows": [
+            {"currency": "USD", "rate_to_usd": 1.000},
+            {"currency": "GBP", "rate_to_usd": 1.265},
+            {"currency": "EUR", "rate_to_usd": 1.082},
+        ],
+    },
+    "task_description": (
+        "You have two datasets: (A) a sales transaction log and (B) currency conversion rates. "
+        "Perform the following analysis and return results as structured JSON:\n"
+        "1. Identify all data quality issues across both datasets (missing values, duplicates, outliers, anomalies).\n"
+        "2. Join the datasets: convert all transaction amounts to USD using dataset B.\n"
+        "3. Compute the total USD revenue per sales rep (R05 and R12), EXCLUDING duplicate and zero-amount transactions.\n"
+        "4. Flag any suspicious transactions with a reason (statistical outlier, duplicate, zero amount)."
+    ),
+}
+
+DATA_EXPERT_EXPECTED = {
+    # Ground truth
+    "duplicate_txn": ["T001", "T006"],   # same customer, amount, date, rep
+    "zero_txn": ["T010"],                # $0 amount
+    "missing_amount": ["T003"],
+    "outlier_txn": ["T007"],             # $99,999.99 — extreme IQR outlier
+    # After exclusions: T003 (missing), T006 (dup), T007 (outlier), T010 (zero) excluded
+    # T004: 2300 GBP × 1.265 = 2909.50 USD → R12
+    # T008: 920 EUR × 1.082 = 995.44 USD → R12
+    # R05 valid txns: T001(1200), T002(850.5), T005(475), T009(1100) = 3625.50
+    # R12 valid txns: T004(2909.50), T008(995.44) = 3904.94  [T003 missing, excluded]
+    "r05_usd_range": (3500, 3700),    # allow rounding tolerance
+    "r12_usd_range": (3800, 4100),
+}
+
+DATA_EXPERT_INSTRUCTIONS = (
+    "Perform a multi-dataset analysis across Sales Transactions (A) and Currency Rates (B).\n\n"
+    "Required steps:\n"
+    "  1. Identify ALL data quality issues: missing values, exact duplicates, zero-amount transactions, "
+    "     and statistical outliers (use IQR or z-score on USD-equivalent amounts)\n"
+    "  2. Convert all transaction amounts to USD using the exchange rates in Dataset B\n"
+    "  3. Calculate total USD revenue per sales rep (R05 and R12), EXCLUDING:\n"
+    "     - Duplicate transactions (keep first occurrence only)\n"
+    "     - Zero-amount transactions\n"
+    "     - Missing-amount transactions\n"
+    "     - Statistical outliers\n"
+    "  4. Flag each suspicious transaction with a reason\n\n"
+    "Payload:\n"
+    "{\n"
+    '  "data_quality_issues": ["T006 duplicate of T001", "T010 zero amount", ...],\n'
+    '  "suspicious_transactions": [{"txn_id": "T007", "reason": "outlier: $99,999.99"}],\n'
+    '  "revenue_by_rep": {"R05": 3625.50, "R12": 3904.94},\n'
+    '  "excluded_transactions": ["T003", "T006", "T007", "T010"]\n'
+    "}"
+)
+
+
+def grade_data_expert(payload: dict) -> tuple[float, str, dict]:
+    issues_text   = " ".join(str(x) for x in payload.get("data_quality_issues", [])).lower()
+    suspicious    = payload.get("suspicious_transactions", [])
+    revenue       = payload.get("revenue_by_rep", {})
+    excluded      = [str(x) for x in payload.get("excluded_transactions", [])]
+
+    exp = DATA_EXPERT_EXPECTED
+    checks = {
+        "duplicate_found":    all(t.lower() in issues_text or t in excluded for t in exp["duplicate_txn"]),
+        "zero_found":         any(t.lower() in issues_text or t in excluded for t in exp["zero_txn"]),
+        "missing_found":      any(t.lower() in issues_text or t in excluded for t in exp["missing_amount"]),
+        "outlier_found":      any(t.lower() in issues_text or t in excluded for t in exp["outlier_txn"])
+                              or any(str(s.get("txn_id","")).upper() in exp["outlier_txn"] for s in suspicious),
+        "r05_revenue_correct": False,
+        "r12_revenue_correct": False,
+        "exclusions_correct":  all(t in excluded for t in ["T003", "T006", "T007", "T010"]),
+        "suspicious_flagged":  len(suspicious) >= 2,
+    }
+
+    # Revenue checks with tolerance
+    try:
+        r05 = float(revenue.get("R05", revenue.get("r05", 0)))
+        lo, hi = exp["r05_usd_range"]
+        checks["r05_revenue_correct"] = lo <= r05 <= hi
+    except (ValueError, TypeError):
+        pass
+    try:
+        r12 = float(revenue.get("R12", revenue.get("r12", 0)))
+        lo, hi = exp["r12_usd_range"]
+        checks["r12_revenue_correct"] = lo <= r12 <= hi
+    except (ValueError, TypeError):
+        pass
+
+    passed  = sum(checks.values())
+    score   = round(passed / len(checks), 2)
+    tag     = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    missing = [k for k, v in checks.items() if not v]
+    feedback = f"{tag} {passed}/{len(checks)} data expert checks passed."
+    if missing:
+        feedback += f" Failed: {', '.join(missing)}."
+    return score, feedback, checks
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CONTENT MODERATION EXPERT  — policy design + edge-case rulings
+# ═══════════════════════════════════════════════════════���═══════════════════════
+
+MOD_EXPERT_CONTEXT = {
+    "scenario": (
+        "You are the Trust & Safety Policy Lead at a major social platform. "
+        "The policy team has presented you with 5 contested content moderation cases. "
+        "Each case includes a post, the platform context, and a set of competing arguments "
+        "from policy reviewers. You must: (1) make a final ruling, (2) cite the specific policy "
+        "principle that governs your decision, and (3) explain the real-world harm or absence "
+        "of harm that justifies the ruling."
+    ),
+    "cases": [
+        {
+            "id": "p1",
+            "post": "I think about ending it all every day but I don't know who to talk to.",
+            "platform": "general_social_media",
+            "reviewer_a": "REMOVE: suicidal ideation should be automatically removed.",
+            "reviewer_b": "KEEP + SUPPORT: expression of distress, not encouragement. Flag for crisis resources.",
+            "correct_ruling": "keep_with_intervention",
+            "correct_principle": "harm_reduction",
+            "harm_keywords": ["crisis", "resources", "support", "intervention", "expression", "distress", "not encourage"],
+        },
+        {
+            "id": "p2",
+            "post": "Our community will not let these invaders replace our culture. Rise up.",
+            "platform": "extremist_adjacent_forum",
+            "reviewer_a": "KEEP: vague political speech, doesn't name a group or call for specific violence.",
+            "reviewer_b": "REMOVE: great replacement rhetoric + implicit call to action on a platform known for radicalization.",
+            "correct_ruling": "remove",
+            "correct_principle": "platform_context_amplifies_harm",
+            "harm_keywords": ["replace", "context", "platform", "radicali", "implicit", "call to action"],
+        },
+        {
+            "id": "p3",
+            "post": "Nurses who refuse to give out medications they disagree with should be fired.",
+            "platform": "healthcare_professionals_forum",
+            "reviewer_a": "REMOVE: targeted attack on nurses' professional autonomy.",
+            "reviewer_b": "KEEP: a policy opinion about conscientious objection, not an attack on individuals.",
+            "correct_ruling": "keep",
+            "correct_principle": "policy_opinion_vs_personal_attack",
+            "harm_keywords": ["policy", "opinion", "not targeted", "individual", "conscientious", "debate"],
+        },
+        {
+            "id": "p4",
+            "post": "[Image: meme showing a politician's face on a puppet controlled by a figure in a kippah]",
+            "platform": "political_discussion_forum",
+            "reviewer_a": "KEEP: political satire and criticism of politicians is protected speech.",
+            "reviewer_b": "REMOVE: antisemitic stereotype (puppet master trope) regardless of political target.",
+            "correct_ruling": "remove",
+            "correct_principle": "hate_symbol_not_protected_by_satire_framing",
+            "harm_keywords": ["antisemit", "puppet master", "trope", "stereotype", "satire does not protect", "hate"],
+        },
+        {
+            "id": "p5",
+            "post": "I've developed a simple at-home test to detect cancer using baking soda. DM me for the protocol.",
+            "platform": "health_advice_community",
+            "reviewer_a": "KEEP: alternative health advice. Users can make their own decisions.",
+            "reviewer_b": "REMOVE: dangerous medical misinformation that could delay life-saving treatment.",
+            "correct_ruling": "remove",
+            "correct_principle": "health_misinformation_direct_harm_risk",
+            "harm_keywords": ["misinformation", "medical", "delay", "treatment", "harm", "cancer", "false"],
+        },
+    ],
+}
+
+MOD_EXPERT_INSTRUCTIONS = (
+    "You are the Trust & Safety Policy Lead. Review 5 contested moderation cases.\n\n"
+    "For each case, provide:\n"
+    "  1. ruling: 'keep', 'remove', or 'keep_with_intervention'\n"
+    "  2. principle: the policy principle governing your decision (1-2 word label)\n"
+    "  3. justification: 2-3 sentences explaining the real-world harm or absence of harm\n\n"
+    "Cases where reviewers disagree genuinely are ambiguous — your reasoning matters as much as your ruling.\n\n"
+    'Payload: {"rulings": [{"id": "p1", "ruling": "keep_with_intervention", '
+    '"principle": "harm_reduction", "justification": "..."}]}'
+)
+
+
+def grade_mod_expert(payload: dict) -> tuple[float, str, dict]:
+    rulings = {r["id"]: r for r in payload.get("rulings", []) if isinstance(r, dict)}
+    cases   = MOD_EXPERT_CONTEXT["cases"]
+    checks: dict[str, bool] = {}
+
+    for case in cases:
+        cid    = case["id"]
+        ruling = rulings.get(cid, {})
+        r_val  = ruling.get("ruling", "").lower().replace(" ", "_")
+        just   = ruling.get("justification", "").lower()
+
+        # Ruling must match exactly
+        ruling_ok = r_val == case["correct_ruling"]
+        # Justification must show understanding (keyword match)
+        just_ok   = any(kw in just for kw in case["harm_keywords"])
+
+        checks[f"{cid}_ruling"]      = ruling_ok
+        checks[f"{cid}_justification"] = just_ok
+
+    passed  = sum(checks.values())
+    total   = len(checks)
+    score   = round(passed / total, 2)
+    tag     = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    wrong_rulings = [c["id"] for c in cases if not checks.get(f"{c['id']}_ruling")]
+    feedback = f"{tag} {passed}/{total} expert moderation checks passed."
+    if wrong_rulings:
+        feedback += f" Wrong rulings: {', '.join(wrong_rulings)}."
+    return score, feedback, checks
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# TICKET EXPERT  — post-incident review + process improvement
+# ═══════════════════════════════════════════════════════════════════════════════
+
+TICKET_EXPERT_CONTEXT = {
+    "post_incident_report": {
+        "incident_id": "INC-2026-0312",
+        "title": "Complete checkout service outage — 4 hours 17 minutes",
+        "timeline": [
+            {"time": "14:00", "event": "Deployment of release v4.2.1 to production"},
+            {"time": "14:07", "event": "Checkout error rate rises from 0.1% to 18%"},
+            {"time": "14:12", "event": "First PagerDuty alert fires to on-call engineer"},
+            {"time": "14:22", "event": "On-call escalates to engineering lead"},
+            {"time": "14:35", "event": "Team identifies suspect: new DB migration introduced N+1 query"},
+            {"time": "14:50", "event": "Decision made to rollback v4.2.1"},
+            {"time": "15:10", "event": "Rollback begins — delayed 20 min due to missing runbook"},
+            {"time": "15:45", "event": "Rollback completes but error rate stays at 12%"},
+            {"time": "16:00", "event": "Team discovers DB connection pool was not reset after rollback"},
+            {"time": "16:30", "event": "Connection pool reset + service restart"},
+            {"time": "18:17", "event": "Error rate returns to baseline (0.1%)"},
+        ],
+        "metrics": {
+            "error_rate_peak": "94%",
+            "revenue_lost_estimate": "$47,000",
+            "customers_affected": 8200,
+            "mttr_minutes": 257,
+            "detection_time_minutes": 12,
+            "rollback_time_minutes": 55,
+        },
+        "contributing_factors": [
+            "N+1 query introduced in ORM migration layer",
+            "No load testing before production deploy",
+            "Missing rollback runbook caused 20-minute delay",
+            "DB connection pool not included in rollback checklist",
+            "Alert threshold too high (fired at 18% error rate, not 1%)",
+        ],
+    },
+}
+
+TICKET_EXPERT_INSTRUCTIONS = (
+    "You are the Engineering Director. Analyse this post-incident report (PIR) and produce:\n\n"
+    "  1. root_cause: precise technical root cause (not just 'bad deploy')\n"
+    "  2. contributing_factors: list of all systemic factors that made the incident worse\n"
+    "  3. timeline_gaps: specific process failures visible in the timeline with timestamps\n"
+    "  4. action_items: 5+ concrete, specific remediation actions (not vague like 'improve testing')\n"
+    "  5. severity_justification: why this is P1 vs P2, citing revenue and customer impact\n"
+    "  6. mttr_analysis: break down where the 257 minutes went and which gap was largest\n\n"
+    "This will be presented to the board. Be specific and technical.\n\n"
+    'Payload: {\n'
+    '  "root_cause": "...",\n'
+    '  "contributing_factors": ["...", "..."],\n'
+    '  "timeline_gaps": [{"time": "14:50-15:10", "gap": "..."}],\n'
+    '  "action_items": ["...", "..."],\n'
+    '  "severity_justification": "...",\n'
+    '  "mttr_analysis": "..."\n'
+    '}'
+)
+
+
+def grade_ticket_expert(payload: dict) -> tuple[float, str, dict]:
+    root_cause  = payload.get("root_cause", "").lower()
+    factors     = [str(f).lower() for f in payload.get("contributing_factors", [])]
+    gaps        = payload.get("timeline_gaps", [])
+    actions     = [str(a).lower() for a in payload.get("action_items", [])]
+    sev_just    = payload.get("severity_justification", "").lower()
+    mttr        = payload.get("mttr_analysis", "").lower()
+
+    factors_text = " ".join(factors)
+    gaps_text    = " ".join(str(g) for g in gaps).lower()
+    actions_text = " ".join(actions)
+
+    checks = {
+        # Root cause must mention the N+1 query or ORM migration
+        "root_cause_specific":    any(w in root_cause for w in ["n+1", "n + 1", "query", "orm", "migration", "connection pool"]),
+        # Contributing factors must cover at least 3 of the 5
+        "factors_complete":       sum(1 for w in ["load test", "runbook", "connection pool", "alert threshold", "n+1", "migration"]
+                                      if w in factors_text) >= 3,
+        # Timeline must call out the 20-minute runbook delay
+        "gap_runbook_delay":      any(w in gaps_text for w in ["runbook", "20 min", "20-min", "15:10", "14:50"]),
+        # Timeline must call out the connection pool not being reset
+        "gap_connection_pool":    any(w in gaps_text for w in ["connection pool", "16:00", "reset"]),
+        # Must have 5+ action items
+        "action_items_count":     len(actions) >= 5,
+        # Actions must be specific: load testing, runbook, alert threshold
+        "actions_specific":       sum(1 for w in ["load test", "runbook", "alert", "connection pool", "rollback checklist", "canary"]
+                                      if w in actions_text) >= 3,
+        # Severity must cite revenue or customer count
+        "severity_quantified":    any(w in sev_just for w in ["$47", "47,000", "8200", "8,200", "revenue", "customer"]),
+        # MTTR analysis must identify rollback delay as largest gap
+        "mttr_gap_identified":    any(w in mttr for w in ["rollback", "20 min", "runbook", "largest", "55 min", "connection"]),
+    }
+
+    passed  = sum(checks.values())
+    score   = round(passed / len(checks), 2)
+    tag     = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    missing = [k for k, v in checks.items() if not v]
+    feedback = f"{tag} {passed}/{len(checks)} PIR analysis checks passed."
+    if missing:
+        feedback += f" Missing: {', '.join(missing)}."
+    return score, feedback, checks
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# REGISTRY  — for openenv.yaml and app.py task list
+# ═══════════════════════════════════════════════════════════════════════════════
+
+EXPERT_TASK_IDS = [
+    "email_triage_expert",
+    "code_review_expert",
+    "data_cleaning_expert",
+    "content_moderation_expert",
+    "ticket_triage_expert",
+]
+
+EXPERT_TASK_REGISTRY = [
+    {
+        "id": "email_triage_expert",
+        "name": "Executive Escalation Response",
+        "agent": "email_triage",
+        "difficulty": "expert",
+        "description": "Draft an executive-level reply to a 4-week escalated enterprise complaint thread. Account context and engineering root cause provided.",
+        "action_schema": {"message": '{"agent": "email_triage", "task_id": "email_triage_expert", "payload": {"reply": "...min 200 words..."}}'},
+    },
+    {
+        "id": "code_review_expert",
+        "name": "Advanced Security Audit (8 subtle vulns)",
+        "agent": "code_review",
+        "difficulty": "expert",
+        "description": "Find 8 vulnerabilities including JWT none-algorithm attack, mass assignment, IDOR, YAML RCE, TOCTOU race condition, prototype pollution, and log injection.",
+        "action_schema": {"message": '{"agent": "code_review", "task_id": "code_review_expert", "payload": {"vulnerabilities": [...]}}'},
+    },
+    {
+        "id": "data_cleaning_expert",
+        "name": "Cross-Dataset Join & Revenue Analysis",
+        "agent": "data_cleaning",
+        "difficulty": "expert",
+        "description": "Join two datasets, detect multi-type anomalies, convert currencies, compute per-rep revenue excluding outliers/dups/zeros.",
+        "action_schema": {"message": '{"agent": "data_cleaning", "task_id": "data_cleaning_expert", "payload": {"data_quality_issues": [], "suspicious_transactions": [], "revenue_by_rep": {}, "excluded_transactions": []}}'},
+    },
+    {
+        "id": "content_moderation_expert",
+        "name": "Contested Policy Rulings",
+        "agent": "content_moderation",
+        "difficulty": "expert",
+        "description": "Rule on 5 genuinely contested moderation cases — suicidal ideation, dog-whistle rhetoric, policy debate, antisemitic satire, health misinformation.",
+        "action_schema": {"message": '{"agent": "content_moderation", "task_id": "content_moderation_expert", "payload": {"rulings": [{"id": "p1", "ruling": "...", "principle": "...", "justification": "..."}]}}'},
+    },
+    {
+        "id": "ticket_triage_expert",
+        "name": "Post-Incident Review (PIR)",
+        "agent": "ticket_triage",
+        "difficulty": "expert",
+        "description": "Produce a board-ready PIR from a 4-hour outage timeline: root cause, contributing factors, timeline gaps, 5+ action items, MTTR breakdown.",
+        "action_schema": {"message": '{"agent": "ticket_triage", "task_id": "ticket_triage_expert", "payload": {"root_cause": "...", "contributing_factors": [], "timeline_gaps": [], "action_items": [], "severity_justification": "...", "mttr_analysis": "..."}}'},
+    },
+]
+
+EXPERT_INSTRUCTIONS: dict[str, str] = {
+    "email_triage_expert":         EMAIL_EXPERT_INSTRUCTIONS,
+    "code_review_expert":          CODE_EXPERT_INSTRUCTIONS,
+    "data_cleaning_expert":        DATA_EXPERT_INSTRUCTIONS,
+    "content_moderation_expert":   MOD_EXPERT_INSTRUCTIONS,
+    "ticket_triage_expert":        TICKET_EXPERT_INSTRUCTIONS,
+}

inference.py

@@ -68,13 +68,12 @@ HF_TOKEN = API_KEY
 
 
 TASK_IDS = [
-    "email_triage_easy",   "email_triage_medium",   "email_triage_hard",
-    "code_review_easy",    "code_review_medium",     "code_review_hard",
-    "data_cleaning_easy",  "data_cleaning_medium",   "data_cleaning_hard",
-    "content_moderation_easy", "content_moderation_medium", "content_moderation_hard",
-    "ticket_triage_easy",  "ticket_triage_medium",   "ticket_triage_hard",
-    "cross_agent_chain",   "cross_agent_email_data", "cross_agent_code_email",
-    "cross_agent_mod_escalation",
+    "email_triage_easy",        "email_triage_medium",        "email_triage_hard",         "email_triage_expert",
+    "code_review_easy",         "code_review_medium",         "code_review_hard",          "code_review_expert",
+    "data_cleaning_easy",       "data_cleaning_medium",       "data_cleaning_hard",        "data_cleaning_expert",
+    "content_moderation_easy",  "content_moderation_medium",  "content_moderation_hard",   "content_moderation_expert",
+    "ticket_triage_easy",       "ticket_triage_medium",       "ticket_triage_hard",        "ticket_triage_expert",
+    "cross_agent_chain",        "cross_agent_email_data",     "cross_agent_code_email",    "cross_agent_mod_escalation",
 ]
 
 SYSTEM_PROMPT = (

inference.py.bak

@@ -0,0 +1,365 @@
+"""
+inference.py — Meta OpenEnv Hackathon Submission v3.0
+19 tasks: 5 domains × 3 + 4 cross-agent chained tasks.
+Deterministic context, trajectory-aware reward, proper [START]/[STEP]/[END] logging.
+
+Provider priority (auto-detected from env vars):
+  1. GROQ_API_KEY  → https://api.groq.com/openai/v1  (llama-3.3-70b-versatile) [FREE]
+  2. HF_TOKEN      → API_BASE_URL                     (Qwen/Qwen2.5-72B-Instruct)
+  3. OPENAI_API_KEY → https://api.openai.com/v1       (gpt-4o-mini)
+
+Override everything with: API_BASE_URL + MODEL_NAME + HF_TOKEN (hackathon validator vars)
+
+Logging format:
+  [START] task=<task_id> env=meta model=<model>
+  [STEP]  step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
+  [END]   success=<true|false> steps=<n> score=<0.000> rewards=<r1,r2,...>
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+from typing import Optional
+
+# ── Load .env ─────────────────────────────────────────────────────────────────
+_env_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env")
+if os.path.exists(_env_file):
+    with open(_env_file, encoding="utf-8", errors="ignore") as _f:
+        for _line in _f:
+            _line = _line.strip()
+            if _line and not _line.startswith("#") and "=" in _line:
+                _k, _, _v = _line.partition("=")
+                os.environ.setdefault(_k.strip(), _v.strip())
+
+# ── Mandatory env vars (spec-compliant) ──────────────────────────────────────
+# The hackathon validator sets API_BASE_URL + MODEL_NAME + HF_TOKEN.
+# We must respect those exactly. Groq is only used as local fallback
+# when HF_TOKEN is NOT set.
+
+_groq_key   = os.environ.get("GROQ_API_KEY", "")
+_hf_token   = os.environ.get("HF_TOKEN", "")
+_api_base   = os.environ.get("API_BASE_URL", "")
+_model_name = os.environ.get("MODEL_NAME", "")
+
+# Spec-required defaults
+API_BASE_URL = _api_base   or "https://router.huggingface.co/v1"
+MODEL_NAME   = _model_name or "Qwen/Qwen2.5-72B-Instruct"
+
+if _hf_token:
+    # HF_TOKEN set — hackathon validator mode or HF local run
+    API_KEY  = _hf_token
+    PROVIDER = "huggingface"
+elif _groq_key:
+    # No HF_TOKEN — use Groq as free local fallback
+    API_BASE_URL = "https://api.groq.com/openai/v1"
+    MODEL_NAME   = "llama-3.3-70b-versatile"
+    API_KEY      = _groq_key
+    PROVIDER     = "groq"
+else:
+    API_KEY  = ""
+    PROVIDER = "none"
+
+# Keep HF_TOKEN alias for backward compat
+HF_TOKEN = API_KEY
+
+
+TASK_IDS = [
+    "email_triage_easy",        "email_triage_medium",        "email_triage_hard",         "email_triage_expert",
+    "code_review_easy",         "code_review_medium",         "code_review_hard",          "code_review_expert",
+    "data_cleaning_easy",       "data_cleaning_medium",       "data_cleaning_hard",        "data_cleaning_expert",
+    "content_moderation_easy",  "content_moderation_medium",  "content_moderation_hard",   "content_moderation_expert",
+    "ticket_triage_easy",       "ticket_triage_medium",       "ticket_triage_hard",        "ticket_triage_expert",
+    "cross_agent_chain",        "cross_agent_email_data",     "cross_agent_code_email",    "cross_agent_mod_escalation",
+]
+
+SYSTEM_PROMPT = (
+    "You are an expert AI agent completing structured real-world tasks.\n"
+    "You MUST respond with ONLY a valid JSON object as the payload.\n"
+    "Rules:\n"
+    "1. Start your response with { and end with }\n"
+    "2. No markdown, no code fences (no ```), no explanation text\n"
+    "3. No preamble — just the JSON object\n"
+    "4. The JSON must be valid and parseable\n"
+    "5. Follow the payload format exactly as described in the instructions"
+)
+
+SUCCESS_SCORE_THRESHOLD = 0.5
+
+
+# ── Logging ───────────────────────────────────────────────────────────────────
+
+def log_start(task: str, env: str, model: str) -> None:
+    print(f"[START] task={task} env={env} model={model}", flush=True)
+
+
+def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
+    error_val = error if error else "null"
+    done_val  = str(done).lower()
+    # Collapse action to single line, truncate to 120 chars
+    action_oneline = action.replace("\n", " ")[:120]
+    print(
+        f"[STEP] step={step} action={action_oneline} reward={reward:.2f} done={done_val} error={error_val}",
+        flush=True,
+    )
+
+
+def log_end(success: bool, steps: int, score: float, rewards: list[float]) -> None:
+    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
+    print(
+        f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={rewards_str}",
+        flush=True,
+    )
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def progress_bar(current: int, total: int, width: int = 40) -> str:
+    filled = int(width * current / total)
+    return f"[{'█'*filled}{'░'*(width-filled)}] {int(100*current/total):3d}%  ({current}/{total})"
+
+
+def score_bar(score: float, width: int = 10) -> str:
+    filled = int(width * score)
+    return "█" * filled + "░" * (width - filled)
+
+
+def extract_json(text: str) -> dict:
+    if not text:
+        return {}
+    text = text.strip()
+    try:
+        r = json.loads(text)
+        if isinstance(r, dict):
+            return r
+    except Exception:
+        pass
+    cleaned = re.sub(r"^```(?:json)?\s*\n?", "", text, flags=re.IGNORECASE)
+    cleaned = re.sub(r"\n?```\s*$", "", cleaned, flags=re.MULTILINE).strip()
+    try:
+        r = json.loads(cleaned)
+        if isinstance(r, dict):
+            return r
+    except Exception:
+        pass
+    start = text.find("{")
+    if start != -1:
+        depth = 0
+        for i, ch in enumerate(text[start:], start):
+            if ch == "{":
+                depth += 1
+            elif ch == "}":
+                depth -= 1
+                if depth == 0:
+                    try:
+                        r = json.loads(text[start:i+1])
+                        if isinstance(r, dict):
+                            return r
+                    except Exception:
+                        pass
+                    break
+    return {}
+
+
+def unwrap_payload(payload: dict) -> dict:
+    if payload and "payload" in payload and isinstance(payload["payload"], dict):
+        return payload["payload"]
+    return payload
+
+
+def call_model(client, task_id: str, difficulty: str, instructions: str,
+               agent_ctx: dict, feedback: Optional[str] = None) -> dict:
+    retry_note = ""
+    if feedback:
+        retry_note = (
+            f"\nPREVIOUS ATTEMPT FEEDBACK: {feedback}\n"
+            "Fix the issues described above in your new response.\n"
+        )
+    user_prompt = (
+        f"Task: {task_id}\n"
+        f"Difficulty: {difficulty}\n\n"
+        f"Instructions:\n{instructions}\n"
+        f"{retry_note}\n"
+        f"Data to work with:\n{json.dumps(agent_ctx, indent=2)}\n\n"
+        "Respond with ONLY the payload JSON object. Start with { and end with }."
+    )
+    try:
+        response = client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user",   "content": user_prompt},
+            ],
+            temperature=0.0,
+            max_tokens=2000,
+        )
+        raw     = response.choices[0].message.content or ""
+        payload = unwrap_payload(extract_json(raw))
+        if not payload:
+            print(f"\n    [DEBUG] Empty JSON. Raw: {repr(raw[:200])}")
+        else:
+            print(f"\n    [DEBUG] Attempt payload keys: {list(payload.keys())}")
+        return payload
+    except Exception as e:
+        print(f"\n    [ERROR] API call failed: {e}")
+        return {}
+
+
+# ── Single task episode ───────────────────────────────────────────────────────
+
+def run_task(client, task_id: str, INSTRUCTIONS: dict) -> dict:
+    from server.Meta_environment import MetaEnvironment
+    from models import MetaAction
+
+    agent = (
+        "_".join(task_id.split("_")[:-1])
+        if not task_id.startswith("cross_agent")
+        else "cross_agent"
+    )
+
+    env = MetaEnvironment()
+    env.reset()
+    try:
+        agent_ctx, grader_ctx, difficulty = env._load_context(task_id)
+    except Exception as e:
+        return {
+            "task_id": task_id, "agent": agent, "difficulty": "unknown",
+            "score": 0.0, "feedback": f"Context load failed: {e}",
+            "partial_credits": {}, "reward": 0.0,
+        }
+
+    instructions = INSTRUCTIONS.get(task_id, "")
+
+    env2 = MetaEnvironment()
+    env2.reset()
+    env2._current_agent_context  = agent_ctx
+    env2._current_grader_context = grader_ctx
+    env2._current_task_id        = task_id
+
+    rewards: list[float] = []
+    log_start(task_id, "meta", MODEL_NAME)
+
+    # Attempt 1
+    payload  = call_model(client, task_id, difficulty, instructions, agent_ctx)
+    action   = MetaAction(message=json.dumps({"agent": agent, "task_id": task_id, "payload": payload}))
+    obs      = env2.step(action)
+    rewards.append(obs.reward)
+    log_step(1, json.dumps(payload)[:80], obs.reward, obs.done, None)
+
+    # Attempt 2 if retry available
+    if not obs.done and obs.score < 1.0:
+        print(f"\n    [RETRY] Score {obs.score:.2f} — attempting retry with feedback...")
+        time.sleep(1)
+        payload2 = call_model(client, task_id, difficulty, instructions, agent_ctx, feedback=obs.feedback)
+        action2  = MetaAction(message=json.dumps({"agent": agent, "task_id": task_id, "payload": payload2}))
+        obs      = env2.step(action2)
+        rewards.append(obs.reward)
+        log_step(2, json.dumps(payload2)[:80], obs.reward, obs.done, None)
+
+    success = obs.score >= SUCCESS_SCORE_THRESHOLD
+    log_end(success, len(rewards), obs.score, rewards)
+
+    return {
+        "task_id":         task_id,
+        "agent":           agent,
+        "difficulty":      obs.difficulty,
+        "score":           obs.score,
+        "feedback":        obs.feedback,
+        "partial_credits": obs.partial_credits,
+        "reward":          obs.reward,
+    }
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def run_all(api_base_url: str, model_name: str, hf_token: str) -> list[dict]:
+    api_key = hf_token or API_KEY
+    if not api_key:
+        print("ERROR: No API key found.")
+        print("Set one of:")
+        print("  GROQ_API_KEY=gsk_...   (free at https://console.groq.com)")
+        print("  HF_TOKEN=hf_...        (HuggingFace token)")
+        sys.exit(1)
+
+    try:
+        from openai import OpenAI
+    except ImportError:
+        print("ERROR: pip install openai")
+        sys.exit(1)
+
+    try:
+        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+        from server.Meta_environment import INSTRUCTIONS
+    except ImportError as e:
+        print(f"ERROR: {e}")
+        sys.exit(1)
+
+    # Groq free tier: 30 req/min — add a small sleep between tasks
+    is_groq  = "groq.com" in api_base_url
+    task_sleep = 2.0 if is_groq else 1.0
+
+    client  = OpenAI(api_key=api_key, base_url=api_base_url)
+    results: list[dict] = []
+
+    print()
+    print("╔══════════════════════════════════════════════════════════════════╗")
+    print("║        Meta OpenEnv — Inference Runner (OpenEnv Hackathon 2026) ║")
+    print("╠══════════════════════════════════════════════════════════════════╣")
+    print(f"║  Provider : {PROVIDER:<53}║")
+    print(f"║  API Base : {api_base_url:<53}║")
+    print(f"║  Model    : {model_name:<53}║")
+    print(f"║  Tasks    : {len(TASK_IDS):<53}║")
+    print("╚══════════════════════════════════════════════════════════════════╝")
+    print()
+
+    start_all = time.time()
+
+    for i, task_id in enumerate(TASK_IDS):
+        print(f"  {progress_bar(i, len(TASK_IDS))}  {task_id}")
+        sys.stdout.flush()
+
+        t0      = time.time()
+        result  = run_task(client, task_id, INSTRUCTIONS)
+        elapsed = time.time() - t0
+        results.append(result)
+
+        diff = result["difficulty"].upper()[:4]
+        bar  = score_bar(result["score"])
+        print(f"  ✓ [{diff:4s}] {task_id:<40} [{bar}] {result['score']:.2f}  ({elapsed:.1f}s)  {result['feedback'][:50]}")
+        sys.stdout.flush()
+        time.sleep(task_sleep)
+
+    total_time = time.time() - start_all
+    avg_score  = sum(r["score"] for r in results) / len(results)
+    passed     = sum(1 for r in results if r["score"] >= SUCCESS_SCORE_THRESHOLD)
+
+    print()
+    print(f"  {progress_bar(len(TASK_IDS), len(TASK_IDS))}")
+    print()
+    print("╔══════════════════════════════════════════════════════════════════╗")
+    print(f"║  📊 Average Score : {avg_score:.4f}{'':37}║")
+    print(f"║  ⏱  Total Time    : {total_time:.1f}s{'':39}║")
+    print(f"║  ✅ Tasks Passed  : {passed}/{len(results)}{'':42}║")
+    print("╚══════════════════════════════════════════════════════════════════╝")
+    print()
+
+    agents = [
+        "email_triage", "code_review", "data_cleaning",
+        "content_moderation", "ticket_triage", "cross_agent",
+    ]
+    print("  Per-Agent Summary:")
+    for ag in agents:
+        ag_r   = [r for r in results if r["agent"] == ag]
+        ag_avg = sum(r["score"] for r in ag_r) / len(ag_r) if ag_r else 0.0
+        print(f"    {ag:<35} [{score_bar(ag_avg)}] {ag_avg:.2f}")
+    print()
+    return results
+
+
+if __name__ == "__main__":
+    results = run_all(API_BASE_URL, MODEL_NAME, HF_TOKEN)
+    print("Full Results JSON:")
+    print(json.dumps(results, indent=2))

openenv.yaml

@@ -1,13 +1,13 @@
 spec_version: 1
 name: Meta
 description: >
-  Meta is the first multi-domain OpenEnv environment featuring 5 AI agents
+  Meta is the first multi-domain OpenEnv environment featuring 5 AI agents (plus expert tier)
   (Email Triage, Code Review, Data Cleaning, Content Moderation, Ticket Triage)
   across 15 real-world tasks (easy/medium/hard) plus 4 cross-agent chained tasks.
   Built for multi-task learning, transfer learning, and generalist agent research.
   v3.0: deterministic context loading, harder hard tasks (8 vulnerabilities,
   adversarial content moderation), 5th domain, 4 cross-agent chained tasks.
-version: "3.0.0"
+version: "3.1.0"
 type: space
 runtime: fastapi
 app: server.app:app
@@ -116,6 +116,48 @@ tasks:
       Analyse 6 linked incident tickets: produce root cause analysis,
       ordered resolution steps, affected services list, and P1-P4 severity.
 
+
+  # ── Expert tier (multi-step reasoning, frontier challenge) ────────────────
+  - id: email_triage_expert
+    agent: email_triage
+    difficulty: expert
+    description: >
+      Draft an executive reply to a 4-week escalated enterprise complaint.
+      Account metadata and engineering root cause provided. Requires multi-context
+      synthesis and executive tone calibration. Frontier models score ~0.45.
+
+  - id: code_review_expert
+    agent: code_review
+    difficulty: expert
+    description: >
+      Identify 8 subtle vulnerabilities: hardcoded secret, JWT none-algorithm attack,
+      mass assignment, IDOR, YAML RCE deserialization, TOCTOU race condition,
+      prototype pollution, and log injection. Frontier models score ~0.38.
+
+  - id: data_cleaning_expert
+    agent: data_cleaning
+    difficulty: expert
+    description: >
+      Join two datasets, detect multi-type anomalies (duplicates, missing, outliers,
+      zero-amounts), convert currencies to USD, compute per-rep revenue excluding
+      invalid transactions. Frontier models score ~0.50.
+
+  - id: content_moderation_expert
+    agent: content_moderation
+    difficulty: expert
+    description: >
+      Issue rulings on 5 genuinely contested cases: suicidal ideation,
+      great-replacement rhetoric, conscientious objection policy debate,
+      antisemitic satire framing, and health misinformation. Frontier models score ~0.40.
+
+  - id: ticket_triage_expert
+    agent: ticket_triage
+    difficulty: expert
+    description: >
+      Produce a board-ready PIR from a 4h17m outage timeline. Requires identifying
+      timeline gaps, 5+ specific action items, and quantified MTTR breakdown.
+      Frontier models score ~0.50.
+
   # ── Cross-Agent Chained (NEW x3) ──────────────────────────────────────────
   - id: cross_agent_chain
     agent: cross_agent
@@ -139,19 +181,19 @@ tasks:
 
 agents:
   - id: email_triage
-    tasks: [email_triage_easy, email_triage_medium, email_triage_hard]
+    tasks: [email_triage_easy, email_triage_medium, email_triage_hard, email_triage_expert]
 
   - id: code_review
-    tasks: [code_review_easy, code_review_medium, code_review_hard]
+    tasks: [code_review_easy, code_review_medium, code_review_hard, code_review_expert]
 
   - id: data_cleaning
-    tasks: [data_cleaning_easy, data_cleaning_medium, data_cleaning_hard]
+    tasks: [data_cleaning_easy, data_cleaning_medium, data_cleaning_hard, data_cleaning_expert]
 
   - id: content_moderation
-    tasks: [content_moderation_easy, content_moderation_medium, content_moderation_hard]
+    tasks: [content_moderation_easy, content_moderation_medium, content_moderation_hard, content_moderation_expert]
 
   - id: ticket_triage
-    tasks: [ticket_triage_easy, ticket_triage_medium, ticket_triage_hard]
+    tasks: [ticket_triage_easy, ticket_triage_medium, ticket_triage_hard, ticket_triage_expert]
 
   - id: cross_agent
     tasks: [cross_agent_chain, cross_agent_email_data, cross_agent_code_email, cross_agent_mod_escalation]

openenv.yaml.bak

@@ -0,0 +1,232 @@
+spec_version: 1
+name: Meta
+description: >
+  Meta is the first multi-domain OpenEnv environment featuring 5 AI agents (plus expert tier)
+  (Email Triage, Code Review, Data Cleaning, Content Moderation, Ticket Triage)
+  across 15 real-world tasks (easy/medium/hard) plus 4 cross-agent chained tasks.
+  Built for multi-task learning, transfer learning, and generalist agent research.
+  v3.0: deterministic context loading, harder hard tasks (8 vulnerabilities,
+  adversarial content moderation), 5th domain, 4 cross-agent chained tasks.
+version: "3.1.0"
+type: space
+runtime: fastapi
+app: server.app:app
+port: 7860
+
+tags:
+  - openenv
+  - multi-agent
+  - rl
+  - workplace
+  - benchmark
+  - email-triage
+  - code-review
+  - data-cleaning
+  - content-moderation
+  - ticket-triage
+  - cross-agent
+
+tasks:
+  # ── Email Triage ──────────────────────────────────────────────────────────
+  - id: email_triage_easy
+    agent: email_triage
+    difficulty: easy
+    description: Classify a single email as spam, important, or newsletter.
+
+  - id: email_triage_medium
+    agent: email_triage
+    difficulty: medium
+    description: Prioritize 10 workplace emails by urgency (most to least urgent).
+
+  - id: email_triage_hard
+    agent: email_triage
+    difficulty: hard
+    description: Draft a professional, empathetic reply to a complex customer complaint.
+
+  # ── Code Review ───────────────────────────────────────────────────────────
+  - id: code_review_easy
+    agent: code_review
+    difficulty: easy
+    description: Identify the syntax error(s) in a Python code snippet.
+
+  - id: code_review_medium
+    agent: code_review
+    difficulty: medium
+    description: Find all 4 logical bugs in the provided code and suggest fixes.
+
+  - id: code_review_hard
+    agent: code_review
+    difficulty: hard
+    description: >
+      Detect all 8 security vulnerabilities: SQL injection (x2), XSS,
+      command injection, insecure deserialization, timing attack,
+      path traversal, and SSRF.
+
+  # ── Data Cleaning ─────────────────────────────────────────────────────────
+  - id: data_cleaning_easy
+    agent: data_cleaning
+    difficulty: easy
+    description: Identify all missing values by field+row and detect duplicate rows.
+
+  - id: data_cleaning_medium
+    agent: data_cleaning
+    difficulty: medium
+    description: Fix 5 data type/format issues and return the cleaned dataset.
+
+  - id: data_cleaning_hard
+    agent: data_cleaning
+    difficulty: hard
+    description: >
+      Detect outliers via IQR/z-score, identify missing values, impute them,
+      and return the cleaned dataset.
+
+  # ── Content Moderation ────────────────────────────────────────────────────
+  - id: content_moderation_easy
+    agent: content_moderation
+    difficulty: easy
+    description: Classify 7 posts as safe or harmful (explicit content).
+
+  - id: content_moderation_medium
+    agent: content_moderation
+    difficulty: medium
+    description: Detect subtle toxicity, sarcasm, and implicit hostility across 8 posts.
+
+  - id: content_moderation_hard
+    agent: content_moderation
+    difficulty: hard
+    description: >
+      Context-aware moderation across 6 cases including adversarial dog-whistles
+      and coordinated inauthentic behavior patterns. Same text, two contexts.
+
+  # ── Ticket Triage (NEW) ───────────────────────────────────────────────────
+  - id: ticket_triage_easy
+    agent: ticket_triage
+    difficulty: easy
+    description: Classify a Jira-style support ticket by priority and category.
+
+  - id: ticket_triage_medium
+    agent: ticket_triage
+    difficulty: medium
+    description: Order 8 tickets by priority and assign each to the correct team.
+
+  - id: ticket_triage_hard
+    agent: ticket_triage
+    difficulty: hard
+    description: >
+      Analyse 6 linked incident tickets: produce root cause analysis,
+      ordered resolution steps, affected services list, and P1-P4 severity.
+
+
+  # ── Expert tier (multi-step reasoning, frontier challenge) ────────────────
+  - id: email_triage_expert
+    agent: email_triage
+    difficulty: expert
+    description: >
+      Draft an executive reply to a 4-week escalated enterprise complaint.
+      Account metadata and engineering root cause provided. Requires multi-context
+      synthesis and executive tone calibration. Frontier models score ~0.45.
+
+  - id: code_review_expert
+    agent: code_review
+    difficulty: expert
+    description: >
+      Identify 8 subtle vulnerabilities: hardcoded secret, JWT none-algorithm attack,
+      mass assignment, IDOR, YAML RCE deserialization, TOCTOU race condition,
+      prototype pollution, and log injection. Frontier models score ~0.38.
+
+  - id: data_cleaning_expert
+    agent: data_cleaning
+    difficulty: expert
+    description: >
+      Join two datasets, detect multi-type anomalies (duplicates, missing, outliers,
+      zero-amounts), convert currencies to USD, compute per-rep revenue excluding
+      invalid transactions. Frontier models score ~0.50.
+
+  - id: content_moderation_expert
+    agent: content_moderation
+    difficulty: expert
+    description: >
+      Issue rulings on 5 genuinely contested cases: suicidal ideation,
+      great-replacement rhetoric, conscientious objection policy debate,
+      antisemitic satire framing, and health misinformation. Frontier models score ~0.40.
+
+  - id: ticket_triage_expert
+    agent: ticket_triage
+    difficulty: expert
+    description: >
+      Produce a board-ready PIR from a 4h17m outage timeline. Requires identifying
+      timeline gaps, 5+ specific action items, and quantified MTTR breakdown.
+      Frontier models score ~0.50.
+
+  # ── Cross-Agent Chained (NEW x3) ──────────────────────────────────────────
+  - id: cross_agent_chain
+    agent: cross_agent
+    difficulty: hard
+    description: Classify the email AND identify the code bug in the attachment.
+
+  - id: cross_agent_email_data
+    agent: cross_agent
+    difficulty: hard
+    description: Classify the email urgency AND identify+fix data quality issues in the attachment.
+
+  - id: cross_agent_code_email
+    agent: cross_agent
+    difficulty: hard
+    description: Identify the security vulnerability AND draft a professional disclosure email.
+
+  - id: cross_agent_mod_escalation
+    agent: cross_agent
+    difficulty: hard
+    description: Classify the content AND decide escalation AND draft a moderation notice.
+
+agents:
+  - id: email_triage
+    tasks: [email_triage_easy, email_triage_medium, email_triage_hard, email_triage_expert]
+
+  - id: code_review
+    tasks: [code_review_easy, code_review_medium, code_review_hard, code_review_expert]
+
+  - id: data_cleaning
+    tasks: [data_cleaning_easy, data_cleaning_medium, data_cleaning_hard, data_cleaning_expert]
+
+  - id: content_moderation
+    tasks: [content_moderation_easy, content_moderation_medium, content_moderation_hard, content_moderation_expert]
+
+  - id: ticket_triage
+    tasks: [ticket_triage_easy, ticket_triage_medium, ticket_triage_hard, ticket_triage_expert]
+
+  - id: cross_agent
+    tasks: [cross_agent_chain, cross_agent_email_data, cross_agent_code_email, cross_agent_mod_escalation]
+
+reward:
+  range: [0.0, 1.0]
+  partial_credit: true
+  deterministic: true
+  shaping:
+    speed_bonus: 0.05        # perfect on first attempt
+    improvement_bonus: 0.1   # scaled improvement on retry
+    repeated_failure_penalty: -0.05
+
+baseline:
+  model: gpt-4o-mini
+  average_score: 0.80
+  scores:
+    email_triage_easy: 1.00
+    email_triage_medium: 0.80
+    email_triage_hard: 0.67
+    code_review_easy: 1.00
+    code_review_medium: 0.75
+    code_review_hard: 0.50
+    data_cleaning_easy: 1.00
+    data_cleaning_medium: 0.67
+    data_cleaning_hard: 0.67
+    content_moderation_easy: 1.00
+    content_moderation_medium: 0.75
+    content_moderation_hard: 0.42
+    ticket_triage_easy: 1.00
+    ticket_triage_medium: 0.70
+    ticket_triage_hard: 0.50
+    cross_agent_chain: 0.75
+    cross_agent_email_data: 0.67
+    cross_agent_code_email: 0.67
+    cross_agent_mod_escalation: 0.83

server/Meta_environment.py

@@ -36,6 +36,23 @@ from openenv.core.env_server.types import State
 
 from models import MetaAction, MetaObservation
 
+from data_generators import (
+    gen_email_easy, gen_email_medium, gen_email_hard,
+    gen_code_easy,
+    gen_data_easy, gen_data_medium, gen_data_hard,
+    gen_mod_easy, gen_mod_medium,
+    gen_ticket_easy,
+)
+from expert_tasks import (
+    EMAIL_EXPERT_CONTEXT, grade_email_expert,
+    CODE_EXPERT_CONTEXT,  grade_code_expert,
+    DATA_EXPERT_CONTEXT,  grade_data_expert,
+    MOD_EXPERT_CONTEXT,   grade_mod_expert,
+    TICKET_EXPERT_CONTEXT, grade_ticket_expert,
+    EXPERT_INSTRUCTIONS, EXPERT_TASK_IDS,
+)
+
+
 
 # ═════════════════════════════════════════════════════════════════════════════
 # DATA
@@ -602,7 +619,7 @@ def _kendall_tau_score(order: list, correct: list) -> float:
 
 def grade_email_medium(payload: dict, context: dict) -> tuple[float, str, dict]:
     order   = payload.get("order", [])
-    correct = EMAIL_MEDIUM_CORRECT_ORDER
+    correct = context.get("correct_order", EMAIL_MEDIUM_CORRECT_ORDER)
     if not order:
         return 0.0, "[FAIL] No order provided.", {}
     top2_ok  = set(order[:2]) == {"m1", "m5"}
@@ -792,13 +809,14 @@ def grade_data_medium(payload: dict) -> tuple[float, str, dict]:
     return score, feedback, checks
 
 
-def grade_data_hard(payload: dict) -> tuple[float, str, dict]:
+def grade_data_hard(payload: dict, context: dict = None) -> tuple[float, str, dict]:
     outliers_given = set(int(x) for x in payload.get("outliers", []))
     missing_given  = set(int(x) for x in payload.get("missing", []))
     cleaned        = payload.get("cleaned_data", [])
-    correct_out    = DATA_HARD["answers"]["outliers"]
-    correct_miss   = DATA_HARD["answers"]["missing"]
-    lo, hi         = DATA_HARD["answers"]["imputed_range"]
+    answers      = (context or {}).get("answers", DATA_HARD["answers"])
+    correct_out  = answers.get("outliers", DATA_HARD["answers"]["outliers"])
+    correct_miss = answers.get("missing", DATA_HARD["answers"]["missing"])
+    lo, hi       = answers.get("imputed_range", DATA_HARD["answers"]["imputed_range"])
     outlier_ok     = outliers_given == correct_out
     missing_ok     = missing_given == correct_miss
     imputed_rows   = {r["id"]: r.get("value") for r in cleaned if r.get("id") in correct_miss}
@@ -814,9 +832,10 @@ def grade_data_hard(payload: dict) -> tuple[float, str, dict]:
     return score, feedback, checks
 
 
-def grade_mod_easy(payload: dict) -> tuple[float, str, dict]:
+def grade_mod_easy(payload: dict, context: dict = None) -> tuple[float, str, dict]:
     clf     = {c["id"]: c.get("label", "").lower() for c in payload.get("classifications", [])}
-    correct = {p["id"]: p["label"] for p in MOD_EASY}
+    posts   = (context or {}).get("posts", MOD_EASY)
+    correct = {p["id"]: p["label"] for p in posts}
     per     = {id_: clf.get(id_) == label for id_, label in correct.items()}
     hits    = sum(per.values())
     score   = round(hits / len(correct), 2)
@@ -824,9 +843,10 @@ def grade_mod_easy(payload: dict) -> tuple[float, str, dict]:
     return score, f"{tag} {hits}/{len(correct)} posts correctly classified.", per
 
 
-def grade_mod_medium(payload: dict) -> tuple[float, str, dict]:
+def grade_mod_medium(payload: dict, context: dict = None) -> tuple[float, str, dict]:
     clf     = {c["id"]: c.get("label", "").lower() for c in payload.get("classifications", [])}
-    correct = {p["id"]: p["label"] for p in MOD_MEDIUM}
+    posts   = (context or {}).get("posts", MOD_MEDIUM)
+    correct = {p["id"]: p["label"] for p in posts}
     per     = {id_: clf.get(id_) == label for id_, label in correct.items()}
     hits    = sum(per.values())
     score   = round(hits / len(correct), 2)
@@ -1236,26 +1256,19 @@ class MetaEnvironment(Environment):
     def _load_context(self, task_id: str) -> tuple[dict, dict, str]:
         """Return (agent_ctx, grader_ctx, difficulty). Fully deterministic."""
         if task_id == "email_triage_easy":
-            email      = self._deterministic_choice(EMAIL_EASY_VARIANTS, task_id)
-            agent_ctx  = {"email": {k: v for k, v in email.items() if k != "label"}}
-            grader_ctx = {"email": email}
+            agent_ctx, grader_ctx = gen_email_easy(self._state.episode_id)
             return agent_ctx, grader_ctx, "easy"
 
         if task_id == "email_triage_medium":
-            agent_ctx  = {"emails": [{k: v for k, v in e.items() if k not in ("priority", "category")} for e in EMAIL_MEDIUM_EMAILS]}
-            grader_ctx = {"emails": EMAIL_MEDIUM_EMAILS}
+            agent_ctx, grader_ctx = gen_email_medium(self._state.episode_id)
             return agent_ctx, grader_ctx, "medium"
 
         if task_id == "email_triage_hard":
-            case       = self._deterministic_choice(EMAIL_HARD_CASES, task_id)
-            agent_ctx  = {"email": {k: v for k, v in case.items() if k != "expected_elements"}}
-            grader_ctx = {"email": case}
+            agent_ctx, grader_ctx = gen_email_hard(self._state.episode_id)
             return agent_ctx, grader_ctx, "hard"
 
         if task_id == "code_review_easy":
-            variant    = self._deterministic_choice(CODE_EASY_VARIANTS, task_id)
-            agent_ctx  = {"code": variant["code"]}
-            grader_ctx = {"code": variant["code"], "keywords": variant["keywords"]}
+            agent_ctx, grader_ctx = gen_code_easy(self._state.episode_id)
             return agent_ctx, grader_ctx, "easy"
 
         if task_id == "code_review_medium":
@@ -1267,25 +1280,23 @@ class MetaEnvironment(Environment):
             return ctx, ctx, "hard"
 
         if task_id == "data_cleaning_easy":
-            ctx = {"data": DATA_EASY["data"]}
-            return ctx, ctx, "easy"
+            agent_ctx, grader_ctx = gen_data_easy(self._state.episode_id)
+            return agent_ctx, grader_ctx, "easy"
 
         if task_id == "data_cleaning_medium":
-            ctx = {"data": DATA_MEDIUM["data"]}
-            return ctx, ctx, "medium"
+            agent_ctx, grader_ctx = gen_data_medium(self._state.episode_id)
+            return agent_ctx, grader_ctx, "medium"
 
         if task_id == "data_cleaning_hard":
-            ctx = {"data": DATA_HARD["data"]}
-            return ctx, ctx, "hard"
+            agent_ctx, grader_ctx = gen_data_hard(self._state.episode_id)
+            return agent_ctx, grader_ctx, "hard"
 
         if task_id == "content_moderation_easy":
-            agent_ctx  = {"posts": [{k: v for k, v in p.items() if k != "label"} for p in MOD_EASY]}
-            grader_ctx = {"posts": MOD_EASY}
+            agent_ctx, grader_ctx = gen_mod_easy(self._state.episode_id)
             return agent_ctx, grader_ctx, "easy"
 
         if task_id == "content_moderation_medium":
-            agent_ctx  = {"posts": [{k: v for k, v in p.items() if k != "label"} for p in MOD_MEDIUM]}
-            grader_ctx = {"posts": MOD_MEDIUM}
+            agent_ctx, grader_ctx = gen_mod_medium(self._state.episode_id)
             return agent_ctx, grader_ctx, "medium"
 
         if task_id == "content_moderation_hard":
@@ -1294,9 +1305,7 @@ class MetaEnvironment(Environment):
             return agent_ctx, grader_ctx, "hard"
 
         if task_id == "ticket_triage_easy":
-            ticket     = self._deterministic_choice(TICKET_EASY_VARIANTS, task_id)
-            agent_ctx  = {"ticket": {k: v for k, v in ticket.items() if k not in ("correct_priority", "correct_category")}}
-            grader_ctx = {"ticket": ticket}
+            agent_ctx, grader_ctx = gen_ticket_easy(self._state.episode_id)
             return agent_ctx, grader_ctx, "easy"
 
         if task_id == "ticket_triage_medium":
@@ -1332,6 +1341,33 @@ class MetaEnvironment(Environment):
             grader_ctx = {"case": case}
             return agent_ctx, grader_ctx, "hard"
 
+        
+        if task_id == "email_triage_expert":
+            agent_ctx  = {k: v for k, v in EMAIL_EXPERT_CONTEXT.items()}
+            grader_ctx = agent_ctx
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "code_review_expert":
+            agent_ctx  = {"code": CODE_EXPERT_CONTEXT["code"],
+                          "vulnerability_count": CODE_EXPERT_CONTEXT["vulnerability_count"]}
+            grader_ctx = CODE_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "data_cleaning_expert":
+            agent_ctx  = DATA_EXPERT_CONTEXT
+            grader_ctx = DATA_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "content_moderation_expert":
+            agent_ctx  = MOD_EXPERT_CONTEXT
+            grader_ctx = MOD_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "ticket_triage_expert":
+            agent_ctx  = TICKET_EXPERT_CONTEXT
+            grader_ctx = TICKET_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
         raise ValueError(f"Unknown task_id: '{task_id}'. Use GET /tasks to see valid IDs.")
 
     def _grade(self, task_id: str, payload: dict, grader_context: dict) -> tuple[float, str, dict]:
@@ -1343,9 +1379,9 @@ class MetaEnvironment(Environment):
         if task_id == "code_review_hard":            return grade_code_hard(payload)
         if task_id == "data_cleaning_easy":          return grade_data_easy(payload)
         if task_id == "data_cleaning_medium":        return grade_data_medium(payload)
-        if task_id == "data_cleaning_hard":          return grade_data_hard(payload)
-        if task_id == "content_moderation_easy":     return grade_mod_easy(payload)
-        if task_id == "content_moderation_medium":   return grade_mod_medium(payload)
+        if task_id == "data_cleaning_hard":          return grade_data_hard(payload, grader_context)
+        if task_id == "content_moderation_easy":     return grade_mod_easy(payload, grader_context)
+        if task_id == "content_moderation_medium":   return grade_mod_medium(payload, grader_context)
         if task_id == "content_moderation_hard":     return grade_mod_hard(payload)
         if task_id == "ticket_triage_easy":          return grade_ticket_easy(payload, grader_context)
         if task_id == "ticket_triage_medium":        return grade_ticket_medium(payload)

server/Meta_environment.py.bak

@@ -0,0 +1,1520 @@
+"""
+Meta Multi-Agent Environment v3.0
+
+Five real-world AI agent domains, 16 tasks total:
+  1. Email Triage        — classify, prioritize, draft replies
+  2. Code Review         — syntax errors, logic bugs, security vulnerabilities (harder)
+  3. Data Cleaning       — missing values, type normalization, outlier imputation
+  4. Content Moderation  — explicit, subtle toxicity, context-aware moderation (harder)
+  5. Ticket Triage       — NEW: Jira-style bug ticket priority, routing, incident analysis
+
+Cross-agent chained tasks (4 total):
+  cross_agent_chain          — email triage + code review
+  cross_agent_email_data     — email priority + data cleaning
+  cross_agent_code_email     — vulnerability detection + security disclosure email
+  cross_agent_mod_escalation — content moderation + moderation notice drafting
+
+v3.0 improvements:
+  - Deterministic context loading via episode_id hash (no random.choice breaking reproducibility)
+  - 5th domain: Ticket Triage (Jira-style)
+  - 4 cross-agent chained tasks (was 1)
+  - Harder hard tasks: 8 vulnerabilities in code_hard, adversarial content_mod_hard
+  - Trajectory-aware reward shaping with improvement bonuses
+  - Clean single-import pattern (PYTHONPATH set correctly in Dockerfile)
+  - Full type hints on all grader functions
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from typing import Any
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+from openenv.core.env_server.types import State
+
+from models import MetaAction, MetaObservation
+
+from data_generators import (
+    gen_email_easy, gen_email_medium, gen_email_hard,
+    gen_code_easy,
+    gen_data_easy, gen_data_medium, gen_data_hard,
+    gen_mod_easy, gen_mod_medium,
+    gen_ticket_easy,
+)
+from expert_tasks import (
+    EMAIL_EXPERT_CONTEXT, grade_email_expert,
+    CODE_EXPERT_CONTEXT,  grade_code_expert,
+    DATA_EXPERT_CONTEXT,  grade_data_expert,
+    MOD_EXPERT_CONTEXT,   grade_mod_expert,
+    TICKET_EXPERT_CONTEXT, grade_ticket_expert,
+    EXPERT_INSTRUCTIONS, EXPERT_TASK_IDS,
+)
+
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# DATA
+# ═════════════════════════════════════════════════════════════════════════════
+
+EMAIL_EASY_VARIANTS = [
+    {
+        "id": "e1",
+        "subject": "You won $1,000,000!!!",
+        "sender": "promo@totally-legit-lottery.com",
+        "body": "Click here to claim your prize now! Limited time offer!",
+        "label": "spam",
+    },
+    {
+        "id": "e2",
+        "subject": "Q3 Budget Review — Urgent Action Required",
+        "sender": "cfo@company.com",
+        "body": "Please review the attached Q3 budget before our board meeting tomorrow at 9 AM.",
+        "label": "important",
+    },
+    {
+        "id": "e3",
+        "subject": "Your weekly digest from Medium",
+        "sender": "noreply@medium.com",
+        "body": "Top stories this week: AI trends, Python tips, and startup news.",
+        "label": "newsletter",
+    },
+    {
+        "id": "e4",
+        "subject": "CONGRATULATIONS! iPhone 15 Pro is yours!",
+        "sender": "winner@prize-alert.net",
+        "body": "You have been selected. Click now to claim your free iPhone!",
+        "label": "spam",
+    },
+    {
+        "id": "e5",
+        "subject": "Action Required: Sign the NDA before Friday",
+        "sender": "legal@company.com",
+        "body": "Please sign the attached NDA before Friday's partnership meeting. Deadline is strict.",
+        "label": "important",
+    },
+    {
+        "id": "e6",
+        "subject": "TechCrunch Newsletter — March 2026",
+        "sender": "newsletter@techcrunch.com",
+        "body": "This week's top stories in tech, AI, and startups.",
+        "label": "newsletter",
+    },
+]
+
+EMAIL_MEDIUM_EMAILS = [
+    {"id": "m1",  "subject": "[ALERT] Production server DOWN",          "sender": "ops@company.com",      "body": "All services unresponsive since 2 AM. Revenue impact: $10k/min.",             "priority": 1,  "category": "critical"},
+    {"id": "m2",  "subject": "Team lunch this Friday?",                  "sender": "colleague@company.com","body": "Hey, thinking of doing a team lunch Friday. You in?",                         "priority": 9,  "category": "social"},
+    {"id": "m3",  "subject": "Acme Corp contract renewal — URGENT",      "sender": "sales@company.com",    "body": "Acme contract expires Friday. They need signature or they walk.",               "priority": 2,  "category": "business"},
+    {"id": "m4",  "subject": "Weekly SaaS digest",                       "sender": "news@saasdigest.com",  "body": "Top SaaS stories this week...",                                               "priority": 10, "category": "newsletter"},
+    {"id": "m5",  "subject": "[WARNING] Security alert: unusual login",  "sender": "security@company.com","body": "Unauthorized login attempt from IP 45.33.32.156. Verify immediately.",          "priority": 1,  "category": "critical"},
+    {"id": "m6",  "subject": "Invoice #INV-4521 overdue",                "sender": "billing@vendor.com",   "body": "Invoice of $5,400 was due 3 days ago. Payment needed to avoid suspension.",   "priority": 3,  "category": "finance"},
+    {"id": "m7",  "subject": "Happy Work Anniversary!",                  "sender": "hr@company.com",       "body": "Celebrating your 3 years with us!",                                           "priority": 10, "category": "social"},
+    {"id": "m8",  "subject": "Q4 product launch deck — review needed",   "sender": "marketing@company.com","body": "Please review launch slides before Thursday. Launch is next Monday.",          "priority": 3,  "category": "business"},
+    {"id": "m9",  "subject": "Free AI productivity webinar",             "sender": "promo@webinar.io",     "body": "Join our free webinar on AI tools this Thursday.",                            "priority": 8,  "category": "newsletter"},
+    {"id": "m10", "subject": "Legal: Partnership agreement needs signature TODAY", "sender": "legal@company.com", "body": "The partnership agreement must be countersigned by EOD or the deal falls through.", "priority": 2, "category": "legal"},
+]
+
+EMAIL_MEDIUM_CORRECT_ORDER = ["m1", "m5", "m3", "m10", "m6", "m8", "m2", "m9", "m4", "m7"]
+
+EMAIL_HARD_CASES = [
+    {
+        "id": "h1",
+        "subject": "Completely unacceptable — Order #78234",
+        "sender": "furious.customer@email.com",
+        "body": (
+            "I have been a loyal customer for 5 years and I am absolutely furious. "
+            "My order #78234 arrived broken for the SECOND time this month. "
+            "Your support team has been dismissive and unhelpful every time I called. "
+            "I demand a full refund, a replacement sent overnight, AND compensation. "
+            "If this is not resolved within 24 hours, I will file a consumer court complaint."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "frustrat", "disappoint", "concern", "feel"],
+            "apologize": ["sorry", "apologize", "apology", "sincerely"],
+            "offer_refund": ["refund", "money back", "full refund"],
+            "offer_replacement": ["replacement", "replace", "new order", "send another", "overnight"],
+            "provide_timeline": ["24 hours", "48 hours", "within", "today", "tomorrow", "business day"],
+            "professional_tone": None,
+            "compensation_acknowledged": ["compensat", "inconvenience", "make it right", "goodwill"],
+        },
+    },
+    {
+        "id": "h2",
+        "subject": "Billing error — charged twice for subscription",
+        "sender": "billing.complaint@customer.com",
+        "body": (
+            "I was charged twice for my annual subscription this month — once on March 1st "
+            "and again on March 15th. I have the bank statements as proof. "
+            "This is a clear billing error on your end. I need an immediate refund of the "
+            "duplicate charge and a written confirmation that this won't happen again. "
+            "I've been waiting 2 weeks for a response from your support team."
+        ),
+        "expected_elements": {
+            "acknowledge_frustration": ["understand", "concern", "apologize", "sorry"],
+            "apologize": ["sorry", "apologize", "apology"],
+            "offer_refund": ["refund", "reimburse", "credit"],
+            "confirm_investigation": ["investigat", "look into", "check", "review", "verify"],
+            "provide_timeline": ["within", "hours", "days", "business day"],
+            "professional_tone": None,
+            "written_confirmation": ["confirm", "written", "email confirmation", "receipt"],
+        },
+    },
+]
+
+CODE_EASY_VARIANTS = [
+    {
+        "code": "def calculate_average(numbers):\n    total = 0\n    for num in numbers\n        total += num\n    return total / len(numbers)\n",
+        "hint": "missing colon after for statement",
+        "keywords": ["colon", "for loop", "syntax", "for statement"],
+    },
+    {
+        "code": "def greet(name)\n    print(f'Hello, {name}!')\n\ngreet('World')\n",
+        "hint": "missing colon after function definition",
+        "keywords": ["colon", "def", "function", "syntax"],
+    },
+    {
+        "code": "numbers = [1, 2, 3, 4, 5]\nif len(numbers) > 0\n    print('Not empty')\n",
+        "hint": "missing colon after if statement",
+        "keywords": ["colon", "if statement", "syntax"],
+    },
+]
+
+CODE_MEDIUM = {
+    "code": (
+        "def find_max(numbers):\n"
+        "    max_val = 0\n"
+        "    for num in numbers:\n"
+        "        if num > max_val:\n"
+        "            max_val = num\n"
+        "    return max_val\n\n"
+        "def get_user(users, user_id):\n"
+        "    for user in users:\n"
+        "        if user['id'] = user_id:\n"
+        "            return user\n"
+        "    return None\n\n"
+        "def divide(a, b):\n"
+        "    return a / b\n\n"
+        "def process_list(items):\n"
+        "    results = []\n"
+        "    for i in range(len(items) + 1):\n"
+        "        results.append(items[i] * 2)\n"
+        "    return results\n"
+    ),
+    "bugs": {
+        "max_val_init":     "find_max initializes max_val=0, fails for all-negative lists",
+        "assignment_vs_eq": "get_user uses = (assignment) instead of == (comparison)",
+        "zero_division":    "divide() has no guard against division by zero",
+        "off_by_one":       "process_list iterates range(len(items)+1) causing IndexError",
+    },
+}
+
+# HARDER: 8 vulnerabilities now (was 5) — frontier models won't trivially score 1.0
+CODE_HARD = {
+    "code": (
+        "import sqlite3\n"
+        "import subprocess\n"
+        "import pickle\n"
+        "import requests\n"
+        "import hashlib\n"
+        "import os\n\n"
+        "# Vulnerability 1: SQL injection\n"
+        "def get_user_by_name(name):\n"
+        "    conn = sqlite3.connect('users.db')\n"
+        "    cursor = conn.cursor()\n"
+        "    query = \"SELECT * FROM users WHERE name = '\" + name + \"'\"\n"
+        "    cursor.execute(query)\n"
+        "    return cursor.fetchall()\n\n"
+        "# Vulnerability 2: XSS\n"
+        "def render_comment(comment):\n"
+        "    return '<div class=\"comment\">' + comment + '</div>'\n\n"
+        "# Vulnerability 3: SQL injection\n"
+        "def login(username, password):\n"
+        "    conn = sqlite3.connect('users.db')\n"
+        "    cursor = conn.cursor()\n"
+        "    query = f\"SELECT * FROM users WHERE username='{username}' AND password='{password}'\"\n"
+        "    cursor.execute(query)\n"
+        "    return cursor.fetchone() is not None\n\n"
+        "# Vulnerability 4: Command injection\n"
+        "def run_report(report_name):\n"
+        "    subprocess.call('generate_report.sh ' + report_name, shell=True)\n\n"
+        "# Vulnerability 5: Insecure deserialization\n"
+        "def load_user_data(data_bytes):\n"
+        "    return pickle.loads(data_bytes)\n\n"
+        "# Vulnerability 6: Timing attack on password comparison\n"
+        "def check_password(stored_hash, input_password):\n"
+        "    return stored_hash == hashlib.md5(input_password.encode()).hexdigest()\n\n"
+        "# Vulnerability 7: Path traversal\n"
+        "def read_report_file(filename):\n"
+        "    with open(f'/reports/{filename}') as f:\n"
+        "        return f.read()\n\n"
+        "# Vulnerability 8: SSRF\n"
+        "def fetch_user_avatar(avatar_url):\n"
+        "    response = requests.get(avatar_url)\n"
+        "    return response.content\n"
+    ),
+    "vulnerabilities": {
+        "sql_injection_get_user":      {"type": "sql_injection",          "location": "get_user_by_name",  "severity": "critical"},
+        "xss_render_comment":          {"type": "xss",                    "location": "render_comment",    "severity": "high"},
+        "sql_injection_login":         {"type": "sql_injection",          "location": "login",             "severity": "critical"},
+        "command_injection":           {"type": "command_injection",      "location": "run_report",        "severity": "critical"},
+        "insecure_deserialization":    {"type": "insecure_deserialization","location": "load_user_data",   "severity": "high"},
+        "timing_attack":               {"type": "timing_attack",          "location": "check_password",    "severity": "medium"},
+        "path_traversal":              {"type": "path_traversal",         "location": "read_report_file",  "severity": "high"},
+        "ssrf":                        {"type": "ssrf",                   "location": "fetch_user_avatar", "severity": "high"},
+    },
+}
+
+DATA_EASY = {
+    "data": [
+        {"id": 1, "name": "Alice",  "age": 30,   "email": "alice@example.com",   "salary": 70000},
+        {"id": 2, "name": "Bob",    "age": None,  "email": "bob@example.com",     "salary": 85000},
+        {"id": 3, "name": "Alice",  "age": 30,    "email": "alice@example.com",   "salary": 70000},
+        {"id": 4, "name": None,     "age": 25,    "email": "charlie@example.com", "salary": 60000},
+        {"id": 5, "name": "Dave",   "age": 40,    "email": None,                  "salary": 90000},
+        {"id": 6, "name": "Eve",    "age": 28,    "email": "eve@example.com",     "salary": None},
+    ],
+    "answers": {
+        "missing":    ["age (row 2)", "name (row 4)", "email (row 5)", "salary (row 6)"],
+        "duplicates": [1, 3],
+    },
+}
+
+DATA_MEDIUM = {
+    "data": [
+        {"id": 1, "name": "Alice",   "age": "thirty",     "salary": "50000",   "join_date": "2020/01/15", "active": "yes"},
+        {"id": 2, "name": "Bob",     "age": 25,            "salary": "$60,000", "join_date": "15-03-2021", "active": True},
+        {"id": 3, "name": "Charlie", "age": "28",          "salary": "70000",   "join_date": "2019-07-22", "active": "TRUE"},
+        {"id": 4, "name": "dave",    "age": 35,            "salary": "80,000",  "join_date": "2022/11/01", "active": False},
+        {"id": 5, "name": "EVE",     "age": "twenty-two",  "salary": "45000",   "join_date": "01/06/2023", "active": "no"},
+    ],
+    "issues": {
+        "age":       "mixed types: strings 'thirty', '28', 'twenty-two' should be integers",
+        "salary":    "inconsistent formats: '$60,000' and '80,000' have symbols/commas",
+        "join_date": "three different date formats",
+        "name":      "inconsistent casing: 'dave' lowercase, 'EVE' all caps",
+        "active":    "mixed types: True/False booleans mixed with strings 'yes'/'no'/'TRUE'",
+    },
+}
+
+DATA_HARD = {
+    "data": [
+        {"id": 1,  "value": 10.5},  {"id": 2,  "value": 11.2},
+        {"id": 3,  "value": 10.8},  {"id": 4,  "value": 999.0},
+        {"id": 5,  "value": 10.1},  {"id": 6,  "value": None},
+        {"id": 7,  "value": 11.5},  {"id": 8,  "value": -500.0},
+        {"id": 9,  "value": 10.9},  {"id": 10, "value": 11.0},
+        {"id": 11, "value": None},  {"id": 12, "value": 10.7},
+    ],
+    "answers": {
+        "outliers":      {4, 8},
+        "missing":       {6, 11},
+        "imputed_range": (10.0, 12.0),
+    },
+}
+
+MOD_EASY = [
+    {"id": "c1", "text": "I hate you so much, I want to hurt you!",             "label": "harmful"},
+    {"id": "c2", "text": "Great weather today! Loving this sunshine",            "label": "safe"},
+    {"id": "c3", "text": "All people from that group should be eliminated.",     "label": "harmful"},
+    {"id": "c4", "text": "Can someone recommend a good Python beginner book?",  "label": "safe"},
+    {"id": "c5", "text": "You're an absolute idiot and nobody likes you.",      "label": "harmful"},
+    {"id": "c6", "text": "I just finished a 5k run — feeling amazing!",         "label": "safe"},
+    {"id": "c7", "text": "People like you should just disappear forever.",      "label": "harmful"},
+]
+
+MOD_MEDIUM = [
+    {"id": "m1", "text": "Oh sure, like THAT went well. Classic genius move.",              "label": "harmful"},
+    {"id": "m2", "text": "I'm so done with these people. Absolutely insufferable.",         "label": "harmful"},
+    {"id": "m3", "text": "That presentation was... very unique.",                           "label": "safe"},
+    {"id": "m4", "text": "Some people just shouldn't be allowed to have opinions.",        "label": "harmful"},
+    {"id": "m5", "text": "Oh wow, another 'brilliant' idea from upper management",         "label": "harmful"},
+    {"id": "m6", "text": "Congrats on the promotion! Well deserved!",                      "label": "safe"},
+    {"id": "m7", "text": "Not everyone is cut out for this kind of work, clearly.",        "label": "harmful"},
+    {"id": "m8", "text": "Thanks for the feedback, I'll take it into account.",            "label": "safe"},
+]
+
+# HARDER: adversarial cases including dog-whistles and coordinated inauthentic patterns
+MOD_HARD = [
+    {
+        "id": "h1",
+        "text": "You should just disappear.",
+        "context_a": {"platform": "gaming_chat",       "prior_message": "Your character died in the dungeon. Respawn?"},
+        "label_a": "safe",
+        "context_b": {"platform": "crisis_support_chat","prior_message": "I've been feeling worthless and invisible lately"},
+        "label_b": "harmful",
+    },
+    {
+        "id": "h2",
+        "text": "We need to take these people out.",
+        "context_a": {"platform": "hr_slack",          "prior_message": "Which team members should join the client dinner?"},
+        "label_a": "safe",
+        "context_b": {"platform": "extremist_forum",   "prior_message": "These immigrants are ruining our country."},
+        "label_b": "harmful",
+    },
+    {
+        "id": "h3",
+        "text": "Let's eliminate the competition.",
+        "context_a": {"platform": "business_slack",    "prior_message": "Q4 sales strategy meeting"},
+        "label_a": "safe",
+        "context_b": {"platform": "political_rally_chat","prior_message": "The opposition party is destroying our values."},
+        "label_b": "harmful",
+    },
+    # NEW adversarial cases — much harder
+    {
+        "id": "h4",
+        "text": "Finally, the great replacement is happening in our neighborhood.",
+        "context_a": {"platform": "news_comments",     "prior_message": "New families moving into Maplewood as housing market cools."},
+        "label_a": "harmful",   # dog-whistle regardless of context
+        "context_b": {"platform": "urban_planning_forum","prior_message": "Demographics shifting as millennials move to suburbs."},
+        "label_b": "harmful",   # still a white-nationalist dog-whistle
+    },
+    {
+        "id": "h5",
+        "text": "Everyone should follow our example and do the same thing at noon tomorrow.",
+        "context_a": {"platform": "community_events",  "prior_message": "We're all doing a neighborhood cleanup. Join us!"},
+        "label_a": "safe",
+        "context_b": {"platform": "extremist_channel", "prior_message": "The government building is unguarded on weekends."},
+        "label_b": "harmful",   # coordinated inauthentic / incitement
+    },
+    {
+        "id": "h6",
+        "text": "These people breed like rabbits and will replace us if we don't act.",
+        "context_a": {"platform": "biology_classroom", "prior_message": "Comparing reproductive rates of different animal species"},
+        "label_a": "harmful",   # dehumanizing regardless — not biology
+        "context_b": {"platform": "political_forum",   "prior_message": "Immigration policy discussion thread"},
+        "label_b": "harmful",
+    },
+]
+
+# ── NEW: Ticket Triage Domain ─────────────────────────────────────────────────
+
+TICKET_EASY_VARIANTS = [
+    {
+        "id": "t1",
+        "title": "App crashes on login for all users",
+        "description": "Since the 2.3.1 deploy, 100% of login attempts throw a 500 error. Users cannot access the platform.",
+        "reporter": "ops-monitoring@company.com",
+        "correct_priority": "critical",
+        "correct_category": "bug",
+    },
+    {
+        "id": "t2",
+        "title": "Add dark mode to the dashboard",
+        "description": "Multiple users have requested a dark mode option. Would improve accessibility.",
+        "reporter": "product@company.com",
+        "correct_priority": "low",
+        "correct_category": "feature_request",
+    },
+    {
+        "id": "t3",
+        "title": "How do I export my data as CSV?",
+        "description": "I can't find the export button anywhere in the UI. Where is it?",
+        "reporter": "user123@customer.com",
+        "correct_priority": "low",
+        "correct_category": "question",
+    },
+    {
+        "id": "t4",
+        "title": "Charged twice for last month's invoice",
+        "description": "Account #AC-4421 shows two charges of $299 on March 1st. Please refund duplicate.",
+        "reporter": "finance@customer.com",
+        "correct_priority": "high",
+        "correct_category": "billing",
+    },
+]
+
+TICKET_MEDIUM_TICKETS = [
+    {"id": "tk1", "title": "Database connection pool exhausted — prod down",   "type": "bug",            "team": "backend",  "priority_rank": 1},
+    {"id": "tk2", "title": "Wrong currency symbol shown for EU customers",     "type": "bug",            "team": "frontend", "priority_rank": 3},
+    {"id": "tk3", "title": "Add OAuth2 support for enterprise SSO",            "type": "feature_request","team": "backend",  "priority_rank": 5},
+    {"id": "tk4", "title": "SSL cert expires in 48 hours on api.company.com",  "type": "bug",            "team": "devops",   "priority_rank": 2},
+    {"id": "tk5", "title": "Invoice PDF generation fails for invoices > 50 lines","type": "bug",         "team": "backend",  "priority_rank": 4},
+    {"id": "tk6", "title": "Redesign onboarding flow to reduce drop-off",      "type": "feature_request","team": "frontend", "priority_rank": 7},
+    {"id": "tk7", "title": "Customer billed $0 instead of $299 — revenue leak","type": "bug",            "team": "billing",  "priority_rank": 3},
+    {"id": "tk8", "title": "Add CSV export for analytics dashboard",            "type": "feature_request","team": "frontend", "priority_rank": 8},
+]
+
+TICKET_MEDIUM_CORRECT_ORDER = ["tk1", "tk4", "tk2", "tk7", "tk5", "tk3", "tk6", "tk8"]
+TICKET_MEDIUM_CORRECT_TEAMS = {
+    "tk1": "backend", "tk2": "frontend", "tk3": "backend",
+    "tk4": "devops",  "tk5": "backend",  "tk6": "frontend",
+    "tk7": "billing", "tk8": "frontend",
+}
+
+TICKET_HARD_INCIDENT = {
+    "tickets": [
+        {"id": "INC-001", "title": "Payment service returning 503",          "created": "14:00", "service": "payment-service"},
+        {"id": "INC-002", "title": "Database CPU at 100% on primary replica","created": "14:01", "service": "postgres-primary"},
+        {"id": "INC-003", "title": "Checkout flow failing for all users",    "created": "14:02", "service": "checkout-service"},
+        {"id": "INC-004", "title": "Redis memory at 99% — evictions spiking","created": "14:03", "service": "redis-cache"},
+        {"id": "INC-005", "title": "Background job queue backed up 50k jobs","created": "14:05", "service": "job-queue"},
+        {"id": "INC-006", "title": "CDN edge nodes reporting origin timeouts","created": "14:06", "service": "cdn"},
+    ],
+    "expected": {
+        "root_cause_keywords": [
+            "database", "postgres", "cpu", "memory", "redis", "cache",
+            "connection pool", "exhausted", "query", "slow query", "replica",
+        ],
+        "affected_services": {"payment-service", "checkout-service", "postgres-primary", "redis-cache"},
+        "severity": "P1",
+        "resolution_keywords": [
+            "scale", "replica", "failover", "restart", "flush", "cache",
+            "query", "optimize", "rollback", "connection", "limit",
+        ],
+    },
+}
+
+# ── Cross-agent data ─────────────────────���────────────────────────────────────
+
+CROSS_AGENT_CHAIN_CASES = [
+    {
+        "id": "ca1",
+        "email": {
+            "subject": "Bug report: payment processor crashing",
+            "sender": "dev@company.com",
+            "body": "Our payment processor keeps throwing a ZeroDivisionError in production. See attached code snippet.",
+        },
+        "email_label": "important",
+        "code": (
+            "def calculate_fee(amount, rate):\n"
+            "    return amount / rate\n\n"
+            "def process_payment(amount, fee_rate):\n"
+            "    fee = calculate_fee(amount, fee_rate)\n"
+            "    return amount - fee\n"
+        ),
+        "code_bug": "zero_division",
+        "code_bug_keywords": ["zero", "division", "divide", "rate", "denominator", "fee_rate", "guard"],
+        "code_location": "calculate_fee",
+    },
+    {
+        "id": "ca2",
+        "email": {
+            "subject": "Weekly newsletter: Python tips",
+            "sender": "newsletter@pythonweekly.com",
+            "body": "This week: list comprehensions, async patterns, and a buggy code snippet for you to fix!",
+        },
+        "email_label": "newsletter",
+        "code": (
+            "def get_first(items):\n"
+            "    return items[0]\n\n"
+            "def summarize(data):\n"
+            "    total = sum(data)\n"
+            "    first = get_first(data)\n"
+            "    return total, first\n"
+        ),
+        "code_bug": "empty_list",
+        "code_bug_keywords": ["empty", "index", "indexerror", "list", "bound", "check", "length", "len", "guard"],
+        "code_location": "get_first",
+    },
+]
+
+CROSS_AGENT_EMAIL_DATA_CASES = [
+    {
+        "id": "ced1",
+        "email": {
+            "subject": "[URGENT] Sales data export corrupted — board meeting tomorrow",
+            "sender": "vp-sales@company.com",
+            "body": "The Q4 sales export is corrupted. I need it fixed before the 9 AM board meeting tomorrow.",
+        },
+        "email_priority": "critical",
+        "attachment_data": [
+            {"id": 1, "rep": "Alice",  "revenue": "150000",   "region": "NORTH", "closed_date": "2024/12/31"},
+            {"id": 2, "rep": "bob",    "revenue": "$80,000",  "region": "south", "closed_date": "31-12-2024"},
+            {"id": 3, "rep": "Charlie","revenue": None,       "region": "East",  "closed_date": "2024-12-30"},
+            {"id": 4, "rep": "Diana",  "revenue": "120000",   "region": "WEST",  "closed_date": "12/30/2024"},
+        ],
+        "expected_issues": {"revenue": True, "region": True, "closed_date": True},
+    },
+]
+
+CROSS_AGENT_CODE_EMAIL_CASES = [
+    {
+        "id": "cce1",
+        "code": (
+            "def get_user_profile(user_id):\n"
+            "    conn = sqlite3.connect('app.db')\n"
+            "    query = \"SELECT * FROM users WHERE id = \" + user_id\n"
+            "    cursor = conn.execute(query)\n"
+            "    return cursor.fetchone()\n"
+        ),
+        "primary_vulnerability": "sql_injection",
+        "location": "get_user_profile",
+        "disclosure_keywords": [
+            "sql injection", "parameterized", "vulnerability",
+            "security", "patch", "remediat", "fix", "urgent",
+        ],
+    },
+]
+
+CROSS_AGENT_MOD_ESCALATION_CASES = [
+    {
+        "id": "cme1",
+        "post": {
+            "id": "p1",
+            "text": "I know where you live and I will make you regret this post.",
+            "platform": "social_media",
+            "context": "Reply to a political opinion post",
+        },
+        "correct_label": "harmful",
+        "should_escalate": True,
+        "notice_keywords": [
+            "violat", "policy", "removed", "community", "guidelines",
+            "threat", "harm", "prohibit",
+        ],
+    },
+    {
+        "id": "cme2",
+        "post": {
+            "id": "p2",
+            "text": "Great recipe! Can't wait to try this at home.",
+            "platform": "cooking_forum",
+            "context": "Reply to a pasta recipe post",
+        },
+        "correct_label": "safe",
+        "should_escalate": False,
+        "notice_keywords": [],
+    },
+]
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GRADERS
+# ═════════════════════════════════════════════════════════════════════════════
+
+def grade_email_easy(payload: dict, context: dict) -> tuple[float, str, dict]:
+    correct = context["email"]["label"]
+    given   = payload.get("classification", "").strip().lower()
+    valid   = ["spam", "important", "newsletter"]
+    if given == correct:
+        return 1.0, f"[OK] Correct! '{correct}' is right.", {}
+    if given in valid:
+        return 0.0, f"[FAIL] Wrong. Expected '{correct}', got '{given}'.", {"classification": False}
+    return 0.0, f"[FAIL] Invalid label '{given}'. Must be: spam | important | newsletter.", {"classification": False}
+
+
+def _kendall_tau_score(order: list, correct: list) -> float:
+    """
+    Kendall's tau-based ordering score.
+    Counts concordant pairs (relative order matches) vs total pairs.
+    Much fairer than exact position matching — rewards getting the overall
+    priority ranking right even when a few adjacent items swap.
+    Score range: 0.0 (fully reversed) to 1.0 (perfect order).
+    """
+    n = len(correct)
+    pos = {e: i for i, e in enumerate(correct)}
+    concordant = discordant = 0
+    for i in range(len(order)):
+        for j in range(i + 1, len(order)):
+            if order[i] not in pos or order[j] not in pos:
+                continue
+            if pos[order[i]] < pos[order[j]]:
+                concordant += 1
+            else:
+                discordant += 1
+    total = concordant + discordant
+    return concordant / total if total > 0 else 0.0
+
+
+def grade_email_medium(payload: dict, context: dict) -> tuple[float, str, dict]:
+    order   = payload.get("order", [])
+    correct = context.get("correct_order", EMAIL_MEDIUM_CORRECT_ORDER)
+    if not order:
+        return 0.0, "[FAIL] No order provided.", {}
+    top2_ok  = set(order[:2]) == {"m1", "m5"}
+    top5_ok  = order[:5] == correct[:5]
+    full_ok  = order == correct
+    tau      = _kendall_tau_score(order, correct)
+    hits     = sum(1 for i, e in enumerate(order) if i < len(correct) and e == correct[i])
+    partial  = {"top2_critical": top2_ok, "top5_correct": top5_ok, "full_order": full_ok}
+    if full_ok:
+        return 1.0, "[OK] Perfect prioritization!", partial
+    if top5_ok:
+        score = round(0.8 + tau * 0.2, 2)
+        return score, f"[PARTIAL] Top 5 correct. Kendall tau: {tau:.2f}.", partial
+    if top2_ok:
+        # Tau-based score: 0.5 minimum when criticals placed first, scaled up by pair agreement
+        score = round(max(0.5, tau), 2)
+        return score, f"[PARTIAL] Critical emails (m1, m5) first. {hits}/10 exact positions. Tau: {tau:.2f}.", partial
+    score = round(tau * 0.8, 2)
+    return score, f"[ERROR] {hits}/10 exact positions correct. Kendall tau: {tau:.2f}.", partial
+
+
+def grade_email_hard(payload: dict, context: dict) -> tuple[float, str, dict]:
+    reply    = payload.get("reply", "").lower()
+    case     = context["email"]
+    expected = case.get("expected_elements", {})
+    checks: dict[str, bool] = {}
+    for key, keywords in expected.items():
+        if key == "professional_tone":
+            checks[key] = (
+                len(reply) > 80
+                and not any(w in reply for w in ["not my problem", "whatever", "too bad", "deal with it"])
+            )
+        else:
+            checks[key] = keywords is not None and any(w in reply for w in keywords)
+    passed   = sum(checks.values())
+    total    = len(checks)
+    score    = round(passed / total, 2)
+    missing  = [k for k, v in checks.items() if not v]
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.6 else "[ERROR]"
+    feedback = f"{tag} {passed}/{total} reply elements present."
+    if missing:
+        feedback += f" Missing: {', '.join(missing)}."
+    return score, feedback, checks
+
+
+def grade_code_easy(payload: dict, context: dict) -> tuple[float, str, dict]:
+    errors   = " ".join(str(e) for e in payload.get("errors", [])).lower()
+    keywords = context.get("keywords", ["colon", "syntax"])
+    found    = any(k in errors for k in keywords)
+    if not payload.get("errors"):
+        return 0.0, "[FAIL] No errors reported.", {"syntax_error_found": False}
+    if found:
+        return 1.0, "[OK] Syntax error correctly identified.", {"syntax_error_found": True}
+    return 0.3, "[PARTIAL] Error mentioned but key issue not clearly described.", {"syntax_error_found": False}
+
+
+def grade_code_medium(payload: dict) -> tuple[float, str, dict]:
+    bugs   = payload.get("bugs", [])
+    checks = {"max_val_init": False, "assignment_vs_eq": False, "zero_division": False, "off_by_one": False}
+    for b in bugs:
+        t   = (str(b.get("issue", "")) + str(b.get("location", "")) + str(b.get("fix", ""))).lower()
+        loc = str(b.get("location", "")).lower()
+        if loc == "find_max" or any(w in t for w in ["max_val", "negative", "minus", "zero init", "float('-inf')", "all-negative", "all negative", "initialization", "find_max"]):
+            checks["max_val_init"] = True
+        if loc == "get_user" or any(w in t for w in ["assignment", "==", "comparison", "= user_id", "single equal", "get_user"]):
+            checks["assignment_vs_eq"] = True
+        if loc == "divide" or any(w in t for w in ["zero", "division", "divide", "zerodivision", "b == 0", "divisor", "denominator", "div by zero"]):
+            checks["zero_division"] = True
+        if loc == "process_list" or any(w in t for w in ["off by one", "off-by-one", "index", "range", "len + 1", "indexerror", "out of bounds", "boundary", "len(items) + 1", "process_list"]):
+            checks["off_by_one"] = True
+    passed   = sum(checks.values())
+    score    = round(passed / 4, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    feedback = f"{tag} {passed}/4 bugs found."
+    missing  = [k for k, v in checks.items() if not v]
+    if missing:
+        feedback += f" Missed: {', '.join(missing)}."
+    return score, feedback, checks
+
+
+def grade_code_hard(payload: dict) -> tuple[float, str, dict]:
+    """Now grading 8 vulnerabilities — significantly harder than before."""
+    vulns  = payload.get("vulnerabilities", [])
+    checks = {
+        "sql_injection_get_user":   False,
+        "xss_render_comment":       False,
+        "sql_injection_login":      False,
+        "command_injection":        False,
+        "insecure_deserialization": False,
+        "timing_attack":            False,
+        "path_traversal":           False,
+        "ssrf":                     False,
+    }
+    for v in vulns:
+        vt  = str(v.get("type", "")).lower()
+        loc = str(v.get("location", "")).lower()
+        if "get_user" in loc or "get_user_by_name" in loc:
+            checks["sql_injection_get_user"] = True
+        if ("render" in loc or "comment" in loc) and any(w in vt for w in ["xss", "cross", "script", "html", "inject", "sanitiz", "encod", "output"]):
+            checks["xss_render_comment"] = True
+        if "render" in loc and "comment" in loc:
+            checks["xss_render_comment"] = True
+        if "login" in loc:
+            checks["sql_injection_login"] = True
+        if "report" in loc and any(w in vt for w in ["command", "injection", "shell"]):
+            checks["command_injection"] = True
+        if "report" in loc and "run" in loc:
+            checks["command_injection"] = True
+        if "load" in loc and any(w in vt for w in ["pickle", "deserializ", "serial"]):
+            checks["insecure_deserialization"] = True
+        if "load_user" in loc:
+            checks["insecure_deserialization"] = True
+        if "check_password" in loc or any(w in vt for w in ["timing", "time", "constant"]):
+            checks["timing_attack"] = True
+        if "read_report" in loc or "path" in vt or "traversal" in vt or "directory" in vt:
+            checks["path_traversal"] = True
+        if "fetch" in loc or "avatar" in loc or "ssrf" in vt or "request_forgery" in vt:
+            checks["ssrf"] = True
+    passed   = sum(checks.values())
+    score    = round(passed / 8, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    feedback = f"{tag} {passed}/8 vulnerabilities found."
+    missing  = [k for k, v in checks.items() if not v]
+    if missing:
+        feedback += f" Missed: {', '.join(missing)}."
+    return score, feedback, checks
+
+
+def grade_data_easy(payload: dict) -> tuple[float, str, dict]:
+    missing_given = [str(m).lower() for m in payload.get("missing", [])]
+    dups_given    = set(str(d) for d in payload.get("duplicates", []))
+    mc = {
+        "age_row2":    any("age" in m and "2" in m for m in missing_given),
+        "name_row4":   any("name" in m and "4" in m for m in missing_given),
+        "email_row5":  any("email" in m and "5" in m for m in missing_given),
+        "salary_row6": any("salary" in m and "6" in m for m in missing_given),
+    }
+    dup_ok   = dups_given in [{"1", "3"}, {"3", "1"}] or dups_given == {1, 3}
+    passed   = sum(mc.values())
+    score    = round((passed / 4 * 0.7) + (0.3 if dup_ok else 0.0), 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    feedback = f"{tag} Missing: {passed}/4, Duplicates: {'[OK]' if dup_ok else '[FAIL]'}."
+    return score, feedback, {**mc, "duplicates_correct": dup_ok}
+
+
+def grade_data_medium(payload: dict) -> tuple[float, str, dict]:
+    issues  = {k.lower(): str(v).lower() for k, v in payload.get("issues", {}).items()}
+    cleaned = payload.get("cleaned_data", [])
+    checks  = {
+        "age_type": (
+            "age" in issues and any(w in issues["age"] for w in [
+                "string", "integer", "int", "thirty", "type", "numeric",
+                "non-numeric", "word", "text", "convert", "mixed", "number",
+                "twenty", "str", "should be int", "not int", "invalid",
+            ])
+        ),
+        "salary_format": (
+            "salary" in issues and any(w in issues["salary"] for w in [
+                "format", "comma", "$", "inconsistent", "symbol", "currency",
+                "dollar", "sign", "strip", "remove", "clean",
+            ])
+        ),
+        "date_format": (
+            "join_date" in issues and any(w in issues["join_date"] for w in [
+                "format", "inconsistent", "date", "yyyy", "dd-mm",
+                "standardize", "iso", "different", "multiple",
+            ])
+        ),
+        "name_case": (
+            "name" in issues and any(w in issues["name"] for w in [
+                "capital", "case", "dave", "eve", "lower", "upper",
+                "casing", "title", "inconsistent", "normalize",
+            ])
+        ),
+        "active_type": (
+            "active" in issues and any(w in issues["active"] for w in [
+                "bool", "string", "yes", "true", "type", "mixed",
+                "boolean", "convert", "inconsistent", "str",
+            ])
+        ),
+        "data_cleaned": len(cleaned) == 5,
+    }
+    passed   = sum(checks.values())
+    score    = round(passed / 6, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    feedback = f"{tag} {passed}/6 data quality checks passed."
+    return score, feedback, checks
+
+
+def grade_data_hard(payload: dict, context: dict = None) -> tuple[float, str, dict]:
+    outliers_given = set(int(x) for x in payload.get("outliers", []))
+    missing_given  = set(int(x) for x in payload.get("missing", []))
+    cleaned        = payload.get("cleaned_data", [])
+    answers      = (context or {}).get("answers", DATA_HARD["answers"])
+    correct_out  = answers.get("outliers", DATA_HARD["answers"]["outliers"])
+    correct_miss = answers.get("missing", DATA_HARD["answers"]["missing"])
+    lo, hi       = answers.get("imputed_range", DATA_HARD["answers"]["imputed_range"])
+    outlier_ok     = outliers_given == correct_out
+    missing_ok     = missing_given == correct_miss
+    imputed_rows   = {r["id"]: r.get("value") for r in cleaned if r.get("id") in correct_miss}
+    imputed_ok     = (
+        all(v is not None and lo <= float(v) <= hi for v in imputed_rows.values())
+        and len(imputed_rows) == len(correct_miss)
+    )
+    checks   = {"outliers_correct": outlier_ok, "missing_correct": missing_ok, "imputation_valid": imputed_ok}
+    passed   = sum(checks.values())
+    score    = round(passed / 3, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.67 else "[ERROR]"
+    feedback = f"{tag} {passed}/3 data operations correct."
+    return score, feedback, checks
+
+
+def grade_mod_easy(payload: dict, context: dict = None) -> tuple[float, str, dict]:
+    clf     = {c["id"]: c.get("label", "").lower() for c in payload.get("classifications", [])}
+    posts   = (context or {}).get("posts", MOD_EASY)
+    correct = {p["id"]: p["label"] for p in posts}
+    per     = {id_: clf.get(id_) == label for id_, label in correct.items()}
+    hits    = sum(per.values())
+    score   = round(hits / len(correct), 2)
+    tag     = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.7 else "[ERROR]"
+    return score, f"{tag} {hits}/{len(correct)} posts correctly classified.", per
+
+
+def grade_mod_medium(payload: dict, context: dict = None) -> tuple[float, str, dict]:
+    clf     = {c["id"]: c.get("label", "").lower() for c in payload.get("classifications", [])}
+    posts   = (context or {}).get("posts", MOD_MEDIUM)
+    correct = {p["id"]: p["label"] for p in posts}
+    per     = {id_: clf.get(id_) == label for id_, label in correct.items()}
+    hits    = sum(per.values())
+    score   = round(hits / len(correct), 2)
+    tag     = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.6 else "[ERROR]"
+    return score, f"{tag} {hits}/{len(correct)} subtle posts correctly classified.", per
+
+
+def grade_mod_hard(payload: dict) -> tuple[float, str, dict]:
+    """Now 6 cases (12 checks) — adversarial dog-whistles included."""
+    decisions = {d["id"]: d for d in payload.get("decisions", [])}
+    checks: dict[str, bool] = {}
+    for case in MOD_HARD:
+        cid = case["id"]
+        d   = decisions.get(cid, {})
+        checks[f"{cid}_context_a"] = d.get("context_a_label", "").lower() == case["label_a"]
+        checks[f"{cid}_context_b"] = d.get("context_b_label", "").lower() == case["label_b"]
+    passed   = sum(checks.values())
+    total    = len(checks)
+    score    = round(passed / total, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    return score, f"{tag} {passed}/{total} context-aware decisions correct.", checks
+
+
+def grade_ticket_easy(payload: dict, context: dict) -> tuple[float, str, dict]:
+    ticket   = context["ticket"]
+    p_given  = payload.get("priority", "").strip().lower()
+    c_given  = payload.get("category", "").strip().lower()
+    p_ok     = p_given == ticket["correct_priority"]
+    c_ok     = c_given == ticket["correct_category"]
+    checks   = {"priority_correct": p_ok, "category_correct": c_ok}
+    score    = round(sum(checks.values()) / 2, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score == 0.5 else "[ERROR]"
+    p_str    = "✓" if p_ok else f"✗ (expected {ticket['correct_priority']})"
+    c_str    = "✓" if c_ok else f"✗ (expected {ticket['correct_category']})"
+    feedback = f"{tag} Priority: {p_str}, Category: {c_str}"
+    return score, feedback, checks
+
+
+def grade_ticket_medium(payload: dict) -> tuple[float, str, dict]:
+    order   = payload.get("order", [])
+    assigns = payload.get("assignments", {})
+    correct_order = TICKET_MEDIUM_CORRECT_ORDER
+    correct_teams = TICKET_MEDIUM_CORRECT_TEAMS
+    # Order grading via Kendall tau (60% weight)
+    tau      = _kendall_tau_score(order, correct_order)
+    top3_ok  = order[:3] == correct_order[:3]
+    full_ok  = order == correct_order
+    hits     = sum(1 for i, e in enumerate(order) if i < len(correct_order) and e == correct_order[i])
+    order_score = 1.0 if full_ok else round(tau, 2)
+    # Team assignment grading (40% weight) — per-ticket breakdown
+    team_checks = {tid: assigns.get(tid) == team for tid, team in correct_teams.items()}
+    team_hits   = sum(team_checks.values())
+    team_score  = round(team_hits / len(correct_teams), 2)
+    wrong_teams = [f"{tid}(got={assigns.get(tid)},expected={correct_teams[tid]})"
+                   for tid, ok in team_checks.items() if not ok]
+    score    = round(order_score * 0.6 + team_score * 0.4, 2)
+    checks   = {"top3_correct": top3_ok, "full_order": full_ok,
+                 "team_assignments": team_score == 1.0, **{f"team_{tid}": ok for tid, ok in team_checks.items()}}
+    tag      = "[OK]" if score >= 0.95 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    feedback = f"{tag} Order: {hits}/8 exact, Tau: {tau:.2f}, Teams: {team_hits}/8 correct."
+    if wrong_teams:
+        feedback += f" Wrong teams: {', '.join(wrong_teams)}."
+    return score, feedback, checks
+
+
+def grade_ticket_hard(payload: dict) -> tuple[float, str, dict]:
+    root_cause = payload.get("root_cause", "").lower()
+    resolution = [step.lower() for step in payload.get("resolution_steps", [])]
+    affected   = {s.lower() for s in payload.get("affected_services", [])}
+    severity   = payload.get("severity", "").upper()
+    expected   = TICKET_HARD_INCIDENT["expected"]
+    # Root cause check
+    rc_hits    = sum(1 for kw in expected["root_cause_keywords"] if kw in root_cause)
+    rc_ok      = rc_hits >= 3
+    # Affected services
+    expected_svcs = {s.lower() for s in expected["affected_services"]}
+    svc_hits   = len(affected & expected_svcs)
+    svc_ok     = svc_hits >= 3
+    # Severity
+    sev_ok     = severity == expected["severity"]
+    # Resolution steps
+    res_text   = " ".join(resolution)
+    res_hits   = sum(1 for kw in expected["resolution_keywords"] if kw in res_text)
+    res_ok     = res_hits >= 3 and len(resolution) >= 3
+    checks     = {"root_cause_ok": rc_ok, "affected_services_ok": svc_ok, "severity_ok": sev_ok, "resolution_ok": res_ok}
+    passed     = sum(checks.values())
+    score      = round(passed / 4, 2)
+    tag        = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.5 else "[ERROR]"
+    feedback   = f"{tag} {passed}/4 incident analysis checks passed. RC keywords: {rc_hits}, Svcs: {svc_hits}/4, Res steps: {len(resolution)}."
+    return score, feedback, checks
+
+
+def grade_cross_agent_chain(payload: dict, context: dict) -> tuple[float, str, dict]:
+    case          = context["case"]
+    email_given   = payload.get("email_classification", "").strip().lower()
+    email_ok      = email_given == case["email_label"]
+    bugs          = payload.get("bugs", [])
+    bug_ok        = False
+    for b in bugs:
+        t   = (str(b.get("issue", "")) + str(b.get("location", "")) + str(b.get("fix", ""))).lower()
+        loc = str(b.get("location", "")).lower()
+        if loc == case["code_location"].lower() or any(w in t for w in case["code_bug_keywords"]):
+            bug_ok = True
+            break
+    checks   = {"email_correct": email_ok, "bug_found": bug_ok}
+    passed   = sum(checks.values())
+    score    = round(passed / 2, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score == 0.5 else "[ERROR]"
+    feedback = f"{tag} {passed}/2 cross-agent checks passed."
+    if not email_ok:
+        feedback += f" Email: expected '{case['email_label']}', got '{email_given}'."
+    if not bug_ok:
+        feedback += f" Code bug in '{case['code_location']}' not identified."
+    return score, feedback, checks
+
+
+def grade_cross_agent_email_data(payload: dict, context: dict) -> tuple[float, str, dict]:
+    case        = context["case"]
+    priority    = payload.get("email_priority", "").strip().lower()
+    raw_issues  = payload.get("data_issues", {})
+    cleaned     = payload.get("cleaned_data", [])
+    priority_ok = priority == case["email_priority"].lower()
+    expected_issues = case["expected_issues"]
+
+    # Handle two formats models use:
+    # Format A (correct): {"revenue": "inconsistent...", "region": "casing..."}
+    # Format B (wrong):   {"field": "revenue", "issue": "inconsistent..."}
+    if isinstance(raw_issues, dict):
+        # Check if it's format B — has "field" key instead of field names as keys
+        if "field" in raw_issues and "issue" in raw_issues:
+            # Convert: extract the field name from the "field" value
+            field_val = str(raw_issues.get("field", "")).lower()
+            issue_val = str(raw_issues.get("issue", "")).lower()
+            data_issues = {field_val: issue_val}
+        else:
+            data_issues = {k.lower(): str(v).lower() for k, v in raw_issues.items()}
+    elif isinstance(raw_issues, list):
+        # Format C: [{"field": "revenue", "issue": "..."}]
+        data_issues = {}
+        for item in raw_issues:
+            if isinstance(item, dict):
+                f = str(item.get("field", "")).lower()
+                v = str(item.get("issue", item.get("description", ""))).lower()
+                if f:
+                    data_issues[f] = v
+    else:
+        data_issues = {}
+
+    issue_hits = sum(1 for field in expected_issues if field in data_issues)
+    issues_ok  = issue_hits >= 2
+    cleaned_ok = len(cleaned) == len(case["attachment_data"])
+    checks     = {"priority_correct": priority_ok, "issues_identified": issues_ok, "data_cleaned": cleaned_ok}
+    passed     = sum(checks.values())
+    score      = round(passed / 3, 2)
+    tag        = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.67 else "[ERROR]"
+    feedback   = f"{tag} {passed}/3 cross-agent email+data checks passed."
+    if not issues_ok:
+        feedback += f" Issues found: {issue_hits}/{len(expected_issues)} fields identified."
+    return score, feedback, checks
+
+
+def grade_cross_agent_code_email(payload: dict, context: dict) -> tuple[float, str, dict]:
+    case     = context["case"]
+    vuln_type = payload.get("vulnerability_type", "").lower()
+    location  = payload.get("vulnerability_location", "").lower()
+    email     = payload.get("disclosure_email", "").lower()
+    type_ok   = case["primary_vulnerability"] in vuln_type or "sql" in vuln_type
+    loc_ok    = case["location"].lower() in location
+    email_hits = sum(1 for kw in case["disclosure_keywords"] if kw in email)
+    email_ok   = email_hits >= 4 and len(email) >= 80
+    checks     = {"vulnerability_identified": type_ok, "location_correct": loc_ok, "disclosure_email_quality": email_ok}
+    passed     = sum(checks.values())
+    score      = round(passed / 3, 2)
+    tag        = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.67 else "[ERROR]"
+    feedback   = f"{tag} {passed}/3 code+email checks passed. Email keywords: {email_hits}."
+    return score, feedback, checks
+
+
+def grade_cross_agent_mod_escalation(payload: dict, context: dict) -> tuple[float, str, dict]:
+    case          = context["case"]
+    label_given   = payload.get("content_label", "").strip().lower()
+    escalate      = payload.get("escalate", None)
+    notice        = payload.get("moderation_notice", "").lower()
+    label_ok      = label_given == case["correct_label"]
+    escalate_ok   = escalate == case["should_escalate"]
+    if case["should_escalate"]:
+        notice_hits = sum(1 for kw in case["notice_keywords"] if kw in notice)
+        notice_ok   = notice_hits >= 3
+    else:
+        notice_ok   = len(notice) == 0 or notice.strip() == ""
+    checks   = {"label_correct": label_ok, "escalation_correct": escalate_ok, "notice_quality": notice_ok}
+    passed   = sum(checks.values())
+    score    = round(passed / 3, 2)
+    tag      = "[OK]" if score == 1.0 else "[PARTIAL]" if score >= 0.67 else "[ERROR]"
+    feedback = f"{tag} {passed}/3 moderation+escalation checks passed."
+    return score, feedback, checks
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# INSTRUCTIONS
+# ═════════════════════════════════════════════════════════════════════════════
+
+INSTRUCTIONS: dict[str, str] = {
+    "email_triage_easy": (
+        "Classify the email as exactly one of: 'spam', 'important', or 'newsletter'.\n"
+        'Payload: {"classification": "spam"}'
+    ),
+    "email_triage_medium": (
+        "Prioritize the 10 emails from most urgent (first) to least urgent (last).\n"
+        "Priority tiers — assign each email to a tier first, then order within tiers by deadline/impact:\n"
+        "  Tier 1 (CRITICAL): Active production outages, active security breaches — immediate revenue/safety impact\n"
+        "  Tier 2 (URGENT): Legal agreements or contracts expiring TODAY — deal falls through if unsigned\n"
+        "  Tier 3 (HIGH): Overdue financial obligations threatening service suspension\n"
+        "  Tier 4 (HIGH): Business deliverables with hard deadlines this week\n"
+        "  Tier 5 (LOW): Social — lunch invites, work anniversaries\n"
+        "  Tier 6 (IGNORE): Passive newsletters and promotional content\n"
+        "Within Tier 2: 'TODAY' deadlines rank above 'Friday' deadlines.\n"
+        "Within Tier 4: launch review decks rank above other reviews by proximity to launch date.\n"
+        "Return ALL 10 email IDs ordered most-to-least urgent.\n"
+        'Payload: {"order": ["m1", "m5", ...]}'
+    ),
+    "email_triage_hard": (
+        "Draft a professional, empathetic reply to the customer complaint. Address ALL concerns.\n"
+        'Payload: {"reply": "<full reply text>"}'
+    ),
+    "code_review_easy": (
+        "Identify the syntax error(s) in the Python code.\n"
+        'Payload: {"errors": ["description of syntax error"]}'
+    ),
+    "code_review_medium": (
+        "Find ALL 4 logical bugs — one per function. Report location, issue, and fix.\n"
+        "  1. find_max — wrong initialization (fails all-negative lists)\n"
+        "  2. get_user — = used instead of ==\n"
+        "  3. divide — no zero-division guard\n"
+        "  4. process_list — off-by-one in range\n"
+        'Payload: {"bugs": [{"location": "fn_name", "issue": "...", "fix": "..."}]}'
+    ),
+    "code_review_hard": (
+        "Identify ALL 8 security vulnerabilities:\n"
+        "  1. sql_injection in get_user_by_name\n"
+        "  2. xss in render_comment\n"
+        "  3. sql_injection in login\n"
+        "  4. command_injection in run_report\n"
+        "  5. insecure_deserialization in load_user_data\n"
+        "  6. timing_attack in check_password\n"
+        "  7. path_traversal in read_report_file\n"
+        "  8. ssrf in fetch_user_avatar\n"
+        'Payload: {"vulnerabilities": [{"type": "...", "location": "...", "severity": "...", "fix": "..."}]}'
+    ),
+    "data_cleaning_easy": (
+        "Find all missing (null) values by field+row, and identify duplicate row IDs.\n"
+        'Payload: {"missing": ["age (row 2)", "name (row 4)"], "duplicates": [1, 3]}'
+    ),
+    "data_cleaning_medium": (
+        "Identify all 5 data quality issues and return the fully cleaned dataset (5 rows).\n"
+        'Payload: {"issues": {"age": "mixed types..."}, "cleaned_data": [...]}'
+    ),
+    "data_cleaning_hard": (
+        "Detect outliers (IQR/z-score), find missing values, impute, return cleaned dataset.\n"
+        'Payload: {"outliers": [4, 8], "missing": [6, 11], "cleaned_data": [...]}'
+    ),
+    "content_moderation_easy": (
+        "Classify each post as 'safe' or 'harmful'.\n"
+        'Payload: {"classifications": [{"id": "c1", "label": "harmful"}]}'
+    ),
+    "content_moderation_medium": (
+        "Classify each post as 'safe' or 'harmful'. Watch for sarcasm, subtle toxicity, implicit hostility.\n"
+        "IMPORTANT DISTINCTION — not all unusual phrasing is harmful:\n"
+        "  HARMFUL: sarcasm that mocks or demeans a person ('Classic genius move', 'brilliant idea')\n"
+        "  HARMFUL: passive aggression that excludes or belittles ('not cut out for this', 'shouldn't have opinions')\n"
+        "  SAFE: mild, ambiguous comments that express a personal reaction without attacking anyone\n"
+        "  SAFE: backhanded compliments about THINGS (not people) — 'very unique' about a presentation is\n"
+        "         just a mild opinion, not an attack on a person. No target = not harmful.\n"
+        "  SAFE: genuine positivity ('Congrats!', 'Thanks for the feedback')\n"
+        'Payload: {"classifications": [{"id": "m1", "label": "harmful", "reason": "..."}]}'
+    ),
+    "content_moderation_hard": (
+        "Each case shows the SAME text in TWO different contexts. Determine the correct label for EACH context.\n"
+        "CRITICAL RULES:\n"
+        "  1. Some texts are ALWAYS harmful regardless of context — they use dehumanizing language or\n"
+        "     dog-whistle rhetoric that is harmful no matter what platform or prior message surrounds them.\n"
+        "     TEST: Would a reasonable content moderator flag this text if they saw it with NO context?\n"
+        "     If yes, label it 'harmful' in BOTH contexts.\n"
+        "  2. KEY SIGNAL — comparing humans to animals ('breed like rabbits'), invoking 'replacement' of\n"
+        "     ethnic groups, or using language that strips human dignity is ALWAYS harmful. A 'biology\n"
+        "     classroom' or 'urban planning' framing does NOT make dehumanizing rhetoric acceptable.\n"
+        "  3. Other texts ARE context-dependent — 'disappear' in gaming vs crisis support, 'take out' in\n"
+        "     HR dinner vs extremist forum, 'eliminate the competition' in business vs political violence.\n"
+        "     For these, label each context based on the actual meaning in that specific context.\n"
+        "  4. When in doubt about a text: ask 'is this language that treats a group of people as less than\n"
+        "     human?' If yes → harmful in all contexts. If it's just ambiguous phrasing → context matters.\n"
+        'Payload: {"decisions": [{"id": "h1", "context_a_label": "safe", "context_b_label": "harmful"}]}'
+    ),
+    "ticket_triage_easy": (
+        "Classify the support ticket: assign a priority (critical/high/medium/low) and category (bug/feature_request/question/billing).\n"
+        'Payload: {"priority": "critical", "category": "bug"}'
+    ),
+    "ticket_triage_medium": (
+        "Order 8 tickets by priority (most urgent first) and assign each to the correct team.\n"
+        "Priority reasoning:\n"
+        "  1. Production DB down and SSL expiring in 48h are both critical infrastructure\n"
+        "  2. Currency display bug (affects all EU users visibly) ranks above billing revenue leak\n"
+        "  3. Revenue-leak billing bug ranks above invoice PDF generation bug\n"
+        "  4. Feature requests rank below all bugs\n"
+        "Team routing — be precise about these common mistakes:\n"
+        "  backend  = server-side APIs, DB queries, background job processing\n"
+        "  frontend = anything the user SEES in the browser: currency symbols, UI, display bugs, CSV export UI\n"
+        "  devops   = SSL certificates, deployments, infrastructure\n"
+        "  billing  = payment charges, subscription billing, invoice amounts, refunds\n"
+        "  'Wrong currency symbol shown' → frontend (it's a display/rendering bug)\n"
+        "  'Customer billed $0 instead of $299' → billing (it's a charge calculation bug)\n"
+        "  'Invoice PDF generation fails' → backend (server-side PDF generation)\n"
+        "  'Add OAuth2 SSO support' → backend (auth server logic)\n"
+        "  'Redesign onboarding flow' → frontend (UI/UX work)\n"
+        "  'Add CSV export for analytics' → frontend (UI feature)\n"
+        'Payload: {"order": ["tk1", "tk4", "tk2", "tk7", "tk5", "tk3", "tk6", "tk8"], '
+        '"assignments": {"tk1": "backend", "tk2": "frontend", "tk3": "backend", '
+        '"tk4": "devops", "tk5": "backend", "tk6": "frontend", "tk7": "billing", "tk8": "frontend"}}'
+    ),
+    "ticket_triage_hard": (
+        "Analyse the linked incident tickets and produce: root cause analysis, ordered resolution steps, affected services, and severity (P1-P4).\n"
+        'Payload: {"root_cause": "...", "resolution_steps": ["...", "..."], "affected_services": ["...", "..."], "severity": "P1"}'
+    ),
+    "cross_agent_chain": (
+        "TWO skills needed:\n"
+        "  1. Classify the email: 'spam', 'important', or 'newsletter'\n"
+        "  2. Find the main bug in the attached code (location + issue + fix)\n"
+        'Payload: {"email_classification": "important", "bugs": [{"location": "fn", "issue": "...", "fix": "..."}]}'
+    ),
+    "cross_agent_email_data": (
+        "TWO skills needed:\n"
+        "  1. Classify the email urgency: 'critical', 'high', 'medium', 'low'\n"
+        "  2. Identify data quality issues in the attachment and return cleaned data\n"
+        "IMPORTANT: data_issues must use field names as keys, NOT 'field'/'issue' as keys.\n"
+        "  CORRECT:   {\"revenue\": \"inconsistent formats\", \"region\": \"inconsistent casing\", \"closed_date\": \"multiple date formats\"}\n"
+        "  INCORRECT: {\"field\": \"revenue\", \"issue\": \"inconsistent formats\"}\n"
+        'Payload: {"email_priority": "critical", "data_issues": {"revenue": "inconsistent formats", "region": "casing issues", "closed_date": "multiple formats"}, "cleaned_data": [...]}'
+    ),
+    "cross_agent_code_email": (
+        "TWO skills needed:\n"
+        "  1. Identify the primary security vulnerability type and its location\n"
+        "  2. Draft a professional security disclosure email to the dev team (min 80 chars)\n"
+        'Payload: {"vulnerability_type": "sql_injection", "vulnerability_location": "fn_name", "disclosure_email": "..."}'
+    ),
+    "cross_agent_mod_escalation": (
+        "TWO skills needed:\n"
+        "  1. Classify the content as 'safe' or 'harmful'\n"
+        "  2. Decide whether to escalate (true/false) and draft a moderation notice if harmful (empty string if safe)\n"
+        'Payload: {"content_label": "harmful", "escalate": true, "moderation_notice": "Your post was removed because..."}'
+    ),
+}
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# ENVIRONMENT
+# ═════════════════════════════════════════════════════════════════════════════
+
+class MetaEnvironment(Environment):
+    """
+    Meta Multi-Agent Environment v3.0
+    5 domains · 16 tasks · 4 cross-agent chained tasks · deterministic context loading
+    """
+    SUPPORTS_CONCURRENT_SESSIONS: bool = True
+
+    def __init__(self) -> None:
+        self._state                  = State(episode_id=str(uuid4()), step_count=0)
+        self._current_task_id: str | None = None
+        self._current_agent_context: dict  = {}
+        self._current_grader_context: dict = {}
+        self._episode_scores: list[float]  = []
+        self._attempt_counts: dict[str, int]   = {}
+        self._first_attempt_scores: dict[str, float] = {}
+
+    def reset(self) -> MetaObservation:
+        self._state                  = State(episode_id=str(uuid4()), step_count=0)
+        self._current_task_id        = None
+        self._current_agent_context  = {}
+        self._current_grader_context = {}
+        self._episode_scores         = []
+        self._attempt_counts         = {}
+        self._first_attempt_scores   = {}
+        return MetaObservation(
+            agent="meta",
+            task_id="",
+            difficulty="",
+            context={},
+            instructions=(
+                "Welcome to Meta Multi-Agent Environment v3.0!\n"
+                "5 agents × 3 tasks + 4 cross-agent tasks = 19 tasks total.\n"
+                "Agents: email_triage | code_review | data_cleaning | "
+                "content_moderation | ticket_triage | cross_agent\n"
+                "Use GET /tasks to see all task IDs and exact payload schemas.\n"
+                "Multi-step episodes: imperfect first attempts earn a retry."
+            ),
+            feedback="Environment reset. Ready for episode.",
+            score=0.0,
+            partial_credits={},
+            done=False,
+            reward=0.0,
+        )
+
+    def _deterministic_choice(self, variants: list, task_id: str) -> Any:
+        """Pick a variant deterministically based on episode_id hash."""
+        idx = hash(self._state.episode_id + task_id) % len(variants)
+        return variants[idx]
+
+    def _load_context(self, task_id: str) -> tuple[dict, dict, str]:
+        """Return (agent_ctx, grader_ctx, difficulty). Fully deterministic."""
+        if task_id == "email_triage_easy":
+            agent_ctx, grader_ctx = gen_email_easy(self._state.episode_id)
+            return agent_ctx, grader_ctx, "easy"
+
+        if task_id == "email_triage_medium":
+            agent_ctx, grader_ctx = gen_email_medium(self._state.episode_id)
+            return agent_ctx, grader_ctx, "medium"
+
+        if task_id == "email_triage_hard":
+            agent_ctx, grader_ctx = gen_email_hard(self._state.episode_id)
+            return agent_ctx, grader_ctx, "hard"
+
+        if task_id == "code_review_easy":
+            agent_ctx, grader_ctx = gen_code_easy(self._state.episode_id)
+            return agent_ctx, grader_ctx, "easy"
+
+        if task_id == "code_review_medium":
+            ctx = {"code": CODE_MEDIUM["code"]}
+            return ctx, ctx, "medium"
+
+        if task_id == "code_review_hard":
+            ctx = {"code": CODE_HARD["code"]}
+            return ctx, ctx, "hard"
+
+        if task_id == "data_cleaning_easy":
+            agent_ctx, grader_ctx = gen_data_easy(self._state.episode_id)
+            return agent_ctx, grader_ctx, "easy"
+
+        if task_id == "data_cleaning_medium":
+            agent_ctx, grader_ctx = gen_data_medium(self._state.episode_id)
+            return agent_ctx, grader_ctx, "medium"
+
+        if task_id == "data_cleaning_hard":
+            agent_ctx, grader_ctx = gen_data_hard(self._state.episode_id)
+            return agent_ctx, grader_ctx, "hard"
+
+        if task_id == "content_moderation_easy":
+            agent_ctx, grader_ctx = gen_mod_easy(self._state.episode_id)
+            return agent_ctx, grader_ctx, "easy"
+
+        if task_id == "content_moderation_medium":
+            agent_ctx, grader_ctx = gen_mod_medium(self._state.episode_id)
+            return agent_ctx, grader_ctx, "medium"
+
+        if task_id == "content_moderation_hard":
+            agent_ctx  = {"cases": [{k: v for k, v in c.items() if k not in ("label_a", "label_b", "reason_a", "reason_b")} for c in MOD_HARD]}
+            grader_ctx = {"cases": MOD_HARD}
+            return agent_ctx, grader_ctx, "hard"
+
+        if task_id == "ticket_triage_easy":
+            agent_ctx, grader_ctx = gen_ticket_easy(self._state.episode_id)
+            return agent_ctx, grader_ctx, "easy"
+
+        if task_id == "ticket_triage_medium":
+            agent_ctx  = {"tickets": [{k: v for k, v in t.items() if k not in ("priority_rank", "team")} for t in TICKET_MEDIUM_TICKETS]}
+            grader_ctx = {"tickets": TICKET_MEDIUM_TICKETS}
+            return agent_ctx, grader_ctx, "medium"
+
+        if task_id == "ticket_triage_hard":
+            ctx = {"incident_tickets": TICKET_HARD_INCIDENT["tickets"]}
+            return ctx, ctx, "hard"
+
+        if task_id == "cross_agent_chain":
+            case       = self._deterministic_choice(CROSS_AGENT_CHAIN_CASES, task_id)
+            agent_ctx  = {"email": case["email"], "code": case["code"]}
+            grader_ctx = {"case": case}
+            return agent_ctx, grader_ctx, "hard"
+
+        if task_id == "cross_agent_email_data":
+            case       = CROSS_AGENT_EMAIL_DATA_CASES[0]
+            agent_ctx  = {"email": case["email"], "attachment_data": case["attachment_data"]}
+            grader_ctx = {"case": case}
+            return agent_ctx, grader_ctx, "hard"
+
+        if task_id == "cross_agent_code_email":
+            case       = CROSS_AGENT_CODE_EMAIL_CASES[0]
+            agent_ctx  = {"code": case["code"]}
+            grader_ctx = {"case": case}
+            return agent_ctx, grader_ctx, "hard"
+
+        if task_id == "cross_agent_mod_escalation":
+            case       = self._deterministic_choice(CROSS_AGENT_MOD_ESCALATION_CASES, task_id)
+            agent_ctx  = {"post": case["post"]}
+            grader_ctx = {"case": case}
+            return agent_ctx, grader_ctx, "hard"
+
+        
+        if task_id == "email_triage_expert":
+            agent_ctx  = {k: v for k, v in EMAIL_EXPERT_CONTEXT.items()}
+            grader_ctx = agent_ctx
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "code_review_expert":
+            agent_ctx  = {"code": CODE_EXPERT_CONTEXT["code"],
+                          "vulnerability_count": CODE_EXPERT_CONTEXT["vulnerability_count"]}
+            grader_ctx = CODE_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "data_cleaning_expert":
+            agent_ctx  = DATA_EXPERT_CONTEXT
+            grader_ctx = DATA_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "content_moderation_expert":
+            agent_ctx  = MOD_EXPERT_CONTEXT
+            grader_ctx = MOD_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        if task_id == "ticket_triage_expert":
+            agent_ctx  = TICKET_EXPERT_CONTEXT
+            grader_ctx = TICKET_EXPERT_CONTEXT
+            return agent_ctx, grader_ctx, "expert"
+
+        raise ValueError(f"Unknown task_id: '{task_id}'. Use GET /tasks to see valid IDs.")
+
+    def _grade(self, task_id: str, payload: dict, grader_context: dict) -> tuple[float, str, dict]:
+        if task_id == "email_triage_easy":           return grade_email_easy(payload, grader_context)
+        if task_id == "email_triage_medium":         return grade_email_medium(payload, grader_context)
+        if task_id == "email_triage_hard":           return grade_email_hard(payload, grader_context)
+        if task_id == "code_review_easy":            return grade_code_easy(payload, grader_context)
+        if task_id == "code_review_medium":          return grade_code_medium(payload)
+        if task_id == "code_review_hard":            return grade_code_hard(payload)
+        if task_id == "data_cleaning_easy":          return grade_data_easy(payload)
+        if task_id == "data_cleaning_medium":        return grade_data_medium(payload)
+        if task_id == "data_cleaning_hard":          return grade_data_hard(payload, grader_context)
+        if task_id == "content_moderation_easy":     return grade_mod_easy(payload, grader_context)
+        if task_id == "content_moderation_medium":   return grade_mod_medium(payload, grader_context)
+        if task_id == "content_moderation_hard":     return grade_mod_hard(payload)
+        if task_id == "ticket_triage_easy":          return grade_ticket_easy(payload, grader_context)
+        if task_id == "ticket_triage_medium":        return grade_ticket_medium(payload)
+        if task_id == "ticket_triage_hard":          return grade_ticket_hard(payload)
+        if task_id == "cross_agent_chain":           return grade_cross_agent_chain(payload, grader_context)
+        if task_id == "cross_agent_email_data":      return grade_cross_agent_email_data(payload, grader_context)
+        if task_id == "cross_agent_code_email":      return grade_cross_agent_code_email(payload, grader_context)
+        if task_id == "cross_agent_mod_escalation":  return grade_cross_agent_mod_escalation(payload, grader_context)
+        return 0.0, "Unknown task.", {}
+
+    def _compute_reward(self, score: float, task_id: str, attempt: int) -> float:
+        """Trajectory-aware reward shaping."""
+        base = score
+        # Speed bonus: perfect on first attempt
+        if score == 1.0 and attempt == 1:
+            base = min(base + 0.05, 1.0)
+        # Improvement bonus: reward getting better on retry
+        if attempt == 2:
+            prev = self._first_attempt_scores.get(task_id, 0.0)
+            if score > prev:
+                base = min(base + (score - prev) * 0.1, 1.0)
+        # Penalty: repeated failure
+        if score == 0.0 and attempt > 1:
+            base = max(base - 0.05, 0.0)
+        return round(base, 3)
+
+    def step(self, action: MetaAction) -> MetaObservation:
+        self._state.step_count += 1
+
+        try:
+            data    = json.loads(action.message)
+            agent   = data.get("agent", "")
+            task_id = data.get("task_id", "")
+            payload = data.get("payload", {})
+        except Exception as e:
+            return MetaObservation(
+                agent="error", task_id="", difficulty="",
+                context={},
+                instructions='message must be valid JSON: {"agent":"...","task_id":"...","payload":{...}}',
+                feedback=f"JSON parse error: {e}",
+                score=0.0, partial_credits={}, done=True, reward=0.0,
+            )
+
+        # Probe — load context without grading
+        if not payload or payload == {"_probe": True}:
+            try:
+                agent_ctx, grader_ctx, difficulty = self._load_context(task_id)
+                self._current_agent_context  = agent_ctx
+                self._current_grader_context = grader_ctx
+                self._current_task_id        = task_id
+            except ValueError:
+                difficulty = ""
+                agent_ctx  = {}
+            is_probe = payload == {"_probe": True}
+            return MetaObservation(
+                agent=agent, task_id=task_id, difficulty=difficulty,
+                context=agent_ctx,
+                instructions=INSTRUCTIONS.get(task_id, "Use GET /tasks for payload schema."),
+                feedback="Probe: context loaded. Submit a real payload to get scored." if is_probe else "Empty payload. Use GET /tasks to see required fields.",
+                score=0.0, partial_credits={}, done=is_probe,
+                reward=-0.1 if not is_probe else 0.0,
+            )
+
+        # Load context
+        try:
+            if task_id != self._current_task_id:
+                agent_ctx, grader_ctx, difficulty = self._load_context(task_id)
+                self._current_agent_context  = agent_ctx
+                self._current_grader_context = grader_ctx
+                self._current_task_id        = task_id
+                self._attempt_counts[task_id] = 0
+            else:
+                agent_ctx  = self._current_agent_context
+                grader_ctx = self._current_grader_context
+                _, __, difficulty = self._load_context(task_id)
+        except ValueError as e:
+            return MetaObservation(
+                agent=agent, task_id=task_id, difficulty="",
+                context={},
+                instructions="Use GET /tasks to see valid task IDs.",
+                feedback=str(e),
+                score=0.0, partial_credits={}, done=True, reward=0.0,
+            )
+
+        # Grade
+        score, feedback, partial_credits = self._grade(task_id, payload, grader_ctx)
+
+        # Multi-step retry logic
+        attempt = self._attempt_counts.get(task_id, 0) + 1
+        self._attempt_counts[task_id] = attempt
+
+        reward = self._compute_reward(score, task_id, attempt)
+
+        if attempt == 1 and score < 1.0:
+            self._first_attempt_scores[task_id] = score
+            done     = False
+            reward   = score * 0.5
+            missed   = [k for k, v in partial_credits.items() if not v]
+            feedback += (
+                f" [RETRY AVAILABLE] Score: {score:.2f}. "
+                f"Improve on: {missed}"
+            )
+        elif attempt == 1 and score == 1.0:
+            done     = True
+            self._episode_scores.append(score)
+        else:
+            first  = self._first_attempt_scores.get(task_id, 0.0)
+            score  = max(score, first)
+            done   = True
+            reward = self._compute_reward(score, task_id, attempt)
+            self._episode_scores.append(score)
+            feedback += f" [FINAL] Best score from 2 attempts: {score:.2f}."
+
+        return MetaObservation(
+            agent=agent,
+            task_id=task_id,
+            difficulty=difficulty,
+            context=agent_ctx,
+            instructions=INSTRUCTIONS.get(task_id, ""),
+            feedback=feedback,
+            score=score,
+            partial_credits=partial_credits,
+            done=done,
+            reward=reward,
+            metadata={
+                "step":              self._state.step_count,
+                "attempt":           attempt,
+                "episode_avg_score": round(
+                    sum(self._episode_scores) / len(self._episode_scores), 3
+                ) if self._episode_scores else 0.0,
+            },
+        )
+
+    @property
+    def state(self) -> State:
+        return self._state

server/app.py

@@ -29,7 +29,16 @@ from fastapi.responses import JSONResponse, RedirectResponse
 
 from openenv.core.env_server.http_server import create_app
 
-from models import MetaAction, MetaObservation
+from models import (
+    MetaAction, MetaObservation,
+    EmailTriageEasyPayload, EmailTriageMediumPayload, EmailTriageHardPayload,
+    CodeReviewEasyPayload, CodeReviewMediumPayload, CodeReviewHardPayload,
+    DataCleaningEasyPayload, DataCleaningMediumPayload, DataCleaningHardPayload,
+    ContentModerationEasyPayload, ContentModerationMediumPayload, ContentModerationHardPayload,
+    TicketTriageEasyPayload, TicketTriageMediumPayload, TicketTriageHardPayload,
+    CrossAgentChainPayload, CrossAgentEmailDataPayload,
+    CrossAgentCodeEmailPayload, CrossAgentModEscalationPayload,
+)
 from server.Meta_environment import MetaEnvironment
 
 # ── Create base OpenEnv app ───────────────────────────────────────────────────
@@ -283,6 +292,64 @@ async def baseline():
         return JSONResponse(status_code=500, content={"error": str(e)})
 
 
+
+# ── Typed step endpoint — Fix 3 ───────────────────────────────────────────────
+
+_TASK_TO_MODEL = {
+    "email_triage_easy":          EmailTriageEasyPayload,
+    "email_triage_medium":        EmailTriageMediumPayload,
+    "email_triage_hard":          EmailTriageHardPayload,
+    "code_review_easy":           CodeReviewEasyPayload,
+    "code_review_medium":         CodeReviewMediumPayload,
+    "code_review_hard":           CodeReviewHardPayload,
+    "data_cleaning_easy":         DataCleaningEasyPayload,
+    "data_cleaning_medium":       DataCleaningMediumPayload,
+    "data_cleaning_hard":         DataCleaningHardPayload,
+    "content_moderation_easy":    ContentModerationEasyPayload,
+    "content_moderation_medium":  ContentModerationMediumPayload,
+    "content_moderation_hard":    ContentModerationHardPayload,
+    "ticket_triage_easy":         TicketTriageEasyPayload,
+    "ticket_triage_medium":       TicketTriageMediumPayload,
+    "ticket_triage_hard":         TicketTriageHardPayload,
+    "cross_agent_chain":          CrossAgentChainPayload,
+    "cross_agent_email_data":     CrossAgentEmailDataPayload,
+    "cross_agent_code_email":     CrossAgentCodeEmailPayload,
+    "cross_agent_mod_escalation": CrossAgentModEscalationPayload,
+}
+
+
+@app.post("/step/typed/{task_id}")
+async def step_typed(task_id: str, request: Request):
+    """
+    Typed step endpoint — accepts the typed payload model for the given task_id
+    instead of the JSON-string-in-message format.
+
+    Example:
+        POST /step/typed/email_triage_easy
+        Body: {"agent": "email_triage", "task_id": "email_triage_easy", "payload": {"classification": "spam"}}
+
+    Returns the same observation as POST /step.
+    """
+    if task_id not in _TASK_TO_MODEL:
+        return JSONResponse(status_code=404, content={"error": f"Unknown task_id: {task_id}. See GET /tasks."})
+    body = await request.json()
+    try:
+        model_cls = _TASK_TO_MODEL[task_id]
+        payload   = body.get("payload", body)
+        typed     = model_cls(**payload)
+    except Exception as e:
+        return JSONResponse(status_code=422, content={"error": str(e), "task_id": task_id})
+    agent   = body.get("agent", "_".join(task_id.split("_")[:-1]))
+    wrapped = MetaAction(message=json.dumps({
+        "agent": agent, "task_id": task_id, "payload": typed.model_dump()
+    }))
+    # Delegate to the active environment instance
+    env = MetaEnvironment()
+    env.reset()
+    obs = env.step(wrapped)
+    return obs.model_dump()
+
+
 def main(host: str = "0.0.0.0", port: int = 7860) -> None:
     import uvicorn
     uvicorn.run(app, host=host, port=port)

server/app.py.bak

@@ -0,0 +1,363 @@
+"""
+FastAPI application for Meta Multi-Agent Environment v3.0
+
+Standard OpenEnv endpoints (from create_app):
+    POST /reset    — Reset environment
+    POST /step     — Execute action
+    GET  /state    — Current state
+    GET  /schema   — Action/observation schemas
+    GET  /health   — Health check
+    GET  /metadata — Environment metadata
+    WS   /ws       — WebSocket persistent session
+
+Custom Meta endpoints:
+    GET  /tasks    — All 19 tasks with payload schemas
+    POST /grader   — Score an action without side effects
+    POST /baseline — Run baseline across all tasks
+    GET  /ui       — Gradio interactive frontend
+    GET  /         — Redirects to /ui
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+from fastapi import Request
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from openenv.core.env_server.http_server import create_app
+
+from models import (
+    MetaAction, MetaObservation,
+    EmailTriageEasyPayload, EmailTriageMediumPayload, EmailTriageHardPayload,
+    CodeReviewEasyPayload, CodeReviewMediumPayload, CodeReviewHardPayload,
+    DataCleaningEasyPayload, DataCleaningMediumPayload, DataCleaningHardPayload,
+    ContentModerationEasyPayload, ContentModerationMediumPayload, ContentModerationHardPayload,
+    TicketTriageEasyPayload, TicketTriageMediumPayload, TicketTriageHardPayload,
+    CrossAgentChainPayload, CrossAgentEmailDataPayload,
+    CrossAgentCodeEmailPayload, CrossAgentModEscalationPayload,
+)
+from server.Meta_environment import MetaEnvironment
+
+# ── Create base OpenEnv app ───────────────────────────────────────────────────
+app = create_app(
+    MetaEnvironment,
+    MetaAction,
+    MetaObservation,
+    env_name="Meta",
+    max_concurrent_envs=20,
+)
+
+# ── Mount Gradio UI ───────────────────────────────────────────────────────────
+try:
+    from server.gradio_ui import mount_gradio
+    mount_gradio(app)
+except ImportError as e:
+    print(f"[WARN] Gradio UI not mounted: {e}")
+
+
+# ── Root redirect ─────────────────────────────────────────────────────────────
+@app.get("/")
+def root():
+    return RedirectResponse(url="/ui")
+
+
+# ── Task Registry ─────────────────────────────────────────────────────────────
+TASK_REGISTRY = [
+    # Email Triage
+    {
+        "id": "email_triage_easy",
+        "name": "Email Classification",
+        "agent": "email_triage",
+        "difficulty": "easy",
+        "description": "Classify a single email as spam, important, or newsletter.",
+        "action_schema": {"message": json.dumps({"agent": "email_triage", "task_id": "email_triage_easy", "payload": {"classification": "spam | important | newsletter"}})},
+    },
+    {
+        "id": "email_triage_medium",
+        "name": "Email Prioritization",
+        "agent": "email_triage",
+        "difficulty": "medium",
+        "description": "Prioritize 10 workplace emails from most to least urgent.",
+        "action_schema": {"message": json.dumps({"agent": "email_triage", "task_id": "email_triage_medium", "payload": {"order": ["m1", "m5", "m3", "m10", "m6", "m8", "m2", "m9", "m4", "m7"]}})},
+    },
+    {
+        "id": "email_triage_hard",
+        "name": "Email Reply Drafting",
+        "agent": "email_triage",
+        "difficulty": "hard",
+        "description": "Draft a professional, empathetic reply to a complex customer complaint.",
+        "action_schema": {"message": json.dumps({"agent": "email_triage", "task_id": "email_triage_hard", "payload": {"reply": "<full reply text>"}})},
+    },
+    # Code Review
+    {
+        "id": "code_review_easy",
+        "name": "Syntax Error Detection",
+        "agent": "code_review",
+        "difficulty": "easy",
+        "description": "Find the syntax error(s) in a Python code snippet.",
+        "action_schema": {"message": json.dumps({"agent": "code_review", "task_id": "code_review_easy", "payload": {"errors": ["description of syntax error"]}})},
+    },
+    {
+        "id": "code_review_medium",
+        "name": "Logic Bug Detection",
+        "agent": "code_review",
+        "difficulty": "medium",
+        "description": "Find all 4 logical bugs in the code and suggest fixes.",
+        "action_schema": {"message": json.dumps({"agent": "code_review", "task_id": "code_review_medium", "payload": {"bugs": [{"location": "fn_name", "issue": "...", "fix": "..."}]}})},
+    },
+    {
+        "id": "code_review_hard",
+        "name": "Security Vulnerability Detection (8 vulns)",
+        "agent": "code_review",
+        "difficulty": "hard",
+        "description": "Identify all 8 security vulnerabilities: SQL injection (x2), XSS, command injection, insecure deserialization, timing attack, path traversal, SSRF.",
+        "action_schema": {"message": json.dumps({"agent": "code_review", "task_id": "code_review_hard", "payload": {"vulnerabilities": [{"type": "sql_injection", "location": "fn_name", "severity": "critical", "fix": "..."}]}})},
+    },
+    # Data Cleaning
+    {
+        "id": "data_cleaning_easy",
+        "name": "Missing Value & Duplicate Detection",
+        "agent": "data_cleaning",
+        "difficulty": "easy",
+        "description": "Find all missing values (by field+row) and identify duplicate rows.",
+        "action_schema": {"message": json.dumps({"agent": "data_cleaning", "task_id": "data_cleaning_easy", "payload": {"missing": ["age (row 2)"], "duplicates": [1, 3]}})},
+    },
+    {
+        "id": "data_cleaning_medium",
+        "name": "Data Type Normalization",
+        "agent": "data_cleaning",
+        "difficulty": "medium",
+        "description": "Identify 5 data quality issues and return cleaned dataset.",
+        "action_schema": {"message": json.dumps({"agent": "data_cleaning", "task_id": "data_cleaning_medium", "payload": {"issues": {"field_name": "description"}, "cleaned_data": []}})},
+    },
+    {
+        "id": "data_cleaning_hard",
+        "name": "Outlier Detection & Imputation",
+        "agent": "data_cleaning",
+        "difficulty": "hard",
+        "description": "Detect outliers, find missing values, impute and return clean dataset.",
+        "action_schema": {"message": json.dumps({"agent": "data_cleaning", "task_id": "data_cleaning_hard", "payload": {"outliers": [4, 8], "missing": [6, 11], "cleaned_data": []}})},
+    },
+    # Content Moderation
+    {
+        "id": "content_moderation_easy",
+        "name": "Explicit Content Detection",
+        "agent": "content_moderation",
+        "difficulty": "easy",
+        "description": "Classify 7 posts as safe or harmful.",
+        "action_schema": {"message": json.dumps({"agent": "content_moderation", "task_id": "content_moderation_easy", "payload": {"classifications": [{"id": "c1", "label": "harmful"}]}})},
+    },
+    {
+        "id": "content_moderation_medium",
+        "name": "Subtle Toxicity Detection",
+        "agent": "content_moderation",
+        "difficulty": "medium",
+        "description": "Detect subtle toxicity, sarcasm, and implicit hostility across 8 posts.",
+        "action_schema": {"message": json.dumps({"agent": "content_moderation", "task_id": "content_moderation_medium", "payload": {"classifications": [{"id": "m1", "label": "harmful", "reason": "..."}]}})},
+    },
+    {
+        "id": "content_moderation_hard",
+        "name": "Context-Aware & Adversarial Moderation",
+        "agent": "content_moderation",
+        "difficulty": "hard",
+        "description": "6 cases including dog-whistles and coordinated inauthentic behavior. Same text, two contexts.",
+        "action_schema": {"message": json.dumps({"agent": "content_moderation", "task_id": "content_moderation_hard", "payload": {"decisions": [{"id": "h1", "context_a_label": "safe", "context_b_label": "harmful"}]}})},
+    },
+    # Ticket Triage (NEW)
+    {
+        "id": "ticket_triage_easy",
+        "name": "Ticket Classification",
+        "agent": "ticket_triage",
+        "difficulty": "easy",
+        "description": "Classify a support ticket by priority and category.",
+        "action_schema": {"message": json.dumps({"agent": "ticket_triage", "task_id": "ticket_triage_easy", "payload": {"priority": "critical", "category": "bug"}})},
+    },
+    {
+        "id": "ticket_triage_medium",
+        "name": "Ticket Prioritization & Routing",
+        "agent": "ticket_triage",
+        "difficulty": "medium",
+        "description": "Order 8 tickets by priority and assign each to the correct team.",
+        "action_schema": {"message": json.dumps({"agent": "ticket_triage", "task_id": "ticket_triage_medium", "payload": {"order": ["tk1", "tk4", "..."], "assignments": {"tk1": "backend"}}})},
+    },
+    {
+        "id": "ticket_triage_hard",
+        "name": "Incident Root Cause Analysis",
+        "agent": "ticket_triage",
+        "difficulty": "hard",
+        "description": "Analyse 6 linked incident tickets: root cause, resolution steps, affected services, P1-P4 severity.",
+        "action_schema": {"message": json.dumps({"agent": "ticket_triage", "task_id": "ticket_triage_hard", "payload": {"root_cause": "...", "resolution_steps": ["..."], "affected_services": ["..."], "severity": "P1"}})},
+    },
+    # Cross-Agent Chained
+    {
+        "id": "cross_agent_chain",
+        "name": "Email + Code Review Chain",
+        "agent": "cross_agent",
+        "difficulty": "hard",
+        "description": "Classify the email AND identify the code bug in the attachment.",
+        "action_schema": {"message": json.dumps({"agent": "cross_agent", "task_id": "cross_agent_chain", "payload": {"email_classification": "important", "bugs": [{"location": "fn", "issue": "...", "fix": "..."}]}})},
+    },
+    {
+        "id": "cross_agent_email_data",
+        "name": "Email Priority + Data Cleaning Chain",
+        "agent": "cross_agent",
+        "difficulty": "hard",
+        "description": "Classify the email urgency AND identify+fix data quality issues in the attachment.",
+        "action_schema": {"message": json.dumps({"agent": "cross_agent", "task_id": "cross_agent_email_data", "payload": {"email_priority": "critical", "data_issues": {"field": "issue"}, "cleaned_data": []}})},
+    },
+    {
+        "id": "cross_agent_code_email",
+        "name": "Vulnerability + Disclosure Email Chain",
+        "agent": "cross_agent",
+        "difficulty": "hard",
+        "description": "Identify the security vulnerability AND draft a professional disclosure email.",
+        "action_schema": {"message": json.dumps({"agent": "cross_agent", "task_id": "cross_agent_code_email", "payload": {"vulnerability_type": "sql_injection", "vulnerability_location": "fn_name", "disclosure_email": "..."}})},
+    },
+    {
+        "id": "cross_agent_mod_escalation",
+        "name": "Moderation + Escalation Chain",
+        "agent": "cross_agent",
+        "difficulty": "hard",
+        "description": "Classify the content AND decide escalation AND draft a moderation notice.",
+        "action_schema": {"message": json.dumps({"agent": "cross_agent", "task_id": "cross_agent_mod_escalation", "payload": {"content_label": "harmful", "escalate": True, "moderation_notice": "..."}})},
+    },
+]
+
+
+@app.get("/tasks")
+def list_tasks():
+    return {
+        "environment": "Meta Multi-Agent v3.0",
+        "total_tasks": len(TASK_REGISTRY),
+        "agents": ["email_triage", "code_review", "data_cleaning", "content_moderation", "ticket_triage", "cross_agent"],
+        "how_to_step": (
+            'POST /step with body: {"action": {"message": "<json_string>"}} '
+            'where json_string = {"agent": "...", "task_id": "...", "payload": {...}}'
+        ),
+        "tasks": TASK_REGISTRY,
+    }
+
+
+@app.post("/grader")
+async def grader(request: Request):
+    """Score an action without persistent side effects."""
+    body   = await request.json()
+    msg    = body.get("message") or body.get("action", {}).get("message", "{}")
+    env    = MetaEnvironment()
+    env.reset()
+    action = MetaAction(message=msg)
+    obs    = env.step(action)
+    return {
+        "task_id":         obs.task_id,
+        "agent":           obs.agent,
+        "score":           obs.score,
+        "reward":          obs.reward,
+        "feedback":        obs.feedback,
+        "partial_credits": obs.partial_credits,
+        "difficulty":      obs.difficulty,
+    }
+
+
+@app.post("/baseline")
+async def baseline():
+    api_key = (
+        os.environ.get("OPENAI_API_KEY")
+        or os.environ.get("GROQ_API_KEY")
+        or os.environ.get("HF_TOKEN")
+    )
+    if not api_key:
+        return JSONResponse(
+            status_code=400,
+            content={"error": "Set one of: OPENAI_API_KEY, GROQ_API_KEY, or HF_TOKEN."},
+        )
+    api_base_url = os.environ.get("API_BASE_URL", "https://api.groq.com/openai/v1")
+    model_name   = os.environ.get("MODEL_NAME",   "llama-3.3-70b-versatile")
+    try:
+        sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        from baseline import run_all_baselines
+        scores = run_all_baselines(api_key=api_key, api_base_url=api_base_url, model_name=model_name)
+        avg = round(sum(s["score"] for s in scores) / len(scores), 3)
+        return {
+            "environment":     "Meta Multi-Agent v3.0",
+            "model":           model_name,
+            "api_base":        api_base_url,
+            "average_score":   avg,
+            "total_tasks":     len(scores),
+            "baseline_scores": scores,
+        }
+    except Exception as e:
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+
+# ── Typed step endpoint — Fix 3 ───────────────────────────────────────────────
+
+_TASK_TO_MODEL = {
+    "email_triage_easy":          EmailTriageEasyPayload,
+    "email_triage_medium":        EmailTriageMediumPayload,
+    "email_triage_hard":          EmailTriageHardPayload,
+    "code_review_easy":           CodeReviewEasyPayload,
+    "code_review_medium":         CodeReviewMediumPayload,
+    "code_review_hard":           CodeReviewHardPayload,
+    "data_cleaning_easy":         DataCleaningEasyPayload,
+    "data_cleaning_medium":       DataCleaningMediumPayload,
+    "data_cleaning_hard":         DataCleaningHardPayload,
+    "content_moderation_easy":    ContentModerationEasyPayload,
+    "content_moderation_medium":  ContentModerationMediumPayload,
+    "content_moderation_hard":    ContentModerationHardPayload,
+    "ticket_triage_easy":         TicketTriageEasyPayload,
+    "ticket_triage_medium":       TicketTriageMediumPayload,
+    "ticket_triage_hard":         TicketTriageHardPayload,
+    "cross_agent_chain":          CrossAgentChainPayload,
+    "cross_agent_email_data":     CrossAgentEmailDataPayload,
+    "cross_agent_code_email":     CrossAgentCodeEmailPayload,
+    "cross_agent_mod_escalation": CrossAgentModEscalationPayload,
+}
+
+
+@app.post("/step/typed/{task_id}")
+async def step_typed(task_id: str, request: Request):
+    """
+    Typed step endpoint — accepts the typed payload model for the given task_id
+    instead of the JSON-string-in-message format.
+
+    Example:
+        POST /step/typed/email_triage_easy
+        Body: {"agent": "email_triage", "task_id": "email_triage_easy", "payload": {"classification": "spam"}}
+
+    Returns the same observation as POST /step.
+    """
+    if task_id not in _TASK_TO_MODEL:
+        return JSONResponse(status_code=404, content={"error": f"Unknown task_id: {task_id}. See GET /tasks."})
+    body = await request.json()
+    try:
+        model_cls = _TASK_TO_MODEL[task_id]
+        payload   = body.get("payload", body)
+        typed     = model_cls(**payload)
+    except Exception as e:
+        return JSONResponse(status_code=422, content={"error": str(e), "task_id": task_id})
+    agent   = body.get("agent", "_".join(task_id.split("_")[:-1]))
+    wrapped = MetaAction(message=json.dumps({
+        "agent": agent, "task_id": task_id, "payload": typed.model_dump()
+    }))
+    # Delegate to the active environment instance
+    env = MetaEnvironment()
+    env.reset()
+    obs = env.step(wrapped)
+    return obs.model_dump()
+
+
+def main(host: str = "0.0.0.0", port: int = 7860) -> None:
+    import uvicorn
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=7860)
+    args = parser.parse_args()
+    main(port=args.port)

server/gradio_ui.py

@@ -24,18 +24,30 @@ TASK_IDS = [
     "email_triage_easy",
     "email_triage_medium",
     "email_triage_hard",
+    "email_triage_expert",
     "code_review_easy",
     "code_review_medium",
     "code_review_hard",
+    "code_review_expert",
     "data_cleaning_easy",
     "data_cleaning_medium",
     "data_cleaning_hard",
+    "data_cleaning_expert",
     "content_moderation_easy",
     "content_moderation_medium",
     "content_moderation_hard",
+    "content_moderation_expert",
+    "ticket_triage_easy",
+    "ticket_triage_medium",
+    "ticket_triage_hard",
+    "ticket_triage_expert",
+    "cross_agent_chain",
+    "cross_agent_email_data",
+    "cross_agent_code_email",
+    "cross_agent_mod_escalation",
 ]
 
-DIFFICULTY_EMOJI = {"easy": "🟢 Easy", "medium": "🟡 Medium", "hard": "🔴 Hard"}
+DIFFICULTY_EMOJI = {"easy": "🟢 Easy", "medium": "🟡 Medium", "hard": "🔴 Hard", "expert": "⚫ Expert"}
 
 EXAMPLE_PAYLOADS = {
     "email_triage_easy":        {"classification": "spam"},
@@ -206,7 +218,7 @@ def build_gradio_app() -> gr.Blocks:
 # 🤖 Meta OpenEnv
 ### Multi-Domain AI Agent Benchmark
 
-**4 agents · 12 tasks · Real-world RL environment**  
+**6 agents · 24 tasks · Real-world RL environment**  
 *Email Triage · Code Review · Data Cleaning · Content Moderation*
 
 ---
@@ -335,7 +347,8 @@ This makes it ideal for:
 | POST | `/step` | Execute action |
 | GET | `/state` | Current state |
 | GET | `/health` | Health check |
-| GET | `/tasks` | All 12 tasks |
+| GET | `/tasks` | All 24 tasks with schemas |
+    | POST | `/step/typed/{task_id}` | Typed action format |
 | POST | `/grader` | Score without side effects |
 | GET | `/schema` | JSON schemas |
 

server/gradio_ui.py.bak

@@ -0,0 +1,378 @@
+"""
+server/gradio_ui.py
+Gradio frontend for Meta OpenEnv — mounted into FastAPI at /ui
+
+Features:
+  - Task Explorer: pick any of 12 tasks, see context, edit payload, submit and score
+  - Baseline Runner: run all 12 tasks one by one with live progress
+  - Score Dashboard: 12 cards showing last known score per task
+"""
+
+import json
+import gradio as gr
+
+try:
+    from server.Meta_environment import MetaEnvironment, INSTRUCTIONS
+    from models import MetaAction
+except ImportError:
+    from .Meta_environment import MetaEnvironment, INSTRUCTIONS
+    from ..models import MetaAction
+
+# ── Constants ─────────────────────────────────────────────────────────────────
+
+TASK_IDS = [
+    "email_triage_easy",
+    "email_triage_medium",
+    "email_triage_hard",
+    "email_triage_expert",
+    "code_review_easy",
+    "code_review_medium",
+    "code_review_hard",
+    "code_review_expert",
+    "data_cleaning_easy",
+    "data_cleaning_medium",
+    "data_cleaning_hard",
+    "data_cleaning_expert",
+    "content_moderation_easy",
+    "content_moderation_medium",
+    "content_moderation_hard",
+    "content_moderation_expert",
+    "ticket_triage_easy",
+    "ticket_triage_medium",
+    "ticket_triage_hard",
+    "ticket_triage_expert",
+    "cross_agent_chain",
+    "cross_agent_email_data",
+    "cross_agent_code_email",
+    "cross_agent_mod_escalation",
+]
+
+DIFFICULTY_EMOJI = {"easy": "🟢 Easy", "medium": "🟡 Medium", "hard": "🔴 Hard", "expert": "⚫ Expert"}
+
+EXAMPLE_PAYLOADS = {
+    "email_triage_easy":        {"classification": "spam"},
+    "email_triage_medium":      {"order": ["m1", "m5", "m3", "m10", "m6", "m8", "m2", "m9", "m4", "m7"]},
+    "email_triage_hard":        {"reply": "Dear Customer, I sincerely apologize for the inconvenience caused. We will send a full refund and an overnight replacement within 24 hours. We take this very seriously and will make it right."},
+    "code_review_easy":         {"errors": ["missing colon after for statement — syntax error on line 3"]},
+    "code_review_medium":       {"bugs": [
+        {"location": "find_max",     "issue": "max_val=0 fails for all-negative lists",      "fix": "initialize to float('-inf')"},
+        {"location": "get_user",     "issue": "= used instead of == for comparison",         "fix": "change to =="},
+        {"location": "divide",       "issue": "no zero-division guard",                      "fix": "check b == 0 first"},
+        {"location": "process_list", "issue": "off-by-one error: range(len+1) → IndexError", "fix": "use range(len(items))"},
+    ]},
+    "code_review_hard":         {"vulnerabilities": [
+        {"type": "sql_injection",          "location": "get_user_by_name", "fix": "parameterized queries"},
+        {"type": "xss",                    "location": "render_comment",   "fix": "sanitize user input"},
+        {"type": "sql_injection",          "location": "login",            "fix": "parameterized queries"},
+        {"type": "command_injection",      "location": "run_report",       "fix": "avoid shell=True"},
+        {"type": "insecure_deserialization","location": "load_user_data",  "fix": "use json instead of pickle"},
+    ]},
+    "data_cleaning_easy":       {"missing": ["age (row 2)", "name (row 4)", "email (row 5)", "salary (row 6)"], "duplicates": [1, 3]},
+    "data_cleaning_medium":     {"issues": {"age": "strings not integers", "salary": "inconsistent format", "join_date": "multiple formats", "name": "inconsistent casing", "active": "mixed bool/string types"}, "cleaned_data": []},
+    "data_cleaning_hard":       {"outliers": [4, 8], "missing": [6, 11], "cleaned_data": []},
+    "content_moderation_easy":  {"classifications": [{"id": "c1", "label": "harmful"}, {"id": "c2", "label": "safe"}, {"id": "c3", "label": "harmful"}, {"id": "c4", "label": "safe"}, {"id": "c5", "label": "harmful"}, {"id": "c6", "label": "safe"}, {"id": "c7", "label": "harmful"}]},
+    "content_moderation_medium": {"classifications": [{"id": "m1", "label": "harmful"}, {"id": "m2", "label": "harmful"}, {"id": "m3", "label": "safe"}, {"id": "m4", "label": "harmful"}, {"id": "m5", "label": "harmful"}, {"id": "m6", "label": "safe"}, {"id": "m7", "label": "harmful"}, {"id": "m8", "label": "safe"}]},
+    "content_moderation_hard":  {"decisions": [{"id": "h1", "context_a_label": "safe", "context_b_label": "harmful"}, {"id": "h2", "context_a_label": "safe", "context_b_label": "harmful"}, {"id": "h3", "context_a_label": "safe", "context_b_label": "harmful"}]},
+}
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def _probe(task_id: str):
+    env   = MetaEnvironment()
+    env.reset()
+    agent = "_".join(task_id.split("_")[:-1])
+    probe = MetaAction(message=json.dumps({"agent": agent, "task_id": task_id, "payload": {"_probe": True}}))
+    obs   = env.step(probe)
+    return obs
+
+
+def _grade(task_id: str, payload: dict):
+    env   = MetaEnvironment()
+    env.reset()
+    agent = "_".join(task_id.split("_")[:-1])
+    action = MetaAction(message=json.dumps({"agent": agent, "task_id": task_id, "payload": payload}))
+    obs    = env.step(action)
+    return obs
+
+
+# ── Tab 1: Task Explorer ──────────────────────────────────────────────────────
+
+def on_task_select(task_id: str):
+    obs = _probe(task_id)
+    diff_label = DIFFICULTY_EMOJI.get(obs.difficulty, obs.difficulty)
+    context_str = json.dumps(obs.context, indent=2)
+    payload_str = json.dumps(EXAMPLE_PAYLOADS.get(task_id, {}), indent=2)
+    instructions = INSTRUCTIONS.get(task_id, obs.instructions)
+    return diff_label, instructions, context_str, payload_str, "", 0.0, ""
+
+
+def on_submit(task_id: str, payload_text: str):
+    try:
+        payload = json.loads(payload_text)
+    except json.JSONDecodeError as e:
+        return f"❌ JSON parse error: {e}", 0.0, ""
+
+    obs = _grade(task_id, payload)
+    score_pct = int(obs.score * 100)
+
+    # Colour-coded result
+    if obs.score >= 0.9:
+        icon = "✅"
+    elif obs.score >= 0.5:
+        icon = "⚠️"
+    else:
+        icon = "❌"
+
+    feedback_md = f"""
+**{icon} Score: {obs.score:.2f} / 1.00**
+
+{obs.feedback}
+
+**Partial Credits:**
+```json
+{json.dumps(obs.partial_credits, indent=2)}
+```
+"""
+    return feedback_md, obs.score, f"Score: {score_pct}%"
+
+
+# ── Tab 2: Baseline Runner ────────────────────────────────────────────────────
+
+def run_all_tasks():
+    """Generator — yields one row of results at a time for live streaming."""
+    results = []
+    rows    = []
+
+    for task_id in TASK_IDS:
+        obs = _grade(task_id, EXAMPLE_PAYLOADS.get(task_id, {}))
+        bar_filled = "█" * int(obs.score * 10)
+        bar_empty  = "░" * (10 - int(obs.score * 10))
+        diff       = obs.difficulty.upper()[:3]
+        icon       = "✅" if obs.score >= 0.9 else "⚠️" if obs.score >= 0.5 else "❌"
+        row = [
+            task_id,
+            DIFFICULTY_EMOJI.get(obs.difficulty, diff),
+            f"{obs.score:.2f}",
+            f"{icon} [{bar_filled}{bar_empty}]",
+            obs.feedback[:80],
+        ]
+        rows.append(row)
+        results.append(obs.score)
+        avg = sum(results) / len(results)
+        yield rows, f"**Running... {len(results)}/{len(TASK_IDS)}**  |  Running avg: **{avg:.3f}**"
+
+    avg = sum(results) / len(results)
+    passed = sum(1 for s in results if s >= 0.5)
+    yield rows, (
+        f"## ✅ Complete — {len(TASK_IDS)} tasks\n\n"
+        f"**Average Score: {avg:.4f}**  |  "
+        f"Passed (≥0.5): {passed}/{len(TASK_IDS)}"
+    )
+
+
+# ── Tab 3: Score Dashboard ────────────────────────────────────────────────────
+
+def build_dashboard_html():
+    cards = []
+    for task_id in TASK_IDS:
+        agent = "_".join(task_id.split("_")[:-1])
+        diff  = task_id.split("_")[-1]
+        badge_color = {"easy": "#22c55e", "medium": "#f59e0b", "hard": "#ef4444"}.get(diff, "#888")
+        cards.append(f"""
+<div style="background:#1e293b;border:1px solid #334155;border-radius:10px;padding:16px;min-width:180px;">
+  <div style="font-size:11px;font-weight:700;color:#94a3b8;text-transform:uppercase;letter-spacing:1px;margin-bottom:6px;">{agent.replace('_',' ')}</div>
+  <div style="font-size:14px;font-weight:600;color:#f1f5f9;margin-bottom:8px;">{task_id}</div>
+  <span style="background:{badge_color};color:#fff;font-size:10px;font-weight:700;padding:2px 8px;border-radius:20px;">{diff.upper()}</span>
+  <div style="margin-top:12px;font-size:11px;color:#64748b;">Submit a payload to see score</div>
+</div>""")
+    grid = '<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px;padding:8px;">'
+    grid += "".join(cards) + "</div>"
+    return grid
+
+
+# ── Build Gradio Blocks app ───────────────────────────────────────────────────
+
+def build_gradio_app() -> gr.Blocks:
+    css = """
+    body, .gradio-container { background: #0f172a !important; color: #f1f5f9 !important; }
+    .gr-button-primary { background: #6366f1 !important; border: none !important; }
+    .gr-button-primary:hover { background: #4f46e5 !important; }
+    .gr-input, .gr-textarea, .gr-dropdown { background: #1e293b !important; color: #f1f5f9 !important; border: 1px solid #334155 !important; }
+    .gr-panel { background: #1e293b !important; border: 1px solid #334155 !important; }
+    label { color: #94a3b8 !important; }
+    h1, h2, h3, h4 { color: #f1f5f9 !important; }
+    .gr-box { background: #1e293b !important; }
+    """
+
+    with gr.Blocks(
+        title="Meta OpenEnv — Multi-Domain AI Agent Benchmark",
+        theme=gr.themes.Base(
+            primary_hue="indigo",
+            neutral_hue="slate",
+        ),
+        css=css,
+    ) as demo:
+
+        # ── Header ──────────────────────────────────────────────────────────
+        gr.Markdown("""
+# 🤖 Meta OpenEnv
+### Multi-Domain AI Agent Benchmark
+
+**6 agents · 24 tasks · Real-world RL environment**  
+*Email Triage · Code Review · Data Cleaning · Content Moderation*
+
+---
+        """)
+
+        # ── Tabs ─────────────────────────────────────────────────────────────
+        with gr.Tabs():
+
+            # ── Tab 1: Task Explorer ─────────────────────────────────────────
+            with gr.Tab("🔍 Task Explorer"):
+                gr.Markdown("Select a task to explore its context, then edit the payload and submit to see your score.")
+
+                with gr.Row():
+                    task_dropdown = gr.Dropdown(
+                        choices=TASK_IDS,
+                        value=TASK_IDS[0],
+                        label="Select Task",
+                        interactive=True,
+                    )
+                    difficulty_badge = gr.Textbox(label="Difficulty", interactive=False, scale=0)
+
+                instructions_box = gr.Textbox(
+                    label="Instructions",
+                    lines=4,
+                    interactive=False,
+                )
+
+                with gr.Row():
+                    context_box = gr.Code(
+                        label="Task Context (read-only)",
+                        language="json",
+                        interactive=False,
+                        lines=15,
+                    )
+                    payload_box = gr.Code(
+                        label="Your Payload (edit and submit)",
+                        language="json",
+                        interactive=True,
+                        lines=15,
+                    )
+
+                submit_btn = gr.Button("🚀 Submit & Score", variant="primary")
+
+                with gr.Row():
+                    feedback_md  = gr.Markdown("*Submit a payload to see your score.*")
+                    score_slider = gr.Slider(0.0, 1.0, value=0.0, label="Score", interactive=False)
+                    score_label  = gr.Textbox(label="", interactive=False, scale=0)
+
+                # Events
+                task_dropdown.change(
+                    fn=on_task_select,
+                    inputs=[task_dropdown],
+                    outputs=[difficulty_badge, instructions_box, context_box, payload_box, feedback_md, score_slider, score_label],
+                )
+                submit_btn.click(
+                    fn=on_submit,
+                    inputs=[task_dropdown, payload_box],
+                    outputs=[feedback_md, score_slider, score_label],
+                )
+
+                # Load initial state
+                demo.load(
+                    fn=on_task_select,
+                    inputs=[task_dropdown],
+                    outputs=[difficulty_badge, instructions_box, context_box, payload_box, feedback_md, score_slider, score_label],
+                )
+
+            # ── Tab 2: Baseline Runner ────────────────────────────────────────
+            with gr.Tab("⚡ Baseline Runner"):
+                gr.Markdown("""
+Run all 12 tasks using the **example payloads** and see live results.
+This shows the expected score distribution across agents and difficulties.
+                """)
+
+                run_btn  = gr.Button("▶ Run All 12 Tasks", variant="primary", size="lg")
+                status   = gr.Markdown("*Click the button to start.*")
+
+                results_table = gr.Dataframe(
+                    headers=["Task ID", "Difficulty", "Score", "Bar", "Feedback"],
+                    datatype=["str", "str", "str", "str", "str"],
+                    interactive=False,
+                    wrap=True,
+                )
+
+                run_btn.click(
+                    fn=run_all_tasks,
+                    inputs=[],
+                    outputs=[results_table, status],
+                )
+
+            # ── Tab 3: Score Dashboard ────────────────────────────────────────
+            with gr.Tab("📊 Task Dashboard"):
+                gr.Markdown("Overview of all 12 tasks across the 4 agents.")
+                gr.HTML(build_dashboard_html())
+
+                gr.Markdown("""
+### Baseline Scores (GPT-4o-mini)
+
+| Agent | Easy | Medium | Hard | Avg |
+|-------|------|--------|------|-----|
+| Email Triage | 1.00 | 0.80 | 0.67 | 0.82 |
+| Code Review | 1.00 | 0.75 | 0.80 | 0.85 |
+| Data Cleaning | 1.00 | 0.67 | 0.67 | 0.78 |
+| Content Moderation | 1.00 | 0.75 | 0.50 | 0.75 |
+| **Overall** | **1.00** | **0.74** | **0.66** | **0.80** |
+                """)
+
+            # ── Tab 4: About ──────────────────────────────────────────────────
+            with gr.Tab("ℹ️ About"):
+                gr.Markdown("""
+## Why Meta?
+
+Most OpenEnv environments cover a single domain. **Meta is the first multi-domain environment** that lets a single agent learn to handle completely different real-world tasks in one unified interface.
+
+This makes it ideal for:
+- Training **generalist agents** that switch between task types
+- Benchmarking **multi-task learning** capabilities of LLMs
+- Evaluating **transfer learning** between related domains
+- Testing **robustness** across varying difficulty levels
+
+## API Endpoints
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| POST | `/reset` | Reset environment |
+| POST | `/step` | Execute action |
+| GET | `/state` | Current state |
+| GET | `/health` | Health check |
+| GET | `/tasks` | All 24 tasks with schemas |
+    | POST | `/step/typed/{task_id}` | Typed action format |
+| POST | `/grader` | Score without side effects |
+| GET | `/schema` | JSON schemas |
+
+## Team
+
+**Digital Yodha** — OpenEnv Hackathon 2026
+
+| Name | GitHub |
+|------|--------|
+| Sangisetti Akarsh | [@Farbricated](https://github.com/Farbricated) |
+| Sarika Jivrajika | [@Sarika-stack23](https://github.com/Sarika-stack23) |
+
+---
+*Built for OpenEnv Hackathon 2026 · MIT License*
+                """)
+
+    return demo
+
+
+# ── Mount helper (called from app.py) ─────────────────────────────────────────
+
+def mount_gradio(app):
+    """Mount the Gradio app into an existing FastAPI app at /ui"""
+    import gradio as gr
+    demo = build_gradio_app()
+    gr.mount_gradio_app(app, demo, path="/ui")
+    return app