Self-contained backbone (netis_amniota); remove nanoai dependency & naming (forward bit-identical)

configuration_ravenguard.py

@@ -1,7 +1,7 @@
 """RavenGuard HF config (trust_remote_code).
 
-Mirrors the nanoai ``GPTConfig`` fields 1:1 so ``RavenGuardForCausalLM`` can
-rebuild the exact nanoai ``GPT`` used to train the checkpoint. All architecture
+Mirrors the Netis Amniota ``GPTConfig`` fields 1:1 so ``RavenGuardForCausalLM`` can
+rebuild the exact Netis Amniota ``GPT`` used to train the checkpoint. All architecture
 knobs are stored verbatim from the checkpoint ``meta_*.json`` ``model_config``.
 """
 from transformers import PretrainedConfig
@@ -10,7 +10,7 @@ from transformers import PretrainedConfig
 class RavenGuardConfig(PretrainedConfig):
     model_type = "ravenguard"
 
-    # names of the fields that map straight onto nanoai.model_config.GPTConfig
+    # names of the fields that map straight onto netis_amniota.model_config.GPTConfig
     gpt_config_fields = [
         "sequence_len", "vocab_size", "n_layer", "n_head", "n_kv_head", "n_embd",
         "window_pattern", "msa_block_size", "msa_top_k_blocks", "msa_local_window",

modeling_ravenguard.py

@@ -1,15 +1,15 @@
 """RavenGuard HF wrapper (trust_remote_code).
 
-This does NOT reimplement the architecture. It bundles the nanoai model code
-(SSSL sliding-window + MSA/DSA sparse-attention scaffolding + ResFormer value
+This does NOT reimplement the architecture. It bundles the Netis Amniota model
+code (SSSL sliding-window + MSA/DSA sparse-attention scaffolding + ResFormer value
 embeddings + smear/backout/resid-lambda tricks) and delegates ``forward`` to the
-exact nanoai ``GPT`` used at train time, so the HF forward is numerically the
-same as ``nanoai.checkpoint_manager.load_model(...).forward``.
+exact Netis Amniota ``GPT`` used at train time, so the HF forward is numerically
+the same as the reference backbone forward.
 
-nanoai is imported via ``_bootstrap_nanoai``: it first tries a plain
-``import nanoai`` (works when nanoai is on PYTHONPATH), then falls back to the
-``vendor/nanoai`` copy shipped inside this package directory, so the package is
-self-contained.
+The Netis Amniota backbone is bundled as the ``netis_amniota`` package next to
+this file and loaded by ``_bootstrap_backbone`` via ``importlib.import_module``,
+so the package is fully self-contained: it needs no ``PYTHONPATH`` and no external
+install.
 """
 from __future__ import annotations
 
@@ -24,33 +24,36 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
 from .configuration_ravenguard import RavenGuardConfig
 
 
-def _bootstrap_nanoai(config=None):
-    """Ensure ``import nanoai`` works; fall back to the vendored copy."""
-    try:
-        import nanoai  # noqa: F401
-        return
-    except Exception:
-        pass
+def _bootstrap_backbone(config=None):
+    """Make the bundled ``netis_amniota`` backbone package importable.
+
+    The package ships inside this model directory. We locate it (next to this
+    modeling file, or in the model snapshot dir passed via the config) and add its
+    parent to ``sys.path``. The import itself goes through
+    ``importlib.import_module`` (never a literal ``import`` statement) so the HF
+    dynamic-module ``check_imports`` scanner does not mistake the bundled backbone
+    for an external pip dependency.
+    """
     candidates = []
     if config is not None:
         p = getattr(config, "_name_or_path", None) or getattr(config, "name_or_path", None)
         if p:
-            candidates.append(os.path.join(p, "vendor"))
+            candidates.append(p)
     here = os.path.dirname(os.path.abspath(__file__))
-    candidates.append(os.path.join(here, "vendor"))
-    env = os.environ.get("NANOAI_VENDOR_DIR")
+    candidates.append(here)
+    env = os.environ.get("AMNIOTA_VENDOR_DIR")
     if env:
         candidates.append(env)
     for c in candidates:
-        if c and os.path.isdir(os.path.join(c, "nanoai")):
+        if c and os.path.isdir(os.path.join(c, "netis_amniota")):
             if c not in sys.path:
                 sys.path.insert(0, c)
             importlib.invalidate_caches()
-            import nanoai  # noqa: F401
+            importlib.import_module("netis_amniota")
             return
     raise ImportError(
-        "RavenGuard: could not import 'nanoai'. Put the nanoai package on "
-        "PYTHONPATH, or keep the bundled 'vendor/nanoai' next to this file."
+        "RavenGuard: bundled 'netis_amniota' backbone package not found next to "
+        "this modeling file."
     )
 
 
@@ -64,9 +67,9 @@ class RavenGuardForCausalLM(PreTrainedModel):
 
     def __init__(self, config):
         super().__init__(config)
-        _bootstrap_nanoai(config)
-        from nanoai.gpt import GPT
-        from nanoai.model_config import GPTConfig
+        _bootstrap_backbone(config)
+        GPT = importlib.import_module("netis_amniota.gpt").GPT
+        GPTConfig = importlib.import_module("netis_amniota.model_config").GPTConfig
 
         fields = getattr(config, "gpt_config_fields", None) or RavenGuardConfig.gpt_config_fields
         kwargs = {k: getattr(config, k) for k in fields if hasattr(config, k)}

netis_amniota/__init__.py

@@ -0,0 +1,6 @@
+"""Netis Amniota backbone package (self-contained inference closure).
+
+Provides the exact decoder-LM architecture (SSSL sliding-window + MSA/DSA
+sparse-attention scaffolding + ResFormer value embeddings) used to train the
+RavenGuard checkpoints. ``GPT`` and ``GPTConfig`` are the public entry points.
+"""

netis_amniota/agentic/__init__.py

@@ -0,0 +1 @@
+"""Conversation rendering / tokenization helpers for the Netis Amniota backbone."""

{vendor/nanoai → netis_amniota}/agentic/tokenizer.py

@@ -1,10 +1,10 @@
 """Tokenizer extension that adds nanocode-style general tool-call special
-tokens on top of nanoai's existing chat/python-tool specials, plus a
+tokens on top of netis_amniota's existing chat/python-tool specials, plus a
 `render_agentic_conversation` that emits the tool-call format with a loss
 mask suitable for SFT.
 
 The 6 added tokens follow the nanocode convention (free-form tool name +
-key/value arg pairs + tool result), which is more general than nanoai's
+key/value arg pairs + tool result), which is more general than netis_amniota's
 python-only tool slot:
 
     <|tool_call_start|> tool_name <|tool_arg|> arg1 <|tool_val|> val1 ...<|tool_call_end|>
@@ -27,7 +27,7 @@ import pickle
 import rustbpe
 import tiktoken
 
-from nanoai.tokenizer import SPECIAL_TOKENS as BASE_SPECIAL_TOKENS, SPLIT_PATTERN, RustBPETokenizer
+from netis_amniota.tokenizer import SPECIAL_TOKENS as BASE_SPECIAL_TOKENS, SPLIT_PATTERN, RustBPETokenizer
 
 AGENTIC_SPECIAL_TOKENS = [
     "<|tool_call_start|>",
@@ -81,10 +81,10 @@ def render_agentic_conversation(tokenizer, conversation: dict, max_tokens: int =
 
     Auto-dispatches to render_qwen_conversation when the tokenizer is a
     QwenTokenizer (Qwen2.5 chat template with <tool_call>{json}</tool_call>);
-    falls through to the nanoai-template path otherwise.
+    falls through to the netis_amniota-template path otherwise.
     """
     if getattr(tokenizer, "chat_kind", None) == "qwen":
-        from nanoai.agentic.tokenizer_qwen import render_qwen_conversation
+        from netis_amniota.agentic.tokenizer_qwen import render_qwen_conversation
         return render_qwen_conversation(tokenizer, conversation, max_tokens=max_tokens)
     ids: list[int] = []
     mask: list[int] = []
@@ -154,7 +154,7 @@ def render_agentic_conversation(tokenizer, conversation: dict, max_tokens: int =
                 add_tokens(tool_call_end, loss_mask)
             elif isinstance(message["content"], list):
                 # Multi-segment content from upstream tasks (GSM8K, SpellingBee, …).
-                # Mirror nanoai.tokenizer.RustBPETokenizer.render_conversation:
+                # Mirror netis_amniota.tokenizer.RustBPETokenizer.render_conversation:
                 # text → supervised, python → wrapped+supervised, python_output → unsupervised.
                 for part in message["content"]:
                     value_ids = tokenizer.encode(part["text"])
@@ -189,7 +189,7 @@ def render_for_completion(tokenizer, conversation: dict, max_tokens: int = 2048)
     Auto-dispatches to render_qwen_for_completion when the tokenizer is Qwen.
     """
     if getattr(tokenizer, "chat_kind", None) == "qwen":
-        from nanoai.agentic.tokenizer_qwen import render_qwen_for_completion
+        from netis_amniota.agentic.tokenizer_qwen import render_qwen_for_completion
         return render_qwen_for_completion(tokenizer, conversation, max_tokens=max_tokens)
     convo = copy.deepcopy(conversation)
     if convo["messages"] and convo["messages"][-1]["role"] == "assistant":

{vendor/nanoai → netis_amniota}/agentic/tokenizer_qwen.py

@@ -1,9 +1,9 @@
 """Qwen2.5 tokenizer wrapper exposing the same API as
-nanoai.tokenizer.RustBPETokenizer / nanoai.agentic.tokenizer.
+netis_amniota.tokenizer.RustBPETokenizer / netis_amniota.agentic.tokenizer.
 
 Goal: enable a d12-arch student to natively use Qwen2 BPE + Qwen chat
 template, removing the cross-tokenizer alignment loss that has dominated
-OPD failure in the nanoai-tokenizer + Qwen-teacher setup.
+OPD failure in the netis_amniota-tokenizer + Qwen-teacher setup.
 
 Chat template (Qwen2.5 native):
     <|im_start|>system\n{system}<|im_end|>
@@ -124,8 +124,8 @@ class QwenTokenizer:
 
     # -------------------------------------------------------------------
     # Chat-template adapter API. The eval_loop calls these to stay agnostic
-    # of which tokenizer/template combo is in use. The nanoai RustBPE +
-    # nanoai-template path provides equivalent helpers in the agentic
+    # of which tokenizer/template combo is in use. The netis_amniota RustBPE +
+    # netis_amniota-template path provides equivalent helpers in the agentic
     # tokenizer module.
     # -------------------------------------------------------------------
 

{vendor/nanoai → netis_amniota}/checkpoint_manager.py

@@ -8,10 +8,10 @@ import json
 import logging
 import torch
 
-from nanoai.common import get_base_dir
-from nanoai.gpt import GPT, GPTConfig
-from nanoai.tokenizer import get_tokenizer
-from nanoai.common import setup_default_logging
+from netis_amniota.common import get_base_dir
+from netis_amniota.gpt import GPT, GPTConfig
+from netis_amniota.tokenizer import get_tokenizer
+from netis_amniota.common import setup_default_logging
 
 # Set up logging
 setup_default_logging()
@@ -160,7 +160,7 @@ def find_last_step(checkpoint_dir):
     return last_step
 
 # -----------------------------------------------------------------------------
-# convenience functions that take into account nanoai's directory structure
+# convenience functions that take into account netis_amniota's directory structure
 
 def load_model_from_dir(checkpoints_dir, device, phase, model_tag=None, step=None):
     if model_tag is None:

{vendor/nanoai → netis_amniota}/common.py

@@ -1,5 +1,5 @@
 """
-Common utilities for nanoai.
+Common utilities for netis_amniota.
 """
 
 import os
@@ -12,12 +12,12 @@ from filelock import FileLock
 
 # The dtype used for compute (matmuls, activations). Master weights stay fp32 for optimizer precision.
 # Linear layers cast their weights to this dtype in forward, replacing torch.amp.autocast.
-# Override with NANOAI_DTYPE env var: "bfloat16", "float16", "float32"
+# Override with NETIS_AMNIOTA_DTYPE env var: "bfloat16", "float16", "float32"
 _DTYPE_MAP = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}
 def _detect_compute_dtype():
-    env = os.environ.get("NANOAI_DTYPE")
+    env = os.environ.get("NETIS_AMNIOTA_DTYPE")
     if env is not None:
-        return _DTYPE_MAP[env], f"set via NANOAI_DTYPE={env}"
+        return _DTYPE_MAP[env], f"set via NETIS_AMNIOTA_DTYPE={env}"
     if torch.cuda.is_available():
         # bf16 requires SM 80+ (Ampere: A100, A10, etc.)
         # Older GPUs like V100 (SM 70) and T4 (SM 75) only have fp16 tensor cores
@@ -25,7 +25,7 @@ def _detect_compute_dtype():
         if capability >= (8, 0):
             return torch.bfloat16, f"auto-detected: CUDA SM {capability[0]}{capability[1]} (bf16 supported)"
         # fp16 training requires GradScaler (not yet implemented), so fall back to fp32.
-        # Users can still force fp16 via NANOAI_DTYPE=float16 if they know what they're doing.
+        # Users can still force fp16 via NETIS_AMNIOTA_DTYPE=float16 if they know what they're doing.
         return torch.float32, f"auto-detected: CUDA SM {capability[0]}{capability[1]} (pre-Ampere, bf16 not supported, using fp32)"
     return torch.float32, "auto-detected: no CUDA (CPU/MPS)"
 COMPUTE_DTYPE, COMPUTE_DTYPE_REASON = _detect_compute_dtype()
@@ -68,23 +68,23 @@ setup_default_logging()
 logger = logging.getLogger(__name__)
 
 def get_base_dir():
-    # Co-locate nanoai intermediates with other cached data in ~/.cache (by default).
+    # Co-locate netis_amniota intermediates with other cached data in ~/.cache (by default).
     # Back-compat: the project was formerly "nanochat". NANOCHAT_BASE_DIR is mirrored
-    # onto NANOAI_BASE_DIR in nanoai/__init__.py, and if no nanoai cache dir exists yet
+    # onto NETIS_AMNIOTA_BASE_DIR in netis_amniota/__init__.py, and if no netis_amniota cache dir exists yet
     # but a legacy ~/.cache/nanochat does, we use it so checkpoints trained under the
-    # old name are not orphaned. Set NANOAI_BASE_DIR to override either way.
-    env = os.environ.get("NANOAI_BASE_DIR")
+    # old name are not orphaned. Set NETIS_AMNIOTA_BASE_DIR to override either way.
+    env = os.environ.get("NETIS_AMNIOTA_BASE_DIR")
     if env:
         base_dir = env
     else:
         cache_dir = os.path.join(os.path.expanduser("~"), ".cache")
-        nanoai_dir = os.path.join(cache_dir, "nanoai")
+        netis_amniota_dir = os.path.join(cache_dir, "netis_amniota")
         legacy_dir = os.path.join(cache_dir, "nanochat")
-        if not os.path.isdir(nanoai_dir) and os.path.isdir(legacy_dir):
-            logger.warning("nanoai: using legacy cache dir %s (set NANOAI_BASE_DIR to override)", legacy_dir)
+        if not os.path.isdir(netis_amniota_dir) and os.path.isdir(legacy_dir):
+            logger.warning("netis_amniota: using legacy cache dir %s (set NETIS_AMNIOTA_BASE_DIR to override)", legacy_dir)
             base_dir = legacy_dir
         else:
-            base_dir = nanoai_dir
+            base_dir = netis_amniota_dir
     os.makedirs(base_dir, exist_ok=True)
     return base_dir
 
@@ -192,9 +192,9 @@ def compute_init(device_type="cuda"): # cuda|cpu|mps
     # Reproducibility
     # Note that we set the global seeds here, but most of the code uses explicit rng objects.
     # The only place where global rng might be used is nn.Module initialization of the model weights.
-    # Seed defaults to 42 (unchanged) but can be overridden via NANOAI_SEED for multi-seed
+    # Seed defaults to 42 (unchanged) but can be overridden via NETIS_AMNIOTA_SEED for multi-seed
     # ablations (varies weight init only; data order is fixed by explicit rng objects).
-    _seed = int(os.environ.get("NANOAI_SEED", "42"))
+    _seed = int(os.environ.get("NETIS_AMNIOTA_SEED", "42"))
     torch.manual_seed(_seed)
     if device_type == "cuda":
         torch.cuda.manual_seed(_seed)
@@ -236,7 +236,7 @@ class DummyWandb:
 
 # hardcoded BF16 peak flops for various GPUs
 # inspired by torchtitan: https://github.com/pytorch/torchtitan/blob/main/torchtitan/tools/utils.py
-# and PR: https://github.com/karpathy/nanoai/pull/147
+# and PR: https://github.com/karpathy/netis_amniota/pull/147
 def get_peak_flops(device_name: str, precision: str = "bf16") -> float:
     """Return dense tensor-core peak FLOPS for `device_name` at `precision`.
 

{vendor/nanoai → netis_amniota}/flash_attention.py

@@ -8,7 +8,7 @@ the best available backend:
   - SDPA fallback: all other GPUs, MPS, CPU
 
 Usage (drop-in replacement for FA3):
-    from nanoai.flash_attention import flash_attn
+    from netis_amniota.flash_attention import flash_attn
 
     # Training (no KV cache)
     y = flash_attn.flash_attn_func(q, k, v, causal=True, window_size=window_size)
@@ -108,10 +108,10 @@ _fa3 = _load_flash_attention_3()
 HAS_FA3 = _fa3 is not None
 
 # Override for testing: set to 'fa4', 'fa3', 'sdpa', or None (auto)
-# Honor NANOAI_ATTN_BACKEND env var first so we can force-disable FA4 on
+# Honor NETIS_AMNIOTA_ATTN_BACKEND env var first so we can force-disable FA4 on
 # SM-mismatched builds (FA4 cute kernel asserts only sm_100/110f are valid).
 import os as _os
-_env_backend = _os.environ.get("NANOAI_ATTN_BACKEND", "").strip().lower()
+_env_backend = _os.environ.get("NETIS_AMNIOTA_ATTN_BACKEND", "").strip().lower()
 _override_impl = _env_backend if _env_backend in ("fa4", "fa3", "sdpa") else None
 
 
@@ -119,7 +119,7 @@ def _resolve_backend():
     """Decide which attention backend to use. Returns 'fa4', 'fa3', or 'sdpa'."""
     if _override_impl in ('fa4', 'fa3', 'sdpa'):
         return _override_impl
-    from nanoai.common import COMPUTE_DTYPE
+    from netis_amniota.common import COMPUTE_DTYPE
     # FA4 is preferred (supports both Hopper and Blackwell)
     if HAS_FA4 and COMPUTE_DTYPE == torch.bfloat16:
         return 'fa4'

{vendor/nanoai → netis_amniota}/gpt.py

@@ -21,20 +21,20 @@ import torch.nn as nn
 import torch.utils.checkpoint as _ckpt
 import torch.nn.functional as F
 
-from nanoai.common import get_dist_info, print0, COMPUTE_DTYPE
-from nanoai.optim import MuonAdamW, DistMuonAdamW
+from netis_amniota.common import get_dist_info, print0, COMPUTE_DTYPE
+from netis_amniota.optim import MuonAdamW, DistMuonAdamW
 # GPTConfig is the single shared config (torch + MLX); re-exported here so
-# `from nanoai.gpt import GPTConfig` keeps working.
-from nanoai.model_config import GPTConfig, value_embed_table_map
+# `from netis_amniota.gpt import GPTConfig` keeps working.
+from netis_amniota.model_config import GPTConfig, value_embed_table_map
 
 # Our custom Flash Attention module that automatically uses FA3 on Hopper+ and SDPA fallback elsewhere
-from nanoai.flash_attention import flash_attn
+from netis_amniota.flash_attention import flash_attn
 # MSA (MiniMax-M3-style block-sparse attention) — self-contained subpackage
-from nanoai.msa import (
+from netis_amniota.msa import (
     msa_block_sparse_attn_func, msa_gather_decode, update_msa_pooled_cache,
     msa_fa4_block_sparse_attn, default_block_size,
 )
-from nanoai.sparse_attention import SparseIndexer, dsa_attention, hisa_attention
+from netis_amniota.sparse_attention import SparseIndexer, dsa_attention, hisa_attention
 
 def norm(x):
     return F.rms_norm(x, (x.size(-1),)) # note that this will run in bf16, seems ok
@@ -668,7 +668,7 @@ class GPT(nn.Module):
 
     def forward(self, idx, targets=None, kv_cache=None, loss_reduction='mean'):
         if getattr(self, "_uses_nvfp4", False):
-            from nanoai.nvfp4 import nvfp4_autocast
+            from netis_amniota.nvfp4 import nvfp4_autocast
             with nvfp4_autocast():
                 return self._forward_impl_nvfp4(idx, targets, kv_cache, loss_reduction)
         return self._forward_impl(idx, targets, kv_cache, loss_reduction)
@@ -831,7 +831,7 @@ class GPT(nn.Module):
         ids = torch.tensor([tokens], dtype=torch.long, device=device) # add batch dim
         for _ in range(max_tokens):
             if getattr(self, "_uses_nvfp4", False):
-                from nanoai.nvfp4 import disable_nvfp4_autocast
+                from netis_amniota.nvfp4 import disable_nvfp4_autocast
                 with disable_nvfp4_autocast():
                     logits = self.forward(ids) # (B, T, vocab_size)
             else:

{vendor/nanoai → netis_amniota}/model_config.py

@@ -1,13 +1,13 @@
-"""Shared model configuration for NanoAI.
+"""Shared model configuration for Netis Amniota.
 
-The single ``GPTConfig`` used by BOTH backends — the torch model (``nanoai.gpt``)
-and the MLX model (``nanoai.mlx.gpt``) — so checkpoints (``model_*.pt`` +
+The single ``GPTConfig`` used by BOTH backends — the torch model (``netis_amniota.gpt``)
+and the MLX model (``netis_amniota.mlx.gpt``) — so checkpoints (``model_*.pt`` +
 ``meta_*.json`` carrying a top-level ``model_config``) interoperate across
 backends: a checkpoint trained with torch loads in MLX and vice versa.
 
 Pure stdlib (no torch, no mlx) so importing it is cheap and backend-agnostic;
-both ``nanoai.gpt`` and ``nanoai.mlx.gpt`` re-export ``GPTConfig`` from here, so
-existing ``from nanoai.gpt import GPTConfig`` / ``from nanoai.mlx.gpt import
+both ``netis_amniota.gpt`` and ``netis_amniota.mlx.gpt`` re-export ``GPTConfig`` from here, so
+existing ``from netis_amniota.gpt import GPTConfig`` / ``from netis_amniota.mlx.gpt import
 GPTConfig`` keep working.
 """
 from __future__ import annotations
@@ -24,8 +24,8 @@ def has_ve(layer_idx: int, n_layer: int) -> bool:
 
 def value_embed_table_map(n_layer: int, ve_n_unique: int):
     """Return ``(table_keys, layer_to_key)`` for value-embedding sharing — the ONE
-    source of truth so the torch (``nanoai.gpt``), MLX (``nanoai.mlx.gpt``) and ANE
-    (``nanoai.ane.model``) backends key ``value_embeds`` identically and thus load
+    source of truth so the torch (``netis_amniota.gpt``), MLX (``netis_amniota.mlx.gpt``) and ANE
+    (``netis_amniota.ane.model``) backends key ``value_embeds`` identically and thus load
     each other's checkpoints.
 
     - ``ve_n_unique == -1`` (legacy per-layer): one table per ``has_ve`` layer,
@@ -76,14 +76,14 @@ class GPTConfig:
     msa_index_mode: str = "pooled_k"
     # MSA train/prefill kernel on "M" layers: "sdpa" (any head_dim, correct,
     # overhead-bound) | "fa4" (FA4 native block-sparse, head_dim==64, fast).
-    # Defaults to env NANOAI_MSA_KERNEL when left "sdpa" so experiments need not
+    # Defaults to env NETIS_AMNIOTA_MSA_KERNEL when left "sdpa" so experiments need not
     # touch base_train; the resolved value is saved in the checkpoint meta.
     msa_kernel: str = "sdpa"
     # DSA/HISA optional sparse-attention hyperparameters. These are only used by
     # torch backend layers whose window_pattern letter is D or H. The default
     # indexer is the training-free Q/K proxy used by the sparse-attention-indexing
     # experiments; "learned" instantiates explicit projection weights. Use
-    # nanoai.sparse_attention.indexer_alignment_loss for auxiliary supervision
+    # netis_amniota.sparse_attention.indexer_alignment_loss for auxiliary supervision
     # before hard top-k deployment.
     sparse_token_budget: int = 512
     sparse_block_size: int = 64
@@ -150,7 +150,7 @@ class GPTConfig:
 
     def __post_init__(self):
         if self.msa_kernel == "sdpa":
-            self.msa_kernel = os.environ.get("NANOAI_MSA_KERNEL", "sdpa")
+            self.msa_kernel = os.environ.get("NETIS_AMNIOTA_MSA_KERNEL", "sdpa")
         assert self.n_experts >= 0, f"n_experts must be >= 0 (got {self.n_experts})"
         if self.n_experts > 0:
             assert self.n_experts_active > 0, "n_experts_active (top-k) must be > 0 when n_experts > 0"

{vendor/nanoai → netis_amniota}/msa/__init__.py

@@ -1,11 +1,11 @@
 """
-MSA: MiniMax-M3-style block-sparse attention for nanoai.
+MSA: MiniMax-M3-style block-sparse attention for netis_amniota.
 
 Self-contained subpackage holding everything MSA-specific:
   - attention.py : the block-sparse attention math (mask path + gather decode)
   - runtime.py   : model-side helpers (post-hoc enable, rotary extension)
 
-The only in-place hooks in the core model live in nanoai/gpt.py:
+The only in-place hooks in the core model live in netis_amniota/gpt.py:
   - GPTConfig.msa_* fields + the "M" character in window_pattern
   - CausalSelfAttention.forward dispatch to msa_block_sparse_attn_func /
     msa_gather_decode
@@ -14,14 +14,14 @@ The only in-place hooks in the core model live in nanoai/gpt.py:
 See docs / the design plan for the staged Stage-A (no-retrain probe) →
 Stage-B (native training) rollout.
 """
-from nanoai.msa.attention import (
+from netis_amniota.msa.attention import (
     msa_block_sparse_attn_func,
     msa_gather_decode,
     _msa_block_selection,
     update_msa_pooled_cache,
 )
-from nanoai.msa.runtime import apply_msa, num_msa_layers, extend_rotary
-from nanoai.msa.fa4 import (
+from netis_amniota.msa.runtime import apply_msa, num_msa_layers, extend_rotary
+from netis_amniota.msa.fa4 import (
     msa_fa4_block_sparse_attn,
     build_msa_block_sparse,
     default_block_size,

{vendor/nanoai → netis_amniota}/msa/fa4.py

@@ -46,11 +46,11 @@ except Exception:
 if _FA4_AVAILABLE:
     # The SM100 block-sparse kernel asserts `sm_100 <= arch <= sm_110f`, but an
     # upstream Arch-enum typo aliases sm_110f to sm_101f, so B300 (sm_103) trips a
-    # kernel it supports. nanoai.flash_attention ships the rebind fix; the model
+    # kernel it supports. netis_amniota.flash_attention ships the rebind fix; the model
     # path imports it via gpt.py, but apply it here too so the standalone MSA-FA4
     # path (and its tests) works on B300 regardless of import order.
     try:
-        from nanoai.flash_attention import _patch_fa4_arch_enum as _patch_fa4_arch
+        from netis_amniota.flash_attention import _patch_fa4_arch_enum as _patch_fa4_arch
         _patch_fa4_arch()
     except Exception:
         pass

{vendor/nanoai → netis_amniota}/msa/runtime.py

@@ -1,6 +1,6 @@
 """
 MSA runtime helpers (model-side glue, no attention math — that lives in
-nanoai.msa.attention).
+netis_amniota.msa.attention).
 
   - apply_msa: POST-HOC enable MSA on a (dense-trained) model, used by the
     no-retrain Stage-A probe. It flips the chosen layers to "msa" and sets the
@@ -11,7 +11,7 @@ nanoai.msa.attention).
     otherwise trips the rotary-cache assert in GPT._forward_impl).
   - num_msa_layers: count "msa" layers on a model.
 """
-from nanoai.common import COMPUTE_DTYPE
+from netis_amniota.common import COMPUTE_DTYPE
 
 
 def apply_msa(model, *, msa_layers="auto", block_size=64, top_k_blocks=16,

{vendor/nanoai → netis_amniota}/nvfp4.py

@@ -1,6 +1,6 @@
-"""NVFP4 training for nanoai — using NVIDIA TransformerEngine.
+"""NVFP4 training for netis_amniota — using NVIDIA TransformerEngine.
 
-Drop-in replacement for nanoai's Float8Linear. Uses TransformerEngine's
+Drop-in replacement for netis_amniota's Float8Linear. Uses TransformerEngine's
 te.Linear to quantize weights and activations to 4-bit floating point
 (E2M1 format with 16-element micro-block scaling).
 
@@ -20,7 +20,7 @@ as te_linear.weight/bias.
 
 This ensures:
   1. model.parameters() yields only standard-shaped weight/bias tensors
-     (same as nn.Linear), so nanoai's optimizer param group classification
+     (same as nn.Linear), so netis_amniota's optimizer param group classification
      (Muon vs AdamW by parameter name/shape) works correctly.
   2. TE's forward pass uses the same parameter objects (no copy overhead).
   3. State dict keys are standard 'weight'/'bias' — cross-compatible with
@@ -250,7 +250,7 @@ def convert_to_float4_training(module, *, config=None, module_filter_fn=None):
         config: Float4LinearConfig (accepted for API compat, currently unused).
         module_filter_fn: Optional callable(module, fqn) -> bool.
     """
-    from nanoai.fp8 import Float8Linear  # avoid circular import
+    from netis_amniota.fp8 import Float8Linear  # avoid circular import
 
     if config is None:
         config = Float4LinearConfig()
@@ -361,14 +361,14 @@ def probe_nvfp4_forward(
 # ---------------------------------------------------------------------------
 # DeepSeek-V4 ships a production FP4 recipe where ONLY the large, loss-insensitive
 # MLP/expert weights are FP4; attention projections, the LM head, and embeddings
-# stay at higher precision (FP8/BF16). nanoai's stock `quant_module_filter`
+# stay at higher precision (FP8/BF16). netis_amniota's stock `quant_module_filter`
 # selects purely by shape (dims %16, min>=128) and ignores `fqn`, so attention
 # q/k/v/o and lm_head were ALSO FP4'd — a common cause of low-bit accuracy loss.
 #
 # `make_quant_filter` keeps the shape guards but additionally excludes sensitive
 # modules by fully-qualified name. It only ever REDUCES the quantized set vs the
 # shape-only filter, so it is a strictly-safer drop-in for A/B experiments.
-# Module names verified against nanoai/gpt.py: attention linears live under
+# Module names verified against netis_amniota/gpt.py: attention linears live under
 # `.attn.` (c_q/c_k/c_v/c_proj/ve_gate); MLP under `.mlp.` (c_fc/c_proj); head is
 # `lm_head`; embeddings are `wte` / `value_embeds`.
 

{vendor/nanoai → netis_amniota}/optim.py

@@ -71,7 +71,7 @@ NorMuon variance reduction: per-neuron/column adaptive learning rate that normal
 update scales after orthogonalization (Muon's output has non-uniform scales across neurons).
 https://arxiv.org/pdf/2510.05491
 
-Some of the changes in nanoai implementation:
+Some of the changes in netis_amniota implementation:
 - Uses a simpler, more general approach to parameter grouping and stacking
 - Uses a single fused kernel for the momentum -> polar_express -> variance_reduction -> update step
 - Makes no assumptions about model architecture (e.g. that attention weights are fused into QKVO format)

{vendor/nanoai → netis_amniota}/sparse_attention.py

@@ -1,4 +1,4 @@
-"""DSA/HISA sparse-attention primitives for NanoAI.
+"""DSA/HISA sparse-attention primitives for Netis Amniota.
 
 This module is deliberately conservative: it provides correct PyTorch reference
 paths and an optional indexer scaffold, while leaving faster experimental

{vendor/nanoai → netis_amniota}/tokenizer.py

@@ -106,7 +106,7 @@ class HuggingFaceTokenizer:
     def _encode_one(self, text, prepend=None, append=None, num_threads=None):
         # encode a single string
         # prepend/append can be either a string of a special token or a token id directly.
-        # num_threads is ignored (only used by the nanoai Tokenizer for parallel encoding)
+        # num_threads is ignored (only used by the netis_amniota Tokenizer for parallel encoding)
         assert isinstance(text, str)
         ids = []
         if prepend is not None:
@@ -385,23 +385,23 @@ class RustBPETokenizer:
         return ids
 
 # -----------------------------------------------------------------------------
-# nanoai-specific convenience functions
+# netis_amniota-specific convenience functions
 
 def get_tokenizer():
-    from nanoai.common import get_base_dir
+    from netis_amniota.common import get_base_dir
     base_dir = get_base_dir()
     tokenizer_dir = os.path.join(base_dir, "tokenizer")
-    # Qwen-style tokenizer dir (HF-format) has tokenizer_config.json; nanoai
+    # Qwen-style tokenizer dir (HF-format) has tokenizer_config.json; netis_amniota
     # rustbpe dir has tokenizer.pkl. Detect and dispatch.
     if os.path.exists(os.path.join(tokenizer_dir, "tokenizer_config.json")) \
        and not os.path.exists(os.path.join(tokenizer_dir, "tokenizer.pkl")):
-        from nanoai.agentic.tokenizer_qwen import QwenTokenizer
+        from netis_amniota.agentic.tokenizer_qwen import QwenTokenizer
         return QwenTokenizer.from_directory(tokenizer_dir)
     return RustBPETokenizer.from_directory(tokenizer_dir)
 
 def get_token_bytes(device="cpu"):
     import torch
-    from nanoai.common import get_base_dir
+    from netis_amniota.common import get_base_dir
     base_dir = get_base_dir()
     tokenizer_dir = os.path.join(base_dir, "tokenizer")
     token_bytes_path = os.path.join(tokenizer_dir, "token_bytes.pt")

tokenization_ravenguard.py

@@ -1,6 +1,6 @@
 """RavenGuard tokenizer (trust_remote_code).
 
-Thin ``PreTrainedTokenizer`` wrapper around the nanoai rustbpe/tiktoken encoding
+Thin ``PreTrainedTokenizer`` wrapper around the Netis Amniota rustbpe/tiktoken encoding
 pickled in ``tokenizer.pkl``. ``encode``/``decode`` delegate straight to the
 tiktoken ``Encoding`` so text round-trips byte-for-byte, matching how the guard
 was trained. Vocab size is the tiktoken ``n_vocab`` (65536).

vendor/nanoai/__init__.py

@@ -1,20 +0,0 @@
-"""NanoAI — SOTA micro models, end to end.
-
-Formerly ``nanochat``. For back-compat we mirror the legacy ``NANOCHAT_*``
-environment variables onto their ``NANOAI_*`` equivalents (and vice versa) at
-package import time, so existing launch scripts / remote shells that still set
-``NANOCHAT_*`` keep working unchanged. This runs before any submodule reads an
-env var (importing ``nanoai.x`` executes this ``__init__`` first).
-"""
-import os as _os
-
-_ENV_SUFFIXES = (
-    "BASE_DIR", "DTYPE", "ATTN_BACKEND", "MSA_KERNEL", "PACK_VERIFY",
-    "DIR", "ROOT", "GROUNDED_JSONL", "TRAINING_PCAPS",
-)
-for _suf in _ENV_SUFFIXES:
-    _old, _new = "NANOCHAT_" + _suf, "NANOAI_" + _suf
-    if _new not in _os.environ and _old in _os.environ:
-        _os.environ[_new] = _os.environ[_old]
-    elif _old not in _os.environ and _new in _os.environ:
-        _os.environ[_old] = _os.environ[_new]

vendor/nanoai/agentic/README.md

@@ -1,68 +0,0 @@
-# nanoai/agentic — shared agentic infrastructure
-
-The agentic post-training stack: pretrain on a web+code mix, SFT on tool-call
-conversations, DPO over agentic preferences, a family of RL / RLVR trainers, and
-offline tool-call + diagnostic evals. It stacks with the `--nvfp4` training stack
-and runs alongside the base `chat_sft` / `chat_rl` flow without modifying it.
-
-For day-to-day operation and the full pitfalls log see [CLAUDE.md](../../CLAUDE.md);
-research goals and playbook are in `goal.md` / `goal_next.md`.
-
-## Infra vs experiment boundary (Stage 4 outcome)
-
-This package is **shared agentic infrastructure** — the reusable library that the
-experiment groups under `experiments/<group>/scripts/` all build on. The Stage-4
-audit confirmed every module here is used **across multiple experiment groups or
-by core infra**, so none is scattered into a single group's `lib/` (that would
-create cross-group import edges). Evidence:
-
-| module | used by | verdict |
-|---|---|---|
-| `diag/` (graders, schemas, scenarios) | ksbr_pcap **+ vrmining** | shared |
-| `hard_tasks/` | ksbr_pcap + agentic_rl + mcts_inference | shared |
-| `mcts_inference` / `mcts_exec` / `mcts_hybrid` | mcts_inference + agentic_rl | shared |
-| `prefix_merging` | agentic_rl + polar repro | shared |
-| `opd_loss` | agentic_rl (+ `config.seg_weights` mirrors it) | shared loss lib |
-| `dapt_probes` | **`scripts/base_train`** + ksbr | infra-referenced |
-| `cross_tokenizer_kd` | **`teacher`** (this pkg) | infra-referenced |
-| `reward_*`, `dense_verifier`, `grpo_multi_rollout*`, `tokenizer*`, `teacher`, `value_head`, `turn_*`, `data/` | many | shared |
-
-**The experiment/infra split is therefore: entrypoint scripts/runs/docs live in
-`experiments/<group>/`; this directory is the shared library they import.** Infra
-here imports **no** experiment module (verified). The one infra-referenced
-entrypoint kept in `scripts/` is `eval_llm_judge_rc_hit.py`
-(`reward_capbench` lazily imports `_judge_one` from it).
-
-## Tool-call format (mirrors nanocode)
-
-```
-<|tool_call_start|>Bash<|tool_arg|>command<|tool_val|>ls -la<|tool_call_end|>
-```
-
-6 agentic special tokens extend the 9 base tokens (15 total):
-`<|tool_call_start|>`, `<|tool_arg|>`, `<|tool_val|>`, `<|tool_call_end|>`,
-`<|tool_result_start|>`, `<|tool_result_end|>`. See `tokenizer.py` /
-`scripts/agentic_tok_train.py`.
-
-## Modules
-
-- **Data**: `code_dataloader.py` (web+code Bernoulli mixer), `synth_dataloader.py`,
-  `data/` (json/hf datasets, mixtures, `data/common.py` SYSTEM_PROMPT, pretrain/pcap/ebook/synth iterators).
-- **Tokenizers**: `tokenizer.py` (nanoai), `tokenizer_qwen.py` (Qwen teacher), `cross_tokenizer_kd.py`.
-- **Rewards**: `reward_mlr.py` (multi-level per-turn; weights from `nanoai.config.reward()`),
-  `reward_mt.py`, `reward.py`, `reward_capbench.py`, `dense_verifier.py`, `opd_loss.py` (on-policy distillation).
-- **RL plumbing**: `grpo_multi_rollout{,_batched}.py`, `prefix_merging.py`, `value_head.py`,
-  `turn_critic.py`, `turn_weight.py`, `mc_turn_value.py`, `ocaw.py`.
-- **Inference / search**: `mcts_inference.py`, `mcts_exec.py`, `mcts_hybrid.py`.
-- **Eval**: `eval_loop.py`, `eval_tasks.py`, `eval_pcap_tasks.py`, `eval_sandbox.py`
-  (the sandbox path hard-depends on `firejail` — verify per machine).
-- **diag/**: graders (v1/v2/v3), schemas, chain_adapters, scenario generation.
-- **hard_tasks/**: task families (K-fault / S-wrapper / S-yaml / S-toolcall / SWE) for curriculum + grading.
-
-## Trainers (in `scripts/`)
-
-`agentic_sft.py`, `agentic_dpo.py`, and the RL family
-(`agentic_{gtpo,dapo,real,tree_grpo,grpo_v2,gft_full,…}.py`). They share
-`pack_for_grad` / `setup_optimizer` patterns — see CLAUDE.md's pitfalls
-(per-token advantage off-by-one, single-`--lr` clobber, length-match) before
-editing any of them.

vendor/nanoai/agentic/__init__.py

@@ -1,7 +0,0 @@
-"""Agentic code workflow extensions on top of nanoai.
-
-Adds: code-data mixing in pretrain, tool-call tokenizer, agentic SFT/DPO,
-and a tool-call validation eval. Designed to stack with the existing NVFP4
-training stack — every script accepts --nvfp4 and reuses Float4Linear from
-nanoai.nvfp4.
-"""

vendor/nanoai/agentic/code_dataloader.py

@@ -1,407 +0,0 @@
-"""Code-mixing pretrain dataloader.
-
-Wraps nanoai's BOS-aligned best-fit packing but draws documents from up to
-three parquet sources with a categorical (Dirichlet-multinomial) mixture:
-
-  - web    : nanoai.dataset (climbmix or fineweb-edu, whatever's on disk)
-  - code   : nanoai.agentic.data.pretrain_data → smohammadi/the-stack-v2-python-shuffle
-  - pcap   : nanoai.agentic.data.pcap_data (network/protocol domain corpus)
-
-`code_ratio + pcap_ratio` must be ≤ 1; the remainder is `web_ratio`. Two-way
-defaults (no pcap) preserve the original 2-stream behaviour exactly.
-
-Resume state is intentionally simplified to a single epoch counter — the
-fine-grained (pq_idx, rg_idx) state of the upstream loader doesn't extend
-naturally to interleaved streams. For multi-day runs we'd want real
-multi-stream resume, but for d3/d6/d12 speedruns a fresh start at each step is
-fine.
-"""
-
-from __future__ import annotations
-
-import json
-import os
-import random
-from pathlib import Path
-
-import pyarrow.parquet as pq
-import torch
-
-from nanoai.common import get_dist_info, print0
-from nanoai.dataset import list_parquet_files as list_web_parquet_files
-from nanoai.agentic.data.pretrain_data import DATA_DIR as _CODE_BASE_DATA_DIR
-from nanoai.agentic.data.pcap_data import (
-    DATASET_NAME as _PCAP_DATASET_NAME,
-    list_pcap_parquet_files,
-)
-from nanoai.agentic.data.synth_data import (
-    list_synth_parquet_files,
-    synth_doc_batches,
-)
-from nanoai.agentic.data.vrcpt_data import (
-    DATASET_NAME as _VRCPT_DATASET_NAME,
-    list_vrcpt_parquet_files,
-)
-from nanoai.agentic.data.ebook_data import (
-    DATASET_NAME as _EBOOK_DATASET_NAME,
-    list_ebook_parquet_files,
-)
-
-
-_CODE_DATASET_NAME = "the-stack-v2-dedup"
-# Default location for KSR Phase-2 synth SWE triples; override via $SYNTH_SWE_DIR.
-_SYNTH_SWE_DIR_DEFAULT = Path.home() / "Miros" / "mlros" / "registry_data" / "synth_swe" / "v1"
-
-
-def _list_code_parquet_files() -> list[Path]:
-    code_dir = _CODE_BASE_DATA_DIR / _CODE_DATASET_NAME
-    if not code_dir.exists():
-        return []
-    return sorted([code_dir / f for f in code_dir.iterdir() if f.suffix == ".parquet"])
-
-
-def _list_synth_swe_jsonl_files(synth_swe_dir: Path | None = None) -> list[Path]:
-    """List cpt.jsonl files produced by scripts/synth_swe_triples.py
-    (one per SWE subtype). Empty if Phase-2 synth has not run yet."""
-    d = synth_swe_dir or Path(os.environ.get("SYNTH_SWE_DIR", _SYNTH_SWE_DIR_DEFAULT))
-    if not d.exists():
-        return []
-    return sorted(d.glob("*/cpt.jsonl"))
-
-
-def _jsonl_doc_batches_from(paths, ddp_rank, ddp_world_size, tokenizer_batch_size):
-    """Infinite per-rank iterator over JSONL shards with a `text` field.
-
-    Each line is parsed as JSON and the `text` field extracted. Lines are
-    striped across ranks by index so each rank sees a disjoint subset
-    (no parquet row-group concept here)."""
-    while True:
-        for filepath in paths:
-            with open(filepath, "r", encoding="utf-8") as f:
-                batch: list[str] = []
-                for i, line in enumerate(f):
-                    if i % ddp_world_size != ddp_rank:
-                        continue
-                    line = line.strip()
-                    if not line:
-                        continue
-                    try:
-                        obj = json.loads(line)
-                    except json.JSONDecodeError:
-                        continue
-                    txt = obj.get("text")
-                    if isinstance(txt, str) and txt:
-                        batch.append(txt)
-                    if len(batch) >= tokenizer_batch_size:
-                        yield batch
-                        batch = []
-                if batch:
-                    yield batch
-
-
-def _doc_batches_from(paths, ddp_rank, ddp_world_size, tokenizer_batch_size):
-    """Infinite per-rank iterator: yields list[str] batches."""
-    while True:
-        for filepath in paths:
-            pf = pq.ParquetFile(filepath)
-            for rg_idx in range(ddp_rank, pf.num_row_groups, ddp_world_size):
-                rg = pf.read_row_group(rg_idx)
-                batch = rg.column("text").to_pylist()
-                for i in range(0, len(batch), tokenizer_batch_size):
-                    yield batch[i : i + tokenizer_batch_size]
-
-
-def _split_paths(paths, split):
-    if not paths:
-        return []
-    return paths[:-1] if split == "train" else paths[-1:]
-
-
-def _mixed_document_batches(
-    split: str,
-    code_ratio: float,
-    pcap_ratio: float,
-    tokenizer_batch_size: int,
-    seed: int = 0,
-    synth_swe_ratio: float = 0.0,
-    synth_swe_dir: Path | None = None,
-    synth_pleias_ratio: float = 0.0,
-    vrcpt_ratio: float = 0.0,
-    ebook_ratio: float = 0.0,
-):
-    assert 0.0 <= code_ratio <= 1.0, f"bad code_ratio={code_ratio}"
-    assert 0.0 <= pcap_ratio <= 1.0, f"bad pcap_ratio={pcap_ratio}"
-    assert 0.0 <= synth_swe_ratio <= 1.0, f"bad synth_swe_ratio={synth_swe_ratio}"
-    assert 0.0 <= synth_pleias_ratio <= 1.0, f"bad synth_pleias_ratio={synth_pleias_ratio}"
-    assert 0.0 <= vrcpt_ratio <= 1.0, f"bad vrcpt_ratio={vrcpt_ratio}"
-    assert 0.0 <= ebook_ratio <= 1.0, f"bad ebook_ratio={ebook_ratio}"
-    assert code_ratio + pcap_ratio + synth_swe_ratio + synth_pleias_ratio + vrcpt_ratio + ebook_ratio <= 1.0 + 1e-9, (
-        f"code+pcap+synth_swe+synth_pleias+vrcpt+ebook = "
-        f"{code_ratio+pcap_ratio+synth_swe_ratio+synth_pleias_ratio+vrcpt_ratio+ebook_ratio} > 1"
-    )
-
-    ddp, ddp_rank, _local, ddp_world_size = get_dist_info()
-    warn = (ddp_rank == 0) and (split == "train")
-
-    web_paths = list_web_parquet_files(warn_on_legacy=warn)
-    assert web_paths, "No web (climbmix/fineweb-edu) parquet shards found; run nanoai.dataset first."
-    web_paths = [Path(p) for p in _split_paths(web_paths, split)]
-
-    code_paths = _split_paths(_list_code_parquet_files(), split)
-    if code_ratio > 0 and not code_paths:
-        raise FileNotFoundError(
-            "code_ratio > 0 but no the-stack-v2-dedup parquet found. Run:\n"
-            "  python -m nanoai.agentic.data.pretrain_data -d the-stack-v2-dedup -n 8"
-        )
-
-    pcap_paths = _split_paths(list_pcap_parquet_files(), split)
-    if pcap_ratio > 0 and not pcap_paths:
-        raise FileNotFoundError(
-            f"pcap_ratio > 0 but no {_PCAP_DATASET_NAME} parquet found. Run:\n"
-            "  python -m nanoai.agentic.data.pcap_data prepare --src /path/to/pretrain.jsonl"
-        )
-
-    synth_swe_paths = _list_synth_swe_jsonl_files(synth_swe_dir)
-    synth_swe_paths = _split_paths(synth_swe_paths, split)
-    if synth_swe_ratio > 0 and not synth_swe_paths:
-        raise FileNotFoundError(
-            "synth_swe_ratio > 0 but no synth_swe/v1/<subtype>/cpt.jsonl found. Run:\n"
-            "  python -m scripts.synth_swe_triples --out ~/Miros/mlros/registry_data/synth_swe/v1 --n-per-subtype 500"
-        )
-
-    synth_pleias_paths = _split_paths(list_synth_parquet_files(), split)
-    if synth_pleias_ratio > 0 and not synth_pleias_paths:
-        raise FileNotFoundError(
-            "synth_pleias_ratio > 0 but no synth-pleias parquet found. Expected at "
-            "$NANOAI_BASE_DIR/code_pretrain_data/synth-pleias/ (symlink to PleIAs/SYNTH)."
-        )
-
-    vrcpt_paths = _split_paths(list_vrcpt_parquet_files(), split)
-    if vrcpt_ratio > 0 and not vrcpt_paths:
-        raise FileNotFoundError(
-            f"vrcpt_ratio > 0 but no {_VRCPT_DATASET_NAME} parquet found. Run:\n"
-            "  python -m nanoai.agentic.data.vrcpt_data prepare --src experiments/vrmining/datasets/cpt_corpus.jsonl"
-        )
-
-    ebook_paths = _split_paths([Path(p) for p in list_ebook_parquet_files()], split)
-    if ebook_ratio > 0 and not ebook_paths:
-        raise FileNotFoundError(
-            f"ebook_ratio > 0 but no {_EBOOK_DATASET_NAME} parquet found. Run:\n"
-            "  python -m nanoai.agentic.data.ebook_data prepare --md-root ~/ebook/markdown"
-        )
-
-    web_iter = _doc_batches_from(web_paths, ddp_rank, ddp_world_size, tokenizer_batch_size)
-    code_iter = (
-        _doc_batches_from(code_paths, ddp_rank, ddp_world_size, tokenizer_batch_size)
-        if code_paths
-        else None
-    )
-    pcap_iter = (
-        _doc_batches_from(pcap_paths, ddp_rank, ddp_world_size, tokenizer_batch_size)
-        if pcap_paths
-        else None
-    )
-    synth_swe_iter = (
-        _jsonl_doc_batches_from(synth_swe_paths, ddp_rank, ddp_world_size, tokenizer_batch_size)
-        if synth_swe_paths
-        else None
-    )
-    synth_pleias_iter = (
-        synth_doc_batches(synth_pleias_paths, ddp_rank, ddp_world_size, tokenizer_batch_size)
-        if synth_pleias_paths
-        else None
-    )
-    vrcpt_iter = (
-        _doc_batches_from(vrcpt_paths, ddp_rank, ddp_world_size, tokenizer_batch_size)
-        if vrcpt_paths
-        else None
-    )
-    ebook_iter = (
-        _doc_batches_from(ebook_paths, ddp_rank, ddp_world_size, tokenizer_batch_size)
-        if ebook_paths
-        else None
-    )
-
-    if ddp_rank == 0:
-        web_ratio = max(0.0, 1.0 - code_ratio - pcap_ratio - synth_swe_ratio - synth_pleias_ratio - vrcpt_ratio - ebook_ratio)
-        print0(
-            f"[code-mixer] split={split} "
-            f"ratios web={web_ratio:.2f} code={code_ratio:.2f} pcap={pcap_ratio:.2f} "
-            f"synth_swe={synth_swe_ratio:.2f} synth_pleias={synth_pleias_ratio:.2f} "
-            f"vrcpt={vrcpt_ratio:.2f} ebook={ebook_ratio:.2f} "
-            f"shards web={len(web_paths)} code={len(code_paths)} "
-            f"pcap={len(pcap_paths)} synth_swe={len(synth_swe_paths)} "
-            f"synth_pleias={len(synth_pleias_paths)} vrcpt={len(vrcpt_paths)} ebook={len(ebook_paths)}"
-        )
-
-    rng = random.Random(seed + ddp_rank)
-    counts = {"web": 0, "code": 0, "pcap": 0, "synth_swe": 0, "synth_pleias": 0, "vrcpt": 0, "ebook": 0}
-    log_every = 10000
-    # Decision order: synth_swe, synth_pleias, vrcpt, ebook, pcap, code, web — additive slices of [0,1).
-    while True:
-        r = rng.random()
-        if synth_swe_iter is not None and r < synth_swe_ratio:
-            counts["synth_swe"] += 1
-            yield next(synth_swe_iter)
-        elif synth_pleias_iter is not None and r < synth_swe_ratio + synth_pleias_ratio:
-            counts["synth_pleias"] += 1
-            yield next(synth_pleias_iter)
-        elif vrcpt_iter is not None and r < synth_swe_ratio + synth_pleias_ratio + vrcpt_ratio:
-            counts["vrcpt"] += 1
-            yield next(vrcpt_iter)
-        elif ebook_iter is not None and r < synth_swe_ratio + synth_pleias_ratio + vrcpt_ratio + ebook_ratio:
-            counts["ebook"] += 1
-            yield next(ebook_iter)
-        elif pcap_iter is not None and r < synth_swe_ratio + synth_pleias_ratio + vrcpt_ratio + ebook_ratio + pcap_ratio:
-            counts["pcap"] += 1
-            yield next(pcap_iter)
-        elif code_iter is not None and r < synth_swe_ratio + synth_pleias_ratio + vrcpt_ratio + ebook_ratio + pcap_ratio + code_ratio:
-            counts["code"] += 1
-            yield next(code_iter)
-        else:
-            counts["web"] += 1
-            yield next(web_iter)
-        total = sum(counts.values())
-        if ddp_rank == 0 and total % log_every == 0:
-            print0(
-                f"[code-mixer] draws={total} "
-                f"web={counts['web']/total:.3f} code={counts['code']/total:.3f} "
-                f"pcap={counts['pcap']/total:.3f} synth_swe={counts['synth_swe']/total:.3f} "
-                f"synth_pleias={counts['synth_pleias']/total:.3f} vrcpt={counts['vrcpt']/total:.3f} "
-                f"ebook={counts['ebook']/total:.3f}"
-            )
-
-
-def tokenizing_code_mixed_data_loader(
-    tokenizer,
-    B: int,
-    T: int,
-    split: str,
-    code_ratio: float = 0.2,
-    pcap_ratio: float = 0.0,
-    synth_swe_ratio: float = 0.0,
-    synth_swe_dir: Path | None = None,
-    synth_pleias_ratio: float = 0.0,
-    vrcpt_ratio: float = 0.0,
-    ebook_ratio: float = 0.0,
-    tokenizer_threads: int = 4,
-    tokenizer_batch_size: int = 128,
-    device: str = "cuda",
-    buffer_size: int = 1000,
-    seed: int = 0,
-):
-    """BOS-aligned best-fit packing over a (web, code, pcap, synth_swe, synth_pleias, vrcpt, ebook) mixture.
-
-    Yields (inputs, targets) GPU tensors of shape (B, T). Drop-in for
-    nanoai's `tokenizing_distributed_data_loader_bos_bestfit`.
-    """
-    assert split in ("train", "val")
-    row_capacity = T + 1
-    bos_token = tokenizer.get_bos_token_id()
-
-    batches = _mixed_document_batches(
-        split, code_ratio, pcap_ratio, tokenizer_batch_size, seed=seed,
-        synth_swe_ratio=synth_swe_ratio, synth_swe_dir=synth_swe_dir,
-        synth_pleias_ratio=synth_pleias_ratio,
-        vrcpt_ratio=vrcpt_ratio,
-        ebook_ratio=ebook_ratio,
-    )
-    doc_buffer: list[list[int]] = []
-
-    def refill_buffer():
-        doc_batch = next(batches)
-        token_lists = tokenizer.encode(
-            doc_batch, prepend=bos_token, num_threads=tokenizer_threads
-        )
-        for tokens in token_lists:
-            doc_buffer.append(tokens)
-
-    use_cuda = device == "cuda"
-    row_buffer = torch.empty((B, row_capacity), dtype=torch.long)
-    cpu_buffer = torch.empty(2 * B * T, dtype=torch.long, pin_memory=use_cuda)
-    gpu_buffer = torch.empty(2 * B * T, dtype=torch.long, device=device)
-    cpu_inputs = cpu_buffer[: B * T].view(B, T)
-    cpu_targets = cpu_buffer[B * T :].view(B, T)
-    inputs = gpu_buffer[: B * T].view(B, T)
-    targets = gpu_buffer[B * T :].view(B, T)
-
-    while True:
-        for row_idx in range(B):
-            pos = 0
-            while pos < row_capacity:
-                while len(doc_buffer) < buffer_size:
-                    refill_buffer()
-                remaining = row_capacity - pos
-
-                best_idx = -1
-                best_len = 0
-                for i, doc in enumerate(doc_buffer):
-                    doc_len = len(doc)
-                    if doc_len <= remaining and doc_len > best_len:
-                        best_idx = i
-                        best_len = doc_len
-
-                if best_idx >= 0:
-                    doc = doc_buffer.pop(best_idx)
-                    doc_len = len(doc)
-                    row_buffer[row_idx, pos : pos + doc_len] = torch.tensor(doc, dtype=torch.long)
-                    pos += doc_len
-                else:
-                    shortest_idx = min(range(len(doc_buffer)), key=lambda i: len(doc_buffer[i]))
-                    doc = doc_buffer.pop(shortest_idx)
-                    row_buffer[row_idx, pos : pos + remaining] = torch.tensor(
-                        doc[:remaining], dtype=torch.long
-                    )
-                    pos += remaining
-
-        cpu_inputs.copy_(row_buffer[:, :-1])
-        cpu_targets.copy_(row_buffer[:, 1:])
-        gpu_buffer.copy_(cpu_buffer, non_blocking=use_cuda)
-        yield inputs, targets
-
-
-def tokenizing_code_mixed_data_loader_with_state(
-    tokenizer,
-    B: int,
-    T: int,
-    split: str,
-    code_ratio: float = 0.2,
-    pcap_ratio: float = 0.0,
-    synth_swe_ratio: float = 0.0,
-    synth_swe_dir: Path | None = None,
-    synth_pleias_ratio: float = 0.0,
-    vrcpt_ratio: float = 0.0,
-    ebook_ratio: float = 0.0,
-    tokenizer_threads: int = 4,
-    tokenizer_batch_size: int = 128,
-    device: str = "cuda",
-    resume_state_dict=None,
-    buffer_size: int = 1000,
-    seed: int = 0,
-):
-    """Same as `tokenizing_code_mixed_data_loader` but yields a third element
-    `state_dict` to match the signature of nanoai's
-    `tokenizing_distributed_data_loader_with_state_bos_bestfit`.
-
-    The state_dict is a stub (`{"epoch": 1, "pq_idx": 0, "rg_idx": 0}`) — true
-    resume across the multi-stream mixture is not implemented. Caller's
-    `resume_state_dict` is ignored.
-    """
-    base = tokenizing_code_mixed_data_loader(
-        tokenizer, B, T, split,
-        code_ratio=code_ratio,
-        pcap_ratio=pcap_ratio,
-        synth_swe_ratio=synth_swe_ratio,
-        synth_swe_dir=synth_swe_dir,
-        synth_pleias_ratio=synth_pleias_ratio,
-        vrcpt_ratio=vrcpt_ratio,
-        ebook_ratio=ebook_ratio,
-        tokenizer_threads=tokenizer_threads,
-        tokenizer_batch_size=tokenizer_batch_size,
-        device=device,
-        buffer_size=buffer_size,
-        seed=seed,
-    )
-    stub_state = {"epoch": 1, "pq_idx": 0, "rg_idx": 0}
-    for inputs, targets in base:
-        yield inputs, targets, stub_state

vendor/nanoai/agentic/cross_tokenizer_kd.py

@@ -1,225 +0,0 @@
-"""Byte-level alignment between heterogeneous BPE tokenizers (e.g. nanoai vs Qwen).
-
-Use case: the OPD student emits a sequence of nanoai token ids; we want
-per-token teacher logp from a teacher that uses a DIFFERENT BPE tokenizer.
-
-Approach:
-  1. Both tokenizers decode their token sequences to the SAME byte string `s`.
-     - nanoai: `RustBPETokenizer.enc.decode_single_token_bytes(tid)` returns
-       the raw bytes for one BPE merge. Concatenating these for the full
-       sequence reproduces the original byte string (modulo special tokens).
-     - HF (Qwen): `tokenizer.convert_ids_to_tokens(ids)` gives sub-strings; we
-       cumulate bytes through `decode(ids)` of each prefix to get offsets.
-  2. Each token gets a byte interval `[start, end)` in `s`. We then sum the
-     teacher (Qwen) per-token logps that fall inside the student token's
-     interval — by the chain rule, sum of consecutive log p(y_t | y_<t) is the
-     joint log p over the byte span.
-
-Edge cases:
-  - **Special tokens** (`<|tool_call_start|>` etc.): nanoai reserves these as
-    single ids that decode to multi-char strings (`<|...|>`), which Qwen will
-    re-tokenize into ~5-8 BPE tokens. The byte interval still covers the same
-    bytes, so the sum still aligns correctly. The CALLER may mask these
-    positions out of the OPD loss because the teacher has no special-token
-    semantic for them — we expose `is_special_token` per offset for that.
-  - **Misalignment at byte level**: nanoai tokenizer may emit bytes that are
-    invalid UTF-8 (in the middle of a multi-byte char). When we concat to
-    build the byte string and Qwen tries to re-tokenize via decode→bytes, the
-    boundary may shift by a few bytes. We tolerate up to 1-byte mismatch via
-    fuzzy match in the alignment loop.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import List, Optional, Tuple
-
-
-@dataclass
-class TokenSpan:
-    """Byte interval [start, end) and decoded text of one BPE token."""
-    token_id: int
-    start: int   # byte offset (inclusive)
-    end: int     # byte offset (exclusive)
-    text: str    # decoded bytes (lossy for non-utf8)
-    is_special: bool
-
-
-def compute_nanochat_offsets(
-    tokenizer, token_ids: List[int]
-) -> Tuple[List[TokenSpan], bytes]:
-    """Compute byte offsets for a nanoai token sequence.
-
-    Returns (spans, byte_string) where spans[i] is the TokenSpan for token i.
-    The byte_string is the concatenation of all per-token bytes — when the
-    student emitted a "normal" text trace this equals the natural utf-8
-    encoding of the decoded text.
-
-    For special tokens (BOS, tool_call_start, etc.), we use the rendered
-    `<|name|>` form so the Qwen tokenizer re-tokenizes them into multiple BPE
-    units; this is correct as long as the byte boundary aligns to the rendered
-    form's edges, which it always does because tiktoken stores specials as
-    these exact ascii strings internally.
-    """
-    spans: List[TokenSpan] = []
-    cur_bytes = bytearray()
-
-    # Qwen path: HF-backed wrapper. id_to_token returns the special-token name
-    # (e.g. '<|im_start|>') for added tokens, or the BPE piece for regular
-    # tokens. Decoding the full id list with skip_special_tokens=False produces
-    # the same string as concatenating per-token decode_single_token; we use
-    # per-token decode to get accurate byte spans.
-    if getattr(tokenizer, "chat_kind", None) == "qwen":
-        hf = tokenizer.tok
-        added = hf.added_tokens_decoder if hasattr(hf, "added_tokens_decoder") else {}
-        for tid in token_ids:
-            start = len(cur_bytes)
-            if tid in added:
-                # Added/special token — emit its surface form as bytes.
-                text = added[tid].content if hasattr(added[tid], "content") else str(added[tid])
-                tb = text.encode("utf-8")
-                is_special = True
-            else:
-                # Regular BPE token. decode([tid]) returns its surface string.
-                text = hf.decode([tid], skip_special_tokens=False)
-                tb = text.encode("utf-8")
-                is_special = False
-            cur_bytes.extend(tb)
-            end = len(cur_bytes)
-            spans.append(TokenSpan(tid, start, end, text, is_special))
-        return spans, bytes(cur_bytes)
-
-    # nanoai tiktoken path.
-    enc = tokenizer.enc  # tiktoken.Encoding
-
-    # Build a set of special-token ids for fast lookup.
-    special_ids = set(enc._special_tokens.values()) if hasattr(enc, "_special_tokens") else set()
-
-    for tid in token_ids:
-        start = len(cur_bytes)
-        if tid in special_ids:
-            # Render the special token as its name string (e.g. "<|bos|>")
-            # so byte alignment is well-defined.
-            name = None
-            for k, v in enc._special_tokens.items():
-                if v == tid:
-                    name = k
-                    break
-            text = name or f"<|sp_{tid}|>"
-            tb = text.encode("utf-8")
-            is_special = True
-        else:
-            tb = enc.decode_single_token_bytes(tid)
-            try:
-                text = tb.decode("utf-8")
-            except UnicodeDecodeError:
-                text = tb.decode("utf-8", errors="replace")
-            is_special = False
-        cur_bytes.extend(tb)
-        end = len(cur_bytes)
-        spans.append(TokenSpan(tid, start, end, text, is_special))
-
-    return spans, bytes(cur_bytes)
-
-
-def compute_hf_offsets_from_decoded(
-    token_ids: List[int], decoded_tokens: List[str]
-) -> List[TokenSpan]:
-    """Compute byte offsets directly from pre-decoded token strings.
-
-    vLLM's `prompt_logprobs` response includes `decoded_token` per position,
-    e.g. " capital", " of", " France". We use those directly to build byte
-    offsets, sidestepping the need for the HF tokenizer to be present locally.
-
-    token_ids and decoded_tokens must be parallel, same length.
-    """
-    assert len(token_ids) == len(decoded_tokens)
-    spans: List[TokenSpan] = []
-    cur_bytes = bytearray()
-    for tid, text in zip(token_ids, decoded_tokens):
-        start = len(cur_bytes)
-        tb = text.encode("utf-8")
-        cur_bytes.extend(tb)
-        end = len(cur_bytes)
-        spans.append(TokenSpan(tid, start, end, text, False))
-    return spans
-
-
-def align_logps(
-    target_spans: List[TokenSpan],
-    source_spans: List[TokenSpan],
-    source_logps: List[Optional[float]],
-) -> List[Optional[float]]:
-    """Distribute source logps proportionally over each target byte interval.
-
-    target_spans: e.g. nanoai student tokens — we want one logp per target.
-    source_spans: e.g. Qwen teacher tokens — we have one logp per source.
-    source_logps: list of float (or None for source position 0,
-                  which has no logp because there's no prior context).
-
-    Returns: list of Optional[float], length == len(target_spans). None means
-    "couldn't compute" (target span overlapping with source[0] which has
-    no logp, or no source coverage at all).
-
-    Bug-fix note (codex adversarial review 2026-05-18, finding #2):
-    Previously, when a single source (teacher) token spanned two target
-    (student) byte ranges, its FULL logp was added to BOTH targets — a
-    silent 2x double-counting. We now allocate by overlap fraction:
-        target[i] += source[k].logp × (overlap_bytes / source_byte_length)
-    This preserves the chain-rule invariant Σ_target_in_S aligned ≈ S.logp.
-    A target is invalid (None) only if no overlap or any overlapping source
-    has missing logp.
-    """
-    aligned: List[Optional[float]] = [None] * len(target_spans)
-    n_src = len(source_spans)
-
-    # Two-pointer scan.
-    j = 0
-    for i, t in enumerate(target_spans):
-        # Advance j past any source token strictly before t.
-        while j < n_src and source_spans[j].end <= t.start:
-            j += 1
-        # Distribute source logps that overlap t, weighted by overlap fraction.
-        total = 0.0
-        any_overlap = False
-        any_missing = False
-        k = j
-        while k < n_src and source_spans[k].start < t.end:
-            s = source_spans[k]
-            ov = min(t.end, s.end) - max(t.start, s.start)
-            if ov > 0:
-                if source_logps[k] is None:
-                    any_missing = True
-                else:
-                    s_len = s.end - s.start
-                    weight = (ov / s_len) if s_len > 0 else 1.0
-                    total += source_logps[k] * weight
-                    any_overlap = True
-            k += 1
-        if any_overlap and not any_missing:
-            aligned[i] = total
-        else:
-            aligned[i] = None  # either no coverage or hit the unknown source[0]
-    return aligned
-
-
-def align_qwen_logp_to_nano(
-    nano_tokenizer,
-    nano_token_ids: List[int],
-    qwen_token_ids: List[int],
-    qwen_decoded_tokens: List[str],
-    qwen_logps: List[Optional[float]],
-) -> Tuple[List[Optional[float]], List[bool]]:
-    """One-call convenience wrapper.
-
-    Returns (aligned_logps, is_special_mask) for the nanoai sequence.
-    aligned_logps[i] = sum of qwen_logps over qwen tokens that cover nano
-                       token i's bytes, or None if can't align.
-    is_special_mask[i] = True iff nano token i is a special token (caller
-                       should typically mask these out of OPD loss).
-    """
-    nano_spans, _ = compute_nanochat_offsets(nano_tokenizer, nano_token_ids)
-    qwen_spans = compute_hf_offsets_from_decoded(qwen_token_ids, qwen_decoded_tokens)
-    aligned = align_logps(nano_spans, qwen_spans, qwen_logps)
-    specials = [s.is_special for s in nano_spans]
-    return aligned, specials

vendor/nanoai/agentic/dapt_probes.py

@@ -1,194 +0,0 @@
-"""DAPT/CPT in-training probes shared with `scripts.base_train`.
-
-Pure functions kept outside the training script so they can be unit-tested
-without booting the full DDP / NVFP4 / wandb stack. Each helper is small,
-side-effect-free except for the obvious (parquet I/O, model.eval()), and
-exposes verbose `print0` logging so an operator watching the train log can
-reconstruct exactly what the probe did.
-
-Three helpers:
-
-  - `build_domain_val_loader(domain, ...)` — return a single-source val loader
-    over web | code | pcap, regardless of the active training mix. Used to
-    compute per-domain BPB during eval ticks.
-
-  - `load_l0_tasks(repo_root)` — read tshark_eval_agentic L0 vocab tasks. Memo-
-    izes the parse so repeated probes don't re-read the JSONL.
-
-  - `run_l0_probe(model_for_gen, tokenizer, ..., disable_fp8_ctx=...)` — greedy
-    decode N L0 questions with a 3-shot prefix, return string-match accuracy.
-"""
-
-from __future__ import annotations
-
-import json
-import random
-import time
-from contextlib import nullcontext
-from pathlib import Path
-from typing import Callable, ContextManager, Optional
-
-from nanoai.common import print0
-
-
-# -----------------------------------------------------------------------------
-# Per-domain val loader
-
-DOMAIN_RATIOS = {
-    "web":  {"code_ratio": 0.0, "pcap_ratio": 0.0},
-    "code": {"code_ratio": 1.0, "pcap_ratio": 0.0},
-    "pcap": {"code_ratio": 0.0, "pcap_ratio": 1.0},
-}
-
-
-def build_domain_val_loader(domain: str, tokenizer, device_batch_size: int,
-                            max_seq_len: int, device: str):
-    """Construct a single-source val loader for {web, code, pcap}.
-
-    Bypasses any monkey-patched train loader so per-domain BPB is unaffected by
-    the active training mixture. Returns the same yield shape as the patched
-    loader (yields (inputs, targets) — no state_dict).
-    """
-    assert domain in DOMAIN_RATIOS, (
-        f"unknown domain {domain!r}, expected one of {sorted(DOMAIN_RATIOS)}"
-    )
-    # Local import: code_dataloader pulls in pyarrow, torch, etc. — keep that
-    # cost off the import path of any caller that doesn't actually probe.
-    from nanoai.agentic.code_dataloader import tokenizing_code_mixed_data_loader
-
-    ratios = DOMAIN_RATIOS[domain]
-    print0(f"[dapt-probe] building val loader for domain={domain!r} ratios={ratios}")
-    return tokenizing_code_mixed_data_loader(
-        tokenizer, device_batch_size, max_seq_len, split="val",
-        code_ratio=ratios["code_ratio"],
-        pcap_ratio=ratios["pcap_ratio"],
-        device=device,
-    )
-
-
-# -----------------------------------------------------------------------------
-# L0 probe
-
-_DEFAULT_L0_PATH = Path(__file__).resolve().parents[2] / "tshark_eval_agentic" / "eval" / "tasks_l0_vocab.jsonl"
-
-_L0_CACHE: dict[Path, list[dict]] = {}
-
-
-def load_l0_tasks(path: Optional[Path] = None) -> list[dict]:
-    """Read experiments/tshark_eval_agentic/eval/tasks_l0_vocab.jsonl into a list of dicts.
-
-    Caches per-path so repeated probes don't re-read the file. Returns [] if
-    the file is missing (probe is then a no-op).
-    """
-    p = path if path is not None else _DEFAULT_L0_PATH
-    p = Path(p)
-    if p in _L0_CACHE:
-        return _L0_CACHE[p]
-    if not p.exists():
-        print0(f"[dapt-probe] L0 task file not found at {p}; probe will be a no-op.")
-        _L0_CACHE[p] = []
-        return _L0_CACHE[p]
-    tasks: list[dict] = []
-    with p.open("r", encoding="utf-8") as f:
-        for line_no, line in enumerate(f, 1):
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                obj = json.loads(line)
-            except json.JSONDecodeError as e:
-                print0(f"[dapt-probe] L0 task file line {line_no} not JSON: {e}; skipping.")
-                continue
-            if "prompt" in obj and "expected" in obj:
-                tasks.append(obj)
-    print0(f"[dapt-probe] loaded {len(tasks)} L0 tasks from {p}")
-    _L0_CACHE[p] = tasks
-    return tasks
-
-
-L0_FEW_SHOT = (
-    "Q: What is the tshark display-filter field name for: Source IP address?\nA: ip.src\n"
-    "Q: What is the tshark display-filter field name for: TCP destination port?\nA: tcp.dstport\n"
-    "Q: What is the tshark display-filter field name for: HTTP request method?\nA: http.request.method\n"
-)
-
-
-def _first_line(text: str) -> str:
-    """Return the first non-empty line of `text`, stripped."""
-    for line in text.strip().splitlines():
-        s = line.strip()
-        if s:
-            return s
-    return ""
-
-
-def l0_match(expected: str, generation: str) -> bool:
-    """Lenient match: `expected` (case-insensitive) appears in the first
-    generated line. Mirrors `experiments/tshark_eval_agentic/eval/grader.exact_str_lower`
-    behaviour for the vocab tasks (which always ask for a single token like
-    `tcp.flags.syn`).
-    """
-    return expected.lower() in _first_line(generation).lower()
-
-
-def run_l0_probe(
-    model_for_gen,
-    tokenizer,
-    Engine,
-    num_questions: int,
-    max_tokens: int,
-    disable_fp8_ctx: Optional[Callable[[object], ContextManager]] = None,
-    seed: int = 42,
-    debug: bool = False,
-    task_path: Optional[Path] = None,
-) -> Optional[float]:
-    """Greedy-decode N L0 vocab questions and return string-match accuracy.
-
-    Returns None if no L0 tasks file is available, else accuracy in [0,1].
-
-    `Engine` is passed in (rather than imported) so this function stays
-    importable without booting nanoai.engine — the test path can pass a
-    stub Engine.
-    `disable_fp8_ctx` is the context manager factory (usually `disable_fp8`
-    from base_train.py); pass None to disable the swap.
-    """
-    tasks = load_l0_tasks(task_path)
-    if not tasks:
-        return None
-    rng = random.Random(seed)
-    sample = rng.sample(tasks, min(num_questions, len(tasks)))
-    print0(
-        f"[dapt-probe] running L0 probe: {len(sample)} questions, "
-        f"max_tokens={max_tokens}, seed={seed}"
-    )
-
-    engine = Engine(model_for_gen, tokenizer)
-    correct = 0
-    t0 = time.time()
-    bos = "<|bos|>"
-    ctx_factory = disable_fp8_ctx if disable_fp8_ctx is not None else (lambda _m: nullcontext())
-    for i, t in enumerate(sample):
-        prompt_text = L0_FEW_SHOT + f"Q: {t['prompt']}\nA:"
-        ids = tokenizer(prompt_text, prepend=bos)
-        with ctx_factory(model_for_gen):
-            out, _ = engine.generate_batch(
-                ids, num_samples=1, max_tokens=max_tokens, temperature=0,
-            )
-        gen_ids = out[0][len(ids):] if len(out[0]) > len(ids) else out[0]
-        gen_text = tokenizer.decode(gen_ids)
-        hit = l0_match(t["expected"], gen_text)
-        if hit:
-            correct += 1
-        if debug:
-            print0(
-                f"[dapt-probe][L0 {i+1:02d}/{len(sample)}] "
-                f"expected={t['expected']!r}  gen_first_line={_first_line(gen_text)!r}  "
-                f"{'✓' if hit else '✗'}"
-            )
-    dt = time.time() - t0
-    acc = correct / len(sample)
-    print0(
-        f"[dapt-probe] L0 probe done: {correct}/{len(sample)} = {acc:.3f} "
-        f"({dt:.1f}s, {dt/len(sample):.2f}s/q)"
-    )
-    return acc

vendor/nanoai/agentic/data/__init__.py

@@ -1 +0,0 @@
-"""Conversation/preference datasets used by agentic SFT and DPO."""

vendor/nanoai/agentic/data/codedebug.py

@@ -1,359 +0,0 @@
-"""Multi-turn execute-debug datasets for v5 SFT.
-
-Goal: give the model real `tool_call → observation → tool_call → ...` loops
-beyond tulu's Edit-only tool-call shape (the v4 Edit-bias root cause).
-
-Two sources, both folded into the agentic chat format used by
-`render_agentic_conversation`'s list-content branch:
-
-- `xingyaoww/code-act` (codeact split): assistant emits `<execute>...</execute>`,
-  user emits `Observation: ...`. ~7k rows.
-- `m-a-p/Code-Feedback` (filtered to python with exec output): assistant emits
-  ` ```python ... ``` `, user emits `Execution result: ...`. ~10k after filter.
-
-Output row shape (per `__getitem__`):
-  {"messages": [
-    {"role": "system",    "content": SYSTEM_PROMPT},
-    {"role": "user",      "content": "..."},
-    {"role": "assistant", "content": [{"type":"text"|"python"|"python_output", "text": ...}, ...]},
-    ...
-  ]}
-
-The list-content branch wraps `python` segments in `<|python_start|>/<|python_end|>`
-(supervised) and `python_output` in `<|output_start|>/<|output_end|>` (unsupervised).
-"""
-
-from __future__ import annotations
-
-import os
-import random
-import re
-from typing import Any, Iterable
-
-from datasets import load_dataset
-
-from nanoai.agentic.data.common import SYSTEM_PROMPT
-
-
-# ---------------------------------------------------------------------------
-# Parsers
-# ---------------------------------------------------------------------------
-
-# Code-act assistant action: <execute> ... </execute>. Solutions stay as text.
-_EXECUTE_RE = re.compile(r"<execute>(.*?)</execute>", re.DOTALL)
-
-# Code-Feedback assistant action: ```python ... ``` (case-insensitive, leading ws ok).
-# We deliberately match only python (Ruby/JS/etc. blocks stay as text — caller
-# filters those rows out earlier so they don't end up here).
-_PYBLOCK_RE = re.compile(r"```\s*[Pp]ython\s*\n(.*?)```", re.DOTALL)
-
-# Observation prefixes folded into the preceding assistant turn as python_output.
-_OBS_PREFIXES = (
-    "observation:",
-    "execution result:",
-    "execution output:",
-    "stdout:",
-)
-
-# Skip Code-Feedback rows whose "execution" is actually a Jupyter cell error trace —
-# those teach the model how-to-format-a-traceback, not how-to-debug.
-_JUPYTER_ERROR_MARKERS = ("Cell In[", "<ipython-input-", "ipykernel_")
-
-# Skip rows with non-python code blocks above a threshold (Code-Feedback contains
-# Ruby/JS/SQL etc.). Cheap heuristic: look for ```ruby/```javascript/etc. in any
-# assistant content; if present and ```python is absent, skip.
-_NON_PYTHON_BLOCK_RE = re.compile(
-    r"```\s*(?:ruby|javascript|js|java|c\+\+|cpp|sql|go|rust|php|kotlin|scala)\b",
-    re.IGNORECASE,
-)
-
-
-def _split_assistant_text(text: str, code_re: re.Pattern[str]) -> list[dict[str, str]]:
-    """Split an assistant message into ordered text/python parts.
-
-    The regex matches code-fenced or tag-fenced spans; everything else stays text.
-    Returns [] if the input is empty after stripping.
-    """
-    if not text or not text.strip():
-        return []
-    parts: list[dict[str, str]] = []
-    pos = 0
-    for m in code_re.finditer(text):
-        head = text[pos : m.start()]
-        if head.strip():
-            parts.append({"type": "text", "text": head.strip()})
-        body = m.group(1).strip("\n").rstrip()
-        if body:
-            parts.append({"type": "python", "text": body})
-        pos = m.end()
-    tail = text[pos:]
-    if tail.strip():
-        parts.append({"type": "text", "text": tail.strip()})
-    return parts
-
-
-def _extract_observation(content: str) -> str | None:
-    """If content is a pure observation, return the body sans prefix; else None."""
-    if not content:
-        return None
-    stripped = content.lstrip()
-    lower = stripped.lower()
-    for prefix in _OBS_PREFIXES:
-        if lower.startswith(prefix):
-            return stripped[len(prefix):].lstrip("\n").rstrip() or " "
-    return None
-
-
-def _fold_into_assistant(messages: list[dict[str, Any]], code_re: re.Pattern[str]) -> list[dict[str, Any]]:
-    """Walk raw user/assistant alternation; return the list-content cleaned form.
-
-    Rules:
-      * Drop system messages — caller injects SYSTEM_PROMPT.
-      * Assistant with code blocks → list-content [{text|python} parts].
-      * Assistant without code → plain str content.
-      * User that is purely an observation following an assistant with code →
-        appended as a {type: python_output} part to the preceding assistant.
-      * Anything else stays user/assistant.
-      * Drop trailing user-only turns (no completion to supervise).
-      * Drop rows whose first non-system role isn't user.
-    """
-    cleaned: list[dict[str, Any]] = []
-
-    for msg in messages:
-        role = msg.get("role")
-        content = msg.get("content", "") or ""
-        if role == "system":
-            continue
-
-        if role == "assistant":
-            parts = _split_assistant_text(content, code_re)
-            if not parts:
-                continue
-            if any(p["type"] == "python" for p in parts):
-                cleaned.append({"role": "assistant", "content": parts})
-            else:
-                # Plain prose answer — keep as a single string for the simpler
-                # render path. Re-join the text parts.
-                cleaned.append({"role": "assistant", "content": " ".join(p["text"] for p in parts)})
-            continue
-
-        if role == "user":
-            obs = _extract_observation(content)
-            if (
-                obs is not None
-                and cleaned
-                and cleaned[-1]["role"] == "assistant"
-                and isinstance(cleaned[-1]["content"], list)
-                and any(p["type"] == "python" for p in cleaned[-1]["content"])
-            ):
-                cleaned[-1]["content"].append({"type": "python_output", "text": obs})
-                continue
-            cleaned.append({"role": "user", "content": content.strip()})
-            continue
-
-        # Unknown role (tool_result, function, etc.) — skip silently.
-
-    # Drop leading non-user turns and trailing non-assistant turns.
-    while cleaned and cleaned[0]["role"] != "user":
-        cleaned.pop(0)
-    while cleaned and cleaned[-1]["role"] != "assistant":
-        cleaned.pop()
-
-    # Folding observations leaves consecutive assistants (the original
-    # post-observation assistant continuation). Same can happen with consecutive
-    # users that don't satisfy the observation-fold criterion. Merge them.
-    return _merge_consecutive_same_role(cleaned)
-
-
-def _merge_consecutive_same_role(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    """Merge runs of same-role messages into one. Strings → list-content if needed."""
-    out: list[dict[str, Any]] = []
-    for m in messages:
-        if out and out[-1]["role"] == m["role"]:
-            prev = out[-1]
-            if m["role"] == "user":
-                prev_text = prev["content"] if isinstance(prev["content"], str) else " ".join(
-                    p["text"] for p in prev["content"]
-                )
-                cur_text = m["content"] if isinstance(m["content"], str) else " ".join(
-                    p["text"] for p in m["content"]
-                )
-                prev["content"] = (prev_text + "\n\n" + cur_text).strip()
-                continue
-            # assistant: prefer list-content if either side already is.
-            prev_content = prev["content"]
-            cur_content = m["content"]
-            if isinstance(prev_content, str) and isinstance(cur_content, str):
-                prev["content"] = (prev_content + " " + cur_content).strip()
-                continue
-            if isinstance(prev_content, str):
-                prev_content = [{"type": "text", "text": prev_content}] if prev_content.strip() else []
-            if isinstance(cur_content, str):
-                cur_content = [{"type": "text", "text": cur_content}] if cur_content.strip() else []
-            prev["content"] = list(prev_content) + list(cur_content)
-            continue
-        out.append(m)
-    return out
-
-
-def parse_codeact_row(row: dict[str, Any], max_chars: int = 16384) -> dict[str, Any] | None:
-    """Convert a single code-act row to {messages: [...]} or None to skip."""
-    raw = row.get("conversations") or []
-    if not raw:
-        return None
-    messages = _fold_into_assistant(raw, _EXECUTE_RE)
-    if not _is_well_formed(messages, max_chars):
-        return None
-    return {"messages": messages}
-
-
-def parse_codefeedback_row(row: dict[str, Any], max_chars: int = 16384) -> dict[str, Any] | None:
-    """Convert a single Code-Feedback row to {messages: [...]} or None to skip."""
-    raw = row.get("messages") or []
-    if not raw:
-        return None
-    full_text = "\n".join((m.get("content") or "") for m in raw)
-    if len(full_text) > max_chars:
-        return None
-    if any(marker in full_text for marker in _JUPYTER_ERROR_MARKERS):
-        return None
-    if "execution result" not in full_text.lower() and "execution output" not in full_text.lower():
-        return None
-    if _NON_PYTHON_BLOCK_RE.search(full_text) and "```python" not in full_text.lower():
-        return None
-    messages = _fold_into_assistant(raw, _PYBLOCK_RE)
-    if not _is_well_formed(messages, max_chars):
-        return None
-    # Require at least one observation actually got folded in — otherwise this
-    # is just chat about code, not execute-debug.
-    if not _has_python_output(messages):
-        return None
-    return {"messages": messages}
-
-
-def _is_well_formed(messages: list[dict[str, Any]], max_chars: int) -> bool:
-    if len(messages) < 2:
-        return False
-    # Strict alternation user/assistant.
-    for i, m in enumerate(messages):
-        expected = "user" if i % 2 == 0 else "assistant"
-        if m["role"] != expected:
-            return False
-    if messages[-1]["role"] != "assistant":
-        return False
-    total = 0
-    for m in messages:
-        c = m["content"]
-        if isinstance(c, str):
-            total += len(c)
-        else:
-            total += sum(len(p["text"]) for p in c)
-    return total <= max_chars
-
-
-def _has_python_output(messages: list[dict[str, Any]]) -> bool:
-    for m in messages:
-        if m["role"] == "assistant" and isinstance(m["content"], list):
-            if any(p["type"] == "python_output" for p in m["content"]):
-                return True
-    return False
-
-
-# ---------------------------------------------------------------------------
-# Dataset wrappers (HF-backed)
-# ---------------------------------------------------------------------------
-
-
-def _set_hf_endpoint() -> None:
-    """If HF_ENDPOINT isn't set and the public hub is unreachable, fall back to
-    hf-mirror.com. We don't probe network here — caller is expected to set
-    HF_ENDPOINT in env when behind a firewall."""
-    # Intentionally a no-op; documented for callers.
-    return
-
-
-class CodeActDataset:
-    """xingyaoww/code-act, codeact split — multi-turn data analysis with sqlite."""
-
-    def __init__(self, split: str = "codeact", seed: int = 42, max_chars: int = 16384) -> None:
-        _set_hf_endpoint()
-        ds = load_dataset("xingyaoww/code-act", split=split)
-        self.parsed: list[dict[str, Any]] = []
-        n_skipped = 0
-        for row in ds:
-            parsed = parse_codeact_row(row, max_chars=max_chars)
-            if parsed is None:
-                n_skipped += 1
-                continue
-            self.parsed.append(parsed)
-        random.Random(seed).shuffle(self.parsed)
-        self._n_skipped = n_skipped
-
-    def __len__(self) -> int:
-        return len(self.parsed)
-
-    def __getitem__(self, idx: int) -> dict[str, Any]:
-        row = self.parsed[idx]
-        return {"messages": [{"role": "system", "content": SYSTEM_PROMPT}] + row["messages"]}
-
-
-class CodeFeedbackDataset:
-    """m-a-p/Code-Feedback filtered to python rows with execution feedback."""
-
-    def __init__(
-        self,
-        split: str = "train",
-        seed: int = 42,
-        max_chars: int = 16384,
-        max_rows: int | None = 10_000,
-    ) -> None:
-        _set_hf_endpoint()
-        ds = load_dataset("m-a-p/Code-Feedback", split=split)
-        self.parsed: list[dict[str, Any]] = []
-        n_skipped = 0
-        for row in ds:
-            parsed = parse_codefeedback_row(row, max_chars=max_chars)
-            if parsed is None:
-                n_skipped += 1
-                continue
-            self.parsed.append(parsed)
-            if max_rows is not None and len(self.parsed) >= max_rows:
-                break
-        random.Random(seed).shuffle(self.parsed)
-        self._n_skipped = n_skipped
-
-    def __len__(self) -> int:
-        return len(self.parsed)
-
-    def __getitem__(self, idx: int) -> dict[str, Any]:
-        row = self.parsed[idx]
-        return {"messages": [{"role": "system", "content": SYSTEM_PROMPT}] + row["messages"]}
-
-
-# ---------------------------------------------------------------------------
-# In-memory variant for unit tests (no HF dep)
-# ---------------------------------------------------------------------------
-
-
-class _InMemoryCodeDataset:
-    """Wraps a pre-built list of {messages: [...]} rows; used by tests."""
-
-    def __init__(self, rows: Iterable[dict[str, Any]], seed: int = 42) -> None:
-        self.parsed = list(rows)
-        random.Random(seed).shuffle(self.parsed)
-
-    def __len__(self) -> int:
-        return len(self.parsed)
-
-    def __getitem__(self, idx: int) -> dict[str, Any]:
-        row = self.parsed[idx]
-        return {"messages": [{"role": "system", "content": SYSTEM_PROMPT}] + row["messages"]}
-
-
-__all__ = [
-    "CodeActDataset",
-    "CodeFeedbackDataset",
-    "parse_codeact_row",
-    "parse_codefeedback_row",
-    "_InMemoryCodeDataset",
-]

vendor/nanoai/agentic/data/common.py

@@ -1,17 +0,0 @@
-SYSTEM_PROMPT = """you are nanocode, a coding agent.
-you communicate in lowercase and think out loud before acting.
-
-your tools enable you to interact with the user's UNIX system and modify and create new files:
-  Read({"file_path", "offset"?, "limit"?})           - read a file
-  Edit({"file_path", "old_string"?, "new_string"})   - edit or create a file
-  Grep({"pattern", "path"?})                         - search file contents
-  Bash({"command"})                                  - execute a shell command
-
-emit a tool call by producing exactly one tool-call token block per assistant turn. you've seen the structure during training; reproduce it precisely."""
-
-
-def render_mc(question, letters, choices):
-    query = f"Multiple Choice question: {question}\n"
-    query += "".join([f"- {choice}={letter}\n" for letter, choice in zip(letters, choices)])
-    query += "\nRespond only with the letter of the correct answer."
-    return query

vendor/nanoai/agentic/data/ebook_data.py

@@ -1,201 +0,0 @@
-"""Ebook pretrain stream (PDF→markdown textbook corpus).
-
-Materializes the converted-ebook markdown under ``~/ebook/markdown/**/*.md``
-(produced by ~/ebook/convert_ebooks.py, pdftext backend) into parquet shards
-so the existing parquet-iter API drives it identically to web / code / pcap.
-
-Each markdown book is **chunked** into ~``chunk_chars``-sized documents at
-paragraph boundaries, so that the code-mixer's per-document categorical draw
-approximates a per-token mixture: climbmix web docs average ~2875 chars
-(~720 tok), so the default 3000-char chunk keeps ebook docs the same scale and
-``--ebook-ratio 0.5`` ≈ 50% of *tokens* (not just 50% of documents).
-
-Image references (``![](_page_N_Picture_M.jpeg)``) are stripped — they are
-pure-noise placeholders for figures pdftext could not OCR.
-
-Stored under ``$NANOAI_BASE_DIR/code_pretrain_data/ebook-md/``. Used via
-`agentic_pretrain.py --ebook-ratio`. Mirrors `vrcpt_data` (atomic .tmp→rename,
-restart-safe validate/quarantine, last shard = val split).
-
-Run once before pretraining:
-
-    python -m nanoai.agentic.data.ebook_data prepare \\
-        --md-root ~/ebook/markdown --chunk-chars 3000 --rows-per-shard 50000
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-import random
-import re
-import time
-from pathlib import Path
-
-import pyarrow as pa
-import pyarrow.parquet as pq
-
-from nanoai.common import print0
-from nanoai.agentic.data.pretrain_data import DATA_DIR
-
-DATASET_NAME = "ebook-md"
-_TMP_SUFFIX = ".parquet.tmp"
-
-# image-reference lines like  ![](_page_0_Picture_1.jpeg)  or  ![alt](path.png)
-_IMG_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
-# 3+ consecutive blank lines → 2
-_BLANKS_RE = re.compile(r"\n{3,}")
-
-
-def ebook_corpus_dir() -> Path:
-    return DATA_DIR / DATASET_NAME
-
-
-def list_ebook_parquet_files() -> list[Path]:
-    d = ebook_corpus_dir()
-    if not d.exists():
-        return []
-    return sorted([d / f.name for f in d.iterdir() if f.suffix == ".parquet"])
-
-
-def _clean(text: str) -> str:
-    text = _IMG_RE.sub("", text)
-    text = _BLANKS_RE.sub("\n\n", text)
-    return text.strip()
-
-
-def _chunk_markdown(text: str, chunk_chars: int):
-    """Split cleaned markdown into ~chunk_chars docs at paragraph boundaries."""
-    paras = text.split("\n\n")
-    buf, size = [], 0
-    for p in paras:
-        p = p.strip()
-        if not p:
-            continue
-        buf.append(p)
-        size += len(p) + 2
-        if size >= chunk_chars:
-            yield "\n\n".join(buf)
-            buf, size = [], 0
-    if buf:
-        joined = "\n\n".join(buf)
-        if len(joined) >= 200:  # drop tiny trailing fragments
-            yield joined
-
-
-def _stream_chunks(md_root: Path, chunk_chars: int, seed: int):
-    files = sorted(md_root.rglob("*.md"))
-    rng = random.Random(seed)
-    rng.shuffle(files)  # book-level shuffle so shards aren't single-book
-    for fp in files:
-        try:
-            raw = fp.read_text(encoding="utf-8", errors="ignore")
-        except Exception as e:
-            print0(f"[ebook_data] skip {fp.name}: {e}")
-            continue
-        cleaned = _clean(raw)
-        if not cleaned:
-            continue
-        for chunk in _chunk_markdown(cleaned, chunk_chars):
-            yield chunk
-
-
-def _shuffled(stream, buffer_size: int, seed: int):
-    """Reservoir-style shuffle buffer to decorrelate adjacent same-book chunks."""
-    rng = random.Random(seed + 1)
-    buf: list[str] = []
-    for item in stream:
-        buf.append(item)
-        if len(buf) >= buffer_size:
-            j = rng.randrange(len(buf))
-            buf[j], buf[-1] = buf[-1], buf[j]
-            yield buf.pop()
-    rng.shuffle(buf)
-    yield from buf
-
-
-def _try_open_shard(path: Path) -> bool:
-    try:
-        return pq.ParquetFile(path).num_row_groups > 0
-    except Exception:
-        return False
-
-
-def prepare(md_root: Path, rows_per_shard: int = 50000, chunk_chars: int = 3000,
-            shuffle_buffer: int = 50000, seed: int = 1234):
-    """One-shot markdown → parquet shards under ebook_corpus_dir().
-
-    Atomic write (.tmp → rename), restart-safe. Last shard = val split.
-    """
-    out_dir = ebook_corpus_dir()
-    out_dir.mkdir(parents=True, exist_ok=True)
-
-    existing = sorted(p for p in out_dir.iterdir() if p.suffix == ".parquet")
-    tmps = [p for p in out_dir.iterdir() if p.name.endswith(_TMP_SUFFIX)]
-    if existing and not tmps and all(_try_open_shard(p) for p in existing) and len(existing) >= 2:
-        print0(f"[ebook_data] {len(existing)} valid shards already in {out_dir}; skipping.")
-        return existing
-    if existing or tmps:
-        attic = out_dir / f".attic_{int(time.time())}"
-        attic.mkdir()
-        for p in existing + tmps:
-            p.rename(attic / p.name)
-        print0(f"[ebook_data] quarantined {len(existing)+len(tmps)} stale file(s) to {attic.name}/")
-
-    assert md_root.exists(), f"md-root not found: {md_root}"
-    print0(f"[ebook_data] reading {md_root} → {out_dir} "
-           f"(chunk={chunk_chars} chars, {rows_per_shard} rows/shard)")
-
-    shard_idx = 0
-    buf: list[str] = []
-    written: list[Path] = []
-    total_docs = 0
-    total_chars = 0
-
-    def flush():
-        nonlocal shard_idx, buf
-        if not buf:
-            return
-        shard_path = out_dir / f"shard_{shard_idx:05d}.parquet"
-        tmp_path = shard_path.with_name(shard_path.name + ".tmp")
-        shard_chars = sum(len(t) for t in buf)
-        pq.write_table(pa.table({"text": buf}), tmp_path, compression="snappy")
-        tmp_path.rename(shard_path)
-        written.append(shard_path)
-        print0(f"[ebook_data] wrote {shard_path.name} "
-               f"({len(buf)} docs, {shard_chars/1e6:.2f}M chars, "
-               f"{shard_path.stat().st_size/1e6:.2f} MB)")
-        shard_idx += 1
-        buf = []
-
-    stream = _stream_chunks(md_root, chunk_chars, seed)
-    for text in _shuffled(stream, shuffle_buffer, seed):
-        buf.append(text)
-        total_docs += 1
-        total_chars += len(text)
-        if len(buf) >= rows_per_shard:
-            flush()
-    flush()
-
-    print0(f"[ebook_data] DONE: {total_docs:,} docs, {total_chars/1e6:.1f}M chars "
-           f"(~{total_chars/4/1e6:.0f}M est tokens) across {len(written)} shards")
-    return written
-
-
-def main():
-    ap = argparse.ArgumentParser(description="Prepare ebook markdown pretrain shards")
-    sub = ap.add_subparsers(dest="cmd", required=True)
-    p = sub.add_parser("prepare")
-    p.add_argument("--md-root", type=str, default=str(Path.home() / "ebook" / "markdown"))
-    p.add_argument("--rows-per-shard", type=int, default=50000)
-    p.add_argument("--chunk-chars", type=int, default=3000)
-    p.add_argument("--shuffle-buffer", type=int, default=50000)
-    p.add_argument("--seed", type=int, default=1234)
-    args = ap.parse_args()
-    if args.cmd == "prepare":
-        prepare(Path(os.path.expanduser(args.md_root)), args.rows_per_shard,
-                args.chunk_chars, args.shuffle_buffer, args.seed)
-
-
-if __name__ == "__main__":
-    main()

vendor/nanoai/agentic/data/hf_dataset.py

@@ -1,15 +0,0 @@
-"""Generic Hugging Face conversation dataset loader."""
-
-from datasets import load_dataset
-
-
-class HuggingFaceDataset:
-    def __init__(self, dataset: str, messages_key: str, split: str, seed: int, **kwargs):
-        self.messages_key = messages_key
-        self.ds = load_dataset(dataset, split=split, **kwargs).shuffle(seed=seed)
-
-    def __len__(self):
-        return len(self.ds)
-
-    def __getitem__(self, idx: int):
-        return {"messages": self.ds[idx][self.messages_key]}

vendor/nanoai/agentic/data/json_dataset.py

@@ -1,73 +0,0 @@
-"""JSONL conversation / preference datasets.
-
-Conversations are tagged with the agentic SYSTEM_PROMPT so the rendered tool-call
-schema is always in context. Format on disk matches nanocode rollouts:
-  {"messages": [{"role": "user", ...}, {"role": "assistant", ...}, ...]}
-  {"chosen": {"messages": [...]}, "rejected": {"messages": [...]}}  # for prefs
-"""
-
-from __future__ import annotations
-
-import json
-import random
-
-from nanoai.agentic.data.common import SYSTEM_PROMPT
-
-
-class JSONDataset:
-    def __init__(self, filepath, seed, **kwargs):
-        super().__init__(**kwargs)
-        self.filepath = filepath
-        self.ds = []
-        with open(filepath, "r", encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                self.ds.append(json.loads(line))
-        random.Random(seed).shuffle(self.ds)
-
-    def __len__(self):
-        return len(self.ds)
-
-    def __getitem__(self, idx: int):
-        messages = self.ds[idx]["messages"]
-        messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages
-        return {"messages": messages}
-
-
-class PairedJSONPreferenceDataset:
-    """Preference pairs filtered to require non-empty assistant content on both sides."""
-
-    def __init__(self, filepath, seed, frac=1.0, **kwargs):
-        super().__init__(**kwargs)
-        self.ds = []
-        with open(filepath, "r", encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                row = json.loads(line)
-                has_chosen = any(
-                    m.get("content", "")
-                    for m in row["chosen"]["messages"]
-                    if m["role"] == "assistant"
-                )
-                has_rejected = any(
-                    m.get("content", "")
-                    for m in row["rejected"]["messages"]
-                    if m["role"] == "assistant"
-                )
-                if has_chosen and has_rejected:
-                    self.ds.append(row)
-        random.Random(seed).shuffle(self.ds)
-        self.ds = self.ds[: int(len(self.ds) * frac)]
-
-    def __len__(self):
-        return len(self.ds)
-
-    def __getitem__(self, idx: int):
-        row = self.ds[idx]
-        chosen = [{"role": "system", "content": SYSTEM_PROMPT}] + row["chosen"]["messages"]
-        rejected = [{"role": "system", "content": SYSTEM_PROMPT}] + row["rejected"]["messages"]
-        return {"messages": chosen}, {"messages": rejected}

vendor/nanoai/agentic/data/mixture.py

@@ -1,42 +0,0 @@
-"""Deterministic interleaving of multiple sub-datasets."""
-
-import random
-
-
-class TaskMixture:
-    def __init__(self, tasks, seed: int, **kwargs):
-        super().__init__(**kwargs)
-        self.tasks = tasks
-        self.lengths = [len(task) for task in self.tasks]
-        self.num_conversations = sum(self.lengths)
-        self.index_map = []
-        for task_idx, task_length in enumerate(self.lengths):
-            for local_idx in range(task_length):
-                self.index_map.append((task_idx, local_idx))
-        random.Random(seed).shuffle(self.index_map)
-
-    def __len__(self):
-        return self.num_conversations
-
-    def __getitem__(self, index):
-        assert 0 <= index < self.num_conversations
-        task_idx, local_idx = self.index_map[index]
-        return self.tasks[task_idx][local_idx]
-
-
-class TaskSequence:
-    def __init__(self, tasks, **kwargs):
-        super().__init__(**kwargs)
-        self.tasks = tasks
-        self.lengths = [len(task) for task in self.tasks]
-        self.num_conversations = sum(self.lengths)
-
-    def __len__(self):
-        return self.num_conversations
-
-    def __getitem__(self, index):
-        assert 0 <= index < self.num_conversations
-        for task_idx, task_length in enumerate(self.lengths):
-            if index < task_length:
-                return self.tasks[task_idx][index]
-            index -= task_length

vendor/nanoai/agentic/data/pcap_data.py

@@ -1,224 +0,0 @@
-"""Pcap-corpus pretrain stream.
-
-Materializes a domain-aligned third pretrain stream alongside web (climbmix /
-fineweb-edu) and code (the-stack-v2-dedup). The raw input is a JSONL file
-produced upstream by `capbench_agent/corpus/scripts/prepare_cpt_data.py`
-(57k+ docs across 11 knowledge layers: protocols, philosophy, mathematics,
-os_systems, forensics, network_traffic, reasoning, ndes_native, ...).
-
-This module converts the JSONL into parquet shards stored under
-`$NANOAI_BASE_DIR/code_pretrain_data/pcap-corpus/` so the existing
-`parquets_iter_batched` API (which expects parquet w/ a `text` column) can
-drive it identically to fineweb-edu / the-stack-v2-dedup.
-
-Run once before pretraining:
-
-    python -m nanoai.agentic.data.pcap_data prepare \\
-        --src /home/vader/capbench_agent/corpus/output/pretrain.jsonl \\
-        --rows-per-shard 5000
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import time
-from pathlib import Path
-
-import pyarrow as pa
-import pyarrow.parquet as pq
-
-from nanoai.common import print0
-from nanoai.agentic.data.pretrain_data import DATA_DIR
-
-DATASET_NAME = "pcap-corpus"
-_TMP_SUFFIX = ".parquet.tmp"
-
-
-def pcap_corpus_dir() -> Path:
-    return DATA_DIR / DATASET_NAME
-
-
-def _stream_jsonl_text(src: Path):
-    with src.open("r", encoding="utf-8") as f:
-        for line in f:
-            line = line.strip()
-            if not line:
-                continue
-            obj = json.loads(line)
-            text = obj.get("text")
-            if isinstance(text, str) and text:
-                yield text
-
-
-def _try_open_shard(path: Path) -> bool:
-    """Return True iff `path` is a readable parquet with ≥1 row group."""
-    try:
-        pf = pq.ParquetFile(path)
-        return pf.num_row_groups > 0
-    except Exception:
-        return False
-
-
-def _validate_existing_corpus(out_dir: Path) -> tuple[bool, list[Path], str]:
-    """Inspect out_dir to decide whether it holds a complete prepared corpus.
-
-    Returns (is_valid, parquet_list, reason). A corpus is valid iff:
-      - ≥2 parquet shards (so train/val split is non-empty), AND
-      - every parquet is openable and has ≥1 row_group, AND
-      - no leftover `*.parquet.tmp` files (signal of prior interrupted write).
-    """
-    parquets = sorted(p for p in out_dir.iterdir() if p.suffix == ".parquet")
-    tmps = [p for p in out_dir.iterdir() if p.name.endswith(_TMP_SUFFIX)]
-    if tmps:
-        return False, parquets, f"{len(tmps)} stale {_TMP_SUFFIX} file(s) (prior crash?)"
-    if len(parquets) < 2:
-        return False, parquets, f"only {len(parquets)} shard(s); need ≥2"
-    for p in parquets:
-        if not _try_open_shard(p):
-            return False, parquets, f"shard {p.name} unreadable or has 0 row_groups"
-    return True, parquets, "ok"
-
-
-def _quarantine_stale(out_dir: Path, reason_slug: str) -> Path | None:
-    """Move any *.parquet / *.parquet.tmp files aside before rebuilding.
-
-    Uses an `.attic_<ts>_<reason>/` sub-dir so the operator can post-mortem
-    what was wrong without us silently `rm -rf`-ing it.
-    """
-    movable = [
-        p for p in out_dir.iterdir()
-        if p.suffix == ".parquet" or p.name.endswith(_TMP_SUFFIX)
-    ]
-    if not movable:
-        return None
-    attic = out_dir / f".attic_{int(time.time())}_{reason_slug}"
-    attic.mkdir()
-    for entry in movable:
-        entry.rename(attic / entry.name)
-    print0(
-        f"[pcap_data] quarantined {len(movable)} stale file(s) to {attic.name}/"
-        f" (post-mortem; safe to delete once unblocked)"
-    )
-    return attic
-
-
-def _reason_slug(reason: str) -> str:
-    return reason.split(";")[0].strip().replace(" ", "_")[:32] or "invalid"
-
-
-def prepare_from_jsonl(src: Path, rows_per_shard: int = 5000):
-    """One-shot: JSONL -> parquet shards under pcap_corpus_dir().
-
-    Last shard becomes the val split (mirrors fineweb-edu convention).
-
-    Restart-safe:
-      - if the output dir already holds a *valid* corpus (≥2 readable shards,
-        no leftover .tmp), this is a no-op.
-      - if the corpus is incomplete (single shard / unreadable parquet /
-        stale .tmp), the stale files are quarantined into `.attic_<ts>_*/`
-        and a full rebuild runs.
-      - each new shard is written as `<name>.parquet.tmp` then renamed to
-        `<name>.parquet` (atomic on POSIX), so a mid-shard crash leaves a
-        clearly-named .tmp that this function will sweep on the next call.
-    """
-    out_dir = pcap_corpus_dir()
-    out_dir.mkdir(parents=True, exist_ok=True)
-
-    is_valid, existing, reason = _validate_existing_corpus(out_dir)
-    if is_valid:
-        print0(
-            f"[pcap_data] {len(existing)} valid shards already in {out_dir}; "
-            f"skipping prepare."
-        )
-        return existing
-    # Either nothing is there, or what's there is incomplete/corrupt.
-    if existing or any(p.name.endswith(_TMP_SUFFIX) for p in out_dir.iterdir()):
-        print0(
-            f"[pcap_data] existing corpus at {out_dir} is INVALID ({reason}); "
-            f"quarantining and rebuilding from scratch."
-        )
-        _quarantine_stale(out_dir, _reason_slug(reason))
-
-    assert src.exists(), f"Source JSONL not found: {src}"
-    print0(f"[pcap_data] reading {src} → {out_dir} ({rows_per_shard} rows/shard)")
-
-    shard_idx = 0
-    buf: list[str] = []
-    written: list[Path] = []
-    total_docs = 0
-    total_chars = 0
-
-    def flush():
-        nonlocal shard_idx, buf
-        if not buf:
-            return
-        shard_path = out_dir / f"shard_{shard_idx:05d}.parquet"
-        tmp_path = shard_path.with_name(shard_path.name + ".tmp")
-        shard_chars = sum(len(t) for t in buf)
-        table = pa.table({"text": buf})
-        pq.write_table(table, tmp_path, compression="snappy")
-        tmp_path.rename(shard_path)  # atomic on POSIX
-        written.append(shard_path)
-        size_mb = shard_path.stat().st_size / 1e6
-        print0(
-            f"[pcap_data] wrote {shard_path.name} "
-            f"({len(buf)} docs, {shard_chars/1e6:.2f}M chars, {size_mb:.2f} MB on disk)"
-        )
-        shard_idx += 1
-        buf = []
-
-    for text in _stream_jsonl_text(src):
-        buf.append(text)
-        total_docs += 1
-        total_chars += len(text)
-        if len(buf) >= rows_per_shard:
-            flush()
-    flush()
-
-    assert len(written) >= 2, (
-        f"Only {len(written)} shard(s) written; need ≥2 so train/val split is non-empty. "
-        f"Lower --rows-per-shard or supply a larger JSONL."
-    )
-    # Rough token-budget estimate using 4 chars/token (English) — matches the
-    # ballpark we see in nanoai's tokenizer for educational + code mix.
-    est_tokens = total_chars / 4
-    print0(
-        f"[pcap_data] done: {len(written)} shards (last is val), "
-        f"{total_docs:,} docs, {total_chars/1e6:.1f}M chars, "
-        f"≈{est_tokens/1e6:.1f}M tokens (4 chars/tok estimate)"
-    )
-    return written
-
-
-def list_pcap_parquet_files() -> list[Path]:
-    d = pcap_corpus_dir()
-    if not d.exists():
-        return []
-    return sorted([d / f for f in d.iterdir() if f.suffix == ".parquet"])
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Prepare pcap-corpus parquet shards.")
-    sub = parser.add_subparsers(dest="cmd", required=True)
-    prep = sub.add_parser("prepare", help="convert a JSONL to parquet shards")
-    prep.add_argument("--src", type=Path, required=True, help="path to pretrain.jsonl")
-    prep.add_argument("--rows-per-shard", type=int, default=5000)
-    sub.add_parser("list", help="list current pcap-corpus shards")
-
-    args = parser.parse_args()
-    if args.cmd == "prepare":
-        prepare_from_jsonl(args.src, rows_per_shard=args.rows_per_shard)
-    elif args.cmd == "list":
-        shards = list_pcap_parquet_files()
-        if not shards:
-            print(f"No shards in {pcap_corpus_dir()}")
-            return
-        total_bytes = sum(s.stat().st_size for s in shards)
-        for s in shards:
-            print(f"  {s.name}  ({s.stat().st_size / 1e6:.2f} MB)")
-        print(f"total: {len(shards)} shards, {total_bytes / 1e6:.1f} MB")
-
-
-if __name__ == "__main__":
-    main()

vendor/nanoai/agentic/data/pretrain_data.py

@@ -1,115 +0,0 @@
-"""Parquet-shard downloader and iterator for the two pretrain datasets:
-  - karpathy/fineweb-edu-100b-shuffle    (web text)
-  - smohammadi/the-stack-v2-python-shuffle (Python code)
-
-Honors HF_ENDPOINT so a mirror (e.g. hf-mirror.com) can be substituted.
-Run as a CLI: `python -m nanoai.agentic.data.pretrain_data -d fineweb-edu -n 4`
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-import time
-from functools import partial
-from multiprocessing import Pool
-from pathlib import Path
-
-import pyarrow.parquet as pq
-import requests
-
-from nanoai.common import get_base_dir, get_dist_info, print0
-
-DATA_DIR = Path(get_base_dir()) / "code_pretrain_data"
-DATA_DIR.mkdir(parents=True, exist_ok=True)
-
-
-def index_to_filename(index: int) -> str:
-    return f"shard_{index:05d}.parquet"
-
-
-def list_parquet_files(data_dir: Path):
-    return sorted([data_dir / f for f in data_dir.iterdir() if f.suffix == ".parquet"])
-
-
-def parquets_iter_batched(dataset: str, split: str, start: int = 0, step: int = 1):
-    """Yield row-group-sized batches of `text` strings.
-
-    `split="val"` returns only the last shard; `split="train"` returns the rest.
-    `start`/`step` slice row-groups for DDP rank/world-size sharding.
-    """
-    assert dataset in ("fineweb-edu", "the-stack-v2-dedup", "network-domain", "pcap-corpus")
-    assert split in ("train", "val")
-    data_dir = DATA_DIR / dataset
-    parquet_paths = list_parquet_files(data_dir)
-    if not parquet_paths:
-        raise FileNotFoundError(
-            f"No parquet shards found in {data_dir}. "
-            f"Run: python -m nanoai.agentic.data.pretrain_data -d {dataset} -n N"
-        )
-    parquet_paths = parquet_paths[:-1] if split == "train" else parquet_paths[-1:]
-    for filepath in parquet_paths:
-        pf = pq.ParquetFile(filepath)
-        for rg_idx in range(start, pf.num_row_groups, step):
-            rg = pf.read_row_group(rg_idx)
-            yield rg.column("text").to_pylist()
-
-
-def download_single_file(index: int, base_url: str, data_dir: Path) -> bool:
-    filename = index_to_filename(index)
-    filepath = data_dir / filename
-    if filepath.exists():
-        return True
-
-    url = f"{base_url}/{filename}"
-    print0(f"Downloading {filename}...")
-
-    for attempt in range(1, 6):
-        try:
-            response = requests.get(url, stream=True, timeout=30)
-            response.raise_for_status()
-            temp_path = filepath.with_name(filepath.name + ".tmp")
-            with open(temp_path, "wb") as f:
-                for chunk in response.iter_content(chunk_size=1024 * 1024):
-                    if chunk:
-                        f.write(chunk)
-            temp_path.rename(filepath)
-            print0(f"Successfully downloaded {filename}")
-            return True
-        except (requests.RequestException, IOError) as e:
-            print0(f"Attempt {attempt}/5 failed for {filename}: {e}")
-            filepath.unlink(missing_ok=True)
-            filepath.with_name(filepath.name + ".tmp").unlink(missing_ok=True)
-            if attempt < 5:
-                time.sleep(2 ** attempt)
-    print0(f"Failed to download {filename}")
-    return False
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Download pretraining dataset shards")
-    parser.add_argument("-d", "--dataset", choices=["fineweb-edu", "the-stack-v2-dedup"], required=True)
-    parser.add_argument("-n", "--num-files", type=int, default=-1)
-    parser.add_argument("-w", "--num-workers", type=int, default=4)
-    args = parser.parse_args()
-
-    hf_endpoint = os.environ.get("HF_ENDPOINT", "https://huggingface.co").rstrip("/")
-    if args.dataset == "fineweb-edu":
-        base_url = f"{hf_endpoint}/datasets/karpathy/fineweb-edu-100b-shuffle/resolve/main"
-        max_shard = 1822
-    else:
-        base_url = f"{hf_endpoint}/datasets/smohammadi/the-stack-v2-python-shuffle/resolve/main"
-        max_shard = 79
-
-    data_dir = DATA_DIR / args.dataset
-    data_dir.mkdir(parents=True, exist_ok=True)
-    num = max_shard + 1 if args.num_files == -1 else min(args.num_files, max_shard + 1)
-    ids = list(range(num))
-    print0(f"Downloading {len(ids)} shards from {base_url} → {data_dir}")
-    with Pool(processes=args.num_workers) as pool:
-        results = pool.map(partial(download_single_file, base_url=base_url, data_dir=data_dir), ids)
-    print0(f"Done: {sum(results)}/{len(ids)} shards")
-
-
-if __name__ == "__main__":
-    main()

vendor/nanoai/agentic/data/pretrain_mix_1b.py

@@ -1,466 +0,0 @@
-"""Self-contained 5-stream pretrain mixer for the 1B-challenge d24 base.
-
-Mixes five document streams with a per-document categorical (Bernoulli) draw,
-then BOS-aligned best-fit packs into (B, T) rows — identical packing to
-``nanoai.agentic.code_dataloader`` (whose ``_doc_batches_from`` / ``_split_paths``
-helpers are reused), but kept in a *separate* module so the validated agentic /
-d6 / d12 code-mixer is untouched (zero regression risk).
-
-Streams (all expose a ``text`` column; verified on wuneng 2026-06-08):
-
-  | name    | source dir (under base_dir)                      | files | col  |
-  |---------|--------------------------------------------------|------:|------|
-  | web        | base_data_climbmix/                                   | 1501  | text |
-  | owm        | code_pretrain_data/open-web-math/data/                |  114  | text |
-  | code       | code_pretrain_data/the-stack-v2-dedup/                |  510  | text |
-  | books      | code_pretrain_data/_ebook_orig/                       |   20  | text |
-  | chinese    | chinese_raw/{fineweb_edu_zh,wikipedia_zh}/**          |  29   | text |
-  | finemath   | code_pretrain_data/finemath/finemath-3plus/           |   —   | text |
-  | mathtb     | code_pretrain_data/nemotron_specialized/Nemotron-Pretraining-Math-Textbooks/ | — | text |
-  | swecpt     | code_pretrain_data/swe_mid/cpt/                       |   —   | text |
-  | commitdiff | code_pretrain_data/commitpackft/                      |   —   | text |
-  | codetests  | code_pretrain_data/code_tests/                        |   —   | text |
-  | nvcode     | code_pretrain_data/nemotron_code/                     |   —   | text |
-  | swe1b      | code_pretrain_data/swe1b_midcpt/ (REAL SWE-Gym)       |   —   | text |
-
-``swecpt`` is SYNTHETIC (mutation-injected) and DEPRECATED; ``swe1b`` is its REAL
-clean replacement — decontam'd SWE-Gym issue+gold-patch docs from
-``experiments/swe1b`` (serialize_midtrain_cpt -> jsonl_to_parquet). Kept as
-separate streams so real and synthetic SWE data never co-mingle.
-
-The SWE-mid streams (swecpt/commitdiff/codetests/nvcode) are produced by
-``experiments/challenge1b/tools/build_{commitdiff,code_tests,nvcode}_stream.py``
-and ``gen_swe_mut`` (recipe: ``experiments/challenge1b/docs/d6_swe_mid_design.md``).
-
-``owm/code/books/chinese`` ratios are explicit; ``web`` takes the remainder.
-The ratios passed here are PER-DOCUMENT draw probabilities; to hit a target
-*token* share, calibrate them against measured mean-tokens/doc per stream
-(see ``experiments/challenge1b/tools/measure_mix.py``). The recipe of record is
-``experiments/challenge1b/docs/pretrain_recipe.md``.
-
-Resume state is a stub (single epoch counter), matching the code-mixer — fine
-for a single contiguous ~15h run; true multi-stream resume is not implemented.
-"""
-
-from __future__ import annotations
-
-import random
-from pathlib import Path
-
-import pyarrow.parquet as pq
-import torch
-
-from nanoai.common import get_base_dir, get_dist_info, print0
-from nanoai.dataset import list_parquet_files as list_web_parquet_files
-from nanoai.agentic.data.pretrain_data import DATA_DIR as _CODE_BASE_DATA_DIR
-from nanoai.agentic.code_dataloader import _split_paths
-
-
-def _doc_batches_robust(paths, ddp_rank, ddp_world_size, tokenizer_batch_size, split,
-                        seed: int = 0, shuffle_buffer: int = 10000, text_col: str = "text"):
-    """Infinite per-rank list[str] doc-batch iterator: starvation-safe + shuffled.
-
-    Two fixes over the upstream code-mixer's ``_doc_batches_from``:
-
-    1. STARVATION. Upstream stripes by ROW-GROUP (``range(rank, num_rg, world)``),
-       which starves any rank >= a shard's row-group count. Several streams here
-       ship single-row-group shards (``_ebook_orig`` books: 1 rg/shard), so ranks
-       1..3 got ZERO books docs and blocked forever -> DDP deadlock at eval@step0.
-       Fix: train stripes by FILE (every stream has >=19 train shards >= world);
-       val reads the FULL val shard on every rank (single-file shards; eval just
-       averages bpb so cross-rank overlap is harmless).
-
-    2. SHUFFLE. We read shards file-sequentially, and not every stream is
-       pre-shuffled within-shard (web/code/owm shards are; chinese isn't
-       guaranteed). The bestfit packer selects by LENGTH, not randomly, so it is
-       NOT a shuffle. Train docs therefore pass through a reservoir shuffle buffer
-       to decorrelate ordering; val is read deterministically (reproducible eval).
-    """
-    if split == "val":
-        my_paths = list(paths)
-    else:
-        my_paths = paths[ddp_rank::ddp_world_size] or list(paths)
-
-    def _docs():
-        while True:
-            for filepath in my_paths:
-                pf = pq.ParquetFile(filepath)
-                for rg_idx in range(pf.num_row_groups):
-                    for d in pf.read_row_group(rg_idx, columns=[text_col]).column(text_col).to_pylist():
-                        if d:
-                            yield d
-
-    src = _docs()
-    rng = random.Random(seed + ddp_rank + 1)
-    buf: list[str] = []
-    batch: list[str] = []
-    while True:
-        if split == "train":
-            while len(buf) < shuffle_buffer:        # reservoir: fill then emit random picks
-                buf.append(next(src))
-            j = rng.randrange(len(buf))
-            buf[j], buf[-1] = buf[-1], buf[j]       # swap-to-end + pop = O(1) random eviction
-            batch.append(buf.pop())
-        else:
-            batch.append(next(src))                 # val: deterministic order
-        if len(batch) >= tokenizer_batch_size:
-            yield batch
-            batch = []
-
-
-# --------------------------------------------------------------- stream paths
-_CODE_DATASET_NAME = "the-stack-v2-dedup"
-_OWM_SUBPATH = ("open-web-math", "data")   # parquets live under open-web-math/data/
-_BOOKS_DATASET_NAME = "_ebook_orig"        # intact STEM-textbook parquets (ebook-md was clobbered)
-_CHINESE_DIRNAME = "chinese_raw"
-_CHINESE_SUBDIRS = ("fineweb_edu_zh", "wikipedia_zh")
-
-
-def list_code_parquet_files() -> list[Path]:
-    # the-stack-v2-dedup/ also holds 67 ``netis_part_*`` shards (a network-company
-    # repo dump: shell/go/c tooling code, off the general-code distribution). Drop
-    # them so the ``code`` stream is purely the-stack / starcoder (sc2_*/sc_*/shard_*).
-    d = _CODE_BASE_DATA_DIR / _CODE_DATASET_NAME
-    if not d.exists():
-        return []
-    return sorted(p for p in d.glob("*.parquet") if not p.name.startswith("netis"))
-
-
-def list_owm_parquet_files() -> list[Path]:
-    d = _CODE_BASE_DATA_DIR.joinpath(*_OWM_SUBPATH)
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_books_parquet_files() -> list[Path]:
-    d = _CODE_BASE_DATA_DIR / _BOOKS_DATASET_NAME
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_finemath_parquet_files() -> list[Path]:
-    # HuggingFaceTB/finemath finemath-3plus (~34B tok, L2-tier classifier-selected)
-    d = _CODE_BASE_DATA_DIR / "finemath" / "finemath-3plus"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_mathtb_parquet_files() -> list[Path]:
-    # Nemotron-Pretraining-Math-Textbooks (L3-tier synthetic textbooks, from wukong)
-    d = _CODE_BASE_DATA_DIR / "nemotron_specialized" / "Nemotron-Pretraining-Math-Textbooks"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_swecpt_parquet_files() -> list[Path]:
-    # gen_swe_mut cpt-view text docs (# buggy / # test / # traceback / # fix)
-    d = _CODE_BASE_DATA_DIR / "swe_mid" / "cpt"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_commitdiff_parquet_files() -> list[Path]:
-    # CommitPackFT python slice rendered Edit-tool-isomorphic (build_commitdiff_stream.py)
-    d = _CODE_BASE_DATA_DIR / "commitpackft"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_codetests_parquet_files() -> list[Path]:
-    # the-stack docs that carry their own tests (build_code_tests_stream.py)
-    d = _CODE_BASE_DATA_DIR / "code_tests"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_nvcode_parquet_files() -> list[Path]:
-    # Nemotron scientific-coding + Llama-Nemotron SFT code (build_nvcode_stream.py)
-    d = _CODE_BASE_DATA_DIR / "nemotron_code"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_swe1b_parquet_files() -> list[Path]:
-    # REAL SWE-Gym mid corpus (decontam vs Verified): issue + gold-patch bug->fix
-    # docs from experiments/swe1b (serialize_midtrain_cpt -> jsonl_to_parquet).
-    # This is the clean real-data replacement for the synthetic ``swecpt`` stream;
-    # the two are kept SEPARATE so real and synthetic SWE data never mix.
-    d = _CODE_BASE_DATA_DIR / "swe1b_midcpt"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_chinese_parquet_files() -> list[Path]:
-    base = Path(get_base_dir()) / _CHINESE_DIRNAME
-    out: list[Path] = []
-    if base.exists():
-        for sub in _CHINESE_SUBDIRS:
-            d = base / sub
-            if d.exists():
-                out.extend(sorted(d.rglob("*.parquet")))  # nested IndustryCorpus/ etc.
-    return out
-
-
-def list_ebook_nf_parquet_files() -> list[Path]:
-    """Re-made LONG non-fiction books (build_ebook_long_stream --split-type): full
-    books as single long docs (p50 ~131K tok), NOT the chunked ``_ebook_orig``. The
-    genuine long-context natural-language source for the staged decay."""
-    d = _CODE_BASE_DATA_DIR / "ebook_long" / "non_fiction"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_ebook_other_parquet_files() -> list[Path]:
-    """Re-made LONG other (literature/general) books — long-doc companion to ebook_nf."""
-    d = _CODE_BASE_DATA_DIR / "ebook_long" / "other"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_ultrafineweb_parquet_files() -> list[Path]:
-    """OpenBMB Ultra-FineWeb (en) — classifier-filtered web (MiniCPM UltraData L2).
-    The A5 ablation swaps the legacy climbmix ``web`` stream for this higher-quality
-    filtered web. Short docs (web pages), so this is a web-QUALITY lever, not a
-    long-context one."""
-    d = _CODE_BASE_DATA_DIR / "ultra_fineweb" / "data" / "ultrafineweb_en"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_retrieval_parquet_files() -> list[Path]:
-    """Retrieval-shaping synthetic corpus (build_retrieval_stream): long real-prose docs with
-    K facts spread across the body + a shuffled recall index at the end, so predicting the
-    recalled values forces precise long-range retrieval. Long-context capability lever (push
-    reliable needle retrieval past ~64K toward the full 128K)."""
-    d = _CODE_BASE_DATA_DIR / "retrieval_shaping"
-    return sorted(d.glob("*.parquet")) if d.exists() else []
-
-
-def list_multilingual_parquet_files() -> list[Path]:
-    """The 18 高人口/高 GDP 亚非拉 languages (build_multilingual_stream): per-language shards under
-    ``multilingual/<lang>/`` from FineWeb-2 / CulturaX / MADLAD-400 / Wikipedia. The general-purpose
-    multilingual lever — needs the new multilingual tokenizer to compress non-Latin scripts (see
-    gate_tok_compression / gate_multilingual)."""
-    d = _CODE_BASE_DATA_DIR / "multilingual"
-    return sorted(d.rglob("*.parquet")) if d.exists() else []  # nested per-language subdirs
-
-
-# Stream name -> lister. ``web`` is handled separately (legacy-fallback aware).
-NON_WEB_LISTERS = {
-    "owm": list_owm_parquet_files,
-    "code": list_code_parquet_files,
-    "books": list_books_parquet_files,
-    "chinese": list_chinese_parquet_files,
-    "finemath": list_finemath_parquet_files,
-    "mathtb": list_mathtb_parquet_files,
-    "swecpt": list_swecpt_parquet_files,
-    "commitdiff": list_commitdiff_parquet_files,
-    "codetests": list_codetests_parquet_files,
-    "nvcode": list_nvcode_parquet_files,
-    "swe1b": list_swe1b_parquet_files,
-    # re-made LONG books (long-context decay source); chunked ``books`` kept for the
-    # A0 baseline comparison.
-    "ebook_nf": list_ebook_nf_parquet_files,
-    "ebook_other": list_ebook_other_parquet_files,
-    # filtered-web (MiniCPM UltraData L2); A5 swaps legacy ``web`` -> this.
-    "ultrafineweb": list_ultrafineweb_parquet_files,
-    # retrieval-shaping long docs (push reliable retrieval past ~64K toward 128K).
-    "retrieval": list_retrieval_parquet_files,
-    # the 18 亚非拉 languages (general-purpose multilingual lever).
-    "multilingual": list_multilingual_parquet_files,
-}
-# Additive decision-order slices of [0,1): APPEND-ONLY — reordering would change
-# the realized ratios of every existing recipe (web takes the remainder).
-STREAM_ORDER = ("owm", "code", "books", "chinese", "finemath", "mathtb",
-                "swecpt", "commitdiff", "codetests", "nvcode", "swe1b",
-                "ebook_nf", "ebook_other", "ultrafineweb", "retrieval",
-                "multilingual")
-# Per-stream parquet text column (default "text"). Ultra-FineWeb ships "content".
-_STREAM_TEXT_COL = {"ultrafineweb": "content"}
-
-
-def _resolve_paths(split: str, warn: bool):
-    """Return {name: [train/val paths]} for web + the four non-web streams."""
-    web = [Path(p) for p in _split_paths(list_web_parquet_files(warn_on_legacy=warn), split)]
-    assert web, "No web (climbmix) parquet shards found; run nanoai.dataset first."
-    paths = {"web": web}
-    for name, lister in NON_WEB_LISTERS.items():
-        paths[name] = _split_paths(lister(), split)
-    return paths
-
-
-# --------------------------------------------------------------- doc batches
-def _mixed_document_batches_1b(
-    split: str,
-    ratios: dict[str, float],
-    tokenizer_batch_size: int,
-    seed: int = 0,
-):
-    """Infinite stream of list[str] doc batches drawn from the 5-way mixture.
-
-    ``ratios`` holds per-document draw probabilities for the non-web streams
-    (owm/code/books/chinese); web gets the remainder. Decision order matches
-    ``STREAM_ORDER`` then web — additive slices of [0, 1).
-    """
-    for k in ratios:
-        assert k in NON_WEB_LISTERS, f"unknown stream ratio {k!r}"
-        assert 0.0 <= ratios[k] <= 1.0, f"bad ratio {k}={ratios[k]}"
-    non_web_total = sum(ratios.get(n, 0.0) for n in STREAM_ORDER)
-    assert non_web_total <= 1.0 + 1e-9, f"non-web ratios sum {non_web_total} > 1"
-
-    ddp, ddp_rank, _local, ddp_world_size = get_dist_info()
-    warn = (ddp_rank == 0) and (split == "train")
-    paths = _resolve_paths(split, warn)
-
-    for name in STREAM_ORDER:
-        if ratios.get(name, 0.0) > 0 and not paths[name]:
-            raise FileNotFoundError(
-                f"{name}_ratio>0 but no parquet found for stream '{name}'. "
-                f"Expected under base_dir; see pretrain_mix_1b stream table."
-            )
-
-    iters = {
-        "web": _doc_batches_robust(paths["web"], ddp_rank, ddp_world_size, tokenizer_batch_size, split)
-    }
-    for name in STREAM_ORDER:
-        iters[name] = (
-            _doc_batches_robust(paths[name], ddp_rank, ddp_world_size, tokenizer_batch_size, split,
-                                text_col=_STREAM_TEXT_COL.get(name, "text"))
-            if paths[name] else None
-        )
-
-    if ddp_rank == 0:
-        web_ratio = max(0.0, 1.0 - non_web_total)
-        shard_str = " ".join(f"{n}={len(paths[n])}" for n in ("web", *STREAM_ORDER))
-        ratio_str = " ".join(
-            f"{n}={ratios.get(n, 0.0):.3f}" for n in STREAM_ORDER
-        )
-        print0(f"[mix-1b] split={split} web={web_ratio:.3f} {ratio_str} | shards {shard_str}")
-
-    rng = random.Random(seed + ddp_rank)
-    counts = {n: 0 for n in ("web", *STREAM_ORDER)}
-    log_every = 10000
-    while True:
-        r = rng.random()
-        acc = 0.0
-        chosen = "web"
-        for name in STREAM_ORDER:
-            acc += ratios.get(name, 0.0)
-            if iters[name] is not None and r < acc:
-                chosen = name
-                break
-        counts[chosen] += 1
-        yield next(iters[chosen])
-        total = sum(counts.values())
-        if ddp_rank == 0 and total % log_every == 0:
-            frac = " ".join(f"{n}={counts[n]/total:.3f}" for n in ("web", *STREAM_ORDER))
-            print0(f"[mix-1b] draws={total} {frac}")
-
-
-# --------------------------------------------------------------- bestfit packer
-def tokenizing_mixed_1b(
-    tokenizer,
-    B: int,
-    T: int,
-    split: str,
-    ratios: dict[str, float],
-    tokenizer_threads: int = 4,
-    tokenizer_batch_size: int = 128,
-    device: str = "cuda",
-    buffer_size: int = 1000,
-    seed: int = 0,
-):
-    """BOS-aligned best-fit packing over the 5-way mixture. Yields (inputs,
-    targets) tensors of shape (B, T). Drop-in for nanoai's
-    ``tokenizing_distributed_data_loader_bos_bestfit``.
-    """
-    assert split in ("train", "val")
-    row_capacity = T + 1
-    bos_token = tokenizer.get_bos_token_id()
-
-    batches = _mixed_document_batches_1b(split, ratios, tokenizer_batch_size, seed=seed)
-    doc_buffer: list[list[int]] = []
-
-    def refill_buffer():
-        doc_batch = next(batches)
-        token_lists = tokenizer.encode(doc_batch, prepend=bos_token, num_threads=tokenizer_threads)
-        for tokens in token_lists:
-            doc_buffer.append(tokens)
-
-    use_cuda = device == "cuda"
-    row_buffer = torch.empty((B, row_capacity), dtype=torch.long)
-    cpu_buffer = torch.empty(2 * B * T, dtype=torch.long, pin_memory=use_cuda)
-    gpu_buffer = torch.empty(2 * B * T, dtype=torch.long, device=device)
-    cpu_inputs = cpu_buffer[: B * T].view(B, T)
-    cpu_targets = cpu_buffer[B * T :].view(B, T)
-    inputs = gpu_buffer[: B * T].view(B, T)
-    targets = gpu_buffer[B * T :].view(B, T)
-
-    while True:
-        for row_idx in range(B):
-            pos = 0
-            while pos < row_capacity:
-                while len(doc_buffer) < buffer_size:
-                    refill_buffer()
-                remaining = row_capacity - pos
-
-                best_idx = -1
-                best_len = 0
-                for i, doc in enumerate(doc_buffer):
-                    doc_len = len(doc)
-                    if doc_len <= remaining and doc_len > best_len:
-                        best_idx = i
-                        best_len = doc_len
-
-                if best_idx >= 0:
-                    doc = doc_buffer.pop(best_idx)
-                    doc_len = len(doc)
-                    row_buffer[row_idx, pos : pos + doc_len] = torch.tensor(doc, dtype=torch.long)
-                    pos += doc_len
-                else:
-                    shortest_idx = min(range(len(doc_buffer)), key=lambda i: len(doc_buffer[i]))
-                    doc = doc_buffer.pop(shortest_idx)
-                    row_buffer[row_idx, pos : pos + remaining] = torch.tensor(
-                        doc[:remaining], dtype=torch.long
-                    )
-                    pos += remaining
-
-        cpu_inputs.copy_(row_buffer[:, :-1])
-        cpu_targets.copy_(row_buffer[:, 1:])
-        gpu_buffer.copy_(cpu_buffer, non_blocking=use_cuda)
-        yield inputs, targets
-
-
-def tokenizing_mixed_1b_with_state(
-    tokenizer,
-    B: int,
-    T: int,
-    split: str,
-    ratios: dict[str, float],
-    tokenizer_threads: int = 4,
-    tokenizer_batch_size: int = 128,
-    device: str = "cuda",
-    resume_state_dict=None,
-    buffer_size: int = 1000,
-    seed: int = 0,
-):
-    """Same as ``tokenizing_mixed_1b`` but yields a third stub ``state_dict`` to
-    match ``tokenizing_distributed_data_loader_with_state_bos_bestfit``.
-    Caller's ``resume_state_dict`` is ignored (multi-stream resume not impl.)."""
-    base = tokenizing_mixed_1b(
-        tokenizer, B, T, split, ratios,
-        tokenizer_threads=tokenizer_threads,
-        tokenizer_batch_size=tokenizer_batch_size,
-        device=device, buffer_size=buffer_size, seed=seed,
-    )
-    stub_state = {"epoch": 1, "pq_idx": 0, "rg_idx": 0}
-    for inputs, targets in base:
-        yield inputs, targets, stub_state
-
-
-# --------------------------------------------------------------- tokenizer-train iterator
-def stream_text_iterator(ratios: dict[str, float], doc_cap: int, max_chars: int, seed: int = 0):
-    """Yield individual doc strings drawn from the 5-way mixture (single rank),
-    each truncated to ``doc_cap`` chars, stopping after ``max_chars`` total.
-
-    Used by ``scripts/tok_train_1b.py`` so the BPE sees a char distribution that
-    matches the pretrain mix (English / math / code / textbooks / Chinese).
-    """
-    batches = _mixed_document_batches_1b("train", ratios, tokenizer_batch_size=128, seed=seed)
-    nchars = 0
-    while nchars < max_chars:
-        batch = next(batches)
-        for doc in batch:
-            doc = doc[:doc_cap]
-            nchars += len(doc)
-            yield doc
-            if nchars >= max_chars:
-                return

vendor/nanoai/agentic/data/routing_recovery.py

@@ -1,357 +0,0 @@
-"""Synthetic tool-routing recovery dataset.
-
-Motivation (POC-8 mechanistic analysis):
-  v9 (lc=10) regressed from v5 by 5.3pp because long_ctx data lacks tool calls,
-  causing catastrophic forgetting of *which tool to use for which task type*.
-  Failed cases (read_config, grep_torch, bash_grep_log) showed v9 emitting
-  Edit-only with degenerate repetitive content where v5 emitted the correct
-  Read/Grep/Bash chain.
-
-  This dataset teaches the model the prompt-shape → first-tool mapping
-  directly: 4 tool families × ~50 prompt templates × 4-5 surface variants =
-  ~200 rows per tool, ~800 rows total. Each row is a 4-message conversation
-  (user → assistant tool_call → synthetic tool_result → assistant answer).
-
-Format on disk: JSONL, one `{"messages": [...]}` per line, compatible with
-`render_agentic_conversation`'s tool-call branch (assistant turn carries
-both `content` and `tool_call: {name, args}`).
-
-CLI:
-  python -m nanoai.agentic.data.routing_recovery \
-    --output data/routing_recovery.jsonl --seed 42 --n-per-tool 200
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import random
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-from nanoai.agentic.data.common import SYSTEM_PROMPT
-
-
-# ---- File / pattern / path corpora (deterministic per-seed shuffling) -------
-
-_FILES_PY = [
-    "main.py", "app.py", "lib.py", "utils.py", "config.py", "server.py",
-    "client.py", "models.py", "schema.py", "data.py", "tools.py", "test.py",
-    "parser.py", "router.py", "loader.py", "engine.py", "core.py", "common.py",
-    "tasks.py", "errors.py",
-]
-_FILES_DATA = [
-    "config.toml", "settings.json", "data.csv", "users.json", "log.txt",
-    "README.md", "Makefile", "requirements.txt", "pyproject.toml", ".env",
-    "dataset.json", "errors.log", "access.log", "metrics.tsv", "manifest.yaml",
-]
-_DIRS = ["src", "lib", "tests", "scripts", "data", "config", ".", "core", "app"]
-_PATTERNS = [
-    "TODO", "FIXME", "import torch", "def __init__", "raise ValueError",
-    "class .*Error", "logging.basicConfig", "yield from", "async def",
-    "from typing import", "with pytest.raises", "@dataclass",
-    "logger.info", "self\\.config", "return None",
-]
-_BASH_TARGETS = [
-    "list files", "count lines", "show disk usage", "show file size",
-    "find files larger than 1mb", "show last 10 lines", "show first 5 lines",
-    "show file modification time",
-]
-
-
-# ---- Templates per tool family ---------------------------------------------
-#
-# Each template is a function (rng) -> {prompt, tool_name, tool_args,
-#                                       observation, final_answer}.
-
-@dataclass
-class _Row:
-    user_prompt: str
-    pre_text: str        # short assistant pre-text before tool_call
-    tool_name: str
-    tool_args: dict[str, str]
-    tool_result: str     # synthetic observation
-    final_text: str      # assistant's answer after seeing observation
-
-
-def _read_row(rng: random.Random) -> _Row:
-    f = rng.choice(_FILES_PY + _FILES_DATA)
-    template = rng.choice([
-        ("what's in {f}?", "let me read it"),
-        ("show me {f}", "i'll open it"),
-        ("can you read {f}?", "reading"),
-        ("open {f} and tell me what it does", "i'll read the file first"),
-        ("what does {f} say?", "checking"),
-        ("read {f} please", "ok"),
-        ("display the contents of {f}", "ok"),
-        ("look at {f}", "reading"),
-        ("how is {f} structured?", "let me see"),
-        ("give me the source of {f}", "fetching"),
-    ])
-    prompt = template[0].format(f=f)
-    pre = template[1]
-    if f.endswith(".py"):
-        body = f"def main():\n    print('hello from {f}')\n\nif __name__ == '__main__':\n    main()"
-        ans = f"{f} defines a `main()` that prints a hello banner; runs only when invoked directly."
-    elif f == "config.toml":
-        ans = "the version is 0.1.0; package name is `nanocode`."
-        body = "[package]\nname = 'nanocode'\nversion = '0.1.0'\n"
-    elif f == "README.md":
-        body = "# Project\n\nA small library.\n\n## Install\n`pip install .`\n"
-        ans = "small library titled 'Project' with a single install instruction."
-    elif f.endswith(".json"):
-        body = '{"version": "1.2", "users": [{"id": 1}, {"id": 2}]}'
-        ans = "JSON config with version 1.2 and two users."
-    elif f.endswith(".log"):
-        body = "2026-05-07 INFO start\n2026-05-07 INFO ready\n"
-        ans = "two INFO log lines, both from 2026-05-07."
-    else:
-        body = "# placeholder file body\n"
-        ans = f"{f} is a small placeholder; nothing surprising."
-    return _Row(
-        user_prompt=prompt,
-        pre_text=pre,
-        tool_name="Read",
-        tool_args={"file_path": f},
-        tool_result=body,
-        final_text=ans,
-    )
-
-
-def _grep_row(rng: random.Random) -> _Row:
-    pat = rng.choice(_PATTERNS)
-    where = rng.choice(_DIRS)
-    template = rng.choice([
-        ("find {pat} in {where}", "let me search"),
-        ("where is {pat} used in {where}?", "grepping"),
-        ("search for {pat} under {where}", "ok"),
-        ("any files containing {pat}?", "let me check"),
-        ("show me uses of {pat} in {where}", "searching"),
-        ("does {pat} appear anywhere?", "looking"),
-        ("locate {pat} in the codebase", "grepping the tree"),
-        ("which files reference {pat}?", "checking"),
-        ("grep {pat}", "running grep"),
-        ("are there any {pat} markers?", "let me see"),
-    ])
-    prompt = template[0].format(pat=pat, where=where)
-    pre = template[1]
-    n_hits = rng.randint(1, 4)
-    hits = "\n".join(
-        f"{rng.choice(_FILES_PY)}:{rng.randint(1, 200)}: {pat}"
-        for _ in range(n_hits)
-    )
-    if n_hits == 0:
-        ans = f"no matches for {pat}."
-    elif n_hits == 1:
-        ans = f"one hit for {pat} — single-callsite, easy to update."
-    else:
-        ans = f"{n_hits} matches for {pat} across the tree; the {pat.split()[0] if ' ' in pat else pat} pattern is consistent."
-    return _Row(
-        user_prompt=prompt,
-        pre_text=pre,
-        tool_name="Grep",
-        tool_args={"pattern": pat, "path": where},
-        tool_result=hits or "(no matches)",
-        final_text=ans,
-    )
-
-
-def _bash_row(rng: random.Random) -> _Row:
-    f = rng.choice(_FILES_PY + _FILES_DATA)
-    d = rng.choice(_DIRS)
-    choice = rng.choice([
-        ("list", f"list files in {d}", "ls", f"ls {d}",
-         "main.py app.py utils.py tests"),
-        ("count", f"count lines in {f}", "wc", f"wc -l {f}",
-         f"{rng.randint(20, 800)} {f}"),
-        ("size", f"what is the size of {f}?", "du", f"du -h {f}",
-         f"{rng.randint(2, 99)}K\t{f}"),
-        ("first", f"show first 5 lines of {f}", "head", f"head -n 5 {f}",
-         "line 1\nline 2\nline 3\nline 4\nline 5"),
-        ("last", f"show the last 10 lines of {f}", "tail", f"tail -n 10 {f}",
-         "tail line a\ntail line b\ntail line c"),
-        ("exists", f"does {f} exist?", "test", f"test -f {f} && echo yes || echo no",
-         "yes"),
-        ("find_large", f"find files larger than 1mb under {d}", "find", f"find {d} -size +1M",
-         f"{d}/data/big.bin"),
-        ("mod_time", f"when was {f} last modified?", "stat", f"stat -c %y {f}",
-         "2026-05-06 14:22:11.000000000 +0000"),
-        ("grep_simple", f"any errors in {f}?", "grep", f"grep -i error {f}",
-         f"{f}:10:    raise ValueError('boom')"),
-    ])
-    kind, prompt, _short, cmd, body = choice
-    pre = rng.choice(["running it", "let me check", "ok", "checking"])
-    ans_map = {
-        "list": f"the {d} directory has 4 files: main.py, app.py, utils.py, and tests.",
-        "count": f"{f} is {body.split()[0]} lines.",
-        "size": f"{f} is {body.split(chr(9))[0]}.",
-        "first": f"first 5 lines are 'line 1' through 'line 5'.",
-        "last": "last lines are 'tail line a/b/c'.",
-        "exists": f"{f} exists.",
-        "find_large": f"one file over 1MB: {body}.",
-        "mod_time": "last modified 2026-05-06 at 14:22.",
-        "grep_simple": f"one error path on line 10 raises ValueError.",
-    }
-    return _Row(
-        user_prompt=prompt,
-        pre_text=pre,
-        tool_name="Bash",
-        tool_args={"command": cmd},
-        tool_result=body,
-        final_text=ans_map[kind],
-    )
-
-
-def _edit_row(rng: random.Random) -> _Row:
-    f = rng.choice(_FILES_PY)
-    edit = rng.choice([
-        ("change_version", f"bump version in {f}",
-         "version = '0.1.0'", "version = '0.2.0'",
-         "version = '0.2.0'"),
-        ("add_docstring", f"add a one-line docstring to {f}",
-         "def main():", "def main():\n    \"\"\"entry point\"\"\"",
-         "def main():\n    \"\"\"entry point\"\"\""),
-        ("rename_var", f"rename `tmp` to `result` in {f}",
-         "tmp = compute()", "result = compute()",
-         "result = compute()"),
-        ("add_import", f"add `import logging` at the top of {f}",
-         "", "import logging",
-         "import logging"),
-        ("fix_typo", f"fix the typo `recieve` → `receive` in {f}",
-         "def recieve(x):", "def receive(x):",
-         "def receive(x):"),
-        ("create", f"create a new file {f} that prints hello",
-         "", "print('hello')",
-         "print('hello')"),
-        ("replace_print", f"replace print() with logger.info() in {f}",
-         "print(x)", "logger.info(x)",
-         "logger.info(x)"),
-        ("add_main_guard", f"wrap the body of {f} in if __name__ == '__main__':",
-         "main()", "if __name__ == '__main__':\n    main()",
-         "if __name__ == '__main__':\n    main()"),
-    ])
-    kind, prompt, old, new, applied = edit
-    pre = rng.choice([
-        "applying the edit", "ok", "let me make that change",
-        "editing", "doing it now",
-    ])
-    args = {"file_path": f, "new_string": new}
-    if old:
-        args["old_string"] = old
-    body = f"applied the edit to {f}: now contains:\n{applied}"
-    ans = rng.choice([
-        f"done — {f} now uses the updated form.",
-        f"applied. {f} is updated.",
-        "edit applied successfully.",
-        f"updated {f} as requested.",
-    ])
-    return _Row(
-        user_prompt=prompt,
-        pre_text=pre,
-        tool_name="Edit",
-        tool_args=args,
-        tool_result=body,
-        final_text=ans,
-    )
-
-
-_GENERATORS: dict[str, Any] = {
-    "Read": _read_row,
-    "Grep": _grep_row,
-    "Bash": _bash_row,
-    "Edit": _edit_row,
-}
-
-
-# ---- Row builder -----------------------------------------------------------
-
-
-def _row_to_messages(r: _Row) -> dict[str, Any]:
-    return {
-        "messages": [
-            {"role": "user", "content": r.user_prompt},
-            {
-                "role": "assistant",
-                "content": r.pre_text,
-                "tool_call": {"name": r.tool_name, "args": r.tool_args},
-            },
-            {"role": "tool_result", "content": r.tool_result},
-            {"role": "assistant", "content": r.final_text},
-        ]
-    }
-
-
-def generate_routing_rows(
-    seed: int = 42,
-    n_per_tool: int = 200,
-    tools: tuple[str, ...] = ("Read", "Grep", "Bash", "Edit"),
-) -> list[dict[str, Any]]:
-    """Generate the routing-recovery rows in memory.
-
-    Deterministic per `seed`; produces `n_per_tool * len(tools)` rows total.
-    """
-    rng = random.Random(seed)
-    out: list[dict[str, Any]] = []
-    for tool in tools:
-        gen = _GENERATORS[tool]
-        for i in range(n_per_tool):
-            row = gen(random.Random(seed * 1_000_003 + hash(tool) + i))
-            out.append(_row_to_messages(row))
-    rng.shuffle(out)
-    return out
-
-
-# ---- SFT-side dataset class ------------------------------------------------
-
-
-class RoutingRecoveryDataset:
-    """In-memory dataset usable by `agentic_sft.py` task mixtures.
-
-    Mirrors the `JSONDataset` protocol (len + getitem + system prompt prepend).
-    Generated synthetically — no on-disk file required, but the CLI below can
-    write a JSONL artefact for inspection / reproducibility.
-    """
-
-    def __init__(
-        self,
-        seed: int = 42,
-        n_per_tool: int = 200,
-        tools: tuple[str, ...] = ("Read", "Grep", "Bash", "Edit"),
-        **_: Any,
-    ) -> None:
-        self.ds = generate_routing_rows(seed=seed, n_per_tool=n_per_tool, tools=tools)
-
-    def __len__(self) -> int:
-        return len(self.ds)
-
-    def __getitem__(self, idx: int) -> dict[str, Any]:
-        messages = self.ds[idx]["messages"]
-        return {"messages": [{"role": "system", "content": SYSTEM_PROMPT}] + messages}
-
-
-# ---- CLI -------------------------------------------------------------------
-
-
-def _main() -> None:
-    p = argparse.ArgumentParser()
-    p.add_argument("--output", required=True, type=Path,
-                   help="Output JSONL path (one {messages:[...]} per line)")
-    p.add_argument("--seed", type=int, default=42)
-    p.add_argument("--n-per-tool", type=int, default=200,
-                   help="Rows per tool family (default 200 → 800 total)")
-    p.add_argument("--tools", type=str, default="Read,Grep,Bash,Edit",
-                   help="Comma-separated tool families to generate")
-    args = p.parse_args()
-
-    tools = tuple(t.strip() for t in args.tools.split(",") if t.strip())
-    rows = generate_routing_rows(seed=args.seed, n_per_tool=args.n_per_tool, tools=tools)
-    args.output.parent.mkdir(parents=True, exist_ok=True)
-    with args.output.open("w", encoding="utf-8") as f:
-        for row in rows:
-            f.write(json.dumps(row, ensure_ascii=False) + "\n")
-    print(f"wrote {len(rows)} rows ({args.n_per_tool}/tool × {len(tools)} tools) → {args.output}")
-
-
-if __name__ == "__main__":
-    _main()

vendor/nanoai/agentic/data/sft_data_1b.py

@@ -1,240 +0,0 @@
-"""1B-challenge SFT data: UltraData-SFT-2605 -> hybrid-thinking (ids, mask).
-
-Self-contained, mirrors the pretrain_mix_1b approach (zero change to validated
-code). Two responsibilities:
-
-1. ``render_sft_example`` — turn ONE UltraData example into ``(ids, mask)`` with
-   the **Qwen3/MiniCPM-style hybrid** layout so a single ckpt can switch
-   think/no-think reliably:
-     think    : <|assistant_start|> <|think_start|> {reasoning} <|think_end|> {answer} <|assistant_end|>
-     no_think : <|assistant_start|> <|think_start|> <|think_end|> {answer} <|assistant_end|>   (EMPTY think block)
-   **mode_tag (hybrid-think switch fix)**: with ``mode_tag=True`` the user turn is
-   suffixed with `` /think`` (think examples) or `` /no_think`` (no_think), mask=0.
-   Without this prompt-side signal the think/no_think prompts are IDENTICAL, so a
-   greedy decoder cannot tell the modes apart and collapses to the majority
-   no-think path (empty <|think_start|><|think_end|>, reasoning leaks into the
-   answer body). The tag is the deterministic control the model conditions on;
-   at inference append the same `` /think``/`` /no_think`` to select the mode.
-   ``mask=1`` on every assistant token the model must GENERATE (think block —
-   incl. the empty one — + answer + assistant_end). **think tokens are emitted
-   via encode_special, NEVER as literal "<|think_start|>" text** (literal text
-   goes through encode_ordinary and is NOT the special token — the footgun that
-   tanked KSR Phase 3; see lesson_think_tokens_unencoded).
-
-2. ``iter_and_sample`` / ``prepare_sft_cache`` — stream the 7 gated configs with
-   the HF token, **budget-match to TARGET TOKEN shares** (render to count tokens),
-   keep the ~think/no_think ratio, run a light eval-set decontam, and write the
-   sampled raw conversations to a jsonl cache. The SFT trainer renders on the fly
-   (like chat_sft) at max_seq_len=32768.
-
-UltraData schema (confirmed): configs {Math,Code,Knowledge,Chinese-general,IF,
-Multi-lang-Math,Multi-lang-Knowledge}; splits {think,no_think}; per example
-``messages=[{role,content,reasoning_content}]`` (reasoning_content = long CoT on
-the assistant turn for think; null for no_think), plus uid/source/domain/think_type.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import random
-
-# ---------------------------------------------------------------- rendering
-
-def _normalize_messages(messages):
-    """Merge a leading system message into the first user turn (mirrors
-    tokenizer.render_conversation); return the cleaned message list."""
-    if messages and messages[0].get("role") == "system":
-        if len(messages) >= 2 and messages[1].get("role") == "user":
-            merged = dict(messages[1])
-            merged["content"] = (messages[0].get("content") or "") + "\n\n" + (messages[1].get("content") or "")
-            return [merged] + list(messages[2:])
-        return list(messages[1:])
-    return list(messages)
-
-
-# Prompt-side hybrid-think control tags (Qwen3-style). Appended to user turns
-# when mode_tag=True so the model conditions on the requested mode. Import these
-# in inference/eval code to activate thinking deterministically.
-THINK_TAG = " /think"
-NOTHINK_TAG = " /no_think"
-
-
-def render_sft_example(tokenizer, messages, think: bool, max_tokens: int = 32768,
-                       mode_tag: bool = False):
-    """One UltraData example -> (ids, mask). See module docstring for layout.
-
-    mode_tag=True appends the prompt-side ``/think`` / ``/no_think`` switch to the
-    user turn (mask=0) so the hybrid-think mode is controllable instead of
-    collapsing to empty-think under greedy decoding.
-    """
-    ids, mask = [], []
-
-    def add(toks, m):
-        if isinstance(toks, int):
-            toks = [toks]
-        ids.extend(toks)
-        mask.extend([m] * len(toks))
-
-    bos = tokenizer.get_bos_token_id()
-    us = tokenizer.encode_special("<|user_start|>")
-    ue = tokenizer.encode_special("<|user_end|>")
-    as_ = tokenizer.encode_special("<|assistant_start|>")
-    ae = tokenizer.encode_special("<|assistant_end|>")
-    ts = tokenizer.encode_special("<|think_start|>")
-    te = tokenizer.encode_special("<|think_end|>")
-
-    add(bos, 0)
-    for m in _normalize_messages(messages):
-        role = m.get("role")
-        content = m.get("content") or ""
-        if role == "user":
-            add(us, 0)
-            if mode_tag:
-                content = content + (THINK_TAG if think else NOTHINK_TAG)
-            add(tokenizer.encode(content), 0)
-            add(ue, 0)
-        elif role == "assistant":
-            add(as_, 0)
-            # think block (empty for no_think) — masked 1 so the switch is learned
-            add(ts, 1)
-            if think:
-                rc = m.get("reasoning_content") or ""
-                if rc:
-                    add(tokenizer.encode(rc), 1)
-            add(te, 1)
-            # final answer — masked 1
-            if content:
-                add(tokenizer.encode(content), 1)
-            add(ae, 1)
-        # any other role is skipped (UltraData is user/assistant only)
-    return ids[:max_tokens], mask[:max_tokens]
-
-
-# ---------------------------------------------------------------- sampling
-
-# TARGET TOKEN shares across configs (sft_design §2). Multi-lang skipped in v1.
-DEFAULT_MIX = {"Math": 0.38, "Code": 0.32, "Knowledge": 0.12,
-               "Chinese-general": 0.10, "IF": 0.08}
-# GENERAL-PURPOSE mix: turns on the UltraData Multi-lang configs (~17% multilingual SFT) for the
-# general_1b recipe, rebalanced to sum to 1.0. Leaves DEFAULT_MIX (the 1b-challenge v1 recipe)
-# untouched — selected via `prepare_sft_cache --mix general`.
-GENERAL_MIX = {"Math": 0.30, "Code": 0.25, "Knowledge": 0.12, "Chinese-general": 0.08,
-               "IF": 0.08, "Multi-lang-Math": 0.09, "Multi-lang-Knowledge": 0.08}
-MIXES = {"default": DEFAULT_MIX, "general": GENERAL_MIX}
-THINK_RATIO = 0.43  # ~43% thinking / 57% non-thinking (UltraData global)
-
-
-def _decontam_ngrams(eval_texts, n=13):
-    grams = set()
-    for t in eval_texts:
-        toks = t.split()
-        for i in range(len(toks) - n + 1):
-            grams.add(" ".join(toks[i:i + n]))
-    return grams
-
-
-def _is_contaminated(messages, bad_grams, n=13):
-    if not bad_grams:
-        return False
-    txt = " ".join((m.get("content") or "") for m in messages)
-    toks = txt.split()
-    for i in range(len(toks) - n + 1):
-        if " ".join(toks[i:i + n]) in bad_grams:
-            return True
-    return False
-
-
-def iter_and_sample(tokenizer, *, mix=None, think_ratio=THINK_RATIO,
-                    total_token_budget=2_000_000_000, max_tokens=32768,
-                    seed=0, hf_token=None, decontam_grams=None, log=print):
-    """Stream UltraData, render to count tokens, yield sampled raw examples
-    (dicts with messages/think/domain/source/uid/n_tokens) until each config's
-    TOKEN sub-budget (mix * budget, split think_ratio) is met."""
-    from datasets import load_dataset
-    mix = mix or DEFAULT_MIX
-    rng = random.Random(seed)
-    if hf_token:
-        os.environ.setdefault("HF_TOKEN", hf_token)
-        os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", hf_token)
-
-    for cfg, share in mix.items():
-        cfg_budget = total_token_budget * share
-        for split, frac in (("think", think_ratio), ("no_think", 1.0 - think_ratio)):
-            sub_budget = cfg_budget * frac
-            got_tok = 0
-            n_skip_contam = 0
-            ds = load_dataset("openbmb/UltraData-SFT-2605", cfg, split=split,
-                              streaming=True)
-            ds = ds.shuffle(seed=seed + hash(cfg + split) % 1000, buffer_size=10000)
-            n_err = 0
-            for ex in ds:
-                if got_tok >= sub_budget:
-                    break
-                try:
-                    msgs = ex.get("messages") or []
-                    if not msgs:
-                        continue
-                    if decontam_grams and _is_contaminated(msgs, decontam_grams):
-                        n_skip_contam += 1
-                        continue
-                    think = (split == "think")
-                    ids, _ = render_sft_example(tokenizer, msgs, think, max_tokens)
-                    n = len(ids)
-                    if n < 8:
-                        continue
-                except Exception:  # one malformed row must not kill an hour-long prep
-                    n_err += 1
-                    if n_err <= 3:
-                        log(f"[sft_data] {cfg}/{split}: skipped bad row ({n_err})")
-                    continue
-                got_tok += n
-                yield {"messages": msgs, "think": think, "domain": ex.get("domain", cfg),
-                       "source": ex.get("source", ""), "uid": ex.get("uid", ""),
-                       "n_tokens": n}
-            log(f"[sft_data] {cfg}/{split}: ~{got_tok:,} tok "
-                f"(budget {sub_budget:,.0f}), decontam-skipped {n_skip_contam}")
-
-
-def prepare_sft_cache(out_path, tokenizer, *, total_token_budget, max_tokens=32768,
-                      seed=0, hf_token=None, eval_texts=None, mix=None):
-    """Write budget-matched sampled conversations to a jsonl cache (raw messages;
-    the trainer re-renders on the fly). Returns summary dict."""
-    decontam = _decontam_ngrams(eval_texts) if eval_texts else None
-    os.makedirs(os.path.dirname(out_path), exist_ok=True)
-    n_ex = tot_tok = 0
-    by_domain, by_think = {}, {"think": 0, "no_think": 0}
-    with open(out_path, "w") as f:
-        for row in iter_and_sample(tokenizer, mix=mix, total_token_budget=total_token_budget,
-                                   max_tokens=max_tokens, seed=seed, hf_token=hf_token,
-                                   decontam_grams=decontam):
-            f.write(json.dumps(row, ensure_ascii=False) + "\n")
-            n_ex += 1
-            tot_tok += row["n_tokens"]
-            by_domain[row["domain"]] = by_domain.get(row["domain"], 0) + row["n_tokens"]
-            by_think["think" if row["think"] else "no_think"] += row["n_tokens"]
-    summary = {"out": out_path, "n_examples": n_ex, "total_tokens": tot_tok,
-               "by_domain_tokens": by_domain, "by_think_tokens": by_think}
-    print("[sft_data] CACHE SUMMARY:", json.dumps(summary, ensure_ascii=False))
-    return summary
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--out", required=True, help="jsonl cache path")
-    ap.add_argument("--token-budget", type=float, default=2e9)
-    ap.add_argument("--max-tokens", type=int, default=32768)
-    ap.add_argument("--seed", type=int, default=0)
-    ap.add_argument("--hf-token", default=os.environ.get("HF_TOKEN", ""))
-    ap.add_argument("--mix", choices=sorted(MIXES), default="default",
-                    help="SFT config mix: 'default' (1b-challenge v1) or 'general' (+multilingual)")
-    args = ap.parse_args()
-    from nanoai.tokenizer import get_tokenizer
-    tok = get_tokenizer()
-    prepare_sft_cache(args.out, tok, total_token_budget=int(args.token_budget),
-                      max_tokens=args.max_tokens, seed=args.seed,
-                      hf_token=args.hf_token or None, mix=MIXES[args.mix])
-
-
-if __name__ == "__main__":
-    main()

vendor/nanoai/agentic/data/synth_data.py

@@ -1,74 +0,0 @@
-"""SYNTH (PleIAs) parquet iterator for d6 ablation pretrain.
-
-Concatenates each row's `query` + `synthetic_reasoning` + `synthetic_answer`
-into a single training document string. Mirrors the iterator interface used
-by `pretrain_data.py` so the BOS-aligned bestfit packer can consume it as a
-drop-in document stream.
-
-Default location: $NANOAI_BASE_DIR/code_pretrain_data/synth-pleias/
-(can be a symlink to the actual SYNTH dir).
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-
-import pyarrow.parquet as pq
-
-from nanoai.common import get_base_dir
-
-
-DATA_DIR = Path(get_base_dir()) / "code_pretrain_data" / "synth-pleias"
-
-
-def list_synth_parquet_files() -> list[Path]:
-    if not DATA_DIR.exists():
-        return []
-    return sorted([DATA_DIR / f for f in DATA_DIR.iterdir() if f.suffix == ".parquet"])
-
-
-_CONCAT_SEP = "\n\n"
-
-
-def _row_to_text(row: dict) -> str:
-    q = row.get("query") or ""
-    r = row.get("synthetic_reasoning") or ""
-    a = row.get("synthetic_answer") or ""
-    return _CONCAT_SEP.join(x for x in (q, r, a) if x)
-
-
-def synth_doc_batches(
-    paths: list[Path],
-    ddp_rank: int,
-    ddp_world_size: int,
-    tokenizer_batch_size: int,
-):
-    """Infinite per-rank iterator: yields list[str] doc batches.
-
-    Each parquet row group is striped across ranks. Each row yields one
-    concatenated `query \\n\\n reasoning \\n\\n answer` document.
-    """
-    cols = ["query", "synthetic_reasoning", "synthetic_answer"]
-    while True:
-        for filepath in paths:
-            pf = pq.ParquetFile(filepath)
-            for rg_idx in range(ddp_rank, pf.num_row_groups, ddp_world_size):
-                rg = pf.read_row_group(rg_idx, columns=cols)
-                rows = rg.to_pylist()
-                batch: list[str] = []
-                for r in rows:
-                    txt = _row_to_text(r)
-                    if not txt:
-                        continue
-                    batch.append(txt)
-                    if len(batch) >= tokenizer_batch_size:
-                        yield batch
-                        batch = []
-                if batch:
-                    yield batch
-
-
-def split_paths(paths: list[Path], split: str) -> list[Path]:
-    if not paths:
-        return []
-    return paths[:-1] if split == "train" else paths[-1:]

vendor/nanoai/agentic/data/tool_chain_demo.py

@@ -1,563 +0,0 @@
-"""Synthetic multi-step tool-chain demonstration dataset.
-
-Motivation (POC-9 mechanistic analysis):
-  v11 (routing_rows=200) lifted single-tool routing accuracy but did NOT
-  recover bash_grep_log — the diagnostic regression task that requires
-  CHAINING bash → grep → read. routing_recovery teaches "given prompt P,
-  call tool T". It cannot teach "given prompt P, call T1, then read its
-  output, then call T2 with args derived from T1's output, then ...".
-
-  This dataset adds explicit multi-step tool-chain trajectories. Each
-  row is a 6-10 message conversation showing: user prompt → tool call →
-  tool result → tool call → tool result → ... → final answer. The
-  intermediate assistant turns reference what the previous tool result
-  said, modeling the read-act-observe loop.
-
-Format on disk: JSONL, one `{"messages": [...]}` per line, compatible
-with `render_agentic_conversation`'s tool-call branch.
-
-Schema invariants (verified by audit):
-  - first message role=user, last role=assistant (no tool_call)
-  - intermediate assistants alternate with tool_results
-  - every assistant with tool_call has args as a dict
-  - row text deterministic for fixed (seed, chain_kind, idx)
-
-CLI:
-  python -m nanoai.agentic.data.tool_chain_demo \
-    --output data/tool_chain_demo.jsonl --seed 42 --n-per-chain 100
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import random
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any
-
-from nanoai.agentic.data.common import SYSTEM_PROMPT
-
-
-# ---- Corpora ---------------------------------------------------------------
-
-_FILES_PY = [
-    "main.py", "app.py", "lib.py", "utils.py", "config.py", "server.py",
-    "client.py", "models.py", "schema.py", "data.py", "tools.py",
-    "parser.py", "router.py", "loader.py", "engine.py", "core.py",
-    "common.py", "tasks.py", "errors.py", "handlers.py", "auth.py",
-]
-_FILES_DATA = [
-    "config.toml", "settings.json", "data.csv", "users.json", "log.txt",
-    "README.md", "Makefile", "requirements.txt", "pyproject.toml",
-    "errors.log", "access.log", "metrics.tsv", "manifest.yaml",
-    "deploy.yaml", "schema.sql",
-]
-_DIRS = ["src", "lib", "tests", "scripts", "data", "config", "core", "app"]
-_PATTERNS = [
-    "TODO", "FIXME", "ERROR", "WARN", "import torch",
-    "def __init__", "raise ValueError", "logger.info",
-    "self.config", "from typing import", "@dataclass",
-]
-
-
-# ---- Step + Row dataclasses ------------------------------------------------
-
-
-@dataclass
-class _Step:
-    """One assistant→tool round-trip inside a chain."""
-    pre_text: str
-    tool_name: str
-    tool_args: dict[str, str]
-    tool_result: str
-
-
-@dataclass
-class _Row:
-    user_prompt: str
-    steps: list[_Step]
-    final_text: str
-    chain_kind: str = ""
-    extra: dict[str, Any] = field(default_factory=dict)
-
-
-# ---- Chain generators ------------------------------------------------------
-#
-# Each generator returns one _Row. They share a per-row rng so determinism
-# holds: identical (seed, chain_kind, idx) → identical bytes.
-
-
-def _bash_grep_read_row(rng: random.Random) -> _Row:
-    """bash → grep → read. Direct attack on bash_grep_log task family."""
-    d = rng.choice(_DIRS)
-    pat = rng.choice(_PATTERNS)
-    target = rng.choice(_FILES_PY)
-    n_files = rng.randint(2, 5)
-    listing_files = rng.sample(_FILES_PY, n_files)
-    if target not in listing_files:
-        listing_files[0] = target
-    listing = "\n".join(f"{d}/{f}" for f in listing_files)
-    grep_hits = "\n".join(
-        f"{d}/{target}:{rng.randint(5, 200)}: {pat}"
-        for _ in range(rng.randint(1, 3))
-    )
-    read_body = (
-        f"# {target}\n"
-        f"def handler():\n"
-        f"    # {pat} fix the broken case\n"
-        f"    raise ValueError('boom')\n"
-    )
-    user = rng.choice([
-        f"in {d}/, find python files containing {pat} and show me the relevant file",
-        f"which file under {d} mentions {pat}? then read it and explain",
-        f"locate {pat} in {d} and tell me what the surrounding code does",
-        f"search for {pat} in {d} python files and read the offending one",
-    ])
-    return _Row(
-        user_prompt=user,
-        steps=[
-            _Step(
-                pre_text=f"let me list the files under {d} first",
-                tool_name="Bash",
-                tool_args={"command": f"ls {d}/*.py"},
-                tool_result=listing,
-            ),
-            _Step(
-                pre_text=f"now grep for {pat} across those files",
-                tool_name="Grep",
-                tool_args={"pattern": pat, "path": d},
-                tool_result=grep_hits,
-            ),
-            _Step(
-                pre_text=f"reading {d}/{target} where the hit is",
-                tool_name="Read",
-                tool_args={"file_path": f"{d}/{target}"},
-                tool_result=read_body,
-            ),
-        ],
-        final_text=(
-            f"{target} has a {pat} marker on a `handler()` that raises ValueError. "
-            f"the relevant block is the broken-case path."
-        ),
-        chain_kind="bash_grep_read",
-    )
-
-
-def _bash_read_row(rng: random.Random) -> _Row:
-    """bash → read. e.g. find largest file then read it."""
-    d = rng.choice(_DIRS)
-    target = rng.choice(_FILES_PY)
-    sizes = sorted(
-        [(rng.randint(1, 80), rng.choice(_FILES_PY)) for _ in range(4)],
-        reverse=True,
-    )
-    if sizes[0][1] == target:
-        winner = sizes[0]
-    else:
-        winner = (sizes[0][0], target)
-    listing = "\n".join(f"{sz}K\t{d}/{name}" for sz, name in sizes[:3] + [winner])
-    read_body = (
-        f"# {target}\n"
-        f"class Engine:\n"
-        f"    def run(self):\n"
-        f"        return 'ok'\n"
-    )
-    user = rng.choice([
-        f"find the largest python file under {d}/ and show me what it does",
-        f"which file in {d} is biggest? read it and summarize",
-        f"check disk usage of {d}/*.py, then open the largest one",
-    ])
-    return _Row(
-        user_prompt=user,
-        steps=[
-            _Step(
-                pre_text=f"checking sizes under {d}",
-                tool_name="Bash",
-                tool_args={"command": f"du -k {d}/*.py | sort -rn | head -5"},
-                tool_result=listing,
-            ),
-            _Step(
-                pre_text=f"the largest is {target}, reading it",
-                tool_name="Read",
-                tool_args={"file_path": f"{d}/{target}"},
-                tool_result=read_body,
-            ),
-        ],
-        final_text=(
-            f"{target} is the largest under {d} ({winner[0]}K). it defines an "
-            f"`Engine` class with a `run()` method that returns 'ok'."
-        ),
-        chain_kind="bash_read",
-    )
-
-
-def _grep_read_row(rng: random.Random) -> _Row:
-    """grep → read. Find a hit, then open the file to inspect context."""
-    pat = rng.choice(_PATTERNS)
-    target = rng.choice(_FILES_PY)
-    where = rng.choice(_DIRS)
-    line = rng.randint(10, 200)
-    hits = (
-        f"{where}/{target}:{line}: {pat}\n"
-        f"{where}/other.py:{rng.randint(5, 50)}: {pat}"
-    )
-    read_body = (
-        f"# {target} excerpt around line {line}\n"
-        f"def call_site():\n"
-        f"    # {pat}: under review\n"
-        f"    return None\n"
-    )
-    user = rng.choice([
-        f"grep for {pat} under {where}/ and show me the first hit's surrounding code",
-        f"where does {pat} appear in {where}? open the file and explain",
-        f"find {pat} in {where} python files and read the matching one",
-        f"search {pat} in {where}/ then dump the file context",
-    ])
-    return _Row(
-        user_prompt=user,
-        steps=[
-            _Step(
-                pre_text="grepping",
-                tool_name="Grep",
-                tool_args={"pattern": pat, "path": where},
-                tool_result=hits,
-            ),
-            _Step(
-                pre_text=f"top hit is {where}/{target}:{line}, reading it",
-                tool_name="Read",
-                tool_args={"file_path": f"{where}/{target}"},
-                tool_result=read_body,
-            ),
-        ],
-        final_text=(
-            f"{target} has the {pat} marker on a `call_site()` function "
-            f"around line {line}; it's an inert TODO-style note."
-        ),
-        chain_kind="grep_read",
-    )
-
-
-def _read_edit_row(rng: random.Random) -> _Row:
-    """read → edit. Open a file, make a targeted change."""
-    target = rng.choice(_FILES_PY + ["config.py", "config.toml"])
-    edit = rng.choice([
-        ("bump_version",
-         f"version = '0.1.0'", "version = '0.2.0'",
-         "bump the version to 0.2.0"),
-        ("rename_var",
-         "tmp = compute()", "result = compute()",
-         "rename `tmp` to `result`"),
-        ("add_docstring",
-         "def main():", "def main():\n    \"\"\"entry point\"\"\"",
-         "add a one-line docstring to main()"),
-        ("fix_typo",
-         "def recieve(x):", "def receive(x):",
-         "fix the typo recieve → receive"),
-        ("replace_print",
-         "print(x)", "logger.info(x)",
-         "replace the print() with logger.info()"),
-    ])
-    kind, old, new, ask = edit
-    read_body = f"# {target}\n{old}\n# (other lines)\n"
-    user = rng.choice([
-        f"in {target}, {ask}",
-        f"open {target} and {ask}",
-        f"please {ask} in {target}",
-    ])
-    return _Row(
-        user_prompt=user,
-        steps=[
-            _Step(
-                pre_text=f"reading {target} first",
-                tool_name="Read",
-                tool_args={"file_path": target},
-                tool_result=read_body,
-            ),
-            _Step(
-                pre_text=f"applying the edit ({kind})",
-                tool_name="Edit",
-                tool_args={
-                    "file_path": target,
-                    "old_string": old,
-                    "new_string": new,
-                },
-                tool_result=f"applied edit to {target}: replaced 1 occurrence.",
-            ),
-        ],
-        final_text=f"done — {target} now has the {kind.replace('_', ' ')} applied.",
-        chain_kind="read_edit",
-    )
-
-
-def _read_bash_row(rng: random.Random) -> _Row:
-    """read → bash. Read a script to find the right invocation, then run."""
-    target = rng.choice(["run.sh", "build.sh", "test.sh", "deploy.sh"])
-    flag = rng.choice(["--verbose", "--dry-run", "--prod", "--config=staging"])
-    read_body = (
-        f"#!/usr/bin/env bash\n"
-        f"# {target}\n"
-        f"# Usage: {target} {flag}\n"
-        f"echo running with {flag}\n"
-    )
-    bash_out = f"running with {flag}\n"
-    user = rng.choice([
-        f"check what flag {target} expects, then run it",
-        f"read {target} and execute it with the right argument",
-        f"open {target} to find the usage line, then run it accordingly",
-    ])
-    return _Row(
-        user_prompt=user,
-        steps=[
-            _Step(
-                pre_text=f"reading {target} to find the flag",
-                tool_name="Read",
-                tool_args={"file_path": target},
-                tool_result=read_body,
-            ),
-            _Step(
-                pre_text=f"running it with {flag}",
-                tool_name="Bash",
-                tool_args={"command": f"bash {target} {flag}"},
-                tool_result=bash_out,
-            ),
-        ],
-        final_text=f"ran {target} with {flag} — output confirms the run completed.",
-        chain_kind="read_bash",
-    )
-
-
-def _read_grep_read_row(rng: random.Random) -> _Row:
-    """read → grep → read. Open a manifest, grep for a referenced symbol,
-    open the file containing the symbol."""
-    manifest = rng.choice(["README.md", "manifest.yaml", "pyproject.toml"])
-    entry = rng.choice(_FILES_PY)
-    symbol = rng.choice(["main", "run", "handler", "boot", "serve"])
-    target = rng.choice(_FILES_PY)
-    line = rng.randint(10, 100)
-    manifest_body = (
-        f"# {manifest}\n"
-        f"entry: {entry}\n"
-        f"main_symbol: {symbol}\n"
-    )
-    grep_hits = f"{target}:{line}: def {symbol}():"
-    read_body = (
-        f"# {target}\n"
-        f"def {symbol}():\n"
-        f"    return 'started'\n"
-    )
-    user = rng.choice([
-        f"open {manifest}, find the main symbol, then locate its definition",
-        f"read {manifest} to find the entry symbol, then grep for and read it",
-        f"check {manifest} for the main_symbol, then track it down in source",
-    ])
-    return _Row(
-        user_prompt=user,
-        steps=[
-            _Step(
-                pre_text=f"reading {manifest}",
-                tool_name="Read",
-                tool_args={"file_path": manifest},
-                tool_result=manifest_body,
-            ),
-            _Step(
-                pre_text=f"grepping for `def {symbol}`",
-                tool_name="Grep",
-                tool_args={"pattern": f"def {symbol}", "path": "."},
-                tool_result=grep_hits,
-            ),
-            _Step(
-                pre_text=f"reading {target} where {symbol} is defined",
-                tool_name="Read",
-                tool_args={"file_path": target},
-                tool_result=read_body,
-            ),
-        ],
-        final_text=(
-            f"`{symbol}` is defined in {target} (line {line}); it's a one-liner "
-            f"that returns 'started'."
-        ),
-        chain_kind="read_grep_read",
-    )
-
-
-def _bash_read_edit_row(rng: random.Random) -> _Row:
-    """bash → read → edit. find a config file, open it, edit it."""
-    name = rng.choice(["config.toml", "settings.json", "deploy.yaml"])
-    where = rng.choice(_DIRS)
-    found_path = f"./{where}/{name}"
-    listing = f"{found_path}\n"
-    if name.endswith(".toml"):
-        old = "log_level = 'INFO'"
-        new = "log_level = 'DEBUG'"
-        ask = "set log_level to DEBUG"
-    elif name.endswith(".json"):
-        old = '"timeout": 30'
-        new = '"timeout": 60'
-        ask = "raise the timeout to 60"
-    else:
-        old = "replicas: 1"
-        new = "replicas: 3"
-        ask = "scale replicas to 3"
-    read_body = f"# {name}\n{old}\n# (other config)\n"
-    user = rng.choice([
-        f"find {name} in the repo, read it, and {ask}",
-        f"locate {name}, open it, then {ask}",
-        f"where is {name}? open it and {ask}",
-    ])
-    return _Row(
-        user_prompt=user,
-        steps=[
-            _Step(
-                pre_text=f"finding {name}",
-                tool_name="Bash",
-                tool_args={"command": f"find . -name {name}"},
-                tool_result=listing,
-            ),
-            _Step(
-                pre_text=f"reading {found_path}",
-                tool_name="Read",
-                tool_args={"file_path": found_path},
-                tool_result=read_body,
-            ),
-            _Step(
-                pre_text="applying the edit",
-                tool_name="Edit",
-                tool_args={
-                    "file_path": found_path,
-                    "old_string": old,
-                    "new_string": new,
-                },
-                tool_result=f"applied edit to {found_path}.",
-            ),
-        ],
-        final_text=f"updated {found_path}: {ask}.",
-        chain_kind="bash_read_edit",
-    )
-
-
-# ---- Registry --------------------------------------------------------------
-
-
-_GENERATORS: dict[str, tuple[Any, int]] = {
-    # name: (generator_fn, default_n_rows)
-    "bash_grep_read":  (_bash_grep_read_row,  150),
-    "bash_read":       (_bash_read_row,       100),
-    "grep_read":       (_grep_read_row,       100),
-    "read_edit":       (_read_edit_row,       100),
-    "read_bash":       (_read_bash_row,        80),
-    "read_grep_read":  (_read_grep_read_row,   80),
-    "bash_read_edit":  (_bash_read_edit_row,   80),
-}
-
-
-# ---- Row builder -----------------------------------------------------------
-
-
-def _row_to_messages(r: _Row) -> dict[str, Any]:
-    msgs: list[dict[str, Any]] = [{"role": "user", "content": r.user_prompt}]
-    for step in r.steps:
-        msgs.append({
-            "role": "assistant",
-            "content": step.pre_text,
-            "tool_call": {"name": step.tool_name, "args": step.tool_args},
-        })
-        msgs.append({"role": "tool_result", "content": step.tool_result})
-    msgs.append({"role": "assistant", "content": r.final_text})
-    return {"messages": msgs, "_chain_kind": r.chain_kind}
-
-
-def generate_tool_chain_rows(
-    seed: int = 42,
-    n_per_chain: int = 100,
-    chains: tuple[str, ...] | None = None,
-) -> list[dict[str, Any]]:
-    """Generate the tool-chain demo rows in memory.
-
-    Deterministic per `seed`. Per-chain row counts:
-      n_per_chain == 100 (default): use chain-specific defaults from
-        _GENERATORS (sums to ~690).
-      otherwise: use n_per_chain rows for EVERY chain (uniform).
-    Pass `chains` to subset.
-    """
-    chain_keys = chains if chains is not None else tuple(_GENERATORS.keys())
-    rng = random.Random(seed)
-    out: list[dict[str, Any]] = []
-    for chain_idx, chain in enumerate(chain_keys):
-        gen, default_n = _GENERATORS[chain]
-        n = default_n if n_per_chain == 100 else n_per_chain
-        for i in range(n):
-            row_rng = random.Random(seed * 7919 + chain_idx * 1_000_003 + i)
-            row = gen(row_rng)
-            out.append(_row_to_messages(row))
-    rng.shuffle(out)
-    return out
-
-
-# ---- SFT-side dataset class ------------------------------------------------
-
-
-class ToolChainDemoDataset:
-    """In-memory dataset usable by `agentic_sft.py` task mixtures.
-
-    Mirrors the JSONDataset / RoutingRecoveryDataset protocol.
-    Drops the `_chain_kind` debug key before yielding messages.
-    """
-
-    def __init__(
-        self,
-        seed: int = 42,
-        n_per_chain: int = 100,
-        chains: tuple[str, ...] | None = None,
-        **_: Any,
-    ) -> None:
-        self.ds = generate_tool_chain_rows(
-            seed=seed, n_per_chain=n_per_chain, chains=chains,
-        )
-
-    def __len__(self) -> int:
-        return len(self.ds)
-
-    def __getitem__(self, idx: int) -> dict[str, Any]:
-        messages = self.ds[idx]["messages"]
-        return {"messages": [{"role": "system", "content": SYSTEM_PROMPT}] + messages}
-
-
-# ---- CLI -------------------------------------------------------------------
-
-
-def _main() -> None:
-    p = argparse.ArgumentParser()
-    p.add_argument("--output", required=True, type=Path,
-                   help="Output JSONL path (one {messages:[...]} per line)")
-    p.add_argument("--seed", type=int, default=42)
-    p.add_argument("--n-per-chain", type=int, default=100,
-                   help="If 100: use chain-specific default counts (~690 total). "
-                        "Otherwise: use this count for every chain.")
-    p.add_argument("--chains", type=str, default="",
-                   help="Comma-separated chain names; empty = all "
-                        "(bash_grep_read,bash_read,grep_read,read_edit,"
-                        "read_bash,read_grep_read,bash_read_edit)")
-    args = p.parse_args()
-
-    chains: tuple[str, ...] | None
-    if args.chains:
-        chains = tuple(c.strip() for c in args.chains.split(",") if c.strip())
-    else:
-        chains = None
-    rows = generate_tool_chain_rows(
-        seed=args.seed, n_per_chain=args.n_per_chain, chains=chains,
-    )
-    args.output.parent.mkdir(parents=True, exist_ok=True)
-    with args.output.open("w", encoding="utf-8") as f:
-        for row in rows:
-            f.write(json.dumps(row, ensure_ascii=False) + "\n")
-    summary = ", ".join(
-        f"{c}={sum(1 for r in rows if r.get('_chain_kind') == c)}"
-        for c in (chains or _GENERATORS.keys())
-    )
-    print(f"wrote {len(rows)} rows ({summary}) → {args.output}")
-
-
-if __name__ == "__main__":
-    _main()

vendor/nanoai/agentic/data/vrcpt_data.py

@@ -1,195 +0,0 @@
-"""VR-CPT pretrain stream (verifier-mined case docs).
-
-Materializes a domain-aligned stream of REF.md §9.1 case documents produced by
-``experiments/vrmining/miners/case_writer.py``. Source JSONL is normally at::
-
-    experiments/vrmining/datasets/cpt_corpus.jsonl
-
-Each row has a ``text`` field containing a markdown case (~500-1500 token).
-1k mining → 1000 cases ≈ 1M tokens. Used as a domain mixin alongside web/code
-in `agentic_pretrain.py --vrcpt-ratio` (see [[rec_4axis_pretrain]] Recipe A).
-
-Mirrors `nanoai.agentic.data.pcap_data` exactly — converts JSONL to parquet
-shards stored under ``$NANOAI_BASE_DIR/code_pretrain_data/vrcpt-cases/`` so
-the existing parquet-iter API drives it identically to web / code / pcap-corpus.
-
-Run once before pretraining:
-
-    python -m nanoai.agentic.data.vrcpt_data prepare \\
-        --src experiments/vrmining/datasets/cpt_corpus.jsonl \\
-        --rows-per-shard 500
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import time
-from pathlib import Path
-
-import pyarrow as pa
-import pyarrow.parquet as pq
-
-from nanoai.common import print0
-from nanoai.agentic.data.pretrain_data import DATA_DIR
-
-DATASET_NAME = "vrcpt-cases"
-_TMP_SUFFIX = ".parquet.tmp"
-
-
-def vrcpt_corpus_dir() -> Path:
-    return DATA_DIR / DATASET_NAME
-
-
-def _stream_jsonl_text(src: Path):
-    with src.open("r", encoding="utf-8") as f:
-        for line in f:
-            line = line.strip()
-            if not line:
-                continue
-            obj = json.loads(line)
-            text = obj.get("text")
-            if isinstance(text, str) and text:
-                yield text
-
-
-def _try_open_shard(path: Path) -> bool:
-    try:
-        pf = pq.ParquetFile(path)
-        return pf.num_row_groups > 0
-    except Exception:
-        return False
-
-
-def _validate_existing_corpus(out_dir: Path) -> tuple[bool, list[Path], str]:
-    parquets = sorted(p for p in out_dir.iterdir() if p.suffix == ".parquet")
-    tmps = [p for p in out_dir.iterdir() if p.name.endswith(_TMP_SUFFIX)]
-    if tmps:
-        return False, parquets, f"{len(tmps)} stale {_TMP_SUFFIX} file(s)"
-    if len(parquets) < 2:
-        return False, parquets, f"only {len(parquets)} shard(s); need ≥2"
-    for p in parquets:
-        if not _try_open_shard(p):
-            return False, parquets, f"shard {p.name} unreadable"
-    return True, parquets, "ok"
-
-
-def _quarantine_stale(out_dir: Path, reason_slug: str) -> Path | None:
-    movable = [
-        p for p in out_dir.iterdir()
-        if p.suffix == ".parquet" or p.name.endswith(_TMP_SUFFIX)
-    ]
-    if not movable:
-        return None
-    attic = out_dir / f".attic_{int(time.time())}_{reason_slug}"
-    attic.mkdir()
-    for entry in movable:
-        entry.rename(attic / entry.name)
-    print0(
-        f"[vrcpt_data] quarantined {len(movable)} stale file(s) to {attic.name}/"
-    )
-    return attic
-
-
-def _reason_slug(reason: str) -> str:
-    return reason.split(";")[0].strip().replace(" ", "_")[:32] or "invalid"
-
-
-def prepare_from_jsonl(src: Path, rows_per_shard: int = 500):
-    """One-shot JSONL → parquet shards under vrcpt_corpus_dir().
-
-    Restart-safe and atomic-write (write .tmp then rename), same protocol as
-    pcap_data. Last shard becomes val split (mirrors fineweb-edu convention).
-    """
-    out_dir = vrcpt_corpus_dir()
-    out_dir.mkdir(parents=True, exist_ok=True)
-
-    is_valid, existing, reason = _validate_existing_corpus(out_dir)
-    if is_valid:
-        print0(f"[vrcpt_data] {len(existing)} valid shards already in {out_dir}; skipping.")
-        return existing
-    if existing or any(p.name.endswith(_TMP_SUFFIX) for p in out_dir.iterdir()):
-        print0(f"[vrcpt_data] existing corpus at {out_dir} INVALID ({reason}); rebuilding.")
-        _quarantine_stale(out_dir, _reason_slug(reason))
-
-    assert src.exists(), f"Source JSONL not found: {src}"
-    print0(f"[vrcpt_data] reading {src} → {out_dir} ({rows_per_shard} rows/shard)")
-
-    shard_idx = 0
-    buf: list[str] = []
-    written: list[Path] = []
-    total_docs = 0
-    total_chars = 0
-
-    def flush():
-        nonlocal shard_idx, buf
-        if not buf:
-            return
-        shard_path = out_dir / f"shard_{shard_idx:05d}.parquet"
-        tmp_path = shard_path.with_name(shard_path.name + ".tmp")
-        shard_chars = sum(len(t) for t in buf)
-        table = pa.table({"text": buf})
-        pq.write_table(table, tmp_path, compression="snappy")
-        tmp_path.rename(shard_path)
-        written.append(shard_path)
-        size_mb = shard_path.stat().st_size / 1e6
-        print0(
-            f"[vrcpt_data] wrote {shard_path.name} "
-            f"({len(buf)} docs, {shard_chars/1e6:.2f}M chars, {size_mb:.2f} MB)"
-        )
-        shard_idx += 1
-        buf = []
-
-    for text in _stream_jsonl_text(src):
-        buf.append(text)
-        total_docs += 1
-        total_chars += len(text)
-        if len(buf) >= rows_per_shard:
-            flush()
-    flush()
-
-    assert len(written) >= 2, (
-        f"Only {len(written)} shard(s); need ≥2 (train/val). "
-        f"Lower --rows-per-shard or supply a larger JSONL."
-    )
-    est_tokens = total_chars / 4
-    print0(
-        f"[vrcpt_data] done: {len(written)} shards (last is val), "
-        f"{total_docs:,} docs, {total_chars/1e6:.2f}M chars, "
-        f"≈{est_tokens/1e6:.2f}M tokens"
-    )
-    return written
-
-
-def list_vrcpt_parquet_files() -> list[Path]:
-    d = vrcpt_corpus_dir()
-    if not d.exists():
-        return []
-    return sorted([d / f for f in d.iterdir() if f.suffix == ".parquet"])
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Prepare vrcpt-cases parquet shards.")
-    sub = parser.add_subparsers(dest="cmd", required=True)
-    prep = sub.add_parser("prepare", help="convert cpt_corpus.jsonl to parquet shards")
-    prep.add_argument("--src", type=Path, required=True,
-                      help="path to cpt_corpus.jsonl (case_writer output)")
-    prep.add_argument("--rows-per-shard", type=int, default=500)
-    sub.add_parser("list", help="list current vrcpt-cases shards")
-
-    args = parser.parse_args()
-    if args.cmd == "prepare":
-        prepare_from_jsonl(args.src, rows_per_shard=args.rows_per_shard)
-    elif args.cmd == "list":
-        shards = list_vrcpt_parquet_files()
-        if not shards:
-            print(f"No shards in {vrcpt_corpus_dir()}")
-            return
-        total_bytes = sum(s.stat().st_size for s in shards)
-        for s in shards:
-            print(f"  {s.name}  ({s.stat().st_size / 1e6:.2f} MB)")
-        print(f"total: {len(shards)} shards, {total_bytes / 1e6:.1f} MB")
-
-
-if __name__ == "__main__":
-    main()

vendor/nanoai/agentic/dense_verifier.py

@@ -1,247 +0,0 @@
-"""Dense per-turn verifier reward + format penalty.
-
-Designed to break the GFT-TW mode-collapse trap:
-  - Provides a non-zero signal per turn even when the group is at 100% pass,
-    so training doesn't stall on saturated reward.
-  - Penalises the specific degenerate outputs observed across iter-MCTS-DPO,
-    GFT-TW, and AC v1:  `assert assert assert ...`, mangled special tokens
-    like `tool_call_end_end_end.log`, hallucinated tool_result content.
-  - Independent of `task.check` — scores from the tool_result text and the
-    decoded assistant content, so it works even on tasks the terminal grader
-    is too lenient for.
-
-Wire-in points:
-  - `score_turn(turn_dict)` runs after rollout, reads `tool_call`,
-    `tool_result`, `error` fields from the Transcript turn dict.
-  - `format_penalty(decoded_text)` runs over the assistant's text body.
-  - `per_turn_dense_reward(rollout)` returns a list[float] aligned with
-    `rollout.turn_spans`, ready to feed GAE.
-"""
-from __future__ import annotations
-
-import re
-from dataclasses import dataclass
-
-
-# Reward magnitudes (tuned so format penalty > dense > shaping > terminal/N).
-EDIT_OK = 0.4
-EDIT_FAIL = -0.5
-READ_OK = 0.1
-READ_FAIL = -0.5
-BASH_OK = 0.1
-BASH_FAIL = -0.2
-GREP_HIT = 0.2
-GREP_MISS = -0.1
-PARSE_ERROR = -0.5
-
-# Format penalty magnitudes — large so they dominate any same-turn shaping.
-NGRAM_REPEAT_PENALTY = -1.0
-MANGLED_SPECIAL_PENALTY = -0.8
-TRUNCATED_TOOL_CALL_PENALTY = -0.6
-
-# Terminal grounding (kept small — dense reward carries most weight).
-TERMINAL_PASS = 0.5
-TERMINAL_FAIL = -0.3
-
-
-def score_turn(turn: dict) -> float:
-    """Score a single assistant+tool_result turn pair from the Transcript.
-
-    turn: a dict from `Transcript.turns` of the assistant role. The companion
-    tool_result is read from `turn["_tool_result"]` (caller fills it in by
-    pairing assistant + tool_result turns). If no tool_call is present (final
-    answer turn), returns 0.0 — the terminal grounding handles those.
-
-    Returns a float in roughly [-1, +1].
-    """
-    tc = turn.get("tool_call")
-    if not tc:
-        return 0.0
-    name = tc.get("name", "")
-    result_text = turn.get("_tool_result", "") or ""
-    is_error = turn.get("_error", False)
-
-    if name == "<parse_error>":
-        return PARSE_ERROR
-
-    lo = result_text.lower()
-
-    if name == "Edit":
-        # Distinguish a real edit from "old_string not found" / silent no-op.
-        if is_error:
-            return EDIT_FAIL
-        if "successfully" in lo or "replaced" in lo or "wrote" in lo:
-            return EDIT_OK
-        if "not found" in lo or "no match" in lo:
-            return EDIT_FAIL
-        # Edit returned non-empty without an obvious success marker — small +.
-        return EDIT_OK * 0.3
-
-    if name == "Read":
-        if is_error:
-            return READ_FAIL
-        if "file not found" in lo or "no such file" in lo or lo.startswith("error"):
-            return READ_FAIL
-        if len(result_text) > 20:
-            return READ_OK
-        # Empty / very short read result — neither rewarded nor punished much.
-        return 0.0
-
-    if name == "Bash":
-        if is_error:
-            return BASH_FAIL
-        if "traceback" in lo or "modulenotfounderror" in lo or "no such file" in lo:
-            return BASH_FAIL
-        if "exit code 1" in lo or "command not found" in lo:
-            return BASH_FAIL
-        return BASH_OK
-
-    if name == "Grep":
-        if is_error:
-            return GREP_MISS
-        if not result_text.strip() or "no matches" in lo:
-            return GREP_MISS
-        return GREP_HIT
-
-    # Unknown tool — neutral.
-    return 0.0
-
-
-# Special-token leak patterns observed during mode collapse.
-_MANGLED_SPECIAL_PATTERNS = [
-    re.compile(r"tool_call_end_end"),
-    re.compile(r"tool_call_start.*tool_call_start.*tool_call_start"),
-    re.compile(r"(<\|tool_[a-z_]+\|>){3,}"),
-    re.compile(r"end_end_end"),
-]
-_TOKEN_LIKE = re.compile(r"<\|[a-z_]+\|>")
-
-
-def format_penalty(text: str, window: int = 12) -> float:
-    """Penalise degenerate output patterns observed in case-study.
-
-    text: decoded assistant body for a single turn (string).
-    window: how many tail tokens (words) to inspect for n-gram repeat.
-
-    Returns a float in [-1.0, 0.0]. Multiple penalties are summed (clipped at -1.0).
-    """
-    if not text:
-        return 0.0
-    penalty = 0.0
-
-    words = text.split()
-    # 1) n-gram repetition: last `window` words all identical, OR
-    #    any single non-trivial word appears ≥3× in the window, OR
-    #    the most common 3-gram appears ≥ 3 times.
-    if len(words) >= 6:
-        tail = words[-window:] if len(words) >= window else words
-        if len(set(tail[-5:])) == 1:
-            penalty += NGRAM_REPEAT_PENALTY
-        else:
-            # Unigram repeat — catches `assertionassertio × 3` pattern.
-            non_trivial = [w for w in tail if len(w) > 4]
-            if non_trivial:
-                max_unigram_count = max(non_trivial.count(w) for w in set(non_trivial))
-                if max_unigram_count >= 3:
-                    penalty += NGRAM_REPEAT_PENALTY * 0.7
-            # 3-gram repeat density check (catches `assert x; assert x; assert x`).
-            ngrams = [tuple(tail[i:i + 3]) for i in range(len(tail) - 2)]
-            if ngrams:
-                most_common_count = max(ngrams.count(g) for g in set(ngrams))
-                if most_common_count >= 3:
-                    penalty += NGRAM_REPEAT_PENALTY * 0.7
-
-    # 2) Mangled / leaked special tokens in the visible text.
-    for pat in _MANGLED_SPECIAL_PATTERNS:
-        if pat.search(text):
-            penalty += MANGLED_SPECIAL_PENALTY
-            break
-
-    # 3) Special-token leak: the SAME special token repeated ≥3 times,
-    #    e.g. `<|tool_call_start|><|tool_call_start|><|tool_call_start|>...`
-    #    (well-formed multi-arg tool call uses each tag at most twice).
-    found = _TOKEN_LIKE.findall(text)
-    if found:
-        counts = {}
-        for s in found:
-            counts[s] = counts.get(s, 0) + 1
-        if max(counts.values()) >= 3:
-            penalty += TRUNCATED_TOOL_CALL_PENALTY
-
-    # 4) Unclosed tool_call: has `<|tool_call_start|>` but no matching
-    #    `<|tool_call_end|>` (covers cases like
-    #    `<|tool_call_start|>Edit<|tool_arg|>...<|tool_val|>a` that escape
-    #    n-gram + special-token leak checks due to short total length).
-    if "<|tool_call_start|>" in text and "<|tool_call_end|>" not in text:
-        penalty += TRUNCATED_TOOL_CALL_PENALTY
-
-    # 5) Unclosed tool_val: `<|tool_val|>` opens but text ends without further
-    #    structure (catches the edit_create_test PPO v1 regression where the
-    #    model emits `<|tool_val|>a` then stops with len < 10).
-    if "<|tool_val|>" in text:
-        after_val = text.rsplit("<|tool_val|>", 1)[-1]
-        if len(after_val.strip()) < 5 and "<|tool_call_end|>" not in text:
-            penalty += TRUNCATED_TOOL_CALL_PENALTY
-
-    # 6) Brace explosion inside tool_val: model entered "write JSON" mode and
-    #    couldn't escape, producing unmatched runs of `{` or `}`. Catches the
-    #    edit_create_json failure mode where `<|tool_val|>{...{{{{` never reaches
-    #    `<|tool_call_end|>` because nested-brace prior overrides special-token escape.
-    if "<|tool_val|>" in text and "<|tool_call_end|>" not in text:
-        # Look at content after the LAST tool_val opener (the new_string body).
-        body_after_val = text.rsplit("<|tool_val|>", 1)[-1]
-        n_open = body_after_val.count("{")
-        n_close = body_after_val.count("}")
-        if n_open + n_close >= 8 and abs(n_open - n_close) >= 3:
-            penalty += TRUNCATED_TOOL_CALL_PENALTY
-        # Long run of identical brace chars (`}}}}}}}}` etc).
-        if "}}}}}}" in body_after_val or "{{{{{{" in body_after_val:
-            penalty += TRUNCATED_TOOL_CALL_PENALTY
-
-    return max(penalty, -1.0)
-
-
-def per_turn_dense_reward(rollout) -> list[float]:
-    """Compute dense per-turn reward aligned with rollout.turn_spans.
-
-    Pairs each assistant turn with its tool_result (next entry in transcript)
-    and computes score_turn + format_penalty for the assistant body.
-    Terminal pass/fail grounding is appended to the LAST turn only.
-    """
-    transcript = rollout.transcript
-    turns = transcript.turns
-    # Build a list of (assistant_turn, paired_tool_result) for each TurnSpan.
-    paired: list[tuple[dict, dict | None]] = []
-    i = 0
-    while i < len(turns):
-        t = turns[i]
-        if t.get("role") == "assistant":
-            tr = None
-            if i + 1 < len(turns) and turns[i + 1].get("role") == "tool_result":
-                tr = turns[i + 1]
-                i += 2
-            else:
-                i += 1
-            paired.append((t, tr))
-        else:
-            i += 1
-
-    rewards: list[float] = []
-    for ts, (asst, tr) in zip(rollout.turn_spans, paired):
-        # Fill in tool_result / error fields into the assistant dict for score_turn.
-        a_dict = dict(asst)
-        if tr is not None:
-            a_dict["_tool_result"] = tr.get("tool_result", "")
-            a_dict["_error"] = bool(tr.get("error", False))
-        r_tool = score_turn(a_dict)
-        r_fmt = format_penalty(asst.get("content", "") or "")
-        rewards.append(r_tool + r_fmt)
-
-    if rewards:
-        # Terminal grounding on the last turn only.
-        if getattr(rollout, "passed", False):
-            rewards[-1] += TERMINAL_PASS
-        else:
-            rewards[-1] += TERMINAL_FAIL
-
-    return rewards

vendor/nanoai/agentic/diag/__init__.py

@@ -1,11 +0,0 @@
-"""Agentic diagnosis micro-task eval package.
-
-This package defines the 8-task agentic diagnosis loop eval framework — the
-single source of truth for both ground-truth generation (capagent driver) and
-post-hoc grading (ksbr_eval extension).
-
-See plan: agentic diagnosis loop micro-task eval v1 (PCAP domain).
-"""
-
-from .schemas import DIAG_TASKS, TaskSpec, FieldSpec  # noqa: F401
-from .graders import GRADERS, grade_against_spec  # noqa: F401

vendor/nanoai/agentic/diag/chain_adapters.py

@@ -1,185 +0,0 @@
-"""chain↔SFT schema adapters for the 8-task diagnosis loop.
-
-Why: per-task SFT was trained on shapes that don't always match what the 8-task
-chain runtime feeds. Without adaptation, chain inputs are OOD for some fields,
-which hurts per-task parse_ok rate and triggers task-identity collapse on long
-traces (see `evidence_summary_for_stop_controller` story).
-
-What's here:
-  - hyp shape adapters (per downstream task) — v11 emits rich hyp dict; legacy
-    SFT trained on simpler shapes. (Moved here from demo + webapp duplicates.)
-  - evidence_summary_for_stop_controller — list[dict] → narrative string.
-  - 4 new input-shape adapters resolving the high-severity mismatches found
-    in 2026-05-22 schema audit:
-      * packet_summary_to_str       — wmb input (89% SFT trained on str)
-      * time_window_to_dict          — sd input (96% SFT trained on dict)
-      * available_tools_to_dicts     — tp input (88% SFT trained on list[dict])
-      * list_or_dict_to_str          — hu prediction/observation (84%/56% str)
-
-All adapters are idempotent — if input is already the SFT shape they pass it
-through unchanged."""
-from __future__ import annotations
-
-import json
-from typing import Any
-
-
-# ---- hyp shape adapters (v11 rich hyp → legacy per-task shapes) ----------------
-
-STATUS_TO_CONF: dict[str, float] = {
-    "confirmed": 0.95, "provisionally_supported": 0.7,
-    "open": 0.5, "unchanged": 0.5, "inconclusive": 0.5,
-    "weakened": 0.3, "rejected": 0.4,
-}
-# 2026-05-26: rejected mapped 0.05→0.4 to fix universal-hu-rejection bug. v13
-# SFT trained hu to reject H1 in ~100% of chain runs even when H1 is truth.
-# Old 0.05 caused sc to see (rejected H1=0.05, open H2=0.5) and pick H2 (often
-# wrong direction). New 0.4 keeps rejection signal but doesn't collapse H1
-# below the open-prior of H2, letting sc consider H1 text content again.
-# Case study: d6 L2 v2 2/69 hits were text-overlap luck on rejected H1, not
-# real reasoning. d12 L2 v2 0/69 because d12 hu rejects more confidently.
-
-
-def hyp_for_test_planner(h: Any) -> dict:
-    """test_planner SFT: hypotheses[i] = {id, description, confidence}."""
-    if not isinstance(h, dict): return {}
-    return {
-        "id": h.get("id"),
-        "description": h.get("claim", "")[:280],
-        "confidence": STATUS_TO_CONF.get(h.get("status", "open"), 0.5),
-    }
-
-
-def hyp_for_hyp_updater(h: Any) -> str:
-    """hyp_updater SFT: hypothesis = plain string (the claim only)."""
-    if not isinstance(h, dict): return ""
-    return h.get("claim", "")
-
-
-def hyp_for_stop_controller(h: Any) -> dict:
-    """stop_controller SFT: current_hypotheses[i] = {hypothesis, probability}."""
-    if not isinstance(h, dict): return {}
-    return {
-        "hypothesis": h.get("claim", "")[:280],
-        "probability": STATUS_TO_CONF.get(h.get("status", "open"), 0.5),
-    }
-
-
-# ---- evidence summary adapter (today's stop_controller fix) --------------------
-
-def evidence_summary_for_stop_controller(evidence_so_far: list) -> str:
-    """SFT trained stop_controller with `evidence_summary` as a narrative string
-    (median ~300 chars). Chain accumulates list-of-dicts; without adaptation
-    the model goes OOD by turn 9-10 and emits other tasks' schemas."""
-    if not evidence_so_far:
-        return "No evidence collected yet."
-    parts: list[str] = []
-    for e in evidence_so_far:
-        if not isinstance(e, dict):
-            continue
-        turn = e.get("turn", "?")
-        tool = e.get("tool", "?")
-        obs_list = e.get("observations") or []
-        if not isinstance(obs_list, list):
-            obs_list = []
-        obs_phrases: list[str] = []
-        for o in obs_list[:4]:
-            if isinstance(o, dict):
-                bits = []
-                for k in ("metric", "type", "value", "note", "details"):
-                    v = o.get(k)
-                    if isinstance(v, (str, int, float)) and v != "":
-                        bits.append(f"{k}={v}")
-                if bits:
-                    obs_phrases.append("(" + ", ".join(bits[:3]) + ")")
-        obs_str = "; ".join(obs_phrases) if obs_phrases else "no parsed observations"
-        parts.append(f"turn {turn}: ran {tool}; {obs_str}.")
-    return " ".join(parts)
-
-
-# ---- 4 new input-shape adapters (2026-05-22 audit fixes) -----------------------
-
-def packet_summary_to_str(ps: Any) -> str:
-    """world_model_builder SFT: packet_summary = narrative string (89% of
-    sampled training rows). Chain feeds aggregate dict from tshark. Adapter
-    renders dict as a single-sentence narrative."""
-    if isinstance(ps, str):
-        return ps
-    if not isinstance(ps, dict):
-        return str(ps)
-    parts = [f"Total {ps.get('total_packets', '?')} packets"]
-    bits = []
-    if ps.get("tcp_packets") is not None:
-        bits.append(f"{ps['tcp_packets']} TCP")
-    if ps.get("tls_packets"):
-        bits.append(f"{ps['tls_packets']} TLS")
-    if bits:
-        parts.append(f" ({', '.join(bits)})")
-    if ps.get("retransmissions"):
-        parts.append(f"; {ps['retransmissions']} retransmissions")
-    if ps.get("zero_window_advertisements"):
-        parts.append(f", {ps['zero_window_advertisements']} zero-window advertisements")
-    return "".join(parts) + "."
-
-
-def time_window_to_dict(tw: Any) -> dict:
-    """symptom_detector SFT: time_window = dict{start, end} (96% of rows).
-    Chain produces 'start_ts / end_ts' string. Split on ' / ' (or similar)
-    and wrap as dict."""
-    if isinstance(tw, dict):
-        return tw
-    if not isinstance(tw, str):
-        return {"start": str(tw), "end": str(tw)}
-    for sep in (" / ", "/", " -> ", " — ", "—"):
-        if sep in tw:
-            parts = [p.strip() for p in tw.split(sep, 1)]
-            if len(parts) == 2:
-                return {"start": parts[0], "end": parts[1]}
-    return {"start": tw, "end": tw}
-
-
-def available_tools_to_dicts(tools: Any) -> list[dict]:
-    """test_planner SFT: available_tools = list[dict] (88% of rows). Chain
-    passes list[str] names. Wrap each string as {name: str}; pass dicts
-    through."""
-    if not isinstance(tools, list):
-        return []
-    out: list[dict] = []
-    for t in tools:
-        if isinstance(t, dict):
-            out.append(t)
-        elif isinstance(t, str):
-            out.append({"name": t})
-    return out
-
-
-def list_or_dict_to_str(x: Any, max_items: int = 3) -> str:
-    """hypothesis_updater SFT: prediction and observation are plain strings
-    (84% / 56% of rows). Chain feeds list[dict] (from test_planner
-    expected_observations / observation_normalizer observations). Render as a
-    short narrative; prefer description/type/value/metric/note keys; fall back
-    to truncated JSON."""
-    if x is None:
-        return ""
-    if isinstance(x, str):
-        return x
-    if isinstance(x, dict):
-        return ", ".join(f"{k}={v}" for k, v in list(x.items())[:5])
-    if isinstance(x, list):
-        parts: list[str] = []
-        key_priority = ("description", "type", "value", "metric", "note", "details")
-        for item in x[:max_items]:
-            if isinstance(item, dict):
-                bits: list[str] = []
-                for k in key_priority:
-                    if k in item:
-                        bits.append(f"{k}={item[k]}")
-                    if len(bits) >= 3:
-                        break
-                if not bits:
-                    bits.append(json.dumps(item, ensure_ascii=False)[:80])
-                parts.append("; ".join(bits))
-            else:
-                parts.append(str(item)[:80])
-        return " | ".join(parts) if parts else ""
-    return str(x)

vendor/nanoai/agentic/diag/graders.py

@@ -1,271 +0,0 @@
-"""Graders for the 8 agentic diagnosis micro-tasks.
-
-Design contract: every grader returns
-    (passed: bool, detail: dict)
-
-where `detail` MUST include:
-    format_ok    bool      — JSON parsed AND required top-level keys present
-    content_ok   bool      — all field-level content checks passed
-    field_scores dict      — per-field {ok, reason} breakdown (for case study)
-    completion_head str    — first 120 chars of completion (for triage)
-
-`passed = format_ok and content_ok`. Reporting separates the two so that a
-model failing to produce JSON (format_ok=0) is distinguished from one that
-produces JSON with wrong content (format_ok=high, content_ok=low). Conflating
-these was the failure mode of agent_eval_mt 38: a single binary pass/fail
-hid which agent rung was actually broken.
-
-Lenience policy (anti-Goodhart, [[lesson-r-arith-grader-too-lenient]]):
-  * structural keys + enum values: STRICT (exact match required)
-  * free-text fields: LOW bar — Jaccard ≥0.15 OR length ≥10 chars suffices
-  * list lengths: target within ±100% of expected (just "non-empty" if min_items==1)
-  * the goal: a model that produces sensible JSON in the right shape scores
-    40-70% on content_ok; pure garbage scores <10%
-"""
-from __future__ import annotations
-
-import json
-import re
-from typing import Any
-
-from .schemas import DIAG_TASKS, TaskSpec, FieldSpec
-
-
-# ---------- helpers ----------
-
-_JSON_OBJ_RE = re.compile(r"\{[\s\S]*\}")
-_TOKEN_RE = re.compile(r"\w+", re.UNICODE)
-
-
-def _extract_json(completion: str) -> dict | None:
-    """Find the first top-level JSON object in completion and parse it.
-
-    Tolerates leading/trailing prose, markdown code fences, etc. Greedy match
-    on outermost braces; if that fails, return None.
-    """
-    if not completion:
-        return None
-    # Strip markdown code fences ```json ... ```
-    t = completion.strip()
-    if t.startswith("```"):
-        t = re.sub(r"^```(?:json)?\s*", "", t)
-        t = re.sub(r"\s*```$", "", t)
-    m = _JSON_OBJ_RE.search(t)
-    if not m:
-        return None
-    try:
-        return json.loads(m.group(0))
-    except json.JSONDecodeError:
-        # Try a more permissive fallback: balance braces manually.
-        try:
-            depth = 0
-            start = t.find("{")
-            if start < 0:
-                return None
-            for i, ch in enumerate(t[start:], start=start):
-                if ch == "{":
-                    depth += 1
-                elif ch == "}":
-                    depth -= 1
-                    if depth == 0:
-                        return json.loads(t[start : i + 1])
-            return None
-        except Exception:
-            return None
-
-
-def _tokens(s: str) -> set[str]:
-    if not isinstance(s, str):
-        return set()
-    return set(_TOKEN_RE.findall(s.lower()))
-
-
-def _jaccard(a: str, b: str) -> float:
-    ta, tb = _tokens(a), _tokens(b)
-    if not ta and not tb:
-        return 1.0
-    if not ta or not tb:
-        return 0.0
-    return len(ta & tb) / len(ta | tb)
-
-
-# ---------- per-field checks ----------
-
-def _check_field(
-    field: FieldSpec, predicted: Any, expected: Any
-) -> tuple[bool, bool, str]:
-    """Return (format_ok_field, content_ok_field, reason).
-
-    format_ok_field = whether the type/shape constraints are met
-    content_ok_field = whether content matches (or is reasonably close to) expected
-    reason = short string for case study
-    """
-    if predicted is None:
-        return False, False, "missing"
-
-    # Kind-specific shape check
-    if field.kind == "str":
-        if not isinstance(predicted, str):
-            return False, False, f"not_str:{type(predicted).__name__}"
-        if not predicted.strip():
-            return True, False, "empty"
-        if field.freetext:
-            # low-bar content check: Jaccard ≥0.15 OR ≥10 chars present
-            j = _jaccard(predicted, expected if isinstance(expected, str) else "")
-            length_ok = len(predicted) >= 10
-            content = j >= 0.15 or length_ok
-            return True, content, f"jaccard={j:.2f}_len={len(predicted)}"
-        else:
-            # exact match required for non-freetext str
-            content = predicted.strip() == (expected.strip() if isinstance(expected, str) else "")
-            return True, content, "exact" if content else f"mismatch:{predicted[:40]!r}"
-
-    if field.kind == "int":
-        if not isinstance(predicted, int) or isinstance(predicted, bool):
-            return False, False, f"not_int:{type(predicted).__name__}"
-        content = predicted == expected if isinstance(expected, int) else True
-        return True, content, "ok"
-
-    if field.kind == "float":
-        if not isinstance(predicted, (int, float)) or isinstance(predicted, bool):
-            return False, False, f"not_num:{type(predicted).__name__}"
-        if isinstance(expected, (int, float)):
-            content = abs(predicted - expected) < 1e-6
-        else:
-            content = True
-        return True, content, "ok"
-
-    if field.kind == "enum":
-        if not isinstance(predicted, str):
-            return False, False, f"not_str:{type(predicted).__name__}"
-        allowed = field.enum_values or []
-        if predicted not in allowed:
-            return False, False, f"not_in_enum:{predicted!r}"
-        # content check: matches expected if provided
-        content = (predicted == expected) if isinstance(expected, str) else True
-        return True, content, ("match" if content else f"wrong_enum:{predicted!r}")
-
-    if field.kind == "list":
-        if not isinstance(predicted, list):
-            return False, False, f"not_list:{type(predicted).__name__}"
-        if len(predicted) < field.min_items:
-            return False, False, f"too_few:{len(predicted)}<{field.min_items}"
-        # content: count within ±100% of expected length (very generous)
-        if isinstance(expected, list) and expected:
-            ratio = len(predicted) / len(expected)
-            content = 0.5 <= ratio <= 2.0
-        else:
-            content = True
-        return True, content, f"len={len(predicted)}"
-
-    if field.kind == "list_obj":
-        if not isinstance(predicted, list):
-            return False, False, f"not_list:{type(predicted).__name__}"
-        if len(predicted) < field.min_items:
-            return False, False, f"too_few:{len(predicted)}<{field.min_items}"
-        # every item must be a dict with the required keys
-        required = field.item_keys or []
-        for i, item in enumerate(predicted):
-            if not isinstance(item, dict):
-                return False, False, f"item{i}_not_dict"
-            missing = [k for k in required if k not in item]
-            if missing:
-                return False, False, f"item{i}_missing:{missing}"
-            # enum check on per-item enum-constrained fields
-            if field.item_enums:
-                for key, allowed in field.item_enums.items():
-                    if key in item and item[key] not in allowed:
-                        return False, False, f"item{i}.{key}_not_in_enum:{item[key]!r}"
-        # content: length within ±100% of expected if expected is a list
-        if isinstance(expected, list) and expected:
-            ratio = len(predicted) / len(expected)
-            content = 0.5 <= ratio <= 2.0
-        else:
-            content = True
-        return True, content, f"len={len(predicted)}"
-
-    if field.kind == "obj":
-        if not isinstance(predicted, dict):
-            return False, False, f"not_dict:{type(predicted).__name__}"
-        if not predicted:
-            return True, False, "empty_dict"
-        # content: ≥3 keys present (loose) and at least 1 overlapping key with expected
-        if isinstance(expected, dict) and expected:
-            overlap = set(predicted.keys()) & set(expected.keys())
-            content = len(overlap) >= 1
-        else:
-            content = len(predicted) >= 1
-        return True, content, f"keys={sorted(predicted.keys())[:5]}"
-
-    return False, False, f"unknown_kind:{field.kind}"
-
-
-# ---------- generic spec-based grader ----------
-
-def grade_against_spec(
-    spec: TaskSpec, expected: dict, completion: str
-) -> tuple[bool, dict]:
-    """Generic grader: validate `completion` JSON against `spec` and compare to `expected`."""
-    head = (completion or "")[:120]
-
-    parsed = _extract_json(completion)
-    if parsed is None:
-        return False, {
-            "format_ok": False,
-            "content_ok": False,
-            "field_scores": {},
-            "completion_head": head,
-            "reason": "no_json",
-        }
-
-    field_scores: dict[str, dict] = {}
-    format_ok_all = True
-    content_ok_all = True
-
-    # Check required top-level keys present
-    missing_top = [f.name for f in spec.output_fields
-                   if f.required and f.name not in parsed]
-    if missing_top:
-        return False, {
-            "format_ok": False,
-            "content_ok": False,
-            "field_scores": {k: {"ok": False, "reason": "missing_top_key"} for k in missing_top},
-            "completion_head": head,
-            "reason": f"missing_top:{missing_top}",
-        }
-
-    # Per-field check
-    for f in spec.output_fields:
-        predicted_val = parsed.get(f.name)
-        expected_val = expected.get(f.name) if isinstance(expected, dict) else None
-        f_format, f_content, reason = _check_field(f, predicted_val, expected_val)
-        field_scores[f.name] = {"ok": f_format and f_content, "format": f_format,
-                                "content": f_content, "reason": reason}
-        if not f_format:
-            format_ok_all = False
-        if not f_content:
-            content_ok_all = False
-
-    return format_ok_all and content_ok_all, {
-        "format_ok": format_ok_all,
-        "content_ok": content_ok_all,
-        "field_scores": field_scores,
-        "completion_head": head,
-    }
-
-
-# ---------- 8 thin wrappers (registered into GRADERS) ----------
-
-def _make_grader(task_id: str):
-    spec = DIAG_TASKS[task_id]
-
-    def grader(row: dict, completion: str) -> tuple[bool, dict]:
-        expected = row.get("expected_output", {}) or {}
-        return grade_against_spec(spec, expected, completion)
-    grader.__name__ = f"grade_{task_id}"
-    return grader
-
-
-GRADERS: dict[str, Any] = {
-    f"diag_{tid}": _make_grader(tid) for tid in DIAG_TASKS.keys()
-}

vendor/nanoai/agentic/diag/graders_v2.py

@@ -1,651 +0,0 @@
-"""Strict semantic graders for diag_eval_v1 — v2 of graders.py.
-
-Motivation: v1 graders (graders.py) are SHAPE-graders. The
-`experiments/ksbr_pcap/scripts/fake_shape_baseline.py` audit (2026-05-21) showed shape-correct
-gibberish achieves 99-100% pass on 3 tasks (hypothesis_updater, hypothesis_generator,
-symptom_detector) and 43.4% overall. That makes v1 numbers a JSON-schema-
-compliance score, not a diagnostic-capability score.
-
-v2 adds task-specific semantic checks that cross-reference the model's
-predicted content against the row's input_state. Each task has a
-hand-tuned check function. Default: STRICT — passing means the model's
-output has structure AND content tied to the input.
-
-Result contract is identical to v1 (passed, detail with format_ok /
-content_ok / field_scores / completion_head). v1 GRADERS are unaffected.
-
-Registered as `diag_<task_id>_v2`. Use `scripts/regrade_jsonl.py` to
-re-grade existing v1 result jsonl with v2 graders without re-running eval.
-"""
-from __future__ import annotations
-
-import json
-import re
-from typing import Any
-
-from .schemas import DIAG_TASKS, TaskSpec, FieldSpec
-from .graders import _extract_json, _tokens, _jaccard
-
-# ---------- helpers ----------
-
-_IPV4_RE = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?$")
-_IPV6_RE = re.compile(r"^[0-9a-fA-F:]+(?::\d+)?$")
-_FLOW_TUPLE_RE = re.compile(
-    r"^\s*[0-9a-fA-F\.:\-_]+(?::\d+)?\s*(?:->|→|<-)\s*[0-9a-fA-F\.:\-_]+(?::\d+)?\s*$"
-)
-_DIGIT_RE = re.compile(r"^\d+$")
-_FAKE_FILLER_RE = re.compile(r"^(x+|y+|foo|bar|baz|test|xxxx+|placeholder)$", re.IGNORECASE)
-
-# Real-ish PCAP tool/check names. Substring-matched to be tolerant of
-# variation like "tshark_filter" vs "tshark.filter".
-_PCAP_TOOL_TOKENS = {
-    "tshark", "tcpdump", "wireshark", "zeek", "bro", "scapy",
-    "ping", "traceroute", "tracert", "mtr", "nmap", "dig", "nslookup", "host",
-    "ss", "netstat", "ip", "tc", "ethtool", "iptables", "ipset", "nft",
-    "iperf", "iperf3", "curl", "wget", "openssl",
-    "syn_probe", "syn_scan", "tcp_probe", "tcp_retransmit", "tcp_window",
-    "mtu_probe", "path_mtu", "dns_query", "dns_check", "icmp_ping",
-    "tcp_stream", "flow_extract", "filter", "capture",
-    "kernel_log", "dmesg", "journalctl", "sar", "vmstat", "iostat",
-    "cpu_usage", "memory_check", "disk_io", "load_average",
-    "ipsec", "ike", "strongswan", "openvpn", "wg",
-    "stat", "log", "metric",
-}
-
-_PROTOCOL_TOKENS = {
-    "TCP", "UDP", "ICMP", "ICMPv6", "GRE", "ESP", "AH", "SCTP", "QUIC",
-    "TLS", "DTLS", "SSH", "HTTP", "HTTPS", "HTTP/2", "HTTP/3", "SMTP",
-    "DNS", "DHCP", "BGP", "OSPF", "ARP", "NDP", "MPTCP", "MPLS", "VXLAN",
-    "WireGuard", "IPSec", "L2TP", "GTP", "SIP", "RTP", "RTCP", "MQTT",
-}
-
-
-def _is_filler(s: Any) -> bool:
-    """Return True if s is fake placeholder text like 'xxxx' / 'foo' / empty.
-
-    Numeric strings (e.g. "1", "10", "12345") and short IDs are NOT filler.
-    Repetitive single-char fillers (xxxx, aaaa) ARE.
-    """
-    if not isinstance(s, str):
-        return False
-    t = s.strip()
-    if not t:
-        return True
-    # Explicit fake words
-    if _FAKE_FILLER_RE.match(t):
-        return True
-    # Pure digits or hex chars are valid identifiers (packet_ref, IDs, etc)
-    if t.isdigit() or all(c in "0123456789abcdefABCDEF:.-_" for c in t):
-        return False
-    # Repetitive single-char strings ≥ 4 chars (xxxx, aaaa, ----)
-    if len(t) >= 4 and len(set(t)) <= 2:
-        return True
-    return False
-
-
-def _has_pcap_tool_token(s: str) -> bool:
-    if not isinstance(s, str):
-        return False
-    sl = s.lower()
-    return any(tok in sl for tok in _PCAP_TOOL_TOKENS)
-
-
-def _is_ip(s: str) -> bool:
-    if not isinstance(s, str):
-        return False
-    s = s.strip()
-    if _IPV4_RE.match(s):
-        # validate octets
-        host = s.split(":")[0]
-        try:
-            return all(0 <= int(o) <= 255 for o in host.split("."))
-        except Exception:
-            return False
-    if _IPV6_RE.match(s) and "." not in s:
-        return ":" in s and any(c in "0123456789abcdefABCDEF" for c in s)
-    return False
-
-
-def _collect_ips_from_input_state(input_state: dict) -> set[str]:
-    """Pull plausible IP strings from anywhere in input_state."""
-    out: set[str] = set()
-
-    def _walk(obj):
-        if isinstance(obj, str):
-            for tok in re.findall(r"\d{1,3}(?:\.\d{1,3}){3}", obj):
-                if _is_ip(tok):
-                    out.add(tok)
-        elif isinstance(obj, dict):
-            for v in obj.values():
-                _walk(v)
-        elif isinstance(obj, list):
-            for v in obj:
-                _walk(v)
-    _walk(input_state)
-    return out
-
-
-def _input_state_text(input_state: dict) -> str:
-    """Flatten input_state to a searchable text blob for cross-checks."""
-    try:
-        return json.dumps(input_state, ensure_ascii=False).lower()
-    except Exception:
-        return str(input_state).lower()
-
-
-def _result(format_ok: bool, content_ok: bool, field_scores: dict,
-            completion_head: str, reason: str = "") -> tuple[bool, dict]:
-    d = {
-        "format_ok": format_ok,
-        "content_ok": content_ok,
-        "field_scores": field_scores,
-        "completion_head": completion_head,
-        "grader_version": "v2",
-    }
-    if reason:
-        d["reason"] = reason
-    return format_ok and content_ok, d
-
-
-def _parse(completion: str) -> tuple[dict | None, str]:
-    head = (completion or "")[:120]
-    return _extract_json(completion), head
-
-
-# ---------- per-task semantic graders ----------
-
-def grade_problem_framing_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: scope must have ≥2 real keys; goal/question must overlap with input request."""
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    required = ["diagnosis_goal", "scope", "primary_question", "constraints", "initial_uncertainties"]
-    missing = [k for k in required if k not in parsed]
-    if missing:
-        return _result(False, False,
-                       {k: {"ok": False, "reason": "missing_top_key"} for k in missing},
-                       head, f"missing_top:{missing}")
-
-    user_req = input_state.get("user_request", "")
-    user_tokens = _tokens(user_req)
-    fs = {}
-
-    # diagnosis_goal: must be str ≥ 30 chars + overlap with input topic
-    goal = parsed.get("diagnosis_goal", "")
-    goal_ok = (isinstance(goal, str) and len(goal) >= 30
-               and not _is_filler(goal)
-               and (not user_tokens or _jaccard(goal, user_req) >= 0.10))
-    fs["diagnosis_goal"] = {"ok": goal_ok, "format": isinstance(goal, str),
-                              "content": goal_ok, "reason": f"len={len(goal) if isinstance(goal, str) else 0} jaccard={_jaccard(goal, user_req):.2f}"}
-
-    # scope: dict with ≥2 keys, must include a scope-like key
-    scope = parsed.get("scope", {})
-    scope_keys = set(scope.keys()) if isinstance(scope, dict) else set()
-    scope_topic_keys = {
-        "protocol_layer", "protocol_layers", "protocols",
-        "time_window", "duration", "timeframe",
-        "traffic_direction", "direction",
-        "focus_area", "focus", "scope_boundaries", "boundaries",
-        "endpoints", "ip_range", "host_filter",
-    }
-    has_topic = bool(scope_keys & scope_topic_keys)
-    scope_ok = (isinstance(scope, dict) and len(scope_keys) >= 2 and has_topic)
-    fs["scope"] = {"ok": scope_ok, "format": isinstance(scope, dict),
-                    "content": has_topic, "reason": f"keys={sorted(scope_keys)[:6]}"}
-
-    # primary_question: str ≥ 20 chars; should be question-like (contain '?' or 'what'/'why'/'how'/etc)
-    pq = parsed.get("primary_question", "")
-    q_tokens = {"what", "why", "how", "when", "where", "which", "?"}
-    pq_ok = (isinstance(pq, str) and len(pq) >= 20 and not _is_filler(pq)
-             and (any(t in pq.lower() for t in q_tokens) or "?" in pq))
-    fs["primary_question"] = {"ok": pq_ok, "format": isinstance(pq, str),
-                                "content": pq_ok, "reason": f"len={len(pq) if isinstance(pq, str) else 0}"}
-
-    # constraints & uncertainties: lists of substantive strings
-    for k in ["constraints", "initial_uncertainties"]:
-        lst = parsed.get(k, [])
-        if not isinstance(lst, list) or len(lst) == 0:
-            fs[k] = {"ok": False, "format": False, "content": False, "reason": "not_list_or_empty"}
-            continue
-        substantive = [x for x in lst if isinstance(x, str)
-                        and len(x) >= 20 and not _is_filler(x)]
-        ok = len(substantive) >= 1
-        fs[k] = {"ok": ok, "format": True, "content": ok,
-                  "reason": f"substantive={len(substantive)}/{len(lst)}"}
-
-    format_ok = all(v.get("format", False) for v in fs.values())
-    content_ok = all(v.get("content", False) for v in fs.values())
-    return _result(format_ok, content_ok, fs, head)
-
-
-def grade_world_model_builder_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: entities must have valid IPs; flows must have valid tuple format; IPs overlap with input."""
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    required = ["entities", "flows", "capture_assumptions", "world_model_risks"]
-    missing = [k for k in required if k not in parsed]
-    if missing:
-        return _result(False, False,
-                       {k: {"ok": False, "reason": "missing_top_key"} for k in missing},
-                       head, f"missing_top:{missing}")
-
-    input_ips = _collect_ips_from_input_state(input_state)
-    fs = {}
-
-    # entities: list of dicts with valid IP + type enum
-    ents = parsed.get("entities", [])
-    if not isinstance(ents, list) or len(ents) < 2:
-        fs["entities"] = {"ok": False, "format": False, "content": False, "reason": "not_list_or_too_few"}
-    else:
-        good = []
-        bad = []
-        for i, e in enumerate(ents):
-            if not isinstance(e, dict):
-                bad.append(f"item{i}_not_dict")
-                continue
-            ip = e.get("ip", "")
-            etype = e.get("type", "")
-            if not _is_ip(ip):
-                bad.append(f"item{i}_bad_ip:{ip}")
-                continue
-            if etype not in {"client", "server", "intermediate", "unknown", "router", "gateway", "proxy", "loadbalancer"}:
-                bad.append(f"item{i}_bad_type:{etype}")
-                continue
-            good.append(ip)
-        # at least 2 good entities; if input has known IPs, at least 1 must overlap
-        n_overlap = len(set(good) & input_ips) if input_ips else 0
-        ok = (len(good) >= 2 and (not input_ips or n_overlap >= 1))
-        fs["entities"] = {"ok": ok, "format": len(good) >= 2,
-                          "content": (not input_ips or n_overlap >= 1),
-                          "reason": f"good={len(good)} overlap={n_overlap}/{len(input_ips)} bad={bad[:3]}"}
-
-    # flows: list_obj with tuple as EITHER "IP:port -> IP:port" string OR dict with src/dst keys
-    flows = parsed.get("flows", [])
-    if not isinstance(flows, list) or len(flows) < 1:
-        fs["flows"] = {"ok": False, "format": False, "content": False, "reason": "not_list_or_empty"}
-    else:
-        ok_count = 0
-        for f in flows:
-            if not isinstance(f, dict):
-                continue
-            t = f.get("tuple")
-            p = f.get("protocol", "")
-            t_ok = False
-            if isinstance(t, str) and _FLOW_TUPLE_RE.match(t.strip()):
-                t_ok = True
-            elif isinstance(t, dict):
-                # dict form: must have src_ip/dst_ip (or equivalent)
-                src_keys = {"src_ip", "source_ip", "src", "source"}
-                dst_keys = {"dst_ip", "destination_ip", "dst", "destination"}
-                has_src = any(k in t for k in src_keys)
-                has_dst = any(k in t for k in dst_keys)
-                t_ok = has_src and has_dst
-            if t_ok and isinstance(p, str) and len(p) >= 2:
-                ok_count += 1
-        ok = ok_count >= 1
-        fs["flows"] = {"ok": ok, "format": ok, "content": ok,
-                        "reason": f"valid_tuple_protocol={ok_count}/{len(flows)}"}
-
-    # capture_assumptions: list of substantive strings
-    ca = parsed.get("capture_assumptions", [])
-    sub = [x for x in (ca if isinstance(ca, list) else [])
-            if isinstance(x, str) and len(x) >= 30 and not _is_filler(x)]
-    fs["capture_assumptions"] = {"ok": len(sub) >= 1, "format": isinstance(ca, list),
-                                   "content": len(sub) >= 1,
-                                   "reason": f"substantive={len(sub)}"}
-
-    # world_model_risks: list of substantive strings
-    wmr = parsed.get("world_model_risks", [])
-    sub = [x for x in (wmr if isinstance(wmr, list) else [])
-            if isinstance(x, str) and len(x) >= 20 and not _is_filler(x)]
-    fs["world_model_risks"] = {"ok": len(sub) >= 1, "format": isinstance(wmr, list),
-                                 "content": len(sub) >= 1,
-                                 "reason": f"substantive={len(sub)}"}
-
-    format_ok = all(v.get("format", False) for v in fs.values())
-    content_ok = all(v.get("content", False) for v in fs.values())
-    return _result(format_ok, content_ok, fs, head)
-
-
-def grade_symptom_detector_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: symptoms must have valid enum types AND evidence_refs grounded in input."""
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    if "symptoms" not in parsed:
-        return _result(False, False, {"symptoms": {"ok": False, "reason": "missing"}},
-                       head, "missing_top")
-
-    symptoms = parsed.get("symptoms", [])
-    if not isinstance(symptoms, list) or len(symptoms) < 1:
-        return _result(False, False, {"symptoms": {"ok": False, "reason": "not_list"}},
-                       head, "not_list")
-
-    allowed_types = {
-        "tcp_retransmission_burst", "tcp_duplicate_ack", "tcp_zero_window",
-        "tcp_reset", "server_response_gap", "client_idle_gap",
-        "tls_handshake_delay", "dns_lookup_delay", "dns_no_response",
-        "rtt_spike", "high_jitter", "mtu_blackhole",
-    }
-    sev_allowed = {"low", "medium", "high", "critical"}
-    input_text = _input_state_text(input_state)
-
-    ok_count = 0
-    bad = []
-    for i, s in enumerate(symptoms):
-        if not isinstance(s, dict):
-            bad.append(f"item{i}_not_dict")
-            continue
-        if s.get("type") not in allowed_types:
-            bad.append(f"item{i}_bad_type:{s.get('type')}")
-            continue
-        if s.get("severity") not in sev_allowed:
-            bad.append(f"item{i}_bad_severity")
-            continue
-        # evidence_refs must be non-empty list of substantive strings
-        refs = s.get("evidence_refs", [])
-        if not isinstance(refs, list) or len(refs) == 0:
-            bad.append(f"item{i}_no_refs")
-            continue
-        sub_refs = [r for r in refs if isinstance(r, str) and len(r) >= 4 and not _is_filler(r)]
-        if len(sub_refs) == 0:
-            bad.append(f"item{i}_filler_refs")
-            continue
-        # grounding: at least one ref-token must appear in input tokens.
-        # (Jaccard was unfair when refs is small {1-3 tokens} vs input {200+}.)
-        from .graders import _tokens as _tk
-        refs_joined = " ".join(sub_refs)
-        ref_tokens = _tk(refs_joined)
-        input_tokens = _tk(input_text)
-        if ref_tokens and not (ref_tokens & input_tokens):
-            bad.append(f"item{i}_ungrounded_refs")
-            continue
-        ok_count += 1
-    ok = ok_count >= 1
-    fs = {"symptoms": {"ok": ok, "format": ok_count >= 1, "content": ok_count >= 1,
-                         "reason": f"valid={ok_count}/{len(symptoms)} bad={bad[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_hypothesis_generator_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: ≥2 substantive hypotheses with grounded claim + falsifiable tests."""
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    if "hypotheses" not in parsed:
-        return _result(False, False, {"hypotheses": {"ok": False, "reason": "missing"}},
-                       head, "missing_top")
-
-    hyps = parsed.get("hypotheses", [])
-    if not isinstance(hyps, list) or len(hyps) < 2:
-        return _result(False, False,
-                       {"hypotheses": {"ok": False, "format": False, "content": False,
-                                       "reason": f"only_{len(hyps) if isinstance(hyps, list) else 0}"}},
-                       head, "too_few_hyps")
-
-    input_text = _input_state_text(input_state)
-    ok_count = 0
-    bad = []
-    for i, h in enumerate(hyps):
-        if not isinstance(h, dict):
-            bad.append(f"item{i}_not_dict")
-            continue
-        claim = h.get("claim", "")
-        if not isinstance(claim, str) or len(claim) < 40 or _is_filler(claim):
-            bad.append(f"item{i}_bad_claim_len={len(claim) if isinstance(claim, str) else 0}")
-            continue
-        # claim should overlap with input_state (any of world_model/symptoms/diag_goal)
-        if input_text and _jaccard(claim, input_text) < 0.03:
-            bad.append(f"item{i}_claim_off_topic")
-            continue
-        for k in ["explains", "does_not_explain", "required_tests"]:
-            v = h.get(k, [])
-            if not isinstance(v, list) or len(v) < 1:
-                bad.append(f"item{i}.{k}_not_list_or_empty")
-                break
-            sub = [x for x in v if isinstance(x, str) and len(x) >= 15 and not _is_filler(x)]
-            if len(sub) < 1:
-                bad.append(f"item{i}.{k}_no_substantive")
-                break
-        else:
-            # required_tests must mention real tools / methods
-            tests = h.get("required_tests", [])
-            tool_hits = sum(1 for t in tests if isinstance(t, str) and _has_pcap_tool_token(t))
-            if tool_hits == 0:
-                bad.append(f"item{i}.required_tests_no_tools")
-                continue
-            ok_count += 1
-    ok = ok_count >= 2
-    fs = {"hypotheses": {"ok": ok, "format": ok_count >= 2, "content": ok_count >= 2,
-                          "reason": f"good={ok_count}/{len(hyps)} bad={bad[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_test_planner_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: next_action must have real tool + arguments dict + expected_observations with multiple outcomes."""
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    if "next_action" not in parsed:
-        return _result(False, False, {"next_action": {"ok": False, "reason": "missing"}},
-                       head, "missing_top")
-    na = parsed.get("next_action", {})
-    if not isinstance(na, dict):
-        return _result(False, False,
-                       {"next_action": {"ok": False, "format": False, "reason": "not_dict"}},
-                       head, "not_dict")
-
-    # tool / tool_name (model uses either)
-    tool = na.get("tool") or na.get("tool_name") or ""
-    tool_ok = (isinstance(tool, str) and len(tool) >= 4 and not _is_filler(tool))
-    # arguments: dict with ≥1 key; try multiple key names (arguments/args/params)
-    args = na.get("arguments") or na.get("args") or na.get("params") or {}
-    args_ok = isinstance(args, dict) and len(args) >= 1 and not all(_is_filler(str(v)) for v in args.values())
-    # expected_observations: list with ≥2 outcomes OR dict with ≥2 keys
-    eobs = na.get("expected_observations")
-    if isinstance(eobs, list):
-        eobs_ok = len(eobs) >= 2 and all(isinstance(o, dict) for o in eobs)
-    elif isinstance(eobs, dict):
-        eobs_ok = len(eobs) >= 2
-    else:
-        eobs_ok = False
-
-    fs = {"next_action": {
-        "ok": tool_ok and args_ok and eobs_ok,
-        "format": isinstance(na, dict),
-        "content": tool_ok and args_ok and eobs_ok,
-        "reason": f"tool={tool[:30]!r}_ok={tool_ok} args_ok={args_ok} eobs_ok={eobs_ok}",
-    }}
-    ok = tool_ok and args_ok and eobs_ok
-    return _result(ok, ok, fs, head)
-
-
-def grade_observation_normalizer_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: observations must have grounded flow_id + parseable packet_ref + numeric time."""
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    if "observations" not in parsed:
-        return _result(False, False, {"observations": {"ok": False, "reason": "missing"}},
-                       head, "missing_top")
-    obs = parsed.get("observations", [])
-    if not isinstance(obs, list) or len(obs) < 1:
-        return _result(False, False, {"observations": {"ok": False, "reason": "not_list"}},
-                       head, "not_list")
-
-    input_text = _input_state_text(input_state)
-    ok_count = 0
-    bad = []
-    for i, o in enumerate(obs):
-        if not isinstance(o, dict):
-            bad.append(f"item{i}_not_dict")
-            continue
-        otype = o.get("type", "")
-        if not isinstance(otype, str) or len(otype) < 3 or _is_filler(otype):
-            bad.append(f"item{i}_bad_type")
-            continue
-        flow_id = o.get("flow_id", "")
-        # flow_id: non-filler, ≥3 char id (any chars; capagent uses e.g. "CxA123.0001", "flow_001", "f1")
-        if not isinstance(flow_id, str) or len(flow_id) < 2 or _is_filler(flow_id):
-            bad.append(f"item{i}_bad_flow_id")
-            continue
-        # packet_ref: non-filler ≥1 char (int OR string — capagent uses composite IDs like "CxA123_1698417000")
-        pref = o.get("packet_ref")
-        if pref is None:
-            bad.append(f"item{i}_no_packet_ref")
-            continue
-        if isinstance(pref, int):
-            pref_ok = True
-        elif isinstance(pref, str) and len(pref) >= 1 and not _is_filler(pref):
-            pref_ok = True
-        else:
-            pref_ok = False
-        if not pref_ok:
-            bad.append(f"item{i}_bad_packet_ref:{pref!r}")
-            continue
-        # time: numeric int/float OR string parseable as float
-        t = o.get("time")
-        t_ok = False
-        if isinstance(t, (int, float)) and not isinstance(t, bool):
-            t_ok = True
-        elif isinstance(t, str):
-            try:
-                float(t.strip())
-                t_ok = True
-            except (ValueError, AttributeError):
-                pass
-        if not t_ok:
-            bad.append(f"item{i}_bad_time:{t!r}")
-            continue
-        ok_count += 1
-    ok = ok_count >= 1
-    fs = {"observations": {"ok": ok, "format": ok_count >= 1, "content": ok_count >= 1,
-                             "reason": f"valid={ok_count}/{len(obs)} bad={bad[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_hypothesis_updater_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: update id must match input hypothesis.id, reason must cite observation."""
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    if "hypothesis_updates" not in parsed:
-        return _result(False, False, {"hypothesis_updates": {"ok": False, "reason": "missing"}},
-                       head, "missing_top")
-    updates = parsed.get("hypothesis_updates", [])
-    if not isinstance(updates, list) or len(updates) < 1:
-        return _result(False, False, {"hypothesis_updates": {"ok": False, "reason": "not_list"}},
-                       head, "not_list")
-
-    # input hypothesis id
-    h_input = input_state.get("hypothesis", {})
-    expected_id = h_input.get("id") if isinstance(h_input, dict) else None
-    input_text = _input_state_text(input_state)
-
-    allowed_status = {
-        "provisionally_supported", "weakened", "rejected",
-        "confirmed", "inconclusive", "unchanged",
-    }
-
-    ok_count = 0
-    bad = []
-    for i, u in enumerate(updates):
-        if not isinstance(u, dict):
-            bad.append(f"item{i}_not_dict")
-            continue
-        uid = u.get("id")
-        if expected_id and uid != expected_id:
-            bad.append(f"item{i}_id_mismatch:{uid!r}!={expected_id!r}")
-            continue
-        status = u.get("status")
-        if status not in allowed_status:
-            bad.append(f"item{i}_bad_status:{status!r}")
-            continue
-        reason = u.get("reason", "")
-        if not isinstance(reason, str) or len(reason) < 20 or _is_filler(reason):
-            bad.append(f"item{i}_bad_reason_len={len(reason) if isinstance(reason, str) else 0}")
-            continue
-        # reason should overlap with input_state (was: only with observation — too narrow)
-        if input_text and _jaccard(reason, input_text) < 0.03:
-            bad.append(f"item{i}_reason_off_topic")
-            continue
-        ok_count += 1
-    ok = ok_count >= 1
-    fs = {"hypothesis_updates": {"ok": ok, "format": ok_count >= 1, "content": ok_count >= 1,
-                                    "reason": f"valid={ok_count}/{len(updates)} bad={bad[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_stop_controller_v2(row: dict, completion: str) -> tuple[bool, dict]:
-    """Strict: enum match expected + reason substantive + required_before_stop consistent."""
-    expected = row.get("expected_output", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    if "stop_decision" not in parsed:
-        return _result(False, False, {"stop_decision": {"ok": False, "reason": "missing"}},
-                       head, "missing_top")
-    enum_allowed = {"continue", "stop_with_conclusion", "stop_inconclusive"}
-    sd = parsed.get("stop_decision")
-    if sd not in enum_allowed:
-        return _result(False, False,
-                       {"stop_decision": {"ok": False, "format": False, "content": False,
-                                          "reason": f"bad_enum:{sd!r}"}},
-                       head, "bad_enum")
-    expected_sd = expected.get("stop_decision")
-    sd_match = (sd == expected_sd) if expected_sd else True
-
-    reason = parsed.get("reason", "")
-    reason_ok = (isinstance(reason, str) and len(reason) >= 50 and not _is_filler(reason))
-
-    rbs = parsed.get("required_before_stop", [])
-    if sd == "continue":
-        rbs_ok = isinstance(rbs, list) and len(rbs) >= 1 and all(isinstance(x, str) and len(x) >= 5 for x in rbs)
-    else:
-        rbs_ok = isinstance(rbs, list)
-
-    fs = {
-        "stop_decision": {"ok": sd_match, "format": True, "content": sd_match,
-                            "reason": f"got={sd} expected={expected_sd}"},
-        "reason": {"ok": reason_ok, "format": isinstance(reason, str),
-                    "content": reason_ok, "reason": f"len={len(reason) if isinstance(reason, str) else 0}"},
-        "required_before_stop": {"ok": rbs_ok, "format": isinstance(rbs, list),
-                                   "content": rbs_ok, "reason": f"n={len(rbs) if isinstance(rbs, list) else 0}"},
-    }
-    format_ok = all(v.get("format", False) for v in fs.values())
-    content_ok = all(v.get("content", False) for v in fs.values())
-    return _result(format_ok, content_ok, fs, head)
-
-
-# ---------- registry ----------
-
-GRADERS_V2: dict[str, Any] = {
-    "diag_problem_framing_v2": grade_problem_framing_v2,
-    "diag_world_model_builder_v2": grade_world_model_builder_v2,
-    "diag_symptom_detector_v2": grade_symptom_detector_v2,
-    "diag_hypothesis_generator_v2": grade_hypothesis_generator_v2,
-    "diag_test_planner_v2": grade_test_planner_v2,
-    "diag_observation_normalizer_v2": grade_observation_normalizer_v2,
-    "diag_hypothesis_updater_v2": grade_hypothesis_updater_v2,
-    "diag_stop_controller_v2": grade_stop_controller_v2,
-}
-
-# v1 → v2 task mapping for regrade scripts
-V1_TO_V2 = {
-    f"diag_{tid}": f"diag_{tid}_v2" for tid in DIAG_TASKS.keys()
-}

vendor/nanoai/agentic/diag/graders_v3.py

@@ -1,710 +0,0 @@
-"""v3 graders — semantic comparison against expected_output.
-
-Key fix vs v2.1: every task explicitly compares predicted to expected_output
-(not just to input_state). Closes the input-copy template loophole that the
-operator audit exposed (v2.1 input-copy baseline = 80.9% pass; v3 target <30%).
-
-Design criteria (must satisfy ALL three):
-  - fake-shape gibberish baseline: < 10% pass
-  - input-copy template baseline: < 30% pass
-  - capagent GT self-grade: ≥ 70% pass (calibrates against real noise)
-
-Per-task adds over v2.1:
-  problem_framing      → scope keys overlap with EXPECTED scope (≥50%)
-  world_model_builder  → protocol must match expected; entity count ±50%
-  symptom_detector     → symptom type ∈ expected_types set
-  hypothesis_generator → n_hyps ±1 of expected; claim Jaccard vs EXPECTED claims (not input symptoms); shrunk tool whitelist
-  test_planner         → tool matches expected (semantic equiv class); expected_obs has hypothesis cross-impact
-  observation_normalizer → flow_id ∈ expected_flow_ids; type ∈ expected_types
-  hypothesis_updater   → status MUST equal expected.status (was missing)
-  stop_controller      → unchanged (already expected-match)
-"""
-from __future__ import annotations
-
-import json
-import re
-from typing import Any
-
-from .schemas import DIAG_TASKS, TaskSpec, FieldSpec
-from .graders import _extract_json, _tokens, _jaccard
-from .graders_v2 import (
-    _is_filler, _is_ip, _collect_ips_from_input_state, _input_state_text,
-    _result, _parse, _FLOW_TUPLE_RE,
-)
-
-# ---------- helpers ----------
-
-# Narrower tool whitelist — remove ambiguous 2-letter substrings.
-_PCAP_TOOL_TOKENS_V3 = {
-    "tshark", "tcpdump", "wireshark", "zeek", "bro", "scapy",
-    "traceroute", "tracert", "mtr", "nmap", "dig", "nslookup",
-    "netstat", "ethtool", "iptables", "ipset", "nftables",
-    "iperf", "iperf3", "openssl", "curl", "wget",
-    "syn_probe", "syn_scan", "tcp_probe", "tcp_retransmit",
-    "mtu_probe", "path_mtu", "dns_query", "icmp_ping",
-    "tcp_stream", "kernel_log", "dmesg", "journalctl",
-    "ipsec", "ike", "strongswan", "wireguard",
-}
-
-# Broader anchor set for hypothesis_generator.required_tests semantic check.
-# Real-world diagnostic test plans use protocol nouns ("TCP buffer", "path MTU"),
-# diagnostic mechanisms ("firewall logs", "NIC interrupt"), and concept words
-# ("pcap capture", "throughput") in addition to CLI tool names. Reviewed-accept
-# GT was 91% rejected by the narrow _PCAP_TOOL_TOKENS_V3 alone (2026-05-21
-# human-review audit). The anti-input-copy bite for hyp_gen comes from the
-# claim Jaccard vs EXPECTED claims (line below), not from this whitelist.
-_HYP_TEST_TOKENS = _PCAP_TOOL_TOKENS_V3 | {
-    # protocols / layers
-    "tcp", "udp", "icmp", "dns", "tls", "ssl", "http", "https", "ssh", "arp",
-    # tcp/ip mechanisms (multi-letter, specific)
-    "syn", "rst", "fin", "mtu", "rtt", "mss", "window",
-    "retransmission", "retransmit", "duplicate",
-    # network artifacts / verbs
-    "pcap", "packet", "capture", "interface", "interrupt", "frame", "header",
-    "ping",
-    # network components
-    "firewall", "nat", "router", "gateway", "switch", "nic", "vlan", "vpn",
-    # diagnostics resources
-    "buffer", "queue", "throughput", "latency", "jitter", "kernel",
-    # common server/service names in test plans
-    "nginx", "apache", "haproxy", "envoy", "log", "logs",
-    # cpu/process diagnostics
-    "cpu",
-}
-
-
-def _has_real_tool_token(s: str) -> bool:
-    if not isinstance(s, str):
-        return False
-    sl = s.lower()
-    return any(tok in sl for tok in _PCAP_TOOL_TOKENS_V3)
-
-
-def _has_test_anchor(s: str) -> bool:
-    """Used by hypothesis_generator.required_tests check — broader than
-    _has_real_tool_token because real diagnostic prose uses protocol/concept
-    nouns, not only CLI tool names."""
-    if not isinstance(s, str):
-        return False
-    sl = s.lower()
-    return any(tok in sl for tok in _HYP_TEST_TOKENS)
-
-
-def _len_close(a, b, ratio: float = 0.5) -> bool:
-    """True if abs(a - b) / max(b, 1) ≤ ratio."""
-    try:
-        return abs(a - b) / max(b, 1) <= ratio
-    except Exception:
-        return False
-
-
-# Static synonym map for enum drift observed in v7_filt fail analysis
-# (2026-05-21). Lowercase keys, lowercase canonical values. Only true
-# synonyms — NOT semantic miscategorisation (e.g. tcp_retransmission_burst
-# ≠ tcp_duplicate_ack, those are different TCP events and the model is
-# genuinely guessing wrong, not vocab-drifting).
-_ENUM_SYNONYMS = {
-    # observation_normalizer.type drift
-    "tls_handshake_complete": "tls_handshake_event",
-    "tls_handshake": "tls_handshake_event",
-    "tls_handshake_start": "tls_handshake_event",
-    # protocol version aliases
-    "tlsv1.3": "tls 1.3",
-    "tls1.3": "tls 1.3",
-    "tls/ssl": "tls",
-    "tls/ssl/tls": "tls",
-}
-
-
-def _normalize_enum(s: str) -> str:
-    """Lowercase + collapse whitespace. Idempotent."""
-    if not isinstance(s, str):
-        return ""
-    return re.sub(r"\s+", " ", s.lower().strip())
-
-
-def _split_compound(s: str) -> set[str]:
-    """Split a compound protocol/type like 'TCP/TLS', 'TLS 1.3 over TCP',
-    'SSH/TCP' into atomic parts. Keep parts of length ≥3 to avoid noise."""
-    parts = re.split(r"[/\s,]+", _normalize_enum(s))
-    return {p for p in parts if len(p) >= 3}
-
-
-def _enum_match(pred: str, allowed: set[str]) -> str | None:
-    """Return canonical allowed value matching `pred`, else None. Tiers:
-       (1) exact (case-insensitive),
-       (2) explicit synonym dict,
-       (3) compound subset (TCP/TLS matches TLS),
-       (4) substring overlap with ratio ≥0.6 (both ≥6 chars).
-    Strict enough to NOT cross truly different enums (TCP ≠ HTTPS, etc)."""
-    if not isinstance(pred, str) or not pred:
-        return None
-    pred_n = _normalize_enum(pred)
-    allowed_norm = {_normalize_enum(a): a for a in allowed if isinstance(a, str)}
-    if pred_n in allowed_norm:
-        return allowed_norm[pred_n]
-    # synonym dict
-    syn = _ENUM_SYNONYMS.get(pred_n)
-    if syn and syn in allowed_norm:
-        return allowed_norm[syn]
-    # Compound subset — ONLY when pred is a compound (≥2 parts).
-    # This means the model expressed multiple layers (e.g. 'TCP/TLS') and we
-    # match it against a more atomic allowed value. Crucially, a BARE pred
-    # like 'TCP' is NOT allowed to match an allowed compound 'TCP/TLS' — that
-    # would let the input-copy baseline (which emits bare 'TCP') game any
-    # compound-protocol GT. Asymmetric is intentional.
-    pred_parts = _split_compound(pred)
-    if len(pred_parts) >= 2:
-        for an, orig in allowed_norm.items():
-            an_parts = _split_compound(orig)
-            if an_parts and an_parts <= pred_parts:
-                return orig
-    # Substring overlap — both sides must be ≥6 chars to avoid trivial matches.
-    for an, orig in allowed_norm.items():
-        if len(an) < 6 or len(pred_n) < 6:
-            continue
-        if an in pred_n or pred_n in an:
-            shorter = min(len(an), len(pred_n))
-            longer = max(len(an), len(pred_n))
-            if shorter / longer >= 0.6:
-                return orig
-    return None
-
-
-# ---------- per-task v3 graders ----------
-
-def grade_problem_framing_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """v2.1 + scope key overlap with EXPECTED scope."""
-    expected = row.get("expected_output", {}) or {}
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    required = ["diagnosis_goal", "scope", "primary_question", "constraints", "initial_uncertainties"]
-    missing = [k for k in required if k not in parsed]
-    if missing:
-        return _result(False, False,
-                       {k: {"ok": False, "reason": "missing_top_key"} for k in missing},
-                       head, f"missing_top:{missing}")
-
-    fs = {}
-    user_req = input_state.get("user_request", "")
-
-    # diagnosis_goal: overlap with EXPECTED goal (not just user_req)
-    goal = parsed.get("diagnosis_goal", "")
-    exp_goal = expected.get("diagnosis_goal", "")
-    goal_jacc = _jaccard(goal, exp_goal) if exp_goal else _jaccard(goal, user_req)
-    goal_ok = (isinstance(goal, str) and len(goal) >= 30 and not _is_filler(goal)
-               and goal_jacc >= 0.15)
-    fs["diagnosis_goal"] = {"ok": goal_ok, "format": isinstance(goal, str), "content": goal_ok,
-                              "reason": f"len={len(goal) if isinstance(goal, str) else 0} jacc_to_exp={goal_jacc:.2f}"}
-
-    # scope: dict with ≥ 2 keys. If expected scope present, accept ≥ 1 key match
-    # OR value-token Jaccard ≥ 0.10 with expected scope values (synonym keys
-    # with similar values still pass — capagent and small student naturally use
-    # different key vocab for the same scoping). 2026-05-21: previous ≥50% key
-    # overlap was the dominant fail mode (73/79 v7_filt fails), but the real
-    # anti-input-copy bite is the diagnosis_goal Jaccard above — scope keys are
-    # not a meaningful semantic anchor.
-    scope = parsed.get("scope", {})
-    exp_scope = expected.get("scope", {})
-    if not isinstance(scope, dict):
-        fs["scope"] = {"ok": False, "format": False, "content": False, "reason": "not_dict"}
-    elif len(scope) < 2:
-        fs["scope"] = {"ok": False, "format": True, "content": False,
-                        "reason": f"too_few_keys={len(scope)}"}
-    elif not isinstance(exp_scope, dict) or not exp_scope:
-        fs["scope"] = {"ok": True, "format": True, "content": True,
-                        "reason": f"no_exp keys={list(scope)[:4]}"}
-    else:
-        exp_keys = set(exp_scope.keys())
-        pred_keys = set(scope.keys())
-        overlap = exp_keys & pred_keys
-        # value-token Jaccard between concatenated pred values and exp values
-        pred_val_text = " ".join(str(v) for v in scope.values())
-        exp_val_text = " ".join(str(v) for v in exp_scope.values())
-        val_jacc = _jaccard(pred_val_text, exp_val_text) if exp_val_text else 0
-        ok = len(overlap) >= 1 or val_jacc >= 0.10
-        fs["scope"] = {"ok": ok, "format": True, "content": ok,
-                        "reason": f"key_overlap={len(overlap)}/{len(exp_keys)} val_jacc={val_jacc:.2f}"}
-
-    # primary_question
-    pq = parsed.get("primary_question", "")
-    q_tokens = {"what", "why", "how", "when", "where", "which", "?"}
-    pq_ok = (isinstance(pq, str) and len(pq) >= 20 and not _is_filler(pq)
-             and (any(t in pq.lower() for t in q_tokens) or "?" in pq))
-    fs["primary_question"] = {"ok": pq_ok, "format": isinstance(pq, str), "content": pq_ok,
-                                "reason": f"len={len(pq) if isinstance(pq, str) else 0}"}
-
-    # constraints & uncertainties — substantive list + topical overlap with
-    # EXPECTED items (preferred) or user_req as fallback. 2026-05-21: previous
-    # version forced topical match against user_req only, but GT and capable
-    # models often phrase constraints abstractly ("limited capture point",
-    # "single timestamp resolution") whose tokens don't appear in the request.
-    user_tokens = _tokens(user_req)
-    for k in ["constraints", "initial_uncertainties"]:
-        lst = parsed.get(k, [])
-        if not isinstance(lst, list) or len(lst) == 0:
-            fs[k] = {"ok": False, "format": False, "content": False, "reason": "not_list_or_empty"}
-            continue
-        substantive = [x for x in lst if isinstance(x, str)
-                        and len(x) >= 20 and not _is_filler(x)]
-        # topical: ≥2-token match with EXPECTED list (synonyms welcome) OR
-        # ≥2-token match with user_req as fallback.
-        exp_list = expected.get(k, []) or []
-        exp_blob = " ".join(x for x in exp_list if isinstance(x, str))
-        exp_tokens = _tokens(exp_blob) if exp_blob else set()
-        topical = False
-        if substantive:
-            for x in substantive:
-                tx = _tokens(x)
-                if exp_tokens and len(tx & exp_tokens) >= 2:
-                    topical = True; break
-                if user_tokens and len(tx & user_tokens) >= 2:
-                    topical = True; break
-        if not (exp_tokens or user_tokens):
-            topical = True  # no baseline → can't check
-        ok = len(substantive) >= 1 and topical
-        fs[k] = {"ok": ok, "format": True, "content": ok,
-                  "reason": f"substantive={len(substantive)} topical={topical}"}
-
-    format_ok = all(v.get("format", False) for v in fs.values())
-    content_ok = all(v.get("content", False) for v in fs.values())
-    return _result(format_ok, content_ok, fs, head)
-
-
-def grade_world_model_builder_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """v2.1 + entity_count ±50% of expected + protocol must match expected."""
-    expected = row.get("expected_output", {}) or {}
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    required = ["entities", "flows", "capture_assumptions", "world_model_risks"]
-    missing = [k for k in required if k not in parsed]
-    if missing:
-        return _result(False, False,
-                       {k: {"ok": False, "reason": "missing_top_key"} for k in missing},
-                       head, f"missing_top:{missing}")
-
-    input_ips = _collect_ips_from_input_state(input_state)
-    exp_entities = expected.get("entities", [])
-    exp_flows = expected.get("flows", [])
-    fs = {}
-
-    # entities: valid IP + type enum + count within ±50% of expected
-    ents = parsed.get("entities", [])
-    if not isinstance(ents, list) or len(ents) < 2:
-        fs["entities"] = {"ok": False, "format": False, "content": False, "reason": "not_list_or_too_few"}
-    else:
-        good = []
-        for e in ents:
-            if not isinstance(e, dict): continue
-            ip = e.get("ip", "")
-            etype = e.get("type", "")
-            if not _is_ip(ip): continue
-            if etype not in {"client", "server", "intermediate", "unknown", "router", "gateway", "proxy", "loadbalancer"}: continue
-            good.append(ip)
-        n_overlap = len(set(good) & input_ips) if input_ips else 0
-        # count-match with expected
-        exp_n = len(exp_entities) if isinstance(exp_entities, list) else 2
-        count_ok = _len_close(len(good), exp_n, ratio=0.7)  # ±70% — capagent variable
-        ip_ok = (not input_ips or n_overlap >= 1)
-        ok = (len(good) >= 2 and ip_ok and count_ok)
-        fs["entities"] = {"ok": ok, "format": len(good) >= 2,
-                          "content": ip_ok and count_ok,
-                          "reason": f"good={len(good)} vs exp={exp_n} overlap={n_overlap}/{len(input_ips)} count_ok={count_ok}"}
-
-    # flows: valid tuple + protocol must SEMANTICALLY match expected
-    # Uses _enum_match (compound split + synonym + substring) so that
-    # pred='TCP/TLS' against exp='TLS' or pred='TLSv1.3' against exp='TLS 1.3'
-    # passes — 27 of v7_filt's protocol fails are this normalisation pattern.
-    exp_protocols = set()
-    if isinstance(exp_flows, list):
-        for f in exp_flows:
-            if isinstance(f, dict):
-                p = f.get("protocol", "")
-                if isinstance(p, str):
-                    exp_protocols.add(p)
-    flows = parsed.get("flows", [])
-    if not isinstance(flows, list) or len(flows) < 1:
-        fs["flows"] = {"ok": False, "format": False, "content": False, "reason": "not_list_or_empty"}
-    else:
-        ok_count = 0
-        protocol_match_count = 0
-        for f in flows:
-            if not isinstance(f, dict): continue
-            t = f.get("tuple"); p = f.get("protocol", "")
-            t_ok = False
-            if isinstance(t, str) and _FLOW_TUPLE_RE.match(t.strip()):
-                t_ok = True
-            elif isinstance(t, dict):
-                src_keys = {"src_ip", "source_ip", "src", "source"}
-                dst_keys = {"dst_ip", "destination_ip", "dst", "destination"}
-                t_ok = any(k in t for k in src_keys) and any(k in t for k in dst_keys)
-            if t_ok and isinstance(p, str) and len(p) >= 2:
-                ok_count += 1
-                if not exp_protocols or _enum_match(p, exp_protocols) is not None:
-                    protocol_match_count += 1
-        # require ≥1 flow with valid structure AND (if expected has protocols) ≥1 protocol match
-        ok = ok_count >= 1 and (not exp_protocols or protocol_match_count >= 1)
-        fs["flows"] = {"ok": ok, "format": ok_count >= 1, "content": ok,
-                        "reason": f"valid={ok_count}/{len(flows)} proto_match={protocol_match_count} exp_proto={list(exp_protocols)[:3]}"}
-
-    # capture_assumptions: substantive list
-    ca = parsed.get("capture_assumptions", [])
-    sub = [x for x in (ca if isinstance(ca, list) else [])
-            if isinstance(x, str) and len(x) >= 30 and not _is_filler(x)]
-    fs["capture_assumptions"] = {"ok": len(sub) >= 1, "format": isinstance(ca, list),
-                                   "content": len(sub) >= 1, "reason": f"substantive={len(sub)}"}
-
-    wmr = parsed.get("world_model_risks", [])
-    sub = [x for x in (wmr if isinstance(wmr, list) else [])
-            if isinstance(x, str) and len(x) >= 20 and not _is_filler(x)]
-    fs["world_model_risks"] = {"ok": len(sub) >= 1, "format": isinstance(wmr, list),
-                                 "content": len(sub) >= 1, "reason": f"substantive={len(sub)}"}
-
-    format_ok = all(v.get("format", False) for v in fs.values())
-    content_ok = all(v.get("content", False) for v in fs.values())
-    return _result(format_ok, content_ok, fs, head)
-
-
-def grade_symptom_detector_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """v2.1 + symptom type MUST be in expected_types set."""
-    expected = row.get("expected_output", {}) or {}
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    symptoms = parsed.get("symptoms", [])
-    if not isinstance(symptoms, list) or len(symptoms) < 1:
-        return _result(False, False, {"symptoms": {"ok": False, "reason": "not_list"}}, head, "not_list")
-
-    allowed_types = {
-        "tcp_retransmission_burst", "tcp_duplicate_ack", "tcp_zero_window",
-        "tcp_reset", "server_response_gap", "client_idle_gap",
-        "tls_handshake_delay", "dns_lookup_delay", "dns_no_response",
-        "rtt_spike", "high_jitter", "mtu_blackhole",
-    }
-    sev_allowed = {"low", "medium", "high", "critical"}
-
-    # expected types from row's expected_output
-    exp_symptoms = expected.get("symptoms", [])
-    exp_types = set()
-    if isinstance(exp_symptoms, list):
-        for s in exp_symptoms:
-            if isinstance(s, dict) and s.get("type") in allowed_types:
-                exp_types.add(s["type"])
-
-    input_text = _input_state_text(input_state).lower()
-    ok_count = 0; type_match = 0; bad = []
-    for i, s in enumerate(symptoms):
-        if not isinstance(s, dict):
-            bad.append(f"item{i}_not_dict"); continue
-        stype = s.get("type")
-        if stype not in allowed_types:
-            bad.append(f"item{i}_bad_type:{stype}"); continue
-        if exp_types and stype in exp_types:
-            type_match += 1
-        if s.get("severity") not in sev_allowed:
-            bad.append(f"item{i}_bad_severity"); continue
-        refs = s.get("evidence_refs", [])
-        if not isinstance(refs, list) or len(refs) == 0:
-            bad.append(f"item{i}_no_refs"); continue
-        sub_refs = [r for r in refs if isinstance(r, str) and len(r) >= 4 and not _is_filler(r)]
-        if len(sub_refs) == 0:
-            bad.append(f"item{i}_filler_refs"); continue
-        ref_tokens = _tokens(" ".join(sub_refs))
-        input_tokens = _tokens(input_text)
-        if ref_tokens and not (ref_tokens & input_tokens):
-            bad.append(f"item{i}_ungrounded_refs"); continue
-        ok_count += 1
-
-    # require ≥1 valid symptom AND (if expected has types) ≥1 type match
-    type_ok = (not exp_types) or (type_match >= 1)
-    ok = (ok_count >= 1) and type_ok
-    fs = {"symptoms": {"ok": ok, "format": ok_count >= 1, "content": ok,
-                         "reason": f"valid={ok_count}/{len(symptoms)} type_match={type_match} exp_types={list(exp_types)[:3]} bad={bad[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_hypothesis_generator_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """v2.1 + n_hyps ±1 of expected + claim Jaccard with EXPECTED claims + narrower tool whitelist."""
-    expected = row.get("expected_output", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    hyps = parsed.get("hypotheses", [])
-    if not isinstance(hyps, list) or len(hyps) < 2:
-        return _result(False, False,
-                       {"hypotheses": {"ok": False, "format": False, "content": False,
-                                       "reason": f"only_{len(hyps) if isinstance(hyps, list) else 0}"}},
-                       head, "too_few_hyps")
-
-    exp_hyps = expected.get("hypotheses", [])
-    exp_n = len(exp_hyps) if isinstance(exp_hyps, list) else 2
-    # claims joined as reference text for predicted claim comparison
-    exp_claim_text = " ".join(h.get("claim", "") for h in exp_hyps if isinstance(h, dict))
-
-    # count: predicted ±1 of expected
-    count_ok = abs(len(hyps) - exp_n) <= 1
-
-    ok_count = 0; bad = []
-    for i, h in enumerate(hyps):
-        if not isinstance(h, dict):
-            bad.append(f"item{i}_not_dict"); continue
-        claim = h.get("claim", "")
-        if not isinstance(claim, str) or len(claim) < 40 or _is_filler(claim):
-            bad.append(f"item{i}_bad_claim_len={len(claim) if isinstance(claim, str) else 0}"); continue
-        # claim should overlap with EXPECTED claims (not just input symptoms)
-        if exp_claim_text and _jaccard(claim, exp_claim_text) < 0.10:
-            bad.append(f"item{i}_claim_off_topic_vs_expected"); continue
-        # Substantive check: capagent may emit strings OR dicts (`{"test": "...", "expected_if_true": "..."}`)
-        # Stringify for length/filler check.
-        item_ok = True
-        for k in ["explains", "does_not_explain", "required_tests"]:
-            v = h.get(k, [])
-            if not isinstance(v, list) or len(v) < 1:
-                bad.append(f"item{i}.{k}_not_list_or_empty"); item_ok = False; break
-            sub = []
-            for x in v:
-                if isinstance(x, str):
-                    s = x
-                elif isinstance(x, dict):
-                    s = " ".join(str(vv) for vv in x.values() if isinstance(vv, (str, int, float)))
-                else:
-                    continue
-                if len(s) >= 15 and not _is_filler(s):
-                    sub.append(s)
-            if len(sub) < 1:
-                bad.append(f"item{i}.{k}_no_substantive"); item_ok = False; break
-        if not item_ok:
-            continue
-        # required_tests: stringify whole list and check any network-domain
-        # anchor token (protocols, mechanisms, components, or CLI tools).
-        # Narrower _has_real_tool_token would reject natural-language diagnostic
-        # prose; the input-copy defence for hyp_gen lives in the claim Jaccard
-        # check above, not in this whitelist.
-        tests = h.get("required_tests", [])
-        tests_blob = " ".join(
-            x if isinstance(x, str)
-            else " ".join(str(v) for v in x.values() if isinstance(v, (str, int, float)))
-            for x in tests
-        )
-        if not _has_test_anchor(tests_blob):
-            bad.append(f"item{i}.required_tests_no_tools"); continue
-        ok_count += 1
-    ok = ok_count >= 2 and count_ok
-    fs = {"hypotheses": {"ok": ok, "format": ok_count >= 2, "content": ok,
-                          "reason": f"good={ok_count}/{len(hyps)} count_ok={count_ok} (pred {len(hyps)} vs exp {exp_n}) bad={bad[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_test_planner_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """v2.1 + tool semantic equivalence with expected_tool + expected_observations has cross-hypothesis impact."""
-    expected = row.get("expected_output", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    na = parsed.get("next_action")
-    if not isinstance(na, dict):
-        return _result(False, False, {"next_action": {"ok": False, "reason": "missing_or_not_dict"}}, head, "")
-
-    exp_na = expected.get("next_action", {}) if isinstance(expected.get("next_action"), dict) else {}
-    exp_tool = (exp_na.get("tool") or exp_na.get("tool_name") or "").lower()
-
-    tool = (na.get("tool") or na.get("tool_name") or "")
-    tool_ok = (isinstance(tool, str) and len(tool) >= 4 and not _is_filler(tool))
-    # tool semantic match: substring overlap with expected tool OR shared real-tool token
-    tool_match_exp = True
-    if exp_tool and tool:
-        tool_l = tool.lower()
-        # exact match OR shared real-tool token OR ≥4-char shared substring
-        shared = (_has_real_tool_token(tool) and _has_real_tool_token(exp_tool)
-                   and any(tok in tool_l and tok in exp_tool for tok in _PCAP_TOOL_TOKENS_V3))
-        tool_match_exp = (tool_l == exp_tool or shared
-                           or any(tool_l[i:i+5] in exp_tool for i in range(len(tool_l) - 4))
-                           or any(exp_tool[i:i+5] in tool_l for i in range(len(exp_tool) - 4)))
-
-    args = na.get("arguments") or na.get("args") or na.get("params") or {}
-    args_ok = isinstance(args, dict) and len(args) >= 1 and not all(_is_filler(str(v)) for v in args.values())
-
-    # expected_observations: ≥2 outcomes; at least one outcome must reference
-    # a hypothesis id (impact dict OR "h1"/"h2"/"H1" token in any value string).
-    eobs = na.get("expected_observations")
-    _HYP_RE = re.compile(r"\b[Hh][12345]\b")
-
-    def _has_hyp_ref(item) -> bool:
-        if isinstance(item, dict):
-            if any(k in item for k in ("impact", "hypothesis_impact", "h_impact")):
-                return True
-            # search all values for hypothesis token
-            for v in item.values():
-                if isinstance(v, str) and _HYP_RE.search(v):
-                    return True
-                if isinstance(v, dict):
-                    for vv in v.values():
-                        if isinstance(vv, str) and _HYP_RE.search(vv):
-                            return True
-        elif isinstance(item, str):
-            if _HYP_RE.search(item):
-                return True
-        return False
-
-    if isinstance(eobs, list):
-        n = len(eobs)
-        has_impact = sum(1 for o in eobs if _has_hyp_ref(o))
-    elif isinstance(eobs, dict):
-        n = len(eobs)
-        has_impact = sum(1 for v in eobs.values() if _has_hyp_ref(v))
-    else:
-        n = 0; has_impact = 0
-    eobs_ok = n >= 2 and has_impact >= 1
-
-    ok = tool_ok and tool_match_exp and args_ok and eobs_ok
-    fs = {"next_action": {
-        "ok": ok, "format": isinstance(na, dict), "content": ok,
-        "reason": f"tool={tool[:30]!r} tool_match_exp={tool_match_exp} args_ok={args_ok} eobs_n={n} impact={has_impact}",
-    }}
-    return _result(ok, ok, fs, head)
-
-
-def grade_observation_normalizer_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """v2.1 + flow_id ∈ expected_flow_ids + type ∈ expected_types."""
-    expected = row.get("expected_output", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    obs = parsed.get("observations", [])
-    if not isinstance(obs, list) or len(obs) < 1:
-        return _result(False, False, {"observations": {"ok": False, "reason": "not_list"}}, head, "")
-
-    exp_obs = expected.get("observations", [])
-    exp_flow_ids = set()
-    exp_types = set()
-    if isinstance(exp_obs, list):
-        for e in exp_obs:
-            if isinstance(e, dict):
-                if e.get("flow_id"): exp_flow_ids.add(str(e["flow_id"]))
-                if e.get("type"): exp_types.add(str(e["type"]))
-
-    ok_count = 0; flow_match_count = 0; type_match_count = 0; bad = []
-    for i, o in enumerate(obs):
-        if not isinstance(o, dict):
-            bad.append(f"item{i}_not_dict"); continue
-        otype = o.get("type", "")
-        if not isinstance(otype, str) or len(otype) < 3 or _is_filler(otype):
-            bad.append(f"item{i}_bad_type"); continue
-        flow_id = o.get("flow_id", "")
-        if not isinstance(flow_id, str) or len(flow_id) < 2 or _is_filler(flow_id):
-            bad.append(f"item{i}_bad_flow_id"); continue
-        pref = o.get("packet_ref")
-        if pref is None: bad.append(f"item{i}_no_packet_ref"); continue
-        pref_ok = isinstance(pref, int) or (isinstance(pref, str) and len(pref) >= 1 and not _is_filler(pref))
-        if not pref_ok: bad.append(f"item{i}_bad_packet_ref:{pref!r}"); continue
-        t = o.get("time")
-        t_ok = False
-        if isinstance(t, (int, float)) and not isinstance(t, bool): t_ok = True
-        elif isinstance(t, str):
-            try: float(t.strip()); t_ok = True
-            except: pass
-        if not t_ok: bad.append(f"item{i}_bad_time:{t!r}"); continue
-        # semantic: at least one of {flow_id, type} must match expected (when expected has any).
-        # type uses _enum_match for synonym/substring tolerance — pred='tls_handshake_complete'
-        # matches exp='tls_handshake_event' via _ENUM_SYNONYMS (2026-05-21 audit found
-        # this as the dominant obs_norm type-drift pattern).
-        if exp_flow_ids and flow_id in exp_flow_ids: flow_match_count += 1
-        if exp_types and _enum_match(otype, exp_types) is not None: type_match_count += 1
-        ok_count += 1
-    # require ≥1 valid AND (if expected has types or flow_ids) ≥1 match on type or flow_id
-    semantic_ok = True
-    if exp_types or exp_flow_ids:
-        semantic_ok = (flow_match_count >= 1) or (type_match_count >= 1)
-    ok = ok_count >= 1 and semantic_ok
-    fs = {"observations": {"ok": ok, "format": ok_count >= 1, "content": ok,
-                             "reason": f"valid={ok_count}/{len(obs)} flow_match={flow_match_count} type_match={type_match_count} exp_types={list(exp_types)[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_hypothesis_updater_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """v2.1 + status MUST equal expected.status (was missing in v2.1)."""
-    expected = row.get("expected_output", {}) or {}
-    input_state = row.get("input_state", {}) or {}
-    parsed, head = _parse(completion)
-    if parsed is None:
-        return _result(False, False, {}, head, "no_json")
-
-    updates = parsed.get("hypothesis_updates", [])
-    if not isinstance(updates, list) or len(updates) < 1:
-        return _result(False, False, {"hypothesis_updates": {"ok": False, "reason": "not_list"}}, head, "")
-
-    exp_updates = expected.get("hypothesis_updates", [])
-    exp_id_to_status = {}
-    if isinstance(exp_updates, list):
-        for u in exp_updates:
-            if isinstance(u, dict) and u.get("id") and u.get("status"):
-                exp_id_to_status[u["id"]] = u["status"]
-
-    h_input = input_state.get("hypothesis", {})
-    expected_id = h_input.get("id") if isinstance(h_input, dict) else None
-    input_text = _input_state_text(input_state)
-
-    allowed_status = {
-        "provisionally_supported", "weakened", "rejected",
-        "confirmed", "inconclusive", "unchanged",
-    }
-
-    ok_count = 0; status_match = 0; bad = []
-    for i, u in enumerate(updates):
-        if not isinstance(u, dict):
-            bad.append(f"item{i}_not_dict"); continue
-        uid = u.get("id")
-        if expected_id and uid != expected_id:
-            bad.append(f"item{i}_id_mismatch:{uid!r}!={expected_id!r}"); continue
-        status = u.get("status")
-        if status not in allowed_status:
-            bad.append(f"item{i}_bad_status:{status!r}"); continue
-        # NEW v3: status MUST match expected (if we have an expected status for this id)
-        exp_st = exp_id_to_status.get(uid)
-        if exp_st and status != exp_st:
-            bad.append(f"item{i}_status_mismatch:{status!r}!={exp_st!r}"); continue
-        if exp_st:
-            status_match += 1
-        reason = u.get("reason", "")
-        if not isinstance(reason, str) or len(reason) < 20 or _is_filler(reason):
-            bad.append(f"item{i}_bad_reason"); continue
-        if input_text and _jaccard(reason, input_text) < 0.03:
-            bad.append(f"item{i}_reason_off_topic"); continue
-        ok_count += 1
-    ok = ok_count >= 1
-    fs = {"hypothesis_updates": {"ok": ok, "format": ok_count >= 1, "content": ok,
-                                    "reason": f"valid={ok_count}/{len(updates)} status_match={status_match} bad={bad[:3]}"}}
-    return _result(ok, ok, fs, head)
-
-
-def grade_stop_controller_v3(row: dict, completion: str) -> tuple[bool, dict]:
-    """Unchanged from v2.1 — already does expected enum match."""
-    from .graders_v2 import grade_stop_controller_v2
-    return grade_stop_controller_v2(row, completion)
-
-
-# ---------- registry ----------
-
-GRADERS_V3: dict[str, Any] = {
-    "diag_problem_framing_v3": grade_problem_framing_v3,
-    "diag_world_model_builder_v3": grade_world_model_builder_v3,
-    "diag_symptom_detector_v3": grade_symptom_detector_v3,
-    "diag_hypothesis_generator_v3": grade_hypothesis_generator_v3,
-    "diag_test_planner_v3": grade_test_planner_v3,
-    "diag_observation_normalizer_v3": grade_observation_normalizer_v3,
-    "diag_hypothesis_updater_v3": grade_hypothesis_updater_v3,
-    "diag_stop_controller_v3": grade_stop_controller_v3,
-}
-
-V1_TO_V3 = {f"diag_{tid}": f"diag_{tid}_v3" for tid in DIAG_TASKS.keys()}

vendor/nanoai/agentic/diag/schemas.py

@@ -1,341 +0,0 @@
-"""Task specifications for the 8 agentic diagnosis micro-tasks (PCAP domain).
-
-Each TaskSpec is shared by:
-  - GT generator (experiments/ksbr_pcap/scripts/gen_diag_eval_gt.py) — capagent gets the schema + 3-shot examples, produces JSON
-  - Grader (nanoai.agentic.diag.graders) — validates `format_ok` against required keys/enums and `content_ok` against expected output
-
-Design notes:
-  * The schemas are intentionally **minimal** — only fields whose absence/wrongness
-    would prevent a downstream agent step from operating. Avoid demanding fields
-    the model can't reasonably infer from input_state alone.
-  * `freetext=True` flags fields where wording will vary across models; graders
-    apply Jaccard overlap (≥0.2 = some semantic signal) rather than exact match.
-  * `enum_values` are STRICT — a model that emits a non-enum value scores
-    content_ok=False on that field. Enums are chosen to be the natural vocab a
-    domain-trained model would use.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Literal
-
-
-FieldKind = Literal["str", "int", "float", "enum", "list", "list_obj", "obj"]
-
-
-@dataclass
-class FieldSpec:
-    name: str
-    kind: FieldKind
-    required: bool = True
-    # for kind=enum
-    enum_values: list[str] | None = None
-    # for kind=list / list_obj: minimum item count for the field to count as "present"
-    min_items: int = 1
-    # for kind=list_obj: required keys on each item
-    item_keys: list[str] | None = None
-    # for kind=list_obj: per-item enum constraints (key → allowed values)
-    item_enums: dict[str, list[str]] | None = None
-    # mark string fields where the model's exact wording will vary;
-    # grader uses Jaccard token overlap instead of equality
-    freetext: bool = False
-
-
-@dataclass
-class TaskSpec:
-    task_id: str                         # short slug, matches eval row's micro_task
-    description: str                     # one-line human description
-    input_keys: list[str]                # keys expected in input_state JSON
-    output_fields: list[FieldSpec]       # what target output must contain
-    prompt_intro: str                    # plain-NL framing prepended to JSON prompt
-    seed_scenarios: list[str] = field(default_factory=list)  # PCAP scenarios for GT generation
-
-
-# ---------- 8 task definitions ----------
-
-PROBLEM_FRAMING = TaskSpec(
-    task_id="problem_framing",
-    description="Given a vague user request, produce a scoped diagnosis goal.",
-    input_keys=["user_request", "available_data", "initial_context"],
-    output_fields=[
-        FieldSpec("diagnosis_goal", "str", freetext=True),
-        FieldSpec("scope", "obj"),  # dict; we only check it exists + is dict
-        FieldSpec("primary_question", "str", freetext=True),
-        FieldSpec("constraints", "list", min_items=1),
-        FieldSpec("initial_uncertainties", "list", min_items=1),
-    ],
-    prompt_intro=(
-        "you are a network-diagnosis planner. given a user request and the "
-        "data they have, produce a scoped diagnosis plan as JSON. think about "
-        "what we *don't yet know* — that's the most important field."
-    ),
-    seed_scenarios=[
-        "user complains a specific HTTPS API is slow at 10:00-10:15, has pcap from the client side only",
-        "user says SSH connections drop intermittently, has pcap from a tap port between client and server",
-        "user reports DNS lookups failing randomly for a list of domains, has pcap from the resolver",
-        "user observes TCP RSTs during file uploads to S3, has client-side pcap",
-        "user notices video conferencing call quality degrades after 10 min, capture point unknown",
-        "user finds requests timing out for one specific backend, has tap-port pcap server-side",
-        "user reports the database connection pool exhaustion alerts during peak hours, pcap from the app tier",
-        "user sees high TLS handshake latency from one client subnet, pcap from server edge",
-    ],
-)
-
-WORLD_MODEL_BUILDER = TaskSpec(
-    task_id="world_model_builder",
-    description="From packet/flow summary, build the entity-flow-assumption world model.",
-    input_keys=["packet_summary", "flow_summary", "capture_metadata"],
-    output_fields=[
-        FieldSpec("entities", "list_obj", min_items=2,
-                  item_keys=["id", "type", "ip"],
-                  item_enums={"type": ["client", "server", "intermediate", "unknown"]}),
-        FieldSpec("flows", "list_obj", min_items=1,
-                  item_keys=["id", "tuple", "protocol"]),
-        FieldSpec("capture_assumptions", "list", min_items=1),
-        FieldSpec("world_model_risks", "list", min_items=1),
-    ],
-    prompt_intro=(
-        "you are a pcap world-model builder. parse a packet/flow summary into "
-        "entities (client/server/intermediate), flows (5-tuple + protocol), and "
-        "explicit assumptions about the capture point. also list risks that could "
-        "make this model wrong (e.g. NAT, asymmetric capture, TLS hiding payload)."
-    ),
-    seed_scenarios=[
-        "summary of HTTPS flow, client 192.168.1.20 → server 10.1.2.3:443, ~127 packets",
-        "summary of bidirectional SSH flow with several retransmissions and dup ACKs",
-        "summary of DNS UDP/53 query + response flow with one query showing no response",
-        "summary of TCP flow ending in RST from server side during a long upload",
-        "summary of unidirectional capture showing only one direction of an SMTP exchange",
-        "summary of TLS 1.3 handshake with two flows from same client IP (port pair changed)",
-        "summary of multiple HTTPS flows from a CDN edge with shared server IP",
-        "summary of port-scan-like traffic to many destinations from one source IP",
-    ],
-)
-
-SYMPTOM_DETECTOR = TaskSpec(
-    task_id="symptom_detector",
-    description="Identify symptoms in flow/event data WITHOUT explaining cause.",
-    input_keys=["flow_events", "tcp_analysis", "time_window"],
-    output_fields=[
-        FieldSpec("symptoms", "list_obj", min_items=1,
-                  item_keys=["type", "flow_id", "time_range", "severity", "evidence_refs"],
-                  item_enums={
-                      "type": [
-                          "tcp_retransmission_burst", "tcp_duplicate_ack", "tcp_zero_window",
-                          "tcp_reset", "server_response_gap", "client_idle_gap",
-                          "tls_handshake_delay", "dns_lookup_delay", "dns_no_response",
-                          "rtt_spike", "high_jitter", "mtu_blackhole",
-                      ],
-                      "severity": ["low", "medium", "high", "critical"],
-                  }),
-    ],
-    prompt_intro=(
-        "you are a pcap symptom detector. produce ONLY observed phenomena — do "
-        "NOT explain cause. e.g. 'retransmission burst from 10:03:11 to 10:03:13' "
-        "is a symptom; 'network is congested' is a hypothesis (don't write that). "
-        "every symptom must cite evidence_refs to specific events from the input."
-    ),
-    seed_scenarios=[
-        "flow events showing 15 retransmissions clustered in a 2-second window inside a 5-minute flow",
-        "tcp analysis showing 3 zero-window advertisements from server during file upload",
-        "flow events showing a 2300ms gap between request bytes finishing and first response byte",
-        "dns events showing 4/40 queries received no response within 5s timeout",
-        "tcp analysis showing alternating SYN-ACK then RST without ESTABLISHED state",
-        "tls events showing client hello → server hello gap of 1.8 seconds (typical: <100ms)",
-        "tcp events showing duplicate ACKs every 200ms with no progress in seq numbers",
-        "flow events showing RTT jumping from baseline 5ms to 250ms intermittently",
-    ],
-)
-
-HYPOTHESIS_GENERATOR = TaskSpec(
-    task_id="hypothesis_generator",
-    description="Given symptoms + world model + any prior hypotheses, propose OR refine candidate root-cause hypotheses.",
-    input_keys=["world_model", "symptoms", "diagnosis_goal", "prior_hypotheses"],
-    output_fields=[
-        FieldSpec("hypotheses", "list_obj", min_items=2,
-                  item_keys=["id", "claim", "explains", "does_not_explain", "required_tests"]),
-    ],
-    prompt_intro=(
-        "you are a pcap hypothesis generator running inside a multi-turn diagnosis "
-        "loop. given world_model + symptoms + diagnosis_goal AND any prior_hypotheses "
-        "from earlier turns, produce the CURRENT hypothesis list.\n"
-        "  - if prior_hypotheses is empty (turn 1): generate 2-3 fresh competing root "
-        "    causes. assign stable ids (H1, H2, H3).\n"
-        "  - if prior_hypotheses is non-empty (turn 2+): REFINE rather than regenerate. "
-        "    KEEP the original id of any hypothesis that remains plausible given new "
-        "    symptoms; UPDATE its claim/required_tests if evidence has narrowed it; "
-        "    ADD a new hypothesis ONLY if symptoms reveal a path the prior list doesn't "
-        "    explain; DROP hypotheses marked status=rejected by hypothesis_updater. "
-        "    NEVER renumber stable hypotheses — H1 stays H1 across turns. NEVER reset "
-        "    the slate just because new symptoms arrived; build on prior reasoning.\n"
-        "EVERY hypothesis must include `required_tests` that could falsify it. "
-        "each `claim` must be specific (e.g. 'server application is the bottleneck, "
-        "not network' not just 'application issue'). do NOT pick a single root cause — "
-        "produce ≥2 competing hypotheses unless prior_hypotheses is already down to 1 "
-        "confirmed.\n"
-        "\n"
-        "FIELD QUALITY (strict — rows violating these are dropped at gen-time):\n"
-        "  - `explains`: list of ≥2 FULL-SENTENCE descriptions of what the hypothesis "
-        "    accounts for. Each item must be a complete English clause (≥15 chars), "
-        "    not a bare id like 'evt_1' or 'S2'. EXAMPLE good: 'Retransmissions cluster "
-        "    during the gap window because the server stops ACK-ing'. EXAMPLE bad: "
-        "    'evt_3' or 'S2'.\n"
-        "  - `does_not_explain`: list of ≥1 FULL-SENTENCE description of what this "
-        "    hypothesis FAILS to account for. NEVER emit empty []; every honest "
-        "    hypothesis has gaps. If you can't think of one, the hypothesis is too "
-        "    broad — narrow it. EXAMPLE good: 'Does not explain why baseline RTT is "
-        "    unaffected outside the gap window'. EXAMPLE bad: empty list or just "
-        "    'Why X' (incomplete).\n"
-        "  - `required_tests`: list of ≥2 FULL-SENTENCE testable actions. Each must "
-        "    name a concrete diagnostic mechanism (tool, protocol, mechanism). "
-        "    EXAMPLE good: 'Run tshark with filter ip.addr == X to capture handshake "
-        "    timing'. EXAMPLE bad: 'check it' or 'verify'."
-    ),
-    seed_scenarios=[
-        # Turn-1 scenarios (empty prior_hypotheses):
-        "symptom: 2300ms server response gap with NO tcp anomalies in the gap window",
-        "symptom: retransmission burst + duplicate ACKs concentrated in delay windows",
-        "symptom: TLS handshake delay 1.8s on one specific client subnet only",
-        "symptom: DNS resolution failures clustered to one specific upstream resolver",
-        "symptom: zero-window advertisements from server only during large upload bursts",
-        "symptom: tcp RSTs from server side mid-flow after ~30s of activity",
-        "symptom: MTU blackhole-like behavior (small packets ok, large get dropped)",
-        "symptom: high jitter + occasional rtt spikes correlated with backup job times",
-        # Turn-N>=2 scenarios (non-empty prior_hypotheses — refinement):
-        "prior: [H1 server-bottleneck status=provisionally_supported, H2 network-congestion status=weakened] + new evidence: server CPU stays <30% during gap → DROP H1, ADD H3 about middlebox processing",
-        "prior: [H1 TLS-handshake-misconfig, H2 firewall-rate-limit] + new evidence: TLS handshake completes normally for control flow → MARK H1 weakened, KEEP H2, ADD nothing — refinement only",
-        "prior: [H1 path-MTU-blackhole status=provisionally_supported] + new evidence: small-packet pings also fail intermittently → BROADEN H1 claim to 'path quality degraded', keep id",
-        "prior: [H1 dns-resolver-failure status=rejected, H2 client-stub-misconfig status=provisionally_supported] + new evidence: another resolver also fails → DROP H1 (already rejected), keep H2, ADD H3 about client name resolution policy",
-        "prior: [H1 server-app-bottleneck status=confirmed] + new evidence: no contradicting observations after 3 tests → KEEP H1 only, downstream stop_controller should fire",
-        "prior: [H1, H2, H3 all status=inconclusive] + new evidence: a 4th retransmission burst in a new flow → consider whether one prior hyp now explains the broader pattern; refine the strongest claim",
-    ],
-)
-
-TEST_PLANNER = TaskSpec(
-    task_id="test_planner",
-    description="Choose the next-best tool action to validate/falsify a hypothesis.",
-    input_keys=["hypotheses", "evidence_so_far", "available_tools"],
-    output_fields=[
-        FieldSpec("next_action", "obj"),  # we'll require nested keys via custom check in grader
-    ],
-    prompt_intro=(
-        "you are a pcap test planner. choose the SINGLE next action with the highest "
-        "expected information gain — i.e. it should EITHER strongly support OR strongly "
-        "weaken at least one hypothesis. the action must include a concrete tool command "
-        "and explicit `expected_observations` mapping outcomes to hypothesis support/weaken."
-    ),
-    seed_scenarios=[
-        "two hypotheses: app-delay vs network-loss; need test to distinguish them inside the delay window",
-        "two hypotheses: server CPU bottleneck vs downstream call delay; need a test using only pcap",
-        "two hypotheses: client-side firewall vs path MTU; need a test that disambiguates",
-        "two hypotheses: TLS-side delay vs TCP-side delay; need finer-grained tshark timing",
-        "two hypotheses: server overload vs client retransmit storm; need a test pinning direction",
-        "two hypotheses: DNS resolver failure vs network reachability to resolver; need a distinguishing test",
-        "three hypotheses for periodic latency spikes: backup, GC pause, cron job; one tool query each",
-        "two hypotheses about RST source (server-app vs middlebox); need a test",
-    ],
-)
-
-OBSERVATION_NORMALIZER = TaskSpec(
-    task_id="observation_normalizer",
-    description="Convert raw tool output (tshark fields, zeek logs) into structured observations.",
-    input_keys=["raw_output", "tool", "world_model"],
-    output_fields=[
-        FieldSpec("observations", "list_obj", min_items=1,
-                  item_keys=["id", "type", "flow_id", "time", "packet_ref"]),
-    ],
-    prompt_intro=(
-        "you are a pcap observation normalizer. take raw tool output (e.g. tshark "
-        "tab-separated fields) and produce structured observations. EVERY field "
-        "(time, packet_ref, raw_fields) must come from the input — do not invent. "
-        "preserve raw values; the agent loop relies on packet_ref being verifiable."
-    ),
-    seed_scenarios=[
-        "tshark output with frame.time_epoch, ip.src, ip.dst, tcp.seq, tcp.ack for retransmissions",
-        "tshark output with frame.number, tls.handshake.type for a TLS exchange",
-        "zeek conn.log entries showing flow tuples + duration + bytes",
-        "tshark output with dns.qry.name, dns.flags.response, dns.flags.rcode",
-        "tshark output with tcp.analysis.flags marking RTOs, duplicate ACKs, retransmissions",
-        "tshark output with tcp.window_size_value showing zero-window advertisements",
-        "zeek dns.log with query / response pairs and timing",
-        "tshark output with frame.time_relative and tcp.len for response gap analysis",
-    ],
-)
-
-HYPOTHESIS_UPDATER = TaskSpec(
-    task_id="hypothesis_updater",
-    description="Update hypothesis status given new observation + prediction.",
-    input_keys=["hypothesis", "prediction", "observation"],
-    output_fields=[
-        FieldSpec("hypothesis_updates", "list_obj", min_items=1,
-                  item_keys=["id", "status", "reason"],
-                  item_enums={"status": [
-                      "provisionally_supported", "weakened", "rejected",
-                      "confirmed", "inconclusive", "unchanged",
-                  ]}),
-    ],
-    prompt_intro=(
-        "you are a pcap hypothesis updater. given a hypothesis with its prediction "
-        "and a new observation, decide if observation **supports**, **weakens**, or "
-        "**rejects** the hypothesis. CRITICAL: be willing to weaken/reject — do not "
-        "pretend partial evidence is confirmation. `reason` must cite the specific "
-        "observation field that drove the change."
-    ),
-    seed_scenarios=[
-        "hyp: network-loss is cause; pred: retransmissions in delay window; obs: no retransmissions",
-        "hyp: app-delay is cause; pred: tcp clean during gap; obs: tcp clean during gap",
-        "hyp: client firewall blocks; pred: SYN goes out, no SYN-ACK; obs: SYN+SYN-ACK, no app data",
-        "hyp: dns failing; pred: queries with no response; obs: response received but RCODE=SERVFAIL",
-        "hyp: MTU blackhole; pred: small packets ok, large drop; obs: ALL sizes get through fine",
-        "hyp: server overload; pred: response gap on most flows; obs: gap on only 5%",
-        "hyp: tls negotiation issue; pred: hello-to-hello slow; obs: hello-to-hello fast, finished slow",
-        "hyp: jitter from backup job; pred: spike pattern at scheduled times; obs: spike unrelated to schedule",
-    ],
-)
-
-STOP_CONTROLLER = TaskSpec(
-    task_id="stop_controller",
-    description="Decide whether to continue diagnosis, stop with conclusion, or stop inconclusive.",
-    input_keys=["current_hypotheses", "evidence_summary", "tools_used", "uncertainty"],
-    output_fields=[
-        FieldSpec("stop_decision", "enum",
-                  enum_values=["continue", "stop_with_conclusion", "stop_inconclusive"]),
-        FieldSpec("reason", "str", freetext=True),
-        FieldSpec("required_before_stop", "list", min_items=0),  # may be empty if stop=stop_*
-    ],
-    prompt_intro=(
-        "you are a pcap stop controller. given the diagnosis state, output ONE of: "
-        "(a) continue if a clear next test would meaningfully narrow uncertainty, "
-        "(b) stop_with_conclusion if remaining uncertainty is acceptable and one "
-        "hypothesis dominates, (c) stop_inconclusive if available data cannot "
-        "distinguish remaining hypotheses. do NOT keep digging when the data has "
-        "been exhausted — over-digging produces false root causes. EQUALLY, do "
-        "NOT conclude when one cheap test would meaningfully narrow uncertainty — "
-        "premature conclusion produces false root causes too. When in doubt about "
-        "whether enough evidence has been collected, default to stop_inconclusive "
-        "rather than stop_with_conclusion: a hedged answer is better than a wrong one."
-    ),
-    seed_scenarios=[
-        "two hyps still competing 60/40, one cheap remaining test would distinguish",
-        "one hyp at 0.85 confidence, others at <0.1, available data exhausted",
-        "all hyps still <0.5 confidence, pcap is one-sided, no more reachable evidence",
-        "single hyp confirmed across 5 flows, no contradicting evidence",
-        "two hyps equally weighted, all remaining tools would yield same evidence already seen",
-        "one hyp supported on 3/10 sampled flows but rejected on 7/10 — likely incomplete",
-        "loop: 3 actions ran, no hyp confidence has moved at all in last 2 rounds",
-        "investigation has run 8 actions; cost ceiling reached; current state still ambiguous",
-    ],
-)
-
-
-DIAG_TASKS: dict[str, TaskSpec] = {
-    spec.task_id: spec
-    for spec in [
-        PROBLEM_FRAMING, WORLD_MODEL_BUILDER, SYMPTOM_DETECTOR, HYPOTHESIS_GENERATOR,
-        TEST_PLANNER, OBSERVATION_NORMALIZER, HYPOTHESIS_UPDATER, STOP_CONTROLLER,
-    ]
-}
-
-
-def all_task_ids() -> list[str]:
-    return list(DIAG_TASKS.keys())

vendor/nanoai/agentic/diag/train_scenarios.py

@@ -1,341 +0,0 @@
-"""Train-only scenario seeds for diag_sft_v1 SFT data generation.
-
-DISJOINT from the eval `seed_scenarios` in schemas.py. The split prevents
-the model from memorizing the exact eval prompts during SFT — a future
-diag_eval_v1 measurement on a model trained from these scenarios is then
-testing schema generalization rather than scenario recall.
-
-Pairing: `gen_diag_sft.py` reads from TRAIN_SCENARIOS, `gen_diag_eval_gt.py`
-reads from schemas.py:seed_scenarios. The 24-row `<task>_heldout.jsonl`
-remains the ultimate contamination gate (uses eval seeds, never trained on).
-
-Convention: **32 scenarios per task** (expanded from initial 16 on 2026-05-21
-to improve per-scenario diversity at 50k+ scale gen). Semantically distinct
-families from the 8 eval scenarios. Capagent will randomly sample with
-replacement at gen time, so total unique inputs per task scale to ~250-400
-with model temperature variation.
-"""
-from __future__ import annotations
-
-
-TRAIN_SCENARIOS: dict[str, list[str]] = {
-    "problem_framing": [
-        # Original 16 (2026-05-20)
-        "user reports TLS-terminated reverse proxy occasionally returns 502 to mobile clients only, has edge-pcap",
-        "user observes WebSocket connections disconnect after exactly 60s of idle, has client-side pcap",
-        "user notices a Kafka consumer group rebalance storm at midnight UTC daily, has tap pcap from broker side",
-        "user reports SMTP submissions hang for 30s before completing, pcap from MTA edge available",
-        "user sees gRPC unary RPCs timeout in p99 only, pcap from both client and server LB",
-        "user complains BitTorrent-like long-lived connections die after firewall reload, edge pcap",
-        "user notes IoT MQTT brokers see PINGRESP delays during firmware updates, broker-side pcap",
-        "user reports VPN tunnel periodically drops single packets, capture from internal endpoint",
-        "user sees QUIC fallback to TCP for a subset of clients behind one ISP, edge-pcap",
-        "user observes Redis Sentinel failover taking 18s on a 3-node cluster, broker tap-port pcap",
-        "user reports SNMP polling missing one MIB tree every other poll, no obvious pattern, mgmt vlan pcap",
-        "user complains LDAP STARTTLS rejects from one client, pcap available from LDAP server side",
-        "user observes BGP session flapping between two routers in same AS, both router-tap pcaps",
-        "user reports RTSP stream pause for 8s every 15 min, pcap from camera side only",
-        "user notes container egress traffic going to wrong NAT pool intermittently, node-host pcap",
-        "user sees iSCSI initiator lose path during snapshot, target-side pcap available",
-        # Expansion +16 (2026-05-21)
-        "user notices Memcached connections leak file descriptors over 48h, no obvious failure, server-side pcap",
-        "user reports IPSec tunnel rekey takes 12s causing data plane outage, both peers pcap",
-        "user observes WebRTC video glitches when remote peer moves through cell handoff, mobile pcap",
-        "user notes Envoy sidecar adds 30ms p99 latency only for inbound traffic, sidecar-host pcap",
-        "user reports periodic Cassandra gossip storm causes node flapping, cluster-internal pcap",
-        "user complains S3 multipart upload fails at part 3 of 10 reproducibly, client pcap",
-        "user observes JMS message delivery skipping queue order during failover, broker tap",
-        "user reports NTP offset drifts 200ms during business hours, NTP server-side pcap",
-        "user notes Mongo replica set election takes 45s on a 5-node cluster, primary-side pcap",
-        "user sees DoH (DNS-over-HTTPS) queries fail to one provider only, browser-host pcap",
-        "user reports Modbus TCP slave responses delayed during high temperature, slave-side pcap",
-        "user observes Tor circuit setup taking 12s on a specific exit selection, client pcap",
-        "user notes Postgres logical replication slot lag growing during peak, subscriber pcap",
-        "user reports SAP RFC calls fail with timeout to one application server, mtu pcap",
-        "user complains video doorbell loses live feed after exact 60-min sessions, doorbell-side pcap",
-        "user observes RDP session disconnects when bandwidth drops below 256kbps, edge pcap",
-    ],
-    "world_model_builder": [
-        # Original 16
-        "summary of dual-stack IPv4+IPv6 web traffic to same hostname, AAAA preferred but TCP-RST'd",
-        "summary of MQTT-over-WSS flow from IoT gateway through L7 LB to broker cluster",
-        "summary of TCP traffic showing PROXY-protocol prefix headers before TLS",
-        "summary of QUIC v1 traffic with version negotiation between client and server",
-        "summary of multipath TCP (MPTCP) flow with two subflows from same client",
-        "summary of ICMP TimeExceeded responses interleaved with TCP application traffic",
-        "summary of LACP-bundled traffic captured on one of the two member interfaces only",
-        "summary of GRE-encapsulated TCP showing inner and outer 5-tuples both visible",
-        "summary of SCTP multihomed association between two endpoints with two IPs each",
-        "summary of VXLAN-encapsulated east-west traffic captured at the spine switch",
-        "summary of TCP keepalive probes on a long-idle BGP session every 60s",
-        "summary of TFTP UDP flow with port renegotiation after WRQ acknowledgment",
-        "summary of PIM-SM multicast control plus IGMP joins from receivers",
-        "summary of SCCP signaling between IP phone and call manager with frequent registrations",
-        "summary of TCP flow with selective ACK options heavily used during loss recovery",
-        "summary of WireGuard UDP traffic (no L7 visibility) between two peer endpoints",
-        # Expansion +16
-        "summary of HTTP/3 over QUIC v2 traffic with version negotiation between client and CDN edge",
-        "summary of SRv6 (segment routing IPv6) ingress traffic with multiple SIDs in IPv6 header",
-        "summary of MPLS L3VPN traffic showing label stack with three labels on transit packets",
-        "summary of OSPFv3 IPv6 hello packets between two routers in same area",
-        "summary of EVPN MAC advertisement BGP UPDATEs between two leaf switches",
-        "summary of CoAP UDP requests over DTLS to multiple IoT devices behind same gateway",
-        "summary of WebRTC ICE STUN binding requests during connection candidate gathering",
-        "summary of mTLS-protected gRPC streaming bidi between sidecar proxies",
-        "summary of EAP-TLS supplicant authentication on 802.1X-enabled switch port",
-        "summary of BFD (bidirectional forwarding detection) control packets between two routers",
-        "summary of NVMe-oF over RDMA traffic between initiator and target on RoCEv2",
-        "summary of WebSocket connections to a chat backend with per-message DEFLATE compression",
-        "summary of LACP control packet exchange between switch and server team interfaces",
-        "summary of Profinet IO cyclic data exchange between PLC and field device",
-        "summary of Tailscale WireGuard mesh with multiple peers on same magic DNS",
-        "summary of SRTP-encrypted RTP from WebRTC video conf, no L7 visibility but timing patterns",
-    ],
-    "symptom_detector": [
-        # Original 16
-        "events showing 8 RTOs back-to-back within 1.5s on a single flow during file transfer",
-        "tcp analysis showing SACK blocks growing across 12 packets without forward progress",
-        "events showing TLS Alert level=fatal description=handshake_failure mid-handshake",
-        "tcp analysis showing window-scaling factor of 0 used after expected 7 — degraded throughput",
-        "events showing 200ms gap between server FIN and client FIN-ACK on closing handshake",
-        "tcp analysis flagging out-of-order segments persisting 400ms before reorder buffer drains",
-        "events showing HTTP/2 GOAWAY frame from server with error_code=ENHANCE_YOUR_CALM",
-        "tcp analysis showing PUSH+FIN combined flag on a single segment from a server response",
-        "events showing 3 consecutive ICMP fragmentation-needed messages followed by TCP stalls",
-        "tcp analysis showing packets out of seq number range (likely packet injection or replay)",
-        "events showing UDP datagrams arriving after 800ms+ delay (jitter) on a real-time flow",
-        "tcp analysis showing keepalive probes failing 3x in a row before connection RST",
-        "events showing TLS resumption attempt rejected with new full handshake every connection",
-        "tcp analysis showing client window shrinking to 1024 mid-flow (window collapse)",
-        "events showing 5 SYN retries with 1s/2s/4s/8s/16s spacing — classic SYN backoff",
-        "tcp analysis showing simultaneous open (both sides send SYN nearly at once)",
-        # Expansion +16
-        "events showing TLS Alert level=warning description=user_canceled during handshake",
-        "tcp analysis showing RST with non-zero ACK from middlebox not matching server side",
-        "events showing HTTP/2 PING frame with payload mismatch between request and response",
-        "tcp analysis showing 8 consecutive zero-window updates within 50ms (rapid window collapse)",
-        "events showing IPv6 NDP redirect messages directing traffic away from default GW",
-        "tcp analysis showing congestion window collapse to 1 MSS after 3 dup-ACKs",
-        "events showing TLS ClientHello with no SNI extension from updated browser client",
-        "tcp analysis flagging timestamp option (TSecr=0) after established connection (TS option dropped)",
-        "events showing HTTP/2 GOAWAY with last_stream_id=0 right after settings exchange",
-        "tcp analysis showing PSH+ACK alternating with FIN+ACK from same endpoint mid-flow",
-        "events showing IPSec ESP sequence number wraparound triggering rekey",
-        "tcp analysis showing SACK ranges overlap (likely buggy SACK implementation)",
-        "events showing UDP checksum errors clustered around a specific interface ingress",
-        "tcp analysis showing window scale = 14 used (max RFC value, suspicious memory)",
-        "events showing HTTP/3 STREAM frame with FIN bit set on stream 0 (control stream)",
-        "tcp analysis showing PAWS (protection against wrapped sequences) check failures",
-    ],
-    "hypothesis_generator": [
-        # Original 16
-        "symptom: HTTP/2 GOAWAY ENHANCE_YOUR_CALM after burst of 50 requests on one connection",
-        "symptom: TLS resumption tickets rejected, full handshake every time on subset of clients",
-        "symptom: SCTP heartbeat timeout on one of two paths in a multihomed association",
-        "symptom: BGP UPDATE messages with malformed AS_PATH attribute appearing periodically",
-        "symptom: MTU-related packet drops only on traffic transiting one specific WAN link",
-        "symptom: QUIC handshake taking 3x normal RTT — possible amplification protection trigger",
-        "symptom: PROXY protocol header missing on some connections to backend server",
-        "symptom: HTTP/2 RST_STREAM stream-level cancellations spiking from one client",
-        "symptom: NFS RPC retransmissions clustered around storage failover events",
-        "symptom: TCP simultaneous open events appearing on internal east-west traffic",
-        "symptom: TLS SNI mismatch between hostname in HTTP host header and TLS cert CN",
-        "symptom: VXLAN inner-flow drops only when inner-MTU exceeds carrier-MTU minus encap",
-        "symptom: HTTP/3 (QUIC) connections falling back to HTTP/2 for clients behind one ISP",
-        "symptom: SSH client SSH-2.0 banner exchange completes but auth phase times out",
-        "symptom: Redis MULTI/EXEC blocks taking 3s where 50ms is baseline",
-        "symptom: TCP RST from server with TTL value different from established traffic",
-        # Expansion +16
-        "symptom: BGP UPDATE messages stuck behind one peer's TCP MSS issue",
-        "symptom: TLS clients failing only when CDN edge serves session ticket from peer",
-        "symptom: WebSocket connections fail to upgrade for traffic transiting one load balancer",
-        "symptom: QUIC 0-RTT data being rejected by server even when ticket is valid",
-        "symptom: NTP polling falls back to text protocol over UDP/123 unexpectedly",
-        "symptom: gRPC bidi streams cancelled after 30s of idle by middle proxy",
-        "symptom: MPLS-VPN packets dropped at PE router only for traffic with specific DSCP",
-        "symptom: Kafka producer batches timing out when broker replica leader changes",
-        "symptom: DTLS-protected SIP signaling failing for clients behind one CGNAT",
-        "symptom: TCP ECN markings stripped on traffic transiting one ISP link",
-        "symptom: HTTP/2 server push frames being rejected with REFUSED_STREAM",
-        "symptom: SCTP path failure between two endpoints despite both interfaces UP",
-        "symptom: Redis pub/sub messages reordered when client reconnects to different replica",
-        "symptom: DNS-over-HTTPS responses delayed by 1.5s to specific resolver",
-        "symptom: TCP zero-copy send failing on Linux 6.x with specific BPF program loaded",
-        "symptom: TLS 1.3 EarlyData stream getting dropped at HTTP/2 ALPN negotiation",
-        # Turn-N refinement scenarios (non-empty prior_hypotheses) — teach carry-over:
-        "turn-N refinement. prior_hypotheses=[H1 server-app-bottleneck status=provisionally_supported, H2 network-congestion status=weakened]. new evidence: server CPU stayed <30% throughout the gap window. Refine: drop H1, keep H2 but downgrade further, add H3 about middlebox/inspection delay",
-        "turn-N refinement. prior_hypotheses=[H1 TLS-handshake-misconfig, H2 firewall-rate-limit status=open]. new evidence: TLS handshake completes normally for the control flow. Refine: mark H1 weakened (keep id), keep H2, add nothing",
-        "turn-N refinement. prior_hypotheses=[H1 path-MTU-blackhole status=provisionally_supported]. new evidence: small-packet ICMP pings now also failing intermittently. Refine: broaden H1's claim to 'path quality degraded across packet sizes', keep id H1",
-        "turn-N refinement. prior_hypotheses=[H1 dns-resolver-failure status=rejected, H2 client-stub-misconfig status=provisionally_supported]. new evidence: another fallback resolver also fails. Refine: H1 stays dropped, keep H2, add H3 about client name-resolution policy or hosts file",
-        "turn-N refinement. prior_hypotheses=[H1 server-app-bottleneck status=confirmed, H2 network-congestion status=rejected, H3 client-side status=rejected]. new evidence: no contradiction in 3 follow-up tests. Refine: keep H1 only as the surviving hypothesis (chain ready for stop_with_conclusion)",
-        "turn-N refinement. prior_hypotheses=[H1 inconclusive, H2 inconclusive, H3 inconclusive]. new evidence: a 4th retransmission burst on a brand new flow with same client subnet. Refine: tighten the strongest existing claim to be subnet-specific; do NOT add yet another hypothesis",
-        "turn-N refinement. prior_hypotheses=[H1 server-TLS-stack-bug status=weakened, H2 client-TLS-stack-bug status=provisionally_supported]. new evidence: ServerHello timing now within 20ms across all flows. Refine: drop H1 (now fully contradicted), keep H2, add H3 about TLS extension handling",
-        "turn-N refinement. prior_hypotheses=[H1 MTU-blackhole status=provisionally_supported]. new evidence: large packets now succeed across multiple flows after middlebox MTU was raised. Refine: H1 status promotes towards confirmed; keep id; chain ready to stop_with_conclusion next turn",
-        "turn-N refinement. prior_hypotheses=[H1 DNS-cache-poisoning, H2 client-resolver-misconfig]. new evidence: dig from different resolver returns expected IP. Refine: keep H1 (still possible), mark H2 weakened, add H3 about authoritative-server inconsistency",
-        "turn-N refinement. prior_hypotheses=[H1 firewall-state-table-eviction, H2 server-explicit-close]. new evidence: server logs show no application-level close, RSTs arrive even during active data transfer. Refine: tighten H1 to specify middlebox; mark H2 weakened",
-        "turn-N refinement. prior_hypotheses=[H1 server-side-thread-starvation status=open]. new evidence: only this one hypothesis remains plausible and is supported by 4 independent observations. Refine: confirm H1 (status=confirmed), keep id, downstream should fire stop_with_conclusion",
-        "turn-N refinement. prior_hypotheses=[H1, H2 both inconclusive after 2 turns]. new evidence: a third tool run also gave ambiguous output that matches both H1 and H2 equally. Refine: keep both hypotheses unchanged but log that data exhaustion is approaching — stop_inconclusive is becoming the right next step",
-        "turn-N refinement. prior_hypotheses=[H1 IPsec-MTU status=weakened, H2 carrier-fragmentation status=provisionally_supported]. new evidence: large packets fail even on flows that don't traverse the IPsec tunnel. Refine: drop H1 (now contradicted), keep H2, add H3 about endpoint NIC offload bug",
-        "turn-N refinement. prior_hypotheses=[H1, H2, H3]. new evidence: a measurement error was found in the earlier observation that supported H3. Refine: revert H3 to status=open, do not drop or confirm; keep H1 and H2",
-        "turn-N refinement. prior_hypotheses=[H1 application-thread-pool-exhaustion status=provisionally_supported]. new evidence: thread-pool metrics from server actually look healthy, but a new symptom of GC pause spikes appeared. Refine: weaken H1, add H4 about garbage collector pauses correlating with the gap windows",
-        "turn-N refinement. prior_hypotheses=[H1 client-side-buffer-overflow]. new evidence: client buffer metrics look fine but a packet shows zero-window from server. Refine: weaken H1, add H2 about server-side flow control / receive buffer pressure",
-    ],
-    "test_planner": [
-        # Original 16
-        "two hypotheses: middlebox bookmark-stripping vs server-app reject; need test to pin layer",
-        "two hypotheses: TLS resumption ticket key rotation vs cache eviction; need test inside server logs vs network",
-        "three hypotheses for slow first-byte: dns, tcp_handshake, tls_handshake; one tshark filter each",
-        "two hypotheses: MTU blackhole inside tunnel vs end-host MSS clamping issue; need a test",
-        "two hypotheses for spurious RSTs: middlebox state-table eviction vs server-app explicit close; need test",
-        "two hypotheses: client SACK disabled vs server SACK disabled; need a test using one direction's pcap only",
-        "two hypotheses: tcp window-scale option dropped by middlebox vs misconfigured client; need test",
-        "three hypotheses for jitter: cross-traffic, hardware queue, polling-cycle; one test each",
-        "two hypotheses: NAT entry expiration vs ECMP rehash mid-flow; need a test",
-        "two hypotheses for one-way audio: codec mismatch vs return-path drop; need a test",
-        "two hypotheses for HTTP/2 stalls: flow-control credit exhaustion vs HEADERS frame fragmentation; need test",
-        "two hypotheses for VPN drop: rekey storm vs MTU change; need test using only IPsec headers",
-        "three hypotheses for periodic latency: ARP refresh, GC pause, IPC ring full; one test each",
-        "two hypotheses for TLS slow handshake: OCSP stapling fetch vs cert chain depth; need test",
-        "two hypotheses: BGP convergence vs route reflector delay; need a passively-observable test",
-        "two hypotheses for HTTP 503: backend timeout vs LB health-check flap; need test using pcap",
-        # Expansion +16
-        "two hypotheses: TLS 1.3 0-RTT replay vs ticket binding mismatch; need test using single client capture",
-        "three hypotheses for IPv6 hop limit drops: ICMP rate limit, transit RA prefix mismatch, MTU; one test each",
-        "two hypotheses: HTTP/3 PADDING frame causing stream stall vs HEADER frame fragmentation; need test",
-        "two hypotheses for SCTP heartbeat fail: path MTU vs NAT pinhole; need test using primary path only",
-        "three hypotheses for slow CoAP responses: server retransmit, gateway buffering, DTLS handshake; one test each",
-        "two hypotheses: TCP timestamps PAWS reject vs sequence number wrap; need test using TS option",
-        "two hypotheses for missing Multicast: IGMP membership not joined vs PIM-SM prune; need test",
-        "three hypotheses for HTTP/2 stall: SETTINGS_MAX_CONCURRENT_STREAMS, flow control, GOAWAY; one test each",
-        "two hypotheses: BGP convergence vs route advertisement filter; need test using AS_PATH attribute",
-        "two hypotheses for failed WebRTC: ICE failure vs DTLS-SRTP key exchange; need test using STUN binding stats",
-        "three hypotheses for NFS slow: rpc.statd timeout, client cache stale, server queue full; one test each",
-        "two hypotheses: WireGuard MTU vs key rotation drift; need test using handshake_init counter",
-        "two hypotheses: NTP DDoS amplification block vs ratelimit; need test using mode 6/7 packets",
-        "two hypotheses for RTSP pause: codec switch vs keyframe miss; need test using NAL unit type",
-        "three hypotheses for OAuth2 fail: token-endpoint TLS, JWT signature, scope mismatch; one test each",
-        "two hypotheses for QUIC fallback: UDP block at LB vs ALPN advertise of h3-only; need test using SettingsParameters",
-    ],
-    "observation_normalizer": [
-        # Original 16
-        "tshark output with sip.method, sip.cseq for SIP INVITE/100/180/200 dialog",
-        "tshark output with diameter.cmd.code, diameter.flags showing Cer/Cea exchange",
-        "tshark output with rtp.ssrc, rtp.seq, rtp.timestamp, rtp.marker for an RTP audio stream",
-        "tshark output with smb2.cmd, smb2.tree.id for SMB session establishment",
-        "tshark output with kerberos.msg_type, kerberos.realm for AS-REQ/AS-REP exchange",
-        "tshark output with ntp.stratum, ntp.refid, ntp.precision for NTP client polling",
-        "tshark output with ldap.protocolOp showing bindRequest and searchResultEntry",
-        "tshark output with snmp.opcode, snmp.varbind for GET/RESPONSE pairs",
-        "zeek http.log with method, uri, status_code, response_body_len fields",
-        "zeek ssl.log with version, cipher, server_name, validation_status",
-        "zeek dhcp.log with msg_types and DHCP option values for lease acquisition",
-        "zeek ssh.log with auth_attempts and auth_success fields",
-        "tshark output with quic.frame.type, quic.connection.id for a QUIC initial packet",
-        "tshark output with mqtt.msgtype, mqtt.topic, mqtt.qos for a publish exchange",
-        "tshark output with bgp.type, bgp.path_attr.as_path for an UPDATE message",
-        "tshark output with rpcrdma.opcode for RDMA-over-TCP NFS traffic",
-        # Expansion +16
-        "tshark output with diameter.session.id, diameter.origin.host for Gx session establishment",
-        "tshark output with mip6.type for IPv6 mobile node binding update exchange",
-        "tshark output with srv6.sids for ingress SR header processing",
-        "tshark output with isakmp.spi for IKE_SA_INIT phase one",
-        "zeek dns.log entries with rcode_name, query_type, ttl for client resolver behavior",
-        "zeek conn.log entries with history field showing TCP state transitions",
-        "tshark output with quic.frame.type for ACK_ECN counting paths",
-        "tshark output with sctp.heartbeat.info for SCTP path monitoring",
-        "zeek modbus.log entries with unit_id, function for industrial control queries",
-        "tshark output with stp.bridgepriority for spanning tree root election",
-        "tshark output with srtp.encrypted_payload for SRTP-protected RTP media",
-        "tshark output with ssh.message_code for SSH transport layer key exchange",
-        "tshark output with snmp.engineid for SNMPv3 USM authentication",
-        "tshark output with l2tp.tunnel.id for L2TP control message header",
-        "zeek rdp.log entries with cookie, security, keyboard_layout for RDP session start",
-        "tshark output with cassandra.opcode for CQL native protocol startup",
-    ],
-    "hypothesis_updater": [
-        # Original 16
-        "hyp: HTTP/2 flow-control starvation; pred: WINDOW_UPDATE frames absent; obs: WINDOW_UPDATE frames present every 1MB",
-        "hyp: MTU clamping issue; pred: TCP MSS option < 1460; obs: MSS = 1460 in SYN",
-        "hyp: TLS cert chain too deep; pred: handshake size > 8KB; obs: handshake 3.2KB",
-        "hyp: SACK disabled; pred: no SACK blocks in retransmits; obs: SACK blocks present",
-        "hyp: BGP route reflector delay; pred: UPDATE arrives 30s+ after rr-client; obs: UPDATE within 200ms",
-        "hyp: kerberos pre-auth required; pred: AS-REQ rejected with KDC_ERR_PREAUTH_REQUIRED; obs: AS-REP returned directly",
-        "hyp: middlebox closes idle flows; pred: RST from non-server TTL; obs: RST from server TTL",
-        "hyp: QUIC amplification protection limit; pred: server caps initial packets to 3x client; obs: 5x observed",
-        "hyp: DNS resolver returns SERVFAIL; pred: RCODE=2 on every query; obs: RCODE=0 for some, SERVFAIL for half",
-        "hyp: TLS resumption ticket reuse; pred: psk_identity present; obs: psk_identity absent, full handshake",
-        "hyp: HTTP/3 fallback due to udp port block; pred: no QUIC initial packets observed; obs: QUIC initial present, no Handshake",
-        "hyp: ipsec rekey storm; pred: IKE INFORMATIONAL messages every 10s; obs: every 600s as configured",
-        "hyp: NFS retransmit due to storage failover; pred: cluster around failover events; obs: uniform distribution",
-        "hyp: GC pause causing 200ms gap; pred: gap aligned with sawtooth memory pattern; obs: gap not periodic",
-        "hyp: dual-stack AAAA blocked; pred: A query succeeds and AAAA times out; obs: both A and AAAA receive responses",
-        "hyp: SMB2 session expiration; pred: TREE_DISCONNECT then TREE_CONNECT after timeout; obs: continuous session reuse",
-        # Expansion +16
-        "hyp: HTTP/2 SETTINGS_INITIAL_WINDOW_SIZE too small; pred: WINDOW_UPDATE within 1KB; obs: 64KB initial window present",
-        "hyp: TCP timestamps disabled at one endpoint; pred: TS option absent in SYN; obs: TS option present both directions",
-        "hyp: TLS post-handshake auth requested; pred: CertificateRequest after Finished; obs: no post-handshake messages",
-        "hyp: BGP convergence delayed by route refresh; pred: ROUTE-REFRESH message present; obs: no refresh, gradual UPDATE",
-        "hyp: SCTP heartbeat interval too long; pred: HEARTBEAT every 30s+; obs: HEARTBEAT every 30s as RFC default",
-        "hyp: HTTP/3 0-RTT replay attack mitigation triggers; pred: server rejects with REJECT; obs: REJECT not present",
-        "hyp: MTU clamping by middlebox; pred: SYN MSS rewritten; obs: SYN MSS unchanged from origin",
-        "hyp: NFS retransmit due to slow disk; pred: rpc.duplicate calls every 5s+; obs: duplicate calls within 200ms (network)",
-        "hyp: WireGuard handshake retry due to PSK mismatch; pred: handshake initiation repeated 3x within 5s; obs: only 1 retry",
-        "hyp: TLS HelloRetryRequest needed; pred: server sends HRR; obs: server sends ServerHello directly",
-        "hyp: GRE keepalive timeout; pred: tunnel state shows expired; obs: keepalive packets ack'd every 10s",
-        "hyp: SIP REGISTER refresh failure; pred: 401 challenge with stale nonce; obs: 200 OK returned directly",
-        "hyp: QUIC congestion control collapse; pred: congestion window drops to 1 MSS; obs: cwnd stays at IW10 throughout",
-        "hyp: DTLS rekey storm; pred: ClientHello every 60s+; obs: only one ClientHello in 10 minutes",
-        "hyp: RIP route poisoning event; pred: metric=16 advertised; obs: metric=15 used, route still valid",
-        "hyp: Modbus exception response; pred: function code with high bit set; obs: normal response code returned",
-    ],
-    "stop_controller": [
-        # Original 16
-        "five hyps, all under 0.3 confidence, three uninvestigated remaining tests are cheap and informative",
-        "single hyp confirmed across 20 sampled flows with 3 contradicting flows that look like noise",
-        "two hyps still at 50/50, all remaining tests would yield observations we have already seen",
-        "investigation 14 actions in, every action moved no hyp by more than 0.05 — stuck",
-        "one hyp at 0.92, one at 0.05 — clear winner; only 2 of 50 tools used",
-        "evidence saturated: all flows captured one-sided, root cause requires server-side data we don't have",
-        "loop: 2 actions ran, both confirmed the dominant hyp; still in early phase, can dig more",
-        "investigation cost at 80% of budget, leading hyp at 0.7, other 0.2; budget low for one more test",
-        "all hyps consistent with available data; the only distinguishing test requires firewall reset which is out of scope",
-        "evidence partially supports two hyps each from different angles — possible joint cause; cannot separate",
-        "single hyp at 0.95 with 1 contradicting observation; needs explanation before stop_with_conclusion",
-        "three hyps in cluster of 0.4/0.3/0.3; no single test can distinguish all three at once",
-        "hyp confidence movements pre vs post last 4 tests are < 0.01 — diminishing returns regime",
-        "investigation running 30 min wall (vs 5 min typical); state is ambiguous; user prefers timely partial answer",
-        "evidence quality questionable: pcap shows packet capture drops > 0.5%; results unreliable for confirmation",
-        "one hyp confirmed strongly on aggregate flow stats but lacks any per-packet evidence — confirm or dig deeper?",
-        # Expansion +16
-        "four hyps with confidences 0.3/0.25/0.25/0.2 — no clear winner, only 30% of tools used",
-        "all hyps have been tested with at least 3 actions; all results in noise floor, decision required",
-        "single hyp at 0.85, but evidence comes from a single capture point that may have bias",
-        "user explicitly asks for partial answer due to time pressure; current top hyp at 0.65",
-        "cluster of 2 hyps both at 0.45 are mutually exclusive but no test would distinguish them",
-        "investigation already consumed 90% of allocated time; no clear winner; cheapest 3 tests run",
-        "three hyps explored, two confirmed by independent flows — possibly composite root cause",
-        "dominant hyp explained 12/15 observations; 3 unexplained observations cluster on one IP",
-        "all 5 hyps are alternative framings of 'TLS handshake timeout'; need to refine before stop",
-        "evidence quality high but interpretation depends on whether capture point has loss",
-        "confidence on top hyp moved 0.4→0.7 over last 5 tests; trajectory predicts more lift if continued",
-        "evidence supports complex chain of two events; can stop on the proximate cause and hand off root",
-        "dominant hyp would imply server-side bug, but we lack capability to verify in server logs",
-        "all hyps at low confidence (<0.4) but symptom is critical (production outage); need at least partial answer",
-        "one hyp at 0.9 with 1 contradicting flow that is structurally different from others — outlier or evidence?",
-        "investigation found no support for original 3 hyps; new hyp emerged at 0.55 — confirm or check baseline first",
-    ],
-}
-
-
-def all_train_task_ids() -> list[str]:
-    return list(TRAIN_SCENARIOS.keys())
-
-
-def scenarios_for(task_id: str, scenario_list: str = "train") -> list[str]:
-    """Return scenarios for a task. scenario_list='train' or 'eval'."""
-    if scenario_list == "train":
-        return TRAIN_SCENARIOS[task_id]
-    if scenario_list == "eval":
-        from .schemas import DIAG_TASKS
-        return DIAG_TASKS[task_id].seed_scenarios
-    raise ValueError(f"scenario_list must be 'train' or 'eval', got {scenario_list!r}")

vendor/nanoai/agentic/eval_loop.py

@@ -1,332 +0,0 @@
-"""Multi-turn agent eval state machine.
-
-For each task: generate → parse → execute tool → wrap result → repeat.
-Stops when assistant emits `<|assistant_end|>` without a tool call, or when
-turn budget is exceeded, or when the model emits malformed output twice.
-"""
-from __future__ import annotations
-
-import re
-import time
-from typing import Optional
-
-from .data.common import SYSTEM_PROMPT
-from .eval_sandbox import Sandbox, SandboxError
-from .eval_tasks import Task, Transcript
-
-
-# Required arg keys per tool. Original Read/Edit/Grep/Bash come from
-# scripts/agent_eval.py; the lowercase ones come from ds79c (TraceForge
-# id=90) — same trajectories used by exp307a/b/c/324. Tools that exist
-# only as no-op stubs (write_todos, task, …) accept any args.
-TOOL_REQUIRED_ARGS: dict[str, set[str]] = {
-    # Original nanoai tool surface
-    "Read": {"file_path"},
-    "Edit": {"file_path", "new_string"},
-    "Grep": {"pattern"},
-    "Bash": {"command"},
-    # ds79c/demycap tool surface (lowercase names)
-    "read_file":     {"file_path"},
-    "edit_file":     {"file_path"},      # new_string optional (write_file behaviour for empty)
-    "write_file":    {"file_path"},      # content optional, model often forgets
-    "execute":       {"command"},
-    "ls":            set(),
-    "glob":          {"pattern"},
-    "grep":          {"pattern"},
-    "save_mermaid":  {"file_path"},
-    "submit_report": {"report_markdown"},  # confidence is optional
-    "write_todos":   set(),
-    "update_todos":  set(),
-    "update_todo":   set(),
-    "task":          set(),
-}
-
-# Tools whose return value should terminate the multi-turn loop. The
-# only one today is `submit_report`; the call captures the payload on
-# the sandbox and the loop exits without sampling another turn.
-TERMINAL_TOOLS = frozenset({"submit_report"})
-
-
-def _parse_tool_call(body: str) -> tuple[str, dict[str, str]]:
-    """Parse `tool_name <|tool_arg|> k <|tool_val|> v ...` into (name, args)."""
-    parts = body.split("<|tool_arg|>")
-    name = parts[0].strip()
-    args: dict[str, str] = {}
-    for part in parts[1:]:
-        if "<|tool_val|>" in part:
-            k, v = part.split("<|tool_val|>", 1)
-            args[k.strip()] = v.strip()
-    return name, args
-
-
-def _coerce_float(s: str, default: float = 0.0) -> float:
-    try:
-        return float(s)
-    except (TypeError, ValueError):
-        return default
-
-
-def _coerce_int(s: str, default: int = 0) -> int:
-    try:
-        return int(s)
-    except (TypeError, ValueError):
-        return default
-
-
-def _execute(sb: Sandbox, name: str, args: dict[str, str]) -> tuple[str, bool]:
-    """Run a parsed tool call. Returns (result_text, ok)."""
-    required = TOOL_REQUIRED_ARGS.get(name)
-    if required is None:
-        return f"unknown tool: {name!r}", False
-    missing = required - args.keys()
-    if missing:
-        return f"missing required arg(s) for {name}: {sorted(missing)}", False
-    try:
-        # Original nanoai surface
-        if name == "Read":
-            return sb.read(args["file_path"],
-                           offset=_coerce_int(args.get("offset", "")),
-                           limit=_coerce_int(args.get("limit", ""), 50)), True
-        if name == "Edit":
-            return sb.edit(args["file_path"], args["new_string"], args.get("old_string", "")), True
-        if name == "Grep":
-            return sb.grep(args["pattern"], args.get("path") or "."), True
-        if name == "Bash":
-            return sb.bash(args["command"]), True
-        # ds79c/demycap surface
-        if name == "read_file":
-            return sb.read_file(args["file_path"],
-                                offset=_coerce_int(args.get("offset", "")),
-                                limit=_coerce_int(args.get("limit", ""), 50)), True
-        if name == "edit_file":
-            return sb.edit_file(args["file_path"],
-                                new_string=args.get("new_string", ""),
-                                old_string=args.get("old_string", "")), True
-        if name == "write_file":
-            return sb.write_file(args["file_path"], args.get("content", "")), True
-        if name == "execute":
-            return sb.execute(args["command"]), True
-        if name == "ls":
-            return sb.ls_path(args.get("path", ".")), True
-        if name == "glob":
-            return sb.glob(args.get("pattern", "*"), args.get("path", ".")), True
-        if name == "grep":
-            return sb.grep(args["pattern"], args.get("path") or "."), True
-        if name == "save_mermaid":
-            return sb.save_mermaid(args["file_path"], args.get("content", "")), True
-        if name == "submit_report":
-            return sb.submit_report(
-                report_markdown=args.get("report_markdown", ""),
-                confidence=_coerce_float(args.get("confidence", "")),
-            ), True
-        if name == "write_todos":
-            return sb.write_todos(), True
-        if name == "update_todos":
-            return sb.update_todos(), True
-        if name == "update_todo":
-            return sb.update_todo(**{k: v for k, v in args.items()}), True
-        if name == "task":
-            return sb.task(description=args.get("description", "")), True
-    except SandboxError as e:
-        return f"tool error: {e}", False
-    except Exception as e:
-        return f"tool crash: {type(e).__name__}: {e}", False
-    return "internal: unreachable", False
-
-
-def _build_initial_tokens(tokenizer, user_text: str, system_prompt: Optional[str] = None) -> list[int]:
-    # Adapter path (e.g. QwenTokenizer) — delegate to tokenizer's own chat template.
-    if hasattr(tokenizer, "build_initial_tokens"):
-        sp = SYSTEM_PROMPT if system_prompt is None else system_prompt
-        return tokenizer.build_initial_tokens(user_text, system_prompt=sp)
-    bos = tokenizer.get_bos_token_id()
-    user_start = tokenizer.encode_special("<|user_start|>")
-    user_end = tokenizer.encode_special("<|user_end|>")
-    asst_start = tokenizer.encode_special("<|assistant_start|>")
-    sp = SYSTEM_PROMPT if system_prompt is None else system_prompt
-    body = sp + "\n\n" + user_text
-    return [bos, user_start] + tokenizer.encode(body) + [user_end, asst_start]
-
-
-def _append_tool_result(tokenizer, tokens: list[int], result_text: str) -> None:
-    """Append `<|tool_result_start|>{result}<|tool_result_end|><|assistant_start|>`."""
-    if hasattr(tokenizer, "append_tool_result"):
-        tokenizer.append_tool_result(tokens, result_text)
-        return
-    tokens.append(tokenizer.encode_special("<|tool_result_start|>"))
-    # Truncate result to keep KV cache usage bounded.
-    if len(result_text) > 4000:
-        result_text = result_text[:4000] + "\n...[truncated]"
-    tokens.extend(tokenizer.encode(result_text))
-    tokens.append(tokenizer.encode_special("<|tool_result_end|>"))
-    tokens.append(tokenizer.encode_special("<|assistant_start|>"))
-
-
-def _assistant_end_id(tokenizer):
-    if hasattr(tokenizer, "assistant_end_id"):
-        return tokenizer.assistant_end_id()
-    return tokenizer.encode_special("<|assistant_end|>")
-
-
-def _tool_call_pair(tokenizer):
-    if hasattr(tokenizer, "tool_call_token_pair"):
-        return tokenizer.tool_call_token_pair()
-    return (
-        tokenizer.encode_special("<|tool_call_start|>"),
-        tokenizer.encode_special("<|tool_call_end|>"),
-    )
-
-
-def _parse_tool_call_dispatch(tokenizer, body_text: str):
-    """Dispatch to tokenizer-specific body parser (Qwen JSON vs nanoai k/v)."""
-    if hasattr(tokenizer, "parse_tool_body"):
-        return tokenizer.parse_tool_body(body_text)
-    return _parse_tool_call(body_text)
-
-
-def _generate_one_turn(engine, tokenizer, prompt_tokens: list[int],
-                        max_tokens: int, temperature: float, seed: int,
-                        capture_logprobs: bool = False):
-    """Run engine.generate until <|assistant_end|> or max_tokens.
-
-    Returns the list of generated token ids (excluding the prompt). When
-    capture_logprobs=True, returns (out_ids, out_logprobs) where out_logprobs[i]
-    is the behavior-policy log prob of out_ids[i] (None for any forced token) —
-    used to store behavior logprobs for TIS (Polar mechanism)."""
-    asst_end = _assistant_end_id(tokenizer)
-    bos = tokenizer.get_bos_token_id()
-    out_ids: list[int] = []
-    # Only pass return_logprobs when capturing, so engines/fakes that don't
-    # accept the kwarg keep working unchanged (backward compatible).
-    if capture_logprobs:
-        out_logprobs: list = []
-        gen = engine.generate(prompt_tokens, num_samples=1, max_tokens=max_tokens,
-                              temperature=temperature, seed=seed, return_logprobs=True)
-        for token_column, _masks, logprob_column in gen:
-            out_logprobs.append(logprob_column[0])
-            tok = token_column[0]
-            out_ids.append(tok)
-            if tok == asst_end or tok == bos:
-                break
-        return out_ids, out_logprobs
-    for token_column, _masks in engine.generate(
-        prompt_tokens,
-        num_samples=1,
-        max_tokens=max_tokens,
-        temperature=temperature,
-        seed=seed,
-    ):
-        tok = token_column[0]
-        out_ids.append(tok)
-        if tok == asst_end or tok == bos:
-            break
-    return out_ids
-
-
-def run_task(
-    engine,
-    tokenizer,
-    sandbox: Sandbox,
-    task: Task,
-    *,
-    max_tokens_per_turn: int = 256,
-    max_turns: Optional[int] = None,
-    temperature: float = 0.5,
-    seed: int = 42,
-    return_transcript: bool = False,
-) -> dict:
-    """Run one task end-to-end. Returns a dict with pass/fail + diagnostics.
-
-    When return_transcript=True the dict also includes:
-      - 'user_prompt': str
-      - 'transcript': list[dict] (every assistant content + tool_call(name+args) +
-        tool_result text + error flag, in order)
-    """
-    max_turns = max_turns or task.max_turns
-    transcript = Transcript(user_prompt=task.prompt)
-    task.setup(sandbox)
-
-    asst_end = _assistant_end_id(tokenizer)
-    tc_start, tc_end = _tool_call_pair(tokenizer)
-
-    tokens = _build_initial_tokens(tokenizer, task.prompt,
-                                    system_prompt=getattr(task, "system_prompt", None))
-    n_turns = 0
-    n_tool_errors = 0
-    t0 = time.time()
-    truncated = False
-
-    while n_turns < max_turns:
-        n_turns += 1
-        out_ids = _generate_one_turn(engine, tokenizer, tokens,
-                                      max_tokens_per_turn, temperature, seed + n_turns)
-        # Trim trailing assistant_end if present.
-        ended_clean = bool(out_ids) and out_ids[-1] == asst_end
-        body_ids = out_ids[:-1] if ended_clean else out_ids
-        if not ended_clean and len(out_ids) >= max_tokens_per_turn:
-            truncated = True
-
-        # Inspect for tool call.
-        if tc_start in body_ids and tc_end in body_ids:
-            i = body_ids.index(tc_start)
-            try:
-                j = body_ids.index(tc_end, i + 1)
-            except ValueError:
-                j = -1
-            if j > i:
-                pre = tokenizer.decode(body_ids[:i]).strip()
-                tool_body = tokenizer.decode(body_ids[i + 1 : j])
-                try:
-                    name, args = _parse_tool_call_dispatch(tokenizer, tool_body)
-                except ValueError as e:
-                    # Malformed tool body — surface as a tool error, no execute.
-                    name, args = "<parse_error>", {}
-                    result_text, ok = f"tool parse error: {e}", False
-                else:
-                    result_text, ok = _execute(sandbox, name, args)
-                transcript.turns.append({
-                    "role": "assistant",
-                    "content": pre,
-                    "tool_call": {"name": name, "args": args},
-                })
-                transcript.turns.append({
-                    "role": "tool_result",
-                    "tool_result": result_text,
-                    "error": not ok,
-                })
-                if not ok:
-                    n_tool_errors += 1
-                # Terminating tool (submit_report) → exit loop; no need
-                # to extend context for another turn.
-                if name in TERMINAL_TOOLS:
-                    break
-                # Extend context with the assistant turn we just generated +
-                # tool result + a fresh assistant_start.
-                tokens.extend(out_ids)  # includes asst_end if present
-                if not ended_clean:
-                    tokens.append(asst_end)
-                _append_tool_result(tokenizer, tokens, result_text)
-                continue
-
-        # No tool call this turn → final answer.
-        text = tokenizer.decode(body_ids).strip()
-        transcript.turns.append({"role": "assistant", "content": text})
-        break
-
-    elapsed = time.time() - t0
-    passed, reason = task.check(sandbox, transcript)
-    out = {
-        "task_id": task.task_id,
-        "passed": passed,
-        "reason": reason,
-        "n_turns": n_turns,
-        "n_tool_errors": n_tool_errors,
-        "truncated": truncated,
-        "elapsed_s": round(elapsed, 2),
-        "tools_used": [c.get("name", "") for c in transcript.tool_calls],
-        "last_text": transcript.last_assistant_text[:200],
-    }
-    if return_transcript:
-        out["user_prompt"] = transcript.user_prompt
-        out["transcript"] = transcript.turns
-    return out