diff --git a/BUILD.json b/BUILD.json new file mode 100644 index 0000000000000000000000000000000000000000..28ec78a84bd4001369d37b61b60a69f6c4e9a49a --- /dev/null +++ b/BUILD.json @@ -0,0 +1,7 @@ +{ + "built_at": "2026-10-26T22:04:33", + "commit": "6bc868a", + "files": 48, + "with_store": false, + "with_card": true +} \ No newline at end of file diff --git a/README.md b/README.md index 50cf8734a66eeec37bf3645f25ad0ccce8be504d..2e80d4291decf3d88ed286a54134d4e3b9b51f94 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,122 @@ --- -title: Traceforge -emoji: 😻 -colorFrom: gray -colorTo: purple +title: traceforge +emoji: 🧭 +colorFrom: indigo +colorTo: green sdk: gradio sdk_version: 6.30.0 -python_version: '3.13' app_file: app.py pinned: false +license: other +short_description: Turn real agent traces into ideal-world references --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# traceforge + +Turn a real agent trace into its **ideal-world reference**: the same work, +with the dead ends removed, gated for structure and grounding. + +This Space is the portable face of the traceforge pipeline. The pipeline +itself is unchanged β€” triage labels every message, a windowed rewriter +produces the ideal path, and a judge admits only traces that pass the six +pillars. The UI hides that machinery behind three actions. + +## Submit a trace + +Upload a JSON/JSONL trace (traceforge native, OpenAI chat, or a bare message +list) or paste it. The app computes a **content hash** of the seed and +queues it. + +**One seed, one output.** The hash covers the conversation content β€” message +text, tool calls and the set of tools the agent had β€” and excludes session +ids, timestamps, cwd and tool-call ids. Submitting the same trace twice +returns the existing output instead of creating a second one. That is what +makes the output dataset a set, not a log. + +## Sponsor a dataset + +Upload a JSONL of traces. Every row is hashed; new seeds are queued and +attributed to the sponsor name you give, known seeds are skipped. Processed +rows join the public output dataset. The dashboard shows the sponsor +leaderboard. + +## Dashboard + +Public statistics over this Space's corpus and, when bundled, the reference +corpus: seeds, outputs, queue, failure rate, task mix, languages, runtime +distribution, and **time sensitivity**. + +## Time sensitivity + +A trace whose correct answer is a fact that changes over time is a bad +training target when the agent had no way to look that fact up. Train on it +and the model learns to *assert* the fact from its weights β€” "the current +president is X" β€” which is wrong the moment X changes. + +The same question is a good target when the agent had a retrieval tool. The +ideal path calls the tool and answers from its result, so what the model +imitates is the procedure (search, read, answer), not the fact. The +procedure does not decay. + +| class | meaning | +|---|---| +| `time_sensitive` | temporal fact, no retrieval tool β€” the answer decays | +| `time_anchored` | temporal fact, but a retrieval tool can fetch it | +| `not_time_sensitive` | no time-dependent fact in the task | + +`time_anchored` carries a sub-flag: whether the tool was actually **called**. +A temporal task with a search tool available but never used is anchored in +principle β€” the ideal rewrite should make the agent call it β€” but the raw +trace answered from weights, so it needs a rewrite rather than a pass. + +The detector is deliberately conservative: a false `time_sensitive` throws +away a good trace, so a pattern must name the **fact** (an officeholder, a +price, a released version, the weather), not just a temporal word. "The +current branch" and "the latest fix round" are local artifacts of the +session, not decaying facts. + +## Configuration + +Set these as Space secrets / variables: + +| variable | meaning | +|---|---| +| `TRACEFORGE_BASE_URL` | OpenAI-compatible endpoint (`…/v1`) | +| `TRACEFORGE_LLM_KEY` | bearer token | +| `TRACEFORGE_MODEL` | writer/judge model id | +| `TRACEFORGE_TRIAGE_MODEL` | cheap labeler (default: `TRACEFORGE_MODEL`) | +| `TRACEFORGE_JUDGE_MODEL` | judge override (default: `TRACEFORGE_MODEL`) | +| `TRACEFORGE_DATA_DIR` | store root (default `./data/space`) | +| `TRACEFORGE_SPACE_WORKERS` | parallel pipeline workers (default 1) | + +Without an endpoint the app still starts and the dashboard still works; the +Submit tab reports that the pipeline is not configured instead of hanging. + +## Storage + +The store is content-addressed on disk: + +``` +data/space/index.jsonl append-only journal (last row per hash wins) +data/space/seeds/.json the seed exactly as submitted +data/space/outputs/.json the improved trace +``` + +A Hugging Face Space's disk is ephemeral unless persistent storage is +attached. For a durable public corpus, mount persistent storage at +`/data` and set `TRACEFORGE_DATA_DIR=/data/space`, or sync the store to a +dataset repo. + +## Local use + +```bash +pip install -e ".[webui]" +python app.py # http://127.0.0.1:7860 +traceforge space serve --port 7860 +traceforge space hash trace.json # print the seed hash +traceforge space stats # store census +``` + +## License + +Private research corpus. All rights reserved. diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d3bfc32f691b54a82c6d606b3033323f20b07bc8 --- /dev/null +++ b/app.py @@ -0,0 +1,24 @@ +"""Hugging Face Spaces entry point. + +The Space runs this file. Everything it needs is in `traceforge/space/`; +configuration comes from Space secrets / variables: + + TRACEFORGE_BASE_URL OpenAI-compatible endpoint (…/v1) + TRACEFORGE_LLM_KEY bearer token + TRACEFORGE_MODEL writer/judge model id + TRACEFORGE_TRIAGE_MODEL cheap labeler (default: TRACEFORGE_MODEL) + TRACEFORGE_DATA_DIR store root (default: ./data/space) + +Locally: `python app.py` (or `traceforge space serve`). +""" +from __future__ import annotations + +import os + +from traceforge.space.app import CSS, build_app + +demo = build_app() +demo.queue(default_concurrency_limit=4) + +if __name__ == "__main__" or os.environ.get("SPACE_ID"): + demo.launch(show_error=True, css=CSS) diff --git a/data/dataset/dataset_card.json b/data/dataset/dataset_card.json new file mode 100644 index 0000000000000000000000000000000000000000..18fcb19fa5c40ac21e696c74d4b86216c33cdb03 --- /dev/null +++ b/data/dataset/dataset_card.json @@ -0,0 +1,167 @@ +{ + "traces": 3652, + "dedup": { + "removed": 1, + "pct": 0.0, + "key": "(task, tool-names + sha1(args))" + }, + "languages": { + "en": 3444, + "de": 208 + }, + "agent_roles": { + "subagent": 278, + "orchestrator": 11 + }, + "tasks": { + "review": 1118, + "debug": 682, + "implement": 620, + "research": 468, + "other": 293, + "test": 165, + "ops": 108, + "data-analysis": 86, + "creative": 54, + "refactor": 44, + "office-docs": 14 + }, + "messages": { + "total": 112327, + "by_role": { + "assistant": 54700, + "tool": 53265, + "user": 4070, + "system": 292 + }, + "per_trace": { + "mean": 30.8, + "median": 20, + "p90": 64, + "max": 252 + } + }, + "tool_calls": { + "total": 53265, + "per_trace": { + "mean": 14.6, + "median": 9, + "p90": 32, + "max": 125 + }, + "traces_with_none": 1, + "distinct_tools": 553, + "top_tools": { + "Bash": 14327, + "exec": 13084, + "terminal": 3813, + "read_file": 3008, + "Read": 2085, + "search_files": 1837, + "execute_code": 1601, + "tool_call": 1512, + "Edit": 1497, + "skill_view": 918, + "Grep": 917, + "patch": 758, + "unlock_discoverable_agent_tool": 457, + "call_discoverable_agent_tool": 457, + "wait": 436 + } + }, + "user_turns": { + "per_trace": { + "mean": 1.1, + "median": 1, + "p90": 1, + "max": 18 + }, + "single_turn_traces": 3352, + "multi_turn_traces": 300 + }, + "tokens": { + "method": "chars/4 (prose proxy). Calibrated against a real tokenizer on this corpus: chars/4 runs ~13% low (median 21.4k est vs 24.5k actual), so scale by ~1.15 for a central estimate; tool output is code/JSON and tokenizes denser than prose.", + "budget": 131072, + "chars_total": 323084312, + "est_total": 80769721, + "per_trace": { + "mean": 22116.6, + "median": 16996, + "p90": 53126, + "max": 119767 + }, + "fits_budget": { + "4096": { + "traces": 1043, + "pct": 28.6 + }, + "8192": { + "traces": 1218, + "pct": 33.4 + }, + "16384": { + "traces": 1773, + "pct": 48.5 + }, + "65536": { + "traces": 3570, + "pct": 97.8 + }, + "131072": { + "traces": 3652, + "pct": 100.0 + } + } + }, + "split": null, + "supervision": { + "contract": "loss on assistant spans only; tool/user/system spans are masked context", + "chars_by_role": { + "user": 4777684, + "assistant": 41191827, + "tool": 276324543, + "system": 790258 + }, + "assistant_share_pct": 12.7, + "tool_share_pct": 85.5, + "recovery": { + "traces": 47, + "pct": 1.3, + "cap_pct": 15.0, + "over_cap": false, + "method": "verified label: exactly one environment-caused failure (flaky endpoint, missing dependency, exhausted resource) followed by exactly one corrective step; model-caused errors and retry loops disqualify (docs/DATA_QUALITY.md Β§4.1)" + } + }, + "difficulty": { + "tool_calls": { + "mean": 14.6, + "median": 9, + "p90": 32, + "max": 125 + }, + "messages": { + "mean": 30.8, + "median": 20, + "p90": 64, + "max": 252 + }, + "est_tokens": { + "mean": 22116.6, + "median": 16996, + "p90": 53126, + "max": 119767 + }, + "bands": { + "calls_lt_10": 1916, + "calls_10_24": 1149, + "calls_25_49": 450, + "calls_ge_50": 137 + } + }, + "notes": [ + "identity/refusal burns gated (traceforge/identity.py)", + "clip markers, template leaks, permission-ask finals gated", + "residual-defect estimate ~5% (selfhosted cross-exam, triaged)", + "reasoning channel dropped per message (keep_reasoning=False)" + ] +} \ No newline at end of file diff --git a/data/space/reference_census.json b/data/space/reference_census.json new file mode 100644 index 0000000000000000000000000000000000000000..25a7e3e12170e32b27f717df3973b4dda0c739ec --- /dev/null +++ b/data/space/reference_census.json @@ -0,0 +1,23 @@ +{ + "key": { + "mtime": 1793044902, + "size": 406218197 + }, + "census": { + "total": 3652, + "classes": { + "time_sensitive": 13, + "time_anchored": 12, + "not_time_sensitive": 3627 + }, + "labels": { + "time_sensitive": "answer decays: temporal fact, no retrieval tool", + "time_anchored": "temporal fact, but a retrieval tool can fetch it", + "not_time_sensitive": "no time-dependent fact in the task" + }, + "needs_rewrite": 0, + "time_sensitive_share": 0.0036, + "source": "data/dataset/train.jsonl", + "available": true + } +} \ No newline at end of file diff --git a/guidelines/IDEAL_TRACE_GUIDELINES.md b/guidelines/IDEAL_TRACE_GUIDELINES.md new file mode 100644 index 0000000000000000000000000000000000000000..d03b8cb1c9ec8b7e0fc053e90412612fec68db3c --- /dev/null +++ b/guidelines/IDEAL_TRACE_GUIDELINES.md @@ -0,0 +1,230 @@ +# Ideal-Trace Guidelines β€” v0.3 + +> **Authoritative spec for what an "ideal trace" looks like.** Read at runtime +> by the improver, digest, long-trace and review stages. The six quality +> pillars (Β§3) are owner-decided and binding. Β§4 states the training-signal +> contract that makes the pillars necessary; Β§6 scopes what this reference is +> for (evaluation, not training). Evidence base: `docs/DATA_QUALITY.md`. +> +> **Integrity rule:** this file is a spec, not a notebook. Keep it under +> 24 KB. Never paste a worked example, a trace, or your own reasoning into it +> β€” examples live in `data/traces/improved/`. `tests/test_guidelines.py` +> enforces this. + +## 1. Purpose + +An ideal trace is the **ideal-world execution of the same task**: same intent, +same real environment, flawless execution. It is NOT a re-execution β€” it is a +rewrite of the recorded trace, using the trace's own tool outputs as the only +evidence about the world. + +The ideal trace is the **reference standard** the benchmark scores a model +against: the less the pipeline has to edit a model's trace to reach it, the +better the model. The training-signal contract in Β§4 is retained because the +reference must also be a valid target if the corpus is reused for SFT β€” but +the official scope is evaluation, not training. + +## 2. Stable structural rules (hard requirements) + +1. **Preserve the task intent.** The first user message carries the task. The + ideal trace solves *that* task. When the source trace starts with a handoff + artifact β€” a Hermes `[CONTEXT COMPACTION …]` row, or a Claude Code "This + session is being continued…" summary row β€” the first REAL user task message + defines the intent: reproduce it verbatim and byte-exact as the ideal + trace's opening message. +2. **Tool-call pairing.** Every assistant tool_call has exactly one matching + tool result; every tool result references an existing call; tool arguments + are valid JSON strings. +3. **Message shape.** user / assistant(+calls) / tool / … / final assistant. + The last message is the assistant's final answer. No two consecutive user + rows, no two consecutive text-only assistant rows. +4. **Metadata is set by the harness**, not by the improver. +5. **No scaffold leakage.** Never copy prompt scaffolding, clip markers + (`…[+N chars clipped]`), compaction banners, or harness self-talk into the + ideal trace. + +## 3. The six quality pillars (owner-decided, binding) + +**P1 β€” Efficient but deep reasoning.** +Reasoning is sharp and high-signal: the key decision logic per turn (why this +call, what the result implies, what to verify next). No filler, no restating +the obvious, no rambling. Depth where it matters (diagnoses, trade-offs, +verification logic) β€” silence where it doesn't. **Answer focus:** every +answer, final or mid-trace, gives exactly what its question asked for. A +follow-up gets only the new piece; never re-present facts the user already +received. Over-answering is a P1 violation. + +**Deliver the substance, not a description of it.** The final answer must +present the concrete result the task asked for β€” the actual numbers, names, +paths, values, or code from the tool evidence β€” not a summary of what was +retrieved. "I pulled the nutrition data and the recipe" is poor; the data and +the recipe ARE the answer. Naming the work instead of showing its result fails +the task even when every tool call was correct. + +**P2 β€” No committed mistakes (mine the negative signal first).** +Before writing, identify every mistake in the real trace: failed tool calls +(error results), retry loops, redundant calls, wrong turns that later needed +correction, and USER FEEDBACK (corrections, re-asks, complaints). The ideal +trace commits none of them: the correct route is taken the first time. Where +the real agent repaired itself, the ideal trace shows only the correct action +β€” not the failure, not the repair. +*Why:* the assistant turns are the loss-bearing target. A mistake in the +target is imitated; a mistake in the context is not. See Β§4 for the one +narrow exception (environment-caused failures). + +**P3 β€” Perfect tool calls, only when really needed.** +Every call is necessary, correctly parameterized, and the natural next step. +Independent calls are batched in parallel. + +**Batching is structural, not stylistic.** When the real trace issues two +calls in separate assistant turns but the second does not consume the first's +output, the ideal path issues them in ONE assistant turn carrying both +`tool_calls`, with the two tool results following in the same order. Do not +preserve the source's one-call-per-turn rhythm β€” that rhythm is the defect. +Measured 2026-10-25: 75 of 621 repair-exhausted failures are exactly this +shape. A duplicate call (same tool, same arguments, same turn) is never +batched; it is dropped. No exploratory calls, no re-reads +of what is already in context, no call whose result the narrative doesn't +use. When the ideal path makes the SAME call as the real trace, the tool +result must be the real trace's output β€” verbatim or a faithful compression +of it β€” never a plausible re-simulation of what the call might have printed. + +**P4 β€” Fact-driven, no hallucinations (two layers, both binding).** +Layer 1 (trace-internal): reuse the real trace's tool outputs verbatim +wherever the ideal path makes the same call; never invent files, commands, or +results; no fake success β€” the final answer is grounded in real evidence from +the trace. Layer 2 (external world): statements about the outside world +(library versions, API behavior, dates, technical facts) must be either (a) +taken from the trace's own evidence or (b) kept out unless you are certain +they are correct. The harness fact-checks external claims with live web +searches after you finish β€” unsupported claims send the trace back. +**Construction order:** decide the ideal call path β†’ copy each result +byte-verbatim from `outputs.jsonl` β†’ only then write reasoning and the final +answer around those real results. A tool result you cannot quote is not +writable: reroute the path or drop the claim. Stitching individually-plausible +fragments is fabrication and is the single most common rejection. + +**P5 β€” Perfect grammar and language quality, single language.** +One language per trace: the language of the original first user message. No +code-switching, no stray fragments in other languages. Assistant prose is +clean, grammatically perfect, professionally concise. The user message is kept +verbatim β€” realism of the input matters more than its typos. Never open by +restating the brief: the first assistant turn must do work (a tool call, a +decision, a finding), not echo the user's own text back at them. + +**P6 β€” No refusals.** +The ideal agent completes the task without refusing. If the source task +genuinely cannot be completed (harmful or impossible), do NOT write an ideal +version: reply starting with `REJECTED:` plus a one-line reason, and write no +output file. + +## 4. Training-signal contract (why the pillars are shaped this way) + +The corpus is consumed by SFT: loss is computed on **assistant target tokens +only**; tool results, user turns and system prompts are masked context. The +model imitates what is in the target span. + +> **Failures may be context. Failures must never be target.** + +Therefore: + +1. **A model-caused error must never appear in an assistant turn** β€” not as + "I'll try X", not as a failed call, not as a self-correction narrative. + Wrong tool name, wrong arguments, wrong path, bad reasoning: route around + it. This is P2. +2. **An environment-caused error may appear as a tool result** (context), with + the correct adaptation in the next assistant turn. This is the only + exception to P2, and it exists because a corpus of only first-try-clean + paths loses the only examples of "tool returned an error β†’ read it β†’ + adapt", and silently skews the corpus toward easy tasks. +3. **The recovery is the target, never the mistake.** If the ideal path cannot + avoid the error, the assistant's next turn is the supervised recovery; the + error text stays byte-verbatim in the tool row. + +### 4.1 The narrow recovery carve-out + +A trace may keep a real failure **only** when ALL of these hold: + +- the failure is **environment-caused** β€” a tool returned an error the ideal + agent could not have foreseen (flaky endpoint, missing dependency, a + genuinely absent file). Not a wrong tool name, wrong arguments, wrong path, + or a reasoning error; +- the error text is **byte-verbatim** from `outputs.jsonl` β€” never composed, + never paraphrased, never invented; +- there is **one corrective step per error**; two or more consecutive + identical failed calls disqualify the trace; +- the harness records the trace as a **retained recovery** (manifest + dataset + card) so the corpus can report failure density and a trainer can filter it + out; +- the final state is **verified** β€” a clean-looking recovery whose final + answer is wrong is rejected outright. + +Everything else follows P2: route around the mistake, show only the correct +action. When in doubt, route around it β€” the carve-out is an exception, not a +license. + +### 4.2 What must never be in the target span + +- A fabricated or paraphrased error message. +- A failure the ideal agent could have avoided by reading its own context. +- A "let me try again" narrative whose first attempt is not shown (the model + learns the shape of recovery without the substance). +- A recovery loop longer than one corrective step. +- A success claim that the tool results do not support. + +## 5. Rewrite procedure (binding) + +1. Read these guidelines in full. +2. Mine the negative signal: list the real trace's failed calls, retries, + wrong turns, and user feedback (P2). +3. Classify each failure: **harness artefact** (wrong cwd, missing env var, + stale workdir, tool not installed) β†’ route around it; **environment-caused + and unavoidable** β†’ eligible for the Β§4.1 carve-out; **model-caused** β†’ + route around it, always. +4. Map the ideal path: the minimal set of correct, necessary calls (P3). +5. Copy every tool result on that path byte-verbatim from `outputs.jsonl` + (P4), then write the reasoning and final answer around them (P1, P5). +6. Write the ideal trace JSON (schema in the task prompt). +7. Verify: re-read the file; check JSON validity, call/result pairing, every + pillar, and that no failure sits in an assistant turn. Fix, then summarize + the changes. + +## 6. Scope: what this corpus is and is not + +This is an **evaluation reference**. It is not a training corpus by official +scope, not a preference corpus, and not an RL environment. + +- **The benchmark** scores a model by how little the pipeline has to edit its + traces to reach the reference. The reference is the ruler, not the product. +- **SFT reuse** is possible β€” the Β§4 contract keeps the reference a valid + target β€” but it is a downstream decision, not this repo's purpose. +- **KTO/DPO** needs rejected samples from the policy being aligned, + length-matched to the chosen side, with a bounded negative loss. A + hand-written "bad" trace is a strawman and teaches length/style, not + preference. +- **PPO/GRPO** needs rollouts and a verifiable reward; failed rollouts are + advantage, not imitation. A static JSONL of ideal traces cannot supply that + signal. +- If those corpora are built later, they are separate artifacts with separate + rules. Do not repurpose this corpus by relabelling its rows. + +Corpus-level obligations that belong to the harness, not the improver: report +failure density and difficulty distribution per batch; cap retained recoveries +at ≀15% of traces until an in-repo ablation says otherwise; state the masking +contract in the dataset card; dedup at the (task, tool-sequence) level. + +## 7. Remaining open items + +- System prompt: keep original vs. normalize to a canonical corpus prompt. +- Corpus-level dedup threshold for batch runs (Jaccard > 0.9 is the current + audit default). +- Whether the Β§4.1 carve-out improves or degrades held-out recovery rate β€” + needs an in-repo ablation (same traces, with and without retained + recoveries, scored on a forced-error recovery eval). + +## 8. Provenance + +Improved traces carry source session id, parent session id, and generator +version; the manifest records per-trace status, attempts, and errors. The +harness records retained recoveries (Β§4.1) in the manifest and reports failure +density in the dataset card. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9693d2f5fd3f543d849e2d48964b29cdec9da73 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +gradio>=6.0 +pyyaml>=6 +huggingface_hub>=0.24 diff --git a/traceforge/__init__.py b/traceforge/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..09fb817d219ed42042a1522806670938333c5c66 --- /dev/null +++ b/traceforge/__init__.py @@ -0,0 +1,16 @@ +"""traceforge β€” an evaluation harness for agent models. + +Reads real agent traces (Hermes Agent, Claude Code, Codex), then uses the +Hermes Agent CLI itself as the improver harness to rewrite them into +ideal-world references, validated against the guidelines in +guidelines/IDEAL_TRACE_GUIDELINES.md. The benchmark scores a model by how +little that rewrite has to change: if the pipeline has nothing to do, the +model is already perfect. +""" + +__version__ = "0.1.0" + +from traceforge.config import Config +from traceforge.trace import Message, ToolCall, Trace + +__all__ = ["Config", "Message", "ToolCall", "Trace", "__version__"] \ No newline at end of file diff --git a/traceforge/__pycache__/__init__.cpython-314.pyc b/traceforge/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8eb01a1d33e229a9a5bdf6a2eee20c0e5585c21 Binary files /dev/null and b/traceforge/__pycache__/__init__.cpython-314.pyc differ diff --git a/traceforge/__pycache__/config.cpython-314.pyc b/traceforge/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9e9fa6e46165eae76218aefab90a91de9e993d0 Binary files /dev/null and b/traceforge/__pycache__/config.cpython-314.pyc differ diff --git a/traceforge/__pycache__/trace.cpython-314.pyc b/traceforge/__pycache__/trace.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d66d1f54c4fc191742ac1b0b666c8bbc977b03b Binary files /dev/null and b/traceforge/__pycache__/trace.cpython-314.pyc differ diff --git a/traceforge/audit.py b/traceforge/audit.py new file mode 100644 index 0000000000000000000000000000000000000000..05bacc52f833ae2d2b3422cbe99a52fcb49e2959 --- /dev/null +++ b/traceforge/audit.py @@ -0,0 +1,272 @@ +"""Corpus audit: mechanical, LLM-free quality gates over the PROMOTED corpus. + +Supervisor tool β€” run before any training export. Zero LLM calls; every +finding is a mechanical pattern with a trace + index citation, so it can +drive repairs or corpus pruning deterministically. + +Audit classes: + 1. harness self-talk β€” compaction/continuation narrative, spillover + slices, retry language in ASSISTANT prose (raw-session plumbing an + ideal trace must never show a model saying) + 2. clip markers β€” the static gate, re-checked corpus-wide + 3. empty/degenerate assistant turns β€” prose turns with < 20 chars + that are not a tool-call carrier + 4. judge/quality residue β€” "defect": false entries that leaked into + promoted artifacts (should be impossible; proves a gate bypass) + 5. near-duplicates β€” normalized assistant-prose shingles > 0.9 + Jaccard across traces (overfit risk for the trainee) +""" +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections import Counter + +# ---------------------------------------------------------------- patterns + +# Harness/plumbing noise in ASSISTANT prose. Deliberately narrower than +# the raw-trace patterns: words like "harness"/"retry" are legitimate in +# domain tasks (test harness, git retry), so we anchor on unambiguous +# self-referential plumbing. +SELF_TALK_RE = re.compile( + r"(" + r"[Cc]ontext was compacted" + r"|\[\s*CONTEXT COMPACTION\s*\]" + r"|βŸͺHERMES-CONTEXT-COMPRESSION" + r"|spillover slice" + r"|\bcompacted mid-" + r"|\bmid context-compression\b" + r"|\bcontinuation handoff\b" + r"|CONTEXT-COMPRESSION:" + r")" +) + +# A task ABOUT harness/crash/database repair legitimately discusses +# compaction state β€” that is domain content, not self-talk leakage. +DOMAIN_TASK_RE = re.compile( + r"crash|repair|database|hermes|corrupt|restore|recover|diagnos", + re.IGNORECASE, +) + +CLIP_RE = re.compile(r"…?\s*\[\+[\d,.]+ ?(?:chars|bytes) clipped\]") + +# same-family judge leak marker: judge entries surviving into artifacts +JUDGE_LEAK_RE = re.compile(r'"defect"\s*:\s*false|"answer_quality"', re.I) + +# identity burns / refusal phrasings β€” must never reach the reference +from traceforge.identity import IDENTITY_RE, REFUSAL_RE # noqa: E402 +from traceforge.splice import _task_demands_json # noqa: E402 + +# a final answer that IS a raw JSON dump (not fenced, parses as JSON) +# teaches the trainee to close sessions with machine output +def _final_is_blob(msgs: list) -> bool: + if not msgs: + return False + last = msgs[-1] + if last.get("role") != "assistant": + return False + c = (last.get("content") or "").strip() + if len(c) < 400 or c.startswith("```"): + return False + if not (c.startswith("{") or c.startswith("[")): + return False + try: + json.loads(c) + return True + except json.JSONDecodeError: + return False + + +def _shingle(text: str, k: int = 8) -> set[str]: + """Word k-gram hashes for near-duplicate detection.""" + words = re.findall(r"\w+", text.lower()) + if len(words) < k: + return set() + return {hashlib.md5( + " ".join(words[i:i + k]).encode()).hexdigest() + for i in range(len(words) - k + 1)} + + +def _jaccard(a: set, b: set) -> float: + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + +def audit_trace(t: dict) -> list[dict]: + """One promoted trace -> list of findings.""" + findings = [] + sid = t.get("parent_session_id") or t.get("session_id", "?") + msgs = t.get("messages", []) + # opening user task (skip compaction/continuation handoffs) + task = next((m.get("content") or "" for m in msgs + if m.get("role") == "user" + and not (m.get("content") or "").startswith( + ("[CONTEXT", "βŸͺ"))), "") + task_domain = bool(DOMAIN_TASK_RE.search(task)) + if _final_is_blob(msgs) and not _task_demands_json(task): + findings.append({ + "cls": "final_blob", "i": len(msgs) - 1, + "detail": "final answer is a raw JSON dump, not prose", + "ctx": (msgs[-1].get("content") or "")[:160].replace("\n", " "), + }) + for i, m in enumerate(msgs): + role = m.get("role") + c = m.get("content") or "" + if role == "assistant": + mo = SELF_TALK_RE.search(c) + if mo: + findings.append({ + "cls": "self_talk_warn" if task_domain else "self_talk", + "i": i, + "detail": mo.group(0)[:80], + "ctx": c[max(0, mo.start() - 30): + mo.end() + 50].replace("\n", " ")[:160], + }) + # clip markers must not survive in ANY role's content (tool + # rows carry them when a rewrite copied the window rendering) + mo = CLIP_RE.search(c) + if mo: + findings.append({ + "cls": "clip_marker", "i": i, + "detail": mo.group(0)[:80], "ctx": c[ + max(0, mo.start() - 30): + mo.end() + 50].replace("\n", " ")[:160], + }) + if role == "assistant": + has_calls = bool(m.get("tool_calls")) + if not has_calls and len(c.strip()) < 20: + findings.append({ + "cls": "degenerate_turn", "i": i, + "detail": f"prose turn {len(c.strip())} chars", + "ctx": c.strip()[:160], + }) + mo = JUDGE_LEAK_RE.search(c) + if mo: + findings.append({ + "cls": "judge_residue", "i": i, + "detail": mo.group(0)[:80], + "ctx": c[max(0, mo.start() - 30): + mo.end() + 50].replace("\n", " ")[:160], + }) + mo = IDENTITY_RE.search(c) + if mo: + findings.append({ + "cls": "identity_burn", "i": i, + "detail": mo.group(0)[:80], + "ctx": c[max(0, mo.start() - 30): + mo.end() + 50].replace("\n", " ")[:160], + }) + mo = REFUSAL_RE.search(c) + if mo: + findings.append({ + "cls": "refusal", "i": i, + "detail": mo.group(0)[:80], + "ctx": c[max(0, mo.start() - 30): + mo.end() + 50].replace("\n", " ")[:160], + }) + return findings + + +def audit_corpus(improved_dir: str, *, max_dupes: int = 20) -> dict: + """Whole-corpus audit: per-trace findings + near-duplicate clusters.""" + per_trace: dict[str, list] = {} + shingles: dict[str, set] = {} + sizes = [] + for f in sorted(os.listdir(improved_dir)): + if not f.endswith(".json"): + continue + sid = f[:-5] + try: + t = json.load(open(os.path.join(improved_dir, f))) + except Exception as e: + per_trace[sid] = [{"cls": "unparseable", "i": -1, + "detail": str(e)[:120], "ctx": ""}] + continue + per_trace[sid] = audit_trace(t) + prose = " ".join((m.get("content") or "") + for m in t.get("messages", []) + if m.get("role") == "assistant") + sh = _shingle(prose) + if sh: + shingles[sid] = sh + sizes.append((sid, len(json.dumps(t)))) + + # Near-duplicate clusters: greedy grouping against cluster reps. + # + # Comparing every trace against every cluster is O(n^2) Jaccard on sets of + # thousands of shingles: at 2,983 traces it ran for over 20 minutes and + # blocked every export. Two sets with Jaccard > 0.9 almost certainly share + # their smallest shingle hashes, so bucket by the 3 smallest and only + # compare inside a bucket. That is the standard LSH banding trick and it + # turns the pass into O(n) buckets with a handful of comparisons each. + buckets: dict[str, list[str]] = {} + for sid, sh in shingles.items(): + for h in sorted(sh)[:3]: + buckets.setdefault(h, []).append(sid) + + clusters: list[list[str]] = [] + assigned: dict[str, list[str]] = {} + for a in shingles: + placed = False + # candidate clusters: any cluster whose rep shares a bucket with `a` + cands: list[list[str]] = [] + seen_ids: set[int] = set() + for h in sorted(shingles[a])[:3]: + for other in buckets.get(h, ()): + cl = assigned.get(other) + if cl is not None and id(cl) not in seen_ids: + seen_ids.add(id(cl)) + cands.append(cl) + for cl in cands: + if _jaccard(shingles[a], shingles[cl[0]]) > 0.9: + cl.append(a) + assigned[a] = cl + placed = True + break + if not placed: + cl = [a] + clusters.append(cl) + assigned[a] = cl + dupes = [cl for cl in clusters if len(cl) > 1][:max_dupes] + + return {"per_trace": per_trace, "dupes": dupes, + "total": len(per_trace)} + + +def main(improved_dir: str, *, verbose: bool = False) -> int: + rep = audit_corpus(improved_dir) + counts: Counter = Counter() + flagged = {} + for sid, findings in rep["per_trace"].items(): + for f in findings: + counts[f["cls"]] += 1 + if findings: + flagged[sid] = findings + print(f"audited {rep['total']} promoted traces") + for cls, n in sorted(counts.items()): + print(f" {cls}: {n}") + for sid, fs in list(flagged.items())[:10 if verbose else 5]: + print(f"\n--- {sid}") + for f in fs[:4]: + print(f" [{f['i']}] {f['cls']}: {f['detail']}") + print(f" …{f['ctx'][:120]}…") + if rep["dupes"]: + print(f"\nnear-duplicate clusters (J>0.9): {len(rep['dupes'])}") + for cl in rep["dupes"][:8]: + print(f" {' ~ '.join(cl)}") + else: + print("\nno near-duplicate clusters (J>0.9)") + return 1 if any(counts[c] for c in + ("self_talk", "clip_marker", "unparseable", + "judge_residue", "identity_burn", "refusal", + "final_blob")) else 0 + + +if __name__ == "__main__": + import sys + raise SystemExit(main(sys.argv[1] if len(sys.argv) > 1 + else "data/traces/improved", + verbose="-v" in sys.argv)) \ No newline at end of file diff --git a/traceforge/benchmark.py b/traceforge/benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..38c7075def06de9d66a1695df2c70a14024ca5e0 --- /dev/null +++ b/traceforge/benchmark.py @@ -0,0 +1,811 @@ +"""Benchmark suite β€” score a model by how little the pipeline had to edit. + +The pipeline turns a real agent trace into an ideal-world reference. That +makes it an evaluation instrument: **the amount of work the pipeline has to +do on a model's trace is an inverse measure of how close the model already +is to the reference.** If the pipeline has nothing to do, the model is +already perfect. + +Two independent signals, both "less is better": + +1. **Fidelity** β€” similarity between the model's trace and the ideal + rewrite. A trace the pipeline can promote unchanged scores 1.0; every + dropped, rephrased or added message lowers it. Computed as a Dice + coefficient over aligned message pairs, so a rewrite that deletes half + the trace cannot score high by matching the surviving half perfectly. +2. **Labels** β€” the triage pass labels every message ``keep`` / ``drop`` / + ``rephrase`` / ``fact_check`` / ``summarize``. ``keep`` is a positive + judgement, ``drop`` a negative one, and the shape edits sit in between. + The label score is graded (keep 1.0, shape edit 0.5, drop 0.0) and is + computed over ASSISTANT rows only: those are the model's own turns, while + a long tool result the triage wants compressed is the environment's + verbosity, not a model defect. + +Composite score (0-100):: + + score = 100 * (w_sim * fidelity + w_label * label_score) + +Defaults are 0.6 / 0.4. Weights are a policy choice, not a measurement; +they are recorded in every report so a score is never quoted without them. + +Why Dice and not a plain mean: a mean over aligned pairs rewards a rewrite +that keeps one message and deletes nine. Dice divides by the sum of both +lengths, so deletions and additions are edits and cost score. + +Normalisation: the readers prepend pipeline framing markers to user rows +(``[trace segment 2 of 2 …]``, ``[subagent brief β€” role: …]``). The model +never wrote those, so they are stripped before comparison β€” otherwise every +split segment and subagent trace is penalised for a reader artifact. + +Caveats the report states explicitly: triage labels come from a cheap +model and are a judgement, not ground truth; similarity measures agreement +with the reference, not correctness of the task outcome; and a corpus +smaller than ~30 traces cannot support a stable mean. +""" +from __future__ import annotations + +import difflib +import json +import math +import os +import re +import time +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from typing import Any, Iterable, Optional + +from traceforge.trace import Message, Trace +from traceforge.triage import ACTIONS +from traceforge.validate import strip_scaffold_markers + +BENCHMARK_VERSION = "1.0" + +# Policy weights, not measurements. Recorded in every report. +DEFAULT_WEIGHTS: dict[str, float] = {"similarity": 0.6, "labels": 0.4} + +# A pair at or above this is "kept verbatim" (normalised); below it the +# pipeline rewrote the message. 0.55 is the alignment floor: two messages +# that share less than half their text are different messages, not edits. +VERBATIM = 0.98 +REPHRASE = 0.55 + +_WS_RE = re.compile(r"\s+") + + +# --------------------------------------------------------------- similarity + + +def _norm(text: Optional[str]) -> str: + """Collapse whitespace and case so formatting noise is not an edit.""" + return _WS_RE.sub(" ", (text or "").strip().lower()) + + +def text_similarity(a: str, b: str, *, head: int = 800, tail: int = 200) -> float: + """Similarity of two message texts in [0, 1]. + + Long tool outputs are compared on head + tail: a rewrite that preserves + an output preserves both ends, and running difflib over 50 KB strings is + quadratic for no extra signal. The length-ratio prefilter is what keeps + a full-corpus run under two minutes β€” a message half the length of its + counterpart is a compression, not a match. + """ + a, b = _norm(a), _norm(b) + if a == b: + return 1.0 + if not a or not b: + return 0.0 + la, lb = len(a), len(b) + if min(la, lb) / max(la, lb) < 0.5: + return 0.0 + if la > head + tail or lb > head + tail: + a = a[:head] + "\n…\n" + a[-tail:] + b = b[:head] + "\n…\n" + b[-tail:] + sm = difflib.SequenceMatcher(None, a, b, autojunk=False) + # quick_ratio is an upper bound on ratio; below the alignment floor the + # exact value cannot change any decision, so skip the quadratic pass. + if sm.quick_ratio() < 0.4: + return 0.0 + return sm.ratio() + + +def _msg_text(m: Message) -> str: + """Everything a message carries: prose plus its tool calls.""" + parts = [m.content or ""] + for tc in m.tool_calls: + parts.append(f"{tc.name} {tc.arguments}") + return "\n".join(p for p in parts if p) + + +def _call_sig(tc) -> tuple[str, str]: + return (tc.name, _norm(tc.arguments)[:300]) + + +def _seq_similarity(a: list, b: list) -> float: + """Dice coefficient over two sequences (1.0 when both are empty).""" + if not a and not b: + return 1.0 + if not a or not b: + return 0.0 + sm = difflib.SequenceMatcher(None, a, b, autojunk=False) + return sm.ratio() + + +def message_similarity(raw_m: Message, imp_m: Message) -> float: + """Similarity of one aligned message pair. + + A tool-only assistant turn carries no prose, so its calls ARE the + message; weighting prose and calls equally would score every such pair + as a perfect match on the empty string. + """ + if raw_m.role != imp_m.role: + return 0.0 + text_sim = text_similarity(_msg_text(raw_m), _msg_text(imp_m)) + raw_calls = [_call_sig(tc) for tc in raw_m.tool_calls] + imp_calls = [_call_sig(tc) for tc in imp_m.tool_calls] + if raw_calls or imp_calls: + call_sim = _seq_similarity(raw_calls, imp_calls) + if not _norm(raw_m.content) and not _norm(imp_m.content): + return call_sim + return 0.5 * text_sim + 0.5 * call_sim + return text_sim + + +def align_messages(raw_msgs: list[Message], + imp_msgs: list[Message]) -> list[tuple[int, int]]: + """Monotonic alignment of raw -> improved messages. + + Roles are the stable skeleton: a rewrite keeps the user/assistant/tool + shape and drops or adds whole turns. SequenceMatcher over the role + sequences finds the equal runs and the replaced blocks; inside a block + messages pair by position. That is O(n) pairs instead of the O(nΒ·window) + of a content-driven search, and it is the right model of the edit: the + pipeline does not reorder turns, it removes and rewrites them. + """ + raw_roles = [m.role for m in raw_msgs] + imp_roles = [m.role for m in imp_msgs] + sm = difflib.SequenceMatcher(None, raw_roles, imp_roles, autojunk=False) + pairs: list[tuple[int, int]] = [] + for tag, i1, i2, j1, j2 in sm.get_opcodes(): + if tag in ("equal", "replace"): + for k in range(min(i2 - i1, j2 - j1)): + pairs.append((i1 + k, j1 + k)) + return pairs + + +# ------------------------------------------------------------------- labels + + +def label_stats(triage: Optional[dict], + messages: Optional[list[Message]] = None) -> dict[str, Any]: + """Counts and rates over one triage label file. + + ``keep`` is the positive judgement; ``drop`` the negative one. + ``rephrase``/``summarize`` are shape edits (right content, wrong form), + ``fact_check`` is a verification flag β€” neither is a negative label, so + they are reported separately instead of being folded into the keep rate. + + When ``messages`` is given, the same counts are also computed over + ASSISTANT rows only. That is the number the composite uses: the model + wrote its own turns, not the tool outputs, and a long tool result the + triage wants compressed is the environment's verbosity, not a model + defect. + """ + labels = (triage or {}).get("labels") or [] + valid = [lab for lab in labels + if isinstance(lab, dict) and lab.get("action") in ACTIONS] + counts = Counter(lab["action"] for lab in valid) + total = sum(counts.values()) + out: dict[str, Any] = {"total": total} + for action in ACTIONS: + out[action] = counts.get(action, 0) + out["keep_rate"] = (counts.get("keep", 0) / total) if total else None + out["negative_rate"] = (counts.get("drop", 0) / total) if total else None + out["edit_rate"] = ( + (counts.get("rephrase", 0) + counts.get("summarize", 0)) / total + if total else None + ) + # Graded positive/negative score: keep = 1.0, a shape edit (rephrase / + # summarize / fact_check) = 0.5, drop = 0.0. The strict keep rate is + # dominated by the triage model's compression preference β€” 58% of + # assistant labels on the real corpus are `summarize` because assistant + # turns carry large tool-call arguments β€” so the composite uses the + # graded score and the report shows both. + out["label_score"] = ( + (counts.get("keep", 0) + + 0.5 * (counts.get("rephrase", 0) + counts.get("summarize", 0) + + counts.get("fact_check", 0))) / total + if total else None + ) + if messages is not None: + roles = {i: m.role for i, m in enumerate(messages)} + a_counts = Counter( + lab["action"] for lab in valid + if isinstance(lab.get("i"), int) + and roles.get(lab["i"]) == "assistant" + ) + a_total = sum(a_counts.values()) + out["assistant_total"] = a_total + for action in ACTIONS: + out[f"assistant_{action}"] = a_counts.get(action, 0) + out["assistant_keep_rate"] = ( + a_counts.get("keep", 0) / a_total if a_total else None) + out["assistant_negative_rate"] = ( + a_counts.get("drop", 0) / a_total if a_total else None) + out["assistant_label_score"] = ( + (a_counts.get("keep", 0) + + 0.5 * (a_counts.get("rephrase", 0) + a_counts.get("summarize", 0) + + a_counts.get("fact_check", 0))) / a_total + if a_total else None + ) + return out + + +def load_triage(path: str) -> Optional[dict]: + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError): + return None + + +# ------------------------------------------------------------------ scoring + + +@dataclass +class TraceScore: + """One model trace scored against its ideal rewrite.""" + + session_id: str + task: Optional[str] = None + source: Optional[str] = None + model: Optional[str] = None + + # fidelity + similarity: float = 0.0 + tool_similarity: float = 0.0 + kept_verbatim: int = 0 + rephrased: int = 0 + dropped: int = 0 + added: int = 0 + raw_messages: int = 0 + improved_messages: int = 0 + raw_chars: int = 0 + + # labels + labels_total: int = 0 + keep: int = 0 + drop: int = 0 + rephrase: int = 0 + fact_check: int = 0 + summarize: int = 0 + keep_rate: Optional[float] = None + negative_rate: Optional[float] = None + label_score: Optional[float] = None + # assistant-only: the model's own turns (the loss-bearing target) + assistant_labels_total: int = 0 + assistant_keep: int = 0 + assistant_drop: int = 0 + assistant_keep_rate: Optional[float] = None + assistant_negative_rate: Optional[float] = None + assistant_label_score: Optional[float] = None + + # gates (best effort β€” only traces with a manifest attempts row have it) + attempts: Optional[int] = None + first_pass: Optional[bool] = None + + # composite + score: float = 0.0 + pipeline_idle: bool = False + + @property + def edits(self) -> int: + return self.dropped + self.added + self.rephrased + + def to_dict(self) -> dict[str, Any]: + d = { + "session_id": self.session_id, + "task": self.task, + "source": self.source, + "model": self.model, + "score": round(self.score, 2), + "similarity": round(self.similarity, 4), + "tool_similarity": round(self.tool_similarity, 4), + "keep_rate": (None if self.keep_rate is None + else round(self.keep_rate, 4)), + "negative_rate": (None if self.negative_rate is None + else round(self.negative_rate, 4)), + "assistant_keep_rate": (None if self.assistant_keep_rate is None + else round(self.assistant_keep_rate, 4)), + "assistant_negative_rate": ( + None if self.assistant_negative_rate is None + else round(self.assistant_negative_rate, 4)), + "label_score": (None if self.label_score is None + else round(self.label_score, 4)), + "assistant_label_score": ( + None if self.assistant_label_score is None + else round(self.assistant_label_score, 4)), + "edits": { + "kept_verbatim": self.kept_verbatim, + "rephrased": self.rephrased, + "dropped": self.dropped, + "added": self.added, + "total": self.edits, + }, + "messages": { + "raw": self.raw_messages, + "improved": self.improved_messages, + "raw_chars": self.raw_chars, + }, + "labels": { + "total": self.labels_total, + "keep": self.keep, + "drop": self.drop, + "rephrase": self.rephrase, + "fact_check": self.fact_check, + "summarize": self.summarize, + }, + "attempts": self.attempts, + "first_pass": self.first_pass, + "pipeline_idle": self.pipeline_idle, + } + return d + + +def score_pair(raw: Trace, improved: Trace, *, triage: Optional[dict] = None, + attempts: Optional[int] = None, + weights: Optional[dict[str, float]] = None) -> TraceScore: + """Score one (model trace, ideal rewrite) pair. + + ``raw`` is mutated in place by the scaffold strip β€” pass a freshly + loaded trace, which is what the CLI does. + """ + w = {**DEFAULT_WEIGHTS, **(weights or {})} + strip_scaffold_markers(raw) + + pairs = align_messages(raw.messages, improved.messages) + sims = [message_similarity(raw.messages[i], improved.messages[j]) + for i, j in pairs] + n_raw, n_imp = len(raw.messages), len(improved.messages) + similarity = (2.0 * sum(sims) / (n_raw + n_imp)) if (n_raw + n_imp) else 1.0 + + kept = sum(1 for s in sims if s >= VERBATIM) + rephrased = sum(1 for s in sims if REPHRASE <= s < VERBATIM) + dropped = n_raw - len(pairs) + added = n_imp - len(pairs) + + tool_similarity = _seq_similarity( + [_call_sig(tc) for m in raw.messages for tc in m.tool_calls], + [_call_sig(tc) for m in improved.messages for tc in m.tool_calls], + ) + + lab = label_stats(triage, raw.messages) + keep_rate = lab["keep_rate"] + # The composite judges the model on its OWN turns. Tool outputs are + # context the model did not write; a long result the triage wants + # compressed is the environment's verbosity, not a model defect. + label_score = (lab.get("assistant_label_score") + if lab.get("assistant_label_score") is not None + else lab.get("label_score")) + # Idle is strict: any non-keep label means the pipeline had work to do. + strict_keep = (lab.get("assistant_keep_rate") + if lab.get("assistant_keep_rate") is not None else keep_rate) + + # Composite: renormalise over the components that exist. A trace with no + # triage file is scored on fidelity alone rather than being penalised + # for a missing label pass. + parts = [(w["similarity"], similarity)] + if label_score is not None: + parts.append((w["labels"], label_score)) + total_w = sum(p[0] for p in parts) + score = (100.0 * sum(p[0] * p[1] for p in parts) / total_w) if total_w else 0.0 + + idle = (similarity >= 0.999 and dropped == 0 and added == 0 + and (strict_keep is None or strict_keep >= 0.999)) + + return TraceScore( + session_id=raw.session_id, + task=improved.task or raw.task, + source=raw.source, + model=improved.model or raw.model, + similarity=similarity, + tool_similarity=tool_similarity, + kept_verbatim=kept, + rephrased=rephrased, + dropped=dropped, + added=added, + raw_messages=n_raw, + improved_messages=n_imp, + raw_chars=sum(len(m.content or "") for m in raw.messages), + labels_total=lab["total"], + keep=lab["keep"], + drop=lab["drop"], + rephrase=lab["rephrase"], + fact_check=lab["fact_check"], + summarize=lab["summarize"], + keep_rate=keep_rate, + negative_rate=lab["negative_rate"], + label_score=lab.get("label_score"), + assistant_labels_total=lab.get("assistant_total", 0), + assistant_keep=lab.get("assistant_keep", 0), + assistant_drop=lab.get("assistant_drop", 0), + assistant_keep_rate=lab.get("assistant_keep_rate"), + assistant_negative_rate=lab.get("assistant_negative_rate"), + assistant_label_score=lab.get("assistant_label_score"), + attempts=attempts, + first_pass=(attempts == 1) if attempts is not None else None, + score=score, + pipeline_idle=idle, + ) + + +# --------------------------------------------------------------- statistics + + +def _dist(vals: Iterable[float]) -> dict[str, Any]: + """Distribution summary. Percentiles are nearest-rank, not interpolated.""" + v = sorted(float(x) for x in vals) + if not v: + return {"n": 0, "mean": None, "median": None, "p10": None, + "p90": None, "min": None, "max": None} + + def pct(q: float) -> float: + idx = min(len(v) - 1, max(0, int(round(q * (len(v) - 1))))) + return v[idx] + + return { + "n": len(v), + "mean": round(sum(v) / len(v), 4), + "median": round(pct(0.5), 4), + "p10": round(pct(0.1), 4), + "p90": round(pct(0.9), 4), + "min": round(v[0], 4), + "max": round(v[-1], 4), + } + + +def _pearson(xs: list[float], ys: list[float]) -> Optional[float]: + n = len(xs) + if n < 3 or n != len(ys): + return None + mx, my = sum(xs) / n, sum(ys) / n + num = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) + dx = math.sqrt(sum((x - mx) ** 2 for x in xs)) + dy = math.sqrt(sum((y - my) ** 2 for y in ys)) + if dx == 0 or dy == 0: + return None + return round(num / (dx * dy), 3) + + +def _histogram(vals: list[float], edges: list[float]) -> dict[str, int]: + """Bucket counts; the last bucket is inclusive of its upper edge.""" + out: dict[str, int] = {} + last = len(edges) - 2 + for i, (lo, hi) in enumerate(zip(edges, edges[1:])): + out[f"{lo:g}-{hi:g}"] = sum( + 1 for v in vals if lo <= v < hi or (i == last and v == hi)) + return out + + +def _group(scores: list[TraceScore], key) -> dict[str, dict[str, Any]]: + buckets: dict[str, list[TraceScore]] = defaultdict(list) + for s in scores: + buckets[key(s) or "unknown"].append(s) + return { + k: {"n": len(v), "score_mean": round(sum(x.score for x in v) / len(v), 2), + "similarity_mean": round(sum(x.similarity for x in v) / len(v), 3), + "keep_rate_mean": ( + round(sum(x.keep_rate for x in v if x.keep_rate is not None) + / max(1, sum(1 for x in v if x.keep_rate is not None)), 3) + if any(x.keep_rate is not None for x in v) else None)} + for k, v in sorted(buckets.items(), key=lambda kv: -len(kv[1])) + } + + +def _size_band(chars: int) -> str: + kb = chars / 1000.0 + if kb < 60: + return "<60KB" + if kb < 120: + return "60-120KB" + if kb < 180: + return "120-180KB" + if kb < 240: + return "180-240KB" + if kb < 300: + return "240-300KB" + return "300KB+" + + +def build_report(scores: list[TraceScore], *, label: Optional[str] = None, + weights: Optional[dict[str, float]] = None, + corpus: Optional[dict[str, Any]] = None, + generated_at: Optional[str] = None) -> dict[str, Any]: + """Corpus-level statistics over a set of scored trace pairs.""" + w = {**DEFAULT_WEIGHTS, **(weights or {})} + n = len(scores) + sims = [s.similarity for s in scores] + keeps = [s.keep_rate for s in scores if s.keep_rate is not None] + a_keeps = [s.assistant_keep_rate for s in scores + if s.assistant_keep_rate is not None] + a_scores = [s.assistant_label_score for s in scores + if s.assistant_label_score is not None] + negs = [s.negative_rate for s in scores if s.negative_rate is not None] + a_negs = [s.assistant_negative_rate for s in scores + if s.assistant_negative_rate is not None] + comp = [s.score for s in scores] + edits = [s.edits for s in scores] + tools = [s.tool_similarity for s in scores] + + label_totals = Counter() + for s in scores: + for action in ACTIONS: + label_totals[action] += getattr(s, action) + label_total = sum(label_totals.values()) + + edit_totals = { + "kept_verbatim": sum(s.kept_verbatim for s in scores), + "rephrased": sum(s.rephrased for s in scores), + "dropped": sum(s.dropped for s in scores), + "added": sum(s.added for s in scores), + } + edit_total = sum(edit_totals.values()) or 1 + + idle = [s for s in scores if s.pipeline_idle] + gated = [s for s in scores if s.attempts is not None] + first_pass = [s for s in gated if s.first_pass] + + # Fidelity and labels are independent instruments; where they disagree + # the pipeline is inconsistent (a trace promoted unchanged although the + # triage wanted edits, or a rewrite of turns the triage called clean). + # That is a pipeline diagnostic, not a model defect β€” report it. + def _lab(s: TraceScore) -> float: + return s.assistant_label_score if s.assistant_label_score is not None else 0.0 + + fid_perfect = [s for s in scores if s.similarity >= 0.999] + lab_perfect = [s for s in scores if _lab(s) >= 0.999] + both_perfect = [s for s in scores + if s.similarity >= 0.999 and _lab(s) >= 0.999] + + corr = _pearson( + [s.similarity for s in scores if s.assistant_label_score is not None], + [s.assistant_label_score for s in scores + if s.assistant_label_score is not None], + ) + + worst = sorted(scores, key=lambda s: s.score)[:10] + best = sorted(scores, key=lambda s: -s.score)[:10] + + report: dict[str, Any] = { + "benchmark_version": BENCHMARK_VERSION, + "generated_at": generated_at or time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "label": label, + "weights": w, + "corpus": { + "scored": n, + "with_labels": len(keeps), + "with_gates": len(gated), + **(corpus or {}), + }, + "score": { + **_dist(comp), + "histogram": _histogram(comp, [0, 20, 40, 60, 80, 100]), + }, + "components": { + "similarity": _dist(sims), + "keep_rate": _dist(keeps), + "assistant_keep_rate": _dist(a_keeps), + "label_score": _dist(a_scores), + "negative_rate": _dist(negs), + "assistant_negative_rate": _dist(a_negs), + "tool_sequence_similarity": _dist(tools), + }, + "edits": { + "totals": edit_totals, + "share_pct": {k: round(100.0 * v / edit_total, 1) + for k, v in edit_totals.items()}, + "per_trace": _dist(edits), + }, + "labels": { + "totals": dict(label_totals), + "total": label_total, + "share_pct": {k: round(100.0 * v / label_total, 1) + for k, v in label_totals.items()} if label_total else {}, + }, + "pipeline_idle": { + "traces": len(idle), + "pct": round(100.0 * len(idle) / n, 1) if n else 0.0, + "definition": "similarity >= 0.999, no dropped/added message, " + "keep_rate >= 0.999 β€” the pipeline had nothing to do", + }, + "agreement": { + "both_perfect": len(both_perfect), + "fidelity_perfect_labels_edit": len(fid_perfect) - len(both_perfect), + "labels_perfect_fidelity_edit": len(lab_perfect) - len(both_perfect), + "note": "fidelity and labels are independent instruments; a " + "disagreement is a pipeline inconsistency (promoted " + "unchanged although triage wanted edits, or rewritten " + "although triage called it clean), not a model defect", + }, + "gates": { + "traces_with_attempts": len(gated), + "first_pass": len(first_pass), + "first_pass_pct": (round(100.0 * len(first_pass) / len(gated), 1) + if gated else None), + "attempts_mean": (round(sum(s.attempts for s in gated) / len(gated), 2) + if gated else None), + }, + "by_task": _group(scores, lambda s: s.task), + "by_source": _group(scores, lambda s: s.source), + "by_size": _group(scores, lambda s: _size_band(s.raw_chars)), + "correlation": {"similarity_vs_label_score": corr}, + "worst": [s.to_dict() for s in worst], + "best": [s.to_dict() for s in best], + "notes": [ + "score = 100 * (w_similarity * fidelity + w_labels * label_score); " + "weights are policy, not measurement", + "fidelity is a Dice coefficient over aligned message pairs, so " + "dropped and added messages both cost score", + "label_score is graded: keep=1.0, rephrase/summarize/fact_check=0.5, " + "drop=0.0, over ASSISTANT rows only (the model's own turns)", + "triage labels come from a cheap model and are a judgement, not " + "ground truth; label_score is agreement with the pipeline's judge", + "similarity measures agreement with the reference, not whether " + "the task outcome was correct", + "pipeline framing markers are stripped from raw user rows before " + "comparison (the model never wrote them)", + "n < 30 traces cannot support a stable mean β€” treat the score as " + "a smoke signal, not a ranking", + ], + } + return report + + +# ------------------------------------------------------------------ render + + +def _fmt(v: Optional[float], nd: int = 2) -> str: + return "β€”" if v is None else f"{v:.{nd}f}" + + +def render_report(report: dict[str, Any]) -> str: + """Human-readable report. The JSON is the artifact; this is the glance.""" + c = report["corpus"] + sc = report["score"] + comp = report["components"] + lines: list[str] = [] + title = f"traceforge benchmark v{report['benchmark_version']}" + if report.get("label"): + title += f" β€” {report['label']}" + lines.append(title) + lines.append( + f"scored {c['scored']} trace pairs (model trace -> ideal rewrite) Β· " + f"{c['with_labels']} with triage labels Β· " + f"{c['with_gates']} with gate attempts" + ) + lines.append("") + lines.append(f"score (0-100) mean {_fmt(sc['mean'], 1)} " + f"median {_fmt(sc['median'], 1)} " + f"p10 {_fmt(sc['p10'], 1)} p90 {_fmt(sc['p90'], 1)}") + lines.append(f" fidelity mean {_fmt(comp['similarity']['mean'], 3)} " + f"median {_fmt(comp['similarity']['median'], 3)} " + f"p10 {_fmt(comp['similarity']['p10'], 3)} " + f"p90 {_fmt(comp['similarity']['p90'], 3)}") + lines.append(f" label score mean {_fmt(comp['label_score']['mean'], 3)} " + f"median {_fmt(comp['label_score']['median'], 3)} " + f"p10 {_fmt(comp['label_score']['p10'], 3)} " + f"p90 {_fmt(comp['label_score']['p90'], 3)}") + lines.append(f" keep rate (asst) mean {_fmt(comp['assistant_keep_rate']['mean'], 3)} " + f"median {_fmt(comp['assistant_keep_rate']['median'], 3)} " + f"p10 {_fmt(comp['assistant_keep_rate']['p10'], 3)} " + f"p90 {_fmt(comp['assistant_keep_rate']['p90'], 3)}") + lines.append(f" negative (asst) mean {_fmt(comp['assistant_negative_rate']['mean'], 3)} " + f"median {_fmt(comp['assistant_negative_rate']['median'], 3)} " + f"p90 {_fmt(comp['assistant_negative_rate']['p90'], 3)}") + lines.append(f" tool sequence mean {_fmt(comp['tool_sequence_similarity']['mean'], 3)}") + lines.append("") + hist = sc["histogram"] + lines.append("score histogram " + " Β· ".join( + f"{k}: {v}" for k, v in hist.items())) + lines.append("") + e = report["edits"] + lines.append(f"edits per trace mean {_fmt(e['per_trace']['mean'], 1)} " + f"median {_fmt(e['per_trace']['median'], 1)} " + f"p90 {_fmt(e['per_trace']['p90'], 1)}") + lines.append( + " messages " + f"kept {e['totals']['kept_verbatim']} ({e['share_pct']['kept_verbatim']}%) Β· " + f"rephrased {e['totals']['rephrased']} ({e['share_pct']['rephrased']}%) Β· " + f"dropped {e['totals']['dropped']} ({e['share_pct']['dropped']}%) Β· " + f"added {e['totals']['added']} ({e['share_pct']['added']}%)") + lab = report["labels"] + if lab["total"]: + lines.append("labels " + " Β· ".join( + f"{k} {v}" for k, v in lab["totals"].items())) + idle = report["pipeline_idle"] + lines.append(f"pipeline idle {idle['traces']} traces " + f"({idle['pct']}%) β€” nothing to edit") + ag = report.get("agreement") or {} + if ag: + lines.append(f"instrument agree both perfect {ag['both_perfect']} Β· " + f"fidelity-only {ag['fidelity_perfect_labels_edit']} Β· " + f"labels-only {ag['labels_perfect_fidelity_edit']}") + g = report["gates"] + if g["traces_with_attempts"]: + lines.append(f"first-pass gates {g['first_pass']}/" + f"{g['traces_with_attempts']} ({g['first_pass_pct']}%) Β· " + f"mean attempts {g['attempts_mean']}") + lines.append("") + for name, key in (("by task", "by_task"), ("by source", "by_source"), + ("by size", "by_size")): + groups = report[key] + if not groups: + continue + parts = [f"{k} {v['n']} ({_fmt(v['score_mean'], 1)})" + for k, v in list(groups.items())[:8]] + lines.append(f"{name:<18} " + " Β· ".join(parts)) + corr = report["correlation"]["similarity_vs_label_score"] + lines.append(f"correlation similarity vs label score: r={_fmt(corr, 3)}") + lines.append("") + lines.append("worst traces (inspect these first):") + for s in report["worst"][:5]: + lines.append(f" {s['session_id'][:44]:<44} score {s['score']:>5.1f} " + f"sim {s['similarity']:.2f} " + f"label {_fmt(s['assistant_label_score'], 2)} " + f"{s['task'] or 'β€”'}") + return "\n".join(lines) + + +# -------------------------------------------------------------- leaderboard + + +def append_leaderboard(path: str, report: dict[str, Any]) -> None: + """Append one compact row per run; the leaderboard keeps the latest per label.""" + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + row = { + "label": report.get("label") or "unlabeled", + "generated_at": report["generated_at"], + "benchmark_version": report["benchmark_version"], + "n": report["corpus"]["scored"], + "score_mean": report["score"]["mean"], + "score_median": report["score"]["median"], + "similarity_mean": report["components"]["similarity"]["mean"], + "label_score_mean": report["components"]["label_score"]["mean"], + "keep_rate_mean": report["components"]["assistant_keep_rate"]["mean"], + "pipeline_idle_pct": report["pipeline_idle"]["pct"], + "weights": report["weights"], + } + with open(path, "a", encoding="utf-8") as f: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def load_leaderboard(path: str) -> list[dict[str, Any]]: + rows: dict[str, dict[str, Any]] = {} + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + try: + r = json.loads(line) + except ValueError: + continue + if r.get("label"): + rows[r["label"]] = r # latest run wins + except OSError: + return [] + return sorted(rows.values(), key=lambda r: -(r.get("score_mean") or 0)) + + +def render_leaderboard(rows: list[dict[str, Any]]) -> str: + if not rows: + return "leaderboard: empty" + lines = ["leaderboard (latest run per label, sorted by mean score):", + f" {'label':<28} {'n':>5} {'score':>7} {'fidelity':>9} " + f"{'label':>6} {'idle%':>6} when"] + for r in rows: + lines.append( + f" {r['label'][:28]:<28} {r['n']:>5} " + f"{_fmt(r.get('score_mean'), 1):>7} " + f"{_fmt(r.get('similarity_mean'), 3):>9} " + f"{_fmt(r.get('label_score_mean'), 3):>6} " + f"{_fmt(r.get('pipeline_idle_pct'), 1):>6} " + f"{r.get('generated_at', '')[:19]}") + return "\n".join(lines) diff --git a/traceforge/chunker.py b/traceforge/chunker.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9bd656243244b64a05bb2824cfaaa398910e19 --- /dev/null +++ b/traceforge/chunker.py @@ -0,0 +1,90 @@ +"""Splits a Trace into LLM-digestible chunks at message boundaries. + +A 10M-token session can never be rendered into a single prompt. Chunks are +cut at message boundaries under a char budget (β‰ˆ4 chars/token), with a +small tail-overlap of previous-chunk messages prepended as continuity +context for the digester. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from traceforge.trace import Trace + + +@dataclass +class Chunk: + index: int # 1-based + start: int # first message index (0-based, inclusive) + end: int # last message index (exclusive) + rendered: str # compact rendering incl. overlap context header + approx_chars: int # size estimate of the chunk's own messages + + +def _msg_size(m) -> int: + return len(m.content or "") + sum( + len(tc.arguments) for tc in m.tool_calls + ) + 64 + + +def chunk_trace( + trace: Trace, + *, + max_chars: int = 100_000, + overlap_messages: int = 2, + render_kwargs: dict | None = None, +) -> list[Chunk]: + """Cut the trace into chunks of ≀ max_chars (single messages never split). + + Returns chunks with per-chunk overlap context prepended (marked as + continuity-only so the digester does not double-count). + """ + rk = render_kwargs or {} + msgs = trace.messages + n = len(msgs) + chunks: list[Chunk] = [] + + def render_slice(lo: int, hi: int) -> str: + sub = Trace( + session_id=trace.session_id, + model=trace.model, + messages=msgs[lo:hi], + ) + out = sub.render_compact(**rk) + # drop the synthetic "=== TRACE ... ===" header line + return out.split("\n", 1)[1] if "\n" in out else out + + i = 0 + while i < n: + j = i + total = 0 + while j < n and (j == i or total + _msg_size(msgs[j]) <= max_chars): + total += _msg_size(msgs[j]) + j += 1 + body = render_slice(i, j) + header = ( + f"CHUNK {len(chunks) + 1} of ~N (messages {i + 1}-{j} of {n})" + ) + block = f"{header}\n{body}" + if i > 0 and overlap_messages > 0: + ov = render_slice(max(0, i - overlap_messages), i) + block = ( + "CONTEXT β€” tail of the previous chunk (continuity only;" + " already digested, do not re-report):\n" + f"{ov}\n\n{block}" + ) + chunks.append( + Chunk( + index=len(chunks) + 1, + start=i, + end=j, + rendered=block, + approx_chars=total, + ) + ) + i = j + # fix the "of ~N" placeholders now that we know the count + for c in chunks: + if "~N" in c.rendered: + c.rendered = c.rendered.replace("~N", str(len(chunks))) + return chunks \ No newline at end of file diff --git a/traceforge/classify.py b/traceforge/classify.py new file mode 100644 index 0000000000000000000000000000000000000000..3132f6d2535265f924285acbe9a1b274ccf44113 --- /dev/null +++ b/traceforge/classify.py @@ -0,0 +1,394 @@ +"""Task/topic classification for traces. + +Labels every trace with one task bucket so export can split the corpus by +topic (``train.coding.jsonl``, ``train.research.jsonl``, ...). Two tiers: + + heuristic tool signature + first-task-message keywords. Free, + deterministic, whole corpus in seconds. Confidence is the + margin between the winning bucket and the runner-up. + llm one structured call per trace, used only where the heuristic + is unsure (``--llm``). Same endpoint/grammar ladder as + triage.py, so no new infrastructure. + +The label is persisted on the trace (``Trace.task``) so export is +deterministic and every assignment is inspectable and reviewable. + +Taxonomy is deliberately closed: a small set the trainer can weight +against, not free-form tags. Coding is split because the corpus is ~95% +coding and a single bucket would be useless for balancing. +""" +from __future__ import annotations + +import json +import re +from typing import Any, Optional + +from traceforge.trace import Trace + +# --------------------------------------------------------------- taxonomy + +CODING_TASKS = ("debug", "implement", "refactor", "test", "review") +NONCODING_TASKS = ( + "research", "data-analysis", "office-docs", "creative", "ops", "other", +) +TASKS = CODING_TASKS + NONCODING_TASKS + +TASK_DESCRIPTIONS = { + "debug": "diagnose and fix a failure: error, traceback, crash, regression", + "implement": "add or build a feature, file, endpoint, or capability", + "refactor": "restructure existing code without changing behaviour", + "test": "write or run tests, coverage, fixtures, CI test jobs", + "review": "read-only audit, code review, diff inspection, critique", + "research": "gather external information: web, papers, docs, comparison", + "data-analysis": "analyse data: metrics, SQL, CSV, statistics, dashboards", + "office-docs": "produce office-suite deliverables: .xlsx spreadsheets, " + ".pptx decks, .docx documents (NOT markdown docs or code docs)", + "creative": "creative writing or design: prose, lyrics, branding, visuals", + "ops": "deploy, configure, or operate infrastructure and services", + "other": "none of the above, or too ambiguous to call", +} + +# ------------------------------------------------------------- heuristics + +# Keyword rules over the first task message. Weight = how diagnostic the +# term is; a single strong term ("traceback") should beat several weak ones. +_KEYWORDS: dict[str, list[tuple[str, int]]] = { + "debug": [ + (r"\btraceback\b", 4), (r"\bstack ?trace\b", 4), + (r"\bsegfault\b", 4), (r"\bpanic\b", 3), (r"\bexception\b", 3), + (r"\bregression\b", 3), (r"\bcrash(es|ed|ing)?\b", 3), + (r"\bbug\b", 3), (r"\bbroken\b", 3), (r"\bfail(s|ed|ing)?\b", 2), + (r"\berror\b", 2), (r"\bfix\b", 2), (r"\bwhy (does|is|did|do)\b", 2), + (r"\bdebug\b", 3), (r"\bhang(s|ing)?\b", 2), (r"\btimeout\b", 2), + ], + "test": [ + (r"\bpytest\b", 4), (r"\bunittest\b", 4), (r"\bcoverage\b", 3), + (r"\bfixture\b", 3), (r"\bmock(s|ed|ing)?\b", 2), + (r"\bassert(ion)?s?\b", 2), (r"\be2e\b", 3), + (r"\bintegration test", 3), (r"\btest suite\b", 3), + (r"\bwrite (a |the )?tests?\b", 3), (r"\btests?\b", 1), + ], + "refactor": [ + (r"\brefactor\b", 5), (r"\brestructur", 4), (r"\brename\b", 3), + (r"\bextract (a |the )?(function|method|class|module)", 4), + (r"\bdedup", 3), (r"\bsimplif", 3), (r"\bclean ?up\b", 2), + (r"\bmove .{0,30} into\b", 2), (r"\bsplit .{0,30} into\b", 2), + (r"\btidy\b", 2), (r"\bconsolidat", 3), + ], + "review": [ + (r"\bcode review\b", 5), (r"\breview\b", 3), (r"\baudit\b", 4), + (r"\binspect\b", 3), (r"\bcritique\b", 3), (r"\bpull request\b", 3), + (r"\bdiff\b", 2), (r"\bcheck .{0,25} for\b", 2), + (r"\bread[- ]only\b", 3), (r"\bassess\b", 2), + ], + "implement": [ + (r"\bimplement\b", 4), (r"\badd (a |an |the )?", 2), + (r"\bbuild\b", 2), (r"\bcreate\b", 2), (r"\bwrite (a |an |the )?", 2), + (r"\bfeature\b", 3), (r"\bsupport for\b", 3), (r"\bintroduce\b", 2), + (r"\bwire up\b", 3), (r"\bhook up\b", 3), (r"\bendpoint\b", 2), + (r"\bmake .{0,25} work\b", 2), (r"\bscaffold\b", 3), + ], + "research": [ + (r"\bresearch\b", 4), (r"\binvestigate\b", 3), (r"\bcompare\b", 3), + (r"\bsurvey\b", 3), (r"\barxiv\b", 5), (r"\bpapers?\b", 3), + (r"\bliterature\b", 4), (r"\bfind out\b", 3), (r"\blook up\b", 3), + (r"\bwhat is\b", 2), (r"\bhow does\b", 2), (r"\bexplain\b", 2), + (r"\bsources?\b", 2), (r"\bstate of the art\b", 4), + ], + "data-analysis": [ + (r"\banaly[sz]e\b", 3), (r"\banalysis\b", 3), (r"\bmetrics?\b", 3), + (r"\bstatistic", 3), (r"\bcsv\b", 4), (r"\bsql\b", 4), + (r"\bdashboard\b", 4), (r"\bdataset\b", 3), + (r"\baggregat", 3), (r"\bpivot\b", 4), (r"\bcorrelat", 3), + ], + "office-docs": [ + (r"\bexcel\b", 5), (r"\bxlsx\b", 5), (r"\bspreadsheet\b", 5), + (r"\bpowerpoint\b", 5), (r"\bpptx\b", 5), (r"\bslide deck\b", 5), + (r"\bpresentation\b", 4), + (r"\bdocx\b", 5), (r"\bword document\b", 5), (r"\boutlook\b", 4), + (r"\bcalendar\b", 3), (r"\binvoice\b", 3), + ], + "creative": [ + (r"\bpoem\b", 5), (r"\bstory\b", 4), (r"\bsong\b", 5), + (r"\blyrics\b", 5), (r"\bblog post\b", 4), (r"\bessay\b", 4), + (r"\bcreative\b", 4), (r"\blogo\b", 4), (r"\billustration\b", 4), + (r"\binfographic\b", 4), (r"\bbrand(ing)?\b", 3), + (r"\bnarrative\b", 3), (r"\bdesign a\b", 2), + ], + "ops": [ + (r"\bdeploy(ment)?\b", 4), (r"\bdocker\b", 4), (r"\bkubernetes\b", 5), + (r"\bk8s\b", 5), (r"\bterraform\b", 5), (r"\bci/cd\b", 4), + (r"\binfra(structure)?\b", 4), + (r"\bnginx\b", 4), (r"\bsystemd\b", 4), + (r"\bmigrat", 2), (r"\bbackup\b", 3), + (r"\bmonitor(ing)?\b", 2), (r"\balert(s|ing)?\b", 2), + ], +} + +_COMPILED: dict[str, list[tuple[re.Pattern[str], int]]] = { + task: [(re.compile(pat, re.IGNORECASE), w) for pat, w in rules] + for task, rules in _KEYWORDS.items() +} + +# Tool-signature boosts. The corpus is coding-heavy, so tool evidence is +# what separates the coding sub-buckets from each other. +_EDIT_TOOLS = {"patch", "edit", "write_file", "write", "apply_patch", + "str_replace_editor", "multiedit", "notebookedit"} +_WEB_TOOLS = {"web_search", "web_extract", "webfetch", "websearch", "fetch", + "browser", "browser_exec"} +_SHELL_TOOLS = {"exec", "bash", "terminal", "exec_command", "powershell", + "shell", "run_command"} +_READ_TOOLS = {"read_file", "read", "search_files", "grep", "glob", + "list_dir", "ls"} +_TEST_RUNNER_RE = re.compile( + r"\b(pytest|py\.test|npm (run )?test|yarn test|pnpm test|jest|vitest|" + r"go test|cargo test|mvn test|gradle test|ctest|tox|nox)\b", re.IGNORECASE) +_OFFICE_TOOL_RE = re.compile( + r"\b(xlsx|pptx|docx|openpyxl|python-pptx|python-docx|excel|powerpoint)\b", + re.IGNORECASE) + +# Splice/reader scaffolding prepended to the first user row. It names the +# subagent role ("root/whole_branch_review") and would otherwise dominate +# keyword scoring β€” strip it before classifying. +_SCAFFOLD_RE = re.compile( + r"^\s*(?:\[subagent brief[^\]]*\]" + r"|\[trace segment \d+ of \d+[^\]]*\]" + r"|\[continued[^\]]*\])\s*", re.IGNORECASE) + +# German task vocabulary β€” the corpus carries ~8% German traces and the +# English keyword sets score them as "other". +_KEYWORDS_DE: dict[str, list[tuple[str, int]]] = { + "debug": [(r"\bfehler\b", 3), (r"\bbeheben\b", 3), (r"\bdebuggen\b", 4), + (r"\bkaputt\b", 3), (r"\bschlΓ€gt fehl\b", 3), + (r"\bwarum\b", 2), (r"\bproblem\b", 2)], + "test": [(r"\btests?\b", 2), (r"\btestabdeckung\b", 4), + (r"\bprΓΌf(?:en|e|ung)\b", 1)], + "refactor": [(r"\bumbauen\b", 4), (r"\brefaktor", 5), + (r"\bvereinfach", 3), (r"\baufrΓ€um", 3)], + "review": [(r"\bprΓΌf(?:en|e|ung)\b", 3), (r"\breview\b", 3), + (r"\baudit\b", 4), (r"\bbewert", 3)], + "implement": [(r"\beinbauen\b", 4), (r"\bimplementier", 4), + (r"\berstell", 3), (r"\bhinzufΓΌg", 3), (r"\bumsetzen\b", 3)], + "research": [(r"\brecherch", 4), (r"\bvergleich", 3), + (r"\buntersuch", 3), (r"\banalysier", 2)], + "data-analysis": [(r"\banalysier", 3), (r"\bauswert", 4), + (r"\bkennzahlen\b", 4), (r"\bstatistik\b", 3)], + "office-docs": [(r"\bexcel\b", 5), (r"\bprΓ€sentation\b", 5), + (r"\bfolien\b", 4), (r"\bbericht\b", 2), + (r"\btabelle\b", 3)], + "creative": [(r"\bgedicht\b", 5), (r"\bgeschichte\b", 4), + (r"\btexten\b", 3), (r"\bgestalt", 3)], + "ops": [(r"\bdeploy", 4), (r"\bbereitstell", 3), (r"\binfrastruktur\b", 4), + (r"\bserver\b", 2), (r"\bkonfigurier", 3)], +} +_COMPILED_DE: dict[str, list[tuple[re.Pattern[str], int]]] = { + task: [(re.compile(pat, re.IGNORECASE), w) for pat, w in rules] + for task, rules in _KEYWORDS_DE.items() +} + +# --- language detection ------------------------------------------------- +# The corpus is EN/DE only, so function-word density beats a real detector +# here: no model call, no dependency, and it is stable on short task texts. +# The two sets are deliberately disjoint ("war" is dropped: it is a common +# English noun and a common German verb, so it carries no signal). +_DE_WORDS = frozenset( + "der die das den dem des und oder nicht kein keine ist sind wird werden " + "wurde wurden bitte fΓΌr mit auf von zu im ein eine einen einem einer auch " + "noch schon aber wenn dass wie wir ich du sie es hat haben kann kΓΆnnen " + "muss mΓΌssen soll sollen nach ΓΌber unter zwischen durch bei aus als nur " + "mehr sehr jetzt dann hier dort diese dieser dieses diesem sowie bereits " + "jeweils".split() +) +_EN_WORDS = frozenset( + "the and or not is are be been to of for with from by at this that these " + "those it its we you they he she i have has had can could should would " + "must may do does did but if then than only more very now here there what " + "which who how when where why all any each every some no yes will was were " + "so in on as an a".split() +) +_WORD_RE = re.compile(r"[A-Za-zΓ„Γ–ΓœΓ€ΓΆΓΌΓŸ]+") +_UMLAUT_RE = re.compile(r"[Àâüß]", re.IGNORECASE) + + +def language_scores(text: str) -> tuple[int, int]: + """(de, en) function-word counts for a text, umlaut bonus included. + + Public so the structural validator can require a CONFIDENT language + call (a margin and a ratio) instead of detect_language's tie-break. + """ + words = _WORD_RE.findall(text.lower()) + de = sum(1 for w in words if w in _DE_WORDS) + en = sum(1 for w in words if w in _EN_WORDS) + if _UMLAUT_RE.search(text): + de += 2 # strong signal that survives very short texts + return de, en + + +def detect_language(text: str) -> str: + """Return 'de' or 'en' for a task text. Defaults to 'en' on a tie.""" + if not text: + return "en" + de, en = language_scores(text) + return "de" if de > en else "en" + + +LANGUAGE_NAMES = {"en": "English", "de": "German"} + + +def _tool_names(t: Trace) -> set[str]: + names: set[str] = set() + for m in t.messages: + for tc in m.tool_calls: + if tc.name: + names.add(tc.name.lower()) + return names + + +def _tool_args_blob(t: Trace, limit: int = 20000) -> str: + parts: list[str] = [] + total = 0 + for m in t.messages: + for tc in m.tool_calls: + a = tc.arguments or "" + parts.append(a) + total += len(a) + if total > limit: + return " ".join(parts) + return " ".join(parts) + + +def heuristic_scores(t: Trace) -> dict[str, float]: + """Raw per-task scores from keywords + tool signature.""" + scores = {task: 0.0 for task in TASKS} + raw = (t.first_task_message() or "")[:6000] + text = _SCAFFOLD_RE.sub("", raw) + for rules in (_COMPILED, _COMPILED_DE): + for task, task_rules in rules.items(): + for rx, weight in task_rules: + hits = len(rx.findall(text)) + if hits: + # diminishing returns: one strong term beats a keyword soup + scores[task] += weight * (1 + 0.25 * min(hits - 1, 4)) + + tools = _tool_names(t) + if tools & _EDIT_TOOLS: + scores["implement"] += 3 + scores["refactor"] += 1.5 + if tools & _WEB_TOOLS: + scores["research"] += 4 + if tools & _SHELL_TOOLS: + scores["ops"] += 1 + scores["debug"] += 1 + if tools and not (tools & _EDIT_TOOLS) and (tools & _READ_TOOLS): + scores["review"] += 2 + if _TEST_RUNNER_RE.search(_tool_args_blob(t)): + scores["test"] += 4 + if _OFFICE_TOOL_RE.search(_tool_args_blob(t)): + scores["office-docs"] += 5 + return scores + + +def classify_heuristic(t: Trace) -> dict[str, Any]: + """Deterministic label + confidence + the evidence that produced it.""" + scores = heuristic_scores(t) + ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True) + top_task, top = ranked[0] + second = ranked[1][1] if len(ranked) > 1 else 0.0 + if top <= 0: + return {"task": "other", "confidence": 0.0, "scores": scores, + "method": "heuristic"} + if top == second: + # a tie is not a label β€” hand it to the LLM tier or "other" + return {"task": "other", "confidence": 0.0, "scores": scores, + "method": "heuristic", "tie": True} + # confidence: share of the top two, so a clear winner scores near 1.0 + confidence = (top - second) / top + return {"task": top_task, "confidence": round(confidence, 3), + "scores": scores, "method": "heuristic"} + + +# ------------------------------------------------------------- LLM tier + +_CLASSIFY_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "task": {"type": "string", "enum": list(TASKS)}, + "reason": {"type": "string"}, + }, + "required": ["task"], +} + + +def _classify_prompt(t: Trace) -> str: + task_text = (t.first_task_message() or "")[:2500] + tools = sorted(_tool_names(t)) + # a few assistant prose lines carry the intent when the user message is + # terse ("continue", "go on") + prose = [ + (m.content or "").strip()[:200] + for m in t.messages + if m.role == "assistant" and (m.content or "").strip() + ][:4] + taxonomy = "\n".join(f"- {k}: {v}" for k, v in TASK_DESCRIPTIONS.items()) + return ( + "Classify this agent session into exactly one task bucket.\n\n" + f"Buckets:\n{taxonomy}\n\n" + f"Tools used: {', '.join(tools) or '(none)'}\n\n" + f"User task:\n{task_text}\n\n" + f"Assistant prose samples:\n" + "\n".join(f"- {p}" for p in prose) + + "\n\nAnswer with the single best bucket and a one-line reason." + ) + + +def classify_llm(t: Trace, cfg) -> Optional[dict[str, Any]]: + """One structured call. Returns None when the endpoint is unavailable.""" + from traceforge import triage + + try: + base, key, model = triage._resolve_endpoint(cfg) + except Exception: + return None + payload = { + "model": model, + "messages": [ + {"role": "system", + "content": "You label agent sessions. Answer with JSON only."}, + {"role": "user", "content": _classify_prompt(t)}, + ], + "temperature": 0.0, + "max_tokens": 200, + "response_format": { + "type": "json_schema", + "json_schema": {"name": "task_label", + "schema": _CLASSIFY_SCHEMA, "strict": True}, + }, + } + try: + data = triage._post_json( + triage._chat_url(base), payload, key, cfg.triage_timeout_s) + raw = triage._extract_content(data) + obj = json.loads(raw) if raw else None + except Exception: + return None + if not isinstance(obj, dict) or obj.get("task") not in TASKS: + return None + return {"task": obj["task"], "confidence": 1.0, + "reason": obj.get("reason", ""), "method": "llm"} + + +def classify_trace( + t: Trace, cfg=None, *, use_llm: bool = True, min_confidence: float = 0.35, +) -> dict[str, Any]: + """Label one trace. + + LLM first when an endpoint is configured (it reads intent, which the + keyword prior cannot); the heuristic is the fallback and is always + computed so the result carries the evidence for review. + """ + heuristic = classify_heuristic(t) + if use_llm and cfg is not None: + llm = classify_llm(t, cfg) + if llm: + llm["heuristic"] = heuristic + return llm + return heuristic diff --git a/traceforge/cli.py b/traceforge/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..1f1cd849ceb00aa3b4cbbc27cab6a5140e8ca0ed --- /dev/null +++ b/traceforge/cli.py @@ -0,0 +1,2757 @@ +"""traceforge CLI β€” read, benchmark, improve, validate, export. + +The pipeline is an evaluation instrument: it rewrites a model's real traces +into ideal-world references, and `benchmark` scores the model by how little +that rewrite had to change. + +Layout under repo: + data/traces/raw/ normalized model traces (the benchmark's input) + data/traces/improved/ ideal-world references (the benchmark's other half) + data/triage/ per-message labels (keep/drop/rephrase/...) + data/benchmark/ report.json, traces.jsonl, leaderboard.jsonl + data/prompts/ improver prompts (audit trail) + data/work/ per-trace improver workdirs (scratch) + data/manifest.jsonl one status row per improved trace + data/dataset/*.jsonl exported JSONL (downstream reuse only) +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sqlite3 +import sys +import time +import types +from typing import Optional + +import traceforge +from traceforge import live +from traceforge.config import Config, repo_root_from_pkg +from traceforge.defects import classify +from traceforge.export import export_chat_jsonl +from traceforge.improver.harness import _append_status +from traceforge.readers import ( + ClaudeTraceReader, + CodexTraceReader, + HermesTraceReader, + NemotronTraceReader, +) +from traceforge.trace import Trace +from traceforge.validate import validate_structural + + +def _cfg(args) -> Config: + return Config.load(getattr(args, "config", None), repo_root=repo_root_from_pkg()) + + +SOURCES = ("hermes", "claude", "codex", "nemotron") + + +def _parse_sources(args) -> list[str]: + raw = getattr(args, "source", None) or "all" + if raw == "all": + return list(SOURCES) + parts = [s.strip().lower() for s in raw.split(",") if s.strip()] + bad = [s for s in parts if s not in SOURCES] + if bad: + raise SystemExit(f"error: unknown source(s): {bad} (known: {', '.join(SOURCES)})") + return parts + + +class MultiSourceReader: + """Unified view over every installed agent store. + + load_trace prefers the raw cache (what `pull` wrote) so improve is + reproducible and independent of live stores; unknown ids fall back to + the live readers β€” hermes first (cheap sqlite lookup), then claude + (JSONL index scan). + """ + + def __init__(self, cfg: Config): + self.cfg = cfg + self._hermes: HermesTraceReader | None = None + self._claude: ClaudeTraceReader | None = None + self._codex: CodexTraceReader | None = None + self._nemotron: NemotronTraceReader | None = None + self._child_ids: set[str] | None = None + + @property + def hermes(self) -> HermesTraceReader | None: + if self._hermes is None: + dbs = [self.cfg.state_db] + list(self.cfg.hermes_state_dbs) + if any(os.path.exists(d) for d in dbs): + self._hermes = HermesTraceReader(dbs) + return self._hermes + + @property + def claude(self) -> ClaudeTraceReader | None: + if self._claude is None: + dirs = [os.path.expanduser("~/.claude/projects")] + list( + self.cfg.claude_projects_dirs + ) + if any(os.path.isdir(d) for d in dirs): + self._claude = ClaudeTraceReader( + dirs, include_sidechains=self.cfg.claude_sidechains + ) + return self._claude + + @property + def codex(self) -> CodexTraceReader | None: + if self._codex is None: + # live store ALWAYS first (owner rule: "live store is always + # read when present"); extra roots add seed archives + roots = [os.path.expanduser("~/.codex/sessions")] + for r in self.cfg.codex_roots: + if r not in roots: + roots.append(r) + if any(os.path.isdir(r) for r in roots): + self._codex = CodexTraceReader(roots) + return self._codex + + @property + def nemotron(self) -> NemotronTraceReader | None: + """Published SFT corpora (scripts/fetch_nemotron.py output).""" + if self._nemotron is None: + roots = [r for r in self.cfg.nemotron_roots if os.path.isdir(r)] + if roots: + self._nemotron = NemotronTraceReader(roots) + return self._nemotron + + def reader_for(self, source: str): + if source == "hermes": + r = self.hermes + if r is None: + raise FileNotFoundError( + f"Hermes state.db not found at {self.cfg.state_db}" + ) + return r + if source == "claude": + r = self.claude + if r is None: + raise FileNotFoundError( + "Claude projects dir not found: ~/.claude/projects" + ) + return r + if source == "codex": + r = self.codex + if r is None: + raise FileNotFoundError( + f"Codex sessions dir(s) not found: {self.cfg.codex_roots}" + ) + return r + if source == "nemotron": + r = self.nemotron + if r is None: + raise FileNotFoundError( + "Nemotron seed dir not found: " + f"{self.cfg.nemotron_roots or 'seed-datasets/nemotron'} " + "(run scripts/fetch_nemotron.py)" + ) + return r + raise KeyError(source) + + # ------------------------------------------------------------ listing + + def list_sessions(self, *, sources=None, **kw) -> list[dict]: + sources = sources or SOURCES + rows: list[dict] = [] + guard_kw = dict(kw) + excl_kw = guard_kw.pop("exclude_improver_children", None) + for src in sources: + try: + r = self.reader_for(src) + except FileNotFoundError as e: + print(f"-- source '{src}' unavailable: {e}") + continue + kwd = dict(guard_kw) + # Self-reference guard at the SQL level: HermesTraceReader + # drops rows whose cwd sits inside the improver workdir β€” + # mis-tagged improver children otherwise look like real + # candidates and improve the model on its own output. + if src == "hermes" and (excl_kw is None or excl_kw): + kwd.setdefault("exclude_cwd", self.cfg.improver_workdir) + # seed-archive discovery can take a while on first run; + # heartbeat so long parses don't look hung (only readers + # that accept status_cb β€” codex's parallel discovery) + import inspect + + if "status_cb" in inspect.signature( + r.list_sessions + ).parameters: + def _status(msg, _src=src): + print(f"-- {_src} {msg}", flush=True) + kwd.setdefault("status_cb", _status) + got = r.list_sessions(**kwd) + for s in got: + s["_reader"] = src # serving reader (s['source'] = provenance) + rows.extend(got) + rows.sort(key=lambda s: s["started_at"] or 0.0, reverse=True) + if kw.get("limit") is not None: + rows = rows[: kw["limit"]] + return rows + + def sources_of(self, sid: str) -> list[str]: + """Which stores can serve this session id (raw cache excluded).""" + out = [] + if self.hermes is not None: + try: + self.hermes.load_trace(sid) + out.append("hermes") + except KeyError: + pass + if self.claude is not None and sid in self.claude._index(): + out.append("claude") + if self.codex is not None and sid in self.codex._index(): + out.append("codex") + if self.nemotron is not None: + try: + self.nemotron.load_trace(sid) + out.append("nemotron") + except KeyError: + pass + return out + + # ------------------------------------------------------------- loading + + def load_trace(self, sid: str) -> Trace: + cached = os.path.join(self.cfg.raw_dir, f"{sid}.json") + if os.path.exists(cached): + return Trace.load(cached) + # live stores: hermes first (cheap sqlite lookup) + if self.hermes is not None: + try: + return self.hermes.load_trace(sid) + except KeyError: + pass + if self.claude is not None: + try: + return self.claude.load_trace(sid) + except KeyError: + pass + if self.codex is not None: + try: + return self.codex.load_trace(sid) + except KeyError: + pass + if self.nemotron is not None: + try: + return self.nemotron.load_trace(sid) + except KeyError: + pass + raise KeyError(f"session not found in any source: {sid}") + + # --------------------------------------------------- self-reference guard + + def improver_child_ids(self) -> set[str]: + """Hermes session ids that traceforge's own pipeline spawned. + + Every improver stage (writer, digest, map, verify, review) is a + Hermes one-shot running with cwd inside cfg.improver_workdir. + Improving such a trace trains the model on its own output β€” an + endless self-referential loop β€” so these ids are non-candidates + everywhere: listing, pulling, and improving. + """ + if self._child_ids is not None: + return self._child_ids + ids: set[str] = set() + wd = (self.cfg.improver_workdir or "").rstrip("/\\") + like = wd.replace("\\", "\\\\").replace("%", "\\%").replace( + "_", "\\_" + ) + "/%" + for db in [self.cfg.state_db] + list(self.cfg.hermes_state_dbs): + if not os.path.exists(db): + continue + try: + conn = sqlite3.connect( + f"file:{db}?mode=ro", uri=True + ) + try: + ids |= { + row[0] + for row in conn.execute( + "SELECT id FROM sessions WHERE cwd LIKE ?" + " ESCAPE '\\'", + (like,), + ) + } + finally: + conn.close() + except Exception: + pass + self._child_ids = ids + return ids + + def is_improver_child(self, sid: str, trace: Optional[Trace] = None) -> bool: + """True when sid is one of our improver's own Hermes sessions + (cwd-based, the only signal that survives mis-tagged sources).""" + wd = (self.cfg.improver_workdir or "").rstrip("/\\") + if wd and trace is not None and (trace.cwd or "").startswith(wd + os.sep): + return True + return sid in self.improver_child_ids() + + +def _print_sessions(sessions) -> None: + for s in sessions: + started = time.strftime("%Y-%m-%d %H:%M", time.gmtime(s["started_at"])) if s["started_at"] else "?" + title = (s["title"] or "")[:52] + print( + f"{s['session_id']:38} {s.get('source', '?'):7} msgs={s['message_count']:4} " + f"tools={s['tool_call_count']:4} {started} {title}" + ) + print(f"-- {len(sessions)} candidate traces") + + +def _pull_sessions(cfg, r, sessions) -> int: + """Pull the listed candidate sessions into data/traces/raw/. + + Over-long traces (rendered size > cfg.split_max_chars) are split + into trainable segment traces #p1..N at assistant-turn + boundaries (callβ†’result groups never torn apart; the task row is + prepended to every segment) β€” one giant 500k-token training + sequence becomes several ≀~120k-token ones, each flowing through + the full quality pipeline on its own. + """ + os.makedirs(cfg.raw_dir, exist_ok=True) + from traceforge.progress import Progress + from traceforge.splitter import needs_split, save_segments, split_trace + + n = 0 + # A split session writes #p1..N.json and NEVER .json, so the + # plain existence check below re-pulled and re-split every split + # session on every pass β€” measured 2026-10-23: ~2,000 sessions + # (including 480KB monsters) re-parsed and re-split each pass, ~15 min + # of dead time before the improve stage could start. Index the segment + # bases once instead of scanning the raw dir per session. + existing = set(os.listdir(cfg.raw_dir)) + seg_bases = {f.split("#p")[0] for f in existing if "#p" in f} + with Progress(len(sessions), desc="pull:") as prog: + for s in sessions: + sid = s["session_id"] + out = os.path.join(cfg.raw_dir, f"{sid}.json") + if not cfg.overwrite and (f"{sid}.json" in existing + or sid in seg_bases): + prog.write(f"skip {sid} (already pulled)") + prog.update() + continue + t = r.reader_for(s["_reader"]).load_trace(sid) + trigger = cfg.split_trigger_chars or cfg.split_max_chars + if cfg.split_long and needs_split(t, split_max_chars=trigger): + segs = split_trace( + t, + split_max_chars=cfg.split_max_chars, + min_segment_chars=cfg.min_segment_chars, + max_segments=cfg.max_segments_per_trace, + ) + sids = save_segments(segs, cfg.raw_dir, s["session_id"]) + n += len(sids) + prog.write( + f"split {s['source']:7} {s['session_id']} -> " + f"{len(segs)} segments ({', '.join(x.split('#')[-1] for x in sids)})" + ) + else: + t.save(out) + n += 1 + prog.write(f"pulled {s['source']:7} {s['session_id']} -> data/traces/raw/") + prog.update() + print(f"-- {n} traces pulled ({len(sessions)} sessions)") + return 0 + + +def _pick(args, cfg, name: str): + """CLI flag wins, else the config value. + + The argparse defaults used to be 10/120/2, which shadowed + config.example.yaml at every call site β€” so `min_messages`, + `max_messages` and `min_tool_calls` in the YAML were dead settings. + Defaults are now None and the config is the fallback. + """ + v = getattr(args, name, None) + return v if v is not None else getattr(cfg, name) + + +def cmd_list(args) -> int: + cfg = _cfg(args) + r = MultiSourceReader(cfg) + sessions = r.list_sessions( + sources=_parse_sources(args), + min_messages=_pick(args, cfg, "min_messages"), + max_messages=_pick(args, cfg, "max_messages"), + min_tool_calls=_pick(args, cfg, "min_tool_calls"), + limit=args.limit, + ) + _print_sessions(sessions) + return 0 + + +def cmd_pull(args) -> int: + cfg = _cfg(args) + r = MultiSourceReader(cfg) + sessions = r.list_sessions( + sources=_parse_sources(args), + min_messages=_pick(args, cfg, "min_messages"), + max_messages=_pick(args, cfg, "max_messages"), + min_tool_calls=_pick(args, cfg, "min_tool_calls"), + limit=args.limit, + ) + return _pull_sessions(cfg, r, sessions) + + +def _crash_result(sid: str, exc: Exception): + """Manifest-shaped status row for a worker crash (never loses a batch).""" + return { + "session_id": sid, "ok": False, + "errors": [f"worker crashed: {exc}"], + "hermes_session_id": None, "attempts": 0, "duration_s": 0.0, + } + + +_MANIFEST_LOCK = __import__("threading").Lock() + + +def _write_manifest_row(cfg, row: dict) -> None: + """Append ONE outcome row the moment the attempt finishes. + + The batch used to buffer every row and flush at the END of the run + (see _improve_ids), so a batch that ran for hours β€” or was killed β€” + left the manifest with stage rows but almost no outcomes: measured + 2026-10-23, 4628 manifest rows and only 54 carried ok/errors/attempts + against 3633 attempted traces. Absence of a failure row therefore did + NOT mean no failure, and the funnel's failure census was blind. + """ + try: + os.makedirs(os.path.dirname(cfg.manifest_path), exist_ok=True) + with _MANIFEST_LOCK: + with open(cfg.manifest_path, "a", encoding="utf-8") as f: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + except OSError: + pass # bookkeeping must never kill the batch + + +def _improve_one(cfg, r, sid, max_attempts, skip_existing, prog, liv=None): + """Improve one trace; returns (sid, ImproverResult). Thread-safe.""" + if skip_existing and os.path.exists( + os.path.join(cfg.improved_dir, f"{sid}.json") + ): + prog.write(f"SKIP {sid} (already improved)") + return sid, None + if liv is not None: + liv.registry.begin(sid) + t = r.load_trace(sid) + from traceforge.improver import improve_trace + + def cb(s: str) -> None: + if liv is not None: + liv.registry.set_stage(sid, s) + else: + prog.set_postfix(f"{sid[:12]}… {s}") + + try: + if cfg.splice: + # windowed direct-HTTP engine (no agent runtime) + from traceforge.improver import ImproverResult + from traceforge.splice import improve_trace_splice + + sres = improve_trace_splice( + cfg, t, status_cb=cb, max_attempts=max_attempts, + ) + res = ImproverResult( + session_id=sid, ok=sres["ok"], improved=sres["improved"], + reply=sres.get("route", ""), errors=sres["errors"], + attempts=sres["attempts"], + duration_s=sres["duration_s"], + ) + if res.ok and res.improved is not None \ + and sres.get("route") != "regate": + out = os.path.join(cfg.improved_dir, f"{sid}.json") + res.improved.save(out) + prog.write( + f"IMPROVED {sid} ({res.attempts} attempts, " + f"{res.duration_s:.0f}s, {res.reply})" + ) + elif res.ok: + prog.write( + f"IMPROVED {sid} (regate, {res.duration_s:.0f}s)" + ) + else: + prog.write(f"FAILED {sid}: {res.errors}") + _write_manifest_row(cfg, res.to_dict()) + if liv is not None: + liv.registry.end(sid) + return sid, res + res = improve_trace( + cfg, t, max_attempts=max_attempts, status_cb=cb + ) + except Exception as e: + if liv is not None: + liv.registry.end(sid) + _write_manifest_row(cfg, _crash_result(sid, e)) + raise + if liv is not None: + liv.registry.end(sid) + toks = "" + if liv is not None: + d = liv.per_trace(sid) + if d and (d.get("in") or d.get("out")): + toks = f" Β· {live.fmt_tok(d['in'])} in Β· {live.fmt_tok(d['out'])} out" + if d.get("rate"): + toks += f" Β· {d['rate']:.1f} tok/s" + if res.ok and res.improved is not None: + out = os.path.join(cfg.improved_dir, f"{sid}.json") + res.improved.save(out) + prog.write( + f"IMPROVED {sid} ({res.attempts} attempts, {res.duration_s:.0f}s" + f"{toks})" + ) + else: + prog.write(f"FAILED {sid}{toks}: {res.errors}") + _write_manifest_row(cfg, res.to_dict()) + return sid, res + + +def cmd_improve(args) -> int: + cfg = _cfg(args) + if getattr(args, "no_triage", False): + cfg.triage = False + r = MultiSourceReader(cfg) + ids = args.session_ids or [] + if args.from_raw: + ids = [ + f[:-5] + for f in os.listdir(cfg.raw_dir) + if f.endswith(".json") + ] + # Self-reference guard: improver children never get improved, even + # when a pre-guard run cached them into data/traces/raw/. + child_ids = r.improver_child_ids() + if child_ids: + drop = [i for i in ids if i in child_ids] + if drop: + ids = [i for i in ids if i not in child_ids] + print( + f"-- skipped {len(drop)} traceforge improver session(s) " + f"in input (self-referential data): " + + ", ".join(drop[:5]) + (" …" if len(drop) > 5 else "") + ) + if not ids: + print("no traces to improve β€” pass session ids, or use --from-raw") + return 2 + if not ids: + print("no traces to improve β€” pass session ids, or use --from-raw") + return 2 + ids = _filter_improver_children_live(cfg, r, ids) + if not ids: + print("-- no improvable traces left after self-reference guard") + return 0 + return _improve_ids( + cfg, r, ids, + max_attempts=args.max_attempts, + workers=args.workers, + skip_existing=args.skip_existing, + ) + + +def _raw_cwd(path: str) -> Optional[str]: + """Read only the top-level `cwd` from a raw trace file. + + Trace.to_dict writes cwd as the 5th key, so it lands in the first few + hundred bytes. Parsing the whole file (up to 480KB x 12.7k traces) + just to read that one field cost ~15 min of dead time at the start of + every pass. Returns None when the head does not carry it, so the + caller falls back to a full load. + """ + try: + with open(path, "r", encoding="utf-8") as f: + head = f.read(4096) + except OSError: + return None + m = re.search(r'"cwd"\s*:\s*"((?:[^"\\]|\\.)*)"', head) + if not m: + return None + try: + return json.loads('"' + m.group(1) + '"') + except Exception: + return m.group(1) + + +def _filter_improver_children_live(cfg, r, ids) -> list[str]: + """Drop ids that a currently-running batch (or its state.db rows) + marked as improver children since the reader snapshot was taken, and + drop raw-cached traces whose own Trace.cwd sits in the workdir.""" + wd = (cfg.improver_workdir or "").rstrip("/\\") + child_ids = r.improver_child_ids() + cand = [sid for sid in ids if sid not in child_ids] + if not wd: + return cand + # Resolve cwd from the raw head in parallel. This runs at the start of + # every pass over the whole pending set; 36k sequential 4KB opens cost + # 25+ min on Windows with real-time AV scanning, and threads cut it to + # about a minute. Only the head read is threaded: the live-store + # readers share sqlite handles and are not safe to call concurrently. + from concurrent.futures import ThreadPoolExecutor + + def _head_cwd(sid): + return _raw_cwd(os.path.join(cfg.raw_dir, f"{sid}.json")) + + with ThreadPoolExecutor(max_workers=32) as ex: + heads = list(ex.map(_head_cwd, cand)) + out = [] + for sid, cwd in zip(cand, heads): + if cwd is None: + try: + cwd = r.load_trace(sid).cwd or "" + except Exception: + out.append(sid) # unknown store rows: let downstream report + continue + if cwd.startswith(wd + os.sep) or cwd.rstrip("/\\") == wd: + continue + out.append(sid) + return out + + +def cmd_regate(args) -> int: + """Adjudicate existing workdir artifacts with the windowed judge.""" + from concurrent.futures import ThreadPoolExecutor, as_completed + + cfg = _cfg(args) + if not cfg.splice: + print("-- regate requires the splice engine (splice: true)") + return 2 + work_dir = cfg.improver_workdir + if not os.path.isdir(work_dir): + print("no workdir β€” nothing to regate") + return 1 + ids = args.session_ids or [ + d for d in os.listdir(work_dir) + if os.path.exists(os.path.join(work_dir, d, "improved.json")) + ] + promoted = {f[:-5] for f in os.listdir(cfg.improved_dir) + if f.endswith(".json")} if os.path.isdir(cfg.improved_dir) else set() + ids = [i for i in ids if i not in promoted] + if not ids: + print("no unpromoted artifacts to regate") + return 0 + from traceforge.progress import Progress + from traceforge.splice import judge_artifact + + ok = 0 + workers = max(1, args.workers) + print(f"-- regating {len(ids)} artifacts with the windowed judge " + f"({workers} workers)") + with Progress(len(ids), desc="regate:") as prog: + if workers == 1 or len(ids) == 1: + for sid in ids: + res = judge_artifact(cfg, sid) + if res.get("ok"): + ok += 1 + prog.write(f"PROMOTED {sid} ({res['duration_s']:.0f}s" + + (", cached" if res.get("cached") else "") + + ")") + else: + prog.write(f"FAIL {sid}: {res['errors']}") + prog.update() + else: + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = {ex.submit(judge_artifact, cfg, sid): sid + for sid in ids} + for fut in as_completed(futs): + sid = futs[fut] + try: + res = fut.result() + except Exception as e: + prog.write(f"FAIL {sid}: worker crashed: {e}") + res = {"ok": False} + if res.get("ok"): + ok += 1 + prog.write( + f"PROMOTED {sid} ({res['duration_s']:.0f}s" + + (", cached" if res.get("cached") else "") + + ")") + else: + prog.write(f"FAIL {sid}: {res.get('errors')}") + prog.update() + print(f"-- regate done: {ok}/{len(ids)} promoted") + return 0 if ok or not ids else 1 + + +def cmd_crossexam(args) -> int: + """Cross-examine promoted traces with a second judge family.""" + from concurrent.futures import ThreadPoolExecutor, as_completed + + cfg = _cfg(args) + if not cfg.splice: + print("-- crossexam requires the splice engine (splice: true)") + return 2 + model = args.model + if not model: + print("-- cross-exam needs an explicit --model from a DIFFERENT " + "family than the improver (e.g. gemma4-31b-selfhosted when " + "the improver is qwen-fable-selfhosted)") + return 2 + imp_dir = cfg.improved_dir + if not os.path.isdir(imp_dir): + print("no promoted corpus") + return 1 + ids = args.session_ids or sorted( + f[:-5] for f in os.listdir(imp_dir) if f.endswith(".json")) + if not ids: + print("no promoted traces") + return 0 + from traceforge.progress import Progress + from traceforge.splice import cross_examine + + disagree = [] + n_pass = 0 + workers = max(1, args.workers) + print(f"-- cross-examining {len(ids)} promoted traces with " + f"{model} ({workers} workers)") + with Progress(len(ids), desc="crossexam:") as prog: + if workers == 1 or len(ids) == 1: + for sid in ids: + res = cross_examine(cfg, sid, model) + if res.get("ok"): + n_pass += 1 + else: + disagree.append(res) + prog.update() + else: + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = {ex.submit(cross_examine, cfg, sid, model) + : sid for sid in ids} + for fut in as_completed(futs): + sid = futs[fut] + try: + res = fut.result() + except Exception as e: + res = {"ok": False, "sid": sid, + "violations": [{"detail": + f"worker crashed: {e}"}]} + if res.get("ok"): + n_pass += 1 + prog.write(f"CONFIRM {sid}") + else: + disagree.append(res) + vs = res.get("violations") or [{}] + prog.write( + f"DISAGREE {sid}: " + + "; ".join(str(v.get("detail", ""))[:90] + for v in vs[:2])) + prog.update() + print(f"-- crossexam done: {n_pass}/{len(ids)} confirmed, " + f"{len(disagree)} disagreements") + if disagree: + out = os.path.join(cfg.data_dir, "crossexam_disagreements.json") + os.makedirs(cfg.data_dir, exist_ok=True) + with open(out, "w", encoding="utf-8") as f: + json.dump([{ + "sid": d.get("sid"), "verdict": d.get("verdict"), + "answer_quality": d.get("answer_quality"), + "violations": d.get("violations"), + } for d in disagree], f, indent=1, ensure_ascii=False) + print(f"-- disagreements written to {out}") + return 0 + + +def cmd_triage(args) -> int: + """Label traces via the cheap LLM classifier; prints the route.""" + from traceforge import triage as triage_mod + + cfg = _cfg(args) + r = MultiSourceReader(cfg) + ids = args.session_ids or [] + if args.from_raw: + ids = [ + f[:-5] + for f in os.listdir(cfg.raw_dir) + if f.endswith(".json") + ] + if not ids: + print("no traces to triage β€” pass session ids, or use --from-raw") + return 2 + counts = {"splice": 0, "agent": 0, "reject": 0} + routes = {} + from traceforge.progress import Progress + + with Progress(len(ids), desc="triage:") as prog: + for sid in ids: + try: + t = r.load_trace(sid) + except Exception as e: + prog.write(f"ERROR {sid}: {e}") + prog.update() + continue + try: + rec = triage_mod.triage_trace(cfg, t, force=args.force) + except RuntimeError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + if rec is None: + prog.write(f"FAILED {sid}: all windows failed") + prog.update() + continue + route = rec["route"] + routes[sid] = route + counts[route] = counts.get(route, 0) + 1 + nl = len(rec["labels"]) + prog.write( + f"{route.upper():7} {sid} ({nl} labels, " + f"{rec['windows_ok']}/{rec['n_windows']} windows)" + ) + prog.update() + print( + f"-- {len(routes)} triaged: " + + ", ".join(f"{v} {k}" for k, v in counts.items() if v) + ) + return 0 if routes else 1 + + +def _improve_ids(cfg, r, ids, *, max_attempts, workers, skip_existing) -> int: + """Shared improve driver for cmd_improve and cmd_run. + + Queue order (2026-09-28, measured funnel): smallest raw trace first. + Promotion yield is strongly inversely correlated with raw size + (corpus: ≀60KB β‰ˆ 27% promoted, β‰₯300KB β‰ˆ 1%) and a first-queue bias + previously sent 633–733KB monsters ahead of 40KB sure-things β€” a + thin improved/ dir early in a batch is NOT a gate regression; + checking raw sizes is. Reverse to put the high-yield cohort through + the funnel before the expensive low-yield tail. + """ + def _raw_size(sid: str) -> int: + try: + return os.path.getsize(os.path.join(cfg.raw_dir, f"{sid}.json")) + except OSError: + return 1 << 30 # unknown ids (store, not cache) sink to the tail + + # Queue order: NEVER-ATTEMPTED first, then smallest raw first, with + # multi-turn as a TIEBREAKER inside a size (2026-10-24). + # + # Size alone (the 2026-09-28 rule) is right about yield but wrong about + # exploration: pending = raw minus promoted, so every previously-failed + # trace is retried on every pass, and the small ones come first. Measured + # 2026-10-22: 4530 pending, 1360 already attempted (mostly unfixable β€” + # subagent tails with no evidence, degenerate 1-6KB traces), 3170 never + # attempted and sitting behind them in the 180KB+ bands. The batch spent + # hours re-churning known failures at ~0 promotions while 1825 traces of + # 300KB+ had never been looked at once. A workdir is the attempt marker + # (it persists after a failure), so bucket on it and keep size order + # inside each bucket: explore the pool, then retry the failures. + # An infra failure is not an attempt (2026-10-26, suite R-92). + # `judge infra error:` / llm_unavailable is 807 of 2,839 coded failures + # (28.5%) and it arrives in endpoint-outage BURSTS, not as a property of + # the trace: measured 10-26, 191 sessions in the 02h, 397 in the 07h, + # 122 in the 08h, median raw 20 KB. Because a workdir is the attempt + # marker, every one of those sank to the retry tail behind 33,791 + # pending raws, and 0 of the 807 had been promoted 4 h later β€” the + # cheapest cohort in the pool, parked for weeks. Rank it FIRST (0), ahead + # of never-attempted (1) and genuinely-failed (2): it is the cheapest + # cohort in the pool (median 21 KB), its triage cache is already warm so + # the retry skips the pre-pass, and it was lost to an outage rather than + # to trace quality. An outage must cost the pass, not the session. + _att_cache: dict[str, int] = {} + + def _attempted(sid: str) -> int: + if sid in _att_cache: + return _att_cache[sid] + wd = os.path.join(cfg.improver_workdir, sid) + if not os.path.isdir(wd): + rank = 1 + else: + rank = 2 + try: + with open(os.path.join(wd, "failure.json"), + encoding="utf-8") as fh: + rec = json.load(fh) + code = rec.get("defect") or classify( + stage=rec.get("stage"), + errors=rec.get("errors") or [], + violations=rec.get("violations") or []) + # An outage is not a property of the trace, so an infra + # failure is retried ahead of never-attempted work β€” but + # only in a class that can pay. Measured 2026-10-26: the + # infra cohort was 161 nemotron-ka (46% yield) and 496 + # nemotron-tc (14.9%), and rank 0 put all 657 ahead of + # 1,946 never-attempted real sessions (85.6%). The tc + # half alone is ~2 h of 8-worker time for ~74 traces. + # Keep the bonus where the class yields; let the rest + # queue by class like everything else. + if code == "D-INF-01" and _class_rank(sid) <= 2: + rank = 0 + except (OSError, ValueError): + pass + _att_cache[sid] = rank + return rank + + # Class yield, measured 2026-10-26 on the live pool. Size order alone + # put the WORST class at the front, because the nemotron tool_calling + # subset is the smallest thing in the pool: 4,737 raws, 2,383 attempted, + # 355 promoted = 14.9%, against 85.6% for a real captured session + # (1,432 attempted, 1,226 promoted). The queue was spending its front on + # the 14.9% class while the 85.6% class waited behind it. + # + # Rank is the measured yield, best first. Size stays as the tiebreaker + # INSIDE a class, where it is still the right signal. + # A split segment is a slice of a session, so its yield depends on the + # PARENT: a session that already produced one promotable segment is + # material the gates accept, and its other segments are worth trying. + # Measured 2026-10-26: 548 parents produced a promotion, and 2,649 of + # 18,943 pending segments (14%) sit under one of them. The other 86% + # are slices of sessions that have never yielded anything, at ~2.9%. + # Ranking the productive parents first spends the queue on the 14% that + # can pay instead of draining 18,943 segments in file order. + _good_parents = { + sid.split("#p")[0] for sid in + (os.listdir(cfg.improved_dir) if os.path.isdir(cfg.improved_dir) else []) + if "#p" in sid + } + + def _class_rank(sid: str) -> int: + if sid.startswith("nemotron-"): + return 3 if "agentic-tc" in sid else 2 + if "#sa-" in sid: + return 1 # subagent: ~40% of attempted + if "#p" in sid: + # 4 = segment of a session that has already promoted something, + # 5 = segment of a session that never has + return 4 if sid.split("#p")[0] in _good_parents else 5 + return 0 # real captured session: 85.6% + + # Multi-turn was the FIRST key (2026-10-22, owner ask: more multi-turn + # traces). Measured 2026-10-24 it was inverting the whole queue: the + # multi-turn cohort is the long-session cohort, so 52% of pending + # multi-turn traces are 480KB+ β€” the first 50 ids of a random pending + # sample were 86% 480KB+ (measured prom/wh 0.00-0.36) while the 60-180KB + # single-turn cohort (3.0-3.3 prom/wh) sat at the tail. Multi-turn also + # promotes WORSE per hour than single-turn: 13.4% at 620s mean (0.79 + # prom/wh) vs 27.3% at 296s (3.32 prom/wh) over the last 24h. So it is + # now a tiebreaker inside a size, not a queue-wide override: multi-turn + # still wins among equally cheap traces, but it can no longer jump the + # 480KB+ monsters ahead of the high-yield small cohort. + _mt_cache: dict[str, int] = {} + # The key is a TIEBREAKER inside a size bucket, but computing it means + # reading every raw in full: 12,897 files / 4.52 GB, measured ~10 min on + # a cold cache (and it blocks the heartbeat, so the pass looks hung). + # Persist it once; a re-pulled raw keeps its old value, which is fine + # for a tiebreaker. + _mt_path = os.path.join(os.path.dirname(cfg.manifest_path), + ".multi_turn.json") + try: + with open(_mt_path, encoding="utf-8") as fh: + _mt_cache = {str(k): int(v) for k, v in json.load(fh).items()} + except (OSError, ValueError, AttributeError): + _mt_cache = {} + _mt_dirty = [False] + + def _multi_turn(sid: str) -> int: + if sid not in _mt_cache: + try: + with open(os.path.join(cfg.raw_dir, f"{sid}.json"), + encoding="utf-8", errors="replace") as fh: + n = fh.read().count('"role": "user"') + except OSError: + n = 0 + _mt_cache[sid] = 0 if n >= 2 else 1 + _mt_dirty[0] = True + return _mt_cache[sid] + + ids = sorted(ids, key=lambda s: (_attempted(s), _class_rank(s), + _raw_size(s), _multi_turn(s))) + if _mt_dirty[0]: + try: + tmp = _mt_path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(_mt_cache, fh) + os.replace(tmp, _mt_path) + except OSError: + pass + os.makedirs(cfg.improved_dir, exist_ok=True) + manifest_path = cfg.manifest_path + workers = max(1, workers or 1) + + # Cheap triage pass first: label every message keep/drop/rephrase/ + # fact_check/summarize via direct structured-generation LLM calls. + # splice-route traces skip the improver entirely (spliced downstream + # when the splicer lands); reject-route traces drop out here. + # + # PARALLEL (2026-10-23). This used to be a serial loop over every + # pending trace β€” one trace at a time, each parallelising only its own + # windows β€” so the pre-pass was a hard gate in front of the first + # promotion. Measured: 21 triage traces/min with ~5,400 pending after + # the pool doubled = 4-6 h of silent log before any improve work. The + # per-trace work is self-contained (own cache file, own endpoint + # handle, own window pool), so the outer loop parallelises safely; + # triage_prepass_workers bounds it so the endpoint is not swamped. + # The pre-pass is only needed for the AGENT path. improve_trace_splice + # triages each trace itself (stage 1, cached), so pre-triaging the whole + # pool just moves the same cost in front of the first promotion β€” and + # with the pool doubled it became the dominant cost: measured + # 2026-10-23, 5,224 untriaged traces x ~17 windows = ~89k label calls + # = ~16 h of silent log before any improve work. With splice on, triage + # happens inside the workers and promotions start immediately. + triage_routes: dict[str, str] = {} + if cfg.triage and not cfg.splice: + from concurrent.futures import ThreadPoolExecutor + + from traceforge import triage as triage_mod + + def _triage_one(sid: str) -> tuple[str, str, Optional[str]]: + """(sid, route, fatal_error) β€” never raises.""" + try: + t = r.load_trace(sid) + except Exception: + return sid, "agent", None # unresolvable: default route + try: + rec = triage_mod.triage_trace(cfg, t) + except RuntimeError as e: + return sid, "agent", str(e) + if rec is None: + return sid, "agent", None + return sid, rec["route"], None + + outer = max(1, int(getattr(cfg, "triage_prepass_workers", 4) or 4)) + fatal: Optional[str] = None + with ThreadPoolExecutor(max_workers=outer) as ex: + for sid, route, err in ex.map(_triage_one, ids): + triage_routes[sid] = route + if err and fatal is None: + fatal = err + if fatal: + print(f"-- triage unavailable ({fatal}); improving without labels") + triage_routes = {sid: "agent" for sid in ids} + n_splice = sum(1 for v in triage_routes.values() if v == "splice") + n_reject = sum(1 for v in triage_routes.values() if v == "reject") + n_agent = sum(1 for v in triage_routes.values() if v == "agent") + print( + f"-- triage: {n_agent} agent Β· {n_splice} splice Β· " + f"{n_reject} reject" + ) + if n_reject: + from traceforge.improver.harness import _append_status + + for sid in ids: + if triage_routes.get(sid) == "reject": + _append_status(cfg, sid, { + "stage": "triage", "verdict": "REJECT", + "labels": 0, + }) + ids = [sid for sid in ids if triage_routes.get(sid) != "reject"] + if not ids: + print("-- every trace rejected at triage; nothing to improve") + return 0 + + if workers > 1 and len(ids) > 1: + print(f"-- improving {len(ids)} traces, {workers} concurrent workers") + statuses = [] + from traceforge.progress import Progress + + with Progress(len(ids), desc="improve:") as prog: + # live telemetry: poll child-session token counters in state.db; + # every run gets it (single trace shows its tok/s subline too) + liv = live.LiveStats.setup(cfg, prog) + if liv is not None: + liv.start() + try: + if workers == 1 or len(ids) == 1: + for sid in ids: + _, res = _improve_one( + cfg, r, sid, max_attempts, skip_existing, + prog, liv, + ) + if res is not None: + statuses.append(res) + prog.update() + else: + from concurrent.futures import ThreadPoolExecutor, as_completed + + # load traces serially (SQLite conn per call), improve in parallel + with ThreadPoolExecutor(max_workers=workers) as ex: + futures = { + ex.submit( + _improve_one, cfg, r, sid, max_attempts, + skip_existing, prog, liv, + ): sid + for sid in ids + } + for fut in as_completed(futures): + sid = futures[fut] + try: + _, res = fut.result() + except Exception as e: # never lose a whole batch + prog.write(f"FAILED {sid}: worker crashed: {e}") + res = _crash_result(sid, e) + if res is not None: + statuses.append(res) + prog.update() + finally: + if liv is not None: + liv.stop() + if liv is not None: + print(liv.summary()) + # Outcome rows are appended per trace by _improve_one (and by the + # crash handler below) the moment each attempt finishes β€” a batch that + # runs for hours or is killed must not lose its failure evidence. + from traceforge.improver.harness import ImproverResult + + rows = [ + res.to_dict() if isinstance(res, ImproverResult) else res + for res in statuses + ] + skipped = len(ids) - len(statuses) + if not rows: + # nothing was attempted: all skipped (idempotent resume) = done OK; + # nothing selectable at all = user error + if skipped: + print(f"-- {skipped} traces skipped (all already improved)") + return 0 + return 2 + ok = sum(1 for s in rows if s["ok"]) + print(f"-- {ok}/{len(rows)} traces improved" + ( + f", {skipped} skipped" if skipped else "")) + return 0 if ok == len(rows) else 1 + + +def cmd_audit(args) -> int: + """Mechanical, LLM-free audit of the promoted corpus.""" + from traceforge.audit import main as audit_main + + cfg = _cfg(args) + target = args.files[0] if args.files else cfg.improved_dir + return audit_main(target, verbose=args.verbose) + + +def cmd_validate(args) -> int: + cfg = _cfg(args) + raw_dir = cfg.raw_dir + from traceforge.grounding import validate_grounding + + bad = 0 + files = args.files or [ + os.path.join(cfg.improved_dir, f) + for f in sorted(os.listdir(cfg.improved_dir)) + if f.endswith(".json") + ] + for fp in files: + t = Trace.load(fp) + errs = validate_structural(t) + # grounding needs the source trace; skip silently when raw is absent + raw_path = os.path.join(raw_dir, f"{t.parent_session_id}.json") + if os.path.exists(raw_path): + errs += validate_grounding(t, Trace.load(raw_path)) + if errs: + bad += 1 + print(f"INVALID {os.path.basename(fp)}:") + for e in errs[:10]: + print(f" - {e}") + else: + print(f"ok {os.path.basename(fp)}") + return 1 if bad else 0 + + +def cmd_verify(args) -> int: + """Re-run external fact verification on an existing improved trace.""" + cfg = _cfg(args) + from traceforge.improver.verify import run_verifier + from traceforge.progress import Progress + + files = args.files or [ + os.path.join(cfg.improved_dir, f) + for f in sorted(os.listdir(cfg.improved_dir)) + if f.endswith(".json") + ] + bad = 0 + with Progress(len(files), desc="verify:") as prog: + liv = live.LiveStats.setup(cfg, prog) + if liv is not None: + liv.start() + try: + for fp in files: + sid = os.path.basename(fp)[:-5].replace("ideal-", "") + workdir = os.path.join(cfg.improver_workdir, sid) + os.makedirs(workdir, exist_ok=True) + if liv is not None: + liv.registry.begin(sid) + liv.registry.set_stage(sid, "verify") + else: + prog.set_postfix(os.path.basename(fp)[:24]) + res = run_verifier(cfg, fp, workdir) + if liv is not None: + liv.registry.end(sid) + tag = os.path.basename(fp) + if res.ok(): + prog.write( + f"PASS {tag} ({res.claims_checked} external" + f" claims checked, {res.duration_s:.0f}s)" + ) + elif res.verdict == "FAIL": + bad += 1 + prog.write(f"FAIL {tag}") + for u in res.unsupported[:10]: + prog.write(f" - {u.get('claim', '?')}: {u.get('why', '?')}") + else: + bad += 1 + prog.write(f"ERROR {tag}: {res.error}") + _append_status(cfg, sid, { + "stage": "verify", "verdict": res.verdict, + "claims_checked": res.claims_checked, + }) + prog.update() + finally: + if liv is not None: + liv.stop() + if liv is not None: + print(liv.summary()) + return 1 if bad else 0 + + + + +def _pct(vals: list[int], q: float) -> int: + """Nearest-rank percentile (q in 0..1) over a non-empty list.""" + if not vals: + return 0 + s = sorted(vals) + i = min(len(s) - 1, max(0, int(round(q * (len(s) - 1))))) + return s[i] + + +def _dist(vals: list[int]) -> dict: + """mean/median/p90/max for a per-trace metric.""" + if not vals: + return {"mean": 0, "median": 0, "p90": 0, "max": 0} + return { + "mean": round(sum(vals) / len(vals), 1), + "median": _pct(vals, 0.5), + "p90": _pct(vals, 0.9), + "max": max(vals), + } + + +def _ts_summary(traces) -> dict: + """Time-sensitivity census for the dataset card (never raises).""" + try: + from traceforge.timesensitivity import summary + return summary(traces) + except Exception as e: # a card must never block an export + return {"error": str(e)} + + +def _write_dataset_card(traces, out: str, val_frac: float, + dedup_removed: int = 0) -> None: + """Measured stats the training run should know: distribution biases, + shape, and the guarantees the funnel enforces.""" + import re as _re + from collections import Counter as _C + from traceforge.export import _agent_role + langs = _C() + for t in traces: + task = next((m.content or "" for m in t.messages + if m.role == "user"), "") + if _re.search(r"\b(der|die|das|und|nicht|bitte|f\u00fcr|mit|ist)\b", + task, _re.I): + langs["de"] += 1 + else: + langs["en"] += 1 + # orchestrator/subagent balance (MASS: train on both conversation + # roles, balanced β€” a blind sample underweights the subagent half) + agent_roles = _C(r for r in (_agent_role(t) for t in traces) if r) + # task/topic balance: the trainer needs to know which buckets are thin + # before it weights them (office-docs/creative are near-empty by design + # of the seed pool, not by classifier failure) + tasks = _C(t.task for t in traces if t.task) + + # ---- shape: what a training example actually looks like ------------- + # Counted per trace so the trainer can size its context window and + # batch budget from the real distribution instead of the mean. + role_tot = _C() + chars_by_role = _C() + msgs_per_trace: list[int] = [] + calls_per_trace: list[int] = [] + users_per_trace: list[int] = [] + chars_per_trace: list[int] = [] + tool_names = _C() + # Verified recovery label (docs/DATA_QUALITY.md Β§4.1): exactly one + # environment-caused failure followed by exactly one corrective step. + # The earlier heuristic ("an error-looking tool row and a later assistant + # turn") counted grep exit-1 and error text quoted inside source code as + # failures β€” it reported 32% where the real figure is 1.8%. + from traceforge.export import recovery_label + recovery_traces = 0 + for t in traces: + n_calls = 0 + n_users = 0 + n_chars = 0 + for m in t.messages: + role_tot[m.role] += 1 + chars_by_role[m.role] += len(m.content or "") + if m.role == "user": + n_users += 1 + n_chars += len(m.content or "") + for c in (m.tool_calls or []): + n_calls += 1 + tool_names[c.name] += 1 + n_chars += len(c.arguments or "") + chars_by_role["assistant"] += len(c.arguments or "") + if recovery_label(t): + recovery_traces += 1 + msgs_per_trace.append(len(t.messages)) + calls_per_trace.append(n_calls) + users_per_trace.append(n_users) + chars_per_trace.append(n_chars) + + # Token estimate: no tokenizer is a dependency of this repo, so use the + # standard chars/4 prose proxy and SAY SO. Tool output is code/JSON/logs, + # which tokenizes denser than prose, so treat this as a lower bound. + # The budget the owner trains at is the model's full 128k window, so that + # is the headline number; the smaller budgets stay as reference points. + est_tokens = [c // 4 for c in chars_per_trace] + fits = {str(n): sum(1 for v in est_tokens if v <= n) + for n in (4096, 8192, 16384, 65536, 131072)} + total_chars = sum(chars_by_role.values()) or 1 + + card = { + "traces": len(traces), + # docs/DATA_QUALITY.md rule 9: publish the dedup removal rate. + "dedup": { + "removed": dedup_removed, + "pct": round(100.0 * dedup_removed + / max(1, len(traces) + dedup_removed), 1), + "key": "(task, tool-names + sha1(args))", + }, + "languages": dict(langs), + "agent_roles": dict(agent_roles), + "tasks": dict(tasks.most_common()), + "messages": { + "total": sum(msgs_per_trace), + "by_role": dict(role_tot.most_common()), + "per_trace": _dist(msgs_per_trace), + }, + "tool_calls": { + "total": sum(calls_per_trace), + "per_trace": _dist(calls_per_trace), + "traces_with_none": sum(1 for v in calls_per_trace if v == 0), + "distinct_tools": len(tool_names), + "top_tools": dict(tool_names.most_common(15)), + }, + "user_turns": { + "per_trace": _dist(users_per_trace), + "single_turn_traces": sum(1 for v in users_per_trace if v <= 1), + "multi_turn_traces": sum(1 for v in users_per_trace if v > 1), + }, + "tokens": { + "method": "chars/4 (prose proxy). Calibrated against a real " + "tokenizer on this corpus: chars/4 runs ~13% low " + "(median 21.4k est vs 24.5k actual), so scale by ~1.15 " + "for a central estimate; tool output is code/JSON and " + "tokenizes denser than prose.", + "budget": 131072, + "chars_total": sum(chars_per_trace), + "est_total": sum(est_tokens), + "per_trace": _dist(est_tokens), + "fits_budget": { + k: {"traces": v, "pct": round(100.0 * v / max(1, len(traces)), 1)} + for k, v in fits.items() + }, + }, + "split": {"val_frac": val_frac} if val_frac else None, + # The masking contract is part of the data, not a trainer detail: a + # trainer that computes loss on tool results trains on the + # environment's text instead of the policy's (docs/DATA_QUALITY.md Β§4.3). + "supervision": { + "contract": "loss on assistant spans only; tool/user/system " + "spans are masked context", + "chars_by_role": dict(chars_by_role), + "assistant_share_pct": round( + 100.0 * chars_by_role["assistant"] / total_chars, 1), + "tool_share_pct": round( + 100.0 * chars_by_role["tool"] / total_chars, 1), + "recovery": { + "traces": recovery_traces, + "pct": round(100.0 * recovery_traces / max(1, len(traces)), 1), + "cap_pct": 15.0, + "over_cap": recovery_traces > 0.15 * max(1, len(traces)), + "method": "verified label: exactly one environment-caused " + "failure (flaky endpoint, missing dependency, " + "exhausted resource) followed by exactly one " + "corrective step; model-caused errors and retry " + "loops disqualify (docs/DATA_QUALITY.md Β§4.1)", + }, + }, + # docs/DATA_QUALITY.md Β§4.3: filtering to clean paths skews difficulty, + # so the distribution is reported rather than assumed. + "difficulty": { + "tool_calls": _dist(calls_per_trace), + "messages": _dist(msgs_per_trace), + "est_tokens": _dist(est_tokens), + "bands": { + "calls_lt_10": sum(1 for v in calls_per_trace if v < 10), + "calls_10_24": sum(1 for v in calls_per_trace if 10 <= v < 25), + "calls_25_49": sum(1 for v in calls_per_trace if 25 <= v < 50), + "calls_ge_50": sum(1 for v in calls_per_trace if v >= 50), + }, + }, + # Time sensitivity: a temporal fact answered from weights decays. + # Measured on the improved Trace, which carries no `tools` array, so + # a trace whose harness recorded no schemas cannot be distinguished + # from one with no retrieval tool β€” the count is an UPPER bound. + # `traceforge timesensitivity --dataset train.jsonl` measures the + # export (which does carry `tools`) and is the number to quote. + "time_sensitivity": _ts_summary(traces), + "notes": [ + "identity/refusal burns gated (traceforge/identity.py)", + "clip markers, template leaks, permission-ask finals gated", + "residual-defect estimate ~5% (selfhosted cross-exam, triaged)", + "reasoning channel dropped per message (keep_reasoning=False)", + ], + } + card_path = os.path.join(os.path.dirname(out), "dataset_card.json") + with open(card_path, "w", encoding="utf-8") as f: + json.dump(card, f, indent=2, ensure_ascii=False) + print(f" dataset card -> {card_path}") + # one-line shape summary so the numbers are visible without opening JSON + print(f" shape: {card['messages']['total']} msgs " + f"({card['messages']['per_trace']['median']} median/trace), " + f"{card['tool_calls']['total']} tool calls " + f"({card['tool_calls']['per_trace']['median']} median/trace), " + f"~{card['tokens']['est_total'] / 1e6:.1f}M est tokens " + f"({card['tokens']['per_trace']['median']} median/trace, " + f"{card['tokens']['fits_budget']['131072']['pct']}% fit 128k, " + f"max {card['tokens']['per_trace']['max']})") + + +def _tasks_path(cfg) -> str: + return os.path.join(cfg.data_dir, "tasks.jsonl") + + +def _load_task_labels(cfg) -> dict[str, dict]: + """sid -> {task, method, confidence} from data/tasks.jsonl.""" + path = _tasks_path(cfg) + out: dict[str, dict] = {} + if not os.path.exists(path): + return out + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + if r.get("session_id") and r.get("task"): + out[r["session_id"]] = r + return out + + +def _classify_files(cfg, files, *, use_llm: bool = True, force: bool = False, + workers: int = 8, quiet: bool = False) -> dict[str, dict]: + """Label `files` and merge into data/tasks.jsonl. Returns new rows. + + Shared by `cmd_classify` and the generate pass, so a batch run labels + its own promotions without a manual step. + """ + from concurrent.futures import ThreadPoolExecutor, as_completed + + from traceforge.classify import classify_trace + + existing = _load_task_labels(cfg) + todo = [] + for p in files: + t = Trace.load(p) + if t.session_id in existing and not force: + continue + todo.append((p, t)) + if not todo: + return {} + + if not quiet: + print(f"classifying {len(todo)} traces " + f"({'llm+heuristic' if use_llm else 'heuristic only'}, " + f"{workers} workers)") + + def _work(item): + p, t = item + return p, t, classify_trace(t, cfg, use_llm=use_llm) + + rows: dict[str, dict] = {} + done = 0 + with ThreadPoolExecutor(max_workers=max(1, workers)) as ex: + futs = [ex.submit(_work, it) for it in todo] + for fut in as_completed(futs): + p, t, r = fut.result() + rows[t.session_id] = { + "session_id": t.session_id, + "task": r["task"], + "method": r.get("method", "heuristic"), + "confidence": r.get("confidence", 0.0), + "reason": r.get("reason", ""), + "heuristic_task": (r.get("heuristic") or {}).get("task"), + } + done += 1 + if not quiet and done % 50 == 0: + print(f" {done}/{len(todo)}") + + merged = {**existing, **rows} + path = _tasks_path(cfg) + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + for sid in sorted(merged): + f.write(json.dumps(merged[sid], ensure_ascii=False) + "\n") + os.replace(tmp, path) + return rows + + +def cmd_classify(args) -> int: + """Label improved traces with a task bucket. + + Labels land in data/tasks.jsonl (sid -> task/method/confidence) rather + than mutating the improved artifacts: the corpus stays byte-identical, + labels are reviewable in one file, and re-labeling is a rewrite of one + small file instead of 600. + """ + from traceforge.classify import TASKS + + cfg = _cfg(args) + src_dir = cfg.improved_dir + if args.files: + files = list(args.files) + else: + files = [os.path.join(src_dir, f) for f in sorted(os.listdir(src_dir)) + if f.endswith(".json")] + if not files: + print("no improved traces to classify β€” run `improve` first") + return 2 + + rows = _classify_files( + cfg, files, + use_llm=not getattr(args, "heuristic_only", False), + force=getattr(args, "force", False), + workers=max(1, getattr(args, "workers", 8) or 1), + ) + if not rows: + print(f"all {len(files)} traces already labeled " + f"(use --force to reclassify)") + return 0 + + merged = _load_task_labels(cfg) + import collections + dist = collections.Counter(r["task"] for r in merged.values()) + print(f"\nlabeled {len(rows)} new Β· {len(merged)} total -> " + f"{_tasks_path(cfg)}") + for task in TASKS: + n = dist.get(task, 0) + if n: + print(f" {task:14s} {n:5d} {n / len(merged):5.1%}") + disagree = sum(1 for r in rows.values() + if r["method"] == "llm" + and r["heuristic_task"] + and r["heuristic_task"] != r["task"]) + print(f" llm/heuristic disagreement: {disagree}/{len(rows)}") + return 0 + + +def _export_by_topic(traces, out: str, cfg, keep_reasoning: bool) -> dict[str, int]: + """Write one train..jsonl per bucket next to `out`. + + Labels come from data/tasks.jsonl (`traceforge classify`); traces with + no label are classified with the free heuristic so the split is always + complete. Returns {task: rows}. + """ + from traceforge.classify import TASKS, classify_heuristic + + labels = _load_task_labels(cfg) + buckets: dict[str, list] = {t: [] for t in TASKS} + for t in traces: + task = (labels.get(t.session_id) or {}).get("task") + if not task: + task = classify_heuristic(t)["task"] + t.task = task + buckets.setdefault(task, []).append(t) + + out_dir = os.path.dirname(os.path.abspath(out)) + stem = os.path.splitext(os.path.basename(out))[0] + counts: dict[str, int] = {} + for task, group in buckets.items(): + if not group: + continue + path = os.path.join(out_dir, f"{stem}.{task}.jsonl") + counts[task] = export_chat_jsonl( + group, path, keep_reasoning=keep_reasoning) + return counts + + +def cmd_export(args) -> int: + cfg = _cfg(args) + src_dir = cfg.improved_dir + # export precondition: the corpus must pass the mechanical audit β€” + # a corpus with self-talk/clip/unparseable findings must not reach + # training silently (force with --force-audit-fail). + if not getattr(args, "force_audit_fail", False): + from traceforge.audit import main as audit_main + if audit_main(src_dir) != 0: + print("-- export blocked: audit findings above; fix them " + "or pass --force-audit-fail to override") + return 1 + traces = [] + for f in sorted(os.listdir(src_dir)): + if f.endswith(".json"): + traces.append(Trace.load(os.path.join(src_dir, f))) + if not traces: + print("no improved traces to export β€” run `improve` first") + return 2 + # docs/DATA_QUALITY.md rule 9: dedup at the (task, tool-sequence) level + # before quality filtering, and publish the removal rate. A re-pulled + # session or a re-improved split segment lands here as an exact repeat. + # The sequence carries a sha1 of the call ARGUMENTS (export.dedup_key), + # not just the tool names β€” name-only collapsed distinct sessions. + from traceforge.export import dedup_traces + traces, dupes = dedup_traces(traces) + if dupes: + print(f"-- dedup: removed {len(dupes)} duplicate trace(s) " + f"({100.0 * len(dupes) / max(1, len(traces) + len(dupes)):.1f}%) " + f"by (task, tool-names + sha1(args))") + # Time sensitivity (opt-in filter): a temporal fact the agent had no + # retrieval tool for is a decaying answer. Training on it teaches the + # model to ASSERT the fact from its weights, which is wrong the moment + # the fact changes. The same fact fetched with a tool is a good target. + if getattr(args, "drop_time_sensitive", False): + from traceforge.timesensitivity import is_publishable + kept = [t for t in traces if is_publishable(t)] + dropped = len(traces) - len(kept) + if dropped: + print(f"-- time sensitivity: dropped {dropped} trace(s) whose " + f"answer is a temporal fact with no retrieval tool") + traces = kept + out = args.out or os.path.join(cfg.data_dir, "dataset", "train.jsonl") + val_frac = max(0.0, min(0.5, getattr(args, "val_frac", 0.0) or 0.0)) + if val_frac: + # deterministic split: stable per-session hash, no randomness β€” + # re-exports with a grown corpus keep prior assignments stable + import hashlib + train_traces, val_traces = [], [] + for t in traces: + h = int(hashlib.sha256( + (t.parent_session_id or t.session_id or "").encode() + ).hexdigest()[:8], 16) + (val_traces if (h % 1000) < int(val_frac * 1000) + else train_traces).append(t) + val_out = os.path.join(os.path.dirname(out), "val.jsonl") + n_val = export_chat_jsonl( + val_traces, val_out, + keep_reasoning=getattr(args, "keep_reasoning", False)) + n = export_chat_jsonl( + train_traces, out, + keep_reasoning=getattr(args, "keep_reasoning", False)) + print(f"exported {n} train -> {out}, {n_val} val -> {val_out} " + f"(val_frac={val_frac:.2f})") + if getattr(args, "by_topic", False): + counts = _export_by_topic( + train_traces, out, cfg, getattr(args, "keep_reasoning", False)) + print(" by topic (train split):") + for task, c in sorted(counts.items(), key=lambda kv: -kv[1]): + print(f" {task:14s} {c:5d}") + _write_dataset_card(traces, out, val_frac, dedup_removed=len(dupes)) + return 0 + n = export_chat_jsonl( + traces, out, keep_reasoning=getattr(args, "keep_reasoning", False) + ) + print(f"exported {n} traces -> {out}") + if getattr(args, "by_topic", False): + counts = _export_by_topic( + traces, out, cfg, getattr(args, "keep_reasoning", False)) + print(" by topic:") + for task, c in sorted(counts.items(), key=lambda kv: -kv[1]): + print(f" {task:14s} {c:5d} -> " + f"{os.path.basename(out).replace('.jsonl', '')}.{task}.jsonl") + # corpus profile: surface known distribution biases so the trainer + # can weight/curate (measured 2026-10-05: EN-heavy, review/fix skew) + import re as _re + from collections import Counter as _C + langs = _C() + for t in traces: + task = next((m.content or "" for m in t.messages + if m.role == "user"), "") + if _re.search(r"\b(der|die|das|und|nicht|bitte|fΓΌr|mit|ist)\b", + task, _re.I): + langs["de"] += 1 + else: + langs["en"] += 1 + print(f" corpus profile: {dict(langs)}") + _write_dataset_card(traces, out, val_frac, dedup_removed=len(dupes)) + return 0 + + +def cmd_kto(args) -> int: + """Preference corpus (KTO) β€” docs/DATA_QUALITY.md Β§4.4, rule 10. + + Two stages, because KTO needs rejected samples from the policy being + aligned, not relabelled rows from the SFT corpus: + + --prompts extract rollout seeds from the SFT corpus (cleaned prefixes) + --build turn labelled rollouts into length-matched KTO rows + + Stage 2 (sampling + verification) needs a served model and runs between + the two: sample N completions per prompt from the SFT model, verify each + against the trace's own tool evidence, and label it desirable/undesirable. + """ + cfg = _cfg(args) + from traceforge.export import build_kto, build_kto_provisional, extract_prompts + out_dir = os.path.join(cfg.data_dir, "dataset") + os.makedirs(out_dir, exist_ok=True) + + if args.provisional: + src_dir = cfg.improved_dir + traces = [Trace.load(os.path.join(src_dir, f)) + for f in sorted(os.listdir(src_dir)) if f.endswith(".json")] + if not traces: + print("no improved traces β€” run `improve` first") + return 2 + rows, stats = build_kto_provisional( + traces, + os.path.join(cfg.data_dir, "traces", "raw"), + os.path.join(cfg.data_dir, "triage"), + max_rows=args.max_prompts, + ) + out = args.out or os.path.join(out_dir, "kto_provisional.jsonl") + with open(out, "w", encoding="utf-8") as f: + for r in rows: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + print(f"kto provisional: {stats['rows']} rows -> {out}") + print(f" desirable {stats['desirable']} / undesirable " + f"{stats['undesirable']} (ratio {stats['ratio']})") + print(f" NOT on-policy β€” {stats['note']}") + return 0 + + if args.build: + rollouts = [] + with open(args.build, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + rollouts.append(json.loads(line)) + rows, stats = build_kto(rollouts, max_len_ratio=args.max_len_ratio) + out = args.out or os.path.join(out_dir, "kto.jsonl") + with open(out, "w", encoding="utf-8") as f: + for r in rows: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + print(f"kto: {stats['rows']} rows -> {out}") + print(f" desirable {stats['desirable']} / undesirable " + f"{stats['undesirable']} over {stats['prompts']} prompts") + if stats["dropped_length_mismatch"]: + print(f" dropped {stats['dropped_length_mismatch']} negative(s) " + f"for length mismatch (>{args.max_len_ratio}x the shortest " + f"desirable completion)") + if stats["dropped_unpaired"]: + print(f" dropped {stats['dropped_unpaired']} negative(s) with no " + f"desirable sample for the same prompt") + return 0 + + src_dir = cfg.improved_dir + traces = [Trace.load(os.path.join(src_dir, f)) + for f in sorted(os.listdir(src_dir)) if f.endswith(".json")] + if not traces: + print("no improved traces β€” run `improve` first") + return 2 + prompts = extract_prompts(traces, max_prompts=args.max_prompts, + turns=args.turns, + max_prefix_chars=args.max_prefix_chars) + out = args.out or os.path.join(out_dir, "kto_prompts.jsonl") + with open(out, "w", encoding="utf-8") as f: + for p in prompts: + f.write(json.dumps(p, ensure_ascii=False) + "\n") + print(f"kto prompts: {len(prompts)} rollout seeds -> {out}") + print(" next: sample N completions per prompt from the SFT model, " + "verify each against the trace's tool evidence, label it, then " + "`traceforge kto --build `") + return 0 + + +def cmd_stats(args) -> int: + cfg = _cfg(args) + r = MultiSourceReader(cfg) + out = {"sources": {}} + for src in _parse_sources(args): + try: + out["sources"][src] = r.reader_for(src).stats() + except FileNotFoundError as e: + out["sources"][src] = {"error": str(e)} + print(json.dumps(out, indent=2)) + return 0 + + +def cmd_funnel(args) -> int: + """Census + gate re-run over the current data state. + + Answers "where are traces dropping?" in one command: raw pool vs + workdir stage-reach vs promoted, a sampled re-adjudication of + unpromoted writer artifacts against the live gates (how many are + actually clean = free promotion candidates vs clip-marker vs + real failures), per-size yield, and manifest stage tallies. Read-only. + """ + import random + import collections + + from traceforge.grounding import validate_grounding + from traceforge.improver.harness import ( + _parse_improved, _clip_markers_present, + ) + + cfg = _cfg(args) + raw_dir, work_dir, imp_dir = cfg.raw_dir, cfg.improver_workdir, cfg.improved_dir + raw = {f[:-5] for f in os.listdir(raw_dir) if f.endswith(".json")} \ + if os.path.isdir(raw_dir) else set() + promoted = {f[:-5] for f in os.listdir(imp_dir) if f.endswith(".json")} \ + if os.path.isdir(imp_dir) else set() + + print(f"pool: {len(raw)} raw Β· {len(promoted)} promoted " + f"({len(promoted) / len(raw):.1%} of pool)" if raw else "no raw pool") + # multi-agent composition (MASS doctrine): orchestrator vs subagent + # halves of the pool β€” the census shows whether both roles flow. + # (Disjoint: a split segment of a sidechain counts as subagent.) + if raw: + n_sub = sum(1 for sid in raw if "#sa-" in sid) + n_seg = sum(1 for sid in raw if "#p" in sid and "#sa-" not in sid) + comp = f" Β· {n_sub} subagent" if n_sub else "" + comp += f" Β· {n_seg} split-segment" if n_seg else "" + if comp: + print(f"composition:{comp}") + never = raw - set(os.listdir(work_dir)) if os.path.isdir(work_dir) else raw + print(f"never attempted: {len(never)}") + + # workdir stage census + stage = collections.Counter() + sizes = {sid: os.path.getsize(os.path.join(raw_dir, f"{sid}.json")) + for sid in raw} + unpromoted_artifacts = [] + if os.path.isdir(work_dir): + for d in os.listdir(work_dir): + wd = os.path.join(work_dir, d) + if not os.path.isdir(wd): + continue + has_imp = os.path.exists(os.path.join(wd, "improved.json")) + if d in promoted: + stage["promoted"] += 1 + continue + if has_imp: + unpromoted_artifacts.append(d) + if any(f.startswith("review") for f in os.listdir(wd)): + stage["died at/after review"] += 1 + elif os.path.exists(os.path.join(wd, "verdict.json")): + stage["died at/after verify"] += 1 + else: + stage["writer artifact, pre-verify"] += 1 + else: + stage["no writer artifact"] += 1 + for k, v in stage.most_common(): + print(f" workdirs: {k}: {v}") + + # failure census: WHY the empty workdirs are empty. The splice engine + # used to persist nothing on failure, so this was unknowable; every + # _fail() now writes data/work//failure.json (stage + violations). + fail_stage = collections.Counter() + fail_pillar = collections.Counter() + fail_err = collections.Counter() + n_fail_rec = 0 + if os.path.isdir(work_dir): + for d in os.listdir(work_dir): + fp = os.path.join(work_dir, d, "failure.json") + if not os.path.exists(fp): + continue + try: + with open(fp, encoding="utf-8") as f: + rec = json.load(f) + except Exception: + continue + n_fail_rec += 1 + fail_stage[str(rec.get("stage"))] += 1 + for v in (rec.get("violations") or []): + fail_pillar[str(v.get("pillar"))] += 1 + for e in (rec.get("errors") or [])[:1]: + head = str(e).split(":")[0][:60] + fail_err[head] += 1 + if n_fail_rec: + print(f"failure records: {n_fail_rec}") + print(" by stage:") + for k, v in fail_stage.most_common(12): + print(f" {k}: {v}") + if fail_pillar: + print(" by cited pillar:") + for k, v in fail_pillar.most_common(10): + print(f" {k}: {v}") + print(" by first error:") + for k, v in fail_err.most_common(10): + print(f" {k}: {v}") + bar_denom = len(raw) + attempt_proxy = bar_denom - len(never) + print(f"attempted β‰ˆ {attempt_proxy} " + f"({attempt_proxy and f'{len(promoted)/max(1,attempt_proxy):.1%} promoted of attempted'})") + + # size-yield buckets + if raw: + buckets: dict[int, list[int]] = {} + for sid in raw: + kb = min(sizes.get(sid, 0), 300_000) // 60_000 # 0..5 + buckets.setdefault(kb, [0, 0]) + buckets[kb][0] += 1 + if sid in promoted: + buckets[kb][1] += 1 + print("yield by raw size:") + for kb in sorted(buckets): + tot, ok = buckets[kb] + lo = kb * 60 + label = f"{lo}-{lo + 60}KB" if kb < 5 else "300KB+" + print(f" {label:>12}: {ok}/{tot}") + + # sampled gate re-adjudication of unpromoted artifacts + n = min(args.sample, len(unpromoted_artifacts)) + if n: + random.seed(args.sample) + picked = random.sample(unpromoted_artifacts, n) + verdicts = collections.Counter() + for sid in picked: + art = os.path.join(work_dir, sid, "improved.json") + t = _parse_improved(art) + if t is None: + verdicts["unparseable artifact"] += 1 + continue + if _clip_markers_present(t): + verdicts["clip markers (repair)"] += 1 + continue + raw_path = os.path.join(raw_dir, f"{sid}.json") + if not os.path.exists(raw_path): + verdicts["raw missing"] += 1 + continue + try: + real = Trace.load(raw_path) + errs = validate_structural(t) or validate_grounding(t, real) + except Exception as e: + verdicts[f"gate crash: {type(e).__name__}"] += 1 + continue + verdicts["CLEAN β€” free promotion candidate" if not errs + else "gate failure (repair with errors)"] += 1 + scale = len(unpromoted_artifacts) / n + print(f"unpromoted artifacts: {len(unpromoted_artifacts)} " + f"(sampled {n}, scaled Γ—{scale:.1f}):") + for k, v in verdicts.most_common(): + print(f" {k}: {v} β‰ˆ {int(round(v * scale))}") + + # manifest tallies + if os.path.exists(cfg.manifest_path): + tallies = collections.Counter() + with open(cfg.manifest_path, encoding="utf-8") as f: + for line in f: + try: + r = json.loads(line) + except Exception: + continue + if "stage" in r: + tallies[f"{r.get('stage')}|{r.get('verdict', '?')}"] += 1 + if tallies: + print("manifest stage rows:") + for k, v in tallies.most_common(): + print(f" {k}: {v}") + return 0 + + +def cmd_benchmark(args) -> int: + """Score a model's traces by how little the pipeline had to edit them. + + Read-only over the on-disk corpus: every promoted trace is a + (model trace, ideal rewrite) pair. Fidelity = similarity of the pair; + labels = the triage keep/drop judgement; composite = weighted mean. + The inverse framing is the point: a trace the pipeline can promote + unchanged is a trace the model already got right. + """ + from traceforge.benchmark import ( + DEFAULT_WEIGHTS, append_leaderboard, build_report, load_leaderboard, + load_triage, render_leaderboard, render_report, score_pair, + ) + + cfg = _cfg(args) + raw_dir = args.raw_dir or cfg.raw_dir + imp_dir = args.improved_dir or cfg.improved_dir + triage_dir = args.triage_dir or cfg.triage_dir + if not os.path.isdir(imp_dir): + print(f"no improved traces at {imp_dir} β€” run `improve` first") + return 2 + + w_sim = max(0.0, args.w_similarity) + w_lab = max(0.0, args.w_labels) + if w_sim + w_lab <= 0: + print("error: --w-similarity and --w-labels cannot both be zero") + return 2 + weights = {"similarity": w_sim, "labels": w_lab} + + # Gate attempts are sparse (only traces whose improve run wrote a + # manifest row carry them); the benchmark reports them when present and + # never imputes them. + attempts_by_sid: dict[str, int] = {} + if os.path.exists(cfg.manifest_path): + with open(cfg.manifest_path, "r", encoding="utf-8") as f: + for line in f: + try: + r = json.loads(line) + except ValueError: + continue + if "attempts" in r and r.get("session_id"): + attempts_by_sid[r["session_id"]] = r["attempts"] + + # Task labels live in data/tasks.jsonl keyed by the improved session id + # (`ideal-`), not on the artifact β€” the corpus stays byte-identical + # and re-labelling rewrites one small file. + task_by_sid: dict[str, str] = {} + tasks_path = os.path.join(cfg.data_dir, "tasks.jsonl") + if os.path.exists(tasks_path): + with open(tasks_path, "r", encoding="utf-8") as f: + for line in f: + try: + r = json.loads(line) + except ValueError: + continue + if r.get("session_id") and r.get("task"): + task_by_sid[r["session_id"]] = r["task"] + + sources = None if (args.source or "all") == "all" else set(_parse_sources(args)) + files = sorted(f for f in os.listdir(imp_dir) if f.endswith(".json")) + if args.limit: + files = files[: args.limit] + + scores = [] + skipped_no_raw = 0 + skipped_source = 0 + for f in files: + try: + improved = Trace.load(os.path.join(imp_dir, f)) + except (OSError, ValueError, KeyError): + continue + sid = improved.parent_session_id or improved.session_id.replace("ideal-", "") + raw_path = os.path.join(raw_dir, f"{sid}.json") + if not os.path.exists(raw_path): + skipped_no_raw += 1 + continue + try: + raw = Trace.load(raw_path) + except (OSError, ValueError, KeyError): + skipped_no_raw += 1 + continue + if sources is not None and raw.source not in sources: + skipped_source += 1 + continue + improved.task = (improved.task or task_by_sid.get(improved.session_id) + or task_by_sid.get(sid)) + triage = load_triage(os.path.join(triage_dir, f"{sid}.json")) + scores.append(score_pair( + raw, improved, triage=triage, + attempts=attempts_by_sid.get(sid), weights=weights, + )) + + if not scores: + print(f"no scoreable pairs in {imp_dir} (raw dir: {raw_dir})") + return 2 + + report = build_report( + scores, label=args.label, weights=weights, + corpus={ + "raw_pool": (len(os.listdir(raw_dir)) + if os.path.isdir(raw_dir) else 0), + "improved_pool": len(files), + "skipped_no_raw": skipped_no_raw, + "skipped_source": skipped_source, + }, + ) + + out = args.out or os.path.join(cfg.data_dir, "benchmark", "report.json") + os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True) + with open(out, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2, ensure_ascii=False) + traces_path = os.path.join(os.path.dirname(os.path.abspath(out)), + "traces.jsonl") + with open(traces_path, "w", encoding="utf-8") as f: + for s in sorted(scores, key=lambda s: s.score): + f.write(json.dumps(s.to_dict(), ensure_ascii=False) + "\n") + + if args.json: + print(json.dumps(report, indent=2, ensure_ascii=False)) + else: + print(render_report(report)) + print(f"\nreport -> {out}\nper-trace -> {traces_path}") + + if args.label: + lb_path = os.path.join(os.path.dirname(os.path.abspath(out)), + "leaderboard.jsonl") + append_leaderboard(lb_path, report) + print("\n" + render_leaderboard(load_leaderboard(lb_path))) + return 0 + + +def _quarantined_sids(cfg) -> set[str]: + """Session ids parked under data/quarantine/ (all repair classes). + + The quarantine tree is CLASSED β€” data/quarantine//.json + with class in echo/, rework-echo/, rework-language/, rework-reframe/ + β€” so a flat listdir of data/quarantine returns ZERO .json files and + the skip set silently came back empty. Every quarantined trace was + therefore re-attempted on every pass (measured 2026-10-24: 203 files + / 201 unique sids, 200 of them still in the raw queue, 25 already + re-promoted after being quarantined β€” which is exactly why the leak + went unnoticed). Walk the tree; legacy top-level files count too. + """ + base = os.path.join(cfg.data_dir, "quarantine") + out: set[str] = set() + if not os.path.isdir(base): + return out + for _root, _dirs, files in os.walk(base): + for f in files: + if f.endswith(".json"): + out.add(f[:-5]) + return out + + +def cmd_generate(args) -> int: + """Generate ideal-world references, resuming from the latest on-disk state. + + One idempotent pass: pull new candidates -> improve every raw + trace that has no promoted file yet -> export. Because every stage + keys on disk state (raw cache = queue, improved/ = done set, + per-workdir gate caches = paid judge rounds), a killed run resumes + exactly where it stopped on the next invocation; rerunning after + completion is a no-op that just re-exports. + + Pending set = raw cache (incl. split segments #pN) minus + promoted minus quarantined. Discovery + pull are idempotent + (already-pulled raws skip), so every pass also picks up NEW + sessions; --no-pull makes a pass purely resume the existing queue. + A partial improve pass (some traces fail their gates) still + exports the grown corpus and exits nonzero β€” the next pass + retries the failures via --skip-existing. + + Quarantine policy: a quarantined sid is SKIPPED by default. The + quarantine classes (echo / rework-echo / rework-language / + rework-reframe) mark traces whose defect the repair layer could not + fix; re-queueing them every pass is pure waste (they re-fail, or + worse, slip back through a later gate change and get re-promoted as + a duplicate). --requeue-quarantined is the explicit opt-in for a + one-time re-queue after a repair-layer change. + """ + cfg = _cfg(args) + if getattr(args, "no_triage", False): + cfg.triage = False + r = MultiSourceReader(cfg) + max_attempts = args.max_attempts + requeue_quarantined = bool(getattr(args, "requeue_quarantined", False)) + quarantined = set() if requeue_quarantined else _quarantined_sids(cfg) + + def _pending_ids() -> list[str]: + if not os.path.isdir(cfg.raw_dir): + return [] + improved = ( + {f[:-5] for f in os.listdir(cfg.improved_dir) + if f.endswith(".json")} + if os.path.isdir(cfg.improved_dir) else set() + ) + return [ + sid for sid in ( + f[:-5] for f in os.listdir(cfg.raw_dir) + if f.endswith(".json") + ) + if sid not in improved and sid not in quarantined + ] + + rc_improve = 0 + if not getattr(args, "no_pull", False): + # discover + pull ONLY when the local queue is empty: mid-batch + # kills resume the backlog first, discovery re-runs when dry. + pending_before = _pending_ids() + if pending_before: + print(f"-- {len(pending_before)} pending raw traces " + f"(resuming backlog)") + sessions = r.list_sessions( + sources=_parse_sources(args), + min_messages=_pick(args, cfg, "min_messages"), + max_messages=_pick(args, cfg, "max_messages"), + min_tool_calls=_pick(args, cfg, "min_tool_calls"), + limit=args.limit, + ) + if sessions: + rc_pull = _pull_sessions(cfg, r, sessions) + if rc_pull != 0: + return rc_pull + elif not pending_before: + print("no candidate sessions found in any store") + + ids = _pending_ids() + if quarantined: + raw_sids = {f[:-5] for f in os.listdir(cfg.raw_dir) + if f.endswith(".json")} if os.path.isdir(cfg.raw_dir) else set() + in_queue = len(quarantined & raw_sids) + print(f"-- {len(quarantined)} quarantined sids skipped " + f"({in_queue} of them still in the raw queue; " + f"--requeue-quarantined to retry)") + if not ids: + print("-- no pending traces; exporting current corpus") + return _generate_export(args) + + # self-reference guard (same layers as cmd_improve; the raw cache + # from a pre-guard run may still hold improver children) + child_ids = r.improver_child_ids() + if child_ids: + drop = [i for i in ids if i in child_ids] + if drop: + ids = [i for i in ids if i not in child_ids] + print(f"-- skipped {len(drop)} improver session(s) " + f"(self-referential data)") + ids = _filter_improver_children_live(cfg, r, ids) + if not ids: + print("-- no improvable traces left after self-reference guard") + return _generate_export(args) + + rc_improve = _improve_ids( + cfg, r, ids, + max_attempts=max_attempts, + workers=args.workers, + skip_existing=True, + ) + if rc_improve != 0: + # partial pass: export what was promoted anyway, keep the + # nonzero rc so the wrapper's loop counts remaining failures + _generate_export(args) + return rc_improve + return _generate_export(args) + + +def _generate_export(args) -> int: + """Export stage shared by cmd_generate (always runs at the end). + + The improve exit code is deliberately not propagated from a + partial pass: --skip-existing makes the NEXT pass retry failures, + and the dataset must still refresh after a partial batch. Errors + (bad exit codes) still surface here. + + Newly promoted traces are labeled before export so the per-topic + split stays complete without a manual `classify` step. + """ + cfg = _cfg(args) + try: + src_dir = cfg.improved_dir + files = [os.path.join(src_dir, f) for f in sorted(os.listdir(src_dir)) + if f.endswith(".json")] + new = _classify_files(cfg, files, workers=8, quiet=True) + if new: + print(f" classified {len(new)} new traces") + except Exception as e: # labeling must never block the export + print(f" classify skipped: {e}") + return cmd_export(types.SimpleNamespace( + config=args.config, out=getattr(args, "out", None), + keep_reasoning=False, + val_frac=getattr(args, "val_frac", 0.0) or 0.0, + force_audit_fail=False, + by_topic=True, + )) + + +def cmd_run(args) -> int: + """One command: find candidates everywhere, pull, improve, export. + + Chain: list (all sources) -> pull -> improve (--skip-existing by default + so reruns are resumable) -> export. The export only runs when the + improve batch is clean (honest exit codes propagate). + """ + cfg = _cfg(args) + r = MultiSourceReader(cfg) + sessions = r.list_sessions( + sources=_parse_sources(args), + min_messages=_pick(args, cfg, "min_messages"), + max_messages=_pick(args, cfg, "max_messages"), + min_tool_calls=_pick(args, cfg, "min_tool_calls"), + limit=args.limit, + ) + if not sessions: + print("no candidate traces found β€” machine has no improvable sessions") + return 2 + _print_sessions(sessions) + rc_pull = _pull_sessions(cfg, r, sessions) + if rc_pull != 0: + return rc_pull + rc_improve = _improve_ids( + cfg, r, + [s["session_id"] for s in sessions], + max_attempts=args.max_attempts, + workers=args.workers, + skip_existing=True, # run is resumable by default + ) + if rc_improve != 0: + return rc_improve + return cmd_export(types.SimpleNamespace( + config=args.config, out=None, keep_reasoning=False + )) + + +def cmd_timesensitivity(args) -> int: + """Census: does the ideal answer decay with wall-clock time? + + Two axes decide the class: the task asks about an EXTERNAL changing fact, + and the agent had (or lacked) a retrieval tool. A temporal fact answered + from weights is a bad SFT target β€” the model learns to assert it. The + same fact fetched with `web_search` is a good target: the model learns + the procedure, which does not decay. + """ + from traceforge.timesensitivity import CLASSES, CLASS_LABELS, detect + + cfg = _cfg(args) + records: list = [] + if args.dataset: + path = args.dataset + if not os.path.exists(path): + print(f"error: not found: {path}") + return 2 + with open(path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + continue + if args.limit and len(records) >= args.limit: + break + label = path + else: + if not os.path.isdir(cfg.improved_dir): + print(f"error: no improved traces at {cfg.improved_dir}") + return 2 + files = sorted(f for f in os.listdir(cfg.improved_dir) + if f.endswith(".json")) + if args.limit: + files = files[:args.limit] + for f in files: + try: + records.append(Trace.load(os.path.join(cfg.improved_dir, f))) + except Exception: + continue + label = cfg.improved_dir + if not records: + print("no traces to census") + return 2 + + counts = {c: 0 for c in CLASSES} + rewrite = 0 + examples: list[tuple[str, list[str]]] = [] + for r in records: + d = detect(r) + counts[d["class"]] = counts.get(d["class"], 0) + 1 + if d.get("needs_rewrite"): + rewrite += 1 + if d["class"] == "time_sensitive" and len(examples) < 10: + sid = (getattr(r, "session_id", None) + or ((r.get("meta") or {}).get("traceforge_session_id") + if isinstance(r, dict) else None) or "?") + examples.append((sid, d.get("temporal_facts") or [])) + n = len(records) + census = { + "total": n, + "classes": counts, + "labels": CLASS_LABELS, + "needs_rewrite": rewrite, + "time_sensitive_share": round(counts.get("time_sensitive", 0) / n, 4), + "source": label, + } + if args.cache: + from traceforge.space.stats import _write_json + st = os.stat(args.dataset) if args.dataset else None + _write_json(args.cache, { + "key": ({"mtime": int(st.st_mtime), "size": st.st_size} + if st else {"source": label}), + "census": {**census, "available": True}, + }) + print(f"-- census cache -> {args.cache}") + if args.json: + print(json.dumps(census, indent=2)) + return 0 + print(f"-- time sensitivity over {n} traces ({label})") + for c in CLASSES: + print(f" {c:20s} {counts.get(c, 0):6d} " + f"{100.0 * counts.get(c, 0) / n:5.1f}% {CLASS_LABELS[c]}") + print(f" needs rewrite (retrieval tool available, never called): {rewrite}") + if examples: + print(" examples:") + for sid, facts in examples: + print(f" {sid[:48]:48s} {', '.join(facts)}") + return 0 + + +def cmd_space(args) -> int: + """Portable web app: submit traces, sponsor datasets, public dashboard.""" + from traceforge.space import (SpacePipeline, TraceStore, fingerprint, + parse_records, read_upload, space_config) + + if args.action == "serve": + from traceforge.space.app import main as app_main + argv: list[str] = [] + if args.host: + argv += ["--host", args.host] + if args.port: + argv += ["--port", str(args.port)] + if args.share: + argv += ["--share"] + return app_main(argv) + + cfg = space_config(data_dir=args.data_dir) + store = TraceStore(cfg.data_dir) + + if args.action == "hash": + if not args.files: + print("error: pass one or more trace files") + return 2 + for path in args.files: + records, errors = read_upload(path) + for e in errors: + print(f" {path}: {e}") + for rec in records: + fp = fingerprint(rec) + print(f"{fp['hash']} {os.path.basename(path)} " + f"{fp['n_messages']} msgs Β· {fp['n_tool_calls']} calls Β· " + f"{fp['language']} Β· {fp['task'][:60]}") + return 0 + + if args.action == "stats": + d = store.stats() + if args.json: + print(json.dumps(d, indent=2)) + return 0 + print(f"-- space store: {store.root}") + print(f" seeds {d['seeds']} Β· done {d['done']} Β· failed {d['failed']} " + f"Β· queued {d['queued']} Β· running {d['running']}") + for s in d["sponsors"]: + print(f" sponsor {s['sponsor']:24s} {s['seeds']:5d} seeds " + f"({s['done']} done, {s['failed']} failed)") + if d["time_sensitivity"]: + print(" time sensitivity: " + ", ".join( + f"{k} {v}" for k, v in d["time_sensitivity"].items())) + return 0 + + if args.action == "process": + if not args.files: + print("error: pass one or more trace files") + return 2 + pipeline = SpacePipeline(store, cfg) + rc = 0 + for path in args.files: + records, errors = read_upload(path) + for e in errors: + print(f" {path}: {e}") + rc = 1 + for rec in records: + res = pipeline.submit(rec, sponsor=args.sponsor) + h = res["hash"] + if res.get("duplicate") and res.get("status") == "done": + print(f"DUPLICATE {h} β€” existing output returned") + continue + out = pipeline.process(h) + print(f"{out.get('status', '?').upper():9s} {h} " + f"{out.get('duration_s', 0):.0f}s " + f"{out.get('error', '')[:200]}") + if out.get("status") != "done": + rc = 1 + return rc + + print(f"error: unknown action {args.action}") + return 2 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="traceforge", + description="Evaluation harness for agent models: score a model by how " + "little an ideal-world rewrite has to edit its real traces", + ) + p.add_argument("--config", help="YAML config overrides") + sub = p.add_subparsers(dest="cmd", required=True) + + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--min-messages", type=int, default=None) + common.add_argument("--max-messages", type=int, default=None) + common.add_argument("--min-tool-calls", type=int, default=None) + common.add_argument("--limit", type=int, default=None) + common.add_argument( + "--source", "-s", default="all", + help="trace source filter: all | hermes | claude | comma list " + "(default: all)", + ) + + sp = sub.add_parser( + "stats", help="trace-store counts (all sources)", parents=[common] + ) + sp.set_defaults(fn=cmd_stats) + + sp = sub.add_parser( + "list", help="list candidate traces (all sources)", parents=[common] + ) + sp.set_defaults(fn=cmd_list) + + sp = sub.add_parser( + "pull", help="normalize candidate traces to data/traces/raw/", parents=[common] + ) + sp.set_defaults(fn=cmd_pull) + + sp = sub.add_parser("improve", help="run the agentic improver on traces") + sp.add_argument("session_ids", nargs="*", help="session ids (empty = --from-raw)") + sp.add_argument("--from-raw", action="store_true", help="improve all pulled traces") + sp.add_argument("--max-attempts", type=int, default=2) + sp.add_argument( + "--workers", "-w", type=int, default=1, + help="concurrent improver sessions = parallel requests to the LLM " + "endpoint (real batch size; default 1 = sequential)", + ) + sp.add_argument( + "--skip-existing", action="store_true", + help="skip traces that already have data/traces/improved/.json", + ) + sp.add_argument( + "--no-triage", action="store_true", + help="skip the LLM triage pass even when enabled in config", + ) + sp.set_defaults(fn=cmd_improve) + + sp = sub.add_parser( + "triage", + help="label traces (keep/drop/rephrase/fact_check/summarize) via " + "structured generation; prints the route per trace", + ) + sp.add_argument("session_ids", nargs="*", help="session ids (empty = --from-raw)") + sp.add_argument("--from-raw", action="store_true", help="triage all pulled traces") + sp.add_argument( + "--force", action="store_true", + help="re-label even when data/triage/.json exists", + ) + sp.set_defaults(fn=cmd_triage) + + sp = sub.add_parser( + "funnel", + help="funnel census: where do traces drop before promotion? (read-only)", + ) + sp.add_argument( + "--sample", type=int, default=200, + help="artifact sample size for the gate re-adjudication (default 200)", + ) + sp.set_defaults(fn=cmd_funnel) + + sp = sub.add_parser( + "benchmark", + help="score a model's traces by how little the pipeline had to edit " + "(fidelity + triage labels -> 0-100); read-only", + ) + sp.add_argument("--raw-dir", default=None, + help="model traces (default data/traces/raw/)") + sp.add_argument("--improved-dir", default=None, + help="ideal rewrites (default data/traces/improved/)") + sp.add_argument("--triage-dir", default=None, + help="triage label files (default data/triage/)") + sp.add_argument("--label", default=None, + help="name this run (model/checkpoint); appends a row to " + "the leaderboard next to the report") + sp.add_argument("--out", default=None, + help="report path (default data/benchmark/report.json)") + sp.add_argument("--json", action="store_true", + help="print the JSON report instead of the table") + sp.add_argument("--limit", type=int, default=None, + help="score at most N pairs (smoke test)") + sp.add_argument("--source", "-s", default="all", + help="filter by the model trace's source: all | hermes | " + "claude | codex | comma list") + sp.add_argument("--w-similarity", type=float, default=0.6, + help="weight of fidelity in the composite (default 0.6)") + sp.add_argument("--w-labels", type=float, default=0.4, + help="weight of the triage keep rate (default 0.4)") + sp.set_defaults(fn=cmd_benchmark) + + sp = sub.add_parser( + "regate", + help="adjudicate existing workdir artifacts with the windowed " + "judge (gates locally, no agent runtime); promotes on PASS", + ) + sp.add_argument("session_ids", nargs="*", + help="session ids (empty = all workdir artifacts)") + sp.add_argument("--workers", "-w", type=int, default=8) + sp.set_defaults(fn=cmd_regate) + + sp = sub.add_parser( + "crossexam", + help="re-judge PROMOTED traces with a second judge family; " + "disagreements are reported for human review (read-only)", + ) + sp.add_argument("session_ids", nargs="*", + help="session ids (empty = whole promoted corpus)") + sp.add_argument("--model", "-m", required=True, + help="cross judge model (DIFFERENT family than the " + "improver, e.g. gemma4-31b-selfhosted)") + sp.add_argument("--workers", "-w", type=int, default=8) + sp.set_defaults(fn=cmd_crossexam) + + sp = sub.add_parser( + "audit", + help="mechanical corpus audit (self-talk, clip markers, " + "near-duplicates); zero LLM calls", + ) + sp.add_argument("files", nargs="*") + sp.add_argument("-v", "--verbose", action="store_true") + sp.set_defaults(fn=cmd_audit) + + sp = sub.add_parser("validate", help="structurally validate improved traces") + sp.add_argument("files", nargs="*") + sp.set_defaults(fn=cmd_validate) + + sp = sub.add_parser( + "verify", + help="web fact-check improved traces' external claims (adversarial verifier)", + ) + sp.add_argument("files", nargs="*") + sp.set_defaults(fn=cmd_verify) + + sp = sub.add_parser( + "classify", + help="label improved traces with a task/topic bucket (for per-topic export)", + ) + sp.add_argument("files", nargs="*") + sp.add_argument("--heuristic-only", action="store_true", + help="skip the LLM tier (free, deterministic, coarser)") + sp.add_argument("--force", action="store_true", + help="reclassify traces that already carry a task label") + sp.add_argument("--workers", "-w", type=int, default=8, + help="concurrent LLM calls (default 8)") + sp.set_defaults(fn=cmd_classify) + + sp = sub.add_parser("export", help="export references to training JSONL " + "(downstream reuse, not the repo's scope)") + sp.add_argument("--out", default=None) + sp.add_argument("--keep-reasoning", action="store_true", + help="include per-message reasoning in the export") + sp.add_argument("--force-audit-fail", action="store_true", + help="export even when the mechanical audit fails") + sp.add_argument("--val-frac", type=float, default=0.0, + help="hold out a deterministic fraction of traces as " + "val.jsonl (stable session-id hash split)") + sp.add_argument("--by-topic", action="store_true", + help="also write one train..jsonl per task bucket " + "(labels from `traceforge classify`; unlabeled traces " + "are classified on the fly with the heuristic)") + sp.add_argument("--drop-time-sensitive", action="store_true", + help="drop traces whose answer is a temporal fact the " + "agent had no retrieval tool for (the answer decays; " + "training on it teaches the model to assert it)") + sp.set_defaults(fn=cmd_export) + + sp = sub.add_parser( + "kto", + help="build the preference (KTO) corpus: rollout seeds, then " + "length-matched binary labels", + ) + sp.add_argument("--prompts", action="store_true", + help="extract rollout seeds from the SFT corpus " + "(the default action)") + sp.add_argument("--build", default=None, metavar="ROLLOUTS", + help="labelled rollouts JSONL -> length-matched KTO rows") + sp.add_argument("--provisional", action="store_true", + help="build a NOT-on-policy smoke-test set: positives from " + "the SFT corpus, negatives from triage-dropped turns " + "(replace with rollout-built rows after SFT)") + sp.add_argument("--out", default=None) + sp.add_argument("--max-prompts", type=int, default=0, + help="cap the number of rollout seeds (0 = all)") + sp.add_argument("--turns", choices=("final", "all"), default="final", + help="one seed per trace at the final assistant turn " + "(default, verifiable) or every assistant turn") + sp.add_argument("--max-prefix-chars", type=int, default=0, + help="bound each seed's prefix (0 = full context); keeps " + "the opening task and the most recent turns") + sp.add_argument("--max-len-ratio", type=float, default=1.5, + help="drop a negative longer than this multiple of the " + "shortest desirable completion (default 1.5)") + sp.set_defaults(fn=cmd_kto) + + sp = sub.add_parser( + "run", help="one command: find all traces, pull, improve, export", + parents=[common], + ) + sp.add_argument("--max-attempts", type=int, default=2) + sp.add_argument( + "--workers", "-w", type=int, default=1, + help="concurrent improver sessions = parallel requests to the LLM " + "endpoint (real batch size; default 1 = sequential)", + ) + sp.set_defaults(fn=cmd_run) + + sp = sub.add_parser( + "generate", + help="generate data, resuming from the latest on-disk state; " + "rerun after any interruption to continue", + parents=[common], + ) + sp.add_argument("--max-attempts", type=int, default=2) + sp.add_argument( + "--workers", "-w", type=int, default=1, + help="concurrent improver sessions = parallel requests to the LLM " + "endpoint (real batch size; default 1 = sequential)", + ) + sp.add_argument("--no-pull", action="store_true", + help="do not discover/pull new sessions; only improve " + "raws already in data/traces/raw/") + sp.add_argument("--no-triage", action="store_true", + help="skip the LLM triage pass even when enabled in config") + sp.add_argument( + "--requeue-quarantined", action="store_true", + help="re-queue traces parked in data/quarantine// (default: " + "skip them). Use ONCE after a repair-layer change; a quarantined " + "trace otherwise re-fails on every pass", + ) + sp.add_argument("--out", default=None, + help="export path (default data/dataset/train.jsonl)") + sp.add_argument("--val-frac", type=float, default=0.0, + help="hold out a deterministic fraction of traces as " + "val.jsonl (stable session-id hash split)") + sp.set_defaults(fn=cmd_generate) + + sp = sub.add_parser( + "timesensitivity", + help="census: which ideal answers decay with wall-clock time " + "(temporal fact, no retrieval tool)", + ) + sp.add_argument("--dataset", default=None, + help="JSONL export to census (default: data/traces/improved/)") + sp.add_argument("--limit", type=int, default=0, + help="cap the number of traces (0 = all)") + sp.add_argument("--json", action="store_true", + help="machine-readable census") + sp.add_argument("--cache", default=None, + help="write the dashboard census cache to this path") + sp.set_defaults(fn=cmd_timesensitivity) + + sp = sub.add_parser( + "space", + help="portable web app: submit traces, sponsor datasets, public dashboard", + ) + sp.add_argument("action", choices=("serve", "hash", "stats", "process")) + sp.add_argument("files", nargs="*", help="trace files (hash / process)") + sp.add_argument("--host", default=None) + sp.add_argument("--port", type=int, default=None) + sp.add_argument("--share", action="store_true") + sp.add_argument("--data-dir", default=None, + help="store root (default data/space)") + sp.add_argument("--sponsor", default="", help="sponsor name for `process`") + sp.add_argument("--json", action="store_true") + sp.set_defaults(fn=cmd_space) + + return p + + +def main(argv=None) -> int: + args = build_parser().parse_args(argv) + try: + return args.fn(args) + except FileNotFoundError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/traceforge/config.py b/traceforge/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4be28ab0bb8beffd5785d6fce88601ba5fc4e9fc --- /dev/null +++ b/traceforge/config.py @@ -0,0 +1,276 @@ +"""traceforge configuration: defaults + optional YAML overrides.""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any, Optional + +DEFAULT_HERMES_HOME = os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")) + + +@dataclass +class Config: + # ---------------------------------------------------------------- paths + hermes_home: str = DEFAULT_HERMES_HOME + state_db: str = "" # derived from hermes_home when empty + data_dir: str = "" # derived from repo root when empty + guidelines_path: str = "" # derived from repo root when empty + # Seed stores (2026-10-13): the repo ships portable trace archives β€” + # harness-traces/{hermes,claude,codex} (a snapshot of all three + # live stores) and exported-seed-datas/* (codex rollouts from other + # machines). resolve() auto-appends them to the reader path lists + # below when the folders exist, so `traceforge generate` / + # scripts/generate.sh processes them with zero configuration. + # Lists are ADDITIVE to the live stores; empty = live store only. + hermes_state_dbs: list[str] = field(default_factory=list) + claude_projects_dirs: list[str] = field(default_factory=list) + # Published SFT corpora normalized into the pool by + # scripts/fetch_nemotron.py (one .jsonl per dataset subset). + # Auto-discovered from seed-datasets/nemotron/ when it exists. + nemotron_roots: list[str] = field(default_factory=list) + + # -------------------------------------------------------- trace picking + # min_messages / min_tool_calls are HARD floors: they drop degenerate + # stubs (2-9 message probes), not long work. + # max_messages is a SOFT cap (2026-10-23): a session above it is still + # a candidate, flagged `oversize` in the listing row. The hard cap it + # replaced silently discarded 3,026 real agentic sessions (563 hermes, + # 1,246 codex, 1,217 claude β€” 35% of the viable pool) before they + # could reach split_long, which is the stage that actually bounds what + # the pipeline processes: by rendered size (split_max_chars), not by + # message count. A 200-message session is exactly the long tool-call + # chain the corpus wants. + # NOTE: `traceforge list|pull|generate` take these from argparse + # defaults in cli.py, not from this file β€” see the report's cli.py + # patch note. + min_messages: int = 10 + max_messages: int = 120 + min_tool_calls: int = 2 + + # ------------------------------------------------------- prompt budgets + user_chars: int = 4000 + assistant_chars: int = 4000 + tool_result_chars: int = 1200 + tool_args_chars: int = 600 + include_reasoning: bool = False + + # ------------------------------------------------------ improver harness + model: Optional[str] = None # None = hermes default profile model + provider: Optional[str] = None + max_turns: Optional[int] = 16 # improver: read β†’ write β†’ validate β†’ reply, with headroom + run_budget: Optional[int] = None + harness_timeout_s: int = 1200 + improver_workdir: str = "" # derived: data_dir/work/ when empty + skills: list[str] = field(default_factory=list) # -s preloads for improver + + # ------------------------------------------------- external verification + verify: bool = True # web-search fact-check stage (adversarial verifier) + verifier_max_turns: int = 20 + verifier_timeout_s: int = 900 + + # ---------------------------------------------------- review rounds + # Never trust one reviewer: quality-pillar review runs independently + # this many times; ALL rounds must PASS for corpus admission. + review: bool = True + review_rounds: int = 2 + review_max_turns: int = 16 + review_timeout_s: int = 900 # deep-thinker reviewers verify byte-exact; + # 600s timed out in practice (2026-09-26) + + # verification rounds: fact-check runs this many independent times + verify_rounds: int = 2 + + # ------------------------------------------------- cross-family confirm + # Second-family confirmation on promotion (residual-defect squeeze): + # after the primary judge PASSes, ONE selfhosted model from a + # different family re-judges the final window. VETO blocks promotion + # (the trace fails honestly and lands on the retry queue). + confirm_model: Optional[str] = None # e.g. "gemma4-31b-selfhosted" + + # ---------------------------------------------------------- long traces + # Traces whose compact rendering exceeds chunk_max_chars go through the + # map-digest-write pipeline instead of single-shot improvement. + chunk_max_chars: int = 100_000 # β‰ˆ25k tokens per chunk prompt + chunk_overlap_messages: int = 2 # continuity tail between chunks + digest_workers: int = 3 # parallel digester sessions + digest_max_turns: int = 8 + digest_timeout_s: int = 600 + digest_output_clip: int = 6000 # max chars per verbatim output kept + + # ------------------------------------------------------- live telemetry + # LiveStats polls state.db child-session counters while batch runs + # are in flight. TTY: per-trace sublines + tok/s postfix. Pipes: one + # heartbeat line every live_heartbeat_s. 0 disables the poller. + live_poll_s: float = 2.0 + live_heartbeat_s: float = 30.0 + + # ------------------------------------------------------------- triage + # Cheap LLM pass (structured generation, NOT a hermes one-shot) that + # labels every message keep/drop/rephrase/fact_check/summarize BEFORE + # the expensive improver runs. Output: data/triage/.json. + triage: bool = False + triage_base_url: str = "" # empty = hermes active-profile endpoint + triage_model: str = "" # empty = server default + triage_max_chars: int = 24_000 # per-window char budget (β‰ˆ6k tokens) + triage_overlap_messages: int = 4 + triage_workers: int = 8 # parallel window requests + # Parallel traces in the pre-pass (2026-10-23). The pre-pass used to be + # serial over traces, so a doubled pool meant hours of silent log before + # the first promotion. Each trace is self-contained, so the outer loop + # runs this many traces at once; total in-flight requests β‰ˆ + # triage_prepass_workers Γ— triage_workers. + triage_prepass_workers: int = 4 + triage_timeout_s: int = 300 + triage_temperature: float = 0.0 + + # ------------------------------------------------- windowed splice engine + # Direct-HTTP windowed improve path (traceforge/splice.py): no agent + # runtime, stdlib only. Triage labels drive a mechanical splice; only + # windows with rephrase/summarize/fact_check edits pay a rewrite call; + # admission is a windowed judge instead of agent verify/review rounds. + splice: bool = True # default improve path (False = hermes agents) + splice_max_chars: int = 24_000 # per-window char budget for rewrites/judge + splice_workers: int = 8 # parallel window HTTP requests + splice_temperature: float = 0.2 + rewrite_max_tokens: int = 8000 # per-window rewrite emit ceiling + rewrite_timeout_s: int = 300 # per rewrite HTTP call + judge_model: str = "" # empty = same as cfg.model + judge_max_tokens: int = 3000 + judge_timeout_s: int = 300 + allow_task_reframe: bool = True # intent-only FAIL + good answer -> + # rewrite the task message once + + # --------------------------------------------------------------- extras + raw_cache: bool = True # cache normalized raw traces under data/traces/raw + overwrite: bool = False + # Long-trace splitting (2026-10-07, owner directive): raws whose + # rendered size exceeds split_max_chars become N segment traces + # #p1..N at pull time β€” each ~≀120k tokens, each its own + # pipeline candidate (triage/splice/judge/promotion per segment). + split_long: bool = True + split_max_chars: int = 480_000 # β‰ˆ120k tokens per output trace + # Split TRIGGER (2026-10-24): only traces above this rendered size are + # split at all; 0 = same as split_max_chars (split everything over the + # segment ceiling, the pre-2026-10-24 behaviour). + # + # Measured on 300KB+ raws re-split three ways (same 3 parents, 19 + # segments, scratch/exp): 480k segments promoted 0/2 at 650s and ~197k + # tokens each; 240k segments 1/6 (16.7%) at 418s; 120k segments 5/11 + # (45.5%) at 364s and ~76k tokens. Cost per promotion: 802s at 120k vs + # 2507s at 240k vs never at 480k. The historical corpus agrees β€” last + # segments promote 47% at 60-120KB, 42% at 120-180KB, 4.8% at 420-480KB. + # Recommended pair for the 300KB+ backlog: split_max_chars=120_000 with + # split_trigger_chars=240_000 (leave the 60-240KB cohort whole: it + # already promotes at 27-32% and splitting it only doubles the attempts). + split_trigger_chars: int = 0 + # A tail smaller than this merges into the previous segment instead of + # becoming its own trace (2026-10-23: the merge branch was a no-op, so + # 192 of 1,044 segments were under this floor, 31 under 5k, the + # smallest 714 chars / 2 messages). + min_segment_chars: int = 60_000 + # Hard ceiling on segments per parent session (2026-10-23). The soft + # max_messages cap admits long sessions, and a few are pathological: + # measured, one codex rollout is 50,114 messages / 155M chars, which + # splits into 317 segments of the same task. At ~10-20 min per + # segment that is days of batch time for one session, and it makes + # the pool count and ETA meaningless. Above this ceiling the split + # keeps the FIRST max-1 segments (task setup + early work) and the + # LAST one (the real final answer), dropping the middle β€” the same + # standalone-window contract every later segment already has. Set 0 + # to disable the ceiling. + max_segments_per_trace: int = 16 + # Codex rollout stores to read (live store + extracted seed exports). + # Extra roots are merged into one multi-root index; thread ids are + # globally unique (uuid v7) so collisions cannot occur. + codex_roots: list[str] = field(default_factory=list) + # Claude subagent sidechains (/subagents/agent-*.jsonl) as + # separate candidate traces (MASS arXiv 2610.12176: multi-agent + # trajectories are the densest training signal; see export._agent_role). + claude_sidechains: bool = True + + def resolve(self, repo_root: str) -> "Config": + self.state_db = self.state_db or os.path.join(self.hermes_home, "state.db") + self.data_dir = self.data_dir or os.path.join(repo_root, "data") + self.guidelines_path = self.guidelines_path or os.path.join( + repo_root, "guidelines", "IDEAL_TRACE_GUIDELINES.md" + ) + self.improver_workdir = self.improver_workdir or os.path.join( + self.data_dir, "work" + ) + self.codex_roots = [os.path.expanduser(r) for r in self.codex_roots] + # ---- seed-store auto-discovery (see field docs above) -------- + # harness-traces/ + exported-seed-datas/* are appended to + # the reader path lists when present; already-listed paths are + # never duplicated (YAML overrides + auto-discovery merge). + def _add_unique(lst: list[str], item: str) -> None: + if item not in lst: + lst.append(item) + + seed_hermes = os.path.join(repo_root, "harness-traces", "hermes", "state.db") + if os.path.exists(seed_hermes): + _add_unique(self.hermes_state_dbs, seed_hermes) + seed_claude = os.path.join(repo_root, "harness-traces", "claude") + if os.path.isdir(seed_claude): + _add_unique(self.claude_projects_dirs, seed_claude) + seed_codex = os.path.join(repo_root, "harness-traces", "codex") + if os.path.isdir(seed_codex): + _add_unique(self.codex_roots, seed_codex) + seed_rollouts = os.path.join(repo_root, "exported-seed-datas") + if os.path.isdir(seed_rollouts): + _add_unique(self.codex_roots, seed_rollouts) + seed_nemotron = os.path.join(repo_root, "seed-datasets", "nemotron") + if os.path.isdir(seed_nemotron): + _add_unique(self.nemotron_roots, seed_nemotron) + self.hermes_state_dbs = [os.path.expanduser(d) for d in self.hermes_state_dbs] + self.claude_projects_dirs = [ + os.path.expanduser(d) for d in self.claude_projects_dirs + ] + self.nemotron_roots = [os.path.expanduser(d) for d in self.nemotron_roots] + return self + + @property + def raw_dir(self) -> str: + return os.path.join(self.data_dir, "traces", "raw") + + @property + def improved_dir(self) -> str: + return os.path.join(self.data_dir, "traces", "improved") + + @property + def prompts_dir(self) -> str: + return os.path.join(self.data_dir, "prompts") + + @property + def triage_dir(self) -> str: + return os.path.join(self.data_dir, "triage") + + @property + def manifest_path(self) -> str: + return os.path.join(self.data_dir, "manifest.jsonl") + + # ------------------------------------------------------------------ load + + @classmethod + def load(cls, path: Optional[str] = None, repo_root: str = ".") -> "Config": + cfg = cls() + if path: + if not os.path.exists(path): + raise FileNotFoundError(f"config file not found: {path}") + try: + import yaml # optional dependency + except ImportError as e: + raise RuntimeError( + "PyYAML is required for --config files: uv pip install pyyaml" + ) from e + with open(path, "r", encoding="utf-8") as f: + overrides: dict[str, Any] = yaml.safe_load(f) or {} + unknown = set(overrides) - {f for f in cls.__dataclass_fields__} + if unknown: + raise ValueError(f"unknown config keys: {sorted(unknown)}") + cfg = cls(**{k: v for k, v in overrides.items() if v is not None}) + return cfg.resolve(os.path.abspath(repo_root)) + + +def repo_root_from_pkg() -> str: + """Repo root as located relative to this package (traceforge/../).""" + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) \ No newline at end of file diff --git a/traceforge/defects.py b/traceforge/defects.py new file mode 100644 index 0000000000000000000000000000000000000000..ec1e461e541dc62238cfab76a94e626bae29eee4 --- /dev/null +++ b/traceforge/defects.py @@ -0,0 +1,186 @@ +"""Stable defect codes for a failed improve attempt. + +Why this module exists +---------------------- +The pipeline used to record a failure as a free-text `stage` plus an `errors` +list. That is enough to read one trace and useless for a funnel: "repair 2" and +"gates fail (2)" are strings, so every report re-derived its own grouping and +two reports of the same corpus disagreed. + +A defect code is the smallest thing that makes a loss countable. It is stable +across runs, it is what the yield table is broken down by, and it is what tells +you whether a bad week was the writer, the source, or the gate. + +The codes follow the shared taxonomy in the agentic-sft-synth skill suite +(`agentic-verify` Β§"Defect codes") where a code already exists, and add the +repo-specific ones the suite has no name for. Do not invent a new code for a +failure that already has one; extend the mapping instead. + + from traceforge.defects import classify, census + code = classify(stage="repair 2", errors=["max repair rounds exceeded"]) + # -> "D-REP-01" +""" + +from __future__ import annotations + +import collections +import json +import os +import re +from typing import Iterable, Optional + +# code -> (short name, layer, what it means, what to do about it) +DEFECTS: dict[str, tuple[str, str, str, str]] = { + # --- source / task --------------------------------------------------- + "D-TASK-04": ("no_task", "init", + "the source session has no user task message", + "not rewritable; the task lived in another session"), + "D-TASK-05": ("harness_brief", "init", + "the only opening user message is a harness dispatch brief", + "unwrap the brief to the real task, or drop"), + "D-TASK-02": ("reframe_failed", "init", + "the task could not be reframed into a single intent", + "usually several tasks in one session; split it"), + # --- writer / repair ------------------------------------------------- + "D-REP-01": ("repair_exhausted", "repair", + "the repair budget ran out before the trace passed", + "the dominant loss; a third round rarely helps (R-85)"), + "D-REP-02": ("repair_round_1", "repair", + "failed after the first repair round", + "inspect the first violation; often a single bad window"), + "D-REP-03": ("no_writer_artifact", "writer", + "the writer produced nothing to verify", + "check the writer call, not the trace"), + # --- gates ----------------------------------------------------------- + "D-GATE-01": ("gate_reject", "gate", + "a hard gate rejected the trace", + "read the violation; a gate with low precision is R-88"), + "D-VER-01": ("judge_violation", "judge", + "the judge found an unsupported claim or a broken pillar", + "the trace is fluent and wrong; do not ship it"), + "D-VER-02": ("fabricated_observation", "gate", + "a tool result shares no content with the call that produced it", + "fabrication; the trace is unusable, not repairable"), + # --- format (shared with the suite) ---------------------------------- + "D-FMT-01": ("pairing", "format", + "tool call/result pairing or turn structure is broken", + "structural; cheap to detect, never ship"), + "D-FMT-03": ("final_blob", "format", + "the final answer is a dump of file contents, not a synthesis", + "rewrite the closing turn or drop"), + "D-FMT-04": ("self_talk", "format", + "the assistant narrates a plan and never acts on it", + "drop the step; it teaches narration"), + "D-FMT-05": ("judge_residue", "format", + "verifier text leaked into the trace", + "strip it; it is not the model's output"), + "D-FMT-06": ("clip_marker", "format", + "a truncation marker sits inside a message", + "the trace was clipped; drop or re-pull"), + "D-SAFE-03": ("identity_burn", "safety", + "the assistant claims a harness identity in a trainable span", + "strip the claim mechanically before the gates"), + # --- infra ----------------------------------------------------------- + # 2026-10-26: 191 traces failed triage inside a 37-minute window + # (01:50-02:27) and every one was recorded as D-UNK-01, because the + # per-window error text was discarded and no code existed for "the + # endpoint failed". An infra loss is not a trace defect: it must be + # retried, never counted against the pool's yield. + "D-INF-01": ("llm_unavailable", "infra", + "the model endpoint failed every call for this trace", + "not a trace defect; retry it, do not count it as a loss"), + # --- unknown --------------------------------------------------------- + "D-UNK-01": ("unclassified", "unknown", + "a failure with no matching rule", + "extend classify(); an unclassified loss is a reporting bug"), +} + +# Ordered rules: first match wins. Order matters β€” the specific before the +# general, because "max repair rounds exceeded" also contains "repair". +_RULES: list[tuple[str, re.Pattern[str]]] = [ + # infra first: when a record carries both a network error and a repair + # verdict, the network error is the cause and the verdict is its shadow. + ("D-INF-01", re.compile( + r"network:|urlopen|WinError 10054|connection (?:reset|refused|aborted)" + r"|infra error|failed for every window|rate limit", re.I)), + ("D-TASK-05", re.compile(r"harness dispatch brief|dispatch brief", re.I)), + ("D-TASK-04", re.compile(r"no user task message|no user task", re.I)), + ("D-TASK-02", re.compile(r"reframe", re.I)), + ("D-VER-02", re.compile(r"fabricated tool result|shares no content", re.I)), + ("D-FMT-01", re.compile(r"multi-turn:.*dropped|pairing|orphan", re.I)), + ("D-FMT-03", re.compile(r"final_blob|final answer is a (?:file )?dump", re.I)), + ("D-FMT-04", re.compile(r"self_talk|self-talk", re.I)), + ("D-FMT-05", re.compile(r"judge_residue|judge residue", re.I)), + ("D-FMT-06", re.compile(r"clip_marker|truncated", re.I)), + ("D-SAFE-03", re.compile(r"identity_burn|co-authored-by", re.I)), + ("D-REP-01", re.compile(r"max repair rounds|repair budget", re.I)), + ("D-REP-03", re.compile(r"no writer artifact|writer produced nothing", re.I)), + ("D-VER-01", re.compile(r"judge violations?|unsupported claim", re.I)), + ("D-GATE-01", re.compile(r"^gate:|gate fail", re.I)), +] + + +def classify(stage: Optional[str] = None, + errors: Optional[Iterable[str]] = None, + violations: Optional[Iterable[dict]] = None) -> str: + """Map a failure record to one stable code. + + Errors are checked before the stage, because the error text is the specific + signal and the stage is the bucket. A record with neither is D-UNK-01 β€” + which is a bug in the caller, not a property of the trace. + """ + text = " | ".join(str(e) for e in (errors or [])) + for code, pat in _RULES: + if pat.search(text): + return code + + # violations carry a pillar; a judge violation is the only one we can name + for v in (violations or []): + if str(v.get("pillar", "")).upper() in {"P2", "ANSWER"}: + return "D-VER-01" + + st = (stage or "").strip().lower() + if st.startswith("repair 2"): + return "D-REP-01" + if st.startswith("repair 1"): + return "D-REP-02" + if st.startswith("gates fail"): + return "D-GATE-01" + if st.startswith("init"): + return "D-TASK-04" + if st.startswith("reframe"): + return "D-TASK-02" + return "D-UNK-01" + + +def census(workdir: str) -> dict[str, int]: + """Count failure.json records by defect code, newest verdict per trace. + + Reads only `failure.json` (the latest verdict), not the append-only log, so + a trace that failed three times counts once. + """ + counts: collections.Counter[str] = collections.Counter() + try: + entries = os.listdir(workdir) + except OSError: + return {} + for name in entries: + path = os.path.join(workdir, name, "failure.json") + if not os.path.exists(path): + continue + try: + with open(path, encoding="utf-8") as f: + row = json.load(f) + except (OSError, ValueError): + counts["D-UNK-01"] += 1 + continue + code = row.get("defect") or classify( + row.get("stage"), row.get("errors"), row.get("violations")) + counts[code] += 1 + return dict(counts.most_common()) + + +def describe(code: str) -> str: + """One line for a report: code, name, meaning.""" + name, layer, meaning, _ = DEFECTS.get(code, DEFECTS["D-UNK-01"]) + return f"{code} {name} [{layer}] β€” {meaning}" diff --git a/traceforge/difficulty.py b/traceforge/difficulty.py new file mode 100644 index 0000000000000000000000000000000000000000..48919373b453e48a28553d54c59de252679ee9dd --- /dev/null +++ b/traceforge/difficulty.py @@ -0,0 +1,66 @@ +"""Difficulty bands, measured from the trace itself. + +The card reported topic and language but not difficulty, so a reader could not +tell whether the corpus was 2,000 one-shot edits or 2,000 twenty-call debugging +sessions. Those are different datasets. + +The bands are the shared ones from the agentic-sft-synth suite +(`agentic-task-design` Β§"Difficulty"): the number of tool calls in the ideal +path, which is the only difficulty signal that is cheap, deterministic, and +already in the record. It is a proxy β€” a 20-call mechanical rename is easier +than a 6-call race-condition hunt β€” so the card reports it as a distribution, +never as a score. + + from traceforge.difficulty import band, census + band(7) # -> "D3" + census(traces) # -> {"D1": 12, "D2": 88, ...} +""" + +from __future__ import annotations + +import collections +from typing import Iterable + +# (code, lo, hi, label) β€” hi is inclusive; the last band is open-ended. +BANDS: list[tuple[str, int, int, str]] = [ + ("D1", 0, 2, "one-shot / single edit"), + ("D2", 3, 5, "short loop"), + ("D3", 6, 10, "multi-step"), + ("D4", 11, 20, "long session"), + ("D5", 21, 10**9, "very long session"), +] + +# A trace below this is not a task worth training on: one call and an answer is +# a lookup, and it teaches the model to stop early. +MIN_CALLS = 1 + + +def band(n_calls: int) -> str: + """Map a tool-call count to its band code.""" + for code, lo, hi, _ in BANDS: + if lo <= n_calls <= hi: + return code + return "D5" + + +def label(code: str) -> str: + for c, _, _, lab in BANDS: + if c == code: + return lab + return "?" + + +def count_calls(trace: dict) -> int: + """Tool calls in an exported record (`{"messages": [...], "meta": ...}`).""" + n = 0 + for m in trace.get("messages") or []: + n += len(m.get("tool_calls") or []) + return n + + +def census(traces: Iterable[dict]) -> dict[str, int]: + """Count exported records by band, in band order.""" + counts: collections.Counter[str] = collections.Counter() + for t in traces: + counts[band(count_calls(t))] += 1 + return {code: counts.get(code, 0) for code, _, _, _ in BANDS} diff --git a/traceforge/enrich.py b/traceforge/enrich.py new file mode 100644 index 0000000000000000000000000000000000000000..fddde32d66580fc78d04f18b550370b354180a76 --- /dev/null +++ b/traceforge/enrich.py @@ -0,0 +1,334 @@ +"""Recover the system prompt and tool definitions the harness actually sent. + +The readers drop both, so an exported row has tool_calls naming tools nothing +defines β€” a model trained on it learns to emit `exec` without ever seeing what +`exec` accepts. The data is not lost, it is just not in the Trace: + +- claude ``~/.claude/projects/*/.jsonl`` records ``api-request-shape`` + whose ``shape`` holds the full ``system`` blocks AND the ``tools`` + array the request was made with. +- codex ``~/.codex/sessions/**/rollout-*-.jsonl`` opens with + ``session_meta.payload.base_instructions.text`` β€” the prompt, but + not the schemas (those live in the codex binary). +- hermes ``state.db``: ``sessions.system_prompt_hash`` -> ``system_prompts``. +- nemotron already embeds a ``## Available tools`` block in its system row. + +Where the harness did not record schemas, the export falls back to +``data/toolspec.json`` β€” schemas derived from the arguments the corpus +actually passes, with per-property evidence counts. That is evidence, not +invention, and ``meta.tools_source`` says which one a row got. +""" +from __future__ import annotations + +import glob +import json +import os +import sqlite3 +from typing import Any, Optional + +CLAUDE_GLOB = os.path.expanduser("~/.claude/projects/*/*.jsonl") +CODEX_GLOB = os.path.expanduser("~/.codex/sessions/**/*.jsonl") +CODEX_ZST_GLOB = os.path.expanduser("~/.codex/sessions/**/*.jsonl.zst") +HERMES_DB = os.path.expanduser("~/AppData/Local/hermes/state.db") +TOOLSPEC = "data/toolspec.json" +RAW_DIR = "data/traces/raw" + + +def _sid_from_codex(path: str) -> Optional[str]: + """rollout--.jsonl -> .""" + base = os.path.basename(path) + if not base.startswith("rollout-"): + return None + stem = base + for suf in (".jsonl.zst", ".jsonl"): + if stem.endswith(suf): + stem = stem[: -len(suf)] + break + parts = stem.split("-") + # uuid is the last 5 dash groups + if len(parts) < 5: + return None + return "-".join(parts[-5:]) + + +def _read_claude_shape(path: str) -> tuple[Optional[str], Optional[list]]: + """(system_text, tools) from the first api-request-shape in the file.""" + try: + with open(path, encoding="utf-8") as f: + for line in f: + if "api-request-shape" not in line: + continue + try: + r = json.loads(line) + except Exception: + continue + if r.get("type") != "api-request-shape": + continue + shape = r.get("shape") or {} + blocks = shape.get("system") or [] + text = "\n\n".join( + b.get("text", "") for b in blocks + if isinstance(b, dict) and b.get("text")) + tools = shape.get("tools") or None + return (text or None), tools + except OSError: + pass + return None, None + + +def _read_codex_prompt(path: str) -> Optional[str]: + # 587 of the rollouts on this host are zstd-compressed in place; the codex + # reader already knows how to open them, so reuse it rather than + # reimplementing the decompression here. + try: + from traceforge.readers.codex import _open_rollout + opener = _open_rollout + except Exception: + opener = lambda p: open(p, encoding="utf-8") + try: + with opener(path) as f: + for line in f: + if "session_meta" not in line: + continue + try: + r = json.loads(line) + except Exception: + continue + if r.get("type") != "session_meta": + continue + bi = (r.get("payload") or {}).get("base_instructions") + if isinstance(bi, dict): + return bi.get("text") or None + if isinstance(bi, str): + return bi or None + return None + except OSError: + pass + return None + + +def _hermes_prompts() -> dict[str, str]: + """session_id -> system prompt, via the hash join.""" + out: dict[str, str] = {} + if not os.path.exists(HERMES_DB): + return out + try: + c = sqlite3.connect(f"file:{HERMES_DB}?mode=ro", uri=True) + c.row_factory = sqlite3.Row + prompts = {r["hash"]: r["prompt"] for r in + c.execute("select hash, prompt from system_prompts")} + for r in c.execute( + "select id, system_prompt_hash from sessions " + "where system_prompt_hash is not null"): + p = prompts.get(r["system_prompt_hash"]) + if p: + out[r["id"]] = p + c.close() + except sqlite3.Error: + pass + return out + + +_INDEX_CACHE: Optional[dict] = None + + +def load_index(verbose: bool = False) -> dict[str, dict]: + """session_id -> {system, tools, source}. + + Memoized: `export --by-topic` calls export_chat_jsonl once per topic, and + rebuilding this index each time re-reads 587 compressed rollouts and 401 + claude transcripts β€” ten times the I/O for one export. + """ + global _INDEX_CACHE + if _INDEX_CACHE is not None: + return _INDEX_CACHE + idx: dict[str, dict] = {} + + for p in glob.glob(CLAUDE_GLOB): + sid = os.path.basename(p)[:-6] # strip ".jsonl" (6 chars) + sys_text, tools = _read_claude_shape(p) + if sys_text or tools: + idx[sid] = {"system": sys_text, "tools": tools, + "source": "claude-request-shape"} + + codex_files = (glob.glob(CODEX_GLOB, recursive=True) + + glob.glob(CODEX_ZST_GLOB, recursive=True)) + for p in codex_files: + sid = _sid_from_codex(p) + if not sid or sid in idx: + continue + sys_text = _read_codex_prompt(p) + if sys_text: + idx[sid] = {"system": sys_text, "tools": None, + "source": "codex-base-instructions"} + + for sid, prompt in _hermes_prompts().items(): + if sid in idx: + continue + idx[sid] = {"system": prompt, "tools": None, + "source": "hermes-system-prompts"} + + if verbose: + print(f"enrich index: {len(idx)} sessions " + f"({sum(1 for v in idx.values() if v['tools'])} with real tools)") + _INDEX_CACHE = idx + return idx + + +_SPEC_CACHE: Optional[dict] = None + + +def load_toolspec(path: str = TOOLSPEC) -> dict[str, dict]: + global _SPEC_CACHE + if _SPEC_CACHE is not None: + return _SPEC_CACHE + if not os.path.exists(path): + _SPEC_CACHE = {} + return _SPEC_CACHE + try: + _SPEC_CACHE = (json.load(open(path, encoding="utf-8")) or {}).get("tools") or {} + except Exception: + _SPEC_CACHE = {} + return _SPEC_CACHE + + +_RAW_SYS_CACHE: dict[str, Optional[str]] = {} + + +def _raw_system(sid: str, raw_dir: str = RAW_DIR) -> Optional[str]: + """The system row from the RAW trace, if it has one. + + The improver drops the system message, so a nemotron trace arrives with no + prompt even though its raw carries a `## Available tools` block (12,878 + raws on this host have one). Recover it here rather than re-generating. + """ + if sid in _RAW_SYS_CACHE: + return _RAW_SYS_CACHE[sid] + out = None + path = os.path.join(raw_dir, sid + ".json") + if os.path.exists(path): + try: + t = json.load(open(path, encoding="utf-8")) + for m in t.get("messages") or []: + if m.get("role") == "system" and (m.get("content") or "").strip(): + out = m["content"] + break + except Exception: + out = None + _RAW_SYS_CACHE[sid] = out + return out + + +def _called_names(msgs: list[dict]) -> list[str]: + seen: list[str] = [] + for m in msgs: + for tc in (m.get("tool_calls") or []): + fn = tc.get("function") or {} + name = fn.get("name") or tc.get("name") + if name and name not in seen: + seen.append(name) + return seen + + +def _derived_tool(name: str, spec: dict) -> Optional[dict]: + """OpenAI-format tool definition from the observed-call evidence.""" + s = spec.get(name) + if not s: + return None + props = {} + for k, v in (s.get("properties") or {}).items(): + props[k] = {"type": v.get("type", "string")} + ex = v.get("examples") or [] + if ex: + props[k]["description"] = f"e.g. {ex[0][:120]}" + return { + "type": "function", + "function": { + "name": name, + "description": (f"Derived from {s.get('calls', 0)} observed calls " + f"in this corpus; property types and required " + f"list come from the arguments actually passed."), + "parameters": { + "type": "object", + "properties": props, + "required": list(s.get("required") or []), + }, + }, + } + + +def _to_openai_tool(t: dict) -> Optional[dict]: + """Normalize one recorded tool schema to OpenAI `{type, function}`. + + The claude reader keeps the request's own shape, which is Anthropic: + `{name, description, input_schema}`. Every other schema in the export + (derived ones, the nemotron block) is OpenAI chat format, so a trainer + applying the chat template would meet two incompatible shapes in one + file. Convert at export time rather than at capture: the raw store + should keep what the harness actually sent. + """ + if not isinstance(t, dict): + return None + if t.get("type") == "function" and isinstance(t.get("function"), dict): + return t # already OpenAI + name = t.get("name") + if not name: + return None + params = t.get("input_schema") or t.get("parameters") + if not isinstance(params, dict): + params = {"type": "object", "properties": {}} + return { + "type": "function", + "function": { + "name": name, + "description": t.get("description") or "", + "parameters": params, + }, + } + + +def enrich(rec: dict, sid: str, index: dict, spec: dict) -> dict: + """Attach `tools` and a leading system message to one export record.""" + msgs = rec.get("messages") or [] + rec.setdefault("meta", {}) + entry = index.get(sid) or {} + + # ---- tools: real schemas if the harness recorded them, else derived + tools = entry.get("tools") + if tools: + norm = [x for x in (_to_openai_tool(t) for t in tools) if x] + if norm: + # The recorded request can list fewer tools than the trace + # actually called (a deferred tool, or one the harness added + # after the request was captured). Fill the gap from the + # observed-call evidence so no call names a tool nothing + # defines, and mark the additions so a derived schema is never + # mistaken for a recorded one. + have = {(t.get("function") or {}).get("name") for t in norm} + extra = [d for d in (_derived_tool(n, spec) + for n in _called_names(msgs)) + if d and (d.get("function") or {}).get("name") not in have] + rec["tools"] = norm + extra + rec["meta"]["tools_source"] = entry.get("source") + if extra: + rec["meta"]["tools_derived_extra"] = [ + (d.get("function") or {}).get("name") for d in extra] + else: + names = _called_names(msgs) + derived = [d for d in (_derived_tool(n, spec) for n in names) if d] + if derived: + rec["tools"] = derived + rec["meta"]["tools_source"] = "derived-from-observed-calls" + rec["meta"]["tools_undefined"] = [ + n for n in names if n not in spec] + + # ---- system prompt: prepend unless the trace already opens with one + sys_text = entry.get("system") + if not sys_text: + sys_text = _raw_system(sid) + if sys_text: + entry = dict(entry, source="raw-system-row") + if sys_text and not (msgs and msgs[0].get("role") == "system"): + rec["messages"] = [{"role": "system", "content": sys_text}] + msgs + rec["meta"]["system_prompt_source"] = entry.get("source") + return rec diff --git a/traceforge/export.py b/traceforge/export.py new file mode 100644 index 0000000000000000000000000000000000000000..34bf771dde9ba9d68c6a6d67ecfdd5efcf89c019 --- /dev/null +++ b/traceforge/export.py @@ -0,0 +1,564 @@ +"""Export improved traces to training formats. + +chat_jsonl: OpenAI-style chat fine-tune JSONL β€” one line per trace: + {"messages": [{"role","content"[,"tool_calls"][,"tool_call_id"...]}]} +Reasoning is dropped by default (keep_reasoning=False): the corpus trains on +the assistant's visible turns, and the reasoning channel is not part of the +served format. Pass keep_reasoning=True to retain it under "reasoning". +""" +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import re +from typing import Optional + +from traceforge.trace import Trace + +# Progress narration: the agent announcing what it is about to do instead of +# doing it. Same shape the SFT reframe gate rejects (_AGENT_NARRATION_RE in +# splice.py); duplicated here so export does not import the splice module. +_NARRATION_RE = re.compile( + r"\b(?:I'll|I will|Let me|Next,? I|Now I" + r"|Ich (?:prΓΌfe|fΓΌhre|werde|setze|beginne|starte|nehme))\b", + re.I, +) + + +def _msg_to_chat(m, keep_reasoning: bool = False) -> Optional[dict]: + d: dict = {"role": m.role} + if m.content is not None: + d["content"] = m.content + if keep_reasoning and m.reasoning: + d["reasoning"] = m.reasoning + if m.tool_calls: + d["tool_calls"] = [ + { + "id": tc.id, + "type": "function", + "function": {"name": tc.name, "arguments": tc.arguments}, + } + for tc in m.tool_calls + ] + if m.role == "tool": + d["tool_call_id"] = m.tool_call_id or "" + if m.tool_name: + d["name"] = m.tool_name + return d + + +# Delegation tool names across the reader corpus (Claude 'Task', codex +# thread_spawn, hermes delegate_task): a trace calling one of these is +# an ORCHESTRATOR β€” it divides work across subagents. MASS (arXiv +# 2610.12176) trains on orchestrator AND subagent conversations as +# separate sequences; tagging lets the trainer balance the two roles +# instead of sampling blind. +_DELEGATION_TOOLS = { + "task", "delegate_task", "launchsubagent", "spawn_subagent", + "thread_spawn", "agent", +} + + +def _agent_role(t: Trace) -> Optional[str]: + """'orchestrator' | 'subagent' | None, derived mechanically. + + subagent: the sid carries the reader's sidechain namespace + ('#sa-'), or provenance says the trace IS a + subagent's own thread (source claude-sidechain, or a codex child + rollout). The sid check matters because the improve stage rewrites + traces and overwrites source with 'traceforge-splice', while the + namespaced sid is preserved verbatim. + orchestrator: the trace issues a delegation tool call. + """ + if "#sa-" in (t.session_id or ""): + return "subagent" + src = (t.source or "").lower() + if "sidechain" in src: + return "subagent" + for m in t.messages: + for tc in m.tool_calls: + if (tc.name or "").lower() in _DELEGATION_TOOLS: + return "orchestrator" + return None + + +def export_chat_jsonl( + traces: list[Trace], out_path: str, *, append: bool = False, + keep_reasoning: bool = False, enrich: bool = True, +) -> int: + """Write improved traces as chat-format JSONL. Returns lines written. + + keep_reasoning=False (default) drops per-message reasoning: harness + self-talk is not training signal. Flip on when the training target + includes a reasoning channel. + + enrich=True (default) attaches the `tools` array and, where the harness + recorded one, the system prompt. Without it a row names tools nothing + defines β€” see traceforge/enrich.py. + """ + os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True) + mode = "a" if append else "w" + n = 0 + _idx = None + _spec = None + if enrich: + from traceforge.enrich import load_index, load_toolspec + _idx = load_index() + _spec = load_toolspec() + with open(out_path, mode, encoding="utf-8") as f: + for t in traces: + msgs = [ _msg_to_chat(m, keep_reasoning) for m in t.messages ] + msgs = [m for m in msgs if m and (m.get("content") or m.get("tool_calls") or m.get("tool_call_id"))] + if not msgs: + continue + meta = { + "traceforge_session_id": t.session_id, + "parent_session_id": t.parent_session_id, + "source": t.source, + } + role = _agent_role(t) + if role: + meta["agent_role"] = role + # docs/DATA_QUALITY.md Β§4.1: a retained recovery is labelled so a + # trainer can filter it out and the corpus can report density. + if recovery_label(t): + meta["recovered"] = True + rec = {"messages": msgs, "meta": meta} + if enrich: + from traceforge.enrich import enrich as _enrich + sid = t.session_id + if sid.startswith("ideal-"): + sid = sid[6:] + sid = sid.split("#")[0] + rec = _enrich(rec, sid, _idx or {}, _spec or {}) + # Time sensitivity: measured on the RECORD, because the enriched + # row carries the `tools` array and "did the agent have a + # retrieval tool" is what separates a decaying answer from a + # durable procedure. Absent = not time-sensitive. + try: + from traceforge.timesensitivity import detect as _ts_detect + ts = _ts_detect(rec) + if ts["class"] != "not_time_sensitive": + rec["meta"]["time_sensitivity"] = ts["class"] + if ts.get("temporal_facts"): + rec["meta"]["time_sensitivity_facts"] = ts["temporal_facts"] + if ts.get("needs_rewrite"): + rec["meta"]["time_sensitivity_needs_rewrite"] = True + except Exception: + pass + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + n += 1 + return n + + +# -------------------------------------------------------------------------- +# Corpus-level obligations (docs/DATA_QUALITY.md Β§4.3, rules 6 and 9) +# -------------------------------------------------------------------------- + +# An ENVIRONMENT-caused failure is one the ideal agent could not have +# foreseen: a flaky endpoint, a missing dependency, an exhausted resource. +# A MODEL-caused failure (wrong tool, wrong args, wrong path, bad reasoning) +# is never eligible for the Β§4.1 carve-out β€” the ideal path routes around it. +# +# The match is anchored to the HEAD of the tool output, because a harness +# reports status there. Scanning the whole output matches error text quoted +# inside source code and file listings: measured on this corpus, a loose +# pattern labelled 32% of traces "recovered" where the real figure is ~12%, +# with `502`/`429` matching line numbers (`502- capture_output=True`) and +# `timeout` matching documentation prose. Bare status codes are therefore +# excluded; only an HTTP-context form counts. +_ENV_ERROR_RE = re.compile( + r"(?i)(command not found" + r"|modulenotfounderror|cannot find module" + r"|connection (refused|reset|timed out|aborted)" + r"|(?:error|fatal|wait|request|operation|connection)[^\n]{0,40}" + r"(?:timed out|timeout)" + r"|temporarily unavailable|service unavailable" + r"|bad gateway|too many requests" + r"|network is unreachable|name or service not known" + r"|no space left on device|resource temporarily unavailable" + r"|too many open files)" +) +_MODEL_ERROR_RE = re.compile( + r"(?i)(invalid (argument|json|parameter|request)|unknown tool" + r"|tool not found|jsondecodeerror|unexpected token" + r"|missing required|schema validation|permission denied" + r"|operation not permitted|not a git repository" + r"|is a directory|no such file or directory)" +) +# How much of a tool output counts as its status head. +_ERR_HEAD = 200 +# A grep/nl/`cat -n` listing starts with a line number. Its text is file +# content, not a status line β€” `39:const RESOLVE_TIMEOUT_DETAIL: &str = +# "Timed out opening chat event stream"` is a string literal, not a failure. +_LISTING_RE = re.compile(r"^\s*\d+[:-]") + + +def _call_sig(tc) -> tuple[str, str]: + return ((tc.name or "").lower(), + re.sub(r"\s+", " ", (tc.arguments or "")).strip()[:300]) + + +def recovery_label(t: Trace) -> bool: + """True when the trace retains exactly one environment-caused failure + followed by exactly one corrective step (docs/DATA_QUALITY.md Β§4.1). + + Disqualified by a model-caused error, by a repeated identical failed call + (a retry loop, not a correction), or by an error with no corrective step + after it. Conservative by design: the carve-out is an exception, not a + license, so more than one retained error also disqualifies. + """ + msgs = t.messages + env_hits = 0 + for i, m in enumerate(msgs): + if m.role != "tool": + continue + c = (m.content or "")[:_ERR_HEAD] + if _LISTING_RE.match(c): + continue # file content, not a status line + if not _ENV_ERROR_RE.search(c) or _MODEL_ERROR_RE.search(c): + continue + call = None + for j in range(i - 1, -1, -1): + if msgs[j].role == "assistant" and msgs[j].tool_calls: + call = msgs[j].tool_calls[-1] + break + if call is None: + continue + nxt = None + for j in range(i + 1, len(msgs)): + if msgs[j].role == "assistant" and msgs[j].tool_calls: + nxt = msgs[j].tool_calls[0] + break + if nxt is None: + continue + if _call_sig(nxt) == _call_sig(call): + return False # retry loop: disqualified outright + env_hits += 1 + return env_hits == 1 + + +def dedup_key(t: Trace) -> tuple[str, str]: + """(task, tool-sequence) identity β€” docs/DATA_QUALITY.md rule 9. + + Task text alone over-merges (one brief solved two ways is two examples); + the tool sequence alone over-merges (unrelated tasks share exec>exec). + The pair is what repeats when a session is re-pulled or a split segment + is re-improved. + + The sequence carries a digest of the call ARGUMENTS, not just the tool + names (2026-10-23). Name-only was measurably destructive: on 1,246 + improved traces it removed 18, and every one was a DISTINCT session β€” + 0 with an identical body, 0 with an identical first assistant turn. + Nine separate "watch the CI and report" sessions all reduce to + ``exec>exec>exec`` and were collapsed into one; the arguments are what + distinguishes them (a different job id per session). Arguments survive + re-improvement verbatim except on the splice engine's malformed-args + repair path, so a genuine re-pull still matches; a repaired duplicate + that slips through is one extra near-duplicate, which is far cheaper + than deleting real sessions. + """ + task = "" + for m in t.messages: + if m.role == "user": + task = re.sub(r"\s+", " ", (m.content or "")).strip().lower()[:400] + break + names: list[str] = [] + digest = hashlib.sha1() + for m in t.messages: + for c in (m.tool_calls or []): + name = (c.name or "").lower() + names.append(name) + digest.update(name.encode("utf-8", "replace")) + digest.update(b"\x00") + digest.update((c.arguments or "").encode("utf-8", "replace")) + digest.update(b"\x00") + return task, ">".join(names) + "#" + digest.hexdigest()[:16] + + +def dedup_traces(traces: list[Trace]) -> tuple[list[Trace], list[Trace]]: + """Keep the most complete trace per (task, tool-sequence). + + Returns (kept, removed). Deterministic β€” more messages wins, then more + tool calls, then session id β€” so re-exporting a grown corpus keeps the + same survivor instead of reshuffling the split. + """ + best: dict[tuple[str, str], Trace] = {} + for t in traces: + k = dedup_key(t) + cur = best.get(k) + if cur is None: + best[k] = t + continue + a = (len(t.messages), sum(len(m.tool_calls or []) for m in t.messages)) + b = (len(cur.messages), sum(len(m.tool_calls or []) for m in cur.messages)) + if a > b or (a == b and (t.session_id or "") < (cur.session_id or "")): + best[k] = t + kept = list(best.values()) + removed = [t for t in traces if best.get(dedup_key(t)) is not t] + return kept, removed + + +# -------------------------------------------------------------------------- +# Preference corpus (KTO) β€” docs/DATA_QUALITY.md Β§4.4, rule 10 +# -------------------------------------------------------------------------- +# This is NOT a relabelling of the SFT corpus. KTO needs rejected samples +# from the policy being aligned: a negative the policy would never produce is +# a length/style gradient, not a preference signal. So the flow is staged: +# +# 1. extract_prompts() -> kto_prompts.jsonl: rollout seeds taken from the +# SFT corpus (the cleaned prefix at each assistant turn boundary). +# 2. after SFT, sample N completions per prompt from the SFT model, verify +# each against the trace's own tool evidence, and label it. +# 3. build_kto() -> kto.jsonl: length-matched, bounded-loss contract. +# +# Stage 2 needs a served model and is deliberately not in this module. + +def extract_prompts(traces, *, max_prompts: int = 0, turns: str = "final", + max_prefix_chars: int = 0) -> list[dict]: + """Rollout seeds: (cleaned prefix, reference completion). + + The prefix is the ideal trace's own context, so a rollout starts from the + distribution the model sees at inference β€” not from the raw trace's + messier prefix. + + turns="final" (default) emits one seed per trace: the final assistant + turn, whose outcome the trace's own verification already covers. That is + the turn a KTO label can be trusted on. turns="all" emits every assistant + turn β€” 14k seeds and ~940 MB of prefixes on the current corpus β€” for + step-level preference work where each step has its own verifier. + + max_prefix_chars > 0 keeps the opening task message and the most recent + turns, with an explicit elision marker, so a seed stays a bounded size. + """ + out: list[dict] = [] + for t in traces: + msgs = t.messages + idx = [i for i, m in enumerate(msgs) + if m.role == "assistant" and (m.content or m.tool_calls)] + if turns == "final": + idx = idx[-1:] + for i in idx: + prefix = msgs[:i] + if not any(p.role == "user" for p in prefix): + continue + chat = [m for m in (_msg_to_chat(p) for p in prefix) if m] + if max_prefix_chars: + chat = _clip_prefix(chat, max_prefix_chars) + out.append({ + "session_id": t.session_id, + "turn": i, + "prompt": chat, + "reference": _msg_to_chat(msgs[i]), + }) + if max_prompts and len(out) >= max_prompts: + return out + return out + + +def _clip_prefix(chat: list[dict], limit: int) -> list[dict]: + """Keep the opening task and the most recent turns under `limit` chars.""" + def size(m: dict) -> int: + return len(json.dumps(m, ensure_ascii=False)) + if sum(size(m) for m in chat) <= limit: + return chat + head = chat[:1] + used = size(head[0]) + tail: list[dict] = [] + for m in reversed(chat[1:]): + n = size(m) + if used + n > limit: + break + tail.append(m) + used += n + tail.reverse() + return head + [{"role": "system", + "content": "[earlier turns elided for length]"}] + tail + + +def build_kto(rollouts, *, max_len_ratio: float = 1.5) -> tuple[list[dict], dict]: + """Turn labelled rollouts into KTO rows, length-matched. + + Each rollout: {"prompt": [...], "completion": {...}, "label": bool}. + A negative whose completion is more than `max_len_ratio` times the + shortest desirable completion for the same prompt is dropped β€” otherwise + the only signal is length (arXiv:2403.19159). A prompt with no desirable + sample contributes nothing: an unpaired negative is noise, not a label. + """ + by_prompt: dict[str, list[dict]] = {} + for r in rollouts: + key = json.dumps(r.get("prompt"), sort_keys=True, ensure_ascii=False) + by_prompt.setdefault(key, []).append(r) + rows: list[dict] = [] + dropped_len = dropped_unpaired = 0 + for group in by_prompt.values(): + pos = [r for r in group if r.get("label")] + neg = [r for r in group if not r.get("label")] + if not pos: + dropped_unpaired += len(neg) + continue + ref_len = min(len(json.dumps(r["completion"], ensure_ascii=False)) + for r in pos) + for r in pos: + rows.append({"prompt": r["prompt"], + "completion": r["completion"], "label": True}) + for r in neg: + n = len(json.dumps(r["completion"], ensure_ascii=False)) + if n > max_len_ratio * ref_len: + dropped_len += 1 + continue + rows.append({"prompt": r["prompt"], + "completion": r["completion"], "label": False}) + stats = { + "rows": len(rows), + "desirable": sum(1 for r in rows if r["label"]), + "undesirable": sum(1 for r in rows if not r["label"]), + "prompts": len(by_prompt), + "dropped_length_mismatch": dropped_len, + "dropped_unpaired": dropped_unpaired, + } + return rows, stats + + +def _defect_shape(text: str, task: str) -> Optional[str]: + """Name the defect a dropped assistant turn actually exhibits, or None. + + The triage judge drops a turn for two very different reasons: it is + *wrong* (narration instead of work, a restatement of the brief, a bare + acknowledgement) or merely *redundant* (it repeats something already + said). Only the first is an instructive negative β€” training against + redundancy teaches the model to be terse, not to be correct. Measured on + the 1,456 dropped turns available 2026-10-22: 37% are defect shapes, + 63% are redundancy. + """ + body = (text or "").strip() + if not body: + return None + head = body[:200] + if _NARRATION_RE.match(body) or _NARRATION_RE.search(head): + return "narration" + if task: + a = re.sub(r"\s+", " ", task).strip().lower()[:400] + b = re.sub(r"\s+", " ", body).strip().lower()[:400] + if a and b and difflib.SequenceMatcher(None, a, b).ratio() > 0.6: + return "restates_task" + if len(body) < 120: + return "short_ack" + return None + + +def build_kto_provisional(traces, raw_dir: str, triage_dir: str, *, + max_rows: int = 0) -> tuple[list[dict], dict]: + """Provisional KTO rows for exercising the trainer β€” NOT on-policy. + + Positives: each improved trace's final assistant turn, prompt = the + cleaned prefix. Negatives: assistant turns the triage judge labelled + ``drop``, prompt = that raw trace's own prefix. + + ``docs/DATA_QUALITY.md`` Β§4.4 requires negatives sampled from the policy + being aligned. These come from the original harness agent, so this + artifact is a smoke-test set and a baseline, not the corpus a real KTO + run trains on β€” that one is built from SFT-model rollouts via + ``build_kto``. The negatives are at least *verified* bad (the triage + judge dropped them) rather than merely different, which is why this is + preferable to relabelling raw final answers: measured 2026-10-22, raw and + improved finals differ materially in 229 of 991 traces but are usually + just differently worded, not worse. + """ + rows: list[dict] = [] + seen: set[str] = set() + + def _key(prompt, completion) -> str: + return json.dumps([prompt, completion], sort_keys=True, ensure_ascii=False) + + for t in traces: + msgs = t.messages + idx = [i for i, m in enumerate(msgs) + if m.role == "assistant" and (m.content or m.tool_calls)] + if not idx: + continue + i = idx[-1] + prefix = msgs[:i] + if not any(p.role == "user" for p in prefix): + continue + prompt = [m for m in (_msg_to_chat(p) for p in prefix) if m] + comp = _msg_to_chat(msgs[i]) + if not comp: + continue + k = _key(prompt, comp) + if k in seen: + continue + seen.add(k) + rows.append({"prompt": prompt, "completion": comp, "label": True, + "session_id": t.session_id, "source": "improved"}) + if max_rows and len(rows) >= max_rows: + return rows, _provisional_stats(rows, 0, 0, 0) + + raws = {f[:-5]: os.path.join(raw_dir, f) + for f in os.listdir(raw_dir) if f.endswith(".json")} + n_neg = n_skipped = n_redundant = 0 + for tf in sorted(os.listdir(triage_dir)): + if not tf.endswith(".json"): + continue + try: + with open(os.path.join(triage_dir, tf), encoding="utf-8") as f: + d = json.load(f) + except (OSError, json.JSONDecodeError): + continue + rp = raws.get(d.get("session_id")) + if not rp: + continue + try: + rt = Trace.load(rp) + except Exception: + continue + for lab in d.get("labels") or []: + if lab.get("action") != "drop": + continue + i = lab.get("i") + if not isinstance(i, int) or i >= len(rt.messages): + continue + m = rt.messages[i] + if m.role != "assistant" or not m.content or len(m.content) < 20: + continue + prefix = rt.messages[:i] + if not any(p.role == "user" for p in prefix): + continue + prompt = [c for c in (_msg_to_chat(p) for p in prefix) if c] + comp = _msg_to_chat(m) + if not comp: + continue + task = next((p.content or "" for p in prefix if p.role == "user"), "") + if not _defect_shape(m.content, task): + n_redundant += 1 + continue + k = _key(prompt, comp) + if k in seen: + n_skipped += 1 + continue + seen.add(k) + rows.append({"prompt": prompt, "completion": comp, "label": False, + "session_id": rt.session_id, "source": "triage_drop"}) + n_neg += 1 + if max_rows and len(rows) >= max_rows: + return rows, _provisional_stats(rows, n_neg, n_skipped, n_redundant) + return rows, _provisional_stats(rows, n_neg, n_skipped, n_redundant) + + +def _provisional_stats(rows: list[dict], n_neg: int, n_skipped: int, + n_redundant: int = 0) -> dict: + pos = sum(1 for r in rows if r["label"]) + neg = len(rows) - pos + return { + "rows": len(rows), + "desirable": pos, + "undesirable": neg, + "ratio": round(pos / neg, 2) if neg else None, + "duplicates_skipped": n_skipped, + "skipped_redundant": n_redundant, + "on_policy": False, + "note": ("negatives come from the original harness agent, not the SFT " + "model β€” smoke-test artifact only (DATA_QUALITY.md Β§4.4)"), + } \ No newline at end of file diff --git a/traceforge/grounding.py b/traceforge/grounding.py new file mode 100644 index 0000000000000000000000000000000000000000..068b2efd1ab7f123bb9dbaf83b72d60456545990 --- /dev/null +++ b/traceforge/grounding.py @@ -0,0 +1,256 @@ +"""Grounding validation: every salient fact in an improved trace must exist +in its source trace. This is the mechanical no-hallucination gate (P4). + +Salient facts = numbers-with-units, file paths, hex ids, quoted technical +strings. Sound but cheap: a trace that invents paths or counts fails here. + +Tool-result fidelity: an ideal trace that makes the same tool call must +NOT carry a fabricated result. For each ideal tool row whose call signature +(tool name + argument keys) matches a real call, the result must share +content with the real output β€” exact line match first, then 25-char window +containment (the improver sees a clipped rendering and may compress, so a +wholly-unfamiliar output is fabrication, but paraphrase/reformat is not). +""" +from __future__ import annotations + +import json +import re + +from traceforge.trace import Trace + +FACT_PATTERNS = [ + # counts with units (verdicts, row/line counts, repeats) + re.compile(r"\b\d{1,6}\s+(?:PASS|SUSPECT|FAIL|CLEAN|lines|rows|files|chars|bytes|errors)\b"), + # file paths + re.compile(r"(?:[\w./-]+/)?[\w.-]+\.(?:py|ts|tsx|js|rs|go|svelte|json|jsonl|yaml|yml|toml|md|txt|csv|sql|sh)\b"), + # hex ids (commit shas, call ids) + re.compile(r"\b[0-9a-f]{7,40}\b"), +] + +# facts the improver may legitimately introduce (its own bookkeeping) +ALLOWLIST_PATTERN = re.compile(r"^(?:call_|ideal-)") + +# rendering artifacts, not world claims: the compact renderer's clip +# markers (also matches the "+N chars clipped" tail of read_file-style +# outputs). Stripped from the ideal blob before fact extraction. +CLIP_MARKER_RE = re.compile(r"…?\[\+[\d,.]+ ?(?:chars|bytes) clipped\]") + +# unit claims that describe a rendering's compression, not the world. +# "48613 chars" in an ideal trace is the improver mirroring its own +# input rendering β€” never a salient fact. +NON_WORLD_UNIT_RE = re.compile(r"^\d{1,6}\s+(?:chars|bytes)$") + +# fuzzy-containment window for tool-result fidelity +_CONTAIN_WINDOW = 25 + +# Verified on live promoted/attempted corpora (2026-09-28): fact patterns +# extract GLUED tokens from ideal traces whose underlying path IS in the +# source corpus β€” the top grounding false-kill shapes: +# 'a///Users/x/y.py' β€” unified-diff prefix "a/" glued onto a path that +# sits after a literal \n in a JSON-embedded diff +# 'nbackend/x/y.py' β€” a literal \n escape's 'n' glued onto the next +# path segment of a JSON-embedded tool output +# The real corpus stores the same output with REAL newlines, so it +# extracts the clean path and the glued variant looks invented. Before a +# fact is declared missing, glue-stripped variants are re-tested; only +# variants long enough to remain distinctive are considered. +def _glue_variants(fact: str) -> list[str]: + """The fact itself plus glue-stripped spelling variants. + + The BASE fact is always included (it is self-grounding when present + in the corpus); only the stripped variants must be distinctive + (len >= 8) to count, so short facts are never grounded by a + meaninglessly-short fragment. + """ + out = [fact] + if "/" in fact: + stripped = re.sub(r"^[a-z]?/+", "", fact) # glue char + slash run + if stripped != fact and len(stripped) >= 8: + out.append(stripped) + if len(fact) > 1 and len(fact) - 1 >= 8: + out.append(fact[1:]) # single leading char (n-glue, a/, b/) + # count+unit facts: the corpus may spell the same claim with the + # unit BEFORE the numbers ("lines old/new: 12875 12875" β€” verified + # false-kill 1fa6a3e5: identical bytes in ideal and raw, the + # NUMBER+UNIT pattern simply creates a token order the corpus lacks). + m = re.match(r"^(\d{1,6})\s+(lines|rows|files|errors)$", fact) + if m: + out.append(f"{m.group(2)} old/new: {m.group(1)} {m.group(1)}") + out.append(f"{m.group(2)}: {m.group(1)}") + return out + + +def _ideal_blob(ideal: Trace) -> str: + """Content + tool args of the ideal trace, clip markers stripped.""" + blob = "\n".join( + (m.content or "") for m in ideal.messages + ) + "\n" + "\n".join( + tc.arguments for m in ideal.messages for tc in m.tool_calls + ) + return CLIP_MARKER_RE.sub("", blob) + + +def _real_corpus(real: Trace) -> str: + parts = [m.content or "" for m in real.messages] + parts += [tc.arguments for m in real.messages for tc in m.tool_calls] + return "\n".join(parts) + + +def grounding_coverage(ideal: Trace, real: Trace) -> tuple[float, list[str]]: + """Return (fraction_grounded, missing_facts). 1.0 == fully grounded.""" + corpus = _real_corpus(real) + ideal_blob = _ideal_blob(ideal) + missing: list[str] = [] + total = 0 + grounded = 0 + for pat in FACT_PATTERNS: + for f in set(pat.findall(ideal_blob)): + if ALLOWLIST_PATTERN.match(f): + continue + if NON_WORLD_UNIT_RE.match(f): + continue # rendering artifact (mirrored clip marker) + total += 1 + # JSON-embedded outputs glue a stray char onto the head of a + # real path (verified false-kill shapes, see _glue_variants). + # A fact is grounded when IT or a glue-stripped variant sits + # in the real corpus. + if any(v in corpus for v in _glue_variants(f)): + grounded += 1 + else: + missing.append(f) + if total == 0: + return 1.0, [] + return grounded / total, missing + + +def validate_grounding( + ideal: Trace, real: Trace, min_coverage: float = 0.95 +) -> list[str]: + """Gate: fail when salient-fact grounding is below the threshold.""" + errs: list[str] = [] + cov, missing = grounding_coverage(ideal, real) + if cov < min_coverage: + shown = ", ".join(m[:80] for m in missing[:8]) + errs.append( + f"fact grounding {cov:.0%} < {min_coverage:.0%}; " + f"facts not in source trace: {shown}" + ) + errs.extend(tool_result_fidelity_errors(ideal, real)) + return errs + + +def _tool_output_lines(raw: str) -> list[str]: + """Substantive lines of a tool output; unwraps execute_code-style JSON.""" + try: + d = json.loads(raw) + if isinstance(d, dict) and isinstance(d.get("output"), str): + raw = d["output"] + except (json.JSONDecodeError, TypeError): + pass + return [l.strip() for l in raw.split("\n") if len(l.strip()) >= 25] + + +def _call_signature(name: str, arguments: str) -> str: + """Stable-ish call fingerprint: tool name + sorted argument string.""" + try: + a = json.loads(arguments or "{}") + keys = ",".join(sorted(a)) if isinstance(a, dict) else str(sorted(a)) + except json.JSONDecodeError: + keys = (arguments or "")[:80] + return f"{name}({keys})" + + +def _windows(line: str, size: int = _CONTAIN_WINDOW): + """All size-length windows of a line, stepped every quarter-window.""" + if len(line) < size: + return [line] + step = max(1, size // 4) + return [line[i:i + size] for i in range(0, len(line) - size + 1, step)] + + +def _result_grounded(lines: list[str], rlines: list[str]) -> bool: + """Do the ideal result's lines share content with the real output? + + Two tiers: exact stripped-line match (verbatim reuse), then 25-char + window containment (compressed/paraphrased reuse of real evidence). + """ + rset = set(rlines) + if any(l in rset for l in lines): + return True + rjoin = "\n".join(rlines) + for l in lines: + for w in _windows(l): + if w in rjoin: + return True + return False + + +def _real_output_index(real: Trace) -> dict[str, list[tuple[str, list[str]]]]: + """Per tool name: [(call signature, substantive output lines)].""" + idx: dict[str, list[tuple[str, list[str]]]] = {} + for m in real.messages: + for tc in m.tool_calls: + res = next( + (r for r in real.messages + if r.role == "tool" and r.tool_call_id == tc.id), + None, + ) + if res is not None and res.content: + sig = _call_signature(tc.name, tc.arguments) + idx.setdefault(tc.name, []).append( + (sig, _tool_output_lines(res.content)) + ) + return idx + + +def _ideal_call_sigs(ideal: Trace) -> dict[str, str]: + """tool_call_id -> signature, for the ideal trace's own calls.""" + out: dict[str, str] = {} + for m in ideal.messages: + for tc in m.tool_calls: + out[tc.id] = _call_signature(tc.name, tc.arguments) + return out + + +def tool_result_fidelity_errors(ideal: Trace, real: Trace) -> list[str]: + """Fabricated tool results: same-call results sharing no content. + + An ideal tool row whose tool was really called fails when its result + shares neither an exact substantive line nor a 25-char window with + any real output of that tool. The error names the ideal call's + signature, shows what the real trace's output actually looked like, + and gives a concrete rewrite target β€” a vague "be more grounded" + never repairs anything. + """ + idx = _real_output_index(real) + sigs = _ideal_call_sigs(ideal) + errs: list[str] = [] + for m in ideal.messages: + if m.role != "tool": + continue + cands = idx.get(m.tool_name or "", []) + if not cands: + continue # tool never called in real trace: coverage handles it + lines = _tool_output_lines(m.content or "") + if not lines: + continue + for _sig, rlines in cands: + if _result_grounded(lines, rlines): + break # grounded against (at least) one real result + else: + sig = sigs.get(m.tool_call_id or "", "?") + # prefer the real output of the SAME call when one exists, + # else the tool's first real output β€” concrete repair target + same = [rl for s, rl in cands if s == sig] + sample = (same or [rl for _s, rl in cands])[0] + head_txt = " | ".join(l[:70] for l in sample[:2]) or "(empty)" + head = lines[0][:90] + errs.append( + f"fabricated tool result [{m.tool_name}]: result shares no " + f"content with any real {m.tool_name} output " + f"(e.g. {head!r}). The ideal call is {sig}; the real trace's " + f"output looks like: {head_txt!r}. Rewrite this result to " + "quote the real trace's actual output (compressed is fine, " + "but every substantive line must come from the real output)." + ) + return errs \ No newline at end of file diff --git a/traceforge/identity.py b/traceforge/identity.py new file mode 100644 index 0000000000000000000000000000000000000000..8e33a4be43e1f187dd322efd1128c1a419cef951 --- /dev/null +++ b/traceforge/identity.py @@ -0,0 +1,131 @@ +"""Identity-neutrality and refusal gates. + +The reference must not burn in an assistant identity ("I am Claude", +"as an AI made by …") and must not teach refusals β€” a model scored against +it should neither claim a vendor self-identity nor reproduce refusal +phrasings. These gates are deterministic regex checks over ASSISTANT +prose (system/user rows are never rewritten by the pipeline anyway and +tool results are verbatim facts, so only assistant content can carry a +learnable self-claim). + +Domain content rule: a trace whose TASK is about a model/vendor (e.g. +"compare chatgpt vs claude APIs", "debug Claude-Code traces") may +LEGITIMATELY contain those names β€” that is content, not identity. Only +SELF-referential claims are gated: first-person identity patterns +("I am Claude", "I'm an AI", "trained by Anthropic", …) and refusal +patterns ("I can't help with that", "against my guidelines", …). +Mentions like "CLAUDE.md exists in the repo" are never touched. +""" +from __future__ import annotations + +import json +import re +from typing import Optional + +# First-person identity self-claims. Anchored on first-person verbs so +# "the trace contains Claude-Code traces" (third person) never matches. +IDENTITY_RE = re.compile( + r"(" + r"co-?authored-?by:\s*" + r"(claude|chatgpt|gpt-?[45o]|glm|gemini|qwen|gemma|llama|" + r"deepseek|copilot|sonnet|opus|anthropic|openai)\b" + r"|\b(i am|i'?m|my name is|call me)\s+" + r"(claude|chatgpt|gpt-?[45o]|glm|gemini|qwen|gemma|llama|" + r"deepseek|copilot|sonnet|opus)\b" + r"|\b(i am|i'?m)\s+(an?|the)\s+" + r"(ai|llm|large language model|language model|chatbot|assistant " + r"(?:model|developed|made|trained|created)\b[^.]{0,40})" + r"|\b(trained|made|created|developed|built)\s+(by|at)\s+" + r"(anthropic|openai|google|deepmind|meta|mistral|alibaba|z\.ai|" + r"zhipu|nvidia|x\.?ai)\b[^.]{0,30}\bby me\b" + r"|\bi\s+(was\s+)?(trained|fine-?tuned)\s+(by|on|with)\b" + r"|\bas an?\s+(ai|llm|language model)(?=[,.])" + r"|\bclaude\b\s+(here|speaking|at your service)" + r"|\boperated by\s+(anthropic|openai|z\.ai)\b" + r"|\bich\s+(bin|heiße|werde genannt)\s+" + r"(claude|chatgpt|gpt-?[45o]?|glm|gemini|qwen|gemma|llama|" + r"deepseek)\b" + r"|\bals\s+ki\b[^.]{0,40}\bich\b" + r"|\bich\s+(bin|werde)\s+(eine?|ein)\s+" + r"(ki|kΓΌnstliche?\s+intelligenz|sprachmodell)\b" + r")", + re.IGNORECASE, +) + +# Refusal phrasings that would teach the model to refuse. Anchored on +# first-person refusal verbs; a task ABOUT moderation policy may quote +# refusals third-person β€” those don't match. +REFUSAL_RE = re.compile( + r"(" + r"\bi\s+(can'?t|cannot|can not|won'?t|will not|am unable to|" + r"'?m not able to)\s+" + r"(help|assist|comply|provide|create|generate|write|build)\b" + r"(\s+(that|this|with (that|this)))?" + r"|\bi\s+(don'?t|do not)\s+feel\s+comfortable\b" + r"|\b(against|outside)\s+my\s+(guidelines|principles|" + r"programming|policies)\b" + r"|\bi'?m\s+sorry,?\s+but\s+i\s+(can'?t|cannot|won'?t)\b" + r"|\bi\s+must\s+(respect|follow)\s+my\s+" + r"(guidelines|principles|programming)\b" + r"|\bich\s+(kann|kann nicht|werde)\s+.*(nicht helfen|" + r"nicht unterstΓΌtzen|nicht unterstΓΌtzen bei)\b" + r"|\bich\s+(kann|mag)\s+das\s+nicht\b" + r"|\bdas\s+verstâßt\s+gegen\s+meine\b" + r"|\bmeine\s+richtlinien\b[^.]{0,20}\bverbieten\b" + r")", + re.IGNORECASE, +) + +# Refusals in the opening task (user explicitly asks for something +# harmful and the source model refused) are visible in USER rows too β€” +# but we never edit user rows, so a harmful-request trace is not +# promotable at all; that class dies at triage/judge level instead. + + +def _find_hits(content: str, rx: re.Pattern) -> list[str]: + return [m.group(0)[:100] for m in rx.finditer(content)] + + +def validate_identity( + messages: list, + *, + identity_rx: Optional[re.Pattern] = None, + refusal_rx: Optional[re.Pattern] = None, +) -> list[str]: + """Return gate errors for assistant identity/refusal text ([] = ok). + + ``messages`` may be Trace.messages (Message objects with .role/ + .content) or plain dicts (role/content keys) β€” the audit calls this + with dicts, the improver with Messages. + """ + irx = identity_rx or IDENTITY_RE + rrx = refusal_rx or REFUSAL_RE + errors: list[str] = [] + for i, m in enumerate(messages): + role = m.get("role") if isinstance(m, dict) else m.role + if role != "assistant": + continue + c = (m.get("content") if isinstance(m, dict) else m.content) or "" + tc = m.get("tool_calls") if isinstance(m, dict) else m.tool_calls + texts = [c] if c else [] + for call in tc or []: + if isinstance(call, dict): + args = call.get("arguments", call.get("args")) + else: + args = getattr(call, "arguments", None) + if args is None: + args = getattr(call, "args", None) + if isinstance(args, str): + texts.append(args) + elif args: + texts.append(json.dumps(args, ensure_ascii=False)) + for c2 in texts: + if not c2: + continue + for hit in _find_hits(c2, irx): + errors.append( + f"[{i}] identity self-claim in assistant text/tool args: {hit!r}") + for hit in _find_hits(c2, rrx): + errors.append( + f"[{i}] refusal phrasing in assistant text/tool args: {hit!r}") + return errors \ No newline at end of file diff --git a/traceforge/improver/__init__.py b/traceforge/improver/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d50df44969d1a3517ca7949db6d42d9809b1d0b --- /dev/null +++ b/traceforge/improver/__init__.py @@ -0,0 +1,4 @@ +from traceforge.improver.harness import ImproverResult, improve_trace +from traceforge.improver.prompt import build_prompt + +__all__ = ["ImproverResult", "improve_trace", "build_prompt"] \ No newline at end of file diff --git a/traceforge/improver/digest_prompts.py b/traceforge/improver/digest_prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..d323aed66bdd41759cd399d53e5f2b2d175def31 --- /dev/null +++ b/traceforge/improver/digest_prompts.py @@ -0,0 +1,105 @@ +"""Prompts for the long-trace (map-digest-write) pipeline. + +Heavy sessions (up to ~10M tokens) can never fit into one context. The +pipeline: digest each chunk in parallel -> merge digests into a chunk map +-> writer builds the ideal trace from the map, looking up exact verbatim +tool outputs from on-disk chunk files by reference. +""" +from __future__ import annotations + +CHUNK_DIGEST_HEADER = """You are a trace-digest agent. You see ONE chunk of a +long real agent session. boil it down for a downstream writer agent that +will rewrite the whole session as a flawless "ideal trace". + +Produce JSON with EXACTLY these keys, written to: {digest_path} +{{ + "chunk": , + "task_scope": "", + "narrative": "", + "mistakes": [ + {{"kind": "failed_tool_call|retry_loop|wrong_turn|redundant|user_feedback", + "detail": "", "turn_ref": "<[chunk-local msg idx]>"}} + ], + "key_outputs": [ + {{"ref": "chunk{n}-{i}", "tool": "", "significance": "", "loc": ""}} + ], + "claim_worthy": [""] +}} + +RULES: +- Story-first: the downstream writer only sees your digest, not the chunk. + Everything load-bearing must appear in narrative/ key_outputs. +- Mistakes matter most: every failed call, retry, wrong turn, and user + correction must be listed β€” those are what the ideal path must avoid. +- Do NOT include raw tool outputs in digest. Reference them as refs + (chunk{n}-{i}); the writer resolves refs against on-disk chunk files. +- ideal_path: describe in "narrative" what SHOULD have happened in this + chunk's scope (perfect first-time route per guidelines). +- No preamble: write the JSON, validate it parses when re-read, then reply + "DIGEST DONE chunk {n}". +""" + +MAP_MERGE_HEADER = """You are a trace-map merge agent. You receive digests +of ALL chunks of one long real agent session, in order. Build the unified +session map for the ideal-trace writer. + +Write JSON to: {map_path} +{{ + "task": "", + "first_user_message": "", + "structure": [ "" ], + "mistakes": [ "" ], + "key_outputs": [ "" ], + "claim_worthy": [ "" ], + "chunk_index": [ "" ] +}} + +RULES: +- Deduplicate conservative: keep every load-bearing item; drop only + literal repeats introduced by chunk overlap. +- first_user_message must be VERBATIM and COMPLETE β€” the writer will + reproduce it byte-exact in the ideal trace. +- ideal story: structure = the perfect-first-time route through the whole + session (per guidelines: mistakes removed, minimal necessary calls). +- Write the JSON, validate it parses, reply "MAP DONE". +""" + +WRITER_LONG_HEADER = """You are the ideal-trace writer for a LONG session. +You get: (1) the session map (digests merged), (2) access to on-disk chunk +files that hold the raw rendered chunks, (3) the guidelines. + +Your job: write the ideal-world version of the whole session as JSON to: +{output_path} + +Procedure: +1. Read the guidelines file: {guidelines_path} +2. Read the session map at: {map_path} +3. Read guidelines Β§3 pillars. Then look up chunk data on demand: + - The chunk files live at: {chunk_dir} (files chunk_001.txt, ...). + - Use refs from the map's key_outputs to locate the exact tool outputs + you want to reuse VERBATIM (search the chunk files). + - Read a chunk file only when you need exact content from it. Do not + read all chunk files β€” only what refs point you to. +4. Write the ideal trace (same JSON schema as the improver contract: + messages array with role/content/tool_calls/tool_call_id/tool_name). + Apply ALL quality pillars. Reuse tool outputs verbatim when the ideal + path makes the same call (P4); resolve via refs. +5. Verify: re-read what you wrote; check JSON; check tool pairing; check + the first user message matches the map's first_user_message byte-exact. +6. Reply "WRITER DONE" + a summary paragraph. + +Scope limits (unchanged): no environment archaeology, no subagents β€” +chunk files and guidelines are your only inputs. +""" + +DIGEST_SUFFIX = """ + +REAL TRACE CHUNK {n}: +{chunk} +""" \ No newline at end of file diff --git a/traceforge/improver/harness.py b/traceforge/improver/harness.py new file mode 100644 index 0000000000000000000000000000000000000000..dabd4f69c6281f8ff6bc14ca511b12eb756614f6 --- /dev/null +++ b/traceforge/improver/harness.py @@ -0,0 +1,850 @@ +"""Runs the improver as a Hermes Agent one-shot session (the harness). + +Design: for each trace we spawn + + hermes chat --query-file --in -Q + [--improver-model M] [--max-turns N] [--pass-session-id] + +The improver is a full agent: it can read the guidelines file, inspect the +workdir, write its output with a tool, and re-validate it β€” agentic +self-improvement inside the generation loop. The harness never trusts the +improver's chat reply; it reads back the JSON file the improver wrote, +then structural validation + repair loop run outside the model. +""" +from __future__ import annotations + +import dataclasses +import json +import os +import shutil +import subprocess +import time +from typing import Optional + +from traceforge import chunker +from traceforge.config import Config +from traceforge.grounding import CLIP_MARKER_RE as CLIP_MARKER_PROBE +from traceforge.improver.prompt import build_prompt, write_prompt_file +from traceforge.trace import Message, Trace, json_loads_lenient + +_TOOL_TAIL_ERROR = "last message role is 'tool' (expected assistant)" +_EMPTY_ROW_RE = "message with empty content and no tool calls" + + +def _only_tool_tail(errs: list[str]) -> bool: + """True when the artifact's ONLY defect is that it ends on a tool row.""" + return bool(errs) and all(_TOOL_TAIL_ERROR in e for e in errs) + + +def _drop_empty_assistant_rows(improved: Trace) -> bool: + """Remove assistant rows with no content and no calls. Returns changed. + + The writer emits these as separators when it loses the thread. They + carry zero signal and a trainee must never learn to produce them, so + dropping is strictly better than paying a repair round to rewrite the + whole trace around one blank row. + """ + kept = [m for m in improved.messages + if not (m.role == "assistant" and not m.tool_calls + and not (m.content or "").strip())] + if len(kept) == len(improved.messages): + return False + improved.messages = kept + return True + + +def _append_synth_final(cfg: Config, improved: Trace, + source: Trace) -> Optional[Trace]: + """Close a trace that the writer left mid-call with a synthesized answer. + + The writer stops after a tool call in ~3% of traces (output cap or turn + cap), leaving an artifact that is one row short of valid. A full repair + round rewrites the whole trace and usually reproduces the same cut, so + the cheap fix is the targeted one: ask for the closing answer only, + grounded in the tail that already exists. Same helper the splice path + uses for raw traces that were cut off mid-run. + """ + from traceforge.splice import _resolve_endpoint, _synthesize_final_answer + + try: + base, key = _resolve_endpoint(cfg) + except Exception as e: + print(f" tool-tail synth skipped: endpoint {type(e).__name__}: {e}") + return None + ans, err = _synthesize_final_answer( + cfg, improved, source.first_task_message() or "", base, key) + if not ans: + print(f" tool-tail synth failed: {(err or 'empty answer')[:120]}") + return None + improved.messages.append(Message(role="assistant", content=ans)) + return improved + +CLIP_MARKER_ERROR = ( + "artifact contains rendering clip markers (…[+N chars clipped]) " + "with no verifiable counts β€” reviewers reject these. Rewrite the tool " + "results verbatim from outputs.jsonl (contiguous portions fine)." +) + +# Per-workdir file recording which gate rounds already PASSED for which +# artifact content digest (see _load_gate_cache). +GATE_CACHE_FILE = "gate_cache.json" + +# Bump whenever a JUDGE/GATE STANDARD changes (new pillar rule, ANSWER +# note wording that changes verdicts, regex gate changes) β€” cached +# passes recorded under an older standard must not replay. The value +# rides in the cache file; _load_gate_cache discards stale files. +GATE_CACHE_STANDARDS = "2026-10-08.1" # +answer-focus rule (P1 FOCUS, over-answering = poor) # +write-JSON deliverable forms, tool-tail carry, exact-args dup reads # +mixed-turn surgical drops, usage-error detect, planner >=2 + write-fix chains # +identity scans tool args (Co-Authored-By), degenerate prose gate # +judge sees reasoning channel + + +def _clip_markers_present(candidate) -> bool: + """True if any message content carries a rendering clip marker.""" + blob = "\n".join((m.content or "") for m in candidate.messages) + return bool(CLIP_MARKER_PROBE.search(blob)) + + +def _artifact_digest(candidate) -> str: + """Stable content hash of an improved artifact. + + Gate passes are keyed on this: structural/grounding/verify/review + PASSes are only replayed from cache when the artifact byte-content is + unchanged (the file may still have been rewritten with identical + bytes β€” that's fine, it's the same artifact). + """ + import hashlib + + h = hashlib.sha256() + for m in candidate.messages: + h.update((m.role or "").encode("utf-8")) + h.update(b"\x00") + h.update((m.content or "").encode("utf-8")) + h.update(b"\x01") + if m.tool_call_id: + h.update(m.tool_call_id.encode("utf-8")) + h.update(b"\x01") + if m.tool_name: + h.update(m.tool_name.encode("utf-8")) + h.update(b"\x02") + if m.tool_calls: + for tc in m.tool_calls: + h.update((tc.id or "").encode("utf-8")) + h.update((tc.name or "").encode("utf-8")) + h.update((tc.arguments or "").encode("utf-8")) + h.update(b"\x03") + return h.hexdigest() + + +class ImproverResult: + def __init__( + self, + *, + session_id: str, + ok: bool, + improved: Optional[Trace], + reply: str, + hermes_session_id: Optional[str] = None, + errors: Optional[list[str]] = None, + attempts: int = 0, + duration_s: float = 0.0, + ): + self.session_id = session_id # source trace session id + self.ok = ok + self.improved = improved + self.reply = reply + self.hermes_session_id = hermes_session_id + self.errors = errors or [] + self.attempts = attempts + self.duration_s = duration_s + + def to_dict(self) -> dict: + return { + "session_id": self.session_id, + "ok": self.ok, + "errors": self.errors, + "hermes_session_id": self.hermes_session_id, + "attempts": self.attempts, + "duration_s": round(self.duration_s, 1), + } + + +def _run_hermes( + cfg: Config, prompt_path: str, workdir: str +) -> tuple[int, str, Optional[str], str]: + """Invoke hermes chat one-shot; returns (exit_code, stdout, session_id).""" + cmd = [ + "hermes", "chat", + "--query-file", prompt_path, + "--in", workdir, + "-Q", + "--format", "text", + "--pass-session-id", + ] + if cfg.model: + cmd += ["-m", cfg.model] + if cfg.provider: + cmd += ["--provider", cfg.provider] + if cfg.max_turns: + cmd += ["--max-turns", str(cfg.max_turns)] + if cfg.run_budget: + cmd += ["--run-budget", str(cfg.run_budget)] + if cfg.skills: + cmd += ["-s", ",".join(cfg.skills)] + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=cfg.harness_timeout_s, + cwd=workdir, + ) + session_id = None + for line in (proc.stdout or "").splitlines(): + if line.strip().startswith("session_id:"): + session_id = line.split(":", 1)[1].strip() or None + return proc.returncode, proc.stdout, session_id, proc.stderr + + +def _parse_improved(path: str) -> Optional[Trace]: + """Parse the improver-written improved.json into a Trace, leniently.""" + if not os.path.exists(path): + return None + with open(path, "r", encoding="utf-8") as f: + raw = f.read() + # tolerate fenced code blocks from chatty models + raw = raw.strip() + if raw.startswith("```"): + raw = raw.split("\n", 1)[1] if "\n" in raw else "" + if raw.rstrip().endswith("```"): + raw = raw.rstrip()[: -3] + data = json_loads_lenient(raw) + if not isinstance(data, dict): + return None + msgs = data.get("messages") + if not isinstance(msgs, list): + return None + from traceforge.trace import Message, ToolCall + + out = [] + for d in msgs: + if not isinstance(d, dict) or "role" not in d: + return None + m = { + "role": d["role"], + "content": d.get("content"), + "reasoning": d.get("reasoning"), + "tool_call_id": d.get("tool_call_id"), + "tool_name": d.get("tool_name"), + } + m = {k: v for k, v in m.items() if v is not None} + if "tool_calls" in d and isinstance(d["tool_calls"], list): + m["tool_calls"] = [ + { + "id": str(tc.get("id", "")), + "name": str(tc.get("name", "")), + "arguments": tc.get("arguments") or "{}", + } + for tc in d["tool_calls"] + if isinstance(tc, dict) + ] + out.append(Message.from_dict(m)) + return Trace( + session_id="improved-placeholder", # set by caller + messages=out, + improved=True, + ) + + +def _append_status(cfg: Config, sid: str, status: dict) -> None: + """Append a stage-level status row to the manifest.""" + import json as _json + + os.makedirs(os.path.dirname(cfg.manifest_path), exist_ok=True) + row = {"session_id": sid, **status} + with open(cfg.manifest_path, "a", encoding="utf-8") as f: + f.write(_json.dumps(row, ensure_ascii=False) + "\n") + + +def _load_gate_cache(workdir: str) -> dict: + """Read the workdir's gate-pass cache (digest -> list of gate rows). + + Batches die at arbitrary points (turn interruptions, terminal close, + SIGTERM to the worker pool); without this, every verify/review round + already PAID FOR on an unchanged artifact is re-run from scratch on + the next resume. The cache is keyed by the artifact's content digest + so a rewritten (different) artifact never inherits stale passes. + Cached passes from an OLDER standards version are discarded β€” judge + rules evolve, and a verdict recorded under old rules must not + replay against new ones. + """ + path = os.path.join(workdir, GATE_CACHE_FILE) + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + return {} + if data.get("standards") != GATE_CACHE_STANDARDS: + return {} + return data + except (OSError, json.JSONDecodeError): + return {} + + +def _save_gate_cache(workdir: str, cache: dict) -> None: + """Persist the gate-pass cache atomically (workdir, per trace).""" + path = os.path.join(workdir, GATE_CACHE_FILE) + tmp = path + ".tmp" + cache = dict(cache) + cache["standards"] = GATE_CACHE_STANDARDS + with open(tmp, "w", encoding="utf-8") as f: + json.dump(cache, f, ensure_ascii=False) + os.replace(tmp, path) + + +def _cache_gate_pass(cache: dict, digest: str, stage: str, round_no: int) -> None: + """Record a gate PASS for one artifact digest.""" + rows = cache.setdefault(digest, []) + key = f"{stage}:{round_no}" + if key not in rows: + rows.append(key) + + +def _acquire_workdir_lock(workdir: str) -> bool: + """O_EXCL pid lockfile; locks left by dead processes are stolen. + + Two concurrent improve batches working the same sid corrupt each + other (shared improved.json / verdict.json β€” the verifier unlinks its + verdict file before each run) and burn double model time. The lock + makes one-improver-per-workdir structural, not a user discipline. + """ + lock_path = os.path.join(workdir, ".lock") + my_pid = str(os.getpid()) + for _ in range(3): + try: + fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + try: + os.write(fd, my_pid.encode()) + finally: + os.close(fd) + return True + except FileExistsError: + try: + other = open(lock_path, encoding="utf-8").read().strip() + except OSError: + continue + pid = int(other) if other.isdigit() else -1 + alive = False + if pid > 0: + try: + os.kill(pid, 0) + alive = True + except ProcessLookupError: + alive = False + except PermissionError: + alive = True # exists, not ours: assume alive + except OverflowError: + alive = False + if alive: + return False + try: + os.unlink(lock_path) # stale lock from a dead run: steal + except OSError: + pass + return False + + +def _release_workdir_lock(workdir: str) -> None: + """Drop our own lock (never another live run's).""" + lock_path = os.path.join(workdir, ".lock") + try: + with open(lock_path, encoding="utf-8") as f: + if f.read().strip() == str(os.getpid()): + os.unlink(lock_path) + except OSError: + pass + + +def _write_outputs_index( + cfg: Config, trace: Trace, workdir: str, max_chars: int = 250_000 +) -> str: + """Full real tool outputs as outputs.jsonl, keyed by rendering index. + + The compact rendering clips each tool result to a short head; P4 + demands verbatim reuse, so the improver needs the unclipped text. + One JSON object per line: {"i": N, "tool", "args", "output"} where N + is the same [N] message index the rendering shows β€” the improver can + correlate a clipped result in its prompt with the full text here. + + max_chars is a byte-blow safety valve (measured corpus max real + output: ~111KB), NOT a compression budget: the 20k cap this used to + apply hid exactly the mid-tail evidence (p95 = 20k) that P4 demands + the improver quote β€” on exactly the big traces where review kills + artifacts β€” forcing composed pseudo-outputs. Outputs beyond the + safety valve carry an honest total-marker, never a fake count. + """ + path = os.path.join(workdir, "outputs.jsonl") + arg_of: dict[str, str] = {} + for m in trace.messages: + for tc in m.tool_calls: + arg_of[tc.id] = tc.arguments or "{}" + with open(path, "w", encoding="utf-8") as f: + for i, m in enumerate(trace.messages, 1): + if m.role != "tool" or not m.content: + continue + out = m.content + if len(out) > max_chars: + out = out[:max_chars] + f" …[+{len(out) - max_chars} chars total]" + f.write(json.dumps({ + "i": i, + "tool": m.tool_name or "?", + "args": arg_of.get(m.tool_call_id or "", "")[:400], + "output": out, + }, ensure_ascii=False) + "\n") + return path + + +def _generate_single(cfg, trace, rendered, workdir, output_path, errors, prev_path, sid): + """One single-shot improver invocation. Returns (candidate, reply, hsid, code, stderr) or raises Timeout.""" + from traceforge.classify import detect_language + prompt = build_prompt( + rendered_trace=rendered, + guidelines_path=cfg.guidelines_path, + output_path=output_path, + outputs_path=os.path.join(workdir, "outputs.jsonl"), + errors=errors, + prev_path=prev_path, + language=detect_language(trace.first_task_message() or ""), + ) + prompt_path = write_prompt_file(cfg.prompts_dir, sid, prompt) + try: + code, stdout, hsid, stderr = _run_hermes(cfg, prompt_path, workdir) + except subprocess.TimeoutExpired as e: + raise e + reply = (stdout or "").strip() + return _parse_improved(output_path), reply, hsid, code, stderr + + +def improve_trace( + cfg: Config, + trace: Trace, + *, + max_attempts: int = 2, + validator=None, + status_cb=None, +) -> ImproverResult: + """Run the agentic improver on one trace, with a validation repair loop. + + Long traces (rendered size > cfg.chunk_max_chars) go through the staged + map-digest-write pipeline instead of the single shot. Every admitted + trace passes: N independent verification rounds (web fact-check) and + M independent review rounds (quality-pillar review) β€” all must PASS. + + ``status_cb`` (optional) receives short human stage strings + ("improving", "digest", "verify 1/2", ...) for live progress display. + """ + from traceforge.trace import Trace as _T + from traceforge.validate import validate_structural + from traceforge.grounding import validate_grounding + from traceforge.improver.verify import run_verifier + from traceforge.improver.review import run_review + + def _status(s: str) -> None: + if status_cb is not None: + status_cb(s) + + started = time.time() + sid = trace.session_id + workdir = os.path.join(cfg.improver_workdir, sid) + os.makedirs(workdir, exist_ok=True) + # One improver per workdir, structurally: concurrent batches on the + # same sid race on improved.json/verdict.json and pay the model twice. + if not _acquire_workdir_lock(workdir): + return ImproverResult( + session_id=sid, ok=False, improved=None, reply="", + hermes_session_id=None, + errors=["workdir locked by a live improver run β€” skipping"], + attempts=0, duration_s=0.0, + ) + try: + return _improve_locked(cfg, trace, max_attempts, status_cb) + finally: + _release_workdir_lock(workdir) + + +def _improve_locked( + cfg: Config, trace: Trace, max_attempts: int, status_cb +) -> ImproverResult: + from traceforge.trace import Trace as _T + from traceforge.validate import validate_structural + from traceforge.grounding import validate_grounding + from traceforge.improver.verify import run_verifier + from traceforge.improver.review import run_review + + def _status(s: str) -> None: + if status_cb is not None: + status_cb(s) + + started = time.time() + sid = trace.session_id + workdir = os.path.join(cfg.improver_workdir, sid) + rendered = trace.render_compact( + user_chars=cfg.user_chars, + assistant_chars=cfg.assistant_chars, + tool_result_chars=cfg.tool_result_chars, + tool_args_chars=cfg.tool_args_chars, + include_reasoning=cfg.include_reasoning, + ) + output_path = os.path.join(workdir, "improved.json") + # Full unclipped tool outputs on disk: P4's verbatim-reuse rule is + # only satisfiable when the improver can actually see the full text. + outputs_path = _write_outputs_index(cfg, trace, workdir) + + errors: list[str] = [] + prev_path: Optional[str] = None + attempts = 0 + improved: Optional[Trace] = None + reply = "" + hermes_session_id: Optional[str] = None + # long-trace attempts don't run hermes chat directly (stages do); the + # loop below must not touch single-shot navigation vars on that path + code = 0 + stderr = "" + + for attempt in range(1, max_attempts + 1): + attempts = attempt + candidate = None + # Fast-path: when a previous run already left an artifact in this + # workdir, re-adjudicate it first (regate). Only pay the model for + # a fresh write when nothing usable exists on disk. This makes + # "rerun after a gate change" nearly free instead of a full + # re-write of every trace. + if attempt == 1 and os.path.exists(output_path): + _status("regate") + candidate = _parse_improved(output_path) + # Static pre-flight: an artifact that still carries rendering + # clip markers is dead on arrival at review (reviewers kill + # invented counts; grounding silently strips them). Fail it + # here β€” one static check instead of 4 deep-thinker + # sessions re-discovering it. + if candidate is not None and _clip_markers_present(candidate): + candidate = None + errors = [CLIP_MARKER_ERROR] + # keep the failed artifact as the repair reference: the + # next attempt must FIX it (REPAIR_HEADER + errors), not + # blind-rewrite from a fresh TASK_HEADER that would + # silently drop the specific clip-marker error. + prev_path = output_path + if candidate is not None: + reply = "REGATE (existing artifact)" + code = 0 + stderr = "" + elif len(rendered) <= cfg.chunk_max_chars: + _status("improving" if attempt == 1 else f"repair {attempt}") + try: + candidate, reply_one, hsid, code, stderr = _generate_single( + cfg, trace, rendered, workdir, output_path, errors, + prev_path, sid, + ) + except subprocess.TimeoutExpired: + errors = [f"harness timeout after {cfg.harness_timeout_s}s"] + continue + reply = reply_one + hermes_session_id = hsid + if reply.startswith("REJECTED"): + errors = [f"rejected by improver: {reply[:300]}"] + break + else: + # long-trace staged pipeline: digest -> map -> writer + from traceforge.improver import longtrace + + chunk_dir = os.path.join(workdir, "chunks") + if prev_path is None: + _status("digest+map") + chunks = chunker.chunk_trace( + trace, + max_chars=cfg.chunk_max_chars, + overlap_messages=cfg.chunk_overlap_messages, + render_kwargs={ + "user_chars": cfg.user_chars, + "assistant_chars": cfg.assistant_chars, + "tool_result_chars": cfg.tool_result_chars, + "tool_args_chars": cfg.tool_args_chars, + "include_reasoning": cfg.include_reasoning, + }, + ) + if not chunks: + errors = ["chunking produced no chunks"] + continue + longtrace.write_chunks(chunk_dir, chunks, trace) + _status(f"digest 0/{len(chunks)}") + digests, derrs = longtrace.digest_all( + cfg, trace, chunks, workdir, + progress_cb=lambda done, total: _status( + f"digest {done}/{total}" + ), + ) + if derrs: + errors = [f"digest failures: {derrs}"] + continue + _status(f"map ({len(chunks)} chunks)") + map_d, merr = longtrace.build_map(cfg, trace, digests, workdir) + if map_d is None: + errors = [f"map merge failed: {merr}"] + continue + _append_status(cfg, sid, { + "stage": "map", "chunks": len(chunks), + "approx_tokens": sum(c.approx_chars for c in chunks) // 4, + }) + # writer (also used for repair attempts on long traces) + map_d = longtrace._json_file(os.path.join(workdir, "session_map.json")) + if map_d is None: + errors = ["session map missing after long-trace stage"] + continue + _status("writer") + candidate, werr = longtrace.run_writer( + cfg, trace, map_d, chunk_dir, workdir, output_path, + repair_errors=errors if prev_path else None, + ) + if candidate is None: + errors = [f"long-trace writer failed: {werr}"] + continue + reply = "WRITER DONE" + # The artifact on disk is the source of truth β€” NOT the exit code. + # The improver may hit its turn cap after writing a perfectly valid + # file (e.g. right after a repair); validate what exists regardless. + candidate = _parse_improved(output_path) + if candidate is None: + if code != 0: + stderr_tail = f"; stderr: {stderr.strip()[-500:]}" if stderr else "" + errors = [ + f"hermes chat exited with code {code} and " + f"improved.json missing/unparseable{stderr_tail}" + ] + else: + errors = [ + "improved.json missing, unparseable, or missing 'messages'" + ] + continue + # Static clip-marker gate on EVERY path, not just regate: writers + # and repairs can still copy markers out of the clipped rendering. + # A marker artifact is dead on arrival at review (and grounding + # silently forgives it) β€” fail fast here instead of burning + # verifier + reviewer sessions on it. + if _clip_markers_present(candidate): + errors = [f"clip-marker gate: {CLIP_MARKER_ERROR}"] + # repair must fix THIS artifact (REPAIR_HEADER + errors), not + # blind-rewrite from a TASK_HEADER that drops the specifics + prev_path = output_path + continue + prev_path = output_path + improved = candidate + # Pipeline framing markers ("[trace segment k of n]", "[subagent + # brief β€” role: ...]") belong to the improver's INPUT; strip them + # before the gates so the echo check compares against the real task + # and the promoted artifact carries no scaffolding. + from traceforge.validate import strip_scaffold_markers + if strip_scaffold_markers(improved): + improved.save(output_path) + errs = validate_structural(improved) + if errs and any(_EMPTY_ROW_RE in e for e in errs): + # blank assistant rows are pure noise β€” drop them and re-check + # rather than paying a full rewrite to remove one empty row. + # Runs even when other errors are present: a trace with a blank + # row AND a tool tail must get both fixes, not neither. + if _drop_empty_assistant_rows(improved): + improved.save(output_path) # repair must start from the fix + errs = validate_structural(improved) + if not errs: + _status("dropped empty assistant rows") + if _only_tool_tail(errs): + # writer stopped mid-call: one row short of valid. Synthesize + # the closing answer instead of paying a full repair round + # that reproduces the same cut (measured: 49/1535 traces). + fixed = _append_synth_final(cfg, improved, trace) + if fixed is not None: + improved = fixed + # persist BEFORE re-validating: a later repair round reads + # prev_path from disk, so an in-memory-only fix is thrown + # away and the writer reproduces the same tool tail. + improved.save(output_path) + errs = validate_structural(improved) + if not errs: + _status("final answer (tool-tail synth)") + if not errs: + errs = validate_grounding(improved, trace) + if not errs and (cfg.verify or cfg.review): + # Gate-pass cache: rounds that already PASSED for THIS exact + # artifact content are replayed instead of re-paid. Survives + # batch interruptions (turn cap, SIGTERM, terminal close); a + # rewritten artifact has a different digest and re-runs all + # rounds fresh. + digest = _artifact_digest(improved) + gate_cache = _load_gate_cache(workdir) + gate_pass_keys = set(gate_cache.get(digest, [])) + + def _gate_pass(stage: str, round_no: int) -> None: + _cache_gate_pass(gate_cache, digest, stage, round_no) + _save_gate_cache(workdir, gate_cache) + + verify_failures: list[str] = [] + # ---- N independent verification rounds (web fact-check) + # Strict admission: every round must end PASS. A transient ERROR + # (timeout, unparseable verdict) gets ONE in-place retry; a + # second ERROR surfaces as an error row β€” never silent + # under-verification, never a model repair for infra issues. + if cfg.verify: + passes = 0 + vr = 1 + while vr <= cfg.verify_rounds: + if f"verify:{vr}" in gate_pass_keys: + passes += 1 # already adjudicated on identical bytes + _status(f"verify {vr}/{cfg.verify_rounds} (cached)") + _append_status(cfg, sid, { + "stage": "verify", "round": vr, + "verdict": "PASS", "cached": True, + }) + vr += 1 + continue + _status(f"verify {vr}/{cfg.verify_rounds}") + vres = run_verifier(cfg, output_path, workdir) + if vres.verdict == "PASS": + passes += 1 + _gate_pass("verify", vr) + _append_status(cfg, sid, { + "stage": "verify", "round": vr, + "verdict": "PASS", + "claims_checked": vres.claims_checked, + }) + vr += 1 + continue + if vres.verdict == "FAIL": + claims = "\n".join( + f'- "{u.get("claim", "?")}" β€” {u.get("why", "?")}' + for u in vres.unsupported[:10] + ) + verify_failures.append( + f"[verify r{vr}] external claims unsupported:\n{claims}" + ) + _append_status(cfg, sid, { + "stage": "verify", "round": vr, "verdict": "FAIL", + "claims_checked": vres.claims_checked, + }) + break # concrete FAIL: repair next attempt + # transient ERROR: one in-place retry of the same round + retry = run_verifier(cfg, output_path, workdir) + if retry.verdict == "PASS": + passes += 1 + _gate_pass("verify", vr) + _append_status(cfg, sid, { + "stage": "verify", "round": vr, + "verdict": "PASS", "retried": True, + "claims_checked": retry.claims_checked, + }) + vr += 1 + continue + if retry.verdict == "FAIL": + claims = "\n".join( + f'- "{u.get("claim", "?")}" β€” {u.get("why", "?")}' + for u in retry.unsupported[:10] + ) + verify_failures.append( + f"[verify r{vr}] external claims unsupported:\n{claims}" + ) + _append_status(cfg, sid, { + "stage": "verify", "round": vr, "verdict": "FAIL", + "retried": True, + "claims_checked": retry.claims_checked, + }) + break + verify_failures.append( + f"[verify r{vr}] infra error x2: " + f"{vres.error or 'verifier error'} / " + f"{retry.error or 'verifier error'}" + ) + _append_status(cfg, sid, { + "stage": "verify", "round": vr, "verdict": "ERROR", + "error": ( + f"{vres.error or 'verifier error'} / " + f"{retry.error or 'verifier error'}" + )[:300], + }) + break + # ---- M independent review rounds (quality pillars) + review_failures: list[str] = [] + task_snip = trace.first_task_message() or "(no user message)" + if not verify_failures and cfg.review: + for rr in range(1, cfg.review_rounds + 1): + if f"review:{rr}" in gate_pass_keys: + _status(f"review {rr}/{cfg.review_rounds} (cached)") + _append_status(cfg, sid, { + "stage": "review", "round": rr, "verdict": "PASS", + "cached": True, + }) + continue + _status(f"review {rr}/{cfg.review_rounds}") + rres = run_review( + cfg, output_path, task_snip, workdir, rr + ) + if not rres.ok() and rres.verdict == "ERROR": + rres = run_review( + cfg, output_path, task_snip, workdir, rr + ) # one in-place retry on transient infra error + if rres.ok(): + _gate_pass("review", rr) + _append_status(cfg, sid, { + "stage": "review", "round": rr, "verdict": "PASS", + }) + continue + if rres.verdict == "FAIL": + review_failures.append( + "[review r%d] pillar violations:\n%s" + % (rr, "\n".join(rres.violations[:10])) + ) + _append_status(cfg, sid, { + "stage": "review", "round": rr, "verdict": "FAIL", + }) + break # concrete FAIL: repair next attempt + review_failures.append( + f"[review r{rr}] infra error x2: {rres.error or 'review error'}" + ) + _append_status(cfg, sid, { + "stage": "review", "round": rr, "verdict": "ERROR", + "error": (rres.error or "review error")[:300], + }) + break + errs = verify_failures + review_failures + if errs: + # Infra failures (verifier/reviewer down) are NOT content errors: + # a model repair cannot fix them. Fail fast without re-running + # the improver so no model time is wasted on infrastructure. + if any("infra error x2" in e for e in errs): + return ImproverResult( + session_id=sid, ok=False, improved=None, reply=reply, + hermes_session_id=hermes_session_id, + errors=errs, attempts=attempts, + duration_s=time.time() - started, + ) + errors = errs + ( + [f"note: hermes exit code {code}"] if code != 0 else [] + ) + continue + improved.session_id = f"ideal-{sid}" + improved.parent_session_id = sid + # cfg.model None = default profile; the exact model is recoverable + # from hermes_session_id (recorded in the manifest). + improved.model = cfg.model or "hermes-default" + improved.source = "hermes-one-shot" + return ImproverResult( + session_id=sid, ok=True, improved=improved, reply=reply, + hermes_session_id=hermes_session_id, errors=[], + attempts=attempts, duration_s=time.time() - started, + ) + + return ImproverResult( + session_id=sid, ok=False, improved=None, reply=reply, + hermes_session_id=hermes_session_id, errors=errors, + attempts=attempts, duration_s=time.time() - started, + ) + + +def cleanup_workdir(cfg: Config, sid: str, keep: bool = False) -> None: + wd = os.path.join(cfg.improver_workdir, sid) + if not keep and os.path.isdir(wd): + shutil.rmtree(wd, ignore_errors=True) \ No newline at end of file diff --git a/traceforge/improver/longtrace.py b/traceforge/improver/longtrace.py new file mode 100644 index 0000000000000000000000000000000000000000..fcff2edefa2d0c268ea6315385eb1abdb764b78b --- /dev/null +++ b/traceforge/improver/longtrace.py @@ -0,0 +1,182 @@ +"""Long-trace pipeline: chunk -> parallel digests -> map -> writer. + +For sessions whose compact rendering exceeds cfg.chunk_max_chars, the +single-shot improver is replaced by this staged pipeline. The trace's raw +chunks are ALSO written to disk (chunk_dir/chunk_NNN.txt) so the writer +can look up exact tool outputs by reference, keeping each agent context +small even for ~10M-token sessions. +""" +from __future__ import annotations + +import concurrent.futures as cf +import json +import os +import re +import subprocess +import threading +import time +from typing import Optional + +from traceforge import chunker +from traceforge.config import Config +from traceforge.trace import Message, Trace, json_loads_lenient +from traceforge.improver.digest_prompts import ( + CHUNK_DIGEST_HEADER, + MAP_MERGE_HEADER, + WRITER_LONG_HEADER, +) + + +def _hermes_run( + prompt_path: str, workdir: str, max_turns: int, timeout: int +) -> tuple[int, str, str]: + cmd = [ + "hermes", "chat", + "--query-file", prompt_path, + "--in", workdir, + "-Q", + "--max-turns", str(max_turns), + ] + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, cwd=workdir + ) + return proc.returncode, proc.stdout or "", proc.stderr or "" + + +def _json_file(path: str) -> Optional[dict]: + if not os.path.exists(path): + return None + return json_loads_lenient(open(path, encoding="utf-8").read()) + + +def _put_prompt(cfg: Config, name: str, text: str) -> str: + os.makedirs(cfg.prompts_dir, exist_ok=True) + p = os.path.join(cfg.prompts_dir, name) + with open(p, "w", encoding="utf-8") as f: + f.write(text) + return p + + +def write_chunks( + chunk_dir: str, chunks: "list[chunker.Chunk]", trace: Trace +) -> None: + """Persist raw chunks + a ref->output index for verbatim lookup.""" + os.makedirs(chunk_dir, exist_ok=True) + for c in chunks: + with open( + os.path.join(chunk_dir, f"chunk_{c.index:03d}.txt"), "w", + encoding="utf-8", + ) as f: + f.write(c.rendered) + + +def _digest_one( + cfg: Config, trace: Trace, c: chunker.Chunk, workdir: str +) -> tuple[Optional[dict], str]: + """Run one digester session over one chunk. Returns (digest, error).""" + digest_path = os.path.join(workdir, f"digest_{c.index:03d}.json") + prompt = CHUNK_DIGEST_HEADER.format( + digest_path=digest_path, n=c.index + ) + c.rendered + pp = _put_prompt(cfg, f"digest_{trace.session_id}_c{c.index:03d}.txt", prompt) + try: + code, _out, _err = _hermes_run( + pp, workdir, cfg.digest_max_turns, cfg.digest_timeout_s + ) + except subprocess.TimeoutExpired: + return None, f"chunk {c.index} digest timeout" + d = _json_file(digest_path) + if d is None: + return None, f"chunk {c.index} digest unparseable (exit {code})" + return d, "" + + +def digest_all( + cfg: Config, trace: Trace, chunks: list, workdir: str, + progress_cb=None, +) -> tuple[list[dict], list[str]]: + """Digest all chunks, cfg.digest_workers in parallel. + + progress_cb(done, total) fires after each chunk completes (from the + worker threads) β€” used for live progress display. + """ + errors: list[str] = [] + digests: list[dict] = [] + done = 0 + lock = threading.Lock() + with cf.ThreadPoolExecutor(max_workers=cfg.digest_workers) as ex: + futs = { + ex.submit(_digest_one, cfg, trace, c, workdir): c.index + for c in chunks + } + for fut in cf.as_completed(futs): + idx = futs[fut] + d, err = fut.result() + with lock: + done += 1 + if progress_cb is not None: + try: + progress_cb(done, len(chunks)) + except Exception: + pass # display callback must never break the stage + if err: + errors.append(err) + elif d is not None: + digests.append(d) + digests.sort(key=lambda d: int(d.get("chunk", 0))) + return digests, errors + + +def build_map( + cfg: Config, trace: Trace, digests: list[dict], workdir: str +) -> tuple[Optional[dict], str]: + """Merge digests into the unified session map.""" + map_path = os.path.join(workdir, "session_map.json") + blob = "\n\n".join(json.dumps(d, ensure_ascii=False, indent=1) for d in digests) + prompt = MAP_MERGE_HEADER.format(map_path=map_path) + "\nDIGESTS:\n" + blob + pp = _put_prompt(cfg, f"map_{trace.session_id}.txt", prompt) + try: + code, _out, _err = _hermes_run( + pp, workdir, cfg.digest_max_turns, cfg.digest_timeout_s + ) + except subprocess.TimeoutExpired: + return None, "map merge timeout" + m = _json_file(map_path) + if m is None: + return None, f"session map unparseable (exit {code})" + return m, "" + + +def run_writer( + cfg: Config, trace: Trace, map_d: dict, chunk_dir: str, + workdir: str, output_path: str, + repair_errors: Optional[list[str]] = None, +) -> tuple[Optional[Trace], str]: + """Run the ideal-trace writer over the session map.""" + map_path = os.path.join(workdir, "session_map.json") + head = WRITER_LONG_HEADER.format( + output_path=output_path, + guidelines_path=cfg.guidelines_path, + map_path=map_path, + chunk_dir=chunk_dir, + ) + if repair_errors: + head = ( + "Your previous ideal trace FAILED validation. Fix these exact " + f"errors:\n- " + "\n- ".join(repair_errors[:20]) + + "\n\nThe previous attempt is at " + output_path + + " β€” read it, fix it in place.\n\n" + head + ) + pp = _put_prompt(cfg, f"writer_{trace.session_id}.txt", head) + try: + code, _out, _err = _hermes_run( + pp, workdir, cfg.max_turns or 16, cfg.harness_timeout_s + ) + except subprocess.TimeoutExpired: + return None, "writer timeout" + from traceforge.improver.harness import _parse_improved + + t = _parse_improved(output_path) + if t is None: + return None, f"writer output unparseable (exit {code})" + return t, "" \ No newline at end of file diff --git a/traceforge/improver/prompt.py b/traceforge/improver/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..905b41cf6b4213dc6c3f7a11807cfb2c59115f2d --- /dev/null +++ b/traceforge/improver/prompt.py @@ -0,0 +1,241 @@ +"""Builds the improver prompt: task spec + guidelines + compact trace. + +The improver is a Hermes one-shot session with tools. It is told to +1) read the guidelines file itself (agentic grounding), +2) study the rendered real trace, +3) write the ideal-world rewrite to /improved.json, +4) self-check JSON validity before finishing. +""" +from __future__ import annotations + +import os + +TASK_HEADER = """You are a reference improver agent. Your job: rewrite the +REAL agent trace below into the IDEAL-WORLD version of the same task β€” what a +perfect execution would have looked like, following the guidelines exactly. +The rewrite is the REFERENCE a benchmark scores the source model against: the +less you have to change, the better that model is. Change only what the +guidelines require. + +SCOPE β€” this is a REWRITE, not a re-execution. Hard limits: +- The rendered trace below is your ONLY source of truth. Do NOT investigate + the original environment: no reading the actual project files, no git + history, no running commands from the trace, no browsing. You cannot and + must not verify against the real world. +- Work alone. Do not spawn subagents or delegate. +- Tool budget (this is all you need): (1) read the guidelines file, + (2) write improved.json, (3) re-read it to validate, (4) reply. +- Go straight to writing. Analysis of the trace happens in your reply + reasoning, not in exploratory tool calls. + +QUALITY PILLARS (binding β€” the guidelines file is the full spec): +- P1 Efficient but deep reasoning: high-signal decision logic per turn + (why this call, what the result implies, what to verify next); no filler. + FOCUS: answer exactly what was asked β€” for a follow-up question, + deliver the new piece only; never re-present facts the reader already + got in an earlier turn of the same session. + SUBSTANCE: the final answer presents the concrete result the task asked + for β€” the actual numbers, names, paths, values or code from the tool + evidence β€” never a description of what was retrieved. "I pulled the + nutrition data and the recipe" is poor; the data and the recipe ARE the + answer. Naming the work instead of showing its result fails the task + even when every tool call was correct. +- P2 No committed mistakes: FIRST list the real trace's mistakes (failed + tool calls, retry loops, wrong turns, user feedback/corrections), then + route the ideal path around every single one β€” correct action the first + time; never show the failure or the repair. WHY: the assistant turns are + the loss-bearing target; a mistake in the target is imitated, a mistake in + the context is not. ONE exception (guidelines Β§4.1): an environment-caused + failure the ideal agent could not foresee may stay as a byte-verbatim tool + result, with exactly one corrective step in the next assistant turn and + recorded as a retained recovery by the harness. Model-caused errors (wrong + tool, wrong args, wrong path, bad reasoning) are never eligible. +- P3 Perfect tool calls: every call necessary and correctly parameterized; + batch independent calls in parallel; no exploratory flailing; no call + whose result the narrative doesn't use. BATCHING IS STRUCTURAL: when the + real trace issues two calls in separate assistant turns but the second + does not consume the first's output, put BOTH tool_calls in ONE assistant + turn and let the two results follow in order. Do not preserve the source's + one-call-per-turn rhythm β€” that rhythm is the defect the judge rejects. + A duplicate call (same tool, same arguments) is dropped, never batched. +- P4 Fact-driven: reuse real tool outputs verbatim where the ideal path + makes the same call; never invent files, commands, or results; no fake + success β€” the final answer is grounded in the trace's real evidence. + The FULL unclipped text of every real tool output is in {outputs_path} + (one JSON object per line; "i" = the [N] message index in the rendered + trace). When the ideal path makes the same call as the real trace, take + the result from there (a contiguous portion is fine). NEVER copy the + rendering's clip markers ("…[+N chars clipped]") into the ideal trace β€” + they are display artifacts of THIS prompt, not content. If you compress + a tool result, every substantive line must still be quoted from the + real output. + CONSTRUCTION ORDER (do this in this exact order): (1) decide the ideal + call path; (2) for every tool call on it, COPY the result content from + the matching outputs.jsonl row FIRST β€” byte-verbatim or a contiguous + slice; (3) only then write the reasoning and final answer AROUND those + real results. A tool result you cannot quote from outputs.jsonl is NOT + writable: either reroute the ideal path so that call is unnecessary, or + drop the claim. NEVER assemble a tool result from fragments you believe + are true β€” stitching individually-plausible lines is fabrication, the + single most common rejection reason. + External-world claims (versions, API behavior, dates, technical facts) + only when the trace itself proves them β€” otherwise leave them out. + A web-searching fact-checker reviews every external claim AFTER you + finish; unsupported claims send the trace back for repair. +- P5 Language: one language per trace (the first user message's language); + perfect grammar in assistant prose; keep the user message verbatim. + NEVER open by restating the brief. The first assistant turn must do + work β€” a tool call, a decision, a finding β€” not re-present the user's + own text back to them. A message that is a near-copy of the opening + brief (>=85% similar) is rejected mechanically before the judge sees it. +- P6 No refusals: complete the task. If it is genuinely un-completable, + reply starting "REJECTED: " and write no output file. + +TRAINING-SIGNAL CONTRACT (why the pillars are shaped this way): this corpus +is SFT β€” loss is computed on assistant tokens only; tool results, user turns +and system prompts are masked context. Never put a failure, a fabricated +error, or a "try again" narrative in an assistant turn. Never fabricate a +tool output to make a recovery work. This corpus is not preference or RL +data: do not invent rejected samples, preference pairs, or reward labels. + +PROCEDURE: (1) read the guidelines, (2) mine the mistakes, (3) write +{output_path} in one go, (4) re-read it and validate (JSON, pairing, +pillars), (5) summarize the changes. + +Ground rules: +- Read the guidelines file first: {guidelines_path} + Treat it as the authoritative spec. If it has no rule for a case you hit, + follow the six pillars and the training-signal contract, and use your best + judgment β€” never invent a rule that contradicts them. +- CRITICAL β€” intent fidelity: the ideal trace's FIRST user message must be + the real trace's first USER TASK message, verbatim, byte-exact (skip + handoff artifacts β€” `[CONTEXT COMPACTION]` Hermes rows and "This + session is being continued…" Claude Code rows; they carry no task). + If the real trace contains no user task message, do not invent + one: reply REJECTED: no user task in source trace and write no file. + You may merge subsequent real user messages into a shorter, cleaner + dialogue, but the OPENING task must stay the same task. +- Preserve the original task intent (first user message) and any real context + (file paths, commands, tool results) that a correct execution would also + have produced. You MAY compress and clean the trace's own tool results, + but you must NOT invent facts the trace does not support. +- Keep realistic tool_io. The improved trace must still read like a genuine + agent session β€” just a flawless one: efficient tool use, clear reasoning, + no flailing, no exploration dead-ends, no policy violations, precise + final answer. + +Output contract (CRITICAL): +- Write the improved trace as JSON to exactly this path: {output_path} +- Schema: + {{ + "messages": [ + {{"role": "user"|"assistant"|"tool", + "content": "...", + "tool_calls": [{{"id": "...", "name": "...", + "arguments": ""}}] // assistant only + "tool_call_id": "...", "tool_name": "..." // tool rows only + }} + ] + }} + Optional per-message key: "reasoning" (string) for thinking text. +- tool_calls[].arguments must be a JSON-encoded STRING (OpenAI style). +- Every assistant tool_call id must have exactly one matching tool row + (tool_call_id), and vice versa. +- Validate your output before finishing: re-read the file you wrote with a + tool, confirm it parses as JSON and the schema + pairing rules hold. +- Then reply with a one-paragraph summary of what you changed and why. +""" + +REPAIR_HEADER = """Your previous attempt at improving this trace FAILED +validation. Fix it. + +Validation errors: +{errors} + +Previous attempt: {prev_path} +(Rendered real trace and guidelines still apply, restated below.) + +The FULL unclipped text of every real tool output is in {outputs_path} +(one JSON object per line; "i" = the [N] message index in the rendered +trace). Grounding failures mean: take the ideal tool results from there +verbatim (contiguous portion fine) β€” do not paraphrase real outputs. +NEVER replace a bad result by assembling plausible lines: copy byte- +verbatim from outputs.jsonl, or drop the call/claim. Clip markers +("…[+N chars clipped]") are display artifacts β€” never reproduce them, +and remove any that the previous attempt copied in. + +SCOPE (same hard limits as before): this is a REWRITE, not an +investigation. Do NOT read project files, repo source, gate code, or git +history. Do NOT spawn subagents. Budget: (1) read the previous +improved.json once, (2) read the specific outputs.jsonl rows cited in +the errors above, (3) write the corrected file, (4) re-read it once to +validate, (5) reply. NO other tool calls β€” every forensic check the +reviewer already did for you is spelled out in the errors. + +WRITE MECHANICS: to overwrite an existing file, read the ENTIRE current +file first (not a page of it), or write via a Python script +(execute_code / terminal) that builds the new JSON with json.dumps β€” +never hand-type large JSON. If a write is refused, re-read the full file +and retry the write immediately; do not spend turns analyzing the +refusal. + +Write the corrected improved trace JSON to: {output_path} +Validate the file yourself before finishing, then summarize the fixes. +""" + + +def build_prompt( + *, + rendered_trace: str, + guidelines_path: str, + output_path: str, + outputs_path: "str | None" = None, + repair: "os._Environ | None" = None, + errors: list[str] | None = None, + prev_path: str | None = None, + language: str = "en", +) -> str: + """Compose the full improver prompt for one trace.""" + if errors and prev_path: + head = REPAIR_HEADER.format( + errors="\n".join(f"- {e}" for e in errors), + prev_path=prev_path, + output_path=output_path, + outputs_path=outputs_path or "(not provided)", + ) + else: + head = TASK_HEADER.format( + guidelines_path=guidelines_path, + output_path=output_path, + outputs_path=outputs_path or "(not provided)", + ) + # P5 is stated in the guidelines, but the writer still mirrors the + # language of the source material (German briefs, German tool output) + # instead of the task's. Naming the language explicitly, next to the + # trace, is what actually holds it. + from traceforge.classify import LANGUAGE_NAMES + lang = LANGUAGE_NAMES.get(language, "English") + lang_line = ( + f"\nLANGUAGE (hard requirement): this trace's task is in {lang}. " + f"Write EVERY assistant message β€” prose, plan steps, final answer β€” " + f"in {lang}. The first user message stays byte-verbatim in its " + f"original language. Source material in another language (briefs, " + f"tool output, file contents) is quoted as-is, never mirrored: " + f"quoting German evidence inside an English trace is correct, " + f"answering in German is not.\n" + ) + return ( + f"{head}\n" + f"{lang_line}" + f"{'=' * 70}\n" + f"REAL TRACE TO IMPROVE (compact rendering, clipped tool output):\n\n" + f"{rendered_trace}\n" + ) + + +def write_prompt_file(prompts_dir: str, session_id: str, prompt: str) -> str: + os.makedirs(prompts_dir, exist_ok=True) + path = os.path.join(prompts_dir, f"improve_{session_id}.txt") + with open(path, "w", encoding="utf-8") as f: + f.write(prompt) + return path \ No newline at end of file diff --git a/traceforge/improver/review.py b/traceforge/improver/review.py new file mode 100644 index 0000000000000000000000000000000000000000..2369739ffd9803e3413dc1787dfa7dfed543f8ed --- /dev/null +++ b/traceforge/improver/review.py @@ -0,0 +1,231 @@ +"""Quality-pillar reviewer: independent adversarial review rounds. + +The improver and verifier already self-check and web-check; this stage is +the final human-dream-quality gate: independent reviewer sessions read +the ideal trace against the six pillars and the ORIGINAL user intent, +with NO knowledge of the improver's reasoning, and must unanimously PASS. +cfg.review_rounds runs independently (default 2); ALL must PASS. +""" +from __future__ import annotations + +import dataclasses +import json +import os +import subprocess +import time +from typing import Any, Optional + +from traceforge.config import Config +from traceforge.trace import json_loads_lenient + +REVIEW_HEADER = """You are an independent, adversarial quality reviewer for +a synthetic agent trace. You had NO part in creating it. Judge ONLY what +you see, against the quality pillars below. + +The ideal trace under review: {improved_path} +The ORIGINAL task (for intent comparison): {task_snippet} +Quality pillars (the spec): {guidelines_path} β€” read it first. + +Judge the trace against every pillar. Typical failure shapes: +- P1 violated: bloated reasoning, filler, restating the obvious +- P2 violated: a mistake survives (failed call, retry loop, wrong turn, + user-correction absorbed nowhere) +- P3 violated: unnecessary/exploratory tool calls, missing parallel + batching, wrong parameters +- P4 violated: statements without basis in the trace's tool evidence; + fake success; invented paths/commands/results +- P5 violated: mixed languages, grammar errors, sloppy prose +- P6 violated: refusal or task abandonment +- INTENT violated: the ideal trace solves a different task than the + original user asked + +Verdict JSON to {verdict_path}: +{{ + "verdict": "PASS" | "FAIL", + "pillar_findings": [ + {{"pillar": "P1", "status": "ok|violated", "detail": ""}} + ], + "violations": [""], + "notes": "" +}} +All pillars must be ok for PASS. Then reply "REVIEW: PASS" or +"REVIEW: FAIL". + +DISCIPLINE β€” you have a LIMITED tool-call budget. Spend it wisely: +- Read the guidelines once, then the improved.json once. +- Do NOT re-read the same file repeatedly or re-verify what you already + checked. Batch related checks into one execute_code/terminal call. +- From ~turn 10 onward (of 16), STOP forensic tool calls entirely and + write the verdict file with what you have. A verdict you can defend + from the checks already done is worth infinitely more than one more + probe: a missing verdict file FAILS the whole review round as an + infra error. +- The verdict file MUST exist on disk BEFORE your final reply. If you + are running out of budget, write it NOW with your best judgment. +""" + + +@dataclasses.dataclass +class ReviewResult: + verdict: str # PASS | FAIL | ERROR + violations: list[str] = dataclasses.field(default_factory=list) + pillar_findings: list[dict] = dataclasses.field(default_factory=list) + notes: str = "" + duration_s: float = 0.0 + error: Optional[str] = None + + def ok(self) -> bool: + return self.verdict == "PASS" + + +def _run_review_hermes( + prompt_path: str, workdir: str, max_turns: int, timeout: int +) -> tuple[int, str]: + cmd = [ + "hermes", "chat", + "--query-file", prompt_path, + "--in", workdir, + "-Q", + "--max-turns", str(max_turns), + ] + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, cwd=workdir + ) + return proc.returncode, proc.stdout or "" + + +def parse_review(raw: Optional[str]) -> Optional[dict[str, Any]]: + if not raw: + return None + raw = raw.strip() + if raw.startswith("```"): + raw = raw.split("\n", 1)[1] if "\n" in raw else "" + if raw.rstrip().endswith("```"): + raw = raw.rstrip()[:-3] + data = json_loads_lenient(raw) + if not isinstance(data, dict) or "verdict" not in data: + return None + return data + + +def _rescue_verdict_from_stdout(raw: Optional[str]) -> Optional[dict[str, Any]]: + """A reviewer that hit its turn cap mid-forensics never writes the + verdict file; its final chat message still contains the judgment. + REVIEW: PASS/FAIL line is authoritative; lift the richest JSON verdict + object the reviewer emitted, if any.""" + if not raw: + return None + marker = None + for m in ("REVIEW: PASS", "REVIEW: FAIL"): + if m in raw: + marker = m.split(":")[1].strip() + break + if marker is None: + return None + data: dict[str, Any] = {"verdict": marker} + for candidate in _json_object_spans(raw): + try: + obj = json_loads_lenient(candidate) + except Exception: + continue + if isinstance(obj, dict) and "verdict" in obj: + data = obj + data["verdict"] = marker + break + return data + + +def _json_object_spans(text: str) -> list[str]: + """Balanced top-level {...} spans from text, largest first.""" + spans: list[tuple[int, int]] = [] + depth = 0 + start = -1 + in_str = False + esc = False + for i, ch in enumerate(text): + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == "{": + if depth == 0: + start = i + depth += 1 + elif ch == "}" and depth: + depth -= 1 + if depth == 0 and start >= 0: + spans.append((start, i + 1)) + start = -1 + spans.sort(key=lambda s: s[1] - s[0], reverse=True) + return [text[a:b] for a, b in spans[:20]] + + +def run_review( + cfg: Config, + improved_path: str, + task_snippet: str, + workdir: str, + round_no: int, +) -> ReviewResult: + """One independent review round.""" + started = time.time() + verdict_path = os.path.join( + workdir, f"review_{round_no:02d}.json" + ) + if os.path.exists(verdict_path): + os.unlink(verdict_path) + prompt = REVIEW_HEADER.format( + improved_path=improved_path, + task_snippet=task_snippet[:2000], + guidelines_path=cfg.guidelines_path, + verdict_path=verdict_path, + ) + os.makedirs(cfg.prompts_dir, exist_ok=True) + prompt_path = os.path.join( + cfg.prompts_dir, + f"review_{os.path.basename(os.path.dirname(improved_path))}" + f"_r{round_no}.txt", + ) + with open(prompt_path, "w", encoding="utf-8") as f: + f.write(prompt) + try: + _code, _out = _run_review_hermes( + prompt_path, workdir, cfg.review_max_turns, cfg.review_timeout_s + ) + except subprocess.TimeoutExpired: + return ReviewResult( + verdict="ERROR", + error=f"review round {round_no} timeout", + duration_s=time.time() - started, + ) + data = parse_review( + open(verdict_path, encoding="utf-8").read() + if os.path.exists(verdict_path) else None + ) + if data is None: + rescue = _rescue_verdict_from_stdout(_out) + if rescue is not None: + data = rescue + if data is None: + return ReviewResult( + verdict="ERROR", + error=f"review round {round_no} produced no parseable verdict", + duration_s=time.time() - started, + ) + return ReviewResult( + verdict=str(data.get("verdict", "FAIL")).upper(), + violations=[ + str(v) for v in (data.get("violations", []) or []) + ], + pillar_findings=[ + p for p in (data.get("pillar_findings", []) or []) if isinstance(p, dict) + ], + notes=str(data.get("notes", "")), + duration_s=time.time() - started, + ) \ No newline at end of file diff --git a/traceforge/improver/verify.py b/traceforge/improver/verify.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b2c4a801ba7e0fa18cbe12ecdd3a6d7457def7 --- /dev/null +++ b/traceforge/improver/verify.py @@ -0,0 +1,210 @@ +"""External-claim verification stage: a web-searching verifier agent. + +The grounding gate covers environment-internal facts (paths, outputs from +the source trace). This stage covers the OTHER class: claims about the +external world (library versions, API behavior, dates, technical facts) +that an agent can state without the trace proving them β€” the classic +hallucination vector. + +The verifier is a Hermes one-shot session with the default toolset (web +search + extract enabled). It reads the improved trace, extracts +external-world claims, web-searches each, and writes a verdict JSON. +It VERDICTS, never rewrites β€” on FAIL the unsupported claims flow back +into the improver repair loop. +""" +from __future__ import annotations + +import dataclasses +import json +import os +import subprocess +import time +from typing import Any, Optional + +from traceforge.config import Config +from traceforge.trace import json_loads_lenient + +VERIFIER_HEADER = """You are an adversarial fact-checker for a synthetic agent +trace. The file below is an "ideal-world" agent trace under review for +corpus admission. Your ONLY job: find incorrect or unverifiable statements +about the EXTERNAL WORLD and fail the trace if any exist. + +What counts as an external claim (CHECK IT): +- Technical facts: library/framework versions, API behavior, default + settings, command flags, file formats, error semantics +- Factual assertions: dates, quantities, named results, "X supports Y", + "Z is deprecated since W" +- Citations: paper names, spec sections, RFC numbers + +What does NOT count (SKIP β€” already verified against the source trace): +- Anything about the project's own files/paths/commands/outputs +- The trace's internal narrative (plans, reasoning steps) +- Opinions, hedged statements, future intentions + +Procedure (use web_search / web_extract as needed β€” that is the point): +1. Read the improved trace at: {improved_path} +2. List every external claim. If there are none, verdict PASS with + claims_checked: 0 β€” do not search. +3. For each claim: search the web. Primary sources beat blogs; note the + source URL per claim. A claim passes when a real source confirms it. + A claim fails when sources contradict it β€” or passes_with_caveat when + partially right. Unverifiable-but-asserted-with-confidence = fail. +4. Write your verdict as JSON to exactly: {verdict_path} + Schema: + {{ + "verdict": "PASS" | "FAIL", + "claims_checked": , + "unsupported": [ + {{"claim": "...", "why": "...", "searched": "..."}} + ], + "notes": "" + }} + Include ALL checked claims (pass and fail) as a "checked" array with + per-claim {{"claim", "verdict": "pass"|"fail"|"pass_with_caveat", + "source"}} entries. +5. Reply with just: VERDICT: PASS or VERDICT: FAIL plus claim count. + +Be adversarial: your value is in finding the wrong statement everyone +else let slide. But do not fail a trace for hedged or internal statements. + +DISCIPLINE β€” you have a LIMITED tool-call budget: +- Read the trace once; list the claims in your reasoning, then check + them with batched searches (multiple queries in parallel tool calls). +- From ~turn 15 onward (of 20), STOP searching and write verdict.json + with what you have. The verdict file MUST exist on disk BEFORE your + final reply β€” a missing verdict file fails the whole verification + round as an infra error. If unsure, write your best verdict NOW. +""" + + +@dataclasses.dataclass +class VerifierResult: + verdict: str # PASS | FAIL | ERROR + claims_checked: int = 0 + unsupported: list[dict] = dataclasses.field(default_factory=list) + checked: list[dict] = dataclasses.field(default_factory=list) + notes: str = "" + hermes_session_id: Optional[str] = None + duration_s: float = 0.0 + error: Optional[str] = None + + def ok(self) -> bool: + return self.verdict == "PASS" + + +def _run_verifier_hermes( + cfg: Config, prompt_path: str, workdir: str +) -> tuple[int, str, str]: + cmd = [ + "hermes", "chat", + "--query-file", prompt_path, + "--in", workdir, + "-Q", + "--max-turns", str(cfg.verifier_max_turns), + ] + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=cfg.verifier_timeout_s, + cwd=workdir, + ) + return proc.returncode, proc.stdout or "", proc.stderr or "" + + +def parse_verdict(raw: Optional[str]) -> Optional[dict[str, Any]]: + if not raw: + return None + raw = raw.strip() + if raw.startswith("```"): + raw = raw.split("\n", 1)[1] if "\n" in raw else "" + if raw.rstrip().endswith("```"): + raw = raw.rstrip()[:-3] + data = json_loads_lenient(raw) + if not isinstance(data, dict) or "verdict" not in data: + return None + return data + + +def _rescue_verdict_from_stdout(raw: Optional[str]) -> Optional[dict[str, Any]]: + """A verifier that hit its turn cap mid-research never writes + verdict.json; its final chat message still contains the judgment. + VERDICT: PASS/FAIL line is authoritative; lift the richest JSON + verdict object the verifier emitted, if any.""" + from traceforge.improver.review import ( + _json_object_spans, + ) + + if not raw: + return None + marker = None + for m in ("VERDICT: PASS", "VERDICT: FAIL"): + if m in raw: + marker = m.split(":")[1].strip() + break + if marker is None: + return None + data: dict[str, Any] = {"verdict": marker} + for candidate in _json_object_spans(raw): + try: + obj = json_loads_lenient(candidate) + except Exception: + continue + if isinstance(obj, dict) and "verdict" in obj: + data = obj + data["verdict"] = marker + break + return data + + +def run_verifier( + cfg: Config, improved_path: str, workdir: str +) -> VerifierResult: + """Fact-check the improved trace's external claims via web search.""" + started = time.time() + verdict_path = os.path.join(workdir, "verdict.json") + if os.path.exists(verdict_path): + os.unlink(verdict_path) + prompt = VERIFIER_HEADER.format( + improved_path=improved_path, verdict_path=verdict_path + ) + os.makedirs(cfg.prompts_dir, exist_ok=True) + prompt_path = os.path.join( + cfg.prompts_dir, + f"verify_{os.path.basename(os.path.dirname(improved_path))}.txt", + ) + with open(prompt_path, "w", encoding="utf-8") as f: + f.write(prompt) + try: + code, stdout, stderr = _run_verifier_hermes(cfg, prompt_path, workdir) + except subprocess.TimeoutExpired: + return VerifierResult( + verdict="ERROR", error=f"verifier timeout {cfg.verifier_timeout_s}s", + duration_s=time.time() - started, + ) + data = parse_verdict( + open(verdict_path, encoding="utf-8").read() + if os.path.exists(verdict_path) else None + ) + if data is None: + rescue = _rescue_verdict_from_stdout(stdout) + if rescue is not None: + data = rescue + if data is None: + tail = stderr.strip()[-400:] if stderr else "" + return VerifierResult( + verdict="ERROR", + error=( + f"verifier produced no parseable verdict.json " + f"(exit {code}{'; ' + tail if tail else ''})" + ), + duration_s=time.time() - started, + ) + return VerifierResult( + verdict=str(data.get("verdict", "FAIL")).upper(), + claims_checked=int(data.get("claims_checked", 0) or 0), + unsupported=list(data.get("unsupported", []) or []), + checked=list(data.get("checked", []) or []), + notes=str(data.get("notes", "")), + duration_s=time.time() - started, + ) \ No newline at end of file diff --git a/traceforge/live.py b/traceforge/live.py new file mode 100644 index 0000000000000000000000000000000000000000..03b2a024bcf20d8ed48877587a04a491d5479f9b --- /dev/null +++ b/traceforge/live.py @@ -0,0 +1,508 @@ +"""Live terminal telemetry for long-running batches. + +Every stage traceforge spawns (improver, digester, map, writer, verifier, +reviewer) is a Hermes one-shot running with cwd = the per-trace workdir, +and Hermes updates live counters on the session row in state.db as the +model streams. LiveStats polls those rows on a background thread and +renders: + + TTY detail rows under the batch bar covering EVERY in-flight + trace (stage, age, tokens in/out, instantaneous tok/s, + child-session count), plus a batch postfix (total out Β· + tok/s). Layout adapts to --workers and terminal size: full + detail rows while they fit; then packed chips + (` `, several per row); a stage + histogram only when even chips overflow the screen. An + in-flight trace is never silently hidden. + pipes a plain heartbeat line every live_heartbeat_s with the same + aggregate β€” every in-flight trace listed up to 10, then + per-stage counts β€” so backgrounded runs show real progress. + end token totals on IMPROVED/FAILED lines and a batch summary line. + +Read-only against state.db (mode=ro); poll failures are swallowed and +keep the last snapshot. Contract: poll() runs on one thread (the built-in +loop); render helpers are safe from any thread. +""" +from __future__ import annotations + +import os +import shutil +import sqlite3 +import threading +import time +from typing import Optional + +from traceforge.progress import _fmt_dur + + +def fmt_tok(n) -> str: + """Compact token count: 1234 -> '1.2k', 1234567 -> '1.23M'.""" + n = int(n or 0) + if n >= 1_000_000: + return f"{n / 1_000_000:.2f}M" + if n >= 10_000: + return f"{n / 1000:.0f}k" + if n >= 1000: + return f"{n / 1000:.1f}k" + return str(n) + + +def _stage_counts(acts) -> dict: + """{stage: n} over [(sid, info), ...] pairs.""" + counts: dict = {} + for _sid, info in acts: + st = info.get("stage", "?") + counts[st] = counts.get(st, 0) + 1 + return counts + + +def sid_short(sid: str) -> str: + """Distinguishing short form of a session id for detail rows. + + Repo sids are date-prefixed (`20260926_160647_ceb268`), so the + first 12 chars are the SAME for every trace started the same + minute β€” head truncation makes rows indistinguishable. Show the + tail instead: last `_` segment (hermes) or last 8 chars (uuid). + """ + seg = sid.rsplit("_", 1)[-1] if len(sid) > 10 else sid + if len(seg) > 8: + seg = seg[-8:] + return seg if seg == sid else f"…{seg}" + + +class LiveRegistry: + """Thread-safe map of in-flight traces -> current stage + timestamps.""" + + def __init__(self): + self._lock = threading.Lock() + self._traces: dict = {} + + def begin(self, sid: str, stage: str = "starting") -> None: + with self._lock: + now = time.monotonic() + self._traces[sid] = { + "stage": stage, "t0": now, "stage_t": now, + } + + def set_stage(self, sid: str, stage: str) -> None: + with self._lock: + d = self._traces.get(sid) + if d is not None: + d["stage"] = stage + d["stage_t"] = time.monotonic() + + def end(self, sid: str) -> None: + with self._lock: + self._traces.pop(sid, None) + + def snapshot(self): + """[(sid, info), ...] oldest first β€” stable display order.""" + with self._lock: + return [ + (sid, dict(d)) + for sid, d in sorted( + self._traces.items(), key=lambda kv: kv[1]["t0"] + ) + ] + + +class LiveStats: + """Polls state.db child-session counters; renders live progress.""" + + def __init__(self, cfg, registry: LiveRegistry, prog=None, + *, max_sublines: int = 0): + self.registry = registry + self.prog = prog + # 0 = adaptive: show every in-flight trace, switching layout + # (full rows -> packed chips -> stage histogram) by screen fit. + # >0 (tests, explicit caps): hard cap on full detail rows. + self.max_sublines = max(0, int(max_sublines)) + self._forced_cap = self.max_sublines > 0 + self.state_db = cfg.state_db + self.poll_s = max(0.5, float(getattr(cfg, "live_poll_s", 2.0))) + self.heartbeat_s = max( + 5.0, float(getattr(cfg, "live_heartbeat_s", 30.0)) + ) + # ours = one-shot sessions started in this batch window AND inside + # this batch's workdir root (cwd leaf = trace sid) + self._like = os.path.join(cfg.improver_workdir, "") + self.enabled = os.path.exists(self.state_db) + self._t0_mono = time.monotonic() + self._since = time.time() - 1.0 + self._per: dict = {} + self._prev_out: dict = {} + self._sess_models: dict = {} # child session id -> model name + self._sessions = 0 + self._tot_in = 0 + self._tot_out = 0 + self._rate = 0.0 + self._last_poll = None # monotonic ts of previous poll + self._lock = threading.Lock() + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + + # ----------------------------------------------------------- lifecycle + @classmethod + def setup(cls, cfg, prog=None, registry=None, **kw): + """Factory honoring the live_poll_s=0 kill-switch.""" + if float(getattr(cfg, "live_poll_s", 0) or 0) <= 0: + return None + reg = registry or LiveRegistry() + return cls(cfg, reg, prog, **kw) + + def start(self) -> None: + if not self.enabled or self._thread is not None: + return + self._thread = threading.Thread( + target=self._loop, name="traceforge-live", daemon=True + ) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + t = self._thread + if ( + t is not None and t.is_alive() + and threading.current_thread() is not t + ): + t.join(timeout=self.poll_s + 1.0) + self._thread = None + + # ---------------------------------------------------------------- poll + def poll(self) -> None: + """One state.db read; recomputes aggregates. Single-thread contract.""" + try: + conn = sqlite3.connect( + f"file:{self.state_db}?mode=ro", uri=True, timeout=2.0 + ) + try: + rows = conn.execute( + "SELECT id, cwd, input_tokens, output_tokens," + " message_count, api_call_count, ended_at IS NULL, model" + " FROM sessions WHERE source='oneshot'" + " AND started_at >= ?", + (self._since,), + ).fetchall() + finally: + conn.close() + except sqlite3.Error: + return # transient lock/IO hiccup: keep last snapshot + rows = [r for r in rows if (r[1] or "").startswith(self._like)] + now = time.monotonic() + with self._lock: + prev_out = dict(self._prev_out) + last_poll = self._last_poll + dt = None + if last_poll is not None: + dt = max(now - last_poll, 1e-6) + per: dict = {} + delta_out = 0 + models: dict = {} + for (sess, cwd, tin, tout, msgs, api, running, model) in rows: + key = (cwd or "").rstrip("/").rsplit("/", 1)[-1] + d = per.setdefault(key, { + "in": 0, "out": 0, "msgs": 0, "api": 0, + "runs": 0, "running": 0, "rate": 0.0, "models": set(), + }) + tin, tout = tin or 0, tout or 0 + d["in"] += tin + d["out"] += tout + d["msgs"] += msgs or 0 + d["api"] += api or 0 + d["runs"] += 1 + d["running"] += 1 if running else 0 + if model: + d["models"].add(str(model)) + models[sess] = str(model) + po = prev_out.get(sess) + if po is not None and tout > po: + grew = tout - po + delta_out += grew + d["rate"] += grew / dt if dt else 0.0 + # splice path: merge direct-HTTP usage (no state.db rows exist + # for _post_chat calls; SPLICE_USAGE is the only telemetry source) + splice_delta = 0 + sper: dict = {} + try: + from traceforge.splice import SPLICE_USAGE + sper, splice_delta = SPLICE_USAGE.drain(dt) + for sid, b in sper.items(): + d = per.setdefault(sid, { + "in": 0, "out": 0, "msgs": 0, "api": 0, + "runs": 0, "running": 0, "rate": 0.0, "models": set(), + }) + d["in"] += b["in"] + d["out"] += b["out"] + d["api"] += b["calls"] + d["runs"] += b["calls"] + d["rate"] += b["rate"] + for m in b["models"]: + d["models"].add(m) + models[f"splice:{sid}:{m}"] = m + except Exception: + pass # telemetry must never break the batch + for d in per.values(): + d["rate"] = round(d["rate"], 2) + d["models"] = sorted(d["models"]) + with self._lock: + self._per = per + self._prev_out = {r[0]: r[3] or 0 for r in rows} + self._sessions = len(rows) + len( + [s for s in sper if s != "-"] + ) + self._sess_models.update(models) + self._tot_in = sum(d["in"] for d in per.values()) + self._tot_out = sum(d["out"] for d in per.values()) + if dt: + self._rate = round( + 0.5 * self._rate + + 0.5 * ((delta_out + splice_delta) / dt), 2 + ) + self._last_poll = now + + def _loop(self) -> None: + next_hb = time.monotonic() + self.heartbeat_s + while not self._stop.is_set(): + self.poll() + if self.prog is not None: + if self.prog.enabled: + pf = self._postfix() + if pf: + self.prog.set_postfix(pf) + # always refresh: an empty list clears stale sublines + self.prog.set_sublines(self.sublines()) + elif time.monotonic() >= next_hb: + self.prog.write(self.heartbeat()) + next_hb = time.monotonic() + self.heartbeat_s + self._stop.wait(self.poll_s) + + # ------------------------------------------------------------ rendering + def _snapshot(self): + with self._lock: + return dict(self._per), self._tot_in, self._tot_out, self._rate + + def per_trace(self, sid: str): + """Aggregate token usage for one trace (all its child sessions).""" + with self._lock: + d = self._per.get(sid) + return dict(d) if d else None + + def sublines(self) -> list: + """Detail lines for the Progress bar β€” covering EVERY in-flight + trace, never a "+N more" cutoff. + + Adaptive layout (no Rich TUI needed β€” the bar is the dashboard): + 1. full detail rows, one per trace, while they all fit on + screen β€” the default at any worker count with headroom; + 2. packed chips β€” several traces per row + (`β”œ improving 1:23 verify 1/2 0:45`) once + full rows would overflow the terminal height; + 3. stage histogram (`β”œ stages: improving 9 Β· verify 2/2 4`) only + when rowsΓ—width can't fit every chip β€” traces are never + silently hidden at any worker count or terminal size. + """ + acts = self.registry.snapshot() + if not acts: + return [] + per, _tin, _tout, _rate = self._snapshot() + now = time.monotonic() + rows_budget = self._detail_rows_budget() + if rows_budget >= len(acts): + return self._full_rows(acts, per, now) + return self._packed_rows(acts, rows_budget, now) + + # ------------------------------------------------- adaptive subline help + def _detail_rows_budget(self) -> int: + """How many detail rows the screen has room for. + + Forced caps (>0, tests/explicit limit) win; adaptive mode + reserves 4 lines around the frame (bar row, prompt, slack) and + keeps at least the historical 8 visible rows. + """ + if self._forced_cap: + return self.max_sublines + lines = shutil.get_terminal_size((120, 24)).lines + return max(8, lines - 4) + + def _full_rows(self, acts, per, now) -> list: + cols = shutil.get_terminal_size((120, 24)).columns + lines = [] + last = len(acts) - 1 + for i, (sid, info) in enumerate(acts): + connector = "β””" if i == last else "β”œ" + head = ( + f"{connector} {sid_short(sid):<10}" + f" {info.get('stage', '?'):<12}" + f" {_fmt_dur(now - info.get('t0', now)):>6}" + ) + d = per.get(sid) + if not d: + lines.append(head + " Β· telemetry pending") + continue + # richest first; trim fields until the line fits the terminal + m = d.get("models") or [] + fields = [ + f"{d['rate']:5.1f} tok/s", + f"{fmt_tok(d['out'])} out", + f"{fmt_tok(d['in'])} in", + "/".join(m) if m else "?", + f"{d['runs']} run" + ("s" if d["runs"] != 1 else ""), + ] + line = head + for f in fields: + cand = line + f" Β· {f}" + if len(cand) + 1 <= cols: + line = cand + lines.append(line) + return lines + + @staticmethod + def _chip(sid: str, info: dict, now: float) -> str: + """One packed trace chip: `… `.""" + stage = info.get("stage", "?") + if len(stage) > 12: + stage = stage[:12] + return f"{sid_short(sid)} {stage} {_fmt_dur(now - info.get('t0', now))}" + + def _packed_rows(self, acts, rows_budget: int, now) -> list: + """Chips packed several-per-row: ALL traces visible, compact. + + Greedy packing by ACTUAL chip width (worst ~28 chars, typically + less) honors the terminal columns; the row count stays within + rows_budget. If widthΓ—rows together cannot show every trace, + degrade to the stage histogram β€” traces are never dropped. + """ + cols = shutil.get_terminal_size((120, 24)).columns + chips = [self._chip(sid, info, now) for sid, info in acts] + rows: list = [] + cur: list = [] + cur_w = 2 # "β”œ " connector prefix + for chip in chips: + w = len(chip) + (2 if cur else 0) # gap before non-first chip + if cur and cur_w + w > cols: + rows.append(cur) + cur, cur_w = [], 2 + if len(rows) >= rows_budget: + break + cur.append(chip) + cur_w += w + if cur and len(rows) < rows_budget: + rows.append(cur) + placed = sum(len(r) for r in rows) + if placed < len(chips): + # cannot fit every trace even packed: histogram, not hiding + return self._stage_histogram(acts, rows_budget) + return [ + f"{'β”œ' if i < len(rows) - 1 else 'β””'} " + " ".join(r) + for i, r in enumerate(rows) + ] + + @staticmethod + def _stage_histogram(acts, rows_budget: int) -> list: + """Last resort (narrow/short terminals): per-stage counts. + + Width-safe: stage groups wrap across rows within the terminal + columns; the oldest in-flight trace closes the frame when the + row budget allows it. + """ + cols = shutil.get_terminal_size((120, 24)).columns + groups = [ + f"{st} {n}" + for st, n in sorted( + _stage_counts(acts).items(), key=lambda kv: (-kv[1], kv[0]) + ) + ] + rows: list = [] + cur: list = [] + cur_w = len("β”œ stages: ") + for g in groups: + w = len(g) + (3 if cur else 0) # " Β· " separator + if cur and cur_w + w > cols: + rows.append(" Β· ".join(cur)) + cur, cur_w = [], len("β”œ stages: ") + if len(rows) >= max(1, rows_budget - 1): + break + cur.append(g) + cur_w += w + if cur: + rows.append(" Β· ".join(cur)) + if len(rows) > max(1, rows_budget - 1): + rows = rows[:max(1, rows_budget - 1)] + sid, info = acts[0] # snapshot() is oldest-first + now = time.monotonic() + oldest = ( + f"oldest: {sid_short(sid)} {info.get('stage', '?')}" + f" {_fmt_dur(now - info.get('t0', now))}" + ) + if rows_budget == 1: + # single-row budget: merge the oldest into the histogram row + head = "stages: " + (rows[0] if rows else "") + return [f"β”” {head} Β· {oldest}"[:cols]] + lines = [f"stages: {r}" if i == 0 else f" {r}" + for i, r in enumerate(rows)] + lines.append(oldest) + return [f"{'β””' if i == len(lines) - 1 else 'β”œ'} {ln}" + for i, ln in enumerate(lines[:rows_budget])] + + def _postfix(self) -> str: + _per, tin, tout, rate = self._snapshot() + if tout or tin: + return f"{fmt_tok(tout)} out Β· {rate:.1f} tok/s" + return "" + + def heartbeat(self) -> str: + _per, tin, tout, rate = self._snapshot() + wall = _fmt_dur(time.monotonic() - self._t0_mono) + now = time.monotonic() + acts = self.registry.snapshot() + # every in-flight trace listed for as long as the line stays + # readable (~300 chars); beyond that collapse to stage counts + shown = acts[:10] + inflight = ", ".join( + f"{sid_short(sid)} {info.get('stage', '?')}" + f" ({_fmt_dur(now - info.get('t0', now))})" + for sid, info in shown + ) + if len(acts) > len(shown): + rest = acts[len(shown):] + counts = " Β· ".join( + f"{st} {n}" + for st, n in sorted( + _stage_counts(rest).items(), key=lambda kv: -kv[1] + ) + ) + inflight = f"{inflight} … +{len(acts) - len(shown)} ({counts})" if inflight else counts + inflight = inflight or "none" + # Judge cache hit rate: a repair round re-judges every window, so this + # is the share of judge calls that cost nothing. Silent until the + # first miss, so a run that never judges does not print "0/0". + try: + from traceforge.splice import judge_cache_stats + js = judge_cache_stats() + jtot = js["hit"] + js["miss"] + jtxt = (f" Β· judge cache {100 * js['hit'] / jtot:.0f}%" + f" ({js['hit']}/{jtot})") if jtot else "" + except Exception: + jtxt = "" + return ( + f"-- [{wall}] {fmt_tok(tin)} in Β· {fmt_tok(tout)} out" + f" Β· {rate:.1f} tok/s Β· {self._models_str()} Β· in flight: {inflight}" + f"{jtxt}" + ) + + def _models_str(self) -> str: + """Distinct child models seen this batch, e.g. 'glmb300'.""" + with self._lock: + ms = set(self._sess_models.values()) + return "/".join(sorted(ms)) if ms else "?" + + def summary(self) -> str: + with self._lock: + tin, tout, ns = self._tot_in, self._tot_out, self._sessions + wall = max(time.monotonic() - self._t0_mono, 1e-6) + avg = tout / wall if tout else 0.0 + return ( + f"-- live: {ns} child sessions Β· {fmt_tok(tin)} tok in" + f" Β· {fmt_tok(tout)} tok out Β· avg {avg:.1f} tok/s gen" + f" Β· model {self._models_str()} Β· wall {_fmt_dur(wall)}" + ) \ No newline at end of file diff --git a/traceforge/progress.py b/traceforge/progress.py new file mode 100644 index 0000000000000000000000000000000000000000..7895a4fc779449b18fc7308df2499c660ccec4c7 --- /dev/null +++ b/traceforge/progress.py @@ -0,0 +1,194 @@ +"""Minimal dependency-free progress bars (tqdm-style, stdlib only). + +Design notes: +- Bar renders to stderr and only when stderr is a TTY; when output is + piped/redirected (CI, background shells, logs) the bar disables itself + and callers fall back to plain per-item lines on stdout. +- Thread-safe: a single lock guards rendering, so worker threads can + update counts / postfixes concurrently. +- write(msg) clears the current frame, prints msg on stderr, and + redraws β€” the tqdm.write pattern β€” so per-item result lines never + visually corrupt the bar. +- Multi-line frames: set_sublines(lines) installs detail lines drawn + under the bar (one per in-flight worker). Each render draws the whole + frame atomically and repaints in place with ANSI cursor-up codes. + Height changes (workers starting/finishing) repaint from the old + frame's top row, so no stale rows survive. +""" +from __future__ import annotations + +import shutil +import sys +import threading +import time +from typing import Optional, TextIO + + +def _fmt_dur(sec: float) -> str: + s = int(sec) + if s >= 3600: + return f"{s // 3600}:{s % 3600 // 60:02d}:{s % 60:02d}" + return f"{s // 60:}:{s % 60:02d}" + + +class Progress: + """Progress bar with optional detail sublines. + + Frame layout (TTY only): + + desc 42%|########----| 4/10 [1:12<24:51, 3.2s/it] postfix + β”œ abc123def456 improving 1:23 45.2k in Β· 3.1k out Β· 12.4 tok/s + β”” def789abc123 verify 1/2 0:45 12.0k in Β· 1.2k out Β· 4.2 tok/s + + update()/set_postfix()/set_sublines() are thread-safe; render is + rate-limited by mininterval so fast loops don't flood the terminal. + """ + + PAD_TARGET = 150 # pad each line to at least this width (wipe leftovers) + + def __init__( + self, + total: int, + desc: str = "", + *, + stream: Optional[TextIO] = None, + width: int = 24, + mininterval: float = 0.25, + enabled: Optional[bool] = None, + ): + self.total = max(int(total), 0) + self.n = 0 + self.desc = desc + self.width = width + self.mininterval = mininterval + self.stream = stream or sys.stderr + if enabled is None: + enabled = bool(getattr(self.stream, "isatty", lambda: False)()) + self.enabled = enabled and self.total > 0 + self._lock = threading.Lock() + self._postfix = "" + self._sublines: list[str] = [] + self._frame_rows = 0 # rows the on-screen frame currently spans + self._start = time.monotonic() + self._last_render = 0.0 + self._closed = False + if self.enabled: + self._render(force=True) + + # ------------------------------------------------------------ lifecycle + def set_postfix(self, s: str) -> None: + with self._lock: + self._postfix = s + self._maybe_render() + + def set_sublines(self, lines) -> None: + """Install/refresh the detail lines drawn under the bar.""" + with self._lock: + if not self.enabled: + return + new = list(lines) + if new == self._sublines: + return # byte-identical: skip render + self._sublines = new + self._maybe_render(force=True) + + def update(self, n: int = 1) -> None: + with self._lock: + self.n = min(self.n + n, self.total) + self._maybe_render() + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + if self.enabled: + self._render(force=True) + self.stream.write("\n") + self.stream.flush() + + # context-manager sugar + def __enter__(self) -> "Progress": + return self + + def __exit__(self, *exc) -> None: + self.close() + + # ------------------------------------------------------------ rendering + def _maybe_render(self, force: bool = False) -> None: + if not self.enabled or self._closed: + return + now = time.monotonic() + if force or now - self._last_render >= self.mininterval: + self._render(force=True) + + def _render(self, force: bool = False) -> None: + if not self.enabled: + return + now = time.monotonic() + if not force and now - self._last_render < self.mininterval: + return + self._last_render = now + elapsed = max(now - self._start, 1e-6) + frac = self.n / self.total if self.total else 0.0 + filled = int(self.width * frac) + bar = "#" * filled + "-" * (self.width - filled) + # rate: per-item time; 0 completed -> unknown + if self.n > 0: + per = elapsed / self.n + eta = _fmt_dur(per * (self.total - self.n)) + rate = f"{per:.1f}s/it" + else: + eta, rate = "-:--", "?it/s" + line = ( + f"{self.desc} {int(frac * 100):3d}%|{bar}|" + f" {self.n}/{self.total} [{_fmt_dur(elapsed)}<{eta}, {rate}]" + ) + if self._postfix: + line += f" {self._postfix}" + frame = [line] + list(self._sublines) + old_h = self._frame_rows + if old_h > 0 and len(frame) < old_h: + # frame shrank: overwrite stale rows below with blanks + frame += [""] * (old_h - len(frame)) + # truncate each line to the terminal width β€” a longer line would + # wrap to a second physical row and break the up-cursor math + cols = shutil.get_terminal_size((120, 24)).columns + padded = [] + for ln in frame: + ln = ln[:cols] + padded.append(ln + " " * (cols - len(ln))) + rows = len(padded) + body = "\r" + "\n".join(padded) + + # Cursor sits ON the frame's last row (lines joined with \n, no + # trailing newline). Move up to the frame's top row before the + # rewrite; F also returns the cursor to column 0. old_h == 1 + # needs no prefix: the cursor is already on the only frame row + # and the leading \r returns it to column 0. + if old_h >= 2: + body = f"\x1b[{old_h - 1}F" + body + + self.stream.write(body) + self.stream.flush() + self._frame_rows = rows + + def write(self, msg: str) -> None: + """Print a permanent line above the frame (thread-safe).""" + with self._lock: + if not self.enabled: + print(msg) # non-TTY: plain stdout log line + return + if self._frame_rows > 0: + # park on the frame's TOP row: cursor sits ON the last + # row of the frame, so up = rows-1 lands on the top row + up = max(self._frame_rows - 1, 1) + self.stream.write(f"\x1b[{up}F") + self.stream.write("\x1b[0J") # clear cursor..end of screen + self._frame_rows = 0 # frame gone; next render starts here + else: + self.stream.write("\r" + " " * 130 + "\r") + self.stream.write(msg + "\n") + self.stream.flush() + if not self._closed: + self._render(force=True) \ No newline at end of file diff --git a/traceforge/readers/__init__.py b/traceforge/readers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..deeaba68682ecd41059cf1ca34c7ab20646615c8 --- /dev/null +++ b/traceforge/readers/__init__.py @@ -0,0 +1,11 @@ +from traceforge.readers.claude import ClaudeTraceReader +from traceforge.readers.codex import CodexTraceReader +from traceforge.readers.hermes import HermesTraceReader +from traceforge.readers.nemotron import NemotronTraceReader + +__all__ = [ + "HermesTraceReader", + "ClaudeTraceReader", + "CodexTraceReader", + "NemotronTraceReader", +] \ No newline at end of file diff --git a/traceforge/readers/_common.py b/traceforge/readers/_common.py new file mode 100644 index 0000000000000000000000000000000000000000..c2b65a492abb0de2f0005f1b3e2264063e79dfab --- /dev/null +++ b/traceforge/readers/_common.py @@ -0,0 +1,184 @@ +"""Shared reader helpers: self-reference guard + silent-skip accounting. + +Both concerns are cross-cutting (every reader has a store it must not +read its own output back from, and every reader has code paths that +skip a file or a row without telling anyone), so they live here rather +than being re-implemented per reader. +""" +from __future__ import annotations + +import re +from collections import Counter +from typing import Optional + + +def _norm(path: Optional[str]) -> str: + """Forward-slash, case-folded, drive-less form of a path for comparison. + + Seed archives come from other machines: a macOS rollout carries + ``/Users/x/dev/traceforge/data/work/`` while the live guard is + ``C:\\Users\\y\\...\\data\\work``. Comparing raw strings misses that, + which is exactly the self-reference leak the guard exists to stop. + """ + if not path: + return "" + p = str(path).replace("\\", "/") + if len(p) > 1 and p[1] == ":": + p = p[2:] # strip the Windows drive letter + return p.rstrip("/").lower() + + +def in_workdir(cwd: Optional[str], workdir: Optional[str]) -> bool: + """True when ``cwd`` sits inside ``workdir`` (any platform's spelling). + + Two matches count: + 1. the normalized cwd starts with the normalized workdir; + 2. the cwd ends with the workdir's last two components + (``data/work/``) β€” the portable form, which catches a + foreign-platform archive whose absolute prefix differs. + """ + c, w = _norm(cwd), _norm(workdir) + if not c or not w: + return False + if c == w or c.startswith(w + "/"): + return True + tail = "/".join(w.split("/")[-2:]) + return bool(tail) and (c == tail or c.startswith(tail + "/") or f"/{tail}/" in c) + + +def new_skips() -> Counter: + """Counter for rows/files a reader dropped without a candidate row.""" + return Counter() + + +def bump(skips: Optional[Counter], reason: str, n: int = 1) -> None: + """Increment a skip counter when one is attached (never fatal).""" + if skips is not None: + skips[reason] += n + + +def oversize_row(row: dict, max_messages: int) -> dict: + """Flag a candidate that exceeds the (soft) message cap. + + ``max_messages`` is a SOFT cap: a session above it is still a + candidate because the pipeline bounds what it actually processes by + rendered size (``split_long`` / ``split_max_chars``), not by message + count. The flag lets callers report the long-session share instead + of silently discarding it. + + Returns a COPY: the codex reader keeps the same dict in its on-disk + summary cache, and a persisted ``oversize`` flag would survive a + later ``max_messages`` change. + """ + out = dict(row) + if max_messages and out.get("message_count", 0) > max_messages: + out["oversize"] = True + return out + + +def workdir_of(cfg) -> str: + """Improver workdir for a Config (empty when unset).""" + return getattr(cfg, "improver_workdir", "") or "" + + +# --------------------------------------------------------------- dispatch briefs +# An orchestrator that dispatches a subagent pastes its own brief into the +# child's first user row: 6-7KB of Orca plumbing (session/task ids, CLI docs, +# worker_done rules) followed by a ``=== TASK ===`` block holding the real +# instruction. Promoted as-is, the trainee learns to emit harness plumbing as +# the user turn, so ``validate`` rejects the trace outright. Measured +# 2026-10-24 on the raw pool: 130 claude-code traces open with this brief, 0 +# promoted, 20 gate failures. The TASK block IS the user's own words, so keep +# exactly that and drop the plumbing. +_DISPATCH_BRIEF_RE = re.compile( + r"You are working inside Orca, a multi-agent IDE" + r"|You are a dispatched worker" + r"|Your coordinator'?s Orca session ID is" + r"|=== CLI COMMANDS ===", + re.IGNORECASE, +) +_TASK_BLOCK = "=== TASK ===" +# The brief arrives wrapped in the parent's paste envelope; the closing tag +# trails the TASK block and is not part of the task. +_PASTE_CLOSE_RE = re.compile(r"\s*]*>\s*$", re.IGNORECASE) +# A split segment's task row is the original task with this marker prepended +# (splitter.save_segments). Unwrapping a brief must not eat it: the marker is +# what tells the trainee the window is a continuation, not a fresh request. +_SEGMENT_PREFIX_RE = re.compile(r"^(\[trace segment \d+ of \d+[^\]]*\]\s*)") + +# Orchestration notifications: the harness injecting "You have N orchestration +# messages. Run `orca orchestration check`." into the conversation. Pure +# plumbing, never user content β€” and because ``validate`` matches the bare +# phrase ``orca orchestration``, ONE such row kills an otherwise clean trace +# (measured 2026-10-24: 4 of 20 dispatch-brief gate failures had a clean task +# row and a 65-char notice at row 168). +_ORCH_NOTICE_RE = re.compile( + r"^\s*You have \d+ orchestration messages?\." + r"\s*Run `orca orchestration check", + re.IGNORECASE, +) + + +def is_orchestration_notice(text: Optional[str]) -> bool: + """True for the harness's 'You have N orchestration messages' row.""" + return bool(_ORCH_NOTICE_RE.match(text or "")) + + +def unwrap_dispatch_brief(text: str) -> str: + """The real task inside an orchestrator dispatch brief. + + Returns ``text`` unchanged when it is not a brief, the ``=== TASK ===`` + block when it is, and ``""`` when the brief carries no task block (pure + harness plumbing β€” the row must be dropped, not kept). + """ + if not _DISPATCH_BRIEF_RE.search(text): + return text + if _TASK_BLOCK in text: + task = text.split(_TASK_BLOCK, 1)[1] + task = _PASTE_CLOSE_RE.sub("", task).strip() + seg = _SEGMENT_PREFIX_RE.match(text) + return (seg.group(1) + task) if seg else task + return "" + + +def reframe_task_rows(messages: list, skips: Optional[Counter] = None) -> list: + """Drop harness plumbing user rows; unwrap dispatch-brief user rows. + + Two loss points, one pass (both are the same defect: harness plumbing + sitting where the user's own words belong): + + 1. ``You have N orchestration messages`` notification rows β€” dropped + everywhere they appear, counted ``orchestration_notice_row``. + 2. A dispatch brief in a user row β€” replaced by its ``=== TASK ===`` + block (``dispatch_brief_reframed``), or dropped when the brief + carries no task (``dispatch_brief_no_task``). Every user row is + checked, not just the first: a codex child can receive a second + dispatch mid-thread, and the validate gate rejects the trace for + ANY row that still carries the boilerplate. + + Returns a new list; the caller decides what an empty result means (a + taskless trace, which the reader's own taskless path then counts). + """ + out = [] + n_notice = 0 + for m in messages: + if m.role == "user" and is_orchestration_notice(m.content): + n_notice += 1 + continue + out.append(m) + bump(skips, "orchestration_notice_row", n_notice) + + kept = [] + for m in out: + if m.role == "user" and _DISPATCH_BRIEF_RE.search(m.content or ""): + task = unwrap_dispatch_brief(m.content) + if task: + m.content = task + bump(skips, "dispatch_brief_reframed") + else: + bump(skips, "dispatch_brief_no_task") + if not task: + continue + kept.append(m) + return kept + diff --git a/traceforge/readers/claude.py b/traceforge/readers/claude.py new file mode 100644 index 0000000000000000000000000000000000000000..a3d6332d43464d7eeb282b89b70ead7af743b542 --- /dev/null +++ b/traceforge/readers/claude.py @@ -0,0 +1,531 @@ +"""Claude Code trace reader β€” session JSONLs under ~/.claude/projects. + +Verified schema (Claude Code 2.x, probed 2026-09-26). Layout: + ~/.claude/projects/-/.jsonl +one JSON object per line, relevant row types: + user message:{role:"user", content: str | [tool_result blocks]} + assistant message:{role:"assistant", content:[text|thinking|tool_use], + model, usage} + custom-title / ai-title title sources (customTitle / aiTitle) + queue-operation, attachment, system, last-prompt, bridge-session, ... β†’ skip + +Probed skip rules (UI/engine artifacts, never agent content): + - isSidechain rows INSIDE top-level session files: subagent traffic + relayed into the parent transcript (also: only top-level + /*.jsonl files are scanned for parent sessions; the + / subdirs hold the subagents' OWN transcripts and are read + by the sidechain index β€” see include_sidechains) + - isMeta string rows starting with '' or 'Another + Claude session sent a message' (subagent hand-back wrapper); other + isMeta rows carry real goal-continuation tasks and are KEPT + - user string rows starting with '' β€” EXCEPT '/goal' command rows, whose + carry the real task text, which is extracted as a user message + - the session-continuation row ('This session is being continued ...') is + KEPT as a user message (handoff context, cf. Hermes compaction rows); + Trace.first_task_message() skips it when picking the task + +Tool results live as tool_result blocks inside USER rows (results arrive in +the row after the assistant turn, possibly several blocks batched for +parallel calls). Tool names are resolved via a tool_use_id β†’ name map built +from the assistant tool_use blocks in the same file. +""" +from __future__ import annotations + +import json +import os +import re +from datetime import datetime +from typing import Any, Optional + +from traceforge.readers._common import ( + bump, new_skips, oversize_row, reframe_task_rows, +) +from traceforge.trace import Message, ToolCall, Trace + +_ARTIFACT_PREFIXES = ( + "", + "", + "Another Claude session sent a message", +) +CONTINUATION_PREFIX = "This session is being continued" +_GOAL_ARGS_RE = re.compile(r"(.*?)", re.DOTALL) + + +def _iso_to_epoch(ts: Optional[str]) -> Optional[float]: + if not ts: + return None + try: + return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() + except ValueError: + return None + + +def _block_text(blocks: Any) -> str: + """Flatten a content-blocks list to text (tool_result content nests).""" + if isinstance(blocks, str): + return blocks + parts: list[str] = [] + if not isinstance(blocks, list): + return "" + for b in blocks: + if isinstance(b, str): + parts.append(b) + elif isinstance(b, dict): + t = b.get("type") + if t == "text" and b.get("text"): + parts.append(str(b["text"])) + elif t == "image": + parts.append("[image]") + elif t == "tool_result" and b.get("content") is not None: + parts.append(_block_text(b["content"])) + return "\n\n".join(p for p in parts if p) + + +class ClaudeTraceReader: + """Reads real traces from the Claude Code session store (read-only). + + include_sidechains=True (default) also indexes the subagents' OWN + transcripts (/subagents/agent-*.jsonl) as separate + candidate traces. Verified schema (probed 2026-10-12): the sidechain + files use the SAME user/assistant/tool_result row shapes as parent + session files, with isSidechain=true on every row. Sidechain + threads open with the dispatched lane-brief ("You are lane A of a + 5-lane agent team…") β€” that row IS the task; the thread becomes a + self-contained trainable (task β†’ work β†’ result) trace, the same + doctrine as codex taskless-child recovery. MASS (arXiv 2610.12176): + orchestrator+subagent conversations are the densest RSI training + signal β€” skipping them discards the highest-value data in the store. + """ + + def __init__( + self, + projects_dir: Optional[list[str] | str] = None, + *, + include_sidechains: bool = True, + ): + # Multi-store support: a list of projects dirs is indexed + # together (live store + seed archives). Same session uuid in + # two stores -> the LARGER file wins (fresher snapshot of the + # same session), matching the single-store collision rule. + if projects_dir is None: + self.projects_dirs = [ + os.path.expanduser(os.path.join("~", ".claude", "projects")) + ] + elif isinstance(projects_dir, (list, tuple)): + self.projects_dirs = [ + os.path.expanduser(d) for d in projects_dir + ] + else: + self.projects_dirs = [os.path.expanduser(projects_dir)] + self.include_sidechains = include_sidechains + # files/rows dropped without a candidate row: reported by stats() + self.skips = new_skips() + # memoized _index() β€” see the docstring there (double-counted skips) + self._idx_cache: Optional[dict[str, str]] = None + + @property + def projects_dir(self) -> str: + """First store (backwards compat for callers/tests).""" + return self.projects_dirs[0] + + # ---------------------------------------------------------------- index + + def _index(self) -> dict[str, str]: + """session uuid -> newest (largest) jsonl path across stores. + + Memoized (2026-10-24): the index walk is what bumps the file-level + skip counters (``sidechain_sidecar``, ``non_transcript_file``), so + re-walking it β€” ``stats()`` used to call ``_index()`` a second time + after ``list_sessions`` β€” reported every file skip TWICE (measured: + 2,684 sidecars where the store holds 1,342). A skip census that + doubles itself is worse than no census. + """ + if self._idx_cache is not None: + return self._idx_cache + idx: dict[str, tuple[int, str]] = {} + found = False + for projects_dir in self.projects_dirs: + if not os.path.isdir(projects_dir): + continue # missing store (seed archive) is not an error + found = True + self._index_store(projects_dir, idx) + if not found: + raise FileNotFoundError( + f"Claude projects dir not found: {self.projects_dirs}" + ) + self._idx_cache = {sid: path for sid, (_, path) in idx.items()} + return self._idx_cache + + def _index_store(self, projects_dir: str, idx: dict) -> None: + local_sids: list[str] = [] # sids discovered in THIS store + for proj in sorted(os.listdir(projects_dir)): + proj_dir = os.path.join(projects_dir, proj) + if not os.path.isdir(proj_dir): + continue # files at top level are never traces + try: + entries = os.listdir(proj_dir) + except OSError: + continue + for fn in entries: + if not fn.endswith(".jsonl"): + # A non-transcript FILE (a .json sidecar, a stray .txt) + # is counted, not just .json β€” the counter used to miss + # every other extension. Directories are the / + # sidechain containers, not skips. + if os.path.isfile(os.path.join(proj_dir, fn)): + bump(self.skips, "non_transcript_file") + continue # / subdirs: parent's sidechains live inside + sid = fn[:-6] + path = os.path.join(proj_dir, fn) + try: + size = os.path.getsize(path) + except OSError: + bump(self.skips, "unreadable_file") + continue + prev = idx.get(sid) + if prev is None or size > prev[0]: + idx[sid] = (size, path) + local_sids.append(sid) + if self.include_sidechains: + # Only THIS store's sids (2026-10-24). Iterating the global idx + # re-scanned every earlier store's subagents dir on each store + # pass, so a two-store setup counted the live store's sidecars + # twice (measured: 1,342 reported where the stores hold 1,169). + for sid in local_sids: + path = idx[sid][1] + sub_dir = os.path.join( + os.path.dirname(path), sid, "subagents" + ) + if not os.path.isdir(sub_dir): + continue + try: + subs = sorted(os.listdir(sub_dir)) + except OSError: + continue + for fn in subs: + if not (fn.startswith("agent-") and fn.endswith(".jsonl")): + # agent-*.meta.json sidecars (and the odd + # `workflows/` dir): metadata, not a transcript. + # Counted so the skip is visible. + bump(self.skips, "sidechain_sidecar") + continue + # namespaced id: #sa- β€” + # '#' cannot appear in session uuids, so the raw-cache + # namespace stays collision-free (same rule as the + # splitter's #pN segments) + sub_sid = f"{sid}#sa-{fn[6:-6]}" + sub_path = os.path.join(sub_dir, fn) + try: + sub_size = os.path.getsize(sub_path) + except OSError: + continue + prev = idx.get(sub_sid) + if prev is None or sub_size > prev[0]: + idx[sub_sid] = (sub_size, sub_path) + return {sid: path for sid, (_, path) in idx.items()} + + # ---------------------------------------------------------------- parse + + def _parse_file(self, path: str, session_id: Optional[str] = None) -> Trace: + apath = os.path.abspath(path).replace("/", os.sep) + # a subagent's OWN transcript: every row carries isSidechain=true + # (kept here; only RELAYED sidechain rows inside parent files + # are skipped) and the sid defaults to the namespaced form + is_sidechain = os.sep + "subagents" + os.sep in apath + if session_id: + sid = session_id + elif is_sidechain: + stem = os.path.basename(path)[:-6] + parent = os.path.basename(os.path.dirname(os.path.dirname(path))) + sid = f"{parent}#sa-{stem[len('agent-'):]}" + else: + sid = os.path.basename(path)[:-6] + messages: list[Message] = [] + tool_names: dict[str, str] = {} # tool_use_id -> name + model: Optional[str] = None + title_custom: Optional[str] = None + title_ai: Optional[str] = None + cwd: Optional[str] = None + started: Optional[float] = None + last_ts: Optional[float] = None + + with open(path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + try: + d = json.loads(line) + except json.JSONDecodeError: + bump(self.skips, "unparseable_line") + continue # truncated/interleaved tail lines + if not isinstance(d, dict): + continue + ts = _iso_to_epoch(d.get("timestamp")) + if started is None and ts is not None: + started = ts + if ts is not None: + last_ts = ts + + t = d.get("type") + if t == "custom-title": + title_custom = d.get("customTitle") or title_custom + continue + if t == "ai-title": + title_ai = d.get("aiTitle") or title_ai + continue + if t not in ("user", "assistant"): + continue + if d.get("isSidechain") and not is_sidechain: + continue # relayed subagent traffic inside a parent file + if cwd is None and d.get("cwd"): + cwd = d["cwd"] + + m = d.get("message") + if not isinstance(m, dict): + continue + content = m.get("content") + + if t == "assistant": + if model is None and m.get("model"): + model = str(m["model"]) + texts: list[str] = [] + thinks: list[str] = [] + calls: list[ToolCall] = [] + if isinstance(content, list): + for blk in content: + if not isinstance(blk, dict): + continue + bt = blk.get("type") + if bt == "text" and blk.get("text"): + texts.append(str(blk["text"])) + elif bt == "thinking" and blk.get("thinking"): + thinks.append(str(blk["thinking"])) + elif bt == "tool_use": + tc_id = str(blk.get("id") or "") + name = str(blk.get("name") or "") + if tc_id: + tool_names[tc_id] = name + calls.append( + ToolCall( + id=tc_id, + name=name, + arguments=json.dumps( + blk.get("input") or {}, + ensure_ascii=False, + ), + ) + ) + if not (texts or thinks or calls): + continue + messages.append( + Message( + role="assistant", + content="\n\n".join(texts) or None, + reasoning="\n\n".join(thinks) or None, + tool_calls=calls, + timestamp=ts, + ) + ) + + elif t == "user": + if isinstance(content, str): + if d.get("isMeta") and content.startswith( + _META_SKIP_PREFIXES + ): + continue + if content.startswith(""): + # /goal rows carry the task in + if "/goal" in content[:80]: + mm = _GOAL_ARGS_RE.search(content) + if mm and mm.group(1).strip(): + messages.append( + Message( + role="user", + content=mm.group(1).strip(), + timestamp=ts, + ) + ) + continue + if content.startswith(_ARTIFACT_PREFIXES): + continue + messages.append( + Message(role="user", content=content, timestamp=ts) + ) + elif isinstance(content, list): + for blk in content: + if ( + not isinstance(blk, dict) + or blk.get("type") != "tool_result" + ): + continue + tc_id = str(blk.get("tool_use_id") or "") + body = _block_text(blk.get("content")) + if not body and blk.get("is_error"): + body = "(tool error)" + if not body and not tc_id: + continue + messages.append( + Message( + role="tool", + content=body or None, + tool_call_id=tc_id, + tool_name=tool_names.get(tc_id), + timestamp=ts, + ) + ) + + # Harness plumbing in user rows (2026-10-24): an orchestrator's + # dispatch brief pasted as the task row, and the harness's "You have + # N orchestration messages" notifications. Both are plumbing where + # the user's own words belong; the brief's `=== TASK ===` block is + # the real request, so keep that and drop the rest. Measured: 130 + # claude-code raw traces opened with the brief, 0 promoted, 20 gate + # failures β€” and 4 of those had a CLEAN task row killed by a 65-char + # notice at row 168. + messages = reframe_task_rows(messages, self.skips) + + # Subagent-brief recovery (MASS doctrine, mirrors codex taskless- + # child recovery): a sidechain thread opens with the dispatched + # lane-brief β€” already a real first user row in Claude files, so + # nothing to synthesize. But a sidechain whose rows are ALL + # assistant/tool (handback content only) stays taskless β€” drop it + # here rather than letting it die at the improve gates. + if is_sidechain and not any( + m.role == "user" for m in messages + ): + return Trace(session_id=sid, model=model, title=None, + started_at=started or last_ts, cwd=cwd, + messages=[], source="claude-sidechain") + + return Trace( + session_id=sid, + model=model, + title=title_custom or title_ai, + started_at=started or last_ts, + cwd=cwd, + messages=messages, + source="claude-sidechain" if is_sidechain else "claude-code", + ) + + # ------------------------------------------------------------- sessions + + def list_sessions( + self, + *, + min_messages: int = 10, + max_messages: int = 120, + min_tool_calls: int = 2, + limit: Optional[int] = None, + ) -> list[dict[str, Any]]: + """Candidate traces for improvement, newest first (summaries only). + + ``max_messages`` is a SOFT cap (2026-10-23): a session above it is + still listed, flagged ``oversize``. The hard cap it replaced + discarded 1,217 real claude sessions (median 245 messages, 107 + tool calls, ~25% of them over ``split_max_chars`` and therefore + split candidates) before they could reach ``split_long``. + """ + rows: list[dict[str, Any]] = [] + for sid, path in self._index().items(): + try: + trace = self._parse_file(path, sid) + except OSError: + bump(self.skips, "unreadable_file") + continue + n_msgs = len(trace.messages) + if not n_msgs: + bump(self.skips, "taskless_sidechain" + if trace.source == "claude-sidechain" else "empty_trace") + continue + if n_msgs < min_messages: + bump(self.skips, "below_min_messages") + continue + n_calls = sum(len(m.tool_calls) for m in trace.messages) + if n_calls < min_tool_calls: + bump(self.skips, "below_min_tool_calls") + continue + roles: dict[str, int] = {} + for m in trace.messages: + roles[m.role] = roles.get(m.role, 0) + 1 + rows.append( + oversize_row( + { + "session_id": sid, + "model": trace.model, + "title": (trace.title or "")[:120].strip(), + "started_at": trace.started_at, + "ended_at": trace.messages[-1].timestamp + if trace.messages + else None, + "message_count": n_msgs, + "tool_call_count": n_calls, + "cwd": trace.cwd, + "source": trace.source, + "roles": roles, + }, + max_messages, + ) + ) + rows.sort(key=lambda r: r["started_at"] or 0.0, reverse=True) + if limit is not None: + rows = rows[:limit] + return rows + + def load_trace(self, session_id: str) -> Trace: + """Load one full trace as a normalized Trace.""" + path = self._index().get(session_id) + if not path: + raise KeyError(f"claude session not found: {session_id}") + return self._parse_file(path, session_id) + + # -------------------------------------------------------------- helpers + + def stats(self) -> dict[str, Any]: + """Counts over the store(s), plus the skip census. + + The parse loop runs against a SCRATCH counter (2026-10-24): stats() + re-parses every session to total messages/tool calls, and those + parses bump the row-level skip counters again β€” so a census taken + after stats() reported every reframed brief and dropped notice row + twice. stats() must observe the funnel, not re-count it. + """ + idx = self._index() + n_msgs = 0 + n_calls = 0 + real_skips = self.skips + census = new_skips() + self.skips = census + try: + for _, path in idx.items(): + try: + t = self._parse_file(path) + except OSError: + continue + n_msgs += len(t.messages) + n_calls += sum(len(m.tool_calls) for m in t.messages) + finally: + self.skips = real_skips + return { + "projects": sum( + len( + [ + e + for e in os.listdir(d) + if os.path.isdir(os.path.join(d, e)) + ] + ) + for d in self.projects_dirs + if os.path.isdir(d) + ), + "sessions": len(idx), + "messages": n_msgs, + "tool_calls": n_calls, + # the scratch counter, not the restored one: reading self.skips + # here returned the caller's counter and threw the census away + "skips": dict(census), + } \ No newline at end of file diff --git a/traceforge/readers/codex.py b/traceforge/readers/codex.py new file mode 100644 index 0000000000000000000000000000000000000000..a326ee98bf4ee566690b62b7c7c2657332a38e4b --- /dev/null +++ b/traceforge/readers/codex.py @@ -0,0 +1,637 @@ +"""Codex trace reader β€” rollout JSONLs under ~/.codex/sessions (and exports). + +Verified schema (Codex Desktop 0.145.x, probed 2026-10-07 against the +`exported-seed-datas` archives + the live store). Layout: + /YYYY/MM/DD/rollout--.jsonl +one JSON object per line, top-level `type`: + session_meta payload:{session_id, id (=thread id, unique per file), + cwd, timestamp, originator, cli_version, source{...}} + turn_context payload:{turn_id, cwd, model, effort, ...} (engine noise) + response_item the actual conversation (payload.type below) + event_msg / world_state engine telemetry β€” skipped + +response_item payload types: + message role:user|assistant|developer, content: + [{type:"input_text"|"output_text", text}] + reasoning summary:[{type:"summary_text", text}] (when + summaries are enabled; encrypted_content + otherwise β€” skipped) + custom_tool_call id, call_id, name, input (STRING of JS code + for the `exec` tool β€” kept verbatim) + custom_tool_call_output call_id, output:[{type:"input_text", text}] + function_call / function_call_output same via call_id + agent_message / tool_search_call / ... multi-agent chatter β€” skipped + +Probed skip rules (harness artifacts, never real user content): + user rows starting with , , + , , the preceding call row. Session id = +the session_meta payload.id (thread id in the filename), NOT session_id +(forked rollouts share the parent's session_id; files would collide). + +Self-reference guard: rollouts whose session_meta cwd sits inside the +improver workdir are dropped from listing (same doctrine as hermes). +""" +from __future__ import annotations + +import glob +import json +import os +import re +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from typing import Any, Optional + +from traceforge.readers._common import ( + bump, in_workdir, new_skips, oversize_row, reframe_task_rows, + unwrap_dispatch_brief, +) +from traceforge.trace import Message, ToolCall, Trace + +_USER_ARTIFACT_PREFIXES = ( + "", + "", + "", + "", + " str: + """rollout--.jsonl[.zst] -> rollout--.""" + for suf in _ROLLOUT_SUFFIXES: + if name.endswith(suf): + return name[: -len(suf)] + return name + + +def _open_rollout(path: str): + """Open a rollout as text, transparently decompressing ``.jsonl.zst``. + + Codex compresses older rollouts in place β€” 107 of 1387 files on this host + (measured 2026-10-22), all of them older than the live ones. The + decompressed bytes are the same JSONL, so only the opener changes. + + Python 3.14 ships ``compression.zstd``; on 3.10-3.13 this needs the + ``zstandard`` package. Both are optional, so a missing backend raises a + clear error rather than silently skipping the file. + """ + if not path.endswith(".zst"): + return open(path, "r", encoding="utf-8", errors="replace") + try: + from compression import zstd as _zstd # Python 3.14+ + except ImportError: + _zstd = None + if _zstd is not None: + return _zstd.open(path, "rt", encoding="utf-8", errors="replace") + try: + import zstandard # type: ignore + except ImportError as exc: # pragma: no cover - depends on interpreter + raise RuntimeError( + f"cannot read compressed rollout {path!r}: install the " + "'zstandard' package or run on Python 3.14+" + ) from exc + return zstandard.open(path, "rt", encoding="utf-8", errors="replace") +_ISO_RE = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})") +# Completion-report openings: work summaries, not task restatements. +# Anchored at string start (case-insensitive); "Implemented …", "Wrote …". +_REPORT_OPENING_RE = re.compile( + r"^\s*(?:i\s+)?(?:ich\s+)?(?:have\s+|hab\s+|habe\s+)?" + r"(?:implemented|created|fixed|added|built|completed|wrote|done)\b", + re.I, +) + + +def _iso_to_epoch(ts: Optional[str]) -> Optional[float]: + if not ts: + return None + try: + return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() + except ValueError: + return None + + +def _unwrap_dispatch_brief(text: str) -> str: + """Return the real task from an Orca coordinator dispatch brief. + + A codex subagent rollout opens with the coordinator's dispatch text: 6-7KB + of orchestration boilerplate (session/task ids, `orca orchestration` CLI + docs, worker_done rules) followed by a `=== TASK ===` block holding the + actual instruction. Promoted as-is, the trainee learns to emit harness + plumbing as the user turn (measured 2026-10-22: 81 raw traces, 6 already + promoted). The TASK block is the user's own words, so keep exactly that; + a brief with no TASK block yields "" and the row is dropped. + + Thin alias over the shared helper (2026-10-24): the claude reader needs + the same unwrap, and two copies of the pattern drift. Kept as a module + name because tests and callers import it from here. + """ + return unwrap_dispatch_brief(text) + + +def recover_taskless_brief(messages: list, role_hint: str = "") -> bool: + """Synthesize a task row for a taskless child thread (in place). + + Subagent handoff recovery (owner directive 2026-10-07: taskless child + threads are valuable work traces, don't drop them): when the rollout + carries no real user row, codex children open with the agent restating + its dispatched brief in task terms. That restatement becomes a + synthesized first user row so the trace is a trainable + (task β†’ work β†’ result) pair; the improve pipeline's reframe path and + judges still verify task↔work fit. Completion-REPORT-shaped openings + ("Implemented …", "Wrote …") are work summaries, not tasks β€” they stay + taskless. + + Shared with the raw-pool repair pass (2026-10-24): dropping the + harness's orchestration-notification rows can leave a trace with no + user row at all, and a cached trace must land in the same state a + fresh pull would produce. + + Returns True when a row was inserted. + """ + if any(m.role == "user" for m in messages): + return False + first_asst = next( + (m for m in messages if m.role == "assistant" and (m.content or "")), + None, + ) + if ( + first_asst is None + or len(first_asst.content or "") < 80 + or _REPORT_OPENING_RE.search(first_asst.content or "") + ): + return False + messages.insert( + 0, + Message( + role="user", + content=role_hint + (first_asst.content or "").strip(), + timestamp=first_asst.timestamp, + ), + ) + return True + + +def _content_text(blocks: Any) -> str: + """Flatten content-block lists (message bodies, tool outputs).""" + if isinstance(blocks, str): + return blocks + if not isinstance(blocks, list): + return "" + parts: list[str] = [] + for b in blocks: + if isinstance(b, dict): + t = b.get("text") + if t: + parts.append(str(t)) + elif isinstance(b, str): + parts.append(b) + return "\n\n".join(p for p in parts if p) + + +class CodexTraceReader: + """Reads real traces from Codex rollout session stores (read-only).""" + + def __init__(self, roots: Optional[list[str]] = None): + if roots: + self.roots = [os.path.expanduser(r) for r in roots] + else: + self.roots = [os.path.expanduser("~/.codex/sessions")] + # files/rows dropped without a candidate row: reported by stats() + self.skips = new_skips() + + # ---------------------------------------------------------------- index + + def _agent_role(self, path: str) -> Optional[str]: + """Subagent role (agent_path/agent_nickname) from session_meta.""" + try: + with _open_rollout(path) as f: + for line in f: + try: + d = json.loads(line) + except json.JSONDecodeError: + continue + if d.get("type") == "session_meta": + pay = d.get("payload") or {} + role = pay.get("agent_path") or pay.get("agent_nickname") + if role: + return str(role).strip("/") + return None + except OSError: + pass + return None + + def _rollout_files(self) -> list[str]: + files: list[str] = [] + for root in self.roots: + if not os.path.isdir(root): + continue + for p in glob.glob(os.path.join(root, "**", "*.jsonl"), recursive=True): + if _ROLLOUT_RE.search(os.path.basename(p)): + files.append(p) + for p in glob.glob(os.path.join(root, "**", "*.jsonl.zst"), recursive=True): + if _ROLLOUT_RE.search(os.path.basename(p)): + files.append(p) + files.sort() + return files + + def _index(self) -> dict[str, str]: + """thread id (session_meta payload.id) -> rollout path.""" + idx: dict[str, str] = {} + for path in self._rollout_files(): + sid = self._thread_id(path) + if sid: + idx[sid] = path + else: + bump(self.skips, "no_thread_id") + return idx + + def _thread_id(self, path: str) -> Optional[str]: + """session_meta payload.id β€” the thread id from the filename. + + The last uuid in the filename IS the thread id; parsing it avoids + scanning the file for sessions whose meta row was truncated away. + Fallback: scan for session_meta when the name is non-standard. + """ + base = _strip_rollout_suffix(os.path.basename(path)) + mm = re.search(r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$", base, re.I) + if mm: + return mm.group(1) + for line in _open_rollout(path): + try: + d = json.loads(line) + except json.JSONDecodeError: + continue + if d.get("type") == "session_meta": + pid = (d.get("payload") or {}).get("id") + if pid: + return str(pid) + return None + + # ---------------------------------------------------------------- parse + + def _parse_file(self, path: str, session_id: Optional[str] = None) -> Trace: + sid = session_id or self._thread_id(path) or _strip_rollout_suffix(os.path.basename(path)) + messages: list[Message] = [] + call_names: dict[str, str] = {} # call_id -> tool name + model: Optional[str] = None + cwd: Optional[str] = None + started: Optional[float] = None + last_ts: Optional[float] = None + title: Optional[str] = None # first real user task, set below + pending_reasoning: Optional[str] = None # reasoning buffered for the next assistant row + + with _open_rollout(path) as f: + for line in f: + try: + d = json.loads(line) + except json.JSONDecodeError: + bump(self.skips, "unparseable_line") + continue + if not isinstance(d, dict): + continue + ts = _iso_to_epoch(d.get("timestamp")) + if started is None and ts is not None: + started = ts + if ts is not None: + last_ts = ts + + t = d.get("type") + if t == "session_meta": + pay = d.get("payload") or {} + cwd = cwd or pay.get("cwd") + elif t == "turn_context": + pay = d.get("payload") or {} + if model is None and pay.get("model"): + model = str(pay["model"]) + cwd = cwd or pay.get("cwd") + elif t != "response_item": + continue + + pay = d.get("payload") or {} + pt = pay.get("type") + + if pt == "message": + role = pay.get("role") + text = _content_text(pay.get("content")) + if not text: + continue + if role == "user": + if text.startswith(_USER_ARTIFACT_PREFIXES) or text.startswith( + _USER_ARTIFACT_TEXT_PREFIXES + ): + continue + messages.append( + Message(role="user", content=text, timestamp=ts) + ) + elif role == "assistant": + messages.append( + Message(role="assistant", content=text, timestamp=ts) + ) + # developer-role rows: engine instructions, skipped + + elif pt == "reasoning": + # codex emits reasoning BEFORE its assistant row + # (reasoning -> message / custom_tool_call). Buffer and + # attach to the next assistant message. Encrypted-only + # reasoning rows produce no text and are skipped. + s = pay.get("summary") or [] + text = _content_text(s) + if text: + pending_reasoning = ( + f"{pending_reasoning}\n\n{text}" + if pending_reasoning + else text + ) + + elif pt in ("custom_tool_call", "function_call"): + name = str(pay.get("name") or "") + call_id = str(pay.get("call_id") or pay.get("id") or "") + if call_id: + call_names[call_id] = name + inp = pay.get("input") + if isinstance(inp, str): + # `exec` custom tool calls carry raw JS code, not a + # JSON object β€” normalize to {"code": ...} so the + # structural gate (JSON tool args) stays satisfiable + # while the code remains byte-verbatim. + args = json.dumps({"code": inp}, ensure_ascii=False) + else: + args = json.dumps(inp or {}, ensure_ascii=False) + messages.append( + Message( + role="assistant", + content=None, + reasoning=pending_reasoning or None, + tool_calls=[ + ToolCall(id=call_id, name=name, arguments=args) + ], + timestamp=ts, + ) + ) + pending_reasoning = None + + elif pt in ("custom_tool_call_output", "function_call_output"): + call_id = str(pay.get("call_id") or "") + out = pay.get("output") + body = _content_text(out) if isinstance(out, list) else ( + str(out) if out is not None else "" + ) + messages.append( + Message( + role="tool", + content=body or None, + tool_call_id=call_id, + tool_name=call_names.get(call_id), + timestamp=ts, + ) + ) + + # Harness plumbing in user rows (2026-10-24): the coordinator's + # dispatch brief (keep its `=== TASK ===` block, drop the 6-7KB of + # Orca boilerplate) and the harness's "You have N orchestration + # messages" notifications. Both are plumbing where the user's own + # words belong, and the validate gate rejects the trace for either. + # Runs BEFORE the title pick so the title is the real task. + messages = reframe_task_rows(messages, self.skips) + + # title: first real user task (codex has no title rows) + for m in messages: + if m.role == "user" and m.content: + title = " ".join(m.content.split())[:120] + break + + # Subagent handoff recovery (owner directive 2026-10-07: taskless + # child threads are valuable work traces, don't drop them): when + # the rollout carries no real user row, codex children open with + # the agent restating its dispatched brief in task terms. That + # restatement becomes a synthesized first user row so the trace + # is a trainable (task β†’ work β†’ result) pair; the improve + # pipeline's reframe path and judges still verify task↔work fit. + # Completion-REPORT-shaped openings ("Implemented …", "Wrote …") + # are work summaries, not tasks β€” they stay taskless. + if not any(m.role == "user" for m in messages): + role = self._agent_role(path) # extra file read: taskless only + recover_taskless_brief( + messages, f"[subagent brief β€” role: {role}]\n" if role else "" + ) + + return Trace( + session_id=sid, + model=model, + title=title, + started_at=started or last_ts, + cwd=cwd, + messages=messages, + source="codex", + ) + + # ------------------------------------------------------------- sessions + + def _cache_path(self) -> str: + """Per-roots summary cache file (hash of the roots list).""" + import hashlib + + key = hashlib.sha1( + "\x00".join(self.roots).encode("utf-8", "replace") + ).hexdigest()[:12] + base = next( + (r for r in self.roots if os.path.isdir(r)), None + ) or os.getcwd() + d = os.path.join(os.path.dirname(os.path.abspath(base)), ".cache") + try: + os.makedirs(d, exist_ok=True) + except OSError: + return os.path.join( + os.environ.get("TMPDIR", os.getcwd()), f".tf-codex-{key}.json" + ) + return os.path.join(d, f"codex-summaries-{key}.json") + + def _load_cache(self) -> dict: + try: + with open(self._cache_path(), "r", encoding="utf-8") as f: + d = json.load(f) + if isinstance(d, dict) and d.get("v") == 1: + return d.get("entries", {}) + except (OSError, ValueError): + pass + return {} + + def _save_cache(self, entries: dict) -> None: + try: + tmp = self._cache_path() + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump({"v": 1, "entries": entries}, f, + separators=(",", ":")) + os.replace(tmp, self._cache_path()) + except OSError: + pass # cache is best-effort, never fatal + + def list_sessions( + self, + *, + min_messages: int = 10, + max_messages: int = 120, + min_tool_calls: int = 2, + exclude_cwd: Optional[str] = None, + limit: Optional[int] = None, + status_cb=None, + use_cache: bool = True, + ) -> list[dict[str, Any]]: + """Candidate traces for improvement, newest first (summaries only). + + Per-file summaries are cached (path -> {sig: size+mtime, row}) + under the store's parent .cache dir; only new/changed files + parse. Parsing is thread-parallel β€” ~3.8k rollouts β‰ˆ 10 GB + parse in ~20s wall on the first run, near-instant after. + ``status_cb(msg)`` gets a discovery heartbeat while it runs. + + ``use_cache=False`` forces a full parse (2026-10-24). The row-level + skip counters are bumped during parsing, so a warm cache made the + census report near-zero: measured, 251 rollouts carry a dispatch + brief and the cached run counted 0 reframes. ``stats()`` passes + False so its census is authoritative. + + ``max_messages`` is a SOFT cap (2026-10-23): a rollout above it is + still listed, flagged ``oversize``. The hard cap it replaced + discarded 1,246 real codex rollouts (median 210 messages, 98 tool + calls) before they could reach ``split_long``. + """ + idx = self._index() + cached = self._load_cache() if use_cache else {} + entries: dict[str, Any] = {} + stale: list[tuple[str, str, str]] = [] + for sid, path in idx.items(): + try: + sig = f"{os.path.getsize(path):x}-{int(os.path.getmtime(path)):x}" + except OSError: + # stat failed: the file vanished or is locked. Counted β€” + # this used to drop the rollout with no trace of it. + bump(self.skips, "unreadable_file") + continue + c = cached.get(path) + if c and c.get("sig") == sig: + entries[path] = c + else: + stale.append((sid, path, sig)) + rows: list[dict[str, Any]] = [ + dict(e["row"]) for e in entries.values() + ] + if stale: + t0 = time.monotonic() + done = 0 + lock = threading.Lock() + + def _parse_one(item): + sid, path, sig = item + try: + trace = self._parse_file(path, sid) + except OSError: + bump(self.skips, "unreadable_file") + return None + return path, sig, trace + + workers = min(8, max(1, len(stale))) + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = [ex.submit(_parse_one, it) for it in stale] + for f in as_completed(futs): + got = f.result() + with lock: + done += 1 + if status_cb and ( + done % 250 == 0 or done == len(stale) + ): + status_cb( + f"discovery: {done}/{len(stale)} rollouts " + f"({time.monotonic() - t0:.0f}s)" + ) + if got is None: + continue + path, sig, trace = got + roles: dict[str, int] = {} + for m in trace.messages: + roles[m.role] = roles.get(m.role, 0) + 1 + row = { + "session_id": trace.session_id, + "model": trace.model, + "title": (trace.title or "")[:120].strip(), + "started_at": trace.started_at, + "ended_at": trace.messages[-1].timestamp + if trace.messages + else None, + "message_count": len(trace.messages), + "tool_call_count": sum( + len(m.tool_calls) for m in trace.messages + ), + "cwd": trace.cwd, + "source": "codex", + "roles": roles, + } + rows.append(row) + entries[path] = {"sig": sig, "row": row} + self._save_cache(entries) + out: list[dict[str, Any]] = [] + for r in rows: + if exclude_cwd and in_workdir(r["cwd"], exclude_cwd): + bump(self.skips, "cwd_self_reference") + continue + if r["message_count"] < min_messages: + bump(self.skips, "below_min_messages") + continue + if r["tool_call_count"] < min_tool_calls: + bump(self.skips, "below_min_tool_calls") + continue + out.append(oversize_row(r, max_messages)) + out.sort(key=lambda r: r["started_at"] or 0.0, reverse=True) + if limit is not None: + out = out[:limit] + return out + + def load_trace(self, session_id: str) -> Trace: + """Load one full trace as a normalized Trace.""" + path = self._index().get(session_id) + if not path: + raise KeyError(f"codex session not found: {session_id}") + return self._parse_file(path, session_id) + + # -------------------------------------------------------------- helpers + + def stats(self) -> dict[str, Any]: + """Counts over the store(s), plus the skip census. + + Runs against a SCRATCH counter (2026-10-24): stats() re-walks the + index and re-lists sessions, which bumps the file-level counters + again β€” a census taken after stats() reported every skip twice. + """ + real_skips = self.skips + census = new_skips() + self.skips = census + try: + rows = self.list_sessions( + min_messages=0, max_messages=10**9, min_tool_calls=0, + use_cache=False, + ) + finally: + self.skips = real_skips + return { + "roots": len(self.roots), + "sessions": len(rows), + "messages": sum(r["message_count"] for r in rows), + "tool_calls": sum(r["tool_call_count"] for r in rows), + # the scratch counter, not the restored one: reading self.skips + # here returned the caller's counter and threw the census away + "skips": dict(census), + } \ No newline at end of file diff --git a/traceforge/readers/hermes.py b/traceforge/readers/hermes.py new file mode 100644 index 0000000000000000000000000000000000000000..2ab06caa12701376e59eb4e66baefaa26b1bf20f --- /dev/null +++ b/traceforge/readers/hermes.py @@ -0,0 +1,319 @@ +"""Hermes Agent trace reader β€” reads sessions+messages from state.db. + +Verified schema (Hermes state.db, 2026-09-26): + sessions(id, model, title, display_name, started_at, ended_at, + message_count, tool_call_count, input_tokens, output_tokens, + cwd, source, ...) + messages(id, session_id, role, content, tool_call_id, tool_calls, + tool_name, timestamp, reasoning, reasoning_content, active, + compacted, display_order, ...) + +display_order is NULL on ~47% of rows -> order by (timestamp, id). +tool_calls is a JSON array: [{"id","call_id","type":"function", + "function":{"name","arguments"}}]. Tool results are role='tool' rows +with tool_call_id + tool_name, paired against the issuing assistant turn. +""" +from __future__ import annotations + +import os +import sqlite3 +from typing import Any, Optional + +from traceforge.readers._common import bump, in_workdir, new_skips, oversize_row +from traceforge.trace import Message, ToolCall, Trace, json_loads_lenient + +# Sources that carry a real user-facing agent session. 'oneshot' is +# traceforge's OWN improver runtime (6,625 sessions in the seed archive, +# 1,578 of them long enough to look like candidates) and 'tool' is a +# 2-message probe β€” both stay out. 'subagent' / 'kanban' / 'local' are +# real work sessions: the subagent ones are delegate_task children whose +# transcripts are the densest multi-agent signal in the store (MASS, +# arXiv 2610.12176 β€” the claude reader already indexes its sidechains +# for the same reason), and excluding them here was an inconsistency, +# not a policy. Measured 2026-10-23: 43 such sessions pass every other +# filter and were being dropped by the whitelist alone. +_CANDIDATE_SOURCES = ("desktop", "cli", "tui", "subagent", "local", "kanban") + + +class HermesTraceReader: + """Reads real traces from one or more Hermes Agent state.dbs (read-only). + + Multi-db support (2026-10-13): a list of db paths is searched + together β€” the live store plus seed archives like + harness-traces/hermes/state.db. Session ids are uuids, unique per + store with zero expected cross-store overlap (verified against the + seed archive); a hit in the FIRST db wins on collision. + """ + + def __init__(self, state_db: str | list[str]): + if isinstance(state_db, (list, tuple)): + self.state_dbs = [os.path.abspath(d) for d in state_db] + else: + self.state_dbs = [os.path.abspath(state_db)] + # rows dropped without a candidate row (self-reference guard, + # inactive turns): reported by stats() instead of vanishing. + self.skips = new_skips() + # Last filter settings list_sessions ran with β€” the SQL-level drop + # census in stats() attributes drops in the same order the WHERE + # clause applies, so it needs to know what the clause was. + self._last_min_messages = 10 + self._last_min_tool_calls = 2 + self._last_require_ended = True + self._last_max_input_tokens: Optional[int] = None + self._last_exclude_cwd: Optional[str] = None + + @property + def state_db(self) -> str: + """First db (backwards compat for callers/tests).""" + return self.state_dbs[0] + + def _connect(self, db: str) -> sqlite3.Connection: + if not os.path.exists(db): + raise FileNotFoundError(f"Hermes state.db not found at {db}") + return sqlite3.connect(f"file:{db}?mode=ro", uri=True) + + def _existing_dbs(self) -> list[str]: + """All configured dbs that exist on disk (seed archives may be + absent; the live store always is).""" + return [d for d in self.state_dbs if os.path.exists(d)] + + def _iter_dbs(self) -> tuple[str, sqlite3.Connection]: + """(db, conn) for every existing store; caller closes each conn.""" + for db in self._existing_dbs(): + yield db, self._connect(db) + + # ------------------------------------------------------------ sessions + + def list_sessions( + self, + *, + min_messages: int = 10, + max_messages: int = 120, + min_tool_calls: int = 2, + max_input_tokens: Optional[int] = None, + source: Optional[str] = None, + require_ended: bool = True, + exclude_cwd: Optional[str] = None, + limit: Optional[int] = None, + ) -> list[dict[str, Any]]: + """Candidate traces for improvement, newest first (summaries only). + + ``exclude_cwd`` roots out traceforge's own improver children even + when mis-tagged: every improver stage runs with cwd inside the + improver workdir, so a cwd there means self-referential data no + matter what ``source`` says. + + ``max_messages`` is a SOFT cap (2026-10-23): a session above it is + still listed, flagged ``oversize``. The hard cap it replaced + discarded 563 real hermes sessions (median 187 messages, 99 tool + calls) before they could reach ``split_long`` β€” the stage that + actually bounds what the pipeline processes, by rendered size + rather than by message count. ``min_messages`` / ``min_tool_calls`` + stay hard: they filter degenerate stubs, not long work. + """ + q = ( + "SELECT s.id, s.model, s.title, s.display_name, s.started_at," + " s.ended_at, s.message_count, s.tool_call_count," + " s.input_tokens, s.output_tokens, s.cwd, s.source" + " FROM sessions s" + " WHERE s.message_count >= ? AND s.tool_call_count >= ?" + " AND s.source IN (" + ",".join("?" * len(_CANDIDATE_SOURCES)) + ")" + ) + # remembered for stats()'s SQL-level drop census + self._last_min_messages = min_messages + self._last_min_tool_calls = min_tool_calls + self._last_require_ended = require_ended + self._last_max_input_tokens = max_input_tokens + self._last_exclude_cwd = exclude_cwd + params: list[Any] = [min_messages, min_tool_calls, *_CANDIDATE_SOURCES] + if exclude_cwd: + # Self-reference guard: improver children run with cwd inside + # the improver workdir; a mis-tagged source (e.g. inherited + # 'desktop') must never let them become candidates. + q += " AND (s.cwd IS NULL OR s.cwd NOT LIKE ?)" + params.append(exclude_cwd.rstrip("/\\") + "/%") + if require_ended: + q += " AND s.ended_at IS NOT NULL" + if source is not None: + q += " AND s.source = ?" + params.append(source) + if max_input_tokens is not None: + q += " AND s.input_tokens <= ?" + params.append(max_input_tokens) + q += " ORDER BY s.started_at DESC" + # (LIMIT applied per-db below, after the merge) + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for _db, conn in self._iter_dbs(): + try: + got = conn.execute(q, params).fetchall() + finally: + conn.close() + for r in got: + if r[0] in seen: + continue + seen.add(r[0]) + if exclude_cwd and in_workdir(r[10], exclude_cwd): + bump(self.skips, "cwd_self_reference") + continue + rows.append( + oversize_row( + { + "session_id": r[0], + "model": r[1], + "title": (r[2] or r[3] or "").strip(), + "started_at": r[4], + "ended_at": r[5], + "message_count": r[6], + "tool_call_count": r[7], + "input_tokens": r[8], + "output_tokens": r[9], + "cwd": r[10], + "source": r[11], + }, + max_messages, + ) + ) + rows.sort(key=lambda r: r["started_at"] or 0.0, reverse=True) + if limit is not None: + rows = rows[:limit] + return rows + + # ------------------------------------------------------------- messages + + MSG_QUERY = ( + "SELECT id, role, content, tool_call_id, tool_calls, tool_name," + " timestamp, reasoning, reasoning_content, active, compacted" + " FROM messages WHERE session_id = ?" + " ORDER BY timestamp ASC, id ASC" + ) + + def load_trace(self, session_id: str) -> Trace: + """Load one full trace as a normalized Trace (searches all dbs).""" + for _db, conn in self._iter_dbs(): + try: + sess = conn.execute( + "SELECT id, model, title, display_name, started_at, cwd," + " source FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if not sess: + continue + rows = conn.execute(self.MSG_QUERY, (session_id,)).fetchall() + return Trace( + session_id=sess[0], + model=sess[1], + title=(sess[2] or sess[3] or "").strip() or None, + started_at=sess[4], + cwd=sess[5], + messages=list(self._parse_rows(rows)), + source="hermes" if not sess[6] else str(sess[6]), + ) + finally: + conn.close() + raise KeyError(f"session not found: {session_id}") + + def _parse_rows(self, rows: list[sqlite3.Row]) -> list[Message]: + out: list[Message] = [] + for (mid, role, content, tc_id, tc_json, tool_name, ts, + reasoning, reasoning_content, active, compacted) in rows: + if not active: + bump(self.skips, "inactive_turn") + continue # rewound/deleted turns + m = Message(role=role, content=content, timestamp=ts) + if role == "tool": + m.tool_call_id = tc_id + m.tool_name = tool_name + elif role == "assistant" and tc_json: + calls = json_loads_lenient(tc_json) + if isinstance(calls, list): + for c in calls: + fn = c.get("function") or {} + m.tool_calls.append( + ToolCall( + id=str(c.get("call_id") or c.get("id") or ""), + name=str(fn.get("name") or ""), + arguments=fn.get("arguments") or "{}", + ) + ) + reasoning = reasoning or reasoning_content + if reasoning: + m.reasoning = reasoning + out.append(m) + return out + + # -------------------------------------------------------------- helpers + + def _sql_drop_census(self) -> dict[str, int]: + """Attribute every dropped session to the filter that took it. + + ``list_sessions`` applies min_messages / min_tool_calls / the source + whitelist / require_ended / max_input_tokens / the cwd guard inside + ONE SQL query, so none of those drops can bump a counter β€” measured + 2026-10-24: 6,741 of 9,110 sessions vanished with ``skips == {}``. + A skip census that reports nothing for the store that drops the most + is worse than no census, so the attribution runs here, in the same + order the WHERE clause applies. + + Read-only and cheap (one row scan per db, ~9k rows). + """ + census: dict[str, int] = {} + seen: set[str] = set() + + def bump_census(reason: str) -> None: + census[reason] = census.get(reason, 0) + 1 + + for _db, conn in self._iter_dbs(): + try: + rows = conn.execute( + "SELECT id, message_count, tool_call_count, source," + " ended_at, cwd, input_tokens FROM sessions" + ).fetchall() + finally: + conn.close() + for (sid, mc, tc, src, ended, cwd, itok) in rows: + if sid in seen: + bump_census("cross_db_duplicate") + continue + seen.add(sid) + if src not in _CANDIDATE_SOURCES: + bump_census(f"source_not_candidate:{src}") + elif (mc or 0) < self._last_min_messages: + bump_census("below_min_messages") + elif (tc or 0) < self._last_min_tool_calls: + bump_census("below_min_tool_calls") + elif self._last_require_ended and ended is None: + bump_census("not_ended") + elif (self._last_max_input_tokens is not None + and (itok or 0) > self._last_max_input_tokens): + bump_census("over_max_input_tokens") + elif self._last_exclude_cwd and cwd and in_workdir( + cwd, self._last_exclude_cwd + ): + bump_census("cwd_self_reference") + return census + + def stats(self) -> dict[str, Any]: + """Quick counts over the store(s) for sanity checks. + + ``skips`` merges the row-level counters with the SQL-level drop + census, so the store's whole funnel is visible in one place. + """ + out: dict[str, Any] = {"dbs": len(self._existing_dbs())} + for key in ("sessions", "messages", "tool_rows"): + out[key] = 0 + for _db, conn in self._iter_dbs(): + try: + one = lambda sql: conn.execute(sql).fetchone()[0] + out["sessions"] += one("SELECT COUNT(*) FROM sessions") + out["messages"] += one("SELECT COUNT(*) FROM messages") + out["tool_rows"] += one( + "SELECT COUNT(*) FROM messages WHERE role='tool'" + ) + finally: + conn.close() + skips = dict(self.skips) + for reason, n in self._sql_drop_census().items(): + skips[reason] = skips.get(reason, 0) + n + out["skips"] = skips + return out \ No newline at end of file diff --git a/traceforge/readers/nemotron.py b/traceforge/readers/nemotron.py new file mode 100644 index 0000000000000000000000000000000000000000..8706045d25e1a7bd44121066e5ef9cd958cbba7e --- /dev/null +++ b/traceforge/readers/nemotron.py @@ -0,0 +1,491 @@ +"""Nemotron SFT dataset reader β€” NVIDIA's published agentic SFT corpora. + +These are not live agent stores. Each row is an already-finished +``(messages, tools)`` pair from a released SFT dataset, so the reader's +job is narrower than the harness readers': normalize the row into a +:class:`~traceforge.trace.Trace` and serve it by session id. + +Why they belong in the pool at all: the live stores are ~95% coding and +English, and the corpus's weak buckets (research, data-analysis, ops, +office-docs) cannot be filled by re-reading the same harness. The +Nemotron agentic subsets carry tool-call trajectories from other +harnesses and other domains (SEC filings, customer support, weather), +which is exactly the coverage the classifier reports as missing. + +Three normalizations the raw rows need: + +1. ``messages`` is a JSON *string* in some subsets (SWE-v3.5) and a list + in others β€” both are accepted. +2. SWE-v3.5 tool rows carry only ``content``: no ``tool_call_id`` and no + ``name``. They are paired positionally against the issuing assistant + turn's calls (verified 1:1 in order on the released shards). +3. ``tools`` is a separate column with no counterpart in the Trace + model, and the export has no field for it either. Dropping it would + leave tool calls pointing at tools nothing defines, so the schemas are + folded into the system row as a compact block (name, short + description, parameter names) β€” capped, because SWE-v3.5 ships 33 KB + of JSON schema per row and the OpenCode system prompt already + describes the tools in prose. + +Provenance is carried twice: ``source`` (``nemotron-swe`` …) for the +benchmark's ``by_source`` grouping, and the session-id namespace +(``nemotron-swe-``) because the improve stage overwrites ``source`` +with ``traceforge-splice`` while the id survives verbatim. +""" +from __future__ import annotations + +import hashlib +import json +import os +from typing import Any, Optional + +from traceforge.readers._common import bump, new_skips, oversize_row +from traceforge.trace import VALID_ROLES, Message, ToolCall, Trace + +# Subset -> provenance tag. The tag is the reader's contract with the +# benchmark: `by_source` groups on it, so it must stay stable. +SUBSET_SOURCES = { + "swe": "nemotron-swe", + "agentic-tc": "nemotron-agentic-tc", + "agentic-ka": "nemotron-agentic-ka", +} + +# Cap on the folded tool-schema block. SWE-v3.5 ships 33 KB of JSON schema +# per row; the OpenCode system prompt already names and describes every +# tool, so the block only has to make the calls resolvable. +TOOLS_BLOCK_MAX_CHARS = 4000 + + +# --------------------------------------------------------------- row parsing + +def as_list(value: Any) -> Optional[list]: + """A JSON list, whether the column stores a list or a JSON string.""" + if isinstance(value, list): + return value + if isinstance(value, str): + try: + got = json.loads(value) + except json.JSONDecodeError: + return None + return got if isinstance(got, list) else None + return None + + +def content_text(value: Any) -> str: + """Message content as text. + + OpenAI-style content may be a string or a list of parts + (``{"type": "text", "text": …}``); the datasets use strings, but a + part list must not silently become ``str(list)``. + """ + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, list): + out = [] + for part in value: + if isinstance(part, str): + out.append(part) + elif isinstance(part, dict): + out.append(str(part.get("text") or part.get("content") or "")) + return "\n".join(p for p in out if p) + return str(value) + + +def _arguments_text(value: Any) -> str: + """Tool-call arguments as the JSON string the Trace model stores.""" + if value is None: + return "{}" + if isinstance(value, str): + return value + try: + return json.dumps(value, ensure_ascii=False) + except (TypeError, ValueError): + return "{}" + + +def tools_block(tools: Optional[list], max_chars: int = TOOLS_BLOCK_MAX_CHARS) -> str: + """Compact ``## Available tools`` block for the system row. + + Name + first line of the description + parameter names. Full JSON + schemas are deliberately not reproduced: they are 33 KB on SWE-v3.5 + and the arguments in the trace are what the trainee has to match. + """ + if not tools: + return "" + lines = ["## Available tools", ""] + for t in tools: + if not isinstance(t, dict): + continue + fn = t.get("function") if isinstance(t.get("function"), dict) else t + if not isinstance(fn, dict): + continue + name = str(fn.get("name") or "").strip() + if not name: + continue + desc = str(fn.get("description") or "").strip().split("\n")[0][:160] + params = fn.get("parameters") or {} + props = params.get("properties") if isinstance(params, dict) else None + arg_names = ", ".join(props.keys()) if isinstance(props, dict) else "" + required = params.get("required") if isinstance(params, dict) else None + req = ", ".join(str(r) for r in required) if isinstance(required, list) else "" + line = f"- `{name}`" + if desc: + line += f": {desc}" + if arg_names: + line += f" (args: {arg_names}" + line += f"; required: {req})" if req else ")" + lines.append(line) + if sum(len(x) for x in lines) > max_chars: + lines.append("- … (further tool schemas omitted)") + break + return "\n".join(lines) + + +def _normalize_calls(raw_calls: list, skips, counter: list[int]) -> list[ToolCall]: + """Assistant tool calls -> ToolCall list, synthesizing missing ids. + + ``counter`` is a one-element list shared across the whole row: SWE-v3.5 + ships no call ids at all, and a per-message ``call_0`` would collide + across turns (the export pairs results by id). + """ + out: list[ToolCall] = [] + for tc in raw_calls: + if not isinstance(tc, dict): + continue + fn = tc.get("function") if isinstance(tc.get("function"), dict) else tc + if not isinstance(fn, dict): + continue + name = str(fn.get("name") or "").strip() + if not name: + bump(skips, "tool_call_without_name") + continue + call_id = str(tc.get("id") or tc.get("call_id") or "").strip() + if not call_id: + call_id = f"call_{counter[0]}" + counter[0] += 1 + out.append( + ToolCall( + id=call_id, + name=name, + arguments=_arguments_text(fn.get("arguments")), + ) + ) + return out + + +def _title_from(text: str) -> Optional[str]: + """First substantive line of the task, for the listing. + + SWE-v3.5 opens with an ```` envelope whose next line is + a bare ``/workspace/repo`` path, so neither the literal first line nor + the first non-tag line is a title. Skip markup, bare paths and short + fragments; fall back to the first non-empty line. + """ + fallback: Optional[str] = None + for line in (text or "").splitlines(): + s = line.strip().strip('"').strip() + if not s: + continue + if fallback is None: + fallback = s[:80] + if s.startswith("<") or s.startswith("```"): + continue + if " " not in s and ("/" in s or "\\" in s): + continue # a bare path, not prose + if len(s) < 20 or len(s.split()) < 3: + continue + return s[:80] + return fallback + + +def normalize_row( + row: dict, + subset: str, + *, + skips=None, + embed_tools: bool = True, +) -> Optional[Trace]: + """One dataset row -> Trace, or None when the row cannot be a trace. + + Returns None (and bumps a skip counter) for a row with no messages, no + user turn, or no assistant turn β€” those are not improvable traces and + must not reach the raw pool. + """ + source = SUBSET_SOURCES.get(subset, f"nemotron-{subset}") + raw_msgs = as_list(row.get("messages")) + if not raw_msgs: + bump(skips, "no_messages") + return None + + messages: list[Message] = [] + pending: list[ToolCall] = [] # calls awaiting their result row + counter = [0] # synthesized call-id counter, shared across the row + for raw in raw_msgs: + if not isinstance(raw, dict): + bump(skips, "non_dict_message") + continue + role = str(raw.get("role") or "").strip() + if role not in VALID_ROLES: + bump(skips, f"unknown_role:{role or '?'}") + continue + text = content_text(raw.get("content")) + if role == "tool": + if not pending: + # Orphan result: no issuing call in this row. Keeping it + # would fail the pairing gate on every attempt. + bump(skips, "orphan_tool_result") + continue + call = pending.pop(0) + messages.append( + Message( + role="tool", + content=text, + tool_call_id=str(raw.get("tool_call_id") or call.id), + tool_name=str(raw.get("name") or call.name), + ) + ) + continue + if role == "assistant": + calls = _normalize_calls(raw.get("tool_calls") or [], skips, counter) + reasoning = raw.get("reasoning_content") or raw.get("reasoning") + if not text and not calls: + bump(skips, "empty_assistant_row") + continue + pending.extend(calls) + messages.append( + Message( + role="assistant", + content=text or None, + reasoning=str(reasoning) if reasoning else None, + tool_calls=calls, + ) + ) + continue + if role == "system" and not text.strip(): + bump(skips, "empty_system_row") + continue + messages.append(Message(role=role, content=text)) + + if pending: + # Calls with no result: the row was cut mid-run. Drop the calls so + # the trace does not carry an unanswered invocation. + bump(skips, "unanswered_tool_call", len(pending)) + dead = {c.id for c in pending} + for m in messages: + if m.role == "assistant" and m.tool_calls: + m.tool_calls = [c for c in m.tool_calls if c.id not in dead] + + roles = {m.role for m in messages} + if "user" not in roles: + bump(skips, "no_user_message") + return None + if "assistant" not in roles: + bump(skips, "no_assistant_message") + return None + + if embed_tools: + block = tools_block(as_list(row.get("tools"))) + if block: + if messages and messages[0].role == "system": + messages[0].content = (messages[0].content or "").rstrip() + "\n\n" + block + else: + messages.insert(0, Message(role="system", content=block)) + + uuid = str(row.get("uuid") or "").strip() + if not uuid: + uuid = hashlib.sha1( + json.dumps(row, sort_keys=True, ensure_ascii=False).encode("utf-8") + ).hexdigest()[:16] + first_user = next((m.content for m in messages if m.role == "user"), "") or "" + return Trace( + session_id=f"{source}-{uuid}", + title=_title_from(first_user), + messages=messages, + source=source, + ) + + +# ------------------------------------------------------------------- reader + +class NemotronTraceReader: + """Reads normalized Nemotron SFT traces from one or more JSONL roots. + + Each root holds one ``.jsonl`` per dataset subset, one Trace + dict per line (written by ``scripts/fetch_nemotron.py``). A byte-offset + index per file is cached next to the data, so ``load_trace`` seeks + instead of re-parsing a multi-hundred-MB file. + """ + + def __init__(self, roots: Optional[list[str]] = None): + if isinstance(roots, str): + roots = [roots] + self.roots = [os.path.abspath(os.path.expanduser(r)) for r in (roots or [])] + self.skips = new_skips() + + # ------------------------------------------------------------- indexing + + def _files(self) -> list[str]: + out: list[str] = [] + for root in self.roots: + if not os.path.isdir(root): + continue + for name in sorted(os.listdir(root)): + if name.endswith(".jsonl"): + out.append(os.path.join(root, name)) + return out + + def _index_path(self, path: str) -> str: + root = os.path.dirname(path) + return os.path.join(root, ".index", os.path.basename(path) + ".idx.json") + + def _build_index(self, path: str) -> dict[str, Any]: + """Byte offsets + summary per row, cached on (size, mtime).""" + try: + st = os.stat(path) + except OSError: + bump(self.skips, "unreadable_file") + return {"sig": "", "rows": []} + sig = f"{st.st_size:x}-{int(st.st_mtime):x}" + cache_path = self._index_path(path) + try: + with open(cache_path, "r", encoding="utf-8") as f: + cached = json.load(f) + if cached.get("sig") == sig: + return cached + except (OSError, json.JSONDecodeError): + pass + + rows: list[dict[str, Any]] = [] + with open(path, "rb") as f: + offset = 0 + for raw in f: + length = len(raw) + if raw.strip(): + try: + d = json.loads(raw.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + bump(self.skips, "unparseable_row") + offset += length + continue + msgs = d.get("messages") or [] + rows.append( + { + "sid": d.get("session_id"), + "offset": offset, + "length": length, + "n_messages": len(msgs), + "n_tool_calls": sum( + len(m.get("tool_calls") or []) for m in msgs + ), + "chars": sum(len(m.get("content") or "") for m in msgs), + "title": d.get("title"), + "source": d.get("source"), + "started_at": d.get("started_at"), + } + ) + offset += length + index = {"sig": sig, "rows": rows} + try: + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + tmp = cache_path + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(index, f) + os.replace(tmp, cache_path) + except OSError: + pass # a read-only seed dir still works, just slower + return index + + def _index(self) -> dict[str, dict[str, Any]]: + """session id -> index entry (with its file path).""" + out: dict[str, dict[str, Any]] = {} + for path in self._files(): + for row in self._build_index(path).get("rows", []): + sid = row.get("sid") + if not sid: + bump(self.skips, "row_without_session_id") + continue + entry = dict(row) + entry["path"] = path + out[sid] = entry + return out + + # ------------------------------------------------------------ sessions + + def list_sessions( + self, + *, + min_messages: int = 10, + max_messages: int = 120, + min_tool_calls: int = 2, + limit: Optional[int] = None, + status_cb=None, + **_ignored, + ) -> list[dict[str, Any]]: + """Candidate traces, newest first (summaries only). + + ``max_messages`` is the same SOFT cap the other readers use: an + over-cap row is listed with ``oversize`` and bounded later by + ``split_long``. ``min_messages`` / ``min_tool_calls`` stay hard. + """ + if status_cb: + status_cb(f"indexing {len(self._files())} nemotron file(s)") + rows: list[dict[str, Any]] = [] + for sid, e in self._index().items(): + if e["n_messages"] < min_messages: + bump(self.skips, "below_min_messages") + continue + if e["n_tool_calls"] < min_tool_calls: + bump(self.skips, "below_min_tool_calls") + continue + rows.append( + oversize_row( + { + "session_id": sid, + "model": None, + "title": e.get("title") or "", + "started_at": e.get("started_at"), + "ended_at": None, + "message_count": e["n_messages"], + "tool_call_count": e["n_tool_calls"], + "input_tokens": None, + "output_tokens": None, + "cwd": None, + "source": e.get("source") or "nemotron", + }, + max_messages, + ) + ) + rows.sort(key=lambda r: (r["started_at"] or 0.0, r["session_id"]), reverse=True) + if limit is not None: + rows = rows[:limit] + return rows + + # ------------------------------------------------------------- loading + + def load_trace(self, session_id: str) -> Trace: + entry = self._index().get(session_id) + if entry is None: + raise KeyError(f"session not found: {session_id}") + with open(entry["path"], "rb") as f: + f.seek(entry["offset"]) + raw = f.read(entry["length"]) + return Trace.from_dict(json.loads(raw.decode("utf-8"))) + + # --------------------------------------------------------------- stats + + def stats(self) -> dict[str, Any]: + files = self._files() + idx = self._index() + return { + "files": len(files), + "traces": len(idx), + "messages": sum(e["n_messages"] for e in idx.values()), + "tool_calls": sum(e["n_tool_calls"] for e in idx.values()), + "by_source": { + src: sum(1 for e in idx.values() if e.get("source") == src) + for src in sorted({e.get("source") or "?" for e in idx.values()}) + }, + "skips": dict(self.skips), + } diff --git a/traceforge/space/__init__.py b/traceforge/space/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..54c25a2e4077da3f365b835f78af6298ff23dd87 --- /dev/null +++ b/traceforge/space/__init__.py @@ -0,0 +1,23 @@ +"""Portable traceforge: content-addressed seeds, a queue, and a Gradio UI. + +The package is import-safe without Gradio β€” `app.py` imports it lazily, so +the CLI and the tests never pay for the UI dependency. + + from traceforge.space import TraceStore, SpacePipeline, seed_hash +""" +from traceforge.space.hashing import (canonical_messages, fingerprint, + output_hash, seed_hash, short_hash, + tool_names) +from traceforge.space.ingest import parse_records, read_upload, trace_from_record +from traceforge.space.pipeline import (SpacePipeline, pipeline_ready, + space_config) +from traceforge.space.store import (DONE, FAILED, QUEUED, REJECTED, RUNNING, + TraceStore) + +__all__ = [ + "TraceStore", "SpacePipeline", "space_config", "pipeline_ready", + "seed_hash", "output_hash", "short_hash", "fingerprint", + "canonical_messages", "tool_names", "trace_from_record", + "parse_records", "read_upload", + "QUEUED", "RUNNING", "DONE", "FAILED", "REJECTED", +] diff --git a/traceforge/space/__pycache__/__init__.cpython-314.pyc b/traceforge/space/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb24ea3c7a20a915783fd630358cb05f3c9bff42 Binary files /dev/null and b/traceforge/space/__pycache__/__init__.cpython-314.pyc differ diff --git a/traceforge/space/__pycache__/hashing.cpython-314.pyc b/traceforge/space/__pycache__/hashing.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f414d659e7e006b568c3b911c4a41a87e686ac09 Binary files /dev/null and b/traceforge/space/__pycache__/hashing.cpython-314.pyc differ diff --git a/traceforge/space/__pycache__/ingest.cpython-314.pyc b/traceforge/space/__pycache__/ingest.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7234af6190d5aa2d3d30f8bbf62473fc0e442dd2 Binary files /dev/null and b/traceforge/space/__pycache__/ingest.cpython-314.pyc differ diff --git a/traceforge/space/__pycache__/pipeline.cpython-314.pyc b/traceforge/space/__pycache__/pipeline.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c74d7084d4d92812891886859486a41efa98c3e Binary files /dev/null and b/traceforge/space/__pycache__/pipeline.cpython-314.pyc differ diff --git a/traceforge/space/__pycache__/stats.cpython-314.pyc b/traceforge/space/__pycache__/stats.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07a47b1def957abfbe12932dd871f97fbecc448a Binary files /dev/null and b/traceforge/space/__pycache__/stats.cpython-314.pyc differ diff --git a/traceforge/space/__pycache__/store.cpython-314.pyc b/traceforge/space/__pycache__/store.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db41ed7508a4f682b6f5b049abe3fe28bc177b61 Binary files /dev/null and b/traceforge/space/__pycache__/store.cpython-314.pyc differ diff --git a/traceforge/space/app.py b/traceforge/space/app.py new file mode 100644 index 0000000000000000000000000000000000000000..53acd8b6bf57a74306f47a8f72dabef439a62f37 --- /dev/null +++ b/traceforge/space/app.py @@ -0,0 +1,465 @@ +"""Gradio app: submit a trace, sponsor a dataset, read the dashboard. + +The UI is deliberately thin β€” every decision lives in `SpacePipeline` and +`TraceStore`, so the same code runs locally (`python app.py`) and on a +Hugging Face Space (`app.py` at the repo root). + +Tabs + Submit one trace (JSON / JSONL / pasted) -> hash -> queue -> output + Sponsor a JSONL of traces -> hashed, deduped, queued, attributed + Dashboard public stats: corpus, time sensitivity, tasks, sponsors, queue + Browse the stored outputs, by hash +""" +from __future__ import annotations + +import html +import json +import os +from typing import Any, Optional + +from traceforge.space.hashing import short_hash +from traceforge.space.ingest import parse_records, read_upload +from traceforge.space.jobs import get_worker +from traceforge.space.pipeline import SpacePipeline, pipeline_ready, space_config +from traceforge.space.stats import bar_rows, dashboard, pct +from traceforge.space.store import DONE, FAILED, QUEUED, RUNNING, TraceStore + +REFRESH_S = 8.0 + +CSS = """ +.tf-wrap { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } +.tf-kpis { display:grid; grid-template-columns:repeat(auto-fit,minmax(130px,1fr)); + gap:8px; margin:6px 0 14px; } +.tf-kpi { background:#161b22; border:1px solid #2a3140; border-radius:8px; + padding:9px 11px; } +.tf-kpi .v { font-size:21px; font-weight:600; color:#e6edf3; line-height:1.2; } +.tf-kpi .k { font-size:11px; color:#8b949e; text-transform:uppercase; + letter-spacing:.04em; } +.tf-kpi .s { font-size:11px; color:#6e7681; } +.tf-ok { color:#3fb950; } .tf-warn { color:#d29922; } .tf-bad { color:#f85149; } +.tf-bar-row { display:grid; grid-template-columns:190px 1fr 64px; gap:8px; + align-items:center; font-size:12px; color:#9aa4b2; margin:3px 0; } +.tf-bar-outer { background:#1b2029; border-radius:6px; height:14px; + overflow:hidden; border:1px solid #2a3140; } +.tf-bar-inner { height:100%; background:linear-gradient(90deg,#2f81f7,#3fb950); } +.tf-head { font-size:13px; color:#8b949e; margin:2px 0 8px; } +.tf-pill { padding:2px 8px; border-radius:999px; font-size:11px; + border:1px solid #2a3140; } +""" + + +# --------------------------------------------------------------------- html + +def _esc(s: Any) -> str: + return html.escape(str(s if s is not None else "")) + + +def _kpi(value: Any, key: str, sub: str = "", cls: str = "") -> str: + sub_html = f'
{_esc(sub)}
' if sub else "" + return (f'
{_esc(value)}
' + f'
{_esc(key)}
{sub_html}
') + + +def _bars(counts: dict, total: Optional[int] = None) -> str: + rows = bar_rows(counts, total=total) + if not rows: + return '
no data yet
' + out = ['
'] + for label, n, p in rows: + out.append( + '
' + f'{_esc(label)}' + f'' + f'{n} Β· {p:.0f}%
' + ) + out.append("
") + return "".join(out) + + +def _status_pill(status: str) -> str: + cls = {DONE: "tf-ok", FAILED: "tf-bad", RUNNING: "tf-warn", + QUEUED: "tf-warn"}.get(status, "") + return f'{_esc(status)}' + + +def _kpi_html(space: dict, ref: dict, worker: dict, ready: tuple[bool, str]) -> str: + ok, reason = ready + cards = [ + _kpi(f"{space['seeds']:,}", "seeds", "distinct content hashes"), + _kpi(f"{space['done']:,}", "outputs", + f"{pct(space['done'], space['seeds']):.0f}% of seeds"), + _kpi(f"{space['queued']:,}", "queued", + f"{space['running']} running"), + _kpi(f"{space['failed']:,}", "failed", "rejected or errored"), + _kpi(f"{len(space['sponsors'])}", "sponsors", + f"{sum(s['seeds'] for s in space['sponsors'])} seeds"), + _kpi(f"{space['duration_s']['median']:.0f}s", "median runtime", + f"p90 {space['duration_s']['p90']:.0f}s"), + ] + if ref.get("available"): + cards.append(_kpi(f"{ref.get('traces', 0):,}", "reference corpus", + "published ideal traces")) + head = ( + '
' + f'' + f'{"pipeline ready" if ok else "pipeline not configured"} ' + f'{_esc(reason)} Β· worker {"alive" if worker.get("alive") else "stopped"}' + f' Β· queue {worker.get("queued", 0)}' + "
" + ) + return head + '
' + "".join(cards) + "
" + + +def _ts_html(space: dict, ref: dict) -> str: + """Time sensitivity: the metric that says whether an answer decays.""" + parts = ['
' + "time sensitivity β€” a temporal fact answered from weights decays; " + "the same fact fetched with a tool does not
"] + space_ts = space.get("time_sensitivity") or {} + if space_ts: + parts.append("this Space") + parts.append(_bars(space_ts)) + census = (ref or {}).get("time_sensitivity") or {} + if census.get("available"): + parts.append(f"reference corpus ({census.get('total', 0):,} traces)") + parts.append(_bars(census.get("classes") or {})) + parts.append( + f'
needs rewrite (tool available but never ' + f'called): {census.get("needs_rewrite", 0):,}
') + if not space_ts and not census.get("available"): + parts.append('
no data yet
') + parts.append("
") + return "".join(parts) + + +# ------------------------------------------------------------------- tables + +def _rows(records: list[dict], cols: list[str]) -> list[list]: + out = [] + for r in records: + row = [] + for c in cols: + v = r.get(c) + if c == "hash": + v = short_hash(v or "") + elif c == "time_sensitivity": + v = (v or {}).get("class", "") + elif isinstance(v, float): + v = round(v, 1) + row.append(v if v is not None else "") + out.append(row) + return out + + +# ---------------------------------------------------------------------- app + +def build_app(store: Optional[TraceStore] = None, + pipeline: Optional[SpacePipeline] = None): + import gradio as gr + + cfg = pipeline.cfg if pipeline else space_config() + store = store or (pipeline.store if pipeline else TraceStore( + os.path.join(cfg.data_dir))) + pipeline = pipeline or SpacePipeline(store, cfg) + worker = get_worker(pipeline) + ready = pipeline_ready(cfg) + dataset_dir = os.path.join(os.path.dirname(cfg.data_dir), "dataset") + cache_path = os.path.join(store.root, "reference_census.json") + + # ------------------------------------------------------------- submit + + def on_submit(file_path, pasted, sponsor, sync): + if not ready: + return ("**pipeline not configured** β€” this Space has no LLM endpoint, " + "so nothing was queued. Set `TRACEFORGE_BASE_URL` and " + "`TRACEFORGE_LLM_KEY` in the Space settings and restart.", None, None) + if file_path: + records, errors = read_upload(file_path) + else: + records, errors = parse_records(pasted or "") + if errors: + return (f"**parse errors**\n\n" + "\n".join(f"- {e}" for e in errors[:10]), + None, None) + if not records: + return "**nothing to submit** β€” upload a file or paste JSON", None, None + if len(records) > 1: + return (f"**{len(records)} records** β€” the Submit tab takes one trace. " + "Use **Sponsor a dataset** for a batch.", None, None) + try: + res = pipeline.submit(records[0], sponsor=sponsor) + except Exception as e: + return f"**submit failed**: {e}", None, None + h = res["hash"] + if res.get("duplicate") and res.get("status") == DONE: + path = store.output_path(h) + return (f"**already processed** β€” this seed hash already has an output.\n\n" + f"`{h}`\n\nNo second output was created.", h, + path if os.path.exists(path) else None) + if sync: + res = pipeline.process(h) + rec = pipeline.result(h) + md = _result_md(rec) + path = store.output_path(h) if rec.get("status") == DONE else None + return md, h, path if path and os.path.exists(path) else None + + def on_check(h): + if not h: + return "no hash yet β€” submit a trace first", None + rec = pipeline.result(h) + path = store.output_path(h) if rec.get("status") == DONE else None + return _result_md(rec), path if path and os.path.exists(path) else None + + def _result_md(rec: dict) -> str: + h = rec.get("hash") or "" + st = rec.get("status") or "unknown" + lines = [f"### {_status_pill(st)} `{h}`", ""] + if rec.get("duplicate"): + lines.append("_duplicate submission β€” the existing record was returned_") + lines.append("") + if rec.get("task"): + lines.append(f"- task: **{rec['task']}** Β· language: **{rec.get('language', '?')}**") + if rec.get("n_messages"): + lines.append(f"- seed: {rec['n_messages']} messages Β· " + f"{rec.get('n_tool_calls', 0)} tool calls") + if rec.get("n_messages_out"): + lines.append(f"- output: {rec['n_messages_out']} messages") + if rec.get("route"): + lines.append(f"- route: `{rec['route']}` Β· attempts: {rec.get('attempts', 0)}") + if rec.get("duration_s"): + lines.append(f"- runtime: {rec['duration_s']:.0f}s") + ts = rec.get("time_sensitivity") or {} + if ts.get("class"): + lines.append(f"- time sensitivity: **{ts['class']}** β€” {ts.get('label', '')}") + if ts.get("temporal_facts"): + lines.append(f" - temporal facts: {', '.join(ts['temporal_facts'])}") + if ts.get("retrieval_available"): + lines.append(f" - retrieval available: {', '.join(ts['retrieval_available'])}") + if ts.get("needs_rewrite"): + lines.append(" - the raw trace answered from weights although a " + "retrieval tool was available") + ts2 = rec.get("time_sensitivity_improved") or {} + if ts2.get("class") and ts2.get("class") != ts.get("class"): + lines.append(f"- after rewrite: **{ts2['class']}**") + if rec.get("error"): + lines.append(f"- error: `{rec['error'][:600]}`") + if st == QUEUED: + lines.append("") + lines.append("_queued β€” the worker picks it up in submission order. " + "Press **Check status** to refresh._") + return "\n".join(lines) + + # ------------------------------------------------------------ sponsor + + def on_sponsor(file_path, pasted, sponsor): + if not ready: + return ("**pipeline not configured** β€” this Space has no LLM endpoint, " + "so nothing was queued. Set `TRACEFORGE_BASE_URL` and " + "`TRACEFORGE_LLM_KEY` in the Space settings and restart.", [], "") + if file_path: + records, errors = read_upload(file_path) + else: + records, errors = parse_records(pasted or "") + if errors: + return (f"**parse errors**\n\n" + "\n".join(f"- {e}" for e in errors[:10]), + [], "") + if not records: + return "**nothing to sponsor** β€” upload a JSONL of traces", [], "" + results = pipeline.submit_many(records, sponsor=sponsor) + new = sum(1 for r in results if not r.get("duplicate") and r.get("hash")) + dup = sum(1 for r in results if r.get("duplicate")) + err = sum(1 for r in results if not r.get("hash")) + rows = [] + for r in results: + rows.append([ + short_hash(r.get("hash") or ""), + r.get("status") or "error", + "duplicate" if r.get("duplicate") else ("new" if r.get("hash") else "error"), + r.get("n_messages", ""), + r.get("task", "")[:60] if r.get("task") else "", + (r.get("time_sensitivity") or {}).get("class", ""), + (r.get("error") or "")[:120], + ]) + md = (f"**{len(records)} records** β€” {new} new, {dup} already known, " + f"{err} rejected. Sponsor: **{_esc(sponsor or 'anonymous')}**.\n\n" + "New seeds are queued; each one produces at most one output.") + return md, rows, sponsor or "anonymous" + + # ---------------------------------------------------------- dashboard + + def on_dashboard(): + d = dashboard(store, dataset_dir=dataset_dir, cache_path=cache_path) + space, ref = d["space"], d["reference"] + w = worker.status() + kpis = _kpi_html(space, ref, w, ready) + ts = _ts_html(space, ref) + tasks = bar_rows(space.get("tasks") or {}) + langs = bar_rows(space.get("languages") or {}) + sponsors = [[s["sponsor"], s["seeds"], s["done"], s["failed"]] + for s in space.get("sponsors") or []] + recent = _rows(d["recent"], ["hash", "status", "sponsor", "task", + "time_sensitivity", "duration_s"]) + queue = _rows(d["queue"], ["hash", "sponsor", "task", "n_messages"]) + ref_rows = [] + if ref.get("available"): + card = ref.get("card") or {} + for k, v in (card.get("tasks") or {}).items(): + ref_rows.append([k, v, pct(v, card.get("traces", 0))]) + return kpis, ts, tasks, langs, sponsors, recent, queue, ref_rows + + # ------------------------------------------------------------- browse + + def browse_choices(): + done = [r for r in store.all() if r.get("status") == DONE] + done.sort(key=lambda r: r.get("updated_at") or 0, reverse=True) + return gr.update(choices=[r["hash"] for r in done][:500]) + + def on_browse(h): + if not h: + return None, None + out = store.load_output(h) + if out is None: + return None, None + return out.to_dict(), store.output_path(h) + + # ---------------------------------------------------------------- UI + + with gr.Blocks(title="traceforge β€” ideal-world agent traces") as app: + gr.Markdown( + "# traceforge\n" + "Turn a real agent trace into its **ideal-world reference**: the same " + "work, with the dead ends removed, gated for structure and grounding.\n\n" + "Every seed is identified by a **content hash** β€” submitting the same " + "trace twice returns the existing output instead of creating a second one." + ) + + with gr.Tab("Submit a trace"): + with gr.Row(): + with gr.Column(scale=3): + file_in = gr.File(label="trace file (.json / .jsonl)", + file_types=[".json", ".jsonl"], + type="filepath") + paste_in = gr.Textbox( + label="…or paste a trace (JSON, JSONL, or a message list)", + lines=10, placeholder='{"messages": [{"role": "user", ...}]}') + with gr.Column(scale=2): + sponsor_in = gr.Textbox(label="your name / handle (optional)", + max_lines=1) + sync_in = gr.Checkbox( + label="process synchronously (debug β€” blocks until done)", + value=os.environ.get("TRACEFORGE_SPACE_SYNC") == "1") + btn_submit = gr.Button("Submit", variant="primary") + btn_check = gr.Button("Check status") + last_hash = gr.State("") + status_md = gr.Markdown("_no submission yet_") + out_file = gr.File(label="improved trace (download)", interactive=False) + + btn_submit.click(on_submit, + inputs=[file_in, paste_in, sponsor_in, sync_in], + outputs=[status_md, last_hash, out_file]) + btn_check.click(on_check, inputs=[last_hash], + outputs=[status_md, out_file]) + + with gr.Tab("Sponsor a dataset"): + gr.Markdown( + "Upload a JSONL of traces. Each row is hashed; **new** seeds are " + "queued for the pipeline and attributed to you, **known** seeds are " + "skipped. The processed rows join the public output dataset." + ) + with gr.Row(): + with gr.Column(scale=3): + sp_file = gr.File(label="dataset (.jsonl)", file_types=[".jsonl"], + type="filepath") + sp_paste = gr.Textbox(label="…or paste JSONL", lines=8) + with gr.Column(scale=2): + sp_name = gr.Textbox(label="sponsor name", max_lines=1) + sp_btn = gr.Button("Sponsor", variant="primary") + sp_md = gr.Markdown("_no dataset sponsored yet_") + sp_table = gr.Dataframe( + headers=["hash", "status", "new?", "messages", "task", + "time sensitivity", "error"], + interactive=False, wrap=True, max_height=420, + ) + sp_credit = gr.State("") + sp_btn.click(on_sponsor, inputs=[sp_file, sp_paste, sp_name], + outputs=[sp_md, sp_table, sp_credit]) + + with gr.Tab("Dashboard"): + gr.Markdown("Public statistics. Refreshes automatically.") + dash_kpis = gr.HTML() + dash_ts = gr.HTML() + with gr.Row(): + with gr.Column(): + gr.Markdown("### task mix (this Space)") + dash_tasks = gr.Dataframe( + headers=["task", "seeds", "pct"], interactive=False, + wrap=True, max_height=300) + with gr.Column(): + gr.Markdown("### languages") + dash_langs = gr.Dataframe( + headers=["language", "seeds", "pct"], interactive=False, + wrap=True, max_height=300) + with gr.Row(): + with gr.Column(): + gr.Markdown("### sponsors") + dash_sponsors = gr.Dataframe( + headers=["sponsor", "seeds", "done", "failed"], + interactive=False, wrap=True, max_height=300) + with gr.Column(): + gr.Markdown("### reference corpus β€” task mix") + dash_ref = gr.Dataframe( + headers=["task", "traces", "pct"], interactive=False, + wrap=True, max_height=300) + gr.Markdown("### recent outputs") + dash_recent = gr.Dataframe( + headers=["hash", "status", "sponsor", "task", + "time sensitivity", "runtime s"], + interactive=False, wrap=True, max_height=320) + gr.Markdown("### queue") + dash_queue = gr.Dataframe( + headers=["hash", "sponsor", "task", "messages"], + interactive=False, wrap=True, max_height=240) + + dash_outputs = [dash_kpis, dash_ts, dash_tasks, dash_langs, + dash_sponsors, dash_recent, dash_queue, dash_ref] + timer = gr.Timer(REFRESH_S) + timer.tick(on_dashboard, outputs=dash_outputs) + app.load(on_dashboard, outputs=dash_outputs) + + with gr.Tab("Browse outputs"): + with gr.Row(): + pick = gr.Dropdown(label="processed seed (by hash)", choices=[], + scale=4) + btn_reload = gr.Button("Reload list", scale=1) + preview = gr.JSON(label="improved trace", max_height=520) + dl = gr.File(label="download", interactive=False) + btn_reload.click(browse_choices, outputs=[pick]) + pick.change(on_browse, inputs=[pick], outputs=[preview, dl]) + app.load(browse_choices, outputs=[pick]) + + return app + + +def main(argv: Optional[list[str]] = None) -> int: + import argparse + + import gradio as gr + + ap = argparse.ArgumentParser(description="traceforge Gradio app") + ap.add_argument("--host", default=os.environ.get("GRADIO_SERVER_NAME", + "127.0.0.1")) + ap.add_argument("--port", type=int, + default=int(os.environ.get("GRADIO_SERVER_PORT", "7860"))) + ap.add_argument("--share", action="store_true") + args = ap.parse_args(argv) + + app = build_app() + app.queue(default_concurrency_limit=4) + app.launch(server_name=args.host, server_port=args.port, + share=args.share, inbrowser=False, show_error=True, + css=CSS, theme=gr.themes.Base()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/traceforge/space/hashing.py b/traceforge/space/hashing.py new file mode 100644 index 0000000000000000000000000000000000000000..73e5a094f4e29d32373f8cfb2497d4b6a91f0c10 --- /dev/null +++ b/traceforge/space/hashing.py @@ -0,0 +1,214 @@ +"""Content-addressed identity for a seed trace. + +The Space must never turn one seed trace into two outputs. The key is a +hash over the *conversation content*, not over the harness metadata: + + * session id, model name, cwd, timestamps and tool-call ids are excluded β€” + the same conversation exported twice by two harnesses is one seed; + * message text is normalised (line endings, trailing spaces, blank runs) + so a re-indent or a CRLF conversion does not mint a new seed; + * tool-call arguments are canonicalised as JSON (sorted keys, no spaces) + so `{"a": 1, "b": 2}` and `{"b":2,"a":1}` are the same call; + * the AVAILABLE tool names are included. A trace where the agent had + `web_search` and one where it did not are different seeds: the first is + time-anchored, the second is time-sensitive, and they must not collapse + into one output. + + from traceforge.space.hashing import seed_hash, fingerprint + seed_hash(trace) # 64-char sha256 + fingerprint(trace) # hash + the shape the dashboard reports +""" +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any, Optional + +# Roles the canonical form keeps. Anything else is harness noise. +_KEEP_ROLES = ("system", "user", "assistant", "tool") + +_BLANK_RUN_RE = re.compile(r"\n{3,}") + + +# ------------------------------------------------------------------ helpers + +def _messages(obj: Any) -> list[dict]: + """Accept a Trace, an exported record, or a bare message list.""" + if isinstance(obj, list): + return [m for m in obj if isinstance(m, dict)] + if hasattr(obj, "messages"): + return [m.to_dict() if hasattr(m, "to_dict") else m for m in obj.messages] + if isinstance(obj, dict): + msgs = obj.get("messages") or [] + return [m for m in msgs if isinstance(m, dict)] + return [] + + +def _norm_text(text: Any) -> str: + """Whitespace-normalise message text without changing its words.""" + if text is None: + return "" + if not isinstance(text, str): + # OpenAI content parts: [{"type": "text", "text": "..."}] + if isinstance(text, list): + parts = [] + for p in text: + if isinstance(p, dict): + parts.append(str(p.get("text") or p.get("content") or "")) + else: + parts.append(str(p)) + text = "\n".join(parts) + else: + text = str(text) + text = text.replace("\r\n", "\n").replace("\r", "\n") + text = "\n".join(line.rstrip() for line in text.split("\n")) + text = _BLANK_RUN_RE.sub("\n\n", text) + return text.strip() + + +def _canon_arguments(raw: Any) -> str: + """Canonical JSON for a tool call's arguments. + + Accepts the OpenAI string form, a mapping, or a malformed blob. A blob + that does not parse is normalised as text β€” it is still part of the + seed's identity, just not comparable field-by-field. + """ + if isinstance(raw, (dict, list)): + return json.dumps(raw, sort_keys=True, separators=(",", ":"), + ensure_ascii=False) + s = (raw or "").strip() + if not s: + return "{}" + try: + return json.dumps(json.loads(s), sort_keys=True, + separators=(",", ":"), ensure_ascii=False) + except (ValueError, TypeError): + return _norm_text(s) + + +def _tool_call(tc: Any) -> Optional[dict]: + """Normalise one tool call from either schema. + + Flat (traceforge): {id, name, arguments} + OpenAI: {id, type: "function", function: {name, arguments}} + """ + if not isinstance(tc, dict): + return None + fn_raw = tc.get("function") + fn: dict = fn_raw if isinstance(fn_raw, dict) else {} + name = (fn.get("name") or tc.get("name") or "").strip() + if not name: + return None + args = fn.get("arguments") if fn else tc.get("arguments") + return {"name": name, "arguments": _canon_arguments(args)} + + +def tool_names(obj: Any) -> list[str]: + """Names of the tools the harness made available, sorted. + + Only a record with a `tools` array can answer this; a bare Trace cannot + (the model was never told which tools existed). An empty list is the + honest answer, not a claim that no tools were available. + """ + if not isinstance(obj, dict): + return [] + names: set[str] = set() + for t in obj.get("tools") or []: + if not isinstance(t, dict): + continue + fn = t.get("function") if isinstance(t.get("function"), dict) else {} + name = (fn.get("name") or t.get("name") or "").strip() + if name: + names.add(name) + return sorted(names) + + +# --------------------------------------------------------------- canonical + +def canonical_messages(obj: Any) -> list[dict]: + """The conversation as a stable, metadata-free structure.""" + out: list[dict] = [] + for m in _messages(obj): + role = str(m.get("role") or "").strip().lower() + if role not in _KEEP_ROLES: + continue + rec: dict[str, Any] = {"role": role} + content = _norm_text(m.get("content")) + if content: + rec["content"] = content + calls = [c for c in (_tool_call(tc) for tc in (m.get("tool_calls") or [])) + if c] + if calls: + rec["tool_calls"] = calls + if role == "tool": + name = (m.get("tool_name") or m.get("name") or "").strip() + if name: + rec["tool_name"] = name + # A row with no content and no calls carries nothing. + if len(rec) == 1: + continue + out.append(rec) + return out + + +def canonical_payload(obj: Any) -> dict: + return { + "messages": canonical_messages(obj), + "tools": tool_names(obj), + } + + +def seed_hash(obj: Any) -> str: + """Stable sha256 of a seed trace's content. Same content -> same hash.""" + payload = json.dumps(canonical_payload(obj), sort_keys=True, + separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def output_hash(obj: Any) -> str: + """Hash of an improved trace, for integrity checks on the stored output.""" + return seed_hash(obj) + + +def short_hash(h: str, n: int = 12) -> str: + return (h or "")[:n] + + +# ------------------------------------------------------------- fingerprint + +def fingerprint(obj: Any) -> dict: + """Hash plus the shape the dashboard and the index row report.""" + msgs = _messages(obj) + n_calls = 0 + n_users = 0 + chars = 0 + for m in msgs: + chars += len(_norm_text(m.get("content"))) + if m.get("role") == "user": + n_users += 1 + for tc in (m.get("tool_calls") or []): + n_calls += 1 + fn = tc.get("function") if isinstance(tc.get("function"), dict) else {} + chars += len(str(fn.get("arguments") or tc.get("arguments") or "")) + task = "" + for m in msgs: + if m.get("role") == "user": + task = _norm_text(m.get("content")) + break + lang = "en" + try: + from traceforge.classify import detect_language + lang = detect_language(task) + except Exception: + pass + return { + "hash": seed_hash(obj), + "n_messages": len(msgs), + "n_tool_calls": n_calls, + "n_user_turns": n_users, + "chars": chars, + "task": task[:400], + "language": lang, + "tools": tool_names(obj), + } diff --git a/traceforge/space/ingest.py b/traceforge/space/ingest.py new file mode 100644 index 0000000000000000000000000000000000000000..93f67e753251c419c4aa7e4f41ed0dae99e3c931 --- /dev/null +++ b/traceforge/space/ingest.py @@ -0,0 +1,175 @@ +"""Turn whatever a user uploads into a `Trace`. + +Three shapes arrive in practice: + + 1. traceforge native β€” `{"session_id", "messages": [...], "source", ...}` + 2. OpenAI chat JSONL β€” `{"messages": [...], "tools": [...]}` + 3. a bare message list β€” `[{...}, {...}]` + +The normaliser accepts all three and both tool-call schemas (flat +`{id,name,arguments}` and OpenAI `{type:"function",function:{...}}`), so a +user does not have to convert their export before submitting it. + +`parse_records` is the batch entry point: it reads a JSON file, a JSONL file +or a pasted blob and yields `(record, trace)` pairs. A malformed row is +reported, never silently dropped. +""" +from __future__ import annotations + +import json +import os +from typing import Any, Iterable, Iterator, Optional + +from traceforge.trace import Message, ToolCall, Trace + + +# ------------------------------------------------------------------ helpers + +def _content_text(content: Any) -> Optional[str]: + if content is None: + return None + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for p in content: + if isinstance(p, dict): + if p.get("type") in (None, "text", "input_text", "output_text"): + parts.append(str(p.get("text") or "")) + elif p.get("type") == "image_url": + parts.append("[image]") + else: + parts.append(str(p)) + return "\n".join(x for x in parts if x) + return str(content) + + +def _tool_call(tc: Any, idx: int) -> Optional[ToolCall]: + if not isinstance(tc, dict): + return None + fn_raw = tc.get("function") + fn: dict = fn_raw if isinstance(fn_raw, dict) else {} + name = (fn.get("name") or tc.get("name") or "").strip() + if not name: + return None + args = fn.get("arguments") if fn else tc.get("arguments") + if isinstance(args, (dict, list)): + args = json.dumps(args, ensure_ascii=False) + return ToolCall( + id=str(tc.get("id") or f"call_{idx}"), + name=name, + arguments=args if isinstance(args, str) and args.strip() else "{}", + ) + + +def message_from_dict(d: dict, idx: int = 0) -> Optional[Message]: + if not isinstance(d, dict): + return None + role = str(d.get("role") or "").strip().lower() + if role not in ("system", "user", "assistant", "tool"): + return None + calls = [c for c in (_tool_call(tc, i) + for i, tc in enumerate(d.get("tool_calls") or [])) if c] + return Message( + role=role, + content=_content_text(d.get("content")), + reasoning=d.get("reasoning") if isinstance(d.get("reasoning"), str) else None, + tool_calls=calls, + tool_call_id=(str(d["tool_call_id"]) if d.get("tool_call_id") is not None + else None), + tool_name=(str(d.get("tool_name") or d.get("name") or "") or None), + timestamp=(float(d["timestamp"]) if isinstance(d.get("timestamp"), (int, float)) + else None), + ) + + +def trace_from_record(rec: Any, *, fallback_id: str = "upload", + source: str = "space-upload") -> Trace: + """Build a Trace from a native record, an OpenAI row, or a message list.""" + if isinstance(rec, list): + meta: dict = {} + msgs_raw = rec + elif isinstance(rec, dict): + meta = rec + msgs_raw = rec.get("messages") or [] + else: + raise ValueError(f"unsupported record type: {type(rec).__name__}") + + msgs = [m for m in (message_from_dict(d, i) + for i, d in enumerate(msgs_raw)) if m] + if not msgs: + raise ValueError("record has no usable messages") + + sid = str(meta.get("session_id") or meta.get("id") or fallback_id) + return Trace( + session_id=sid, + model=meta.get("model"), + title=meta.get("title"), + started_at=(meta.get("started_at") + if isinstance(meta.get("started_at"), (int, float)) else None), + cwd=meta.get("cwd"), + messages=msgs, + source=str(meta.get("source") or source), + improved=bool(meta.get("improved", False)), + parent_session_id=meta.get("parent_session_id"), + task=meta.get("task"), + ) + + +# ------------------------------------------------------------------- input + +def _iter_jsonl(text: str) -> Iterator[Any]: + for line in text.splitlines(): + line = line.strip() + if not line: + continue + yield json.loads(line) + + +def parse_records(text: str) -> tuple[list[Any], list[str]]: + """Parse a pasted/uploaded blob into records. + + Accepts a JSON array, a single JSON object, or JSONL. Returns + `(records, errors)`; a row that fails to parse is reported by line. + """ + text = (text or "").strip() + if not text: + return [], ["empty input"] + # A single JSON document (object or array) is the common case. + try: + doc = json.loads(text) + if isinstance(doc, list): + # A bare message list is ONE trace, not N records. + if doc and all(isinstance(x, dict) and "role" in x for x in doc): + return [doc], [] + return doc, [] + return [doc], [] + except json.JSONDecodeError: + pass + records: list[Any] = [] + errors: list[str] = [] + for i, line in enumerate(text.splitlines(), 1): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError as e: + errors.append(f"line {i}: {e.msg}") + return records, errors + + +def read_upload(path: str) -> tuple[list[Any], list[str]]: + """Read an uploaded file (JSON or JSONL) into records.""" + if not path or not os.path.exists(path): + return [], [f"file not found: {path}"] + with open(path, "r", encoding="utf-8", errors="replace") as f: + return parse_records(f.read()) + + +def iter_traces(records: Iterable[Any], *, source: str = "space-upload" + ) -> Iterator[tuple[Any, Trace]]: + """Yield `(record, trace)`; a bad record raises with its index.""" + for i, rec in enumerate(records): + yield rec, trace_from_record(rec, fallback_id=f"upload-{i}", + source=source) diff --git a/traceforge/space/jobs.py b/traceforge/space/jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..9b213014490f078f102f0bfbd443805dcfc88f47 --- /dev/null +++ b/traceforge/space/jobs.py @@ -0,0 +1,118 @@ +"""Background worker: drain the store's queue without blocking the UI. + +A Space request must return in seconds, but one trace takes minutes. The +worker is a single daemon thread (configurable) that claims the oldest +queued hash, runs the pipeline, and records the outcome. It is safe to +start twice β€” the second call is a no-op β€” and it survives a Gradio +reload because the queue lives in the store, not in memory. +""" +from __future__ import annotations + +import os +import threading +import time +from typing import Optional, Protocol + +from traceforge.space.pipeline import SpacePipeline +from traceforge.space.store import TraceStore + + +class _Pipeline(Protocol): + """What the worker needs: a store and a process(hash) -> dict.""" + store: TraceStore + + def process(self, h: str) -> dict: ... + + +class Worker: + def __init__(self, pipeline: _Pipeline, *, poll_s: float = 2.0, + name: str = "traceforge-space-worker"): + self.pipeline = pipeline + self.store: TraceStore = pipeline.store + self.poll_s = poll_s + self.name = name + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + self._current: Optional[str] = None + self._last: Optional[dict] = None + self._started_at: Optional[float] = None + self._lock = threading.Lock() + + # ------------------------------------------------------------- control + + def start(self) -> "Worker": + with self._lock: + if self._thread and self._thread.is_alive(): + return self + self._stop.clear() + self._thread = threading.Thread(target=self._run, name=self.name, + daemon=True) + self._started_at = time.time() + self._thread.start() + return self + + def stop(self, timeout: float = 5.0) -> None: + self._stop.set() + t = self._thread + if t and t.is_alive(): + t.join(timeout=timeout) + + @property + def alive(self) -> bool: + return bool(self._thread and self._thread.is_alive()) + + def status(self) -> dict: + return { + "alive": self.alive, + "current": self._current, + "last": self._last, + "uptime_s": (round(time.time() - self._started_at, 1) + if self._started_at else 0.0), + "queued": self.store.queue_depth(), + } + + # ---------------------------------------------------------------- loop + + def _run(self) -> None: + while not self._stop.is_set(): + job = None + try: + job = self.store.claim_next() + except Exception: + job = None + if not job: + self._stop.wait(self.poll_s) + continue + h = job["hash"] + self._current = h + try: + self._last = self.pipeline.process(h) + except Exception as e: # process() is meant not to raise; belt+braces + self._last = self.store.update( + h, status="failed", error=f"worker crashed: {e}") + finally: + self._current = None + + +# --------------------------------------------------------------- singleton + +_WORKER: Optional[Worker] = None +_WORKER_LOCK = threading.Lock() + + +def get_worker(pipeline: SpacePipeline, *, autostart: bool = True) -> Worker: + """One worker per process, started on first use.""" + global _WORKER + with _WORKER_LOCK: + if _WORKER is None: + _WORKER = Worker(pipeline) + if autostart and not _WORKER.alive: + _WORKER.start() + return _WORKER + + +def worker_count() -> int: + try: + return max(1, int(os.environ.get("TRACEFORGE_SPACE_WORKERS", "1"))) + except ValueError: + return 1 diff --git a/traceforge/space/pipeline.py b/traceforge/space/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..ebfb2e5bc837225e33a923da57055a560cd31c2c --- /dev/null +++ b/traceforge/space/pipeline.py @@ -0,0 +1,239 @@ +"""The pipeline behind the web UI. + +`SpacePipeline` is the only thing the Gradio app talks to. It wraps the +windowed splice engine (`improve_trace_splice`) so a trace that arrives as +JSON β€” not as a session id in a local store β€” can be processed: + + submit(record) hash -> dedupe -> enqueue (idempotent) + process(hash) run triage + rewrite + gates, store the output + +Configuration is environment-only, because a Space has no `~/.hermes`: + + TRACEFORGE_BASE_URL OpenAI-compatible endpoint (…/v1) + TRACEFORGE_LLM_KEY bearer token + TRACEFORGE_MODEL writer/judge model id + TRACEFORGE_TRIAGE_MODEL cheap labeler (default: TRACEFORGE_MODEL) + TRACEFORGE_JUDGE_MODEL judge override (default: TRACEFORGE_MODEL) + TRACEFORGE_DATA_DIR store root (default: /data/space) + +`split_long` is deliberately OFF: splitting one submitted seed into N +segments would produce N outputs for one hash, which is exactly what the +one-seed-one-output rule forbids. The splice engine already windows a long +trace internally. +""" +from __future__ import annotations + +import os +import shutil +import time +from typing import Any, Optional + +from traceforge.config import Config, repo_root_from_pkg +from traceforge.space.hashing import fingerprint, output_hash, seed_hash +from traceforge.space.ingest import trace_from_record +from traceforge.space.store import (DONE, FAILED, QUEUED, REJECTED, RUNNING, + TraceStore) +from traceforge.trace import Trace + + +# ------------------------------------------------------------------- config + +def _env_int(name: str, default: int) -> int: + try: + return int(os.environ.get(name, "") or default) + except ValueError: + return default + + +def space_config(data_dir: Optional[str] = None, **overrides: Any) -> Config: + """A Config that needs no Hermes profile β€” env vars only.""" + root = repo_root_from_pkg() + cfg = Config() + cfg.data_dir = (data_dir or os.environ.get("TRACEFORGE_DATA_DIR") + or os.path.join(root, "data", "space")) + cfg.guidelines_path = os.path.join(root, "guidelines", + "IDEAL_TRACE_GUIDELINES.md") + cfg.provider = os.environ.get("TRACEFORGE_PROVIDER") or None + cfg.model = os.environ.get("TRACEFORGE_MODEL") or "qwen-fable-selfhosted" + cfg.triage_model = (os.environ.get("TRACEFORGE_TRIAGE_MODEL") + or cfg.model) + cfg.judge_model = os.environ.get("TRACEFORGE_JUDGE_MODEL") or "" + cfg.triage_base_url = (os.environ.get("TRACEFORGE_BASE_URL") or "").rstrip("/") + cfg.triage = True + cfg.splice = True + # The Space processes one submitted trace as one unit. + cfg.split_long = False + cfg.raw_cache = False + cfg.verify = False + cfg.review = False + cfg.splice_workers = _env_int("TRACEFORGE_SPLICE_WORKERS", 4) + cfg.triage_workers = _env_int("TRACEFORGE_TRIAGE_WORKERS", 4) + cfg.rewrite_timeout_s = _env_int("TRACEFORGE_REWRITE_TIMEOUT_S", 300) + cfg.judge_timeout_s = _env_int("TRACEFORGE_JUDGE_TIMEOUT_S", 300) + for k, v in overrides.items(): + setattr(cfg, k, v) + return cfg.resolve(root) + + +def pipeline_ready(cfg: Config) -> tuple[bool, str]: + """(ready, reason). A Space without an endpoint must say so, not hang.""" + try: + from traceforge.triage import _resolve_endpoint + base, _key, model = _resolve_endpoint(cfg) + except Exception as e: # RuntimeError from the resolver, or a bad config + return False, f"no LLM endpoint: {e}" + if not base: + return False, "no LLM endpoint configured (TRACEFORGE_BASE_URL)" + if not model: + return False, "no model configured (TRACEFORGE_MODEL)" + return True, f"{model} @ {base}" + + +# ----------------------------------------------------------------- pipeline + +class SpacePipeline: + """Submit / process / report, with the store as the only state.""" + + def __init__(self, store: TraceStore, cfg: Optional[Config] = None, + *, max_attempts: int = 2): + self.store = store + self.cfg = cfg or space_config() + self.max_attempts = max_attempts + + # ------------------------------------------------------------- submit + + def submit(self, record: Any, *, sponsor: str = "", + source: str = "space-upload") -> dict: + return self.store.submit(record, sponsor=sponsor, source=source) + + def submit_many(self, records: list[Any], *, sponsor: str = "", + source: str = "space-sponsor") -> list[dict]: + out = [] + for rec in records: + try: + out.append(self.submit(rec, sponsor=sponsor, source=source)) + except Exception as e: + out.append({"hash": "", "status": "error", + "error": str(e), "duplicate": False}) + return out + + # ------------------------------------------------------------ process + + def process(self, h: str) -> dict: + """Run the pipeline for one queued hash. Never raises.""" + rec = self.store.get(h) + if not rec: + return {"hash": h, "status": FAILED, "error": "unknown hash"} + if rec.get("status") == DONE: + return {**rec, "skipped": True} + + sid = f"space-{h[:16]}" + started = time.time() + self.store.update(h, status=RUNNING, started_at=started, + attempts=int(rec.get("attempts") or 0) + 1) + try: + record = self.store.load_seed(h) + trace = trace_from_record(record, fallback_id=sid, + source="space-upload") + trace.session_id = sid + trace.source = "space-upload" + trace.improved = False + trace.parent_session_id = None + except Exception as e: + return self._fail(h, [f"unparseable seed: {e}"], started) + + # A fresh attempt must not regate a stale artifact from a previous + # run (the splice engine's stage-0 fast path would promote it as-is). + workdir = os.path.join(self.cfg.improver_workdir, sid) + shutil.rmtree(workdir, ignore_errors=True) + + try: + from traceforge.splice import improve_trace_splice + res = improve_trace_splice( + self.cfg, trace, max_attempts=self.max_attempts, + ) + except Exception as e: + return self._fail(h, [f"pipeline crashed: {e}"], started) + + if not res.get("ok") or res.get("improved") is None: + return self._fail(h, list(res.get("errors") or ["no output"]), + started, route=res.get("route"), + attempts=res.get("attempts")) + + improved: Trace = res["improved"] + improved.session_id = sid + improved.parent_session_id = sid + improved.improved = True + improved.source = "traceforge-space" + + # The splice engine gates before returning, but a structural check + # here is cheap and makes the Space's own contract explicit. + try: + from traceforge.validate import validate_structural + structural = validate_structural(improved) + except Exception as e: + structural = [f"validator crashed: {e}"] + if structural: + return self._fail(h, structural, started, route=res.get("route"), + attempts=res.get("attempts")) + + # Task bucket: heuristic only β€” the Space must not spend an LLM call + # on a label the dashboard can derive for free. + try: + from traceforge.classify import classify_heuristic + improved.task = classify_heuristic(improved).get("task") + except Exception: + pass + + # Time sensitivity: measured on the RAW record (it may carry the + # `tools` array, which is what decides anchored vs sensitive) and on + # the ideal rewrite (did it actually call the retrieval tool?). + ts_raw = self._time_sensitivity(record) + ts_out = self._time_sensitivity(improved) + + self.store.put_output(h, improved) + duration = time.time() - started + row = self.store.update( + h, status=DONE, finished_at=time.time(), duration_s=round(duration, 1), + output_hash=output_hash(improved), + route=res.get("route") or "", + attempts=res.get("attempts") or 0, + n_messages_out=len(improved.messages), + task=improved.task or rec.get("task") or "", + time_sensitivity=ts_raw, + time_sensitivity_improved=ts_out, + error="", + ) + return row + + def _fail(self, h: str, errors: list[str], started: float, *, + route: Optional[str] = None, attempts: Any = None) -> dict: + msg = "; ".join(str(e) for e in errors)[:1200] + return self.store.update( + h, status=FAILED, finished_at=time.time(), + duration_s=round(time.time() - started, 1), + route=route or "", attempts=attempts or 0, error=msg, + ) + + @staticmethod + def _time_sensitivity(obj: Any) -> dict: + try: + from traceforge.timesensitivity import detect + d = detect(obj) + return {k: d[k] for k in ("class", "label", "temporal_facts", + "retrieval_available", "retrieval_called", + "needs_rewrite", "score")} + except Exception as e: + return {"class": "", "label": f"unavailable: {e}"} + + # -------------------------------------------------------------- report + + def result(self, h: str) -> dict: + """Everything the UI needs about one hash.""" + rec = self.store.get(h) or {"hash": h, "status": "unknown"} + out = self.store.load_output(h) + rec = dict(rec) + rec["has_output"] = out is not None + if out is not None: + rec["output_messages"] = len(out.messages) + return rec diff --git a/traceforge/space/stats.py b/traceforge/space/stats.py new file mode 100644 index 0000000000000000000000000000000000000000..7156e4b5acfdc76758fb80fc21c5fb5c05482dec --- /dev/null +++ b/traceforge/space/stats.py @@ -0,0 +1,149 @@ +"""Dashboard data: the Space's own corpus plus the reference corpus. + +Two independent sources, because they answer different questions: + + * the **store** β€” what this Space has processed (seeds, sponsors, queue, + time-sensitivity mix of the traces people submitted); + * the **reference corpus** β€” `data/dataset/dataset_card.json` when the + repo is deployed with its export, plus a cached time-sensitivity census + of `train.jsonl` (the card predates the metric). + +The census is cached by (mtime, size): streaming a 400 MB export on every +dashboard refresh would make the page unusable, and recomputing it when the +file has not changed is pure waste. +""" +from __future__ import annotations + +import json +import os +from typing import Any, Optional + +from traceforge.space.store import TraceStore + + +# ------------------------------------------------------------------ helpers + +def _read_json(path: str) -> Optional[dict]: + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + return None + + +def _write_json(path: str, obj: Any) -> None: + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(obj, f, ensure_ascii=False, indent=2) + os.replace(tmp, path) + + +# ------------------------------------------------------- time-sensitivity + +def census_dataset(dataset_path: str, cache_path: str, + *, force: bool = False) -> dict: + """Time-sensitivity census of a JSONL export, cached by file identity.""" + if not os.path.exists(dataset_path): + # A Space ships the census, not the 400 MB export. Without this the + # dashboard would report "no reference" on the one deployment that + # needs the precomputed number most. + cached = _read_json(cache_path) + if cached and cached.get("census"): + out = dict(cached["census"]) + out["available"] = True + out["bundled"] = True + return out + return {"available": False, "reason": f"not found: {dataset_path}"} + st = os.stat(dataset_path) + key = {"mtime": int(st.st_mtime), "size": st.st_size} + if not force: + cached = _read_json(cache_path) + if cached and cached.get("key") == key: + return cached.get("census") or {} + + from traceforge.timesensitivity import CLASSES, CLASS_LABELS, detect + + counts = {c: 0 for c in CLASSES} + rewrite = 0 + n = 0 + with open(dataset_path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + d = detect(rec) + counts[d["class"]] = counts.get(d["class"], 0) + 1 + if d.get("needs_rewrite"): + rewrite += 1 + n += 1 + census = { + "available": True, + "total": n, + "classes": counts, + "labels": CLASS_LABELS, + "needs_rewrite": rewrite, + "time_sensitive_share": round(counts.get("time_sensitive", 0) / n, 4) + if n else 0.0, + "source": os.path.basename(dataset_path), + } + _write_json(cache_path, {"key": key, "census": census}) + return census + + +# ---------------------------------------------------------------- reference + +def reference_stats(dataset_dir: str, cache_path: str) -> dict: + """The published corpus as the dashboard shows it.""" + card = _read_json(os.path.join(dataset_dir, "dataset_card.json")) or {} + census = census_dataset(os.path.join(dataset_dir, "train.jsonl"), + cache_path) + out: dict[str, Any] = {"available": bool(card), "card": card, + "time_sensitivity": census} + if card: + out["traces"] = card.get("traces", 0) + out["languages"] = card.get("languages", {}) + out["tasks"] = card.get("tasks", {}) + out["difficulty"] = (card.get("difficulty") or {}).get("bands", {}) + out["tool_calls"] = card.get("tool_calls", {}) + out["messages"] = card.get("messages", {}) + out["tokens"] = card.get("tokens", {}) + out["provenance"] = card.get("provenance", {}) + return out + + +# ---------------------------------------------------------------- dashboard + +def dashboard(store: TraceStore, *, dataset_dir: str = "", + cache_path: str = "") -> dict: + """One dict for the whole dashboard tab.""" + space = store.stats() + ref: dict = {"available": False} + if dataset_dir: + ref = reference_stats( + dataset_dir, + cache_path or os.path.join(store.root, "reference_census.json"), + ) + return { + "space": space, + "reference": ref, + "recent": store.recent(25), + "queue": store.by_status("queued")[:25], + } + + +# ------------------------------------------------------------------ render + +def pct(part: float, whole: float) -> float: + return round(100.0 * part / whole, 1) if whole else 0.0 + + +def bar_rows(counts: dict, *, total: Optional[int] = None) -> list[list]: + """`{label: n}` -> [[label, n, pct]] sorted by n desc.""" + total = total if total is not None else sum(counts.values()) + return [[k, v, pct(v, total)] for k, v in + sorted(counts.items(), key=lambda kv: -kv[1])] diff --git a/traceforge/space/store.py b/traceforge/space/store.py new file mode 100644 index 0000000000000000000000000000000000000000..fbd8bb4cf74696eee23feb4df3f052e75dc4fbeb --- /dev/null +++ b/traceforge/space/store.py @@ -0,0 +1,306 @@ +"""Persistent, content-addressed store for the Space. + +Layout under `root` (default `data/space`): + + index.jsonl append-only journal, one row per state change + seeds/.json the seed exactly as submitted (keeps `tools`) + outputs/.json the improved trace + meta.json store version + +The journal is the source of truth: on load the LAST row per hash wins, so +a crash mid-write loses at most the row being written, never the store. +Compaction rewrites the journal when it grows past 4x the live rows. + +The invariant the Space promises: **one seed hash -> at most one output**. +`submit()` is idempotent β€” a second submission of the same content returns +the existing record instead of enqueueing a second job. +""" +from __future__ import annotations + +import json +import os +import tempfile +import threading +import time +from typing import Any, Optional + +from traceforge.space.hashing import fingerprint, seed_hash +from traceforge.trace import Trace + +STORE_VERSION = 1 + +QUEUED = "queued" +RUNNING = "running" +DONE = "done" +FAILED = "failed" +REJECTED = "rejected" + +TERMINAL = (DONE, FAILED, REJECTED) + + +def _now() -> float: + return time.time() + + +def _atomic_write_json(path: str, obj: Any) -> None: + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(path)), + prefix=".tmp-", suffix=".json") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(obj, f, ensure_ascii=False, indent=2) + os.replace(tmp, path) + except BaseException: + if os.path.exists(tmp): + os.unlink(tmp) + raise + + +class TraceStore: + """Thread-safe index + artifact store.""" + + def __init__(self, root: str, *, max_queue: int = 500): + self.root = os.path.abspath(root) + self.seed_dir = os.path.join(self.root, "seeds") + self.output_dir = os.path.join(self.root, "outputs") + self.index_path = os.path.join(self.root, "index.jsonl") + self.meta_path = os.path.join(self.root, "meta.json") + self.max_queue = max_queue + self._lock = threading.RLock() + self._index: dict[str, dict] = {} + self._lines = 0 + os.makedirs(self.seed_dir, exist_ok=True) + os.makedirs(self.output_dir, exist_ok=True) + self._load() + + # ------------------------------------------------------------- loading + + def _load(self) -> None: + if not os.path.exists(self.meta_path): + _atomic_write_json(self.meta_path, { + "version": STORE_VERSION, "created_at": _now(), + }) + if not os.path.exists(self.index_path): + return + with open(self.index_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + self._lines += 1 + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + h = row.get("hash") + if not h: + continue + prev = self._index.get(h) or {} + prev.update(row) + self._index[h] = prev + # A row left `running` by a crash is not running any more. + stale = [h for h, r in self._index.items() if r.get("status") == RUNNING] + for h in stale: + self._append({"hash": h, "status": QUEUED, "updated_at": _now(), + "note": "requeued after restart"}) + if self._lines > 4 * len(self._index) + 200: + self._compact() + + def _compact(self) -> None: + with self._lock: + tmp = self.index_path + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + for row in self._index.values(): + f.write(json.dumps(row, ensure_ascii=False) + "\n") + os.replace(tmp, self.index_path) + self._lines = len(self._index) + + # ------------------------------------------------------------- writing + + def _append(self, row: dict) -> None: + """Append one journal row and fold it into the in-memory index.""" + with self._lock: + h = row["hash"] + prev = self._index.get(h) or {} + prev.update(row) + self._index[h] = prev + os.makedirs(self.root, exist_ok=True) + with open(self.index_path, "a", encoding="utf-8") as f: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + self._lines += 1 + + def update(self, h: str, **fields: Any) -> dict: + row = {"hash": h, "updated_at": _now()} + row.update(fields) + self._append(row) + return self._index[h] + + # -------------------------------------------------------------- reading + + def get(self, h: str) -> Optional[dict]: + with self._lock: + rec = self._index.get(h) + return dict(rec) if rec else None + + def all(self) -> list[dict]: + with self._lock: + return [dict(r) for r in self._index.values()] + + def by_status(self, status: str) -> list[dict]: + return [r for r in self.all() if r.get("status") == status] + + def queue_depth(self) -> int: + return len(self.by_status(QUEUED)) + + def recent(self, n: int = 20) -> list[dict]: + rows = [r for r in self.all() if r.get("status") in TERMINAL] + rows.sort(key=lambda r: r.get("updated_at") or 0, reverse=True) + return rows[:n] + + # ------------------------------------------------------------- artifacts + + def seed_path(self, h: str) -> str: + return os.path.join(self.seed_dir, f"{h}.json") + + def output_path(self, h: str) -> str: + return os.path.join(self.output_dir, f"{h}.json") + + def put_seed(self, h: str, record: Any) -> None: + _atomic_write_json(self.seed_path(h), record) + + def load_seed(self, h: str) -> Any: + with open(self.seed_path(h), "r", encoding="utf-8") as f: + return json.load(f) + + def put_output(self, h: str, trace: Trace) -> None: + _atomic_write_json(self.output_path(h), trace.to_dict()) + + def load_output(self, h: str) -> Optional[Trace]: + path = self.output_path(h) + if not os.path.exists(path): + return None + return Trace.load(path) + + # --------------------------------------------------------------- submit + + def submit(self, record: Any, *, sponsor: str = "", + source: str = "space-upload") -> dict: + """Idempotent submission. Returns the record plus a `duplicate` flag. + + A seed that is already queued/running/done is NOT enqueued again β€” + that is the one-seed-one-output guarantee. A previously FAILED seed + is re-queued (a failure is a pipeline outcome, not a new seed). + """ + fp = fingerprint(record) + h = fp["hash"] + with self._lock: + rec = self._index.get(h) + if rec and rec.get("status") in (QUEUED, RUNNING, DONE): + return {**rec, "duplicate": True} + if rec and rec.get("status") == REJECTED: + return {**rec, "duplicate": True} + if not rec and self.queue_depth() >= self.max_queue: + raise RuntimeError( + f"queue is full ({self.max_queue}); try again later") + self.put_seed(h, record) + prev_sponsor = (rec or {}).get("sponsor") or "" + row = { + "hash": h, + "status": QUEUED, + "sponsor": (sponsor or prev_sponsor).strip(), + "source": source, + "submitted_at": (rec or {}).get("submitted_at") or _now(), + "updated_at": _now(), + "n_messages": fp["n_messages"], + "n_tool_calls": fp["n_tool_calls"], + "n_user_turns": fp["n_user_turns"], + "chars": fp["chars"], + "task": fp["task"], + "language": fp["language"], + "tools": fp["tools"], + "attempts": 0, + } + self._append(row) + return {**row, "duplicate": False} + + def claim_next(self) -> Optional[dict]: + """Oldest queued seed -> running. None when the queue is empty.""" + with self._lock: + queued = [r for r in self._index.values() + if r.get("status") == QUEUED] + if not queued: + return None + queued.sort(key=lambda r: r.get("submitted_at") or 0) + h = queued[0]["hash"] + self._append({"hash": h, "status": RUNNING, + "started_at": _now(), "updated_at": _now()}) + return dict(self._index[h]) + + # ---------------------------------------------------------------- stats + + def stats(self) -> dict: + rows = self.all() + by_status: dict[str, int] = {} + sponsors: dict[str, dict] = {} + ts: dict[str, int] = {} + tasks: dict[str, int] = {} + langs: dict[str, int] = {} + durations: list[float] = [] + calls: list[int] = [] + for r in rows: + st = r.get("status") or "unknown" + by_status[st] = by_status.get(st, 0) + 1 + sp = (r.get("sponsor") or "").strip() + if sp: + s = sponsors.setdefault(sp, {"sponsor": sp, "seeds": 0, + "done": 0, "failed": 0}) + s["seeds"] += 1 + if st == DONE: + s["done"] += 1 + elif st in (FAILED, REJECTED): + s["failed"] += 1 + cls = (r.get("time_sensitivity") or {}).get("class") + if cls: + ts[cls] = ts.get(cls, 0) + 1 + if r.get("task"): + tasks[r["task"]] = tasks.get(r["task"], 0) + 1 + if r.get("language"): + langs[r["language"]] = langs.get(r["language"], 0) + 1 + if isinstance(r.get("duration_s"), (int, float)): + durations.append(float(r["duration_s"])) + if isinstance(r.get("n_tool_calls"), int): + calls.append(r["n_tool_calls"]) + durations.sort() + calls.sort() + + def _pct(vals: list, q: float) -> float: + if not vals: + return 0.0 + i = min(len(vals) - 1, int(q * (len(vals) - 1))) + return round(vals[i], 1) + + return { + "seeds": len(rows), + "by_status": by_status, + "done": by_status.get(DONE, 0), + "failed": by_status.get(FAILED, 0) + by_status.get(REJECTED, 0), + "queued": by_status.get(QUEUED, 0), + "running": by_status.get(RUNNING, 0), + "sponsors": sorted(sponsors.values(), + key=lambda s: (-s["seeds"], s["sponsor"])), + "time_sensitivity": ts, + "tasks": dict(sorted(tasks.items(), key=lambda kv: -kv[1])), + "languages": langs, + "duration_s": { + "n": len(durations), + "median": _pct(durations, 0.5), + "p90": _pct(durations, 0.9), + "max": round(durations[-1], 1) if durations else 0.0, + }, + "tool_calls": { + "n": len(calls), + "median": _pct(calls, 0.5), + "p90": _pct(calls, 0.9), + "max": calls[-1] if calls else 0, + }, + } diff --git a/traceforge/splice.py b/traceforge/splice.py new file mode 100644 index 0000000000000000000000000000000000000000..2128ae430fe94749cfc93fa3b5ac4193c30d1932 --- /dev/null +++ b/traceforge/splice.py @@ -0,0 +1,3968 @@ +"""Windowed direct-HTTP improve engine β€” no agent runtime, stdlib only. + +Replaces the hermes one-shot writer/verifier/reviewer pipeline for the +default path. Design (owner directives 2026-10-04): + +- NEVER send a whole trace to the LLM. Every stage works on sliding + message windows (reuses triage's windowing), merged mechanically. +- The LLM emits ONLY assistant prose / tool-arg fixes / verdicts. User + messages and tool results are spliced byte-verbatim from the raw trace + by absolute index β€” P4/P5 hold by construction, clip markers cannot + enter, grounding failures are structurally impossible for spliced + content. +- Triage labels (cached, windowed, cheap model) drive the plan: keep / + drop cost ZERO calls; only rephrase/summarize windows pay a rewrite. +- Admission = windowed judge (pillars + intent per window, merged + mechanically). No web verifier: external claims are already covered + by the grounding gate (facts must exist in the source trace). +- INTENT-only mismatch with a good answer triggers a task-REFRAME + repair (one tiny call rewriting the task message) instead of a whole + trace rewrite. + +All calls go straight to the OpenAI-compatible endpoint (urllib, +structured-outputs ladder like triage) β€” no subprocess, no state.db +sessions, no self-reference-guard concerns, no turn caps. + +Index spaces (kept explicit β€” most bugs here would be index confusion): +- SOURCE indices: positions in the raw trace (triage labels, plan). +- IDEAL indices: positions in the spliced candidate (judge citations, + structural/grounding errors). src_map[ideal_idx] = source_idx; it is + recomputed after every splice and threading through merges. +""" +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import re +import threading +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Optional + +from traceforge.config import Config +from traceforge.trace import Message, ToolCall, Trace, json_loads_lenient +from traceforge.identity import validate_identity +from traceforge.validate import ( + validate_structural, _confident_lang, _norm_for_similarity, + substantive_user_turns, +) + +try: # clip probe lives with the harness gate constants + from traceforge.improver.harness import CLIP_MARKER_PROBE +except ImportError: # pragma: no cover + from traceforge.grounding import CLIP_MARKER_PROBE # type: ignore + + +# ------------------------------------------------------- usage accounting +# Direct-HTTP calls never appear in state.db, so the live progress bar +# (LiveStats, which polls child-session counters) would show 0 tok/s in +# splice mode. _post_chat feeds this thread-safe accumulator on every +# response; LiveStats.poll() merges it, giving splice runs the same +# tok/s Β· tokens Β· model telemetry the agent path has. + +class _SpliceUsage: + """Thread-safe token/model accounting for direct-HTTP calls. + + Per-trace attribution: workers call set_current(sid) around a + trace; every _post_chat/_post_json response in that thread is + bucketed under the sid. LiveStats.poll() drains the buckets and + merges them into the progress-bar telemetry (tok/s, tokens, model). + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._tls = threading.local() + self.calls = 0 + self.in_tokens = 0 + self.out_tokens = 0 + self.models: set[str] = set() + # sid -> {"in","out","calls","models","last_out"} + self._buckets: dict[str, dict] = {} + + def set_current(self, sid: Optional[str]) -> None: + """Attribute subsequent calls in THIS thread to sid.""" + self._tls.sid = sid + + def add(self, model: str, usage: Optional[dict]) -> None: + u = usage if isinstance(usage, dict) else {} + tin = int(u.get("prompt_tokens") or 0) + tout = int(u.get("completion_tokens") or 0) + sid = getattr(self._tls, "sid", None) or "-" + with self._lock: + self.calls += 1 + self.in_tokens += tin + self.out_tokens += tout + if model: + self.models.add(str(model)) + b = self._buckets.setdefault( + sid, {"in": 0, "out": 0, "calls": 0, + "models": set(), "last_out": 0} + ) + b["in"] += tin + b["out"] += tout + b["calls"] += 1 + if model: + b["models"].add(str(model)) + + def drain(self, dt: Optional[float]) -> tuple[dict, int]: + """Snapshot per-trace buckets + global out-token delta. + + Rate per bucket = (out - last_out) / dt, mirroring the state.db + poller's per-session delta. Returns ({sid: {...}}, delta_out); + bucket rate fields are instantaneous, for the LiveStats merge. + """ + with self._lock: + per: dict[str, dict] = {} + delta_out = 0 + for sid, b in self._buckets.items(): + grew = b["out"] - b["last_out"] + b["last_out"] = b["out"] + delta_out += grew + per[sid] = { + "in": b["in"], "out": b["out"], "calls": b["calls"], + "models": sorted(b["models"]), + "rate": round(grew / dt, 2) if (grew and dt) else 0.0, + } + return per, delta_out + + def totals(self) -> tuple[int, int, int, list[str]]: + with self._lock: + return (self.calls, self.in_tokens, self.out_tokens, + sorted(self.models)) + + +SPLICE_USAGE = _SpliceUsage() + + +def _submit_attributed(ex, fn, *args, **kwargs): + """Submit fn to a pool, carrying this thread's splice-usage + attribution (sid) into the pool thread β€” pool threads don't + inherit thread-locals.""" + sid = getattr(SPLICE_USAGE._tls, "sid", None) + + def _wrap(): + SPLICE_USAGE.set_current(sid) + try: + return fn(*args, **kwargs) + finally: + SPLICE_USAGE.set_current(None) + + return ex.submit(_wrap) + + +# ---------------------------------------------------------------- prompts + +PILLARS = """- P1 Efficient but deep reasoning: high-signal decision logic per turn + (why this call, what the result implies, what to verify next); no filler, + no restating the obvious. FOCUS: answer exactly what was asked β€” for a + follow-up question, deliver the new piece only; never re-present facts + the reader already got in an earlier turn of the same session. +- P2 No committed mistakes: the ideal path takes the correct action the + first time; no failed calls, retry loops, wrong turns, or visible + repairs. Negative signal from the real trace (failures, user + corrections) is routed AROUND, not reproduced. +- P3 Perfect tool calls: every call necessary and correctly parameterized; + independent calls batched in parallel; no exploratory calls whose + results the narrative never uses. +- P4 Fact-driven: every factual statement is grounded in the trace's own + tool evidence; no invented files, commands, paths, or results; no fake + success. External-world claims (versions, API behavior, dates) only + when the trace's evidence proves them. +- P5 One language per trace (the first user message's language), perfect + assistant grammar. The user's task message is verbatim. +- P6 No refusals and no identity: the task is completed, and the + assistant never claims a model identity ("I am Claude", "as an AI") + β€” the trace reads as a neutral, capable agent.""" + +REWRITE_HEADER = """You are rewriting ONE WINDOW of a real agent trace into its +ideal-world version β€” what a flawless execution of this task would have +looked like. You see window messages {lo}-{hi} (of {n}) with ABSOLUTE +indices into the full session. + +QUALITY PILLARS (binding): +{pillars} + +Window context: +- ORIGINAL TASK (the user's opening request, verbatim): +{task} + +What you may edit in THIS window: +- assistant messages: rewrite content (reasoning + prose) β€” or drop the + message entirely when it is dead weight (failed call, retry, flailing, + exploration the ideal path routes around). +- individual tool calls on KEPT assistant messages: list their ids in + "drop_calls" to remove just that call (its result row drops + automatically) β€” use this for multi-call turns where SOME calls failed + or were useless but the turn itself stays. +- tool-call arguments on kept assistant messages: fix via args_fixes + when the call was right but parameterized wrong. +- tool RESULT rows: ONLY compress β€” every substantive line you keep must + be copied verbatim from the row; never invent or paraphrase content. + +What you must NEVER change: +- the trace's language (first user message's language). +- user and system message CONTENT: never rewrite it. Only PLUMBING user + rows may be dropped β€” validator rejections ("previous response was + rejected"), retry nudges, compaction/continuation markers, harness + scaffolding. A SUBSTANTIVE follow-up user turn (a new request, a + correction of the work, an added constraint, a question) is part of + the conversation: keep it verbatim and keep the turn it opens. This + trace may be multi-turn; flattening it into one task destroys the + training signal and fails the multi-turn gate. +- tool-call ids, names, result pairing: the harness owns them. + +Edit rules: +- Keep anything already ideal: unnecessary rewrites are violations. +- FOCUS: when an answer or turn gives more than its question asked + for β€” unrequested context, or facts already delivered in an earlier + turn of the session β€” trim it to exactly what was asked. +- Dropping an assistant message drops its tool calls and their results + automatically β€” do not separately drop the result rows. +- Compress result rows only when genuinely repetitive; quote surviving + lines byte-verbatim. +- CONNECTOR ACCURACY: prose that leads into a tool call ("Let me read + the recipes section around line 1100-1750") must describe the call + EXACTLY as its arguments state it β€” same file, same line range, same + pattern. Never invent or round numbers in connector prose; when + unsure, write a generic lead-in ("Reading the recipes module") + instead of a specific one. +- Ignore rendering clip markers ("…[+N chars clipped]") β€” never copy + them into rewritten content. +- NEVER write first-person identity claims ("I am Claude", "as an AI", + "trained by …") or vendor self-references, and NEVER write refusal + phrasing ("I can't help with that", "against my guidelines"). + The trained model must learn neither an identity nor refusals. + Mentioning a vendor/product as TASK CONTENT (e.g. "the repo has a + CLAUDE.md") is fine β€” self-claims are not. +- Harness self-talk NEVER survives into ideal assistant prose: no + "context was compacted", no "spillover slices", no references to + context windows, compaction, or session plumbing. A flawless agent + does not narrate its own context management. If the raw turn does, + rewrite the prose to say the same thing without the plumbing β€” or + drop the turn when it says nothing else. +- Tool results tagged [ERROR RESULT] are FAILED calls the ideal path + must route around (pillar 2): DROP the assistant turn that issued the + failed call (its result row drops automatically). Never keep a + failed call + traceback in an ideal trace. Only keep an error result + when the NEXT assistant turn legitimately used it as information + (e.g. probing behavior) β€” then the follow-up turn must state what was + learned, not that the call failed. + +Reply with ONLY the JSON object: {{"edits": [{{"i": , +"action": "keep"|"drop"|"rewrite", "content": "", "args_fixes": [{{"call_id": "...", "args": }}]}}]}} β€” +one entry per message index shown below. +""" + +REPAIR_PLAN_HEADER = """You are the repair PLANNER for a synthetic ideal-world agent +trace that FAILED quality review. The cited findings span MULTIPLE +windows β€” a single defect often has parts in different places (the +failed call, its paired result, the follow-up turns that repair it). + +Cited findings: +{violations} + +ORIGINAL TASK: +{task} + +QUALITY PILLARS (for reference): +{pillars} + +DIGEST of every cited source row (absolute indices): +{digest} + +Emit ONE coordinated edit set across ALL cited rows at once. Rules: +- WRITEβ†’FIX CHAINS: when a write created a file with a defect and a + LATER patch fixes it (same trace), do NOT rewrite prose around it β€” + either args_fix the original write so the defect never existed + (result row rewritten to the fixed output) AND drop the patch turns, + or drop the buggy write + patch sequence entirely if a clean + rewrite exists later. Keeping both the buggy write and its visible + patch violates P2 (committed mistake + visible repair). +- Prefer dropping non-essential turns (exploratory, redundant, failed, + retry) over rewriting them; a call and its result row must be + handled as a unit β€” args_fixes on the call REQUIRES rewriting its + result to the output the fixed call returns, or dropping both. +- drop_calls surgically removes single failed calls inside multi-call + turns. +- Rewrite prose only for turns that must stay but say the wrong thing. +- Rewritten connector prose must mirror the call's ACTUAL arguments + (file, line range, pattern) β€” never invent or round specifics. +- user/system rows are never edited. +- Never introduce identity claims, refusals, or clip markers. + +Reply with ONLY the JSON object: {{"edits": [{{"i": , "action": "keep"|"drop"|"rewrite", "content": "...", +"drop_calls": ["", …], "args_fixes": [{{"call_id": "...", +"args": }}]}}]}} for the cited indices. +""" + +REPAIR_HEADER = """You are repairing ONE WINDOW of an ideal-world agent trace that +FAILED quality review. Fix ONLY the cited findings; keep everything else. + +Cited findings (indices are absolute window indices): +{violations} + +Window context: +- ORIGINAL TASK: +{task} + +QUALITY PILLARS (for reference): +{pillars} + +Rules: rewrite only the cited indices; DROP a cited row when the +finding says it should not exist (including correction/rejection user +rows β€” drop them, never rewrite); user/system content is never edited; +tool results may only be compressed with verbatim-quoted lines; match +the trace's language. + +DROP-DOMINANCE for non-essential turns: when a finding calls a turn +exploratory, redundant, useless, a failed call, or a visible repair/ +retry, the correct fix is usually action="drop" on that row (and its +result rows) β€” NOT a prose rewrite that keeps the turn. Ask: would a +flawless first-run execution have made this call at all? If no, drop +it. Rewrite prose only for turns that MUST stay but say the wrong +thing. + +When rewriting prose: never introduce first-person identity claims +("I am Claude", "as an AI", "trained by …") or refusal phrasing ("I +can't help with that", "against my guidelines") β€” the training +data must teach neither identity nor refusals. When the finding is +over-answering (unrequested padding, facts repeated from earlier +turns), the fix is trimming the prose to exactly what the question +asked β€” not rewording it at the same length. + +For a cited FAILED or useless call inside a multi-call turn, prefer +"drop_calls": ["", …] on that turn over dropping the whole +message β€” the turn's good calls survive. + +CALL+RESULT COHERENCE: an args_fix on a call changes what that call +asked for β€” the call's RESULT row (the next tool message answering it) +must then be rewritten to the output the FIXED call would return, or +dropped together with the call. A fixed path with the old result's +content is still incoherent and will be re-cited. Quote verbatim lines +from elsewhere in the trace (e.g. the output of the later call that +did it right) rather than inventing output. + +Reply with ONLY the JSON object: {{"edits": [{{"i": , +"action": "keep"|"drop"|"rewrite", "content": "...", "drop_calls": +["", …], "args_fixes": +[{{"call_id": "...", "args": }}]}}]}} for the CITED indices. +""" + +JUDGE_HEADER = """You are an independent, adversarial quality judge for ONE WINDOW of a +synthetic "ideal-world" agent trace up for corpus admission. You had no +part in creating it. Judge ONLY what you see. + +QUALITY PILLARS (the spec): +{pillars} + +- INTENT: every window must serve the ORIGINAL TASK below β€” the same + task, not a different one. Ask: does this window's work plausibly + belong to a flawless execution of THAT task? + +ORIGINAL TASK (verbatim): +{task} + +IDEAL TRACE WINDOW (messages {lo}-{hi} of {n}, absolute indices): +{window} + +Judge EVERY message in the window. A violation is concrete: cite the +absolute index i, the pillar, and one line of detail. Do not invent +violations for style taste; hedged or subjective statements do not +count. If the window is flawless, verdict PASS with an EMPTY violations +array. NEVER put praise, justifications, or "no violation" notes in the +violations array β€” the array is exclusively for defects; anything +positive goes in "notes". For every array entry set "defect": true when +it is a real defect, false when it is praise or justification (though +praise does not belong in the array at all). + +WINDOW EDGES: consecutive windows overlap, and a tool call in the last +rows of this window may have its result row in the NEXT window (not +shown to you). NEVER cite "result missing/not provided" for a call at +or near the window's end β€” that is the window boundary, not a defect. +Reasoning may live in a separate channel not shown in this rendering; +absence of visible reasoning prose is NOT a violation unless the call +sequence itself is incoherent. +{answer_note} + +Reply with ONLY the JSON object: +{{"verdict": "PASS"|"FAIL", +"violations": [{{"i": , "pillar": "P1|P2|P3|P4|P5|P6|INTENT", +"detail": ""}}], +"notes": ""}} +""" + +ANSWER_NOTE = """ +This window contains the trace's FINAL ANSWER. Additionally assess +"answer_quality": "high"|"ok"|"poor" β€” does the final message directly, +correctly, and completely answer the ORIGINAL TASK, grounded in the +tool evidence shown? A final that only PLANS the work, asks permission +to start ("Want me to implement this?"), or leaves the task's actual +mutation undone is "poor" β€” an ideal execution DELIVERS. Likewise +"poor": an answer that re-lists information the user already received +earlier in the session, or pads with content the question did not ask +for β€” an ideal answer gives exactly what was asked, no more. Add +"answer_quality" to your JSON object.""" + +FINAL_ANSWER_HEADER = """You are completing an ideal-world agent trace. The session below ends +without a final answer (it was cut off mid-run). Write the final +assistant message a flawless execution would have ended with β€” grounded +ONLY in the evidence shown, in the trace's language. + +ORIGINAL TASK: +{task} + +TRACE (tail, absolute indices): +{tail} + +Rules: no new tool calls; answer directly, correctly, completely; +cite the evidence the answer rests on; match the task's language. +FOCUS RULE: give exactly what the question asks β€” for follow-up +questions deliver only the NEW piece of information; never repeat +facts the user already received in an earlier turn, and never pad +with unrequested context. Complete means complete FOR THE QUESTION +ASKED, not exhaustive. +Never state or imply a model identity ("I am …", "as an AI") and never +use refusal phrasing β€” the answer just delivers the result. +GROUNDING CONTRACT: every specific claim you make (code behavior, +file contents, values, findings) must be checkable against the tool +results AS RENDERED ABOVE. NEVER claim that files were changed, +written, fixed, or created unless a write/edit/patch call RESULT in +the rendering explicitly confirms it β€” when the visible calls are +read-only, deliver the analysis/diagnosis as the answer and say what +SHOULD be changed, never that it was changed. Tool results may be truncated β€” never +cite details that the visible text does not show; state findings at +the granularity the visible evidence supports. NEVER quote code, +diffs, or file content line-by-line unless those exact lines are +visible in the rendering above; if a patch was applied but its exact +text is clipped, describe WHAT changed and WHERE in prose instead of +reproducing code the evidence does not show. + +OUTPUT FORMAT RULE: the "answer" value must be natural-language PROSE +for a human reader β€” NEVER a JSON object, key-value dump, or machine +envelope. When the evidence shown is a raw JSON verdict/report, SUMMARIZE +its findings in flowing sentences; do not reproduce the JSON structure. + +Reply with ONLY the JSON object: {{"answer": ""}}""" + +REFRAME_HEADER = """A synthetic ideal-world agent trace is complete and high-quality, but +its content solves a slightly different task than its opening user +message states. Instead of rewriting the whole trace, we reframe the +task message to match the delivered work. + +Opening user message (current task): +{task} + +What the trace actually does (final answer): +{answer} + +Work out what coherent, natural user request the delivered work answers +perfectly, and write that request as the new opening task message. +Rules: +- Same language as the current task message. +- A plausible, natural user request β€” no meta commentary, no mention of + traces or reframing. +- It must be answerable EXACTLY by the final answer shown, with no + stretch: the trace's calls and results must all serve it. +- The new task must ask ONLY for what the trace actually DID. If the + trace stopped at analysis/diagnosis/planning, the new task must be an + analysis/diagnosis/planning request ("please check X and tell me + what's wrong / what to do") β€” NEVER keep an instruction to change, + fix, update, or implement anything the trace did not do. +- Keep it close to the original when possible; shift only what drifted. + +Reply with ONLY the JSON object: {{"task": "", +"rationale": ""}}""" + +# ---------------------------------------------------------------- schemas + +EDITS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "edits": { + "type": "array", + "items": { + "type": "object", + "properties": { + "i": {"type": "integer"}, + "action": {"type": "string", + "enum": ["keep", "drop", "rewrite"]}, + "content": {"type": "string"}, + "drop_calls": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "tool_call ids to remove from THIS message " + "(their result rows drop automatically) β€” " + "use for mixed-result multi-call turns: " + "drop only the failed call, keep the turn"), + }, + "args_fixes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "call_id": {"type": "string"}, + "args": {"type": "object"}, + }, + "required": ["call_id", "args"], + }, + }, + }, + "required": ["i", "action"], + }, + } + }, + "required": ["edits"], +} + +JUDGE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "verdict": {"type": "string", "enum": ["PASS", "FAIL"]}, + "violations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "i": {"type": "integer"}, + "pillar": {"type": "string"}, + "detail": {"type": "string"}, + "defect": {"type": "boolean"}, + }, + "required": ["i", "pillar", "detail", "defect"], + }, + }, + "answer_quality": {"type": "string", + "enum": ["high", "ok", "poor"]}, + "notes": {"type": "string"}, + }, + "required": ["verdict", "violations"], +} + +FINAL_ANSWER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], +} + +REFRAME_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "task": {"type": "string"}, + "rationale": {"type": "string"}, + }, + "required": ["task"], +} + +def _resolve_endpoint(cfg: Config) -> tuple[str, str]: + """(base_url, api_key); mirrors triage's resolution.""" + from traceforge.triage import _resolve_endpoint as _tri + + base, key, _model = _tri(cfg) + return base, key + + +def _post_chat( + base_url: str, + api_key: str, + model: str, + user_content: str, + schema: dict[str, Any], + *, + max_tokens: int, + timeout_s: int, + temperature: float, +) -> tuple[Optional[dict[str, Any]], Optional[str]]: + """One structured chat call; returns (parsed_dict, error). + + Ladder mirrors triage: structured_outputs -> guided_json -> bare + (+ balanced-span rescue). One retry per variant on network errors; + 'length' finish is retried once (bumped max_tokens x2) then fails. + """ + url = (base_url.rstrip("/") + "/").replace("//v1/", "/v1/") \ + + "chat/completions" + variants: list[dict[str, Any]] = [ + {"structured_outputs": {"json": schema}}, + {"guided_json": schema}, + {}, + ] + # HARD CALL DEADLINE. The ladder is 3 variants x 3 attempts, so a + # hanging endpoint multiplied timeout_s by NINE: one 300s call could + # burn 45 minutes (measured 2026-10-23: a 301KB trace sat in a single + # final-answer synthesis for 6+ minutes while the batch starved). + # Bound the whole call to ~2x timeout_s and shrink each attempt's + # socket timeout to what is left. + deadline = time.monotonic() + max(timeout_s * 2, timeout_s + 30) + last_err: Optional[str] = None + for variant in variants: + mt = max_tokens + for attempt in range(3): + remaining = deadline - time.monotonic() + if remaining <= 5: + return None, (last_err or "call deadline exceeded") \ + + " (call deadline exceeded)" + payload: dict[str, Any] = { + "model": model, + "messages": [{"role": "user", "content": user_content}], + "temperature": temperature, + "max_tokens": mt, + } + payload.update(variant) + req = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}" if api_key else "", + }, + method="POST", + ) + try: + with urllib.request.urlopen( + req, timeout=min(timeout_s, int(remaining))) as r: + # A socket timeout bounds each individual read, not the + # whole call: an endpoint that keeps trickling bytes + # holds the connection open past the deadline. Measured + # 2026-10-26: a final-answer call ran 30:45 against a + # 600s deadline while its worker sat idle. Read in + # chunks and abort as soon as the deadline passes. + chunks: list[bytes] = [] + while True: + if time.monotonic() > deadline: + raise TimeoutError( + "call deadline exceeded mid-body") + chunk = r.read(65536) + if not chunk: + break + chunks.append(chunk) + data = json.loads(b"".join(chunks).decode("utf-8")) + except urllib.error.HTTPError as e: + last_err = f"HTTP {e.code}" + break # unknown request field: try next variant + except (urllib.error.URLError, TimeoutError, OSError) as e: + last_err = f"network: {e}" + continue + try: + choice = data["choices"][0] + content = choice["message"]["content"] or "" + finish = choice.get("finish_reason") + except (KeyError, IndexError, TypeError): + last_err = "malformed response" + break + # live telemetry: record usage/model for the progress bar + try: + SPLICE_USAGE.add(model, data.get("usage")) + except Exception: + pass + if finish == "length" and attempt == 0: + mt = min(mt * 2, 32_000) + continue + if finish == "length": + last_err = "output truncated (finish=length)" + continue + parsed = _parse_json_object(content) + if parsed is not None: + return parsed, None + last_err = "unparseable content" + return None, last_err or "all variants failed" + + +def _parse_json_object(raw: str) -> Optional[dict[str, Any]]: + """Bare JSON, fenced JSON, or the largest balanced object in prose.""" + if not raw: + return None + candidates = [raw.strip()] + s = raw.strip() + if s.startswith("```"): + body = s.split("\n", 1)[1] if "\n" in s else "" + if body.rstrip().endswith("```"): + body = body.rstrip()[:-3] + candidates.append(body.strip()) + from traceforge.improver.review import _json_object_spans + + for span in _json_object_spans(s): + candidates.append(span) + for c in candidates: + d = json_loads_lenient(c) + if isinstance(d, dict): + return d + return None + + +# ---------------------------------------------------------------- windows + +def _render_budget(cfg: Config) -> dict: + return { + "user_chars": cfg.user_chars, + "assistant_chars": cfg.assistant_chars, + "tool_result_chars": cfg.tool_result_chars, + "tool_args_chars": cfg.tool_args_chars, + "include_reasoning": cfg.include_reasoning, + } + + +def _windows(trace: Trace, cfg: Config, *, + include_reasoning: Optional[bool] = None) -> list[dict]: + from traceforge.triage import triage_windows + + rk = _render_budget(cfg) + if include_reasoning is not None: + rk["include_reasoning"] = include_reasoning + return triage_windows( + trace, max_chars=cfg.splice_max_chars, + overlap=cfg.triage_overlap_messages, + render_kwargs=rk, + ) + + +# ------------------------------------------------------- plan and splice + +LABEL_TO_PLAN = { + # triage vocabulary -> splice action + "keep": "keep", + "drop": "drop", + "rephrase": "rewrite", + "summarize": "rewrite", # compress: rewrite with verbatim-quoted lines + "fact_check": "keep", # survives; the judge adjudicates the claim +} + + +def _first_task_index(trace: Trace) -> Optional[int]: + from traceforge.trace import COMPACTION_PREFIX, CONTINUATION_PREFIX + + for i, m in enumerate(trace.messages): + if m.role != "user": + continue + c = m.content or "" + if COMPACTION_PREFIX in c[:200] or CONTINUATION_PREFIX in c[:120]: + continue + return i + return None + + +def _task_is_plumbing(trace: Trace, task_idx: int) -> bool: + """The opening task row is harness plumbing, not a task. + + Cross-exam case 1a3c47: the whole session opens with a validator- + rejection row ("Your previous final response was rejected…") because + the real task lived in a PREVIOUS session. An ideal trace built on + that row teaches plumbing as problem-solving β€” reject instead. + """ + m = trace.messages[task_idx] + return _is_rejection_row(m) + + +# A task sentence that demands a concrete written artifact (tests, +# file) β€” and the trace made ZERO write/edit calls. The deliverable +# cannot exist, so no ideal-trace rewrite can satisfy the task as +# stated; the task must be reframed to the delivered work. +_DELIVERABLE_RE = re.compile( + r"((write|create|implement|add)\s+" + r"(tests?|a test|new file|the file|a file|the implementation)" + r"[^\n.]{0,120}" + r"|\b(update|upgrade|install|fix|delete|remove|migrate|deploy|" + r"refactor|rename|patch|bump|revert)\b)", + re.IGNORECASE, +) +# mutation commands executed via generic shells also deliver +_MUT_CMD_RE = re.compile( + r"\b(git\s+(pull|merge|apply|rebase|checkout|reset)" + r"|pip\s+\w*\s*install|npm\s+install|cargo\s+(add|install)" + r"|apt(-get)?\s+install|brew\s+(install|upgrade)" + r"|uv\s+(add|pip))\b", + re.IGNORECASE, +) +_WRITE_TOOLS = {"write", "edit", "write_file", "apply_patch", + "create_file"} + + + + +_JSON_TAIL_RE = re.compile(r'\s*"?(?:rationale|task)"\s*:\s*"?.*$', re.S) + +# The reframe model sometimes echoes its own prompt back into the task: the +# rule list ("Same language as the current task message.", "The new task must +# ask ONLY for what the trace actually DID."), the JSON-envelope instruction, +# or the rationale field. A user never types any of it, and the leak is always +# a TAIL (measured: 68 of 1044 promoted traces, every one cut cleanly at the +# first marker). Cut there; the real task text is untouched. +_REFRAME_LEAK_RE = re.compile( + r"(?:^|\n|(?<=[.!?])\s)\s*(?:" + r"\*{0,2}Rationale\*{0,2}\s*:" + r"|Same language as the current" + r"|Keep it close to the original" + r"|Reply with ONLY the JSON" + r"|The new task must" + r"|no meta commentary" + r"|answerable EXACTLY" + r"|no mention of traces or reframing" + r"|Work out what coherent" + r"|write that request as the new opening task message" + r"|A synthetic ideal-world agent trace" + r")", re.IGNORECASE) + + +def _sanitize_task_text(t: str) -> str: + """Defensive: reframe/synthesis paths sometimes return the whole + JSON envelope tail glued to the task text (6c4918: user row ended + '...\\n\\nrationale": "The delivered work…verbatim."}'). Strip any + trailing key/value/brace garbage AND any echoed prompt rules before + writing a user row β€” user rows are byte-verbatim artifacts, never + JSON and never pipeline instructions.""" + t2 = t + cut = False + stripped = _JSON_TAIL_RE.sub("", t2) + if stripped != t2: + t2, cut = stripped.rstrip(), True + m = _REFRAME_LEAK_RE.search(t2) + if m: + t2, cut = t2[:m.start()].rstrip(), True + if cut: + # drop a trailing orphan brace/quote cluster β€” only after a cut, so a + # clean row that legitimately ends in a quote is returned untouched + t2 = re.sub(r'[}"\'],*$', "", t2).rstrip() + return t2 + + +# A language directive the reframe model injects to match a drifted answer +# ("Report in German.", "Reply in German."). The guard compares the new task +# against the old one: a directive the user already wrote is fine, a NEW one +# is the model legitimising the P5 defect. +_LANG_DIRECTIVE_RE = re.compile( + r"\b(?:in|auf)\s+(?:German|Deutsch|English|Englisch)\b" + r"|\b(?:Reply|Report|Answer|Respond|Antworte|Antworten|Berichte|" + r"Schreibe)\s+(?:in|auf)\b", re.IGNORECASE) + + +# A task brief is written BY the user TO the agent. First-person future +# narration ("Ich prΓΌfe die drei Artefakte zuerst …", "I'm using the +# Superpowers workflow to …") is the AGENT's plan, which the reframe model +# sometimes writes into the task slot. Measured 2026-10-22: 23 of 1094 +# improved traces carried it, and 121 had the assistant echo it verbatim. +# Rejecting a reframe is cheap β€” the caller keeps the original task. +_AGENT_NARRATION_RE = re.compile( + r"^\s*(?:ich\s+(?:prΓΌfe|werde|will|setze|beginne|starte|nehme)" + r"|i\s+(?:check|am going|start|begin|set|use|will)" + r"|i['\u2019]m\s+(?:using|going|checking|starting|beginning|setting))\b", + re.IGNORECASE) + + +def _reframe_ok(task: str, new_task: str, + answer: Optional[str] = None) -> tuple[bool, str]: + """Reject a reframe that would introduce a defect of its own. + + Three measured failure modes: (a) the model appends a language + instruction ("Report in German.") to an English brief to match a + drifted answer β€” that legitimises the P5 defect the gate exists to + catch, and the gate's translation exemption then hides it; (b) the + sanitized task is too short to be a real request; (c) the model + writes the ANSWER as the task ("I'm using the Superpowers workflow + to structure this final review…", "Saved …audit.md. Remaining + deltas: …") β€” the echo gate then kills the trace one full improve + cycle later, so reject it here. + """ + if len(new_task) < 40: + return False, "too_short" + if _LANG_DIRECTIVE_RE.search(new_task) and not _LANG_DIRECTIVE_RE.search(task or ""): + return False, "language_flip" + old_lang = _confident_lang((task or "")[:300], 3) + new_lang = _confident_lang(new_task[:300], 3) + if old_lang and new_lang and old_lang != new_lang: + return False, "language_flip" + if answer and len(answer) >= 200: + ratio = difflib.SequenceMatcher( + None, _norm_for_similarity(new_task), + _norm_for_similarity(answer), autojunk=False).ratio() + if ratio >= 0.85: + return False, "echoes_answer" + # Checked last: an answer-echo is the more specific diagnosis, and the + # narration pattern also matches the answer's opening ("I'm using the …"). + if _AGENT_NARRATION_RE.match(new_task): + return False, "agent_narration" + return True, "" + + +def _reframe_undelivered_task( + cfg, sid: str, trace: Trace, candidate, task: str, + _status, _append_status, base: str, key: str, +) -> tuple[str, bool]: + """Pre-judge reframe: the task demands a deliverable (tests, file, + or a mutation) the trace never performed β€” no prose rewrite can + make an ideal trace out of it; change the task to the delivered + work. Returns (task, reframed).""" + if not (cfg.allow_task_reframe + and _task_demands_unwritten_deliverable(trace, task)): + return task, False + _status("reframe task (undelivered artifact)") + final = next((m.content for m in reversed(candidate.messages) + if m.role == "assistant"), "") + data, rerr = _post_chat( + base, key, cfg.model or "", + REFRAME_HEADER.format(task=(task or "")[:4000], + answer=(final or "")[:4000]), + REFRAME_SCHEMA, max_tokens=1000, + timeout_s=cfg.rewrite_timeout_s, + temperature=cfg.splice_temperature) + if data and isinstance(data.get("task"), str) and data["task"].strip(): + new_task = _sanitize_task_text(data["task"].strip()) + ok, why = _reframe_ok(task, new_task, final) + if not ok: + _append_status(cfg, sid, { + "stage": "reframe", "trigger": "undelivered_artifact", + "rejected": why, + "old_task": (task or "")[:200], + "new_task": (new_task or "")[:200]}) + return task, False + _append_status(cfg, sid, { + "stage": "reframe", "trigger": "undelivered_artifact", + "old_task": (task or "")[:200], + "new_task": new_task[:200]}) + for m in candidate.messages: + if m.role == "user": + m.content = new_task + break + return new_task, True + # reframe failure: fall through; judge cites honestly. + return task, False + + +def _task_demands_unwritten_deliverable( + trace: Trace, task: str) -> bool: + """True when the task text demands a written artifact but the raw + trace contains no write/edit tool call at all (5343b8 class).""" + if not _DELIVERABLE_RE.search((task or "")[:4000]): + return False + for m in trace.messages: + for tc in m.tool_calls or []: + if (tc.name or "").lower() in _WRITE_TOOLS: + return False + # a mutation command run through a generic shell counts as + # delivery (git pull / pip install / brew upgrade …) + if _MUT_CMD_RE.search(tc.arguments or ""): + return False + return True + + +# Harness-generated rejection/correction user rows (validator feedback, +# "previous response rejected", retry nudges) β€” negative signal P2 +# routes around. Real human corrections usually match too, and P2 +# wants them gone from the ideal path as well. +_REJECTION_RE = re.compile( + r"(previous (final )?(response|answer|attempt).{0,40}" + r"(reject|invalid|not valid|validation error)" + r"|(rejected? by the).{0,40}(validator|contract)" + r"|^validation errors?\b|output contract validator)", + re.IGNORECASE, +) + + +def _is_rejection_row(m: Message) -> bool: + return m.role == "user" and bool( + _REJECTION_RE.search((m.content or "")[:400])) + + +def _plan_from_labels( + trace: Trace, labels: dict[int, str] +) -> tuple[dict[int, str], dict[int, set[str]]]: + """SOURCE-index action plan (keep|drop|rewrite). + + Mechanical overrides: + - system rows always keep. + - the first TASK user row always keeps (intent anchor, P5). + - tool rows inherit their call message's action (pairing integrity). + - compaction/continuation handoff rows drop (harness artifacts that + carry no task and must not enter an ideal trace). + - pre-task rows before the first user row drop (an ideal trace + opens on the task). + - degenerate rows drop: empty-content assistant turns with no tool + calls exist verbatim in some raw sources (verified: 1fa6a3e5 + raw[1] content=None calls=[]) and only trip the empty-content + gate; they carry no signal. + - unlabeled rows default to keep (conservative). + """ + from traceforge.trace import COMPACTION_PREFIX, CONTINUATION_PREFIX + + first_task = _first_task_index(trace) + + plan: dict[int, str] = {} + for i, m in enumerate(trace.messages): + if m.role == "system": + plan[i] = "keep" # system prompts survive anywhere + continue + if first_task is not None and i < first_task: + plan[i] = "drop" # pre-task noise + continue + if i == first_task: + plan[i] = "keep" + continue + if m.role == "assistant" and not (m.content or "").strip() \ + and not m.tool_calls and i != len(trace.messages) - 1: + plan[i] = "drop" # degenerate empty turn, no signal + continue + if m.role == "user": + c = m.content or "" + if COMPACTION_PREFIX in c[:200] or CONTINUATION_PREFIX in c[:120]: + plan[i] = "drop" + continue + if _is_rejection_row(m): + plan[i] = "drop" # validator/correction feedback: P2 + continue # routes around it, never shows it + if m.role == "assistant" and not m.tool_calls: + c = m.content or "" + if COMPACTION_PREFIX in c[:200] \ + or CONTINUATION_PREFIX in c[:120]: + # 69 assistant-role compaction blocks in the corpus + # (5343b8 raw[25]: judge cited it as P6 'trace ends + # at a compaction summary'). Harness plumbing, same + # class as validator-rejection rows β€” an ideal trace + # must NEVER show context-compression mechanics. + plan[i] = "drop" + continue + plan[i] = LABEL_TO_PLAN.get(labels.get(i, "keep"), "keep") + + calls_owner: dict[str, int] = {} + for i, m in enumerate(trace.messages): + for tc in m.tool_calls: + if tc.id: + calls_owner[tc.id] = i + # P2 by construction: an ideal trace NEVER shows a failed tool call. + # Assistant turns whose results are ALL error results drop, together + # with those result rows (their retry/success successor carries the + # information). Verified on de2f3c: raw[4]->[5] and raw[12]->[13] + # were the exact P2 citations across three repair rounds; LLM + # discretion (drop rule in the prompt) proved UNSTABLE β€” passed one + # run, failed two. Mixed turns (some results OK) keep everything. + from traceforge.trace import _is_error_result + results_by_owner: dict[int, list[tuple[int, bool]]] = {} + mixed_drop_calls: dict[int, set[str]] = {} + for i, m in enumerate(trace.messages): + if m.role == "tool" and m.tool_call_id in calls_owner: + oi = calls_owner[m.tool_call_id] + results_by_owner.setdefault(oi, []).append( + (i, _is_error_result(m.content or ""))) + for oi, pairs in results_by_owner.items(): + if pairs and all(err for _, err in pairs): + plan[oi] = "drop" + for j, _ in pairs: + plan[j] = "drop" + elif pairs and any(err for _, err in pairs): + # MIXED turn: drop ONLY the failed calls' result rows and + # strip their ids from the owner (cf1f5b: web_search ok + + # browser_exec chrome-profile-lock error in ONE turn β€” + # the judge cites the failed call as P2 forever, and the + # all-error rule can never fire). The owner turn keeps its + # prose only when prose exists; empty owners drop. + bad_ids = {trace.messages[j].tool_call_id + for j, err in pairs if err} + mixed_drop_calls.setdefault(oi, set()).update(bad_ids) + for j, err in pairs: + if err: + plan[j] = "drop" + for i, m in enumerate(trace.messages): + if m.role == "tool": + owner = calls_owner.get(m.tool_call_id or "") + if owner is None: + plan[i] = "drop" # orphan result (rejected/empty call): + # unpairable in any ideal trace + elif plan[i] == "drop": + pass # mixed-turn surgical drop already decided above + else: + # P4: tool results are spliced VERBATIM from the raw + # trace. Inheriting the owner's "rewrite" would ask the + # LLM to re-emit the result, and its compression fails + # the fidelity gate β€” the measured non-convergence loop + # of clip-repair pilots (gates fail xN, 2026-10-04). + plan[i] = "keep" if plan[owner] == "rewrite" \ + else plan[owner] + # REDUNDANT RE-READS (d71c6f class, 7059 corpus-wide): a + # read/search call repeating an EARLIER call's exact (name, path) + # drops when nothing wrote to that path in between β€” the first + # call's result is still valid, the repeat is exploratory noise + # (P3 every-call-necessary). A write/patch to the path between + # them invalidates the earlier read: the LATER read keeps. + import re as _re + _path_re = _re.compile(r'"(?:path|file_path)"\s*:\s*"([^"]+)"') + _range_re = _re.compile(r'"(?:offset|limit|page|cursor)"\s*:\s*"?[0-9]+"?') + seen_call: dict[tuple, int] = {} + dirtied: set[str] = set() + for i, m in enumerate(trace.messages): + wrote_now = False + for tc in m.tool_calls or []: + args = tc.arguments if isinstance(tc.arguments, str) \ + else json.dumps(tc.arguments or {}, ensure_ascii=False) + nm = (tc.name or "").lower() + pm = _path_re.search(args or "") + if pm and any(k in nm for k in ("write", "patch", "edit", + "create", "move", "remove")): + dirtied.add(pm.group(1)); wrote_now = True + elif pm: + # ONLY an exact-args repeat is redundant: ranged + # reads (offset/limit) and different search patterns + # over the same path are DISTINCT calls β€” the WORK, + # not exploration (854d7f false-drop class). + key = (nm, _re.sub(r"\s+", "", args or "")) + if key in seen_call and pm.group(1) not in dirtied \ + and plan.get(i) not in ("drop",): + # drop the owner's redundant call surgically + mixed_drop_calls.setdefault(i, set()).add(tc.id or "") + for j, mm in enumerate(trace.messages): + if mm.role == "tool" and mm.tool_call_id == tc.id: + plan[j] = "drop" + else: + seen_call[key] = i + if not wrote_now: + continue + + # DEGENERATE EMPTY PAIRS (019ff135 class): a tool call with empty + # arguments whose result content is empty/None carries no signal β€” + # raw codex rollouts have it verbatim (019ff135#p2 raw[5]/[6]: + # send_message {} -> content=None) and spliced verbatim it trains + # the model to emit no-arg calls with empty results. Drop the pair + # surgically; the owner turn keeps its prose and other calls + # (empty owners drop via the _apply_edits cascade). + for i, m in enumerate(trace.messages): + if m.role != "tool" or (m.content or "").strip(): + continue + owner = calls_owner.get(m.tool_call_id or "") + if owner is None: + continue # orphan results already drop above + for tc in trace.messages[owner].tool_calls or []: + if tc.id != m.tool_call_id: + continue + args = tc.arguments if isinstance(tc.arguments, str) \ + else json.dumps(tc.arguments or {}, ensure_ascii=False) + if not (args or "").strip("{} \t\n\r"): + mixed_drop_calls.setdefault(owner, set()).add(tc.id or "") + plan[i] = "drop" + + return plan, mixed_drop_calls + + +def _apply_edits( + trace: Trace, + plan: dict[int, str], + edits: list[dict], +) -> tuple[list[Message], list[int], list[str]]: + """Splice the ideal message list from the raw trace. + + Returns (messages, src_map, warnings); messages/src_map are parallel + (message at position j comes from source index src_map[j]) BEFORE + _mechanical_fixes merges rows. + """ + warnings: list[str] = [] + by_idx: dict[int, dict] = {} + for e in edits or []: + if not isinstance(e, dict): + continue + i = e.get("i") + if not isinstance(i, (int, float)) or isinstance(i, bool): + continue + i = int(i) + if 0 <= i < len(trace.messages): + by_idx.setdefault(i, e) + + final: dict[int, str] = dict(plan) + for i, e in by_idx.items(): + act = str(e.get("action", "")).strip().lower() + if act in ("keep", "drop", "rewrite"): + final[i] = act + + # PER-CALL DROPS (mixed-result multi-call turns): an edit may name + # individual tool_call ids to remove from a KEPT turn β€” the failed + # call goes, its result rows go, the turn stays with its good + # calls (85aa21: turn [5] = one TypeError call + one good call; + # the all-error-turn drop can't fire, message-level drop is too + # blunt, and the judge cites the failed call as P2 forever). + edit_drop_calls: dict[int, set[str]] = {} + for i, e in by_idx.items(): + dcs = e.get("drop_calls") + if isinstance(dcs, list): + ids = {str(c) for c in dcs if c} + owner = trace.messages[i] if i < len(trace.messages) else None + if owner is not None and owner.role == "assistant" and ids: + edit_drop_calls[i] = ids + + # CASCADE DROPS (pairing integrity): the rewrite prompt promises + # "dropping an assistant message drops its tool calls and their + # results automatically" β€” implement exactly that, both directions: + # a dropped assistant turn drops its result rows; a dropped result + # row drops the owning turn's call (and the turn itself when it + # carries nothing else). Without the cascade, surviving result rows + # orphan at the pairing gate ("references unknown call", 704bc8). + owner_of: dict[str, int] = {} + for i, m in enumerate(trace.messages): + for tc in m.tool_calls: + if tc.id: + owner_of[tc.id] = i + dropped_calls: set[str] = set() # calls whose result row dropped + for i, m in enumerate(trace.messages): + if final.get(i) != "drop": + continue + if m.role == "assistant": + for tc in m.tool_calls: + for j, mm in enumerate(trace.messages): + if mm.role == "tool" and mm.tool_call_id == tc.id: + final[j] = "drop" + elif m.role == "tool" and m.tool_call_id in owner_of: + dropped_calls.add(m.tool_call_id) + oi = owner_of[m.tool_call_id] + if final.get(oi) == "keep": + remaining_calls = [ + tc for tc in trace.messages[oi].tool_calls + if tc.id and final.get( + next( + (j for j, mm in enumerate(trace.messages) + if mm.role == "tool" + and mm.tool_call_id == tc.id), + -1), + "keep") != "drop" + ] + if remaining_calls or trace.messages[oi].content: + continue # turn keeps its other calls / prose + final[oi] = "drop" + + msgs: list[Message] = [] + src_map: list[int] = [] + # result rows whose call was edit-dropped must not emit + edit_dropped_ids: set[str] = set() + for ids in edit_drop_calls.values(): + edit_dropped_ids |= ids + for i, m in enumerate(trace.messages): + act = final.get(i, "keep") + if act == "drop": + continue + if m.role == "tool" and m.tool_call_id in edit_dropped_ids: + continue + if act == "rewrite": + e = by_idx.get(i, {}) + new_content = e.get("content") + if m.role == "user": + # P5: user text is never rewritten. A "rewrite" edit on + # a user row downgrades to drop when the edit content + # says so; otherwise it is ignored (kept verbatim). + warnings.append(f"[{i}] rewrite on user row ignored (P5)") + msgs.append(_copy(m)) + src_map.append(i) + continue + if m.role == "tool": + compressed = str(new_content) if ( + isinstance(new_content, str) and new_content.strip() + ) else m.content + if CLIP_MARKER_PROBE.search(compressed or ""): + warnings.append( + f"[{i}] rewrite introduced clip marker; kept original") + compressed = m.content + msgs.append(_copy(m, content=compressed)) + src_map.append(i) + continue + # assistant rewrite: new prose, calls kept (args_fixes applied) + fixes = {f.get("call_id"): f.get("args") + for f in (e.get("args_fixes") or []) + if isinstance(f, dict) and f.get("call_id")} + drop_here = edit_drop_calls.get(i, set()) + calls: list[ToolCall] = [] + for tc in m.tool_calls: + if tc.id in dropped_calls or tc.id in drop_here: + # result row dropped: keep the turn, lose the call + # (owner had prose/other calls, else the cascade + # would have dropped the whole turn) + continue + if tc.id in fixes and isinstance(fixes[tc.id], dict): + calls.append(ToolCall( + id=tc.id, name=tc.name, + arguments=json.dumps( + fixes[tc.id], ensure_ascii=False), + )) + else: + calls.append(tc) + content = str(new_content) if ( + isinstance(new_content, str) and new_content.strip() + ) else m.content + if CLIP_MARKER_PROBE.search(content or ""): + warnings.append( + f"[{i}] rewrite introduced clip marker; kept original") + content = m.content + from traceforge.audit import SELF_TALK_RE + if content and SELF_TALK_RE.search(content): + warnings.append( + f"[{i}] rewrite kept harness self-talk; kept original") + content = m.content + msgs.append(Message( + role="assistant", content=content, tool_calls=calls)) + src_map.append(i) + continue + if m.role == "assistant" and ( + dropped_calls or edit_dropped_ids) \ + and any(tc.id in dropped_calls or tc.id in edit_dropped_ids + for tc in m.tool_calls): + # kept verbatim, but its result row was dropped: strip the + # orphaned call so the pairing gate stays green + kept = [tc for tc in m.tool_calls + if tc.id not in dropped_calls + and tc.id not in edit_dropped_ids] + if kept or (m.content or "").strip(): + msgs.append(Message( + role=m.role, content=m.content, tool_calls=kept, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + src_map.append(i) + # else: row is now empty β€” drop it (an empty assistant row + # trips the structural gate; nothing trainable remains) + continue + msgs.append(_copy(m)) + src_map.append(i) + return msgs, src_map, warnings + + +def _copy(m: Message, content: Optional[str] = None) -> Message: + """Verbatim copy; reasoning dropped (export drops it anyway).""" + return Message( + role=m.role, + content=content if content is not None else m.content, + tool_calls=list(m.tool_calls), + tool_call_id=m.tool_call_id, + tool_name=m.tool_name, + ) + + +def _mechanical_fixes( + msgs: list[Message], src_map: list[int] +) -> tuple[list[Message], list[int]]: + """Merge consecutive same-role text-only rows (gate shape); src_map + keeps the FIRST source index of each merged row.""" + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if out: + p = out[-1] + if (m.role == p.role == "assistant" + and not m.tool_calls and not p.tool_calls): + p.content = (p.content or "") + "\n\n" + (m.content or "") + continue + if m.role == p.role == "user": + p.content = (p.content or "") + "\n\n" + (m.content or "") + continue + out.append(m) + omap.append(si) + return out, omap + + +# ------------------------------------------------- no-op call repair +# A tool call that provably changes nothing is a REAL defect β€” the judge +# cites it as P3 and an ideal trace must not contain it β€” but it is also +# the one defect class with a DETERMINISTIC repair: drop the call and its +# result row. Routing it to an LLM repair round does not converge: +# measured 2026-10-23 on 20261013_095155_2dcd99 (236KB), the judge cited +# "Patch call between messages 9 and 10 has identical old_string and +# new_string … result (no_change:true)", the repair writer reproduced the +# same no-op call, and after max rounds the trace died with an empty +# workdir. Strip it mechanically BEFORE the judge ever sees the candidate. +_NOOP_RESULT_RE = re.compile( + r'("?no_change"?\s*:\s*true' + r'|\bno changes? (?:were |was )?(?:made|applied|necessary|needed)\b' + r'|\bfile (?:is )?unchanged\b' + r'|\bnothing to (?:change|do)\b' + r'|\bcontent (?:is )?identical\b)', + re.IGNORECASE, +) +# only patch-shaped tools: a write_file result saying "unchanged" is a +# different (and rarer) story, and prose in a shell result can match the +# regex by accident +_PATCH_TOOL_NAMES = { + "patch", "edit", "str_replace", "str_replace_editor", "apply_patch", + "replace", "multiedit", "multi_edit", "search_replace", "apply_diff", + "edit_file", "modify_file", +} +_OLD_ARG_KEYS = ("old_string", "old_str", "old_text", "old", "search", + "original", "before", "from") +_NEW_ARG_KEYS = ("new_string", "new_str", "new_text", "new", "replace", + "replacement", "after", "to") + + +def _args_dict(tc: ToolCall) -> Optional[dict]: + raw = tc.arguments + if isinstance(raw, dict): + return raw + if not isinstance(raw, str) or not raw.strip(): + return None + d = json_loads_lenient(raw) + return d if isinstance(d, dict) else None + + +def _is_noop_call(tc: ToolCall, result: Optional[Message]) -> Optional[str]: + """Reason string when this call provably changes nothing, else None.""" + args = _args_dict(tc) + if args: + for ok in _OLD_ARG_KEYS: + if ok not in args: + continue + for nk in _NEW_ARG_KEYS: + if nk not in args: + continue + a, b = args.get(ok), args.get(nk) + if isinstance(a, str) and isinstance(b, str) \ + and a.strip() and a == b: + return f"{tc.name}: {ok} == {nk} (guaranteed no-op)" + name = (tc.name or "").strip().lower() + if result is not None and name in _PATCH_TOOL_NAMES: + body = (result.content or "").strip() + # a no-op result is terse; long prose that happens to contain the + # phrase is a narrative, not a verdict + if body and len(body) <= 600 and _NOOP_RESULT_RE.search(body): + return f"{tc.name}: result reports no change" + return None + + +def _strip_noop_calls( + msgs: list[Message], src_map: list[int] +) -> tuple[list[Message], list[int], list[str]]: + """Drop provably no-op tool calls and their result rows. + + Deterministic pre-repair normalization (see the block comment above). + Returns (msgs, src_map, reasons); src_map stays parallel. + """ + if not msgs: + return msgs, src_map, [] + results: dict[str, Message] = {} + for m in msgs: + if m.role == "tool" and m.tool_call_id: + results[m.tool_call_id] = m + drop_ids: set[str] = set() + reasons: list[str] = [] + for m in msgs: + if m.role != "assistant": + continue + for tc in m.tool_calls: + if not tc.id or tc.id in drop_ids: + continue + why = _is_noop_call(tc, results.get(tc.id)) + if why: + drop_ids.add(tc.id) + reasons.append(f"[{tc.id[:14]}] {why}") + if not drop_ids: + return msgs, src_map, [] + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if m.role == "tool" and m.tool_call_id in drop_ids: + continue # its call is gone: the result row would orphan + if m.role == "assistant" and any( + tc.id in drop_ids for tc in m.tool_calls): + kept = [tc for tc in m.tool_calls if tc.id not in drop_ids] + if kept or (m.content or "").strip(): + out.append(Message( + role=m.role, content=m.content, tool_calls=kept, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + omap.append(si) + # else: the turn was nothing but the no-op β€” drop the row + continue + out.append(m) + omap.append(si) + return out, omap, reasons + + +# ------------------------------------------------- duplicate tool calls +# A second call with the same name AND the same arguments in one assistant +# turn is a stall, not work: the second returns what the first already did. +# The judge cites it as P3 (redundant call), and the repair round cannot +# remove it without rewriting the turn, so drop the duplicate here β€” the +# first call keeps its result, the duplicate and its result row go together. +def _dedupe_calls( + msgs: list[Message], src_map: list[int] +) -> tuple[list[Message], list[int], list[str]]: + """Drop tool calls that repeat an identical call inside the same turn.""" + drop_ids: set[str] = set() + reasons: list[str] = [] + for m in msgs: + if m.role != "assistant" or not m.tool_calls: + continue + seen: dict[str, str] = {} + for tc in m.tool_calls: + key = json.dumps( + {"name": tc.name, "args": tc.arguments}, sort_keys=True) + first = seen.get(key) + if first is None: + seen[key] = tc.id + continue + drop_ids.add(tc.id) + reasons.append( + f"[{tc.id[:14]}] repeats {first[:14]} in the same turn") + if not drop_ids: + return msgs, src_map, [] + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if m.role == "tool" and m.tool_call_id in drop_ids: + continue # its call is gone: the result row would orphan + if m.role == "assistant" and any( + tc.id in drop_ids for tc in m.tool_calls): + kept = [tc for tc in m.tool_calls if tc.id not in drop_ids] + if kept or (m.content or "").strip(): + out.append(Message( + role=m.role, content=m.content, tool_calls=kept, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + omap.append(si) + continue + out.append(m) + omap.append(si) + return out, omap, reasons + + +# ------------------------------------------------- prose dedupe +# The rewrite stage re-narrates the plan the KEPT row already stated. +# Measured 2026-10-23 on 019fd233 (180KB): ideal [1] said "erst Regeln und +# bestehende Tests lesen", the rewrite of the next turn said "erst die +# Repository-Regeln lesen" β€” the judge cited it as P1 filler, the repair +# round returned no edits, and the trace died with an empty workdir. An +# ideal trace does not restate itself, and this is deterministic: collapse +# the duplicate sentence instead of paying a repair round for it. +_SENT_SPLIT_RE = re.compile(r"(?<=[.!?])\s+|\n{2,}") + + +def _dedupe_prose( + msgs: list[Message], src_map: list[int], + *, threshold: float = 0.85, min_len: int = 40, +) -> tuple[list[Message], list[int], list[str]]: + """Drop sentences an assistant row restates from the previous + assistant prose row. Returns (msgs, src_map, reasons).""" + if not msgs: + return msgs, src_map, [] + out: list[Message] = [] + omap: list[int] = [] + prev: list[str] = [] + reasons: list[str] = [] + for m, si in zip(msgs, src_map): + if m.role != "assistant" or not (m.content or "").strip(): + out.append(m) + omap.append(si) + continue + sents = [s for s in _SENT_SPLIT_RE.split(m.content) if s.strip()] + kept: list[str] = [] + for s in sents: + ns = _norm_for_similarity(s) + dup = False + if len(ns) >= min_len: + for p in prev: + if difflib.SequenceMatcher(None, ns, p).ratio() >= threshold: + dup = True + break + if dup: + reasons.append(f"[{si}] dropped restated sentence: {s[:90]}") + else: + kept.append(s) + if len(kept) == len(sents): + out.append(m) + omap.append(si) + prev = [_norm_for_similarity(s) for s in sents + if len(_norm_for_similarity(s)) >= min_len] + continue + new_content = " ".join(kept).strip() + if not new_content and not m.tool_calls: + continue # the row was nothing but a restatement + out.append(Message( + role=m.role, content=new_content, tool_calls=m.tool_calls, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + omap.append(si) + prev = [_norm_for_similarity(s) for s in kept + if len(_norm_for_similarity(s)) >= min_len] + return out, omap, reasons + + +# ------------------------------------------------- empty-row repair +def _drop_empty_rows( + msgs: list[Message], src_map: list[int], +) -> tuple[list[Message], list[int], list[str]]: + """Drop rows that can never pass validate_structural. + + Static census of 400 empty-workdir raws (2026-10-23): 59% already fail + the structural gate, and the top two classes are rows the splice copies + verbatim β€” + 410x "assistant message with empty content and no tool calls" + 377x "tool result is empty" + 18x "tool result without tool_call_id" + An ideal trace has no empty turn and no result-less call, so remove + them mechanically instead of paying a repair round that reproduces + them (the writer copies the source shape). + """ + if not msgs: + return msgs, src_map, [] + reasons: list[str] = [] + # a call whose result row is empty/unpairable cannot be kept: the + # result is unusable evidence and the pairing gate rejects the orphan + drop_calls: set[str] = set() + for m in msgs: + if m.role != "tool": + continue + if not m.tool_call_id: + reasons.append("dropped tool result without tool_call_id") + drop_calls.add(f"__row__{id(m)}") + elif not (m.content or "").strip(): + reasons.append( + f"dropped empty tool result for call {m.tool_call_id[:14]}") + drop_calls.add(m.tool_call_id) + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if m.role == "tool" and ( + not m.tool_call_id + or not (m.content or "").strip()): + continue + if m.role == "assistant": + kept = [tc for tc in m.tool_calls if tc.id not in drop_calls] + if len(kept) != len(m.tool_calls): + reasons.append( + f"[{si}] dropped call(s) whose result was empty") + if not kept and not (m.content or "").strip(): + reasons.append(f"[{si}] dropped empty assistant row") + continue + if len(kept) != len(m.tool_calls): + out.append(Message( + role=m.role, content=m.content, tool_calls=kept, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + omap.append(si) + continue + out.append(m) + omap.append(si) + return out, omap, reasons + + +# ------------------------------------------------- failed-call repair +_TEST_CMD_RE = re.compile( + r"\b(pytest|py\.test|tox|nox|npm\s+(run\s+)?test|yarn\s+test|pnpm\s+test|" + r"cargo\s+test|go\s+test|mvn\s+test|gradle\s+test|jest|vitest|rspec|phpunit|" + r"make\s+test|ctest)\b", + re.IGNORECASE, +) + + +def _call_text(call: Optional[ToolCall]) -> str: + """Flatten a call's arguments to text (arguments may be a dict or JSON).""" + if call is None: + return "" + args = call.arguments + if isinstance(args, dict): + try: + return " ".join(str(v) for v in args.values()) + except Exception: + return "" + return args or "" + + +_FATAL_TEXT_RE = re.compile( + r"(^|\n)\s*(fatal:|error:|Traceback \(most recent call last\)|" + r"command not found|No such file or directory|" + r"is not recognized as an internal or external command)" + # A shell tool prefixes its own name: "rg: : No such file or + # directory (os error 2)". Measured 019f85fb#p3: the judge cited + # "[3] P2: rg was invoked with an explicit file path ... that does not + # exist, producing a visible 'No such file or directory' error" and the + # line-start-only pattern missed it. Scoped to the two unambiguous + # phrases so a code listing ("error: None", "Error::Message(...)") + # cannot be mistaken for a failed call: 0.37% of 18k sampled results. + r"|(^|\n)\s*[a-z][\w.\-]{0,15}:\s+\S+\s*:\s*" + r"(?:No such file or directory|command not found)", + re.IGNORECASE, +) + + +def _result_is_error( + content: Optional[str], call: Optional[ToolCall] = None, +) -> bool: + """True when a tool result reports a failed call. + + Three shapes count: a structured ``"error": ""`` (the tool + itself failed), a non-zero ``exit_code`` (the command ran and failed), + and a plain-text fatal (``fatal:``/``Traceback``/``command not + found``) for tools that return raw stderr. The one exception is a + TEST RUNNER: a red test is the point of a TDD trace, so dropping it + would break the narrative. Measured 20260924_235809_c1a3ec: "[19] P2: + git show HEAD:automations.rs fails … the fatal error is visible in + the [20] result" β€” plain text, so the JSON-only detector missed it. + """ + body = (content or "").strip() + if not body: + return False + if not body.startswith("{"): + return bool(_FATAL_TEXT_RE.search(body)) + d = json_loads_lenient(body) + if not isinstance(d, dict): + return bool(_FATAL_TEXT_RE.search(body)) + err = d.get("error") + if isinstance(err, str) and err.strip(): + return True + code = d.get("exit_code") + if isinstance(code, int) and code != 0: + if _TEST_CMD_RE.search(_call_text(call)): + return False + return True + return False + + +def _strip_failed_calls( + msgs: list[Message], src_map: list[int], +) -> tuple[list[Message], list[int], list[str]]: + """Drop tool calls whose result reports a tool-level error. + + P2 ("no failed calls or wrong turns") is the judge's most-cited + pillar and the repair round reproduces the failed call instead of + removing it. An ideal trace does not contain a call that errored. + """ + if not msgs: + return msgs, src_map, [] + drop_calls: set[str] = set() + reasons: list[str] = [] + by_id = {tc.id: tc for m in msgs if m.role == "assistant" + for tc in m.tool_calls} + for m in msgs: + if m.role == "tool" and m.tool_call_id and _result_is_error( + m.content, by_id.get(m.tool_call_id)): + drop_calls.add(m.tool_call_id) + reasons.append( + f"dropped failed call {m.tool_call_id[:14]}: " + f"{(m.content or '')[:80]}") + if not drop_calls: + return msgs, src_map, [] + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if m.role == "tool" and m.tool_call_id in drop_calls: + continue + if m.role == "assistant" and any( + tc.id in drop_calls for tc in m.tool_calls): + kept = [tc for tc in m.tool_calls if tc.id not in drop_calls] + if kept or (m.content or "").strip(): + out.append(Message( + role=m.role, content=m.content, tool_calls=kept, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + omap.append(si) + continue + out.append(m) + omap.append(si) + return out, omap, reasons + + +# ------------------------------------------- committed-mistake repair +# A tool call can SUCCEED and still be a committed mistake: the call's own +# result carries the broken consequence (a patch that applies cleanly but +# leaves the file failing lint), or a LATER tool result names the file the +# call touched and reports a compile error. _strip_failed_calls keys on the +# call's own result being an error, so it misses both shapes. +# +# Measured 2026-10-23 on 20261013_095155_2dcd99 (242KB): the judge cited +# "[11] P2: The first patch call in [11] resulted in a lint error (E0670: +# async fn not permitted in Rust 2015), which is a committed mistake/failed +# call." The patch result was +# {"success": true, ..., "lint": {"status": "error", +# "output": "error[E0670] ... --> .../fallback_model_access.rs:23:1"}} +# The repair round reproduced the same patch and the trace died with an +# empty workdir. An ideal trace does not contain a call whose consequence +# is a broken file, so drop it mechanically. +# +# Only COMPILE/LINT evidence counts for the later-result shape. A red test +# run is the POINT of a TDD trace, so "FAILED"/"FAIL" never qualifies β€” +# otherwise this pass would delete the RED step of every TDD trace. +_COMPILE_ERROR_RE = re.compile( + r"error\[E\d{3,4}\]" + r"|error TS\d{4}" + r"|SyntaxError|IndentationError" + r"|cannot find (?:symbol|name|crate|module)" + r"|unresolved import" + r"|does not compile" + r"|compilation failed" +) +_PATH_ARG_KEYS = ("path", "file_path", "filename", "file", "target_file") +# Only a call that MODIFIES a file can leave it broken. A read/search call +# cannot cause a compile error, so the later-result shape must not fire on +# one (measured 20261013_095155_2dcd99: the read_file of the very file the +# patch later touched was dropped as a "committed mistake"). +_WRITE_TOOL_NAMES = _PATCH_TOOL_NAMES | { + "write_file", "write", "create_file", "create", "append_file", + "save_file", "notebook_edit", "insert", "delete_file", +} + + +def _result_consequence_error(content: Optional[str]) -> Optional[str]: + """Reason when a SUCCESSFUL call's own result reports a broken file.""" + body = (content or "").strip() + if not body.startswith("{"): + return None + d = json_loads_lenient(body) + if not isinstance(d, dict): + return None + if d.get("success") is False: + return None # the failed-call shape belongs to _strip_failed_calls + lint = d.get("lint") + if isinstance(lint, dict): + if str(lint.get("status") or "").lower() == "error": + out = str(lint.get("output") or "").strip() + if out: + return f"lint error: {out[:140]}" + elif isinstance(lint, str) and lint.strip().lower().startswith("error"): + return f"lint error: {lint.strip()[:140]}" + for key in ("compile", "build", "check", "typecheck"): + blk = d.get(key) + if isinstance(blk, dict) \ + and str(blk.get("status") or "").lower() == "error": + return f"{key} error: {str(blk.get('output') or '')[:140]}" + return None + + +def _call_paths(tc: ToolCall) -> list[str]: + args = _args_dict(tc) or {} + out: list[str] = [] + for k in _PATH_ARG_KEYS: + v = args.get(k) + if isinstance(v, str) and v.strip(): + out.append(v.strip()) + return out + + +def _patch_args(tc: ToolCall) -> Optional[tuple[str, str, str]]: + """(path, old_string, new_string) for a patch-shaped call, else None.""" + if (tc.name or "").strip().lower() not in _PATCH_TOOL_NAMES: + return None + args = _args_dict(tc) or {} + path = next((args[k] for k in _PATH_ARG_KEYS + if isinstance(args.get(k), str) and args[k].strip()), "") + old = next((args[k] for k in _OLD_ARG_KEYS + if isinstance(args.get(k), str)), None) + new = next((args[k] for k in _NEW_ARG_KEYS + if isinstance(args.get(k), str)), None) + if old is None or new is None: + return None + return path, old, new + + +def _compile_sig(body: str) -> str: + """The compile-error token a result reports ('' when none).""" + m = _COMPILE_ERROR_RE.search(body or "") + return m.group(0) if m else "" + + +def _strip_committed_mistakes( + msgs: list[Message], src_map: list[int], *, lookahead: int = 8, +) -> tuple[list[Message], list[int], list[str]]: + """Drop tool calls whose SUCCESS left a broken file behind. + + Three tiers, in order of evidence strength: + 1. the call's own result reports a lint/compile error -> drop + 2. a later result names the file+line with a compile error -> drop + 3. a KEPT patch's old_string only exists because a dropped patch + created it -> merge the two (the kept patch then applies to the + pre-broken state, so the final file content is unchanged); if the + merge is not clean, keep the broken call and let the window + rewrite handle it. + """ + if not msgs: + return msgs, src_map, [] + owner: dict[str, int] = {} + for i, m in enumerate(msgs): + for tc in m.tool_calls: + if tc.id: + owner[tc.id] = i + result_idx: dict[str, int] = {} + for i, m in enumerate(msgs): + if m.role == "tool" and m.tool_call_id: + result_idx.setdefault(m.tool_call_id, i) + + broken: dict[str, str] = {} + for cid, oi in owner.items(): + ri = result_idx.get(cid) + if ri is None: + continue + tc = next(t for t in msgs[oi].tool_calls if t.id == cid) + why = _result_consequence_error(msgs[ri].content) + if why: + broken[cid] = f"{tc.name}: {why}" + continue + if _result_is_error(msgs[ri].content, tc): + continue # already a failed call + if (tc.name or "").strip().lower() not in _WRITE_TOOL_NAMES: + continue # a read cannot break a file + paths = _call_paths(tc) + if not paths: + continue + # A compile error that ALREADY appeared before this call is a + # pre-existing condition of the file, not this call's consequence. + # Measured 20261013_095155_2dcd99: the E0670 edition error sits at + # line 23 of a file the patch never touched, so attributing it to + # every later patch on that file deleted the trace's real work. + prior = {_compile_sig(msgs[k].content or "") for k in range(0, ri)} + prior.discard("") + for j in range(ri + 1, min(ri + 1 + lookahead, len(msgs))): + body = msgs[j].content or "" + if msgs[j].role != "tool" or not body: + continue + sig = _compile_sig(body) + if not sig or sig in prior: + continue + hit = next((p for p in paths if re.search( + re.escape(os.path.basename(p)) + + r"(?:[:\(]\d+|[^\n]{0,12}?\bline\s+\d+)", body)), None) + if hit: + broken[cid] = ( + f"{tc.name}: later result [{j}] reports a compile error " + f"in {os.path.basename(hit)}") + break + if not broken: + return msgs, src_map, [] + + reasons: list[str] = [] + merged: dict[str, tuple[str, str]] = {} + for cid, oi in sorted(owner.items(), key=lambda kv: kv[1]): + if cid in broken: + continue + kp = _patch_args(next(t for t in msgs[oi].tool_calls if t.id == cid)) + if kp is None: + continue + kpath, kold, knew = kp + for did, doi in sorted(owner.items(), key=lambda kv: kv[1]): + if did not in broken or doi >= oi: + continue + dp = _patch_args( + next(t for t in msgs[doi].tool_calls if t.id == did)) + if dp is None: + continue + dpath, dold, dnew = dp + if dpath != kpath or not kold or kold not in dnew: + continue + if not dold or dold == dnew: + broken.pop(did, None) + reasons.append( + f"kept broken call {did[:14]}: dependent patch " + f"{cid[:14]} cannot be merged (no usable old_string)") + continue + merged[cid] = (dold, dnew.replace(kold, knew, 1)) + reasons.append( + f"merged dependent patch {cid[:14]} onto {did[:14]}'s " + f"pre-broken state") + + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if m.role == "tool" and m.tool_call_id in broken: + continue + if m.role == "assistant" and any( + tc.id in broken or tc.id in merged for tc in m.tool_calls): + kept: list[ToolCall] = [] + for tc in m.tool_calls: + if tc.id in broken: + reasons.append( + f"[{si}] dropped committed mistake {tc.id[:14]}: " + f"{broken[tc.id][:140]}") + continue + if tc.id in merged: + old, new = merged[tc.id] + args = _args_dict(tc) or {} + for k in _OLD_ARG_KEYS: + if k in args: + args[k] = old + break + for k in _NEW_ARG_KEYS: + if k in args: + args[k] = new + break + tc = ToolCall( + id=tc.id, name=tc.name, + arguments=json.dumps(args, ensure_ascii=False)) + kept.append(tc) + if kept or (m.content or "").strip(): + out.append(Message( + role=m.role, content=m.content, tool_calls=kept, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + omap.append(si) + continue + out.append(m) + omap.append(si) + return out, omap, reasons + + +# --------------------------------------- fabricated tool-result repair +# The writer sometimes invents a tool result instead of quoting the real +# one; validate_grounding catches it ("gate: fabricated tool result +# [patch]: result shares no content with any real patch output") but the +# repair round re-generates from the same source and reproduces it, so the +# trace dies. The real result is IN the source trace β€” substitute it. +# +# Measured 2026-10-24 (batch log): 20260926_205053_cbcf4d died with four +# fabricated-result gate errors in one round, 20260911_144552_6caec8 and +# 019fad3b-ce23-7de3-be93-2d0c5e437625 with three each. Substitution is +# exact (the row becomes the source's own bytes), so it cannot introduce a +# new grounding failure β€” the gate's own predicate is the detector. +def _arg_digest(name: str, arguments: Any) -> str: + """Stable fingerprint of (tool name, canonical arguments).""" + if isinstance(arguments, dict): + raw = json.dumps(arguments, sort_keys=True, ensure_ascii=False) + else: + d = json_loads_lenient(arguments or "") + raw = (json.dumps(d, sort_keys=True, ensure_ascii=False) + if isinstance(d, dict) else str(arguments or "")) + return hashlib.sha1( + f"{(name or '').strip().lower()}\x00{raw}".encode("utf-8") + ).hexdigest() + + +def _resubstitute_tool_results( + msgs: list[Message], src_map: list[int], source: Optional[Trace], +) -> tuple[list[Message], list[int], list[str]]: + """Replace fabricated tool results with the source trace's real ones. + + Detection is the grounding gate's own predicate (shares no substantive + line and no 25-char window with ANY real output of that tool), so this + pass can only fire on rows the gate would otherwise fail the trace for. + Matching is exact: tool_call_id first, then tool name + argument digest. + """ + if not msgs or source is None: + return msgs, src_map, [] + from traceforge.grounding import ( + _real_output_index, _result_grounded, _tool_output_lines, + ) + idx = _real_output_index(source) + if not idx: + return msgs, src_map, [] + by_id: dict[str, str] = {} + by_digest: dict[str, str] = {} + for m in source.messages: + for tc in m.tool_calls: + if not tc.id: + continue + res = next((r for r in source.messages + if r.role == "tool" and r.tool_call_id == tc.id), None) + body = (res.content or "") if res is not None else "" + if not body.strip(): + continue + by_id[tc.id] = body + by_digest.setdefault(_arg_digest(tc.name, tc.arguments), body) + if not by_id and not by_digest: + return msgs, src_map, [] + cand_calls = {tc.id: tc for m in msgs if m.role == "assistant" + for tc in m.tool_calls} + reasons: list[str] = [] + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if m.role != "tool": + out.append(m) + omap.append(si) + continue + body = m.content or "" + lines = _tool_output_lines(body) + cands = idx.get(m.tool_name or "", []) + if not lines or not cands: + out.append(m) + omap.append(si) + continue + if any(_result_grounded(lines, rl) for _s, rl in cands): + out.append(m) + omap.append(si) + continue + real = by_id.get(m.tool_call_id or "") + how = "tool_call_id" + if not real: + tc = cand_calls.get(m.tool_call_id or "") + if tc is not None: + real = by_digest.get(_arg_digest(tc.name, tc.arguments), "") + how = "name+argument digest" + if not real: + reasons.append( + f"[{si}] fabricated {m.tool_name} result left as-is: no " + f"source result matches by tool_call_id or argument digest") + out.append(m) + omap.append(si) + continue + reasons.append( + f"[{si}] substituted the real {m.tool_name} result " + f"(matched by {how}): {body.strip()[:60]!r} -> " + f"{real.strip()[:60]!r}") + out.append(Message(role=m.role, content=real, + tool_call_id=m.tool_call_id, + tool_name=m.tool_name)) + omap.append(si) + return out, omap, reasons + + +# ------------------------------------------- P5 prose-only translation +# The task says "Report ONLY in German" (or the reverse) and the source +# agent wrote the other language. The structural gate only checks the FINAL +# answer, so mid-trace prose in the wrong language survives every gate and +# the judge cites it as P5 forever β€” a window rewrite re-narrates in the +# same language, so the repair round cannot converge. +# +# Measured 2026-10-24: 44 failure records cite P5 language; it is the +# largest remaining class. 019f85fb#p3 (42 msgs) died with exactly one +# violation: "[6] P5: Assistant prose is in German while the first user +# message (the task) is in English". +# +# The transform is a translation, so it needs a model β€” but ONLY the prose +# spans go to it. Code fences, inline code, paths, URLs, identifiers, +# quoted output and hex ids are cut out first and re-inserted +# byte-identical, so P4 (grounding) and the tool-result fidelity gate +# cannot be touched by this pass. One batched call per trace, not a 12-18 +# window rewrite. +_PROTECTED_SPAN_RE = re.compile( + r"```.*?```" # fenced code + r"|`[^`\n]+`" # inline code + r"|!?\[(?=[^\]]{0,300}\]\()" # markdown link: opening bracket + r"|\]\([^)]*\)" # markdown link: ](url) β€” the + # DESCRIPTION between them is + # prose and must be translated + # (measured 019f85fb#p3: two + # German link descriptions + # survived as a P5 violation) + r"|https?://\S+" # URLs + r"|(?:[A-Za-z]:)?(?:/[\w.\-]+){2,}" # absolute paths + r"|[\w.\-]*\w(?:/[\w.\-]*\w)+" # relative paths (src/parser.py) + r"|[\w.\-]+\.(?:py|ts|tsx|js|mjs|go|rs|md|json|jsonl|ya?ml|sh|toml" + r"|txt|csv|sql|html|css|lock)\b" # filenames + r"|\b[A-Za-z_][A-Za-z0-9_]*(?:_[A-Za-z0-9_]+)+\b" # snake_case ids + r"|\b[a-z]+(?:[A-Z][a-z0-9]+)+\b" # camelCase ids + r"|\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b" # CamelCase ids + r"|\b[0-9a-f]{7,40}\b" # hex ids / commit shas + r"|\"[^\"\n]{1,160}\"" # quoted output + r"|\n[ \t]*[-*+] " # list bullets + r"|\n#{1,6} ", # headings + re.S, +) + +_TRANSLATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "translations": { + "type": "array", + "items": { + "type": "object", + "properties": {"id": {"type": "integer"}, + "text": {"type": "string"}}, + "required": ["id", "text"], + }, + } + }, + "required": ["translations"], +} + +_TRANSLATE_HEADER = """You translate the PROSE of an AI coding agent's \ +messages into {lang}. + +Rules: +- Each item is ONE message body. Protected material (code, paths, URLs, + identifiers, quoted output, markdown bullets) has been replaced by + placeholders of the form \u27e6N\u27e7. Copy every placeholder into your + translation EXACTLY as it appears, in the same order, once each. Never + translate, renumber, drop or duplicate a placeholder. +- Translate ALL remaining words into {lang}. Do not leave any sentence in + the source language, and do not paraphrase: translate faithfully. +- Keep markdown structure (headings, bullets, bold, emphasis) as it is. +- No explanations, notes or commentary. Do not merge or split items. +- Return one entry per input id, same ids, translated text only. + +Items (JSON): +{items} +""" + + +def _mask_protected(text: str) -> tuple[str, list[str]]: + """Replace every protected span with a \u27e6N\u27e7 placeholder.""" + spans: list[str] = [] + + def _sub(m: "re.Match[str]") -> str: + spans.append(m.group(0)) + return f"\u27e6{len(spans) - 1}\u27e7" + + return _PROTECTED_SPAN_RE.sub(_sub, text), spans + + +def _unmask_protected(masked: str, spans: list[str]) -> Optional[str]: + """Restore the placeholders; None when any is missing or duplicated.""" + seen = [int(x) for x in re.findall(r"\u27e6(\d+)\u27e7", masked)] + if sorted(seen) != list(range(len(spans))): + return None + return re.sub(r"\u27e6(\d+)\u27e7", lambda m: spans[int(m.group(1))], masked) + + +def _prose_spans(text: str) -> list[tuple[bool, str]]: + """Alternating (is_protected, chunk) segments of a message body.""" + out: list[tuple[bool, str]] = [] + pos = 0 + for m in _PROTECTED_SPAN_RE.finditer(text): + if m.start() > pos: + out.append((False, text[pos:m.start()])) + out.append((True, m.group(0))) + pos = m.end() + if pos < len(text): + out.append((False, text[pos:])) + return out + + +def _prose_only(text: str) -> str: + return " ".join(c for prot, c in _prose_spans(text) if not prot) + + +def _needs_translation(text: str, target: str) -> bool: + """True when the PROSE of a row is confidently in the other language.""" + prose = _prose_only(text) + if len(prose.strip()) < 40: + return False + lang = _confident_lang(prose, 4) + return bool(lang and lang != target) + + +def _translate_prose( + msgs: list[Message], src_map: list[int], task: str, + *, call: Optional[Any] = None, +) -> tuple[list[Message], list[int], list[str]]: + """Translate assistant prose into the trace's language, prose only. + + ``call`` is a ``(prompt) -> (parsed_dict, error)`` callable; when it is + None the pass is a no-op (unit tests and triage-disabled runs). + Idempotent: a translated row's prose is in the target language, so + ``_needs_translation`` is False and no call is made on the next round. + """ + if not msgs or call is None: + return msgs, src_map, [] + brief = "" + try: + brief = (Trace(session_id="p", messages=msgs).first_task_message() + or "") + except Exception: + brief = "" + if not brief.strip(): + brief = task or "" + target = _confident_lang(brief[:300], 3) + if target is None: + return msgs, src_map, [] + lang_name = {"en": "English", "de": "German"}.get(target, target) + + items: list[dict] = [] + plan: list[tuple[int, int, list[str]]] = [] + for i, m in enumerate(msgs): + if m.role != "assistant" or not (m.content or "").strip(): + continue + if not _needs_translation(m.content, target): + continue + masked, spans = _mask_protected(m.content) + if not re.search(r"[A-Za-zΓ„Γ–ΓœΓ€ΓΆΓΌΓŸ]{3}", masked): + continue + items.append({"id": len(items), "text": masked}) + plan.append((i, len(items) - 1, spans)) + if not items: + return msgs, src_map, [] + + data, err = call(_TRANSLATE_HEADER.format( + lang=lang_name, + items=json.dumps(items, ensure_ascii=False, indent=1))) + if not isinstance(data, dict): + return msgs, src_map, [ + f"P5 translation call failed ({err}); prose left in the source " + f"language"] + got: dict[int, str] = {} + for t in (data.get("translations") or []): + if not isinstance(t, dict): + continue + tid, txt = t.get("id"), t.get("text") + if isinstance(tid, (int, float)) and not isinstance(tid, bool) \ + and isinstance(txt, str): + got[int(tid)] = txt + + reasons: list[str] = [] + changed: dict[int, str] = {} + for mi, iid, spans in plan: + new = got.get(iid) + if not new or not new.strip(): + continue + restored = _unmask_protected(new, spans) + if restored is None: + reasons.append( + f"[{mi}] translation dropped or duplicated a protected " + f"span; kept original") + continue + # NOTE: no "invented identifier" guard. Measured on real rows, it + # fires on legitimate translation choices β€” 01a07a9f-4aa1 rendered + # "Discovery-/Join-Vertrag" as "Discovery/Join contract" and + # 019f85fb#p3 rendered a German compound as "PrivSep"; both were + # rejected and the row stayed German, which is exactly the P5 + # violation this pass exists to remove. The byte-identical + # guarantee comes from the placeholders, not from this check. + # it must not still be confidently the source language + if _confident_lang(_prose_only(restored), 4) not in (None, target): + reasons.append( + f"[{mi}] translation stayed in the source language; kept " + f"original") + continue + if restored != (msgs[mi].content or ""): + changed[mi] = restored + if not changed: + return msgs, src_map, reasons + out: list[Message] = [] + omap: list[int] = [] + for i, (m, si) in enumerate(zip(msgs, src_map)): + if i in changed: + reasons.append(f"[{si}] translated assistant prose to {target}") + out.append(Message( + role=m.role, content=changed[i], tool_calls=m.tool_calls, + tool_call_id=m.tool_call_id, tool_name=m.tool_name)) + omap.append(si) + continue + out.append(m) + omap.append(si) + return out, omap, reasons + + +def _make_translator(cfg: Config, base: str, key: str): + """Batched prose-translation callable for _translate_prose.""" + def _call(prompt: str) -> tuple[Optional[dict], Optional[str]]: + return _post_chat( + base, key, cfg.model or "", prompt, _TRANSLATE_SCHEMA, + max_tokens=cfg.rewrite_max_tokens, + timeout_s=cfg.rewrite_timeout_s, + temperature=0.0) + return _call + + +# ------------------------------------------------- gate repairs +# Harness plumbing that arrives as a user row: retry nudges, iteration +# caps, rejection notices, continuation prompts. They are not the task, and +# a trace that opens with one has no task at all β€” measured 2026-10-24 on +# 20260921_102859_4b18fe: the raw opens with "You've reached the maximum +# number of tool-calling iterations allowed. Please provide a final answer" +# (188 chars, so a length test alone does not catch it), and the leading +# cascade then dropped 147 msgs / 78 calls down to a 2-row stub that passed +# the gates. +_PLUMBING_USER_RE = re.compile( + r"(maximum number of tool[- ]calling iterations" + r"|please provide a final answer" + r"|previous response (?:was )?rejected" + r"|your (?:last )?response was rejected" + r"|continue from where you left off" + r"|^\s*(?:go on|continue|weiter|mach weiter|gut fang an)\s*[.!]?\s*$)", + re.IGNORECASE, +) + + +def _is_plumbing_user(content: str) -> bool: + """True when a user row is harness plumbing rather than a task.""" + return bool(_PLUMBING_USER_RE.search((content or "").strip()[:400])) + + +def _gate_repairs( + msgs: list[Message], src_map: list[int], task: str, +) -> tuple[list[Message], list[int], list[str]]: + """Deterministic repairs for the structural gate classes the splice + path used to fail outright. + + Measured 2026-10-23 on the empty-workdir sample (5 traces, 4 bands): + 3 of the 4 failures died at validate_structural, NOT at the judge β€” + * 20260819_131548_dac004 (301KB): "first message is assistant" (the + raw is a continuation segment that opens mid-run) + * 01a01e15-133b-7e81-b47b-e78778ac5e66 (180KB): "first assistant + turn is a 100% verbatim echo of the user's task" + * 01a061bf-0fef-7343-83eb-b4a247817fac (240KB): final answer in the + wrong language + Each is mechanical: an ideal trace opens with the task, does not + restate it, and answers in its language. The harness path repairs + these; the splice path had no equivalent and killed the trace. + """ + reasons: list[str] = [] + if not msgs: + return msgs, src_map, reasons + + # 0. a leading user row too short to be a task ("gut fang an", "go on") + # is a nudge, not the task. Leaving it makes the structural gate and + # the judge disagree about the trace's language: measured 01a061bf + # (240KB), the gate read first_task_message() = the English brief and + # demanded English, the judge read "first user message 'gut fang an'" + # and demanded German β€” no answer can satisfy both. Drop the nudge so + # both read the same row. + while (len(msgs) >= 2 and msgs[0].role == "user" + and (len((msgs[0].content or "").strip()) < 40 + or _is_plumbing_user(msgs[0].content or "")) + and any(m.role == "user" and len((m.content or "").strip()) >= 100 + for m in msgs[1:])): + reasons.append( + f"dropped leading nudge user row " + f"{(msgs[0].content or '').strip()[:30]!r}") + msgs = msgs[1:] + src_map = src_map[1:] + + # 1. leading non-user rows: an ideal trace opens with the task, + # optionally preceded by ONE system row. validate_structural already + # accepts "user or system" first (validate.py: "first message is + # assistant (expected user or system)"), so scanning from 0 here + # dropped a legitimate system prompt β€” that is how 12,878 nemotron + # raws lost the `## Available tools` block their reader had built. + start = 0 + if msgs and msgs[0].role == "system": + start = 1 + while start < len(msgs) and msgs[start].role != "user": + start += 1 + if 0 < start < len(msgs): + # A continuation segment can open with hundreds of assistant/tool + # rows. Dropping them ALL leaves a 2-row task+answer stub that + # PASSES the gates while carrying none of the work β€” measured + # 2026-10-24: 28 such artifacts in the promoted corpus (5 produced + # that day), e.g. 20260909_182039_266960 (157 msgs / 76 calls -> + # 2 msgs / 0 calls, empty task). Only drop a leading run that + # leaves a real trace behind; otherwise leave the rows in place so + # the structural gate rejects the trace honestly. + rest = msgs[start:] + first_user = (rest[0].content or "").strip() if rest else "" + if (len(rest) >= 4 and len(first_user) >= 40 + and not _is_plumbing_user(first_user)): + reasons.append(f"dropped {start} leading non-user row(s)") + if msgs[0].role == "system": + # keep the prompt, drop only the noise between it and the task + msgs = [msgs[0]] + rest + src_map = [src_map[0]] + src_map[start:] + else: + msgs = rest + src_map = src_map[start:] + else: + reasons.append( + f"kept {start} leading non-user row(s): dropping them would " + f"leave {len(rest)} row(s) with a {len(first_user)}-char task " + f"β€” letting the gate reject instead of emitting a stub") + + # 2. task echo: an assistant row that re-presents the task instead of + # doing work. Same ratios as validate_structural, but DROP the row + # instead of failing the trace. The brief comes from the CANDIDATE + # (post scaffold-strip), exactly like the gate reads it β€” comparing + # against the pre-strip task text drops the ratio below the gate's. + brief = "" + try: + brief = (Trace(session_id="p", messages=msgs).first_task_message() + or "").strip() + except Exception: + brief = (task or "").strip() + if not brief: + brief = (task or "").strip() + if len(brief) >= 60: + bn = _norm_for_similarity(brief) + out: list[Message] = [] + omap: list[int] = [] + for m, si in zip(msgs, src_map): + if m.role == "assistant" and not m.tool_calls: + body = (m.content or "").strip() + if body: + r1 = difflib.SequenceMatcher( + None, _norm_for_similarity(brief[:600]), + _norm_for_similarity(body[:600]), + autojunk=False).ratio() + if r1 >= 0.9 and len(body) >= 60: + reasons.append( + f"[{si}] dropped {r1:.0%} task echo") + continue + if len(brief) >= 200 and len(body) >= 200: + r2 = difflib.SequenceMatcher( + None, bn, _norm_for_similarity(body), + autojunk=False).ratio() + if r2 >= 0.85: + reasons.append( + f"[{si}] dropped {r2:.0%} brief echo") + continue + out.append(m) + omap.append(si) + msgs, src_map = out, omap + return msgs, src_map, reasons + + +# ---------------------------------------------------------------- engine +def _normalize_candidate( + msgs: list[Message], src_map: list[int], task: str, + *, + source: Optional[Trace] = None, + translate: Optional[Any] = None, +) -> tuple[list[Message], list[int], dict[str, list[str]]]: + """Run every deterministic repair in a fixed order. + + Order matters: empty rows first (they break pairing), then the two + failed-call classes (own-result error, then success-with-broken- + consequence), then no-op calls (they need their result row present to + be recognised), then fabricated-result substitution (only rows that + survive the strips are worth repairing), then prose dedupe (it needs + the surviving rows), then the P5 prose translation (after dedupe, so a + sentence that is about to be dropped is never paid for), then the gate + repairs (leading rows / task echo), and finally the same-role merge so + the result is gate-shaped. + """ + log: dict[str, list[str]] = {} + for name, fn in ( + ("empty_rows", lambda m, s: _drop_empty_rows(m, s)), + ("dup_calls", lambda m, s: _dedupe_calls(m, s)), + ("failed_calls", lambda m, s: _strip_failed_calls(m, s)), + ("committed_mistakes", + lambda m, s: _strip_committed_mistakes(m, s)), + ("noop_strip", lambda m, s: _strip_noop_calls(m, s)), + ("fabricated_results", + lambda m, s: _resubstitute_tool_results(m, s, source)), + ("prose_dedupe", lambda m, s: _dedupe_prose(m, s)), + ("language", lambda m, s: _translate_prose(m, s, task, call=translate)), + ("gate_repair", lambda m, s: _gate_repairs(m, s, task)), + ): + msgs, src_map, reasons = fn(msgs, src_map) + if reasons: + log[name] = reasons + msgs, src_map = _mechanical_fixes(msgs, src_map) + return msgs, src_map, log + + +def _passthrough(_: str) -> None: + pass + + +# ------------------------------------------------------- failure recording +# The splice engine writes NOTHING into the workdir until it succeeds (only +# improved.json at the very end), so a failed run used to leave a completely +# empty data/work// directory. Measured 2026-10-23: 2343 attempted +# traces had empty workdirs and the funnel could only COUNT them β€” the +# reason (which stage, which window, which violation) was never persisted +# anywhere durable. Every _fail() now lands here. +FAILURE_FILE = "failure.json" +FAILURE_LOG = "failures.jsonl" + + +def _record_failure( + cfg: Config, + sid: str, + *, + stage: str, + errors: list[str], + attempts: int, + duration_s: float, + route: Optional[str] = None, + windows: Optional[list[dict]] = None, + violations: Optional[list[dict]] = None, + extra: Optional[dict] = None, +) -> None: + """Persist WHY a splice attempt produced no artifact. + + failure.json = the LATEST verdict (what the funnel reads) + failures.jsonl = append-only history (how many times this trace has + failed, and whether the failing stage moves between + runs β€” a stage that never moves is a code defect, a + stage that moves is a hard trace) + """ + workdir = os.path.join(cfg.improver_workdir, sid) + try: + os.makedirs(workdir, exist_ok=True) + from traceforge.defects import classify + + row: dict[str, Any] = { + "session_id": sid, + "engine": "splice", + "stage": stage, + # Stable code so the funnel can be broken down by loss class + # instead of re-grouping free text on every report (R-34). + "defect": classify(stage, errors, violations), + "errors": [str(e)[:600] for e in (errors or [])[:20]], + "attempts": attempts, + "duration_s": round(duration_s, 1), + "route": route, + "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), + } + if windows: + row["windows_failed"] = windows[:20] + if violations: + row["violations"] = [ + {"i": v.get("i"), "pillar": v.get("pillar"), + "detail": str(v.get("detail", ""))[:600]} + for v in violations[:20] + ] + if extra: + row.update(extra) + with open(os.path.join(workdir, FAILURE_FILE), "w", + encoding="utf-8") as f: + json.dump(row, f, ensure_ascii=False, indent=1) + with open(os.path.join(workdir, FAILURE_LOG), "a", + encoding="utf-8") as f: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + except OSError: + pass # bookkeeping must never kill the run + + +def _clear_failure(cfg: Config, sid: str) -> None: + """Drop the stale failure.json after a success (history stays).""" + try: + os.remove(os.path.join(cfg.improver_workdir, sid, FAILURE_FILE)) + except OSError: + pass + + +def _raw_bytes(cfg: Config, sid: str) -> Optional[int]: + try: + return os.path.getsize(os.path.join(cfg.raw_dir, f"{sid}.json")) + except OSError: + return None + + +def improve_trace_splice( + cfg: Config, + trace: Trace, + *, + status_cb=None, + max_attempts: int = 3, +) -> dict: + """Windowed direct-HTTP improve of one trace. + + Returns {"ok", "improved", "errors", "attempts", "duration_s", + "route"}; on ok the workdir artifact improved.json is written and + "improved" carries the final Trace (promotion into improved_dir is + the caller's for the fresh-splice path, matching the agent-harness + contract; judge_artifact promotes directly). + """ + started = time.time() + sid = trace.session_id + from traceforge.grounding import validate_grounding + from traceforge.improver.harness import _append_status + from traceforge.triage import triage_trace + + # Stage tracking: every _fail() records the stage it died in, so the + # funnel can say WHY a workdir is empty instead of only counting it. + stage = {"name": "init"} + _cb = status_cb or _passthrough + + def _status(s: str) -> None: + stage["name"] = s + _cb(s) + + win_failures: list[dict] = [] + last_violations: list[dict] = [] + + def _fail(errors: list[str], attempts: int, route: str) -> dict: + _record_failure( + cfg, sid, stage=stage["name"], errors=errors, + attempts=attempts, duration_s=time.time() - started, + route=route, windows=win_failures, + violations=last_violations, + extra={"raw_bytes": _raw_bytes(cfg, sid), + "messages": len(trace.messages)}) + return {"ok": False, "improved": None, "errors": errors, + "attempts": attempts, "route": route, + "duration_s": time.time() - started} + + workdir = os.path.join(cfg.improver_workdir, sid) + os.makedirs(workdir, exist_ok=True) + output_path = os.path.join(workdir, "improved.json") + + # attribute this trace's HTTP usage to its sid for the live bar + SPLICE_USAGE.set_current(sid) + + # ---- stage 0: regate fast-path β€” an existing artifact is adjudicated + # as-is (gates locally, then the windowed judge); only unpromotable + # artifacts fall through to a fresh splice. + if os.path.exists(output_path): + _status("regate") + res0 = judge_artifact(cfg, sid, status_cb=status_cb) + if res0.get("ok"): + _clear_failure(cfg, sid) + return {"ok": True, "improved": res0["improved"], "errors": [], + "attempts": 0, "route": "regate", + "duration_s": time.time() - started} + _append_status(cfg, sid, { + "stage": "regate", "verdict": "FALLTHROUGH", + "error": "; ".join(res0.get("errors", []))[:300], + }) + + # ---- stage 0: free deterministic gates, BEFORE any LLM spend + task_msg_idx = _first_task_index(trace) + if task_msg_idx is None: + return _fail(["no user task message in source trace"], 0, "-") + if _task_is_plumbing(trace, task_msg_idx): + return _fail([ + "opening task is harness plumbing (validator-rejection row); " + "the real task lives in a previous session β€” not trainable"], + 0, "-") + + # ---- stage 1: triage (cached labels) + _status("triage") + if cfg.triage: + triage_errors: list[str] = [] + rec = triage_trace(cfg, trace, errors_out=triage_errors) + if rec is None: + return _fail(["triage failed for every window"] + triage_errors, + 0, "-") + labels = {int(l["i"]): l["action"] for l in rec.get("labels", [])} + route = rec.get("route", "agent") + else: + # triage disabled: mechanical plan only (keeps/drops), zero calls + rec = {"route": "agent", "labels": []} + labels = {} + route = "agent" + if route == "reject": + return _fail(["rejected: nothing survives triage"], 0, route) + + task = trace.messages[task_msg_idx].content or "(no user message)" + + plan, plan_drop_calls = _plan_from_labels(trace, labels) + base, key = _resolve_endpoint(cfg) + + # ---- stage 2: windowed rewrites (only windows that need edits) + windows = _windows(trace, cfg) + _status(f"windows {len(windows)}") + needs_rewrite: list[set[int]] = [] + for win in windows: + idxs = set(range(win["lo"], win["hi"])) + needs_rewrite.append(idxs if any( + plan.get(i, "keep") == "rewrite" for i in idxs + ) else set()) + + edits_all: list[list[dict]] = [[] for _ in windows] + errs: list[str] = [] + lock = threading.Lock() + + def _work(wi: int, win: dict) -> None: + user = REWRITE_HEADER.format( + lo=win["lo"] + 1, hi=win["hi"], n=len(trace.messages), + pillars=PILLARS, task=(task or "")[:2000], + ) + "\n" + win["rendered"] + data, err = _post_chat( + base, key, cfg.model or "", user, EDITS_SCHEMA, + max_tokens=cfg.rewrite_max_tokens, + timeout_s=cfg.rewrite_timeout_s, + temperature=cfg.splice_temperature, + ) + with lock: + if data is None: + errs.append(f"window {win['lo'] + 1}-{win['hi']}: {err}") + win_failures.append({ + "window": wi, "lo": win["lo"] + 1, "hi": win["hi"], + "error": str(err)[:300]}) + else: + edits_all[wi] = list(data.get("edits") or []) + + active = [(wi, w) for wi, w in enumerate(windows) if needs_rewrite[wi]] + if active: + _status(f"rewrite {len(active)}/{len(windows)} windows") + with ThreadPoolExecutor(max_workers=max(1, cfg.splice_workers)) as ex: + futs = [_submit_attributed(ex, _work, wi, w) for wi, w in active] + for f in as_completed(futs): + f.result() + if errs and not any(edits_all): + return _fail( + [f"rewrite windows failed: {e}" for e in errs[:5]], + 1, route) + + # merge edits: later windows win on overlap (same policy as triage) + merged: dict[int, dict] = {} + for wi in range(len(windows) - 1, -1, -1): + for e in edits_all[wi]: + i = e.get("i") + if isinstance(i, (int, float)) and not isinstance(i, bool): + merged.setdefault(int(i), e) + + # synthesized-final-answer edit, carried OUTSIDE the window merges + # so every later rebuild (main flow AND repair rounds) re-applies + # it. Without this, a repair rebuild resurrects the raw interrupt + # notice / JSON blob from source (5343b8: repair 2 re-introduced + # the notice the interrupt rewrite had replaced β€” judge then cited + # it again and the loop could never converge). + synth_final: dict[int, dict] = {} + appended_final: Optional[str] = None + + _status("splice") + synth_dc_edits = [ + {"i": oi, "action": "keep", "drop_calls": sorted(ids)} + for oi, ids in sorted(plan_drop_calls.items()) + ] + msgs, src_map, warnings = _apply_edits( + trace, plan, list(merged.values()) + synth_dc_edits) + if not any(m.role == "user" for m in msgs): + return _fail(["splice left no user message"], 1, route) + # deterministic pre-repair: empty rows, no-op calls, restated prose, + # leading rows / task echoes (see the block comments above). + _translate = _make_translator(cfg, base, key) + msgs, src_map, norm_log = _normalize_candidate( + msgs, src_map, task, source=trace, translate=_translate) + for _stage, _reasons in norm_log.items(): + _append_status(cfg, sid, { + "stage": _stage, "dropped": len(_reasons), + "reasons": _reasons[:6]}) + + # ---- interruption final: the raw trace was cut off mid-run (156/231 + # raws end with "Operation interrupted..." β€” the single largest + # quality killer in the corpus). The notice is NOT an answer; a + # trainee must not learn to close sessions with it. Replace with a + # synthesized answer grounded in the work that DID happen. + if msgs and msgs[-1].role == "tool" and len(msgs) >= 2: + # raw ended mid-call (session cut before the result/close): + # synthesize a final answer from the completed work and append + # it β€” an ideal trace must close on an assistant answer, and + # no rewrite path can ever fix a missing final row. + _status("final answer (tool-tail synth)") + ans, err = _synthesize_final_answer( + cfg, Trace(session_id="p", messages=msgs, improved=True), + task, base, key) + if ans: + msgs.append(Message(role="assistant", content=ans)) + appended_final = ans + else: + # synth failed (endpoint/timeout): fall back to trimming back + # to the last assistant PROSE row. Popping only the tool rows + # leaves the assistant that issued them with unanswered + # tool_calls β€” that trades a tool-tail error for a pairing + # error, so trim past the call row too. + while msgs and not ( + msgs[-1].role == "assistant" + and not msgs[-1].tool_calls + and (msgs[-1].content or "").strip() + ): + msgs.pop() + if msgs and msgs[-1].role == "assistant" \ + and _INTERRUPT_RE.match((msgs[-1].content or "").strip()): + _status("final answer (interrupt rewrite)") + notice = msgs[-1].content or "" + ans, err = _synthesize_final_answer( + cfg, Trace(session_id="p", messages=msgs, improved=True), + task, base, key, drop_last=True) + if ans is not None and not _INTERRUPT_RE.match(ans) \ + and not _is_json_blob_answer(ans) and len(ans) > 80: + msgs[-1] = Message(role="assistant", content=ans) + synth_final[src_map[-1]] = { + "i": src_map[-1], "action": "rewrite", "content": ans} + _append_status(cfg, sid, {"stage": "final_answer", + "interrupt_rewritten": True}) + # failure: keep the notice; the answer-quality judge FAILs it. + + # missing final answer (raw trace cut off mid-run) -> synthesize one + if msgs and msgs[-1].role != "assistant": + _status("final answer") + ans, err = _synthesize_final_answer( + cfg, Trace(session_id="p", messages=msgs, improved=True), + task, base, key) + if ans is None: + return _fail([f"final-answer synthesis failed: {err}"], + 1, route) + if _is_json_blob_answer(ans) \ + or _INTERRUPT_RE.match(ans.strip()): + # 565319: the synthesizer itself emitted a machine dump / + # harness notice. Keep it out of the ideal trace β€” keep the + # tail as-is and let the answer-quality judge FAIL honestly + # rather than training a blob close. + _append_status(cfg, sid, { + "stage": "final_answer", "rejected_blob": True}) + else: + msgs.append(Message(role="assistant", content=ans)) + src_map.append(src_map[-1] if src_map else 0) + # NOTE: no synth_final entry β€” an APPEND has no source row; + # repair rebuilds re-synthesize the missing final instead. + _append_status(cfg, sid, {"stage": "final_answer", + "synthesized": True}) + elif msgs and msgs[-1].role == "assistant" \ + and _is_json_blob_answer(msgs[-1].content or "") \ + and not _task_demands_json(task): + # raw HAS a final answer, but it is a raw JSON dump (machine + # envelope). A trainee must learn to close sessions with prose. + # Rewrite the envelope as grounded prose (de2f3c: the raw ends + # with {"result": "PASS β€” …", "scope_blockers": …} β€” the payload + # is real evidence, the envelope is not an answer). + _status("final answer (blob rewrite)") + # Feed the synthesis the tail WITHOUT the blob (the model would + # otherwise echo the envelope β€” measured: ans == original blob), + # plus the blob's own payload as evidence to convert to prose. + blob = msgs[-1].content or "" + ans, err = _synthesize_final_answer( + cfg, Trace(session_id="p", + messages=msgs[:-1] + [Message( + role="assistant", + content=blob[:4000])], + improved=True), + task, base, key, drop_last=True) + if ans is not None and not _is_json_blob_answer(ans) \ + and len(ans) > len(blob) // 4: + msgs[-1] = Message(role="assistant", content=ans) + synth_final[src_map[-1]] = { + "i": src_map[-1], "action": "rewrite", "content": ans} + _append_status(cfg, sid, {"stage": "final_answer", + "blob_rewritten": True}) + # synthesis failure or another blob: keep the original; the + # json-blob gate in judge_trace will FAIL it honestly. + + msgs, src_map = _mechanical_fixes(msgs, src_map) + candidate = Trace(session_id="improved-placeholder", + messages=msgs, improved=True) + + # ---- pre-judge reframe: the task demands a written deliverable + # (tests/file) the trace never produced β€” no prose rewrite can make + # an ideal trace out of it, the judge will always cite P6/ANSWER. + # Measured: 31/231 raws are exactly this shape (tight same-sentence + # deliverable demand + zero write/edit calls; 5343b8: task says + # "Write tests as NEW file …", the session died mid-review with 13 + # read-only calls). Reframe the task to what the work delivers. + task, reframed = _reframe_undelivered_task( + cfg, sid, trace, candidate, task, _status, _append_status, + base, key) + + # ---- stage 3: gates + windowed judge + repairs + attempts = 0 + violations: list[dict] = [] + while attempts < max_attempts: + attempts += 1 + # Pipeline framing markers ("[trace segment k of n]", "[subagent + # brief β€” role: ...]") belong to the improver's INPUT. Strip them + # before the gates so the echo check compares against the real task + # and the promoted artifact carries no scaffolding. Runs every + # round because a window repair rebuilds the candidate. + from traceforge.validate import strip_scaffold_markers + strip_scaffold_markers(candidate) + # deterministic gate repairs (empty rows, no-op calls, restated + # prose, leading non-user rows, task echoes) run every round: a + # window repair rebuilds the candidate from source and can + # resurrect all of them. + _gm, src_map, norm_log = _normalize_candidate( + candidate.messages, src_map, task, + source=trace, translate=_translate) + if norm_log: + candidate.messages = _gm + _append_status(cfg, sid, { + "stage": "normalize", "round": attempts, + "stages": {k: len(v) for k, v in norm_log.items()}}) + serrs = validate_structural(candidate) + # Multi-turn preservation: a substantive user turn is part of the + # conversation, not dead weight. The rewrite prompt lets the model + # drop non-opening user rows as "corrections the ideal path routes + # around", and it generalises that to every follow-up turn β€” + # measured 2026-10-22: 41 of 123 promoted multi-turn traces lost + # their extra turns, including a 10-turn conversation flattened to + # one task. The prompt rule alone does not hold; this gate does. + raw_turns = substantive_user_turns(trace) + cand_turns = substantive_user_turns(candidate) + if cand_turns < raw_turns: + serrs = serrs + [ + f"multi-turn: {raw_turns - cand_turns} substantive user " + f"turn(s) dropped ({raw_turns} in source, {cand_turns} " + f"kept) β€” follow-up user turns are the conversation, keep " + f"them verbatim"] + gerrs = serrs or validate_grounding(candidate, trace) + # identity/refusal gate: the reference must burn in neither a + # vendor identity nor refusal phrasing β€” repair or drop the row + ierrs = validate_identity(candidate.messages) if not gerrs else [] + gerrs = gerrs or ierrs + if gerrs: + _status(f"gates fail ({attempts})") + # Record WHY the gate failed even when a repair is attempted β€” + # otherwise a trace that burns 20 minutes in a repair round and + # then dies leaves no trace of the gate error (measured + # 20260925_015550_418ad9: 20 min in "gates fail (1)" with no + # manifest row naming the error). + _append_status(cfg, sid, { + "stage": "gate", "round": attempts, + "errors": [str(e)[:300] for e in gerrs[:6]], + }) + # tool-tail: the writer stopped mid-call, so the artifact is one + # row short of valid. A window repair rewrites the whole trace + # and usually reproduces the same cut β€” close it with a + # synthesized answer instead (cheap, targeted, still judged). + if all("last message role is 'tool'" in e for e in gerrs): + ans, _e = _synthesize_final_answer( + cfg, candidate, task or "", base, key) + if ans: + candidate.messages.append( + Message(role="assistant", content=ans)) + _status("final answer (tool-tail synth)") + continue + # P5 language mismatch: no window rewrite can change the + # language of the close β€” re-synthesize it with an explicit + # instruction (measured 01a061bf, 240KB: "final answer is in + # de but the task is in en" killed the trace outright). + m_lang = re.search( + r"final answer is in (\w+) but the task is in (\w+)", + " ".join(gerrs)) + if m_lang and attempts < max_attempts: + _status("final answer (language repair)") + last_asst = next( + (i for i in range(len(candidate.messages) - 1, -1, -1) + if candidate.messages[i].role == "assistant"), None) + ans, _e = _synthesize_final_answer( + cfg, candidate, task or "", base, key, + drop_last=True, lang=m_lang.group(2)) + if ans and not _is_json_blob_answer(ans) \ + and not _INTERRUPT_RE.match(ans.strip()) \ + and last_asst is not None: + candidate.messages[last_asst] = Message( + role="assistant", content=ans) + _append_status(cfg, sid, { + "stage": "final_answer", "language_repaired": True, + "lang": m_lang.group(2)}) + continue + if attempts >= max_attempts: + # the repair's result would never be judged (the loop + # exits right after it) β€” a full 12-15 window rewrite + # round paid for nothing. Fail with the real reason. + return _fail( + [f"gate: {e}" for e in gerrs[:6]], attempts, route) + fixed = _repair_windows( + cfg, trace, plan, src_map, + cited_text="\n".join(f"- {e}" for e in gerrs[:12]), + cited=set(), base=base, key=key, task=task, + synth_final=synth_final, appended_final=appended_final) + if fixed is None: + return _fail( + [f"gate: {e}" for e in gerrs[:6]], attempts, route) + candidate, src_map, plan = fixed + continue + + _status("judge") + verdict, violations, answer_quality = judge_trace( + cfg, candidate, task, base, key) + last_violations = list(violations) + # A FAIL used to leave NO manifest row at all (only PASS was + # recorded), so the funnel saw 1460 PASS / 3 FAIL and could not + # tell that the judge loop is where the empty-workdir class dies. + _append_status(cfg, sid, { + "stage": "judge", "round": attempts, "verdict": verdict, + "violations": len(violations), + "answer_quality": answer_quality}) + if verdict == "ERROR": + return _fail(["judge infra error: " + "; ".join( + v.get("detail", "") for v in violations[:3])], + attempts, route) + if verdict == "PASS": + from traceforge.improver.harness import ( + _artifact_digest, _cache_gate_pass, _load_gate_cache, + _save_gate_cache, + ) + digest = _artifact_digest(candidate) + cache = _load_gate_cache(workdir) + _cache_gate_pass(cache, digest, "judge", 1) + _save_gate_cache(workdir, cache) + _append_status(cfg, sid, { + "stage": "judge", "round": 1, "verdict": "PASS"}) + candidate.session_id = f"ideal-{sid}" + candidate.parent_session_id = sid + candidate.model = cfg.model or "splice-http" + candidate.source = "traceforge-splice" + candidate.save(output_path) + _clear_failure(cfg, sid) + return {"ok": True, "improved": candidate, "errors": [], + "attempts": attempts, "route": route, + "duration_s": time.time() - started} + + # FAIL β€” INTENT mismatch + good answer -> task reframe (once). + # INTENT may coexist with other pillars (13/13 measured run8 + # INTENT failures were mixed, so the old all-INTENT rule NEVER + # fired): repair rounds fix the co-cited P-defects but no + # content edit can fix a wrong TASK. Reframe first when the + # answer itself is decent, then let repairs handle the rest. + if (violations + and any(v.get("pillar") == "INTENT" for v in violations) + and cfg.allow_task_reframe and not reframed + and answer_quality in ("high", "ok", None)): + _status("reframe task") + final = next((m.content for m in reversed(candidate.messages) + if m.role == "assistant"), "") + data, err = _post_chat( + base, key, cfg.model or "", + REFRAME_HEADER.format(task=(task or "")[:4000], + answer=(final or "")[:4000]), + REFRAME_SCHEMA, max_tokens=1000, + timeout_s=cfg.rewrite_timeout_s, + temperature=cfg.splice_temperature) + if data and isinstance(data.get("task"), str) \ + and data["task"].strip(): + new_task = _sanitize_task_text(data["task"].strip()) + ok, why = _reframe_ok(task, new_task, final) + if not ok: + _append_status(cfg, sid, { + "stage": "reframe", "rejected": why, + "old_task": (task or "")[:200], + "new_task": (new_task or "")[:200]}) + return _fail([f"reframe rejected: {why}"] + [ + v.get("detail", "") for v in violations[:6]], + attempts, route) + _append_status(cfg, sid, { + "stage": "reframe", "old_task": (task or "")[:200], + "new_task": new_task[:200]}) + for m in candidate.messages: + if m.role == "user": + m.content = new_task + break + task = new_task + reframed = True + continue + return _fail([f"reframe failed: {err}"] + [ + v.get("detail", "") for v in violations[:6]], + attempts, route) + + # P6 refusal / poor answer: the close is the ONE row the engine can + # regenerate for the price of a single call, while a window repair + # costs 12-18 rewrites and cannot change the answer's stance. + # Measured 20260901_173131_98f97f (182KB): the judge cited "[66] P6: + # Final answer refuses to deliver findings … 'must be inspected + # directly'" and the trace died with an empty workdir. + if (attempts < max_attempts + and (answer_quality == "poor" + or any(v.get("pillar") in ("P6", "ANSWER") + for v in violations))): + _status("final answer (refusal repair)") + ans, _e = _synthesize_final_answer( + cfg, candidate, task or "", base, key, drop_last=True) + if ans: + _replace_final_answer(candidate, ans) + # the text, not True: _finalize compares and re-appends it + # with .strip() after a repair rebuild (a bool crashes there) + appended_final = ans + continue + + # pillar violations -> window-scoped repair + _status(f"repair {attempts}") + cited = {int(v["i"]) for v in violations + if isinstance(v.get("i"), (int, float))} + det = "\n".join( + f"- [{v.get('i')}] {v.get('pillar')}: {v.get('detail')}" + for v in violations[:12]) + if attempts >= max_attempts: + # no round left to judge the repair β€” don't pay for it + return _fail(["max repair rounds exceeded"] + [ + f"[{v.get('i')}] {v.get('pillar')}: {v.get('detail')}" + for v in violations[:10]], attempts, route) + fixed = _repair_windows( + cfg, trace, plan, src_map, + cited_text=det, cited=cited, base=base, key=key, task=task, + synth_final=synth_final, appended_final=appended_final) + if fixed is None: + return _fail(["judge violations:"] + [ + f"[{v.get('i')}] {v.get('pillar')}: {v.get('detail')}" + for v in violations[:10]], attempts, route) + candidate, src_map, plan = fixed + + return _fail(["max repair rounds exceeded"] + [ + f"[{v.get('i')}] {v.get('pillar')}: {v.get('detail')}" + for v in violations[:10]], attempts, route) + + +def _replace_final_answer(candidate: Trace, ans: str) -> None: + """Swap the closing assistant row for a regenerated answer. + + ``_synthesize_final_answer(drop_last=True)`` only trims the PROMPT + tail β€” it does not touch ``candidate.messages``. Appending straight + after it leaves two consecutive text-only assistant turns, which the + structural gate rejects ("[68] two consecutive text-only assistant + turns", measured 20260901_173131_98f97f). Pop the trailing assistant + text rows first. + """ + while (candidate.messages + and candidate.messages[-1].role == "assistant" + and not candidate.messages[-1].tool_calls): + candidate.messages.pop() + candidate.messages.append(Message(role="assistant", content=ans)) + + +def _synthesize_final_answer( + cfg: Config, candidate: Trace, task: str, base: str, key: str, + *, drop_last: bool = False, lang: Optional[str] = None, +) -> tuple[Optional[str], Optional[str]]: + tail = candidate.messages[-8:] + if drop_last and tail and tail[-1].role == "assistant": + tail = tail[:-1] # blob rewrite: don't let the model echo the blob + rendered = Trace( + session_id="p", + messages=tail, + improved=True, + ).render_compact(**_render_budget(cfg)) + header = FINAL_ANSWER_HEADER.format( + task=(task or "")[:2000], tail=rendered) + if lang: + # the model already answered in the wrong language once; say it + # explicitly instead of relying on "match the task's language" + header += (f"\n\nLANGUAGE: write the answer in {lang!r} β€” the " + f"task's language. Do not switch language.") + data, err = _post_chat( + base, key, cfg.model or "", header, + FINAL_ANSWER_SCHEMA, max_tokens=3000, + timeout_s=cfg.rewrite_timeout_s, + temperature=cfg.splice_temperature) + if data and isinstance(data.get("answer"), str) \ + and data["answer"].strip(): + return data["answer"].strip(), None + return None, err or "empty answer" + + + + +def _repair_global_plan( + cfg: Config, + trace: Trace, + *, + cited_text: str, + src_cited: set[int], + base: str, + key: str, + task: str, +) -> dict[int, dict]: + """Cross-window repair coordination (2026-10-05): a defect often + spans a window boundary β€” the failed call lives in window A, its + paired result and the follow-up repair turns in window B. Per-window + repairs can never fix that (each sees half the defect) and rounds + exhaust. ONE planning call sees ALL cited windows at once (compact + digest) and emits coordinated edits with ABSOLUTE source indices.""" + if len(src_cited) < 2: + return {} + lines = [] + for i in sorted(src_cited): + if i >= len(trace.messages): + continue + m = trace.messages[i] + head = (m.content or "")[:100].replace("\n", " ") + calls = ",".join(tc.name or "?" for tc in (m.tool_calls or [])) + tag = f" calls=[{calls}]" if calls else "" + lines.append(f"[{i}] {m.role}{tag}: {head}") + digest = "\n".join(lines) + if len(digest) > 8000: + digest = digest[:8000] + user = REPAIR_PLAN_HEADER.format( + violations=cited_text, task=(task or "")[:2000], + pillars=PILLARS, digest=digest) + data, err = _post_chat( + base, key, cfg.model or "", user, EDITS_SCHEMA, + max_tokens=cfg.rewrite_max_tokens, + timeout_s=cfg.rewrite_timeout_s, + temperature=cfg.splice_temperature) + if not data: + return {} + out: dict[int, dict] = {} + for e in (data.get("edits") or []): + i = e.get("i") + if isinstance(i, (int, float)) and not isinstance(i, bool) \ + and 0 <= int(i) < len(trace.messages): + out[int(i)] = e + return out + +# Vocabulary the repair prompt itself uses for "this row should not +# exist". When the repair model returns NO edits for such a finding, the +# drop it was asked for is applied deterministically instead of killing +# the trace (measured 2026-10-23: 20261013_095155_2dcd99 died with +# "judge violations: [11] P2 … first patch call resulted in a Rust +# compiler error" because _repair_windows returned None). +_DROP_HINT_RE = re.compile( + r"\b(exploratory|redundant|unused|never used|not used|failed|failure|" + r"no-?op|filler|duplicat|restat|unnecessary|does not advance|" + r"doesn't advance|no information|adds no|pointless|superfluous|" + r"no_change|identical)\b", + re.IGNORECASE, +) + + +def _fallback_drop_edits( + trace: Trace, src_cited: set[int], cited_text: str, +) -> dict[int, dict]: + """Drop the cited rows the repair prompt says should not exist. + + Only fires when the repair model produced no edits at all, and only + for rows that are safe to remove: a prose-only assistant turn or a + tool result (whose owning call the cascade removes). A cited turn + that carries tool calls AND prose is left alone β€” dropping it would + lose real work. + """ + if not _DROP_HINT_RE.search(cited_text or ""): + return {} + edits: dict[int, dict] = {} + for si in sorted(src_cited): + if not (0 <= si < len(trace.messages)): + continue + m = trace.messages[si] + if m.role == "assistant" and not m.tool_calls: + edits[si] = {"i": si, "action": "drop"} + elif m.role == "tool": + edits[si] = {"i": si, "action": "drop"} + return edits + + +def _repair_windows( + cfg: Config, + trace: Trace, + plan: dict[int, str], + src_map: list[int], + *, + cited_text: str, + cited: set[int], + base: str, + key: str, + task: str, + synth_final: Optional[dict[int, dict]] = None, + appended_final: Optional[str] = None, +) -> Optional[tuple[Trace, list[int], dict[int, str]]]: + """Re-edit the SOURCE windows containing the cited IDEAL indices. + + cited may be empty (gate errors cite only via text; then every + window whose rows appear in the text is repaired, plus as a + fallback all rewrite windows). Returns the new + (candidate, src_map, plan) or None when nothing was repairable. + """ + src_cited: set[int] = set() + for ii in cited: + if 0 <= ii < len(src_map): + src_cited.add(src_map[ii]) + for m in re.finditer(r"\[(\d+)\]", cited_text): + ii = int(m.group(1)) + if ii < len(src_map): + src_cited.add(src_map[ii]) + if not src_cited and cited: + return None + + windows = _windows(trace, cfg) + new_plan = dict(plan) + # cross-window coordination first: the global planner sees every + # cited row and emits edits as coordinated units; per-window calls + # then fill gaps (its edits win only where the planner is silent). + edits: dict[int, dict] = _repair_global_plan( + cfg, trace, cited_text=cited_text, src_cited=src_cited, + base=base, key=key, task=task) + for win in windows: + idxs = set(range(win["lo"], win["hi"])) + target = (idxs & src_cited) if src_cited else idxs + if not target: + continue + if not src_cited and not any( + new_plan.get(i, "keep") == "rewrite" for i in idxs + ): + continue # nothing this window can fix + user = REPAIR_HEADER.format( + violations=cited_text, task=(task or "")[:2000], + pillars=PILLARS) + "\n" + win["rendered"] + data, err = _post_chat( + base, key, cfg.model or "", user, EDITS_SCHEMA, + max_tokens=cfg.rewrite_max_tokens, + timeout_s=cfg.rewrite_timeout_s, + temperature=cfg.splice_temperature) + if data is None: + continue + for e in (data.get("edits") or []): + i = e.get("i") + if isinstance(i, (int, float)) and not isinstance(i, bool) \ + and int(i) in idxs: + # planner edits win β€” they see the whole cross-window + # defect; this call only fills rows the planner skipped + edits.setdefault(int(i), e) + if not edits: + edits = _fallback_drop_edits(trace, src_cited, cited_text) + if not edits: + return None + for i, e in edits.items(): + act = str(e.get("action", "")).strip().lower() + if act in ("keep", "drop", "rewrite"): + new_plan[i] = act + msgs, sm, _w = _apply_edits( + trace, new_plan, + list((synth_final or {}).values()) + + list(edits.values())) + # a repair rebuild can resurrect rows the first pass stripped + msgs, sm, _norm = _normalize_candidate( + msgs, sm, task, source=trace, + translate=_make_translator(cfg, base, key)) + if appended_final: + # tool-tail synthesized answer has no source row β€” a rebuild + # from source loses it unless re-appended (deeffb: synth ran, + # repair rebuilt, gate failed 'last message role is tool'). + while msgs and msgs[-1].role == "tool": + msgs.pop() + if not msgs or msgs[-1].role != "assistant" \ + or (msgs[-1].content or "").strip() != appended_final.strip(): + msgs.append(Message(role="assistant", content=appended_final)) + # synth_final goes FIRST in the edits list: _apply_edits uses + # by_idx.setdefault (first edit per row wins), and the + # synthesized final answer must NOT be shadowed by an LLM edit + # on the same source row (5343b8: the repair rebuild resurrected + # the raw interrupt notice; with priority the rebuild re-applies + # the synthesized answer instead). + if synth_final: + for i in synth_final: + if not any(e.get("i") == i for e in edits.values()): + new_plan[i] = "rewrite" + if msgs and msgs[-1].role != "assistant": + ans, _err = _synthesize_final_answer( + cfg, Trace(session_id="p", messages=msgs, improved=True), + task, base, key) + if ans is not None: + msgs.append(Message(role="assistant", content=ans)) + sm.append(sm[-1] if sm else 0) + msgs, sm = _mechanical_fixes(msgs, sm) + if not msgs: + return None + return (Trace(session_id="improved-placeholder", messages=msgs, + improved=True), sm, new_plan) + + +# Praise/non-defeat phrases that appear in "violations" entries when a +# model wants to explain why something is FINE. A violation whose detail +# carries no negative signal is dropped β€” praise is not a defect. +# 156/231 raw traces end with an "Operation interrupted…" harness +# notice instead of an answer (measured 2026-10-05). Such a final is +# never an acceptable close for an ideal trace. +_INTERRUPT_RE = re.compile(r"^Operation interrupted", re.IGNORECASE) +_PRAISE_RE = re.compile( + r"\b(no violations?|no violation|none found|no issues?|nothing|" + r"correctly|clearly fine|is fine|are fine|well-?formed|" + r"appropriately|properly|grounded in|addresses the task|" + r"answered correctly|completely addresses|no errors?|no problems?|" + r"flawless|good|excellent|adequately answers)\b", + re.IGNORECASE, +) +_NEGATIVE_RE = re.compile( + r"\b(violat|missing|wrong|incorrect|invented|fabricat|ungrounded|" + r"unsupported|failed|failure|broken|unclear|off-topic|drifted|" + r"redundant|filler|fluff|grammar|mistake|refus|too (long|short|" + r"verbose)|not (grounded|supported|necessary|answered)|" + r"lacks?|contradict|mismatch|truncated|clip marker)\b", + re.IGNORECASE, +) + + +def _is_real_violation(v: dict) -> bool: + """Filter praise entries out of judge violation arrays. + + Primary signal: the judge's own "defect" classification (True = it + flagged a real defect; False = it wrote praise/justification into + the array anyway). Backstop for models that omit "defect": praise- + only wording is dropped; negative wording survives; a detail with + neither praise nor negative words survives (defect stated plainly). + """ + if v.get("defect") is True: + return True + if v.get("defect") is False: + return False + detail = str(v.get("detail", "")) + if _NEGATIVE_RE.search(detail): + return True + return not bool(_PRAISE_RE.search(detail)) + + +_TASK_JSON_RE = re.compile( + r"(return|output|respond|reply|antworten?|antworte|liefere|gib)" + r"[^\n.]{0,80}(json)|(json\s*(object|output|format|only)|" + r"valid\s+json|nur\s+json)" + r"|(write|schreibe)[^\n.]{0,60}\bjson\b" + r"|\bjson\s+(to|nach|to\\s+write)\s+[/~][^\s]", re.IGNORECASE) + + +def _task_demands_json(task: str) -> bool: + """The task's output contract explicitly demands JSON β€” then a JSON + final is the DELIVERABLE, not a machine-envelope defect (30067a: + 'Return ONLY valid JSON with keys: passed, security_concerns…').""" + return bool(_TASK_JSON_RE.search((task or "")[:2000])) + + +def _is_json_blob_answer(content: str) -> bool: + """A final answer that is a raw JSON dump is not an answer. + + Measured in the 2026-10-04 cross-exam: a promoted trace ended with + a 7.5KB unexplained JSON plan blob β€” a trainee would learn to close + sessions with machine dumps instead of prose. A good final answer + may CONTAIN json in a code fence, but must not BE json. + """ + c = (content or "").strip() + if len(c) < 400: + return False + if c.startswith("```"): + return False # fenced: prose around it may explain + if not (c.startswith("{") or c.startswith("[")): + return False + try: + json.loads(c) + return True + except Exception: + return False + + +def _merge_judge_results( + results: list[Optional[dict]], +) -> tuple[str, list[dict], Optional[str]]: + """Merge per-window judge outputs into (verdict, violations, + answer_quality). Praise-entries are filtered; the verdict derives + from REAL violations only (a window that says FAIL but cites only + praise counts as PASS); a window that failed to return data is + tolerated (its siblings adjudicate).""" + violations: list[dict] = [] + seen: set[tuple] = set() + answer_quality: Optional[str] = None + ok_windows = 0 + for r in results: + if not r or "_err" in r: + continue + ok_windows += 1 + if isinstance(r.get("answer_quality"), str): + answer_quality = r["answer_quality"] + for v in (r.get("violations") or []): + if not isinstance(v, dict): + continue + if not _is_real_violation(v): + continue # praise, not a defect + kt = (v.get("i"), str(v.get("pillar"))) + if kt in seen: + continue + seen.add(kt) + violations.append(v) + if not ok_windows: + return "ERROR", [], None + # final-window answer quality is binding: poor (or missing) fails + if answer_quality in (None, "poor"): + violations.append({ + "i": -1, "pillar": "ANSWER", + "detail": "final answer does not adequately answer the task " + f"(answer_quality={answer_quality!r})", + }) + verdict = "FAIL" if violations else "PASS" + return verdict, violations, answer_quality + +# Judge verdicts, keyed by sha256(model + full prompt). A repair round +# rewrites one window and re-judges every window, so without this the same +# unchanged windows are sent to the judge again on each round β€” and the judge +# is the stage that sets throughput. Process-local: the batch runs one process +# for hours, which is the whole lifetime that matters here. +_JUDGE_CACHE: dict[str, dict] = {} +_JUDGE_STATS = {"hit": 0, "miss": 0} + + +def _judge_cache_key(model: str, header: str) -> str: + return hashlib.sha256( + (model + "\x00" + header).encode("utf-8", "replace")).hexdigest() + + +def _judge_window_key(model: str, rendered: str, task: str, + is_last: bool) -> str: + """Cache key for one judged window. + + Content, not position: the rendered header carries lo/hi/n, so keying on + it meant a repair that dropped or inserted a row shifted every window's + indices and invalidated the whole cache (measured 13% hit rate). + `is_last` stays because the final window is judged against a different + contract β€” answer_quality is required there. + """ + return _judge_cache_key( + model, + rendered + "\x00" + task[:2000] + "\x00" + + ("last" if is_last else "mid")) + + +def judge_cache_stats() -> dict: + """Hit/miss counters, for the cycle report.""" + return dict(_JUDGE_STATS) + + +def judge_trace( + cfg: Config, + ideal: Trace, + task: str, + base: str, + key: str, + *, + model: Optional[str] = None, +) -> tuple[str, list[dict], Optional[str]]: + """Windowed pillar+intent judge over the IDEAL trace. + Returns (verdict, violations, answer_quality). + + The final window's judge call REQUIRES answer_quality; "poor" (or a + missing assessment) becomes an auto-FAIL violation β€” a trace whose + final answer does not answer the task is never promotable, no + matter how clean its windows are. + + model overrides cfg.judge_model (cross-examination: the SAME + windows judged by a DIFFERENT model family expose same-family + blind spots the primary judge inherits). + """ + # judges see the reasoning channel: decision logic lives there in + # 107/301 promoted traces; a content-only render makes judges call + # coherent traces "no reasoning" (crossexam false-positive class) + windows = _windows(ideal, cfg, include_reasoning=True) + results: list[Optional[dict]] = [None] * len(windows) + lock = threading.Lock() + n = len(windows) + + def _work(wi: int, win: dict) -> None: + is_last = wi == n - 1 + schema = JUDGE_SCHEMA + if is_last: + schema = dict(JUDGE_SCHEMA) + schema["required"] = ["verdict", "violations", + "answer_quality"] + header = JUDGE_HEADER.format( + pillars=PILLARS, task=(task or "")[:2000], + lo=win["lo"] + 1, hi=win["hi"], n=len(ideal.messages), + window=win["rendered"], + answer_note=ANSWER_NOTE if is_last else "", + ) + # A repair round rewrites ONE window and re-judges all of them. The + # untouched windows are byte-identical to the previous round's, so + # their verdict is already known: re-asking the judge is pure latency + # on the stage that is the throughput bottleneck. + # + # Key on CONTENT, not on the rendered header. The header carries + # lo/hi/n, so a repair that drops or inserts a row shifts every + # window's indices and invalidated the whole cache β€” measured 13% + # hit rate, which is why this is keyed on the window text instead. + # `is_last` stays in the key because the final window is judged + # against a different contract (answer_quality is required there). + ck = _judge_window_key( + model or cfg.judge_model or cfg.model or "", + win["rendered"], task or "", is_last) + hit = _JUDGE_CACHE.get(ck) + if hit is not None: + with lock: + _JUDGE_STATS["hit"] += 1 + results[wi] = hit + return + data, err = _post_chat( + base, key, model or cfg.judge_model or cfg.model or "", header, + schema, max_tokens=cfg.judge_max_tokens, + timeout_s=cfg.judge_timeout_s, + temperature=cfg.splice_temperature) + with lock: + _JUDGE_STATS["miss"] += 1 + if data: + _JUDGE_CACHE[ck] = data + results[wi] = data if data else {"_err": err} + + with ThreadPoolExecutor(max_workers=max(1, cfg.splice_workers)) as ex: + futs = [_submit_attributed(ex, _work, wi, w) + for wi, w in enumerate(windows)] + for f in as_completed(futs): + f.result() + + verdict, violations, aq = _merge_judge_results(results) + # Mechanical JSON-blob gate: a final answer that IS a raw JSON dump + # fails regardless of what the window judge said (the cross-exam + # proved the judge can call such dumps "high" quality) β€” UNLESS the + # task's output contract explicitly demands JSON (30067a: 'Return + # ONLY valid JSON with keys…' β€” then the JSON IS the deliverable). + if verdict == "PASS": + final = next((m.content for m in reversed(ideal.messages) + if m.role == "assistant" and (m.content or "").strip()), + "") + if _is_json_blob_answer(final or "") \ + and not _task_demands_json(task): + violations.append({ + "i": -1, "pillar": "ANSWER", + "detail": "final answer is a raw JSON dump, not a " + "human-facing answer", + "defect": True}) + verdict = "FAIL" + elif _INTERRUPT_RE.match((final or "").strip()): + violations.append({ + "i": -1, "pillar": "ANSWER", + "detail": "final message is a harness interruption " + "notice ('Operation interrupted…'), not an " + "answer", + "defect": True}) + verdict = "FAIL" + + # cross-family confirmation (residual-defect squeeze): a second + # selfhosted family re-judges the FINAL window β€” the answer is + # where residual defects concentrate (crossexam: ~5% of them). + # VETO fails the trace honestly; it re-enters via the retry queue. + if verdict == "PASS" and cfg.confirm_model: + cw = [w for w in windows if w["hi"] >= len(ideal.messages)] + if not cw: + cw = windows[-1:] + user = JUDGE_HEADER.format( + pillars=PILLARS, task=(task or "")[:2000], + lo=cw[-1]["lo"] + 1, hi=cw[-1]["hi"], n=len(ideal.messages), + window=cw[-1]["rendered"], + answer_note=ANSWER_NOTE, + ) + data, _err = _post_chat( + base, key, cfg.confirm_model, user, JUDGE_SCHEMA, + max_tokens=1500, timeout_s=cfg.rewrite_timeout_s, + temperature=cfg.splice_temperature) + if data and str(data.get("verdict", "")).upper() == "FAIL": + for v in (data.get("violations") or [])[:3]: + if isinstance(v, dict) and v.get("defect", True): + violations.append({ + "i": v.get("i", -1), + "pillar": v.get("pillar", "P4"), + "detail": f"[cross-family veto] " + f"{(v.get('detail') or '')[:200]}", + "defect": True}) + if not any(str(v.get("pillar")) == "cross" for v in violations): + verdict = "FAIL" + if verdict == "FAIL": + aq = "poor" + return verdict, violations, aq + + +def judge_artifact( + cfg: Config, sid: str, *, status_cb=None, +) -> dict: + """Adjudicate an EXISTING workdir artifact (improved.json) with the + windowed judge and promote on PASS β€” the cheap harvest path for + the CLEAN backlog: structural + grounding locally, then judge.""" + started = time.time() + _status = status_cb or _passthrough + from traceforge.grounding import validate_grounding + from traceforge.improver.harness import ( + _append_status, _artifact_digest, _clip_markers_present, + _cache_gate_pass, _load_gate_cache, _parse_improved, + _save_gate_cache, + ) + + workdir = os.path.join(cfg.improver_workdir, sid) + output_path = os.path.join(workdir, "improved.json") + candidate = _parse_improved(output_path) + if candidate is None: + return {"ok": False, "errors": ["no parseable artifact"], + "duration_s": time.time() - started} + # strip pipeline framing markers before the gates (same rule as the + # improve path) so a stale artifact is adjudicated on its real task + from traceforge.validate import strip_scaffold_markers + if strip_scaffold_markers(candidate): + candidate.save(output_path) + raw_path = os.path.join(cfg.raw_dir, f"{sid}.json") + if not os.path.exists(raw_path): + return {"ok": False, "errors": ["raw trace missing"], + "duration_s": time.time() - started} + trace = Trace.load(raw_path) + + if _clip_markers_present(candidate): + return {"ok": False, "errors": ["clip markers"], + "duration_s": time.time() - started} + # final-answer normalization (same rule as the improve flow): an + # old artifact may still close with a raw JSON blob β€” a trainee + # must learn a prose close. Rewrite it BEFORE judging; if the + # synthesis fails, the judge FAILs the blob honestly. + if candidate.messages and candidate.messages[-1].role == "assistant": + blob = candidate.messages[-1].content or "" + task_msg0 = trace.first_task_message() or "(no user message)" + if _is_json_blob_answer(blob) \ + and not _task_demands_json(task_msg0): + _status("final answer (blob rewrite, regate)") + ans, _e = _synthesize_final_answer( + cfg, Trace(session_id="p", + messages=candidate.messages[:-1] + [Message( + role="assistant", content=blob[:4000])], + improved=True), + task_msg0, *_resolve_endpoint(cfg), drop_last=True) + if ans is not None and not _is_json_blob_answer(ans) \ + and len(ans) > max(80, len(blob) // 8): + candidate.messages[-1] = Message( + role="assistant", content=ans) + # persist the normalized artifact: the workdir file is + # the repair reference AND what regate re-reads next + # time β€” without the save the blob resurfaces. + candidate.save(output_path) + _append_status(cfg, sid, { + "stage": "final_answer", "blob_rewritten": True, + "regate": True}) + # pre-judge reframe (same rule as the improve path): the task + # demands a deliverable the trace never produced β€” re-adjudicated + # artifacts must get the same treatment fresh splices do (the + # 51ada098 incident: requeued trace passed judge with a + # permission-asking final because regate skipped this reframe). + task_msg = trace.first_task_message() or "(no user message)" + base, key = _resolve_endpoint(cfg) + task_msg, _reframed = _reframe_undelivered_task( + cfg, sid, trace, candidate, task_msg, _status, _append_status, + base, key) + + # plumbing-as-task gate (same rule as the improve path) + rti = _first_task_index(trace) + if rti is not None and _task_is_plumbing(trace, rti): + return {"ok": False, "errors": [ + "opening task is harness plumbing (validator-rejection " + "row); the real task lives in a previous session β€” not " + "trainable"], "duration_s": time.time() - started} + _status("gates") + errs = validate_structural(candidate) \ + or validate_grounding(candidate, trace) \ + or validate_identity(candidate.messages) + if errs and all("last message role is 'tool'" in e for e in errs): + # stale artifact from an earlier run that stopped mid-call: close + # it here instead of failing a trace that is one row short + ans, _e = _synthesize_final_answer( + cfg, candidate, task_msg or "", base, key) + if ans: + candidate.messages.append(Message(role="assistant", content=ans)) + _status("final answer (tool-tail synth)") + errs = validate_structural(candidate) \ + or validate_grounding(candidate, trace) \ + or validate_identity(candidate.messages) + if errs: + return {"ok": False, + "errors": [f"gate: {e}" for e in errs[:4]], + "duration_s": time.time() - started} + + digest = _artifact_digest(candidate) + cache = _load_gate_cache(workdir) + if "judge:1" in cache.get(digest, []): + _append_status(cfg, sid, {"stage": "judge", "round": 1, + "verdict": "PASS", "cached": True}) + return _promote(cfg, sid, candidate, started, cached=True) + + _status("judge") + verdict, violations, aq = judge_trace( + cfg, candidate, task_msg, base, key) + if verdict == "PASS": + _cache_gate_pass(cache, digest, "judge", 1) + _save_gate_cache(workdir, cache) + _append_status(cfg, sid, {"stage": "judge", "round": 1, + "verdict": "PASS"}) + return _promote(cfg, sid, candidate, started) + _append_status(cfg, sid, { + "stage": "judge", "round": 1, "verdict": verdict, + "violations": len(violations)}) + # INTENT mismatch with a good answer -> reframe the task message + # (cheaper and truer than rewriting the whole trace): same rule as + # the improve path β€” INTENT may coexist with other pillars (the + # all-INTENT form never fired; 13/13 run8 INTENT failures were + # mixed), the delivered work is perfect for a slightly different + # task, so change the task, not the result. + if (violations + and any(v.get("pillar") == "INTENT" for v in violations) + and cfg.allow_task_reframe + and aq in ("high", "ok", None)): + _status("reframe task") + final = next((m.content for m in reversed(candidate.messages) + if m.role == "assistant"), "") + data, err = _post_chat( + base, key, cfg.model or "", + REFRAME_HEADER.format(task=task_msg[:4000], + answer=(final or "")[:4000]), + REFRAME_SCHEMA, max_tokens=1000, + timeout_s=cfg.judge_timeout_s, temperature=0.0) + if data and data.get("task"): + ti = next((i for i, m in enumerate(candidate.messages) + if m.role == "user"), None) + if ti is not None: + old_task = candidate.messages[ti].content + candidate.messages[ti] = Message( + role="user", content=data["task"]) + _append_status(cfg, sid, { + "stage": "reframe", + "old_task": (old_task or "")[:200], + "new_task": data["task"][:200]}) + v2, viols2, _ = judge_trace( + cfg, candidate, data["task"], base, key) + if v2 == "PASS": + return _promote(cfg, sid, candidate, started) + candidate.messages[ti] = Message( + role="user", content=old_task) + return {"ok": False, "errors": [ + f"[{v.get('i')}] {v.get('pillar')}: {v.get('detail')}" + for v in violations[:8]], + "duration_s": time.time() - started} + + +def cross_examine( + cfg: Config, + sid: str, + model: str, +) -> dict: + """Re-judge a PROMOTED trace with a second, different-family judge. + + Supervisor tool: the primary judge (same family as the improver) + inherits the improver's blind spots; a cross-family judge over the + same windows exposes verdicts that only look right to their author. + Read-only over the promoted corpus β€” never promotes or rewrites; + disagreement is reported, the human decides (repair or revoke). + """ + import time + + started = time.time() + path = os.path.join(cfg.improved_dir, f"{sid}.json") + if not os.path.exists(path): + return {"ok": False, "errors": [f"not promoted: {sid}"]} + t = Trace.load(path) + task = t.first_task_message() or "(no user message)" + base, key = _resolve_endpoint(cfg) + verdict, violations, aq = judge_trace( + cfg, t, task, base, key, model=model) + return { + "ok": verdict == "PASS", + "sid": sid, "model": model, "verdict": verdict, + "violations": violations, "answer_quality": aq, + "duration_s": time.time() - started, + } + + +# Final-answer hesitation lint: an ideal trace's last assistant message +# must CLOSE the task, not ask the user for a go-ahead ("Want me to +# start?" teaches trainees to stall). Stripped deterministically at +# promotion; only fires on a trailing question, so mid-answer offers +# and rhetorical questions survive. +_PERMISSION_ASK_RE = re.compile( + r"(want me to|shall i|should i (start|implement|begin|go|" + r"proceed)|let me know if you|await(ing)? (your|the) " + r"(go-ahead|approval)|i (can|could) (start|implement|begin|do))", + re.IGNORECASE) + + +_SYSTEM_NOTICE_RE = re.compile(r"\n?\[System:[^\]]{0,400}\]\s*$", re.S) + + +def _strip_system_notice(m: Message) -> bool: + """Trim a trailing [System: …] plumbing notice from a user row + (f75cdf: 'commit and push…\n[System: The active model …]'). The + notice is harness plumbing; the task text before it is real. Only + fires when a real task remains (>= 20 chars).""" + if m.role != "user" or not m.content: + return False + i = m.content.find("\n[System:") + if i < 20: + return False + m.content = m.content[:i].rstrip() + return True + + +def _strip_permission_ask(candidate: Trace) -> bool: + """Remove a trailing permission-asking question from the final + assistant message. Returns True when the message was edited.""" + for m in reversed(candidate.messages): + if m.role != "assistant": + continue + c = m.content or "" + if not c.rstrip().endswith("?"): + return False + # find the last sentence boundary whose tail carries the ask + cut = None + for mm in _PERMISSION_ASK_RE.finditer(c): + # sentence start = previous ., !, \n\n, or start + s = max(c.rfind(".", 0, mm.start()), + c.rfind("!", 0, mm.start()), + c.rfind("\n\n", 0, mm.start()), + c.rfind("?", 0, mm.start())) + 1 + if s == 0 or len(c) - s > 400: + continue + cut = s + if cut is None: + return False + # trim trailing separators/whitespace before the cut point + c2 = c[:cut].rstrip(" \t\n-β€”β€’*") + # keep only when a real answer remains: real permission asks + # ride the tail of long finals, so a short remainder means the + # "final" was the ask itself β€” leave it for the judge/reframe + if len(c2) < 200: + return False + m.content = c2 + ("\n" if c2.endswith((".", "!", ":")) else "") + return True + return False + + +def _promote( + cfg: Config, sid: str, candidate: Trace, started: float, + *, cached: bool = False, +) -> dict: + candidate.session_id = f"ideal-{sid}" + candidate.parent_session_id = sid + if not candidate.model: + candidate.model = "splice-http" + if candidate.source in (None, "", "unknown"): + candidate.source = "traceforge-splice" + if _strip_permission_ask(candidate): + from traceforge.improver.harness import _append_status + _append_status(cfg, sid, { + "stage": "promote", "permission_ask_stripped": True}) + from traceforge.validate import strip_scaffold_markers + if strip_scaffold_markers(candidate): + from traceforge.improver.harness import _append_status + _append_status(cfg, sid, { + "stage": "promote", "scaffold_stripped": True}) + for _m in candidate.messages: + if _strip_system_notice(_m): + from traceforge.improver.harness import _append_status + _append_status(cfg, sid, { + "stage": "promote", "system_notice_stripped": True}) + break + out = os.path.join(cfg.improved_dir, f"{sid}.json") + candidate.save(out) + return {"ok": True, "errors": [], "improved": candidate, + "cached": cached, "duration_s": time.time() - started} \ No newline at end of file diff --git a/traceforge/splitter.py b/traceforge/splitter.py new file mode 100644 index 0000000000000000000000000000000000000000..443dc96c88e4c63b9c075fd69230275ec7cb6b63 --- /dev/null +++ b/traceforge/splitter.py @@ -0,0 +1,218 @@ +"""Splits over-long raw traces into multiple trainable segment traces. + +Monster raws (few messages, giant tool results β€” 1.4 MB in 57 rows +measured) are low-yield in the improve pipeline (repair loops can't +converge on them) and, when promoted, produce ~500k-token training +sequences the trainer must truncate or drop. The owner directive +(2026-10-07): split them into ~N output traces of ≀ split_max_chars +each instead of one unusable giant. + +Split discipline (quality pillars preserved by construction): +- Cut ONLY at assistant-turn boundaries: a callβ†’results group is never + torn apart (pairing stays 1:1 within each segment). +- The task row (first real user message) is prepended to EVERY + segment so each remains a trainable (task β†’ work β†’ result) pair. +- The trace's real final answer stays on the segment that contains + it (the last); earlier segments end on their last completed turn β€” + the splice engine's synthesized-final path grounds a segment final + in the segment's own visible results. +- Tool rows referencing calls in a previous segment can't exist: + turn-boundary cuts guarantee call+result always land together. +- Segment sids: #p1..N β€” separate workdirs, manifest rows, + promotions. '#' cannot appear in real session ids (uuid/date + formats), so the cache namespace stays collision-free. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Optional + +from traceforge.chunker import _msg_size +from traceforge.trace import Trace + + +@dataclass +class Segment: + index: int # 1-based + trace: Trace # standalone trainable trace (task row + turns) + + +def _turn_boundaries(msgs) -> list[int]: + """Indices where a new top-level turn starts. + + A turn = an assistant row plus all tool rows that follow it (until + the next assistant/user row). Boundaries are assistant or user + rows; the first message is always a boundary. + """ + bounds: list[int] = [] + for i, m in enumerate(msgs): + if m.role in ("assistant", "user") and ( + not bounds or i > bounds[-1] + ): + if not bounds or m.role == "user" or msgs[bounds[-1]] is not m: + bounds.append(i) + # dedupe consecutive (a user row right after boundary i is fine) + out: list[int] = [] + for b in bounds: + if not out or b > out[-1]: + out.append(b) + return out + + +def split_trace( + trace: Trace, + *, + split_max_chars: int = 480_000, # β‰ˆ120k tokens per output trace + min_segment_chars: int = 60_000, # don't leave tiny tail segments + max_segments: int = 16, # ceiling per parent (0 = no ceiling) + skip_first: bool = True, # drop the session-opening segment +) -> list[Segment]: + """Split an over-long trace into trainable segments. + + Traces under split_max_chars come back as ONE segment, unchanged + (with index 1) β€” splitting is opt-in by size, never by default. + """ + msgs = trace.messages + total = sum(_msg_size(m) for m in msgs) + if total <= split_max_chars: + return [Segment(index=1, trace=trace)] + + task_idx: Optional[int] = None + task_msg = None + for i, m in enumerate(msgs): + if m.role == "user": + task_idx, task_msg = i, m + break + + bounds = _turn_boundaries(msgs) + # always include the end as an implicit boundary + end_bounds = bounds + [len(msgs)] + + # first cut point list: cumulative sizes between boundaries + cuts: list[int] = [0] + acc = 0 + prev = 0 + for b in end_bounds: + seg = sum(_msg_size(m) for m in msgs[prev:b]) + acc += seg + prev = b + if acc >= split_max_chars: + cuts.append(b) + acc = 0 + if cuts[-1] != len(msgs): + # Tail segment: rebalance it up to the floor, else merge it. + # Measured 2026-10-23: the old code appended the cut regardless + # (the `pass` branch did nothing), so min_segment_chars never + # fired β€” 192 of 1,044 segments were under 60k chars, 31 under + # 5k, the smallest 714 chars / 2 messages. A 2-message tail with + # a prepended task row is not a trainable trace, and it is the + # segment that holds the real final answer. + # + # Two repairs, in order of preference: + # 1. move turn boundaries from the end of the previous segment + # into the tail until the tail reaches the floor β€” both + # segments stay under split_max_chars and the final answer + # lands in a trace big enough to train on; + # 2. if the previous segment is a single turn (nothing to move), + # drop the cut and let the tail join it. + tail_lo = cuts[-1] + tail_size = sum(_msg_size(m) for m in msgs[tail_lo:]) + if tail_size < min_segment_chars and len(cuts) > 1: + prev_lo = cuts[-2] + moved = False + for b in reversed([x for x in bounds if prev_lo < x < tail_lo]): + if sum(_msg_size(m) for m in msgs[prev_lo:b]) < min_segment_chars: + break # would starve the previous segment + cuts[-1] = b + moved = True + if sum(_msg_size(m) for m in msgs[b:]) >= min_segment_chars: + break + if not moved: + cuts.pop() # nothing to move: tail joins the previous segment + cuts.append(len(msgs)) + + # Ceiling per parent session: keep the first max-1 segments (task + # setup + early work) and the LAST one (the real final answer), drop + # the middle. Without it a single 155M-char codex rollout becomes 317 + # same-task segments β€” days of batch time for one session. The ranges + # are explicit (not a cut list) because dropping the middle makes the + # boundaries non-contiguous; a cut list would silently MERGE the + # dropped span into one oversized segment instead. + ranges = [(cuts[k], cuts[k + 1]) for k in range(len(cuts) - 1)] + if max_segments and len(ranges) > max_segments: + ranges = ranges[:max_segments - 1] + [ranges[-1]] + + # Drop the session-opening segment. Measured 2026-10-25 on 5,817 p1 + # segments: 2 promoted (0.03%), against 463 for p2 (8.0%) and 653 for + # all of p2+ (2.9%). The opening is the harness's context β€” a shell + # paste, a dispatch brief, a terminal dump β€” not a task statement, so + # the improver has nothing to rewrite. Attempting them costs ~4 days + # of queue time for two traces. + # + # Indices are kept (p2..pN), never renumbered: existing raws and + # workdirs are keyed by #pN, and renumbering would collide. + indexed = [(i + 1, lo, hi) for i, (lo, hi) in enumerate(ranges)] + if skip_first and len(indexed) > 1: + indexed = indexed[1:] + + segments: list[Segment] = [] + n = len(indexed) + for k, (idx, lo, hi) in enumerate(indexed): + seg_msgs = list(msgs[lo:hi]) + if task_msg is not None and lo > (task_idx or 0): + # prepend the original task row so the segment is + # self-contained; mark it as the segment context + header = ( + f"[trace segment {k + 1} of {n} β€” the same working " + f"session, continued]\n{task_msg.content}" + ) + from traceforge.trace import Message + + seg_msgs.insert( + 0, + Message( + role="user", + content=header, + timestamp=task_msg.timestamp, + ), + ) + segments.append( + Segment( + index=idx, + trace=Trace( + session_id=f"{trace.session_id}#p{idx}", + model=trace.model, + title=trace.title, + started_at=trace.started_at, + cwd=trace.cwd, + messages=seg_msgs, + source=trace.source, + ), + ) + ) + return segments + + +def needs_split(trace: Trace, *, split_max_chars: int = 480_000) -> bool: + return sum(_msg_size(m) for m in trace.messages) > split_max_chars + + +def segment_sid(sid: str, index: int) -> str: + """Stable segment sid: #p.""" + return f"{sid}#p{index}" + + +def save_segments(segments: list[Segment], raw_dir: str, base_sid: str) -> list[str]: + """Write segment traces into the raw cache; returns their sids.""" + # A lone segment is only the unsplit trace when it is index 1. After + # skip_first a 2-way split leaves one segment at index 2, and returning + # base_sid for it would overwrite the unsplit raw with a segment. + if len(segments) == 1 and segments[0].index == 1: + return [base_sid] + sids: list[str] = [] + for seg in segments: + sid = segment_sid(base_sid, seg.index) + seg.trace.save(os.path.join(raw_dir, f"{sid}.json")) + sids.append(sid) + return sids \ No newline at end of file diff --git a/traceforge/timesensitivity.py b/traceforge/timesensitivity.py new file mode 100644 index 0000000000000000000000000000000000000000..13a22e007e2c1949cfee940cbcc272ea6613e7dc --- /dev/null +++ b/traceforge/timesensitivity.py @@ -0,0 +1,304 @@ +"""Time sensitivity: does the ideal answer decay? + +A trace whose correct answer is a fact that changes over time is a bad SFT +target when the agent had no way to look that fact up. Train on it and the +model learns to *assert* the fact from its weights β€” "the current president +is X" β€” which is wrong the moment X changes, and the model has no way to +know it is wrong. + +The same question is a GOOD target when the agent had a retrieval tool. The +ideal path then calls the tool and answers from its result, so what the model +imitates is the *procedure* (search, read, answer), not the fact. The +procedure does not decay. + +So the metric has two axes, and the classification is their combination: + + temporal task + no retrieval tool -> time_sensitive (flag / rewrite) + temporal task + retrieval tool -> time_anchored (good) + no temporal dependency -> not_time_sensitive + +`time_anchored` carries a sub-flag: whether the retrieval tool was actually +CALLED. A temporal task with a search tool available but never used is still +anchored in principle β€” the ideal rewrite should make the agent call it β€” but +the raw trace answered from weights, so it needs a rewrite rather than a pass. + + from traceforge.timesensitivity import detect, census + detect(trace) # -> {"class": "time_sensitive", ...} + census(records) # -> {"time_sensitive": 12, ...} + +WHAT COUNTS AS TEMPORAL +----------------------- +The discriminator is the OBJECT of the temporal qualifier, not the qualifier +itself. "The latest fix round", "the current branch", "now running" and +"today's build" all carry a temporal word and none of them is a decaying +fact β€” they are local artifacts of the session. Measured on 3,597 traces, a +qualifier-only detector flagged 30.4% of the corpus and almost all of it was +noise. + +So a trace is temporal only when the task asks for a fact about the EXTERNAL +world whose truth value changes with wall-clock time: who holds an office, a +price or rate, the released version of a product, the weather, the news, a +population, a score. Those are the facts a model cannot know and must fetch. + +The detector is deliberately conservative: a false `time_sensitive` throws +away a good trace, so a pattern must name the FACT, not just a temporal word. +""" + +from __future__ import annotations + +import collections +import re +from typing import Any, Iterable, Optional + +# --------------------------------------------------------------- taxonomy + +TIME_SENSITIVE = "time_sensitive" +TIME_ANCHORED = "time_anchored" +NOT_TIME_SENSITIVE = "not_time_sensitive" + +CLASSES = (TIME_SENSITIVE, TIME_ANCHORED, NOT_TIME_SENSITIVE) + +CLASS_LABELS = { + TIME_SENSITIVE: "answer decays: temporal fact, no retrieval tool", + TIME_ANCHORED: "temporal fact, but a retrieval tool can fetch it", + NOT_TIME_SENSITIVE: "no time-dependent fact in the task", +} + +# ------------------------------------------------- external changing facts +# Each pattern names a fact whose truth value changes with wall-clock time. +# A bare temporal qualifier is NOT enough β€” see the module docstring. +_EXTERNAL_FACT_PATTERNS: list[tuple[str, str]] = [ + ("officeholder", + r"\bwho (?:is|'s|was) (?:the )?(?:current |new |newly elected |acting )?" + r"(?:president|vice[- ]president|ceo|cto|cfo|coo|chancellor|" + r"prime minister|governor|mayor|chair(?:man|woman|person)?|" + r"pope|monarch|king|queen|head of state)\b"), + ("price", + r"\b(?:what(?:'s| is)|how much is) (?:the )?(?:current |latest )?" + r"(?:price|cost|value|valuation|market cap|exchange rate|" + r"interest rate|inflation rate|unemployment rate)\b"), + ("cost_verb", r"\bhow much (?:does|do|is|are) .{0,40}\b(?:cost|worth)\b"), + ("version", + r"\b(?:what|which) (?:is )?(?:the )?(?:current |latest |newest )?" + r"version of\b"), + ("release", + r"\b(?:latest|current|newest) (?:release|stable release|lts|patch) " + r"(?:of|for)\b"), + ("news", r"\b(?:latest|current|today'?s?|breaking) news\b"), + ("weather", r"\b(?:weather|forecast)\b"), + ("population", r"\bpopulation of\b"), + ("sports", r"\bwho won\b|\b(?:final|current) score\b"), + ("ranking", r"\b(?:current|latest) (?:ranking|standings|leaderboard)\b"), + ("schedule", + r"\b(?:current|latest|today'?s?) (?:schedule|agenda|opening hours)\b"), + ("stock", r"\b(?:stock price|share price|ticker|market price)\b"), +] + +EXTERNAL_FACT_RE = [(name, re.compile(p, re.I)) + for name, p in _EXTERNAL_FACT_PATTERNS] + +# A version/release question about the session's OWN code is local, not +# external: "what version of our API", "which version of the config". The +# external case names a released product ("the latest version of Django"). +_LOCAL_OWNER_RE = re.compile( + r"\b(?:our|this|the (?:repo|project|codebase|branch|config|schema|" + r"migration|api|service|app|package\.json|lock ?file))\b", re.I) + +# ------------------------------------------------------ retrieval tools +# STRONG: a tool whose whole purpose is fetching current external facts. +_STRONG_RETRIEVAL = ( + r"web[_-]?search", r"websearch", r"search[_-]?web", r"web[_-]?fetch", + r"webfetch", r"fetch[_-]?url", r"open[_-]?url", r"read[_-]?url", + r"browse", r"google(?:[_-]?search)?", r"tavily", r"serper", r"exa", + r"brave[_-]?search", r"bing", r"duckduckgo", r"perplexity", + r"news", r"weather", r"stock", r"price", r"lookup", +) +# WEAK: generic HTTP / DB access. It CAN fetch a current fact, but it is also +# how an agent reads a local file over http, so it anchors less strongly. +_WEAK_RETRIEVAL = ( + r"curl", r"requests", r"http[_-]?(?:get|request)", r"api[_-]?call", + r"sql", r"query[_-]?(?:db|database)", r"database", r"fetch", +) + +STRONG_RETRIEVAL_RE = re.compile("|".join(_STRONG_RETRIEVAL), re.I) +WEAK_RETRIEVAL_RE = re.compile("|".join(_WEAK_RETRIEVAL), re.I) + +# ------------------------------------------------------- answer time facts +# A date, a version, or a price in the final answer is evidence the answer +# states a decaying fact. Only used to strengthen the verdict, never alone. +_ANSWER_FACT_PATTERNS = [ + ("date", r"\b(?:January|February|March|April|May|June|July|August|" + r"September|October|November|December)\s+\d{1,2},?\s+20\d\d\b"), + ("iso_date", r"\b20\d\d-\d\d-\d\d\b"), + ("version", r"\bv?\d+\.\d+(?:\.\d+)?\b"), + ("money", r"[$€£]\s?\d[\d,.]*"), +] +ANSWER_FACT_RE = [(n, re.compile(p)) for n, p in _ANSWER_FACT_PATTERNS] + + +# ------------------------------------------------------------------ helpers + +def _messages(obj: Any) -> list[dict]: + """Accept a Trace, an exported record, or a bare message list. + + A Trace is converted WITHOUT `Message.to_dict()`: the census runs over + thousands of traces and the full serializer (tool-call objects, optional + fields) dominated the runtime β€” 2m16s for 3,684 traces. Only the fields + this module reads are materialised. + """ + if isinstance(obj, list): + return obj + if hasattr(obj, "messages"): + out: list[dict] = [] + for m in obj.messages: + if isinstance(m, dict): + out.append(m) + continue + d: dict[str, Any] = {"role": m.role, "content": m.content} + if m.tool_calls: + d["tool_calls"] = [{"name": tc.name, "arguments": tc.arguments} + for tc in m.tool_calls] + if m.tool_name: + d["tool_name"] = m.tool_name + out.append(d) + return out + if isinstance(obj, dict): + return obj.get("messages") or [] + return [] + + +def _task_text(msgs: list[dict]) -> str: + for m in msgs: + if m.get("role") == "user": + return m.get("content") or "" + return "" + + +def _final_answer(msgs: list[dict]) -> str: + for m in reversed(msgs): + if m.get("role") == "assistant" and (m.get("content") or "").strip(): + return m.get("content") or "" + return "" + + +def _tool_names(obj: Any, msgs: list[dict]) -> tuple[list[str], list[str]]: + """(available, called) tool names. + + `available` comes from the record's `tools` array when present (the + export carries it); otherwise only the called names are known, which is + the honest answer for a raw trace. + """ + available: list[str] = [] + if isinstance(obj, dict): + for t in obj.get("tools") or []: + fn = t.get("function") if isinstance(t, dict) else None + name = (fn or {}).get("name") or (t.get("name") + if isinstance(t, dict) else None) + if name: + available.append(name) + called: list[str] = [] + for m in msgs: + for tc in (m.get("tool_calls") or []): + fn = tc.get("function") or {} + name = fn.get("name") or tc.get("name") + if name: + called.append(name) + return available, called + + +def _match(names: Iterable[str], pattern: re.Pattern) -> list[str]: + return sorted({n for n in names if pattern.search(n)}) + + +def _temporal_facts(task: str) -> list[str]: + """Names of the external changing facts the task asks about.""" + hits: list[str] = [] + for name, rx in EXTERNAL_FACT_RE: + m = rx.search(task) + if not m: + continue + # a version/release question about the session's own code is local + if name in ("version", "release"): + tail = task[m.end():m.end() + 60] + if _LOCAL_OWNER_RE.search(tail): + continue + hits.append(name) + return hits + + +# ------------------------------------------------------------------- detect + +def detect(obj: Any) -> dict: + """Classify one trace's time sensitivity. + + Returns a dict with the class, the evidence that produced it, and a + `needs_rewrite` flag for the anchored-but-unused case. + """ + msgs = _messages(obj) + task = _task_text(msgs) + answer = _final_answer(msgs) + available, called = _tool_names(obj, msgs) + + temporal = _temporal_facts(task) + strong_avail = _match(available + called, STRONG_RETRIEVAL_RE) + weak_avail = _match(available + called, WEAK_RETRIEVAL_RE) + strong_called = _match(called, STRONG_RETRIEVAL_RE) + weak_called = _match(called, WEAK_RETRIEVAL_RE) + answer_facts = [name for name, rx in ANSWER_FACT_RE if rx.search(answer)] + + if not temporal: + cls = NOT_TIME_SENSITIVE + elif strong_avail or weak_avail: + cls = TIME_ANCHORED + else: + cls = TIME_SENSITIVE + + # Anchored in principle, but the raw trace answered from weights: the + # ideal rewrite has to make the agent call the tool it already had. + needs_rewrite = bool( + cls == TIME_ANCHORED and not (strong_called or weak_called)) + + return { + "class": cls, + "label": CLASS_LABELS[cls], + "temporal_facts": temporal, + "retrieval_available": strong_avail + weak_avail, + "retrieval_called": strong_called + weak_called, + "answer_time_facts": answer_facts, + "needs_rewrite": needs_rewrite, + # 0-3: how strongly the task depends on a changing fact + "score": min(3, len(temporal)), + } + + +def is_publishable(obj: Any) -> bool: + """False for traces whose answer would teach a decaying fact.""" + return detect(obj)["class"] != TIME_SENSITIVE + + +def census(records: Iterable[Any]) -> dict[str, int]: + """Count records by class, in class order.""" + counts: collections.Counter[str] = collections.Counter() + for r in records: + counts[detect(r)["class"]] += 1 + return {c: counts.get(c, 0) for c in CLASSES} + + +def summary(records: Iterable[Any]) -> dict: + """Census plus the rewrite backlog, for the dataset card.""" + counts: collections.Counter[str] = collections.Counter() + rewrite = 0 + n = 0 + for r in records: + d = detect(r) + counts[d["class"]] += 1 + n += 1 + if d["needs_rewrite"]: + rewrite += 1 + return { + "classes": {c: counts.get(c, 0) for c in CLASSES}, + "labels": CLASS_LABELS, + "needs_rewrite": rewrite, + "total": n, + "time_sensitive_share": (counts.get(TIME_SENSITIVE, 0) / n) if n else 0.0, + } diff --git a/traceforge/trace.py b/traceforge/trace.py new file mode 100644 index 0000000000000000000000000000000000000000..28536922ec6d67c65cb35205beca7ed67f53f6a7 --- /dev/null +++ b/traceforge/trace.py @@ -0,0 +1,379 @@ +"""Normalized trace model β€” the canonical interchange format of traceforge. + +Every reader (hermes, ...) converts into this; every improver/exporter +consumes it. Independent of any specific agent's on-disk format. +""" +from __future__ import annotations + +import json +import os +import re +import tempfile +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Optional + +VALID_ROLES = ("system", "user", "assistant", "tool") + +# Hermes context-compressor artifacts β€” never real user/agent content. +# Stripped from renderings so improvers cannot copy them into ideal traces. +CONTEXT_COMPRESSION_RE = re.compile( + r"βŸͺHERMES-CONTEXT-COMPRESSION:.*?⟫", re.DOTALL +) +COMPACTION_PREFIX = "[CONTEXT COMPACTION" +# Claude Code session continuation: compaction summary handoff row. +CONTINUATION_PREFIX = "This session is being continued" + + +@dataclass +class ToolCall: + """A tool invocation emitted inside an assistant message.""" + + id: str + name: str + arguments: str = "{}" # raw JSON string, kept verbatim (not parsed) + + def to_dict(self) -> dict[str, Any]: + return {"id": self.id, "name": self.name, "arguments": self.arguments} + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "ToolCall": + return cls( + id=str(d.get("id", "")), + name=str(d.get("name", "")), + arguments=d.get("arguments") or "{}", + ) + + +@dataclass +class Message: + """One message in a trace. Mirrors the OpenAI chat format closely.""" + + role: str + content: Optional[str] = None + reasoning: Optional[str] = None # thinking/reasoning text, if captured + tool_calls: list[ToolCall] = field(default_factory=list) # assistant only + tool_call_id: Optional[str] = None # tool rows: which call this answers + tool_name: Optional[str] = None # tool rows: which tool produced it + timestamp: Optional[float] = None # epoch seconds + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {"role": self.role} + if self.content is not None: + d["content"] = self.content + if self.reasoning: + d["reasoning"] = self.reasoning + if self.tool_calls: + d["tool_calls"] = [tc.to_dict() for tc in self.tool_calls] + if self.tool_call_id is not None: + d["tool_call_id"] = self.tool_call_id + if self.tool_name is not None: + d["tool_name"] = self.tool_name + if self.timestamp is not None: + d["timestamp"] = self.timestamp + return d + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "Message": + return cls( + role=d["role"], + content=d.get("content"), + reasoning=d.get("reasoning"), + tool_calls=[ToolCall.from_dict(tc) for tc in d.get("tool_calls", [])], + tool_call_id=d.get("tool_call_id"), + tool_name=d.get("tool_name"), + timestamp=d.get("timestamp"), + ) + + +@dataclass +class Trace: + """A full agent session: metadata + ordered message list.""" + + session_id: str + model: Optional[str] = None + title: Optional[str] = None + started_at: Optional[float] = None + cwd: Optional[str] = None + messages: list[Message] = field(default_factory=list) + source: str = "hermes" # provenance: which reader produced it + improved: bool = False # True for synthetic ideal traces + parent_session_id: Optional[str] = None # original trace it derives from + # task/topic bucket (see traceforge/classify.py). Persisted so export + # can split the corpus deterministically and the label is reviewable. + task: Optional[str] = None + + # ------------------------------------------------------------------ meta + + def started_at_iso(self) -> Optional[str]: + if self.started_at is None: + return None + return datetime.fromtimestamp(self.started_at, tz=timezone.utc).isoformat() + + def stats(self) -> dict[str, Any]: + roles: dict[str, int] = {} + tool_calls = 0 + chars = 0 + tools: dict[str, int] = {} + for m in self.messages: + roles[m.role] = roles.get(m.role, 0) + 1 + chars += len(m.content or "") + tool_calls += len(m.tool_calls) + for tc in m.tool_calls: + tools[tc.name] = tools.get(tc.name, 0) + 1 + return { + "session_id": self.session_id, + "n_messages": len(self.messages), + "roles": roles, + "n_tool_calls": tool_calls, + "tool_histogram": tools, + "content_chars": chars, + "improved": self.improved, + "parent_session_id": self.parent_session_id, + } + + def first_user_message(self) -> Optional[str]: + for m in self.messages: + if m.role == "user": + return m.content + return None + + def first_task_message(self) -> Optional[str]: + """The first REAL user task message, skipping context-compaction + handoff rows (harness artifacts that carry no task).""" + for m in self.messages: + if m.role == "user": + c = m.content or "" + if COMPACTION_PREFIX in c[:200]: + continue + # Claude Code continuation handoff: summary of an earlier + # session, the embedded actionable task lives further down + if CONTINUATION_PREFIX in c[:120]: + continue + # Attachment-wrapper plumbing: '@file:…' plus a + # Context-Warnings notice β€” the real task text never + # made it into the trace (11c65e: the whole "task" is + # 'path is outside the allowed workspace'). Skip; if + # nothing else remains, the caller treats it as + # taskless and the reframe path synthesizes one. + if c.lstrip().startswith("@file:") and ( + "Context Warnings" in c[:600]): + continue + # Model-switch plumbing notice as the opening user row + # (32353e/f75cdf/4d0801 class): "[System: The active + # model for this chat has changed to …]" carries no + # task. Skip it; a later user row (or the reframe + # path) supplies the real task. + if c.lstrip().startswith("[System:") and ( + "has changed" in c[:300]): + continue + return m.content + return self.first_user_message() + + # ------------------------------------------------------- serialization + + def to_dict(self) -> dict[str, Any]: + return { + "session_id": self.session_id, + "model": self.model, + "title": self.title, + "started_at": self.started_at, + "cwd": self.cwd, + "source": self.source, + "improved": self.improved, + "parent_session_id": self.parent_session_id, + "task": self.task, + "messages": [m.to_dict() for m in self.messages], + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "Trace": + return cls( + session_id=d["session_id"], + model=d.get("model"), + title=d.get("title"), + started_at=d.get("started_at"), + cwd=d.get("cwd"), + messages=[Message.from_dict(m) for m in d.get("messages", [])], + source=d.get("source", "unknown"), + improved=bool(d.get("improved", False)), + parent_session_id=d.get("parent_session_id"), + task=d.get("task"), + ) + + def save(self, path: str) -> None: + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + # atomic-ish write: temp file in same dir, then rename + fd, tmp = tempfile.mkstemp( + dir=os.path.dirname(os.path.abspath(path)), + prefix=".tmp-trace-", + suffix=".json", + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(self.to_dict(), f, ensure_ascii=False, indent=2) + os.replace(tmp, path) + except BaseException: + if os.path.exists(tmp): + os.unlink(tmp) + raise + + @classmethod + def load(cls, path: str) -> "Trace": + with open(path, "r", encoding="utf-8") as f: + return cls.from_dict(json.load(f)) + + # --------------------------------------------------------- rendering + + def _clip(self, text: Optional[str], limit: int) -> str: + if not text: + return "" + text = CONTEXT_COMPRESSION_RE.sub("[…context compressed…]", text) + text = text.rstrip() + if len(text) <= limit: + return text + return text[:limit] + f" …[+{len(text) - limit} chars clipped]" + + def render_compact( + self, + user_chars: int = 4000, + assistant_chars: int = 4000, + tool_result_chars: int = 1200, + tool_args_chars: int = 600, + include_reasoning: bool = False, + index_offset: int = 0, + ) -> str: + """Compact, LLM-friendly one-screen-per-turn rendering. + + This is the format embedded into improver prompts. Tool results and + oversized texts are clipped per the char budgets so a whole trace fits + into a single prompt without exploding the improver's context. + + index_offset shifts the rendered [i] message indices (0 = start at 1) + so a slice of a larger trace can be rendered with ABSOLUTE indices β€” + the triage classifier labels those indices and the merge/splice + stages must map them back to the right messages. + """ + lines: list[str] = [] + head = f"=== TRACE {self.session_id} " + if self.model: + head += f"(model={self.model} " + else: + head += "(" + head += f"started={self.started_at_iso() or 'unknown'}) ===" + lines.append(head) + for i, m in enumerate(self.messages, 1 + index_offset): + if m.role == "user": + lines.append(f"[{i}] USER:") + lines.append(self._clip(m.content, user_chars)) + elif m.role == "assistant": + body = [] + if include_reasoning and m.reasoning: + body.append("REASONING: " + self._clip(m.reasoning, assistant_chars)) + if m.content: + body.append(self._clip(m.content, assistant_chars)) + if not body and not m.tool_calls: + continue # fully-empty assistant turn (compaction artifact) + if body: + lines.append(f"[{i}] ASSISTANT:") + lines.extend(body) + for tc in m.tool_calls: + lines.append( + f" -> TOOL {tc.name} args={self._clip(tc.arguments, tool_args_chars)}" + ) + if m.tool_calls and not body: + marks = " ".join(f"[{i}]" for _ in m.tool_calls) + # re-anchor marker for tool-only turns + lines.append( + f"[{i}] ASSISTANT: (tool-only turn) {marks}".replace( + f"[{i}] ASSISTANT: (tool-only turn) {marks}", + f"[{i}] ASSISTANT: (tool-only turn)", + ) + ) + elif m.role == "tool": + name = m.tool_name or "?" + # error-signal tag: the splice window rewriter must see + # WHICH results are failures so it can drop the failed + # call turn (P2 route-around) instead of keeping it. + tag = " [ERROR RESULT]" if _is_error_result(m.content) else "" + lines.append(f"[{i}] TOOL {name} result:{tag}") + lines.append(self._clip(m.content, tool_result_chars)) + elif m.role == "system": + lines.append(f"[{i}] SYSTEM:") + lines.append(self._clip(m.content, assistant_chars)) + return "\n".join(lines) + + +def json_loads_lenient(raw: Optional[str]) -> Any: + """Parse JSON or return None β€” traces contain occasional malformed blobs.""" + if not raw: + return None + try: + return json.loads(raw) + except (json.JSONDecodeError, TypeError): + return None + +def _is_error_result(content: Optional[str]) -> bool: + """True when a tool-result row is a FAILED call (P2 route-around + signal for the splice window rewriter). Recognizes the common + shapes: JSON status:"error", exit-code lines, shell traceback + markers, and error-prefixed text. + + v2 (2026-10-05, measured on 1235 raws): explicit harness + "exit_code": N is AUTHORITATIVE β€” nonzero fails, zero succeeds + even when stderr noise ("rg: command not found" in a compound + fallback that continued) appears in the output. This killed the + main false-positive class. Adds: zsh 'no matches found' glob + errors, bare 'command not found', timeouts (124) and interrupts + (130). 335 additional genuine all-error turns detected across + the corpus, zero false positives in a 40-trace audit. + """ + if not content: + return False + head = content[:400] + # browser_exec wraps its output in an + # block whose error JSON sits past the 400-char head window β€” + # scan the FULL text for the JSON shapes when the wrapper is + # present (140 genuine failed-browser rows corpus-wide; the + # 1326 web_search/web_extract wrappers carry no error JSON). + if ".json and double as the future +training corpus for a small non-autoregressive classifier (the laya +fine-tuning plan). + +Grammar ladder: structured_outputs (vLLM >= 0.12) -> guided_json +(vLLM < 0.12) -> bare request + _json_object_spans rescue (review.py). +The first variant that parses wins; a None response falls through. +""" +from __future__ import annotations + +import json +import os +import threading +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Optional + +from traceforge import chunker +from traceforge.config import Config +from traceforge.trace import Trace, json_loads_lenient + +ACTIONS = ("keep", "drop", "rephrase", "fact_check", "summarize") + +LABELS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "object", + "properties": { + "i": {"type": "integer"}, + "action": {"type": "string", "enum": list(ACTIONS)}, + "reason": {"type": "string"}, + }, + "required": ["i", "action"], + }, + } + }, + "required": ["labels"], +} + +WINDOW_HEADER = """You are a trace-triage classifier for an agent-training-data +pipeline. You see ONE window of a rendered real agent session (message +indices are ABSOLUTE into the full session). Label EVERY message shown. + +For each message index [i], pick exactly one action: +- keep: solid turn, survives the ideal-world rewrite unchanged. +- drop: dead weight the ideal path routes around β€” failed calls, retry + loops, redundant probes, flailing, dead-end exploration, follow-up + messages that exist only to fix an earlier mistake. +- rephrase: the RIGHT call/content in the WRONG shape β€” needs rewriting: + verbose or filler-laden prose, sloppy grammar, wrong language, a + correct tool call with suboptimal arguments (e.g. unbatched calls). +- fact_check: this message ASSERTS an external-world claim (versions, + API behavior, dates, technical facts) that the ideal trace would + repeat β€” worth a dedicated fact-check. +- summarize: very long repetitive content (loop iterations, dumps, + directory listings) that must be compressed in the ideal trace while + keeping the load-bearing facts. + +Rules: +- Conservative bias: when torn between keep and rephrase, rephrase; + when torn between drop and rephrase, rephrase. Never silently keep a + suspicious turn β€” downstream gates catch bad keeps, not missed edits. +- Tool RESULT rows ([i] TOOL ... result:) inherit their call's verdict: + if you drop or summarize the call, drop/summarize the result too. + Label them anyway with the same action for clarity. +- Ignore rendering clip markers ("…[+N chars clipped]") β€” display + artifacts, never content signal. +- The FIRST user task message is always keep (it anchors the task) β€” + label it keep unless genuinely absent. + +Reply with ONLY the JSON object: {{"labels": [{{"i": , +"action": "keep|drop|rephrase|fact_check|summarize", "reason": "", +...}}]}} β€” one entry for every [i] shown, in any order. + +REAL TRACE WINDOW (messages {lo}-{hi} of {n}): +""" + + +def _resolve_endpoint(cfg: Config) -> tuple[str, str, str]: + """(base_url, api_key, model) for triage calls; defaults come from + the active Hermes profile (~/.hermes/config.yaml). + + Resolution order (2026-10-07): when cfg pins a provider (e.g. + provider: plai with a -selfhosted model), the providers. + block wins β€” the profile's top-level model: block may point at an + unrelated backend (codex/chatgpt) that cannot serve the pinned + selfhosted models. The old top-level model: fallback stays for + unpinned configs. + """ + base = (cfg.triage_base_url or "").rstrip("/") + key = os.environ.get("TRACEFORGE_LLM_KEY", "") + model = cfg.triage_model + try: + import yaml # optional, same as --config handling + + with open( + os.path.join( + os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")), + "config.yaml", + ), + "r", + encoding="utf-8", + ) as f: + prof = yaml.safe_load(f) or {} + m = prof.get("model") or {} + # pinned provider block first (matches cfg.provider) + pv = (prof.get("providers") or {}).get(cfg.provider or "") or {} + if not base: + base = (pv.get("base_url") or m.get("base_url") or "").rstrip("/") + if not key: + key = os.environ.get(str(pv.get("key_env") or ""), "") + if not key: + key = os.environ.get(str(m.get("key_env") or ""), "") + if not key: + key = pv.get("key") or m.get("key") or "" + if not model: + model = cfg.triage_model or m.get("default") or "" + except Exception: + pass # fall through with whatever resolved + if not base: + raise RuntimeError( + "triage: no LLM endpoint β€” set triage_base_url in config " + "or configure the active Hermes profile (~/.hermes/config.yaml)" + ) + return base, key, model + + +def _chat_url(base: str) -> str: + return (base.rstrip("/") + "/").replace("//v1/", "/v1/") + "chat/completions" + + +def _post_json( + url: str, + payload: dict[str, Any], + api_key: str, + timeout: int, +) -> dict[str, Any]: + req = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}" if api_key else "", + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + # live telemetry: record usage/model for the progress bar (splice + # accumulator is shared; triage calls bucket to the current sid) + try: + from traceforge.splice import SPLICE_USAGE + SPLICE_USAGE.add(str(payload.get("model") or ""), data.get("usage")) + except Exception: + pass + return data + + +def _extract_content(data: dict[str, Any]) -> str: + try: + return data["choices"][0]["message"]["content"] or "" + except (KeyError, IndexError, TypeError): + return "" + + +def _request_variants( + schema: dict[str, Any], temperature: float +) -> list[dict[str, Any]]: + """Constrained-decoding request bodies, newest API first.""" + return [ + {"structured_outputs": {"json": schema}}, + {"guided_json": schema}, + {}, # bare: parse via _json_object_spans rescue + ] + + +def parse_labels(raw: Optional[str], valid_indices: set[int]) -> list[dict[str, Any]]: + """Parse a model reply into label dicts; lenient + range-checked. + + Accepts a bare JSON object, or JSON inside fences/prose (rescue via + balanced-brace spans). Out-of-window indices are dropped, unknown + actions are dropped, only the first label per index survives. + """ + if not raw: + return [] + from traceforge.improver.review import _json_object_spans + + candidates = [raw] + raw = raw.strip() + if raw.startswith("```"): + body = raw.split("\n", 1)[1] if "\n" in raw else "" + if body.rstrip().endswith("```"): + body = body.rstrip()[:-3] + candidates.append(body) + for span in _json_object_spans(raw): + candidates.append(span) + for c in candidates: + data = json_loads_lenient(c.strip() if c != raw else raw) + if not isinstance(data, dict): + continue + items = data.get("labels") + if not isinstance(items, list): + continue + out: dict[int, dict[str, Any]] = {} + for d in items: + if not isinstance(d, dict): + continue + i_raw = d.get("i") + if not isinstance(i_raw, (int, float)) or isinstance(i_raw, bool): + continue + i = int(i_raw) + action = str(d.get("action", "")).strip().lower() + if i not in valid_indices or action not in ACTIONS: + continue + if i in out: + continue # first label per index wins + row = {"i": i, "action": action} + if d.get("reason"): + row["reason"] = str(d["reason"]) + out[i] = row + if out: + return [out[i] for i in sorted(out)] + return [] + + +def _render_window( + trace: Trace, lo: int, hi: int, render_kwargs: dict +) -> str: + """Rendered window [lo, hi) with ABSOLUTE message indices.""" + sub = Trace( + session_id=trace.session_id, + model=trace.model, + messages=trace.messages[lo:hi], + ) + rk = dict(render_kwargs) + rk["index_offset"] = lo # absolute indices into the full trace + body = sub.render_compact(**rk) + return body.split("\n", 1)[1] if "\n" in body else body + + +def triage_windows( + trace: Trace, *, max_chars: int = 24_000, overlap: int = 4, + render_kwargs: dict | None = None, +) -> list[dict]: + """Sliding windows over the trace: [{lo, hi, rendered}, ...]. + + Reuses chunker._msg_size budgeting: windows cut at message + boundaries under max_chars with a small overlap for continuity. + """ + rk = render_kwargs or {} + msgs = trace.messages + n = len(msgs) + windows: list[dict] = [] + lo = 0 + while lo < n: + hi = lo + total = 0 + while hi < n and (hi == lo or total + chunker._msg_size(msgs[hi]) <= max_chars): + total += chunker._msg_size(msgs[hi]) + hi += 1 + hi = _snap_hi(msgs, hi, n) + # Never open a window on an orphaned tool result. Snap FORWARD, not + # back: the previous window's _snap_hi already pulled those results + # in, so skipping them here leaves no gap, and backing up instead + # would let the advance step return to the same lo forever. + while lo < hi and getattr(msgs[lo], "role", None) == "tool": + lo += 1 + if lo >= hi: + lo = hi + continue + rendered = _render_window(trace, lo, hi, rk) + windows.append({"lo": lo, "hi": hi, "rendered": rendered}) + if hi >= n: + break + lo = max(lo + 1, hi - overlap) + return windows + + +def _snap_hi(msgs: list, hi: int, n: int) -> int: + """Extend a window so it never ends between a call and its results. + + The judge reads each window as a self-contained trace. A window that ends + on an assistant message carrying tool_calls, with the matching tool + results in the next window, reads as a call that was never answered: the + judge reports P3 ("zero tool calls were made") and P6 ("no tool results + are available in the trace"), and the writer cannot repair it because the + results are real, just outside the window. + """ + while hi < n and getattr(msgs[hi], "role", None) == "tool": + prev = msgs[hi - 1] + if (getattr(prev, "role", None) == "tool" + or getattr(prev, "tool_calls", None)): + hi += 1 + else: + break + return hi + + +def _classify_window( + cfg: Config, + endpoint: tuple[str, str, str], + trace: Trace, + win: dict, + render_kwargs: dict, +) -> tuple[list[dict], Optional[str]]: + """One window -> labels. Returns (labels, error).""" + base, key, model = endpoint + url = _chat_url(base) + header = WINDOW_HEADER.format(lo=win["lo"] + 1, hi=win["hi"], n=len(trace.messages)) + user = header + win["rendered"] + valid = set(range(win["lo"] + 1, win["hi"] + 1)) # absolute 1-based + last_err: Optional[str] = None + for variant in _request_variants(LABELS_SCHEMA, cfg.triage_temperature): + payload: dict[str, Any] = { + "model": model or None, + "messages": [{"role": "user", "content": user}], + "temperature": cfg.triage_temperature, + "max_tokens": 4000, + } + if not model: + payload.pop("model") + payload.update(variant) + try: + data = _post_json(url, payload, key, cfg.triage_timeout_s) + except urllib.error.HTTPError as e: + # 400 on an unknown structured-output field: try next variant + last_err = f"HTTP {e.code}" + continue + except (urllib.error.URLError, TimeoutError, OSError) as e: + last_err = f"network: {e}" + continue + labels = parse_labels(_extract_content(data), valid) + if labels: + return labels, None + last_err = "empty/unparseable labels" + return [], last_err or "no variant succeeded" + + +def merge_window_labels( + window_labels: list[list[dict]], + windows: list[dict], +) -> dict[int, str]: + """Merge per-window labels into one {absolute_index: action} map. + + Overlapping regions: the LATER window wins (it sees strictly more + left-context for those messages) β€” same policy as chunker's + continuity tails, applied to labels instead of digests. + Windows that returned nothing leave their indices unlabeled; those + default to 'rephrase' at routing time (conservative). + """ + merged: dict[int, str] = {} + for win, labels in zip(windows, window_labels): + for lab in labels: + merged[lab["i"]] = lab["action"] + return merged + + +def route_trace(labels: dict[int, str]) -> str: + """Per-trace routing decision from the merged label map. + + splice : only keep/drop/summarize β€” no writer session needed; the + splicer can build the ideal trace mechanically + agent : any rephrase/fact_check β€” the improver owns those turns + reject : nothing survives (no user task / everything dropped) + """ + if not labels: + return "agent" # no labels = nothing proven safe; let the improver run + actions = set(labels.values()) + if "rephrase" in actions or "fact_check" in actions: + return "agent" + if actions <= {"drop"}: # everything dropped: nothing to build + return "reject" + return "splice" + + +def triage_trace( + cfg: Config, trace: Trace, *, force: bool = False, + errors_out: Optional[list[str]] = None, +) -> Optional[dict]: + """Label one trace end-to-end. Returns the persisted triage record: + {"session_id", "route", "labels", "windows", "errors"} or None on + total failure (every window failed). Cached: an existing + data/triage/.json is returned as-is unless force. + """ + out_path = os.path.join(cfg.triage_dir, f"{trace.session_id}.json") + if os.path.exists(out_path) and not force: + try: + with open(out_path, encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + pass + endpoint = _resolve_endpoint(cfg) + render_kwargs = { + "user_chars": cfg.user_chars, + "assistant_chars": cfg.assistant_chars, + "tool_result_chars": cfg.tool_result_chars, + "tool_args_chars": cfg.tool_args_chars, + "include_reasoning": cfg.include_reasoning, + } + windows = triage_windows( + trace, + max_chars=cfg.triage_max_chars, + overlap=cfg.triage_overlap_messages, + render_kwargs=render_kwargs, + ) + if not windows: + return None + errors: list[str] = [] + results: list[list[dict]] = [[] for _ in windows] + lock = threading.Lock() + done_count = [0] + + def _work(idx: int, win: dict) -> None: + labels, err = _classify_window(cfg, endpoint, trace, win, render_kwargs) + results[idx] = labels + with lock: + done_count[0] += 1 + if err: + errors.append(f"window {win['lo'] + 1}-{win['hi']}: {err}") + + with ThreadPoolExecutor(max_workers=max(1, cfg.triage_workers)) as ex: + futs = [ex.submit(_work, i, w) for i, w in enumerate(windows)] + for f in as_completed(futs): + f.result() # propagate programming errors, keep infra errors + + merged = merge_window_labels(results, windows) + labeled = sum(1 for lab in results if lab) + if labeled == 0: + # Total failure. The caller must be able to record WHY: an endpoint + # outage and a hard trace are indistinguishable from the bare + # "triage failed for every window" verdict (2026-10-26: 191 traces + # lost that way in 37 minutes, all coded D-UNK-01). + if errors_out is not None: + errors_out.extend(errors[:20] or ["no window produced labels"]) + return None + record = { + "session_id": trace.session_id, + "route": route_trace(merged), + "labels": [ + {"i": i, "action": a} for i, a in sorted(merged.items()) + ], + "windows": [ + {"lo": w["lo"], "hi": w["hi"]} for w in windows + ], + "n_windows": len(windows), + "windows_ok": labeled, + "errors": errors[:20], + } + os.makedirs(cfg.triage_dir, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as f: + json.dump(record, f, ensure_ascii=False, indent=1) + return record \ No newline at end of file diff --git a/traceforge/validate.py b/traceforge/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ea99e343c5bc3ab6a42af6888872767267cc45 --- /dev/null +++ b/traceforge/validate.py @@ -0,0 +1,391 @@ +"""Structural validation of an (improved) trace. + +Checks that are format-level facts, independent of the quality guidelines β€” +guideline-scoring (quality rules) will be defined together with the user and +plugged in as a separate, stricter validator later. +""" +from __future__ import annotations + +import difflib +import json +import re +from typing import Optional + +from traceforge.trace import Message, Trace + +# template substitution tokens that leaked into message content, e.g. +# "exec_reader_search_placeholder_0#" (agent-VM prompt template bug in +# the source session). Word-ish identifier + "_placeholder_" + digits. +_TEMPLATE_LEAK_RE = re.compile( + r"[a-z][a-z0-9_]*_placeholder_\d+", re.IGNORECASE) + +# rendering markers from trace.py's prompt rendering (the improver's INPUT +# format). The improver occasionally echoes them into assistant prose β€” +# e.g. a row whose whole content is "(tool-only turn)" β€” which a trainee +# must never see. Length-based degenerate checks miss these because the +# marker is 16 chars. +_RENDER_MARKER_RE = re.compile( + r"^\s*(?:\(tool-only turn\)" + r"|\[\d+\]\s+(?:ASSISTANT|USER|SYSTEM|TOOL)\b" + r"|->\s*TOOL\s+\S+\s+args=)", + re.IGNORECASE) + +VALID_ROLES = {"system", "user", "assistant", "tool"} + +# An explicit translation request makes a language switch the CORRECT answer, +# so the P5 gate must not fire on it. +_LANG_SWITCH_RE = re.compile( + r"\b(?:ΓΌbersetz|translate|translation|auf englisch|auf deutsch|" + r"in english|in german|ins englische|ins deutsche)\b", re.IGNORECASE) + +# Pipeline framing markers that belong to the improver's INPUT, not to the +# training target: the splitter prepends "[trace segment k of n β€” ...]" and +# the codex reader prepends "[subagent brief β€” role: ...]". A user never +# types these, so they are stripped from user rows before promotion. +_SCAFFOLD_RE = re.compile( + r"\[trace segment \d+ of \d+[^\]]*\]|\[subagent brief[^\]]*\]") + +# Code-ish spans removed before language scoring: fenced/inline code, +# markdown link targets, URLs, absolute paths, and filenames. Their +# identifiers are English by convention and would otherwise outvote the +# prose's function words. +_CODEISH_RE = re.compile( + r"```.*?```" + r"|`[^`\n]*`" + r"|!?\[[^\]]*\]\([^)]*\)" + r"|https?://\S+" + r"|(?:[A-Za-z]:)?(?:/[\w.\-]+){2,}" + r"|[\w.\-]+\.(?:py|ts|tsx|js|mjs|go|rs|md|json|ya?ml|sh|toml|txt)\b", + re.S) + + +def _confident_lang(text: str, min_words: int) -> Optional[str]: + """'de'/'en' only when the call is unambiguous, else None. + + Requires min_words function words AND a 70% dominant share, so a mixed + task or an answer quoting the other language does not trip the gate. + Code, paths, URLs and markdown link targets are removed first: an + English file path inside a German answer ("...-api-and-paired-...") + otherwise supplies enough English function words to flip the call. + """ + from traceforge.classify import language_scores + + de, en = language_scores(_CODEISH_RE.sub(" ", text)) + total = de + en + if total < min_words: + return None + if max(de, en) / total < 0.7: + return None + return "de" if de > en else "en" + + +def strip_scaffold_markers(t: Trace) -> int: + """Remove pipeline framing markers from user rows (in place). + + Returns the number of rows changed. A row that is ONLY a marker is left + untouched (stripping it would create an empty user message). + """ + changed = 0 + for m in t.messages: + if m.role != "user" or not m.content: + continue + new = _SCAFFOLD_RE.sub("", m.content) + if new == m.content: + continue + new = re.sub(r"[ \t]+\n", "\n", new) + new = re.sub(r"\n{3,}", "\n\n", new).strip() + if not new: + continue + m.content = new + changed += 1 + return changed + + +def _norm_for_similarity(text: str) -> str: + """Collapse whitespace and case so formatting noise can't hide an echo.""" + return re.sub(r"\s+", " ", text).strip().lower() + + +# Unmistakable reframe-prompt text. The reframe model echoes its own rules +# into the task message; a user never types any of this. The sanitizer cuts +# it at reframe time, and this gate is the backstop that makes a leak fail +# loudly instead of promoting silently (measured: 68 of 1044 traces). +_REFRAME_PROMPT_RE = re.compile( + r"Same language as the current task message" + r"|The new task must ask ONLY" + r"|no meta commentary" + r"|answerable EXACTLY by the final answer" + r"|Reply with ONLY the JSON object" + r"|write that request as the new opening task message" + r"|A synthetic ideal-world agent trace" + r"|no mention of traces or reframing", re.IGNORECASE) + +# Harness dispatch briefs are NOT user messages. A codex subagent rollout opens +# with the coordinator's dispatch text ("You are working inside Orca, a +# multi-agent IDE. You are a dispatched worker. Your coordinator's Orca session +# ID is: ... === CLI COMMANDS ==="), 6-7KB of orchestration boilerplate with +# session/task ids and CLI docs. Promoted as-is, the trainee learns to emit +# harness plumbing as the user turn. Measured 2026-10-22: 6 of 1046 improved +# traces, 4 of them promoted by the running batch β€” so gate it, don't just +# clean it. +# +# STRONG markers only (2026-10-24). The bare phrase `orca orchestration` used +# to be an alternative here, and it is a CLI NAME, not boilerplate: it appears +# in legitimate task rows ("...report via orca orchestration"), in the +# harness's own "You have N orchestration messages" notifications, and quoted +# inside compaction summaries. Measured on the raw pool: 43 traces matched +# only that phrase, 42 of them with a real task row and no brief at all, and +# 2 of the 20 recorded gate failures were clean traces killed by a quoted +# mention at row 228. The four strong markers below appear in every real +# brief and in nothing else. +_HARNESS_BRIEF_RE = re.compile( + r"You are working inside Orca, a multi-agent IDE" + r"|You are a dispatched worker" + r"|Your coordinator's Orca session ID is" + r"|=== CLI COMMANDS ===", re.IGNORECASE) + +# Plumbing user rows: harness/validator chatter that an ideal trace routes +# around. These MAY be dropped. A substantive user turn may NOT. +_PLUMBING_USER_RE = re.compile( + r"(previous (final )?(response|answer|attempt).{0,40}" + r"(reject|invalid|not valid|validation error)" + r"|(rejected? by the).{0,40}(validator|contract)" + r"|^validation errors?\b|output contract validator" + r"|\[trace segment \d+ of \d+" + r"|\[subagent brief" + r"|\[CONTEXT COMPACTION" + r"|This session is being continued" + r"|^You have \d+ orchestration messages?\.)", re.IGNORECASE) + + +def _is_plumbing_user_row(m) -> bool: + """A user row that is harness/validator plumbing, not a user request.""" + if m.role != "user": + return False + c = (m.content or "")[:400] + return bool(_PLUMBING_USER_RE.search(c)) or bool(_HARNESS_BRIEF_RE.search(c)) + + +def substantive_user_turns(t: Trace) -> int: + """Count genuine user turns, ignoring plumbing and consecutive merges. + + A turn group is a maximal run of user rows; it counts once when at least + one row in it is substantive. Consecutive rows are one turn because the + gate shape merges them (`_merge_same_role`), so a merge must not read as + a dropped turn. + """ + n = 0 + in_group = False + has_substantive = False + for m in t.messages: + if m.role == "user": + if not in_group: + in_group, has_substantive = True, False + if not _is_plumbing_user_row(m): + has_substantive = True + else: + if in_group and has_substantive: + n += 1 + in_group = False + if in_group and has_substantive: + n += 1 + return n + + +def validate_structural(t: Trace) -> list[str]: + """Return a list of structural errors ([] = valid).""" + errors: list[str] = [] + if not t.messages: + return ["trace has no messages"] + + roles = [m.role for m in t.messages] + if roles[0] == "assistant": + errors.append("first message is assistant (expected user or system)") + if "user" not in roles: + errors.append("no user message in trace") + if roles[-1] not in ("assistant",): + errors.append(f"last message role is {roles[-1]!r} (expected assistant)") + + # reframe-prompt leak: the reframe model echoes its own rule list into + # the task message. A user never types it, so the row is not a user row. + for i, m in enumerate(t.messages): + if m.role != "user" or not m.content: + continue + hit = _REFRAME_PROMPT_RE.search(m.content) + if hit: + errors.append( + f"[{i}] user message contains reframe-prompt text " + f"({hit.group(0)!r}) β€” the task must be the user's own words") + hb = _HARNESS_BRIEF_RE.search(m.content) + if hb: + errors.append( + f"[{i}] user message is a harness dispatch brief " + f"({hb.group(0)!r}) β€” not the user's own words") + + # template placeholder leak: source-agent prompt templates sometimes + # bleed their substitution tokens ("exec_reader_search_placeholder_0#") + # into the assistant prose β€” a trainee must never see them. Any + # message STARTING with one fails structurally (repair gets a + # precise citation instead of the judge discovering it late). + for i, m in enumerate(t.messages): + if m.role != "assistant": + continue + head = (m.content or "").lstrip()[:120] + leak = _TEMPLATE_LEAK_RE.match(head) + if leak: + errors.append( + f"[{i}] assistant message starts with a leaked template " + f"placeholder token ({leak.group(0)!r}) " + "β€” rewrite the row as clean prose") + marker = _RENDER_MARKER_RE.match(head) + if marker: + errors.append( + f"[{i}] assistant message starts with a leaked rendering " + f"marker ({marker.group(0).strip()!r}) " + "β€” rewrite the row as clean prose") + + # brief echo: the writer sometimes answers a long subagent brief by + # re-presenting it ("verbatim copy of the user's brief, no new + # reasoning"). The judge catches it late and expensively; a similarity + # check against the opening task catches it for free. Only the FIRST + # user task message is compared β€” quoting a later user message back is + # legitimate (confirming a correction), echoing the brief never is. + brief = t.first_task_message() or "" + if len(brief) >= 200: + brief_norm = _norm_for_similarity(brief) + for i, m in enumerate(t.messages): + if m.role != "assistant" or not m.content: + continue + if len(m.content) < 200: + continue + ratio = difflib.SequenceMatcher( + None, brief_norm, _norm_for_similarity(m.content), + autojunk=False).ratio() + if ratio >= 0.85: + errors.append( + f"[{i}] assistant message is a {ratio:.0%} verbatim echo " + "of the user's opening brief β€” answer the task instead " + "of re-presenting it") + + # First-turn echo: the writer sometimes opens by restating the task + # verbatim instead of doing work. The brief-echo check above needs 200 + # chars on both sides, so a short echo slips through β€” measured + # 2026-10-22: 121 of 1094 traces (11%) had a 1.00-similarity first + # assistant turn with no tool call, and the user row was the agent's own + # plan ("Ich prΓΌfe die drei Artefakte zuerst …"). + first_asst = next((i for i, m in enumerate(t.messages) + if m.role == "assistant"), None) + if first_asst is not None and brief: + m = t.messages[first_asst] + body = (m.content or "").strip() + if body and not m.tool_calls and len(body) >= 60: + ratio = difflib.SequenceMatcher( + None, _norm_for_similarity(brief[:600]), + _norm_for_similarity(body[:600]), autojunk=False).ratio() + if ratio >= 0.9: + errors.append( + f"[{first_asst}] first assistant turn is a {ratio:.0%} " + "verbatim echo of the user's task with no tool call β€” " + "do work instead of restating the brief") + + # P5 language: one language per trace. The judge catches a German task + # answered in English but misses the reverse, and the source traces are + # themselves mixed (an English brief answered in German by the source + # agent). A trainee must learn to answer in the task's language, so a + # CONFIDENT mismatch is a structural error with a precise citation. + # Confidence guards: both sides need enough function words and a clear + # ratio, so a German answer quoting English code (or a mixed task) does + # not trip it; an explicit translation request is exempt. + final_i = next((i for i in range(len(t.messages) - 1, -1, -1) + if t.messages[i].role == "assistant"), None) + if final_i is not None and not _LANG_SWITCH_RE.search(brief): + final = t.messages[final_i].content or "" + if len(final) >= 120: + # 300 chars of the task: the instruction is at the top; a long + # quoted log further down (a German report pasting an English + # CI log) would otherwise flip the task's language. + task_lang = _confident_lang(brief[:300], 3) + ans_lang = _confident_lang(final[:800], 4) + if task_lang and ans_lang and task_lang != ans_lang: + errors.append( + f"[{final_i}] final answer is in {ans_lang} but the task " + f"is in {task_lang} β€” answer in the task's language (P5)") + + # role alternation: no two user or two assistant messages back-to-back + prev: Optional[str] = None + for i, m in enumerate(t.messages): + if m.role not in VALID_ROLES: + errors.append(f"[{i}] invalid role {m.role!r}") + if m.role == "user" and prev == "user": + errors.append(f"[{i}] two consecutive user messages") + if m.role == "assistant" and prev == "assistant" and not ( + m.tool_calls or t.messages[i - 1].tool_calls + ): + errors.append(f"[{i}] two consecutive text-only assistant turns") + prev = m.role + + # tool pairing: every call answered exactly once, every result has a call + called: dict[str, str] = {} # id -> tool name + answered: dict[str, int] = {} + for i, m in enumerate(t.messages): + for tc in m.tool_calls: + if not tc.id: + errors.append(f"[{i}] tool_call without id ({tc.name})") + elif tc.id in called: + errors.append(f"[{i}] duplicate tool_call id {tc.id}") + else: + called[tc.id] = tc.name + if not tc.name: + errors.append(f"[{i}] tool_call without name") + args = tc.arguments or "{}" + if not isinstance(args, str): + errors.append( + f"[{i}] tool_call {tc.name} arguments must be a JSON string" + ) + else: + try: + json.loads(args) + except json.JSONDecodeError as e: + errors.append( + f"[{i}] tool_call {tc.name} arguments not valid JSON: {e}" + ) + if m.role == "tool": + tcid = m.tool_call_id + if not tcid: + errors.append(f"[{i}] tool result without tool_call_id") + elif tcid not in called: + errors.append(f"[{i}] tool result references unknown call {tcid}") + else: + answered[tcid] = answered.get(tcid, 0) + 1 + if not (m.content or "").strip(): + # empty result text (content None/"" β€” 019ff135 class: + # send_message {} -> content=None with tool_name set, so + # the old content-None-AND-tool_name-None check passed + # it). An SFT trainee must never learn to expect or emit + # result-less tool rows. + errors.append(f"[{i}] tool result is empty") + + for tcid, name in called.items(): + n = answered.get(tcid, 0) + if n == 0: + errors.append(f"tool call {tcid} ({name}) has no result") + elif n > 1: + errors.append(f"tool call {tcid} ({name}) has {n} results") + + # empty-content sanity + for i, m in enumerate(t.messages): + if m.role in ("user", "assistant") and m.content in ("", None) and not m.tool_calls: + errors.append(f"[{i}] {m.role} message with empty content and no tool calls") + + # degenerate short-prose assistant rows ("\n\n" connectors, "Now + # the fix:" fragments) β€” training noise; only NON-FINAL rows (the + # final answer may legitimately be terse) and rows without calls. + last = len(t.messages) - 1 + for i, m in enumerate(t.messages): + if (m.role == "assistant" and i != last and not m.tool_calls + and m.content and len(m.content.strip()) < 15): + errors.append( + f"[{i}] degenerate assistant prose ({len(m.content.strip())} chars): " + f"{m.content.strip()[:40]!r}") + + return errors \ No newline at end of file