flozi00 commited on
Commit
a8fbc2f
·
verified ·
1 Parent(s): 2b7058b

traceforge space

Browse files
BUILD.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
- "built_at": "2026-10-27T13:19:56",
3
- "commit": "b412454",
4
  "files": 49,
5
  "with_store": false,
6
  "with_card": true
 
1
  {
2
+ "built_at": "2026-10-27T17:07:51",
3
+ "commit": "173cb13",
4
  "files": 49,
5
  "with_store": false,
6
  "with_card": true
data/dataset/dataset_card.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
- "traces": 3875,
3
  "languages": {
4
- "en": 3720,
5
  "de": 155
6
  },
7
  "language_pct": {
@@ -15,34 +15,34 @@
15
  ],
16
  "rows": {
17
  "review": {
18
- "en": 1177,
19
  "de": 63,
20
- "total": 1240
21
  },
22
  "implement": {
23
- "en": 647,
24
  "de": 39,
25
- "total": 686
26
  },
27
  "debug": {
28
- "en": 630,
29
  "de": 22,
30
- "total": 652
31
  },
32
  "research": {
33
- "en": 516,
34
  "de": 6,
35
- "total": 522
36
  },
37
  "other": {
38
- "en": 282,
39
  "de": 4,
40
- "total": 286
41
  },
42
  "test": {
43
- "en": 147,
44
  "de": 11,
45
- "total": 158
46
  },
47
  "ops": {
48
  "en": 105,
@@ -55,9 +55,9 @@
55
  "total": 87
56
  },
57
  "refactor": {
58
- "en": 74,
59
  "de": 3,
60
- "total": 77
61
  },
62
  "creative": {
63
  "en": 49,
@@ -71,31 +71,31 @@
71
  }
72
  },
73
  "totals": {
74
- "en": 3720,
75
  "de": 155,
76
- "total": 3875
77
  }
78
  },
79
  "tools_and_prompt": {
80
  "tools": {
81
- "rows": 3875,
82
- "with_tools": 3874,
83
  "pct": 100.0,
84
  "by_source": {
85
- "derived-from-observed-calls": 3469,
86
- "claude-request-shape": 405,
87
  "none": 1
88
  }
89
  },
90
  "system_prompt": {
91
- "rows": 3875,
92
- "with_system": 1716,
93
- "pct": 44.3,
94
  "by_source": {
95
- "none": 2159,
96
  "raw-system-row": 663,
97
- "claude-request-shape": 405,
98
- "codex-base-instructions": 354,
99
  "in-trace": 292,
100
  "hermes-system-prompts": 2
101
  }
@@ -104,8 +104,8 @@
104
  "provenance": {
105
  "basis": "session-id prefix (the improver overwrites meta.source)",
106
  "counts": {
107
- "real-session": 1720,
108
- "subagent": 691,
109
  "nemotron-agentic-ka": 581,
110
  "split-segment": 509,
111
  "nemotron-agentic-tc": 370,
@@ -115,11 +115,11 @@
115
  "nemotron-swe-e8ff278b": 1
116
  },
117
  "pct": {
118
- "real-session": 44.4,
119
- "subagent": 17.8,
120
- "nemotron-agentic-ka": 15.0,
121
- "split-segment": 13.1,
122
- "nemotron-agentic-tc": 9.5,
123
  "nemotron-swe-3c63eaaa": 0.0,
124
  "nemotron-swe-6ca796dd": 0.0,
125
  "nemotron-swe-bc9ee6b7": 0.0,
@@ -127,58 +127,58 @@
127
  }
128
  },
129
  "agent_roles": {
130
- "orchestrator": 3184,
131
- "subagent": 691
132
  },
133
  "tasks": {
134
- "review": 1240,
135
- "implement": 686,
136
- "debug": 652,
137
- "research": 522,
138
- "other": 286,
139
- "test": 158,
140
  "ops": 107,
141
  "data-analysis": 87,
142
- "refactor": 77,
143
  "creative": 49,
144
  "office-docs": 11
145
  },
146
  "messages": {
147
- "total": 127999,
148
  "by_role": {
149
- "system": 1716,
150
- "user": 4294,
151
- "assistant": 62022,
152
- "tool": 59967
153
  },
154
  "per_trace": {
155
- "mean": 33.0,
156
- "median": 22,
157
- "p90": 70,
158
  "max": 252
159
  }
160
  },
161
  "tool_calls": {
162
- "total": 59967,
163
  "per_trace": {
164
- "mean": 15.5,
165
- "median": 10,
166
- "p90": 33,
167
  "max": 125
168
  },
169
  "traces_with_none": 1,
170
  "distinct_tools": 553,
171
  "top_tools": {
172
- "Bash": 22114,
173
- "exec": 11489,
174
  "terminal": 3660,
175
  "read_file": 2835,
176
- "Read": 2551,
177
- "Edit": 2053,
178
  "search_files": 1716,
179
  "execute_code": 1581,
180
  "tool_call": 1511,
181
- "Grep": 914,
182
  "skill_view": 861,
183
  "patch": 738,
184
  "unlock_discoverable_agent_tool": 457,
@@ -189,43 +189,43 @@
189
  "user_turns": {
190
  "per_trace": {
191
  "mean": 1.1,
192
- "median": 1,
193
  "p90": 1,
194
  "max": 18
195
  },
196
- "single_turn_traces": 3570,
197
- "multi_turn_traces": 305
198
  },
199
  "tokens": {
200
  "method": "chars/4 (prose proxy); scale by ~1.15 for a central estimate",
201
  "budget": 131072,
202
- "chars_total": 313006272,
203
- "est_total": 78251568,
204
  "per_trace": {
205
- "mean": 20193.6,
206
- "median": 16324,
207
- "p90": 46390,
208
  "max": 121769
209
  },
210
  "fits_budget": {
211
  "4096": {
212
  "traces": 1026,
213
- "pct": 26.5
214
  },
215
  "8192": {
216
  "traces": 1264,
217
- "pct": 32.6
218
  },
219
  "16384": {
220
- "traces": 1944,
221
- "pct": 50.2
222
  },
223
  "65536": {
224
- "traces": 3826,
225
- "pct": 98.7
226
  },
227
  "131072": {
228
- "traces": 3875,
229
  "pct": 100.0
230
  }
231
  }
@@ -234,10 +234,10 @@
234
  "supervision": {
235
  "contract": "loss on assistant spans only; tool/user/system spans are masked context",
236
  "chars_by_role": {
237
- "system": 12352139,
238
- "user": 5725036,
239
- "assistant": 11998958,
240
- "tool": 282930139
241
  },
242
  "assistant_share_pct": 3.8,
243
  "tool_share_pct": 90.4
@@ -248,54 +248,54 @@
248
  "D1": {
249
  "label": "one-shot / single edit",
250
  "traces": 111,
251
- "pct": 2.9
252
  },
253
  "D2": {
254
  "label": "short loop",
255
  "traces": 680,
256
- "pct": 17.5
257
  },
258
  "D3": {
259
  "label": "multi-step",
260
- "traces": 1246,
261
- "pct": 32.2
262
  },
263
  "D4": {
264
  "label": "long session",
265
- "traces": 840,
266
- "pct": 21.7
267
  },
268
  "D5": {
269
  "label": "very long session",
270
- "traces": 998,
271
- "pct": 25.8
272
  }
273
  },
274
  "tool_calls": {
275
- "mean": 15.5,
276
- "median": 10,
277
- "p90": 33,
278
  "max": 125
279
  },
280
  "messages": {
281
- "mean": 33.0,
282
- "median": 22,
283
- "p90": 70,
284
  "max": 252
285
  },
286
  "est_tokens": {
287
- "mean": 20193.6,
288
- "median": 16324,
289
- "p90": 46390,
290
  "max": 121769
291
  }
292
  },
293
  "safety_filter": {
294
  "applied": true,
295
- "traces_in": 4103,
296
- "traces_kept": 3875,
297
- "traces_excluded": 228,
298
- "excluded_secret": 204,
299
  "excluded_personal": 24,
300
  "rule": "drop any trace carrying credential material (passwords, tokens, private keys, connection strings with real credentials) or third-party personal data (private legal correspondence, home addresses, phone numbers, IBANs). Code, internal hostnames, business contact addresses and the owner's own identity stay.",
301
  "report": "safety_report.json"
@@ -304,7 +304,7 @@
304
  "classes": {
305
  "time_sensitive": 13,
306
  "time_anchored": 12,
307
- "not_time_sensitive": 3850
308
  },
309
  "labels": {
310
  "time_sensitive": "answer decays: temporal fact, no retrieval tool",
 
1
  {
2
+ "traces": 3924,
3
  "languages": {
4
+ "en": 3769,
5
  "de": 155
6
  },
7
  "language_pct": {
 
15
  ],
16
  "rows": {
17
  "review": {
18
+ "en": 1202,
19
  "de": 63,
20
+ "total": 1265
21
  },
22
  "implement": {
23
+ "en": 657,
24
  "de": 39,
25
+ "total": 696
26
  },
27
  "debug": {
28
+ "en": 634,
29
  "de": 22,
30
+ "total": 656
31
  },
32
  "research": {
33
+ "en": 521,
34
  "de": 6,
35
+ "total": 527
36
  },
37
  "other": {
38
+ "en": 283,
39
  "de": 4,
40
+ "total": 287
41
  },
42
  "test": {
43
+ "en": 149,
44
  "de": 11,
45
+ "total": 160
46
  },
47
  "ops": {
48
  "en": 105,
 
55
  "total": 87
56
  },
57
  "refactor": {
58
+ "en": 76,
59
  "de": 3,
60
+ "total": 79
61
  },
62
  "creative": {
63
  "en": 49,
 
71
  }
72
  },
73
  "totals": {
74
+ "en": 3769,
75
  "de": 155,
76
+ "total": 3924
77
  }
78
  },
79
  "tools_and_prompt": {
80
  "tools": {
81
+ "rows": 3924,
82
+ "with_tools": 3923,
83
  "pct": 100.0,
84
  "by_source": {
85
+ "derived-from-observed-calls": 3509,
86
+ "claude-request-shape": 414,
87
  "none": 1
88
  }
89
  },
90
  "system_prompt": {
91
+ "rows": 3924,
92
+ "with_system": 1726,
93
+ "pct": 44.0,
94
  "by_source": {
95
+ "none": 2198,
96
  "raw-system-row": 663,
97
+ "claude-request-shape": 414,
98
+ "codex-base-instructions": 355,
99
  "in-trace": 292,
100
  "hermes-system-prompts": 2
101
  }
 
104
  "provenance": {
105
  "basis": "session-id prefix (the improver overwrites meta.source)",
106
  "counts": {
107
+ "real-session": 1723,
108
+ "subagent": 737,
109
  "nemotron-agentic-ka": 581,
110
  "split-segment": 509,
111
  "nemotron-agentic-tc": 370,
 
115
  "nemotron-swe-e8ff278b": 1
116
  },
117
  "pct": {
118
+ "real-session": 43.9,
119
+ "subagent": 18.8,
120
+ "nemotron-agentic-ka": 14.8,
121
+ "split-segment": 13.0,
122
+ "nemotron-agentic-tc": 9.4,
123
  "nemotron-swe-3c63eaaa": 0.0,
124
  "nemotron-swe-6ca796dd": 0.0,
125
  "nemotron-swe-bc9ee6b7": 0.0,
 
127
  }
128
  },
129
  "agent_roles": {
130
+ "orchestrator": 3187,
131
+ "subagent": 737
132
  },
133
  "tasks": {
134
+ "review": 1265,
135
+ "implement": 696,
136
+ "debug": 656,
137
+ "research": 527,
138
+ "other": 287,
139
+ "test": 160,
140
  "ops": 107,
141
  "data-analysis": 87,
142
+ "refactor": 79,
143
  "creative": 49,
144
  "office-docs": 11
145
  },
146
  "messages": {
147
+ "total": 130671,
148
  "by_role": {
149
+ "system": 1726,
150
+ "user": 4345,
151
+ "assistant": 63391,
152
+ "tool": 61209
153
  },
154
  "per_trace": {
155
+ "mean": 33.3,
156
+ "median": 22.0,
157
+ "p90": 71,
158
  "max": 252
159
  }
160
  },
161
  "tool_calls": {
162
+ "total": 61209,
163
  "per_trace": {
164
+ "mean": 15.6,
165
+ "median": 10.0,
166
+ "p90": 34,
167
  "max": 125
168
  },
169
  "traces_with_none": 1,
170
  "distinct_tools": 553,
171
  "top_tools": {
172
+ "Bash": 22977,
173
+ "exec": 11517,
174
  "terminal": 3660,
175
  "read_file": 2835,
176
+ "Read": 2673,
177
+ "Edit": 2210,
178
  "search_files": 1716,
179
  "execute_code": 1581,
180
  "tool_call": 1511,
181
+ "Grep": 939,
182
  "skill_view": 861,
183
  "patch": 738,
184
  "unlock_discoverable_agent_tool": 457,
 
189
  "user_turns": {
190
  "per_trace": {
191
  "mean": 1.1,
192
+ "median": 1.0,
193
  "p90": 1,
194
  "max": 18
195
  },
196
+ "single_turn_traces": 3617,
197
+ "multi_turn_traces": 307
198
  },
199
  "tokens": {
200
  "method": "chars/4 (prose proxy); scale by ~1.15 for a central estimate",
201
  "budget": 131072,
202
+ "chars_total": 318237136,
203
+ "est_total": 79559284,
204
  "per_trace": {
205
+ "mean": 20274.7,
206
+ "median": 16510.5,
207
+ "p90": 46090,
208
  "max": 121769
209
  },
210
  "fits_budget": {
211
  "4096": {
212
  "traces": 1026,
213
+ "pct": 26.1
214
  },
215
  "8192": {
216
  "traces": 1264,
217
+ "pct": 32.2
218
  },
219
  "16384": {
220
+ "traces": 1950,
221
+ "pct": 49.7
222
  },
223
  "65536": {
224
+ "traces": 3875,
225
+ "pct": 98.8
226
  },
227
  "131072": {
228
+ "traces": 3924,
229
  "pct": 100.0
230
  }
231
  }
 
234
  "supervision": {
235
  "contract": "loss on assistant spans only; tool/user/system spans are masked context",
236
  "chars_by_role": {
237
+ "system": 12435083,
238
+ "user": 5887077,
239
+ "assistant": 12168420,
240
+ "tool": 287746556
241
  },
242
  "assistant_share_pct": 3.8,
243
  "tool_share_pct": 90.4
 
248
  "D1": {
249
  "label": "one-shot / single edit",
250
  "traces": 111,
251
+ "pct": 2.8
252
  },
253
  "D2": {
254
  "label": "short loop",
255
  "traces": 680,
256
+ "pct": 17.3
257
  },
258
  "D3": {
259
  "label": "multi-step",
260
+ "traces": 1253,
261
+ "pct": 31.9
262
  },
263
  "D4": {
264
  "label": "long session",
265
+ "traces": 857,
266
+ "pct": 21.8
267
  },
268
  "D5": {
269
  "label": "very long session",
270
+ "traces": 1023,
271
+ "pct": 26.1
272
  }
273
  },
274
  "tool_calls": {
275
+ "mean": 15.6,
276
+ "median": 10.0,
277
+ "p90": 34,
278
  "max": 125
279
  },
280
  "messages": {
281
+ "mean": 33.3,
282
+ "median": 22.0,
283
+ "p90": 71,
284
  "max": 252
285
  },
286
  "est_tokens": {
287
+ "mean": 20274.7,
288
+ "median": 16510.5,
289
+ "p90": 46090,
290
  "max": 121769
291
  }
292
  },
293
  "safety_filter": {
294
  "applied": true,
295
+ "traces_in": 4155,
296
+ "traces_kept": 3924,
297
+ "traces_excluded": 231,
298
+ "excluded_secret": 207,
299
  "excluded_personal": 24,
300
  "rule": "drop any trace carrying credential material (passwords, tokens, private keys, connection strings with real credentials) or third-party personal data (private legal correspondence, home addresses, phone numbers, IBANs). Code, internal hostnames, business contact addresses and the owner's own identity stay.",
301
  "report": "safety_report.json"
 
304
  "classes": {
305
  "time_sensitive": 13,
306
  "time_anchored": 12,
307
+ "not_time_sensitive": 3899
308
  },
309
  "labels": {
310
  "time_sensitive": "answer decays: temporal fact, no retrieval tool",
scratch/unify_space3/meta.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "version": 1,
3
+ "created_at": 1793117281.461422
4
+ }
traceforge/cli.py CHANGED
@@ -449,37 +449,46 @@ def _improve_one(cfg, r, sid, max_attempts, skip_existing, prog, liv=None):
449
 
450
  try:
451
  if cfg.splice:
452
- # windowed direct-HTTP engine (no agent runtime)
 
 
453
  from traceforge.improver import ImproverResult
454
- from traceforge.splice import improve_trace_splice
455
 
456
- sres = improve_trace_splice(
457
- cfg, t, status_cb=cb, max_attempts=max_attempts,
458
- )
459
  res = ImproverResult(
460
- session_id=sid, ok=sres["ok"], improved=sres["improved"],
461
- reply=sres.get("route", ""), errors=sres["errors"],
462
- attempts=sres["attempts"],
463
- duration_s=sres["duration_s"],
464
  )
465
- if res.ok and res.improved is not None \
466
- and sres.get("route") != "regate":
467
  out = os.path.join(cfg.improved_dir, f"{sid}.json")
468
  res.improved.save(out)
469
- prog.write(
470
- f"IMPROVED {sid} ({res.attempts} attempts, "
471
- f"{res.duration_s:.0f}s, {res.reply})"
472
- )
473
- elif res.ok:
474
- prog.write(
475
- f"IMPROVED {sid} (regate, {res.duration_s:.0f}s)"
476
- )
 
 
477
  else:
478
  prog.write(f"FAILED {sid}: {res.errors}")
479
- _write_manifest_row(cfg, res.to_dict())
 
 
 
 
480
  if liv is not None:
481
  liv.registry.end(sid)
482
  return sid, res
 
 
 
483
  res = improve_trace(
484
  cfg, t, max_attempts=max_attempts, status_cb=cb
485
  )
 
449
 
450
  try:
451
  if cfg.splice:
452
+ # The SAME pipeline the Gradio app runs. One implementation, so
453
+ # a gate, a no-op rule or a measurement can never drift between
454
+ # the batch and the Space again.
455
  from traceforge.improver import ImproverResult
456
+ from traceforge.space.pipeline import process_trace
457
 
458
+ pres = process_trace(cfg, t, max_attempts=max_attempts,
459
+ seed=t, label_task=False,
460
+ reset_workdir=False, status_cb=cb)
461
  res = ImproverResult(
462
+ session_id=sid, ok=pres.ok, improved=pres.improved,
463
+ reply=pres.route, errors=pres.errors,
464
+ attempts=pres.attempts, duration_s=pres.duration_s,
 
465
  )
466
+ if res.ok and res.improved is not None:
 
467
  out = os.path.join(cfg.improved_dir, f"{sid}.json")
468
  res.improved.save(out)
469
+ note = (" · no change (the raw trace already reads as ideal)"
470
+ if pres.no_change else "")
471
+ if pres.route == "regate":
472
+ prog.write(f"IMPROVED {sid} (regate, "
473
+ f"{res.duration_s:.0f}s){note}")
474
+ else:
475
+ prog.write(
476
+ f"IMPROVED {sid} ({res.attempts} attempts, "
477
+ f"{res.duration_s:.0f}s, {res.reply}){note}"
478
+ )
479
  else:
480
  prog.write(f"FAILED {sid}: {res.errors}")
481
+ row = res.to_dict()
482
+ row["route"] = pres.route
483
+ row["no_change"] = pres.no_change
484
+ row["time_sensitivity"] = pres.time_sensitivity
485
+ _write_manifest_row(cfg, row)
486
  if liv is not None:
487
  liv.registry.end(sid)
488
  return sid, res
489
+ # Legacy agent engine (`splice: false`). It is not the pipeline the
490
+ # Space runs, so it keeps its own behaviour; the batch only reaches
491
+ # it when the config explicitly turns the splice engine off.
492
  res = improve_trace(
493
  cfg, t, max_attempts=max_attempts, status_cb=cb
494
  )
traceforge/defects.py CHANGED
@@ -116,7 +116,8 @@ _RULES: list[tuple[str, re.Pattern[str]]] = [
116
  ("D-TASK-05", re.compile(r"harness dispatch brief|dispatch brief", re.I)),
117
  ("D-TASK-04", re.compile(r"no user task message|no user task", re.I)),
118
  ("D-TASK-02", re.compile(r"reframe", re.I)),
119
- ("D-VER-02", re.compile(r"fabricated tool result|shares no content", re.I)),
 
120
  ("D-FMT-01", re.compile(r"multi-turn:.*dropped|pairing|orphan", re.I)),
121
  ("D-FMT-03", re.compile(r"final_blob|final answer is a (?:file )?dump", re.I)),
122
  ("D-FMT-04", re.compile(r"self_talk|self-talk", re.I)),
 
116
  ("D-TASK-05", re.compile(r"harness dispatch brief|dispatch brief", re.I)),
117
  ("D-TASK-04", re.compile(r"no user task message|no user task", re.I)),
118
  ("D-TASK-02", re.compile(r"reframe", re.I)),
119
+ ("D-VER-02", re.compile(r"fabricated tool result|shares no content"
120
+ r"|self-grounding", re.I)),
121
  ("D-FMT-01", re.compile(r"multi-turn:.*dropped|pairing|orphan", re.I)),
122
  ("D-FMT-03", re.compile(r"final_blob|final answer is a (?:file )?dump", re.I)),
123
  ("D-FMT-04", re.compile(r"self_talk|self-talk", re.I)),
traceforge/space/__init__.py CHANGED
@@ -9,13 +9,15 @@ from traceforge.space.hashing import (canonical_messages, fingerprint,
9
  output_hash, seed_hash, short_hash,
10
  tool_names)
11
  from traceforge.space.ingest import parse_records, read_upload, trace_from_record
12
- from traceforge.space.pipeline import (SpacePipeline, pipeline_ready,
 
13
  space_config)
14
  from traceforge.space.store import (DONE, FAILED, QUEUED, REJECTED, RUNNING,
15
  TraceStore)
16
 
17
  __all__ = [
18
  "TraceStore", "SpacePipeline", "space_config", "pipeline_ready",
 
19
  "seed_hash", "output_hash", "short_hash", "fingerprint",
20
  "canonical_messages", "tool_names", "trace_from_record",
21
  "parse_records", "read_upload",
 
9
  output_hash, seed_hash, short_hash,
10
  tool_names)
11
  from traceforge.space.ingest import parse_records, read_upload, trace_from_record
12
+ from traceforge.space.pipeline import (ProcessResult, SpacePipeline,
13
+ pipeline_ready, process_trace,
14
  space_config)
15
  from traceforge.space.store import (DONE, FAILED, QUEUED, REJECTED, RUNNING,
16
  TraceStore)
17
 
18
  __all__ = [
19
  "TraceStore", "SpacePipeline", "space_config", "pipeline_ready",
20
+ "process_trace", "ProcessResult",
21
  "seed_hash", "output_hash", "short_hash", "fingerprint",
22
  "canonical_messages", "tool_names", "trace_from_record",
23
  "parse_records", "read_upload",
traceforge/space/pipeline.py CHANGED
@@ -27,6 +27,7 @@ import json
27
  import os
28
  import shutil
29
  import time
 
30
  from typing import Any, Optional
31
 
32
  from traceforge.config import Config, repo_root_from_pkg
@@ -126,16 +127,139 @@ def pipeline_ready(cfg: Config) -> tuple[bool, str]:
126
  return True, f"{model} @ {base}"
127
 
128
 
129
- def _same_conversation(record: dict, improved: Trace) -> bool:
130
- """True when the rewrite is the seed, message for message."""
 
 
 
 
131
  try:
132
  from traceforge.space.hashing import canonical_messages
133
- return (canonical_messages(record) ==
 
134
  canonical_messages(improved.to_dict()))
135
  except Exception:
136
  return False
137
 
138
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
139
  # ----------------------------------------------------------------- pipeline
140
 
141
  class SpacePipeline:
@@ -203,62 +327,18 @@ class SpacePipeline:
203
 
204
  # A fresh attempt must not regate a stale artifact from a previous
205
  # run (the splice engine's stage-0 fast path would promote it as-is).
206
- workdir = os.path.join(self.cfg.improver_workdir, sid)
207
- shutil.rmtree(workdir, ignore_errors=True)
 
 
 
208
 
209
- try:
210
- from traceforge.splice import improve_trace_splice
211
- res = improve_trace_splice(
212
- self.cfg, trace, max_attempts=self.max_attempts,
213
- )
214
- except Exception as e:
215
- return self._fail(h, [f"pipeline crashed: {e}"], started)
216
-
217
- if not res.get("ok") or res.get("improved") is None:
218
- return self._fail(h, list(res.get("errors") or ["no output"]),
219
- started, route=res.get("route"),
220
- attempts=res.get("attempts"))
221
-
222
- improved: Trace = res["improved"]
223
  improved.session_id = sid
224
  improved.parent_session_id = sid
225
  improved.improved = True
226
  improved.source = "traceforge-space"
227
 
228
- # The splice engine gates before returning, but a structural check
229
- # here is cheap and makes the Space's own contract explicit.
230
- try:
231
- from traceforge.validate import validate_structural
232
- structural = validate_structural(improved)
233
- except Exception as e:
234
- structural = [f"validator crashed: {e}"]
235
- if not structural:
236
- # The ideal must be self-contained: a rewrite that drops the
237
- # evidence but keeps the claim passes the source-grounding
238
- # gate, yet teaches the model to assert facts it never saw.
239
- try:
240
- from traceforge.grounding import self_grounding_errors
241
- structural = self_grounding_errors(improved)
242
- except Exception as e:
243
- structural = [f"self-grounding check crashed: {e}"]
244
- if structural:
245
- return self._fail(h, structural, started, route=res.get("route"),
246
- attempts=res.get("attempts"))
247
-
248
- # Task bucket: heuristic only — the Space must not spend an LLM call
249
- # on a label the dashboard can derive for free.
250
- try:
251
- from traceforge.classify import classify_heuristic
252
- improved.task = classify_heuristic(improved).get("task")
253
- except Exception:
254
- pass
255
-
256
- # Time sensitivity: measured on the RAW record (it may carry the
257
- # `tools` array, which is what decides anchored vs sensitive) and on
258
- # the ideal rewrite (did it actually call the retrieval tool?).
259
- ts_raw = self._time_sensitivity(record)
260
- ts_out = self._time_sensitivity(improved)
261
-
262
  # A no-op is not an ideal rewrite. The splice engine can return the
263
  # seed unchanged when the writer decides nothing needs changing, and
264
  # the judge accepts it because it violates nothing — but the Space
@@ -269,12 +349,12 @@ class SpacePipeline:
269
  # It is not a failure either: a seed that already reads as ideal
270
  # legitimately needs no edit. Record it as `no_change` so the UI can
271
  # say so instead of counting it against the visitor.
272
- if _same_conversation(record, improved):
273
  return self._reject(
274
  h, "no change needed: the rewrite is identical to the seed, "
275
  "so the seed already reads as an ideal trace", started,
276
- route=res.get("route"), attempts=res.get("attempts"),
277
- time_sensitivity=ts_raw, no_change=True,
278
  )
279
 
280
  self.store.put_output(h, improved)
@@ -282,12 +362,12 @@ class SpacePipeline:
282
  row = self.store.update(
283
  h, status=DONE, finished_at=time.time(), duration_s=round(duration, 1),
284
  output_hash=output_hash(improved),
285
- route=res.get("route") or "",
286
- attempts=res.get("attempts") or 0,
287
  n_messages_out=len(improved.messages),
288
- task=improved.task or rec.get("task") or "",
289
- time_sensitivity=ts_raw,
290
- time_sensitivity_improved=ts_out,
291
  error="",
292
  )
293
  return row
@@ -311,17 +391,6 @@ class SpacePipeline:
311
  route=route or "", attempts=attempts or 0, error=reason, **extra,
312
  )
313
 
314
- @staticmethod
315
- def _time_sensitivity(obj: Any) -> dict:
316
- try:
317
- from traceforge.timesensitivity import detect
318
- d = detect(obj)
319
- return {k: d[k] for k in ("class", "label", "temporal_facts",
320
- "retrieval_available", "retrieval_called",
321
- "needs_rewrite", "score")}
322
- except Exception as e:
323
- return {"class": "", "label": f"unavailable: {e}"}
324
-
325
  # -------------------------------------------------------------- report
326
 
327
  def result(self, h: str) -> dict:
 
27
  import os
28
  import shutil
29
  import time
30
+ from dataclasses import dataclass, field
31
  from typing import Any, Optional
32
 
33
  from traceforge.config import Config, repo_root_from_pkg
 
127
  return True, f"{model} @ {base}"
128
 
129
 
130
+ def _same_conversation(record: Any, improved: Trace) -> bool:
131
+ """True when the rewrite is the seed, message for message.
132
+
133
+ ``record`` may be the submitted dict or the raw ``Trace`` — the batch
134
+ passes the trace it loaded, the Space passes the stored seed.
135
+ """
136
  try:
137
  from traceforge.space.hashing import canonical_messages
138
+ seed = record.to_dict() if isinstance(record, Trace) else record
139
+ return (canonical_messages(seed) ==
140
  canonical_messages(improved.to_dict()))
141
  except Exception:
142
  return False
143
 
144
 
145
+ # ------------------------------------------------------- the one pipeline
146
+
147
+ @dataclass
148
+ class ProcessResult:
149
+ """What one trace produced. The only shape a caller has to understand."""
150
+
151
+ ok: bool
152
+ improved: Optional[Trace] = None
153
+ route: str = ""
154
+ attempts: int = 0
155
+ duration_s: float = 0.0
156
+ errors: list[str] = field(default_factory=list)
157
+ no_change: bool = False
158
+ task: str = ""
159
+ time_sensitivity: dict = field(default_factory=dict)
160
+ time_sensitivity_improved: dict = field(default_factory=dict)
161
+
162
+
163
+ def _time_sensitivity(obj: Any) -> dict:
164
+ try:
165
+ from traceforge.timesensitivity import detect
166
+ d = detect(obj)
167
+ return {k: d[k] for k in ("class", "label", "temporal_facts",
168
+ "retrieval_available", "retrieval_called",
169
+ "needs_rewrite", "score")}
170
+ except Exception as e:
171
+ return {"class": "", "label": f"unavailable: {e}"}
172
+
173
+
174
+ def process_trace(cfg: Config, trace: Trace, *, max_attempts: int = 2,
175
+ seed: Any = None, reset_workdir: bool = False,
176
+ label_task: bool = True,
177
+ status_cb: Any = None) -> ProcessResult:
178
+ """THE pipeline: one trace in, one ideal trace out (or a reason).
179
+
180
+ Both the Gradio app (``SpacePipeline.process``) and the local batch
181
+ (``cli._improve_one``) call this, so the rewrite, the gates, the no-op
182
+ check and the time-sensitivity measurement exist once.
183
+
184
+ ``seed`` is the submitted record when the caller has one (the Space
185
+ does, and it may carry a ``tools`` array the Trace lost); the raw
186
+ ``trace`` is used otherwise. ``reset_workdir`` is True for the Space,
187
+ where a re-queued seed must not regate a stale artifact, and False for
188
+ the batch, where the workdir cache is the resume mechanism.
189
+ """
190
+ started = time.time()
191
+ sid = trace.session_id
192
+ if reset_workdir:
193
+ shutil.rmtree(os.path.join(cfg.improver_workdir, sid),
194
+ ignore_errors=True)
195
+
196
+ try:
197
+ from traceforge.splice import improve_trace_splice
198
+ raw = improve_trace_splice(cfg, trace, max_attempts=max_attempts,
199
+ status_cb=status_cb)
200
+ res = {"ok": raw.get("ok"), "improved": raw.get("improved"),
201
+ "errors": list(raw.get("errors") or []),
202
+ "route": raw.get("route") or "",
203
+ "attempts": raw.get("attempts") or 0,
204
+ "duration_s": raw.get("duration_s") or 0.0}
205
+ except Exception as e:
206
+ return ProcessResult(ok=False, errors=[f"pipeline crashed: {e}"],
207
+ duration_s=time.time() - started)
208
+
209
+ if not res.get("ok") or res.get("improved") is None:
210
+ return ProcessResult(
211
+ ok=False, errors=list(res.get("errors") or ["no output"]),
212
+ route=res.get("route") or "", attempts=res.get("attempts") or 0,
213
+ duration_s=time.time() - started)
214
+
215
+ improved: Trace = res["improved"]
216
+
217
+ # The splice engine gates before returning; re-checking here makes the
218
+ # pipeline's own contract explicit and catches a regate path that
219
+ # bypassed the engine's gate chain.
220
+ try:
221
+ from traceforge.validate import validate_structural
222
+ errs = validate_structural(improved)
223
+ except Exception as e:
224
+ errs = [f"validator crashed: {e}"]
225
+ if not errs:
226
+ # The ideal must carry its own evidence: a rewrite that drops the
227
+ # tool result but keeps the claim passes source-grounding yet
228
+ # teaches the model to assert facts it never saw.
229
+ try:
230
+ from traceforge.grounding import self_grounding_errors
231
+ errs = self_grounding_errors(improved)
232
+ except Exception as e:
233
+ errs = [f"self-grounding check crashed: {e}"]
234
+ if errs:
235
+ return ProcessResult(
236
+ ok=False, errors=errs, route=res.get("route") or "",
237
+ attempts=res.get("attempts") or 0,
238
+ duration_s=time.time() - started)
239
+
240
+ # Task bucket: heuristic only — neither caller should spend an LLM call
241
+ # on a label the dashboard can derive for free. The batch turns this off
242
+ # because `traceforge classify` writes the authoritative LLM label and
243
+ # the benchmark prefers `trace.task` when it is set.
244
+ if label_task:
245
+ try:
246
+ from traceforge.classify import classify_heuristic
247
+ improved.task = classify_heuristic(improved).get("task") or ""
248
+ except Exception:
249
+ pass
250
+
251
+ seed_obj = seed if seed is not None else trace
252
+ return ProcessResult(
253
+ ok=True, improved=improved,
254
+ route=res.get("route") or "", attempts=res.get("attempts") or 0,
255
+ duration_s=time.time() - started,
256
+ no_change=_same_conversation(seed_obj, improved),
257
+ task=improved.task or "",
258
+ time_sensitivity=_time_sensitivity(seed_obj),
259
+ time_sensitivity_improved=_time_sensitivity(improved),
260
+ )
261
+
262
+
263
  # ----------------------------------------------------------------- pipeline
264
 
265
  class SpacePipeline:
 
327
 
328
  # A fresh attempt must not regate a stale artifact from a previous
329
  # run (the splice engine's stage-0 fast path would promote it as-is).
330
+ res = process_trace(self.cfg, trace, max_attempts=self.max_attempts,
331
+ seed=record, reset_workdir=True)
332
+ if not res.ok or res.improved is None:
333
+ return self._fail(h, res.errors, started, route=res.route,
334
+ attempts=res.attempts)
335
 
336
+ improved: Trace = res.improved
 
 
 
 
 
 
 
 
 
 
 
 
 
337
  improved.session_id = sid
338
  improved.parent_session_id = sid
339
  improved.improved = True
340
  improved.source = "traceforge-space"
341
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
342
  # A no-op is not an ideal rewrite. The splice engine can return the
343
  # seed unchanged when the writer decides nothing needs changing, and
344
  # the judge accepts it because it violates nothing — but the Space
 
349
  # It is not a failure either: a seed that already reads as ideal
350
  # legitimately needs no edit. Record it as `no_change` so the UI can
351
  # say so instead of counting it against the visitor.
352
+ if res.no_change:
353
  return self._reject(
354
  h, "no change needed: the rewrite is identical to the seed, "
355
  "so the seed already reads as an ideal trace", started,
356
+ route=res.route, attempts=res.attempts,
357
+ time_sensitivity=res.time_sensitivity, no_change=True,
358
  )
359
 
360
  self.store.put_output(h, improved)
 
362
  row = self.store.update(
363
  h, status=DONE, finished_at=time.time(), duration_s=round(duration, 1),
364
  output_hash=output_hash(improved),
365
+ route=res.route,
366
+ attempts=res.attempts,
367
  n_messages_out=len(improved.messages),
368
+ task=res.task or rec.get("task") or "",
369
+ time_sensitivity=res.time_sensitivity,
370
+ time_sensitivity_improved=res.time_sensitivity_improved,
371
  error="",
372
  )
373
  return row
 
391
  route=route or "", attempts=attempts or 0, error=reason, **extra,
392
  )
393
 
 
 
 
 
 
 
 
 
 
 
 
394
  # -------------------------------------------------------------- report
395
 
396
  def result(self, h: str) -> dict:
traceforge/splice.py CHANGED
@@ -2940,6 +2940,14 @@ def improve_trace_splice(
2940
  # vendor identity nor refusal phrasing — repair or drop the row
2941
  ierrs = validate_identity(candidate.messages) if not gerrs else []
2942
  gerrs = gerrs or ierrs
 
 
 
 
 
 
 
 
2943
  if gerrs:
2944
  _status(f"gates fail ({attempts})")
2945
  # Record WHY the gate failed even when a repair is attempted —
@@ -3954,6 +3962,9 @@ def judge_artifact(
3954
  errs = validate_structural(candidate) \
3955
  or validate_grounding(candidate, trace) \
3956
  or validate_identity(candidate.messages)
 
 
 
3957
  if errs and all("last message role is 'tool'" in e for e in errs):
3958
  # stale artifact from an earlier run that stopped mid-call: close
3959
  # it here instead of failing a trace that is one row short
@@ -3965,6 +3976,9 @@ def judge_artifact(
3965
  errs = validate_structural(candidate) \
3966
  or validate_grounding(candidate, trace) \
3967
  or validate_identity(candidate.messages)
 
 
 
3968
  if errs:
3969
  return {"ok": False,
3970
  "errors": [f"gate: {e}" for e in errs[:4]],
 
2940
  # vendor identity nor refusal phrasing — repair or drop the row
2941
  ierrs = validate_identity(candidate.messages) if not gerrs else []
2942
  gerrs = gerrs or ierrs
2943
+ # Self-grounding: the ideal must carry its own evidence. A rewrite
2944
+ # that drops the tool result but keeps the claim passes
2945
+ # validate_grounding (the fact IS in the source trace) yet teaches
2946
+ # the model to assert facts it never saw. Measured on 150 promoted
2947
+ # traces before this gate existed: 6% would have failed it.
2948
+ if not gerrs:
2949
+ from traceforge.grounding import self_grounding_errors
2950
+ gerrs = self_grounding_errors(candidate)
2951
  if gerrs:
2952
  _status(f"gates fail ({attempts})")
2953
  # Record WHY the gate failed even when a repair is attempted —
 
3962
  errs = validate_structural(candidate) \
3963
  or validate_grounding(candidate, trace) \
3964
  or validate_identity(candidate.messages)
3965
+ if not errs:
3966
+ from traceforge.grounding import self_grounding_errors
3967
+ errs = self_grounding_errors(candidate)
3968
  if errs and all("last message role is 'tool'" in e for e in errs):
3969
  # stale artifact from an earlier run that stopped mid-call: close
3970
  # it here instead of failing a trace that is one row short
 
3976
  errs = validate_structural(candidate) \
3977
  or validate_grounding(candidate, trace) \
3978
  or validate_identity(candidate.messages)
3979
+ if not errs:
3980
+ from traceforge.grounding import self_grounding_errors
3981
+ errs = self_grounding_errors(candidate)
3982
  if errs:
3983
  return {"ok": False,
3984
  "errors": [f"gate: {e}" for e in errs[:4]],