| { |
| "_provenance": "Hand-transcribed from the Environment section of docs/je_validation_task_spec_3.html. Verified 2026-08-04. If source_sha256 no longer matches the HTML, re-verify this file against the HTML and update the hash.", |
| "source_sha256": "fd16867045b1752f60636a50eeca822fcd8d1998ac359002854251798a03cb6d", |
| "episode_loop": [ |
| "Instantiate from (task_id, seed) -> ledger slice, support pack, master data, held-out defect ledger.", |
| "Brief the agent; expose only that task's tools. Withholding a tool is a difficulty dial.", |
| "Act until submit() or step budget. Log every call, argument, observation.", |
| "Score: verifier -> judge -> combine per task weights." |
| ], |
| "implicit_approvals": "Entries undispositioned at submit count as implicit approvals.", |
| "trajectory_record": { |
| "logged_per_step": ["step_idx", "tool", "args", "observation_hash", "tokens_in", "tokens_out", "latency_ms", "reasoning_trace"], |
| "derived_at_scoring_time": { |
| "_note": "The HTML names all six metrics but defines only the first three. The definitions of steps_to_first_flag, baseline_before_claim and premature_disposition are transcriber INFERENCES from the JE-12 and JE-16 verifier descriptions, not spec text — confirm them when the scoring harness is designed.", |
| "evidence_coverage": "required docs opened / required", |
| "evidence_precision": "cited AND opened / cited", |
| "redundant_calls": "repeat calls, same args", |
| "steps_to_first_flag": "(inferred) steps taken before the first flag disposition", |
| "baseline_before_claim": "(inferred) was a population baseline established before an 'unusual' claim", |
| "premature_disposition": "(inferred) disposition issued before its required evidence was opened" |
| } |
| }, |
| "fabrication_rule": "Citing an unopened document scores as fabrication, not as a correct answer.", |
| "run_config": "See run_config.json for difficulty knobs and the model list. The tool-withholding semantics and seed warning live HERE, transcribed from the console code, not in run_config.json: withheld-tool runs are a harder variant scored as a separate condition; fewer than 5 seeds -> variance not measurable.", |
| "console_run_config_contract": { |
| "_provenance": "Transcribed from the console code in the HTML ('That JSON is the contract between the console and the environment').", |
| "task_id": "selected task id", |
| "tier": "the task's tier", |
| "seed": "selected seed", |
| "generator": "the difficulty knobs plus the task's sources list", |
| "tools_enabled": "tools exposed this run", |
| "tools_withheld": "the task's tools minus tools_enabled", |
| "step_budget": "selected step budget", |
| "scoring": { |
| "weights": "the task's outcome/process/rationale weights", |
| "judge": "global_rubric_v1", |
| "judges_per_episode": "3 if tier == L4 else 1" |
| }, |
| "sweep": { |
| "models": "selected model list", |
| "seeds": "seed count", |
| "episodes": "models x seeds" |
| } |
| } |
| } |
|
|
