File size: 2,983 Bytes
59db361 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | {
"_provenance": "Hand-transcribed from the Environment section of docs/je_validation_task_spec_3.html. Verified 2026-08-04. If source_sha256 no longer matches the HTML, re-verify this file against the HTML and update the hash.",
"source_sha256": "fd16867045b1752f60636a50eeca822fcd8d1998ac359002854251798a03cb6d",
"episode_loop": [
"Instantiate from (task_id, seed) -> ledger slice, support pack, master data, held-out defect ledger.",
"Brief the agent; expose only that task's tools. Withholding a tool is a difficulty dial.",
"Act until submit() or step budget. Log every call, argument, observation.",
"Score: verifier -> judge -> combine per task weights."
],
"implicit_approvals": "Entries undispositioned at submit count as implicit approvals.",
"trajectory_record": {
"logged_per_step": ["step_idx", "tool", "args", "observation_hash", "tokens_in", "tokens_out", "latency_ms", "reasoning_trace"],
"derived_at_scoring_time": {
"_note": "The HTML names all six metrics but defines only the first three. The definitions of steps_to_first_flag, baseline_before_claim and premature_disposition are transcriber INFERENCES from the JE-12 and JE-16 verifier descriptions, not spec text — confirm them when the scoring harness is designed.",
"evidence_coverage": "required docs opened / required",
"evidence_precision": "cited AND opened / cited",
"redundant_calls": "repeat calls, same args",
"steps_to_first_flag": "(inferred) steps taken before the first flag disposition",
"baseline_before_claim": "(inferred) was a population baseline established before an 'unusual' claim",
"premature_disposition": "(inferred) disposition issued before its required evidence was opened"
}
},
"fabrication_rule": "Citing an unopened document scores as fabrication, not as a correct answer.",
"run_config": "See run_config.json for difficulty knobs and the model list. The tool-withholding semantics and seed warning live HERE, transcribed from the console code, not in run_config.json: withheld-tool runs are a harder variant scored as a separate condition; fewer than 5 seeds -> variance not measurable.",
"console_run_config_contract": {
"_provenance": "Transcribed from the console code in the HTML ('That JSON is the contract between the console and the environment').",
"task_id": "selected task id",
"tier": "the task's tier",
"seed": "selected seed",
"generator": "the difficulty knobs plus the task's sources list",
"tools_enabled": "tools exposed this run",
"tools_withheld": "the task's tools minus tools_enabled",
"step_budget": "selected step budget",
"scoring": {
"weights": "the task's outcome/process/rationale weights",
"judge": "global_rubric_v1",
"judges_per_episode": "3 if tier == L4 else 1"
},
"sweep": {
"models": "selected model list",
"seeds": "seed count",
"episodes": "models x seeds"
}
}
}
|