ParthKulshreshtha's picture
Upload folder using huggingface_hub
324b1af verified
Raw
History Blame Contribute Delete
3.68 kB
"""What the console can run: built tasks (JE-01..10), their spec JSON, and the
generator dispatch. New tasks appear here as later phases wire them."""
from __future__ import annotations
import json
from functools import cache
from pathlib import Path
from je_validation.envir.briefs import brief_for # noqa: F401 (re-export)
from je_validation.envir.run_config import DEFAULT_KNOBS
from je_validation.generator import (
je01,
je02,
je03,
je04,
je05,
je06,
je07,
je08,
je09,
je10,
je11,
je12,
je13,
je14,
je15,
)
from je_validation.generator.persist import spec_task_path
GENERATORS = {
"JE-01": je01.generate, "JE-02": je02.generate, "JE-03": je03.generate,
"JE-04": je04.generate, "JE-05": je05.generate, "JE-06": je06.generate,
"JE-07": je07.generate, "JE-08": je08.generate, "JE-09": je09.generate,
"JE-10": je10.generate, "JE-11": je11.generate, "JE-12": je12.generate,
"JE-13": je13.generate, "JE-14": je14.generate, "JE-15": je15.generate,
}
# The population knob is editable only for L1; L2 populations are spec-pinned
# in the generators (JE-06/07/10 accept n_entries but the pinned default is
# canonical; JE-08/09 take none) — dispatch never passes it for L2.
POPULATION_EDITABLE = frozenset(f"JE-{i:02d}" for i in range(1, 6))
# Canonical populations: every surface (task defaults, budget prefills, launch
# contract, generation, scoring config) reads THIS, so they cannot disagree.
# Measured directly by generating seeds 1-5 against the real DS-A snapshot:
# JE-06/07/08/09 produced a constant entry count across all five seeds; JE-10's
# count varied by seed (272, 281, 277, 288, 279), so its value here (280) is a
# NOMINAL prefill/display figure only, not a byte-exact guarantee — see
# POPULATION_SEED_VARIABLE below.
_L2_POPULATIONS: dict[str, int] = {
"JE-06": 60, "JE-07": 304, "JE-08": 221, "JE-09": 199, "JE-10": 280,
}
# Tasks whose actual generated entry count varies by seed (spec-pinned
# knobs/rng interact with seed, not just population). default_population is
# nominal for these; the episode path (Task 6) uses the real scoring-side
# count from the generated instance and skips the hard equality assert that
# would otherwise compare it against this nominal value.
POPULATION_SEED_VARIABLE: frozenset[str] = frozenset({"JE-10"})
# L3 populations are spec-pinned like L2; all five generators fill to an
# exact count, so none joins POPULATION_SEED_VARIABLE.
_L3_POPULATIONS: dict[str, int] = {
"JE-11": 260, "JE-12": 320, "JE-13": 600, "JE-14": 240, "JE-15": 380,
}
# JE-01's spec scenario is "a batch of 400 posted journal entries"; at the
# generic DEFAULT_KNOBS population (60) the generator can't find its required
# 15 clean multi-line distractor entries (seed 1 yields only 14), so
# generation fails. Other L1 tasks (JE-02..05) have no spec-stated batch size
# and keep the generic default.
_L1_POPULATIONS: dict[str, int] = {"JE-01": 400}
def default_population(task_id: str) -> int:
if task_id in POPULATION_EDITABLE:
return _L1_POPULATIONS.get(task_id, int(DEFAULT_KNOBS["population"]))
return {**_L2_POPULATIONS, **_L3_POPULATIONS}[task_id]
@cache
def built_tasks() -> dict[str, dict]:
return {tid: json.loads(spec_task_path(tid).read_text(encoding="utf-8"))
for tid in GENERATORS}
def generate_bundle(task_id: str, seed: int, snapshot: Path,
population: int, out_dir: Path) -> None:
if task_id in POPULATION_EDITABLE:
GENERATORS[task_id](seed, snapshot, n_entries=population, out_dir=out_dir)
else:
GENERATORS[task_id](seed, snapshot, out_dir=out_dir)