| """Deterministically select fresh, test-backed tasks for Study 4. |
| |
| This outcome-blind selector reads repository history and test outcomes only. It |
| excludes every commit already present in the task catalog, validates that public |
| tests pass at the parent, hidden tests expose the bug, and the gold production |
| patch repairs it, then retains every candidate decision. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| from dataclasses import asdict |
| import json |
| from pathlib import Path |
|
|
| from build_study2_tasks import ( |
| candidate_commits, |
| changed_symbols, |
| manifest_text, |
| precheck, |
| run_git, |
| validate_candidate, |
| load_repository, |
| ) |
| from agent_harness.specs import load_tasks |
|
|
|
|
| REPOSITORIES = ("R001", "R002", "R003") |
|
|
|
|
| def select_fresh(root: Path, repository_id: str, count: int, write: bool) -> dict: |
| rule = load_repository(root, repository_id) |
| catalog = load_tasks(root) |
| excluded_commits = {task.gold_commit for task in catalog.values()} |
| selected: list[str] = [] |
| audits = [] |
| manifest_dir = root / "tasks" / "manifests" |
| patch_dir = root / "tasks" / "patches" |
| validation_dir = root / "tasks" / "validation" / "study4" |
| selection_dir = root / "tasks" / "selection" / "study4" |
| if write: |
| validation_dir.mkdir(parents=True, exist_ok=True) |
| selection_dir.mkdir(parents=True, exist_ok=True) |
|
|
| for commit in candidate_commits(rule): |
| if len(selected) >= count: |
| break |
| audit = precheck(rule, commit, excluded_commits) |
| audits.append(audit) |
| if audit.status != "eligible": |
| continue |
| assert audit.parent is not None |
| source_patch = str( |
| run_git(rule.path, "diff", "--binary", audit.parent, commit, "--", *audit.source_paths) |
| ) |
| test_patch = str( |
| run_git(rule.path, "diff", "--binary", audit.parent, commit, "--", *audit.test_paths) |
| ) |
| validation = validate_candidate(rule, audit, source_patch, test_patch) |
| audit.validation = validation |
| audit.elapsed_seconds += float(validation["elapsed_seconds"]) |
| if not validation["valid"]: |
| audit.status = "rejected" |
| audit.reason = str(validation["reason"]) |
| print(f"REJECT {repository_id} {commit[:12]} {audit.reason}", flush=True) |
| continue |
|
|
| task_id = f"TASK_S4_{repository_id}_{len(selected) + 1:03d}" |
| source_name = f"{task_id}_source.patch" |
| test_name = f"{task_id}_tests.patch" |
| symbols = changed_symbols(rule, audit) |
| audit.task_id = task_id |
| audit.status = "selected" |
| audit.reason = None |
| selected.append(task_id) |
| excluded_commits.add(commit) |
| if write: |
| text = manifest_text( |
| rule, audit, task_id, source_name, test_name, symbols |
| ).replace( |
| "deterministic Study 2 held-out validation", |
| "deterministic Study 4 fresh-task validation", |
| ) |
| (manifest_dir / f"{task_id}.toml").write_text(text, encoding="utf-8") |
| (patch_dir / source_name).write_text(source_patch, encoding="utf-8") |
| (patch_dir / test_name).write_text(test_patch, encoding="utf-8") |
| (validation_dir / f"{task_id}.json").write_text( |
| json.dumps( |
| { |
| "schema_version": 1, |
| "study": "Study 4 fresh-task retrieval replication", |
| "task_id": task_id, |
| "repository_id": repository_id, |
| "commit": commit, |
| "parent": audit.parent, |
| "valid_end_to_end": True, |
| "validation": validation, |
| }, |
| indent=2, |
| sort_keys=True, |
| ) |
| + "\n", |
| encoding="utf-8", |
| ) |
| print( |
| f"SELECT {task_id} {commit[:12]} source={len(audit.source_paths)} " |
| f"tests={len(audit.test_paths)} lines={audit.source_changed_lines}", |
| flush=True, |
| ) |
|
|
| report = { |
| "schema_version": 1, |
| "study": "Study 4 fresh-task retrieval replication", |
| "repository": asdict(rule) | {"path": str(rule.path)}, |
| "selection_policy": { |
| "fresh_count": count, |
| "excluded_prior_task_commits": len({task.gold_commit for task in catalog.values()}), |
| "order": "reverse chronological first-parent history since 2023-01-01", |
| "outcome_blind": True, |
| }, |
| "selected_task_ids": selected, |
| "selected_count": len(selected), |
| "complete": len(selected) == count, |
| "audits": [asdict(item) for item in audits], |
| } |
| if write: |
| (selection_dir / f"{repository_id}_selection.json").write_text( |
| json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" |
| ) |
| if len(selected) != count: |
| raise RuntimeError(f"{repository_id} yielded {len(selected)}/{count} fresh valid tasks") |
| return report |
|
|
|
|
| def freeze_split(root: Path, count: int) -> tuple[str, ...]: |
| catalog = load_tasks(root) |
| expected = [ |
| f"TASK_S4_{repository_id}_{number:03d}" |
| for repository_id in REPOSITORIES |
| for number in range(1, count + 1) |
| ] |
| missing = [task_id for task_id in expected if task_id not in catalog] |
| if missing: |
| raise RuntimeError(f"cannot freeze Study 4 split; missing {missing}") |
| path = root / "tasks" / "splits" / "study4_fresh.txt" |
| path.write_text( |
| "# Fresh Study 4 split; frozen before any E10 LLM inference.\n" |
| + "\n".join(expected) |
| + "\n", |
| encoding="utf-8", |
| ) |
| return tuple(expected) |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) |
| parser.add_argument("--repository-id", choices=REPOSITORIES) |
| parser.add_argument("--count", type=int, default=10) |
| parser.add_argument("--write", action="store_true") |
| parser.add_argument("--freeze-split", action="store_true") |
| args = parser.parse_args() |
| root = args.root.resolve() |
| if args.freeze_split: |
| split = freeze_split(root, args.count) |
| print(json.dumps({"split_count": len(split), "task_ids": split}, indent=2)) |
| return |
| if not args.repository_id: |
| parser.error("--repository-id is required unless --freeze-split is used") |
| report = select_fresh(root, args.repository_id, args.count, args.write) |
| print(json.dumps({"repository_id": args.repository_id, "selected": report["selected_count"]})) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
