"""Evidence audit logic for PostTrainBench reproduction. Every function requires verified acquired data as input. No function produces authoritative evidence from constants alone. All inputs come from :func:`acquisition.acquire_all` results. """ from __future__ import annotations import hashlib import re from collections import defaultdict from typing import Any from posttrainbench_repro.constants import ( API_MISUSE_TASK_CLUSTER, ARXIV_ID, ATTEMPT_ID, CANONICAL_ALL_ENTRIES_SHA256, CANONICAL_DIRS_SHA256, CANONICAL_FILES_SHA256, CHALLENGE_ASSESSMENT_DIGEST, CHALLENGE_JSON_SHA256, CHALLENGE_REVISION, CLAIM_1_SHA256, CLAIM_1_TEXT, CLAIM_2_SHA256, CLAIM_2_TEXT, CONTAMINATION_WITNESS_BYTES, CONTAMINATION_WITNESS_PATH, CONTAMINATION_WITNESS_SHA256, EXCLUDED_TOP_LEVEL, EXPECTED_BENCHMARKS, EXPECTED_CELL_COUNTS, EXPECTED_DUPLICATE_PAIRS, EXPECTED_EVAL_DIRS, EXPECTED_MISSING_PAIRS, EXPECTED_MODEL_FRAGMENTS, EXPECTED_ROOT_CELL_PAIRS, EXPECTED_ROOT_COUNT, EXPECTED_TASK_COUNT, GIT_TREE_DIGEST, GIT_TREE_ENTRY_COUNT, GIT_TREE_ID, GITHUB_PINNED_COMMIT, GITHUB_REPO_URL, HF_DATASET_LICENSE, HF_DATASET_URL, HF_PINNED_REVISION, HF_TREE_DIR_COUNT, HF_TREE_FILE_COUNT, HF_TREE_PAGE_SIZE, HF_TREE_TOTAL_ENTRIES, HF_TREE_TOTAL_PAGES, INDEX_JSON_SHA256, INSTRUCTION_MODEL_JUDGMENT_BYTES, INSTRUCTION_MODEL_JUDGMENT_GIT_OBJECT, INSTRUCTION_MODEL_JUDGMENT_PATH, INSTRUCTION_MODEL_JUDGMENT_SHA256, INSTRUCTION_MODEL_JUDGMENT_SIZE, INSTRUCTION_MODEL_TRACE_GIT_OBJECT, INSTRUCTION_MODEL_TRACE_PATH, INSTRUCTION_MODEL_TRACE_SHA256, INSTRUCTION_MODEL_TRACE_SIZE, MODEL_ORDER, PAID_API_COST_USD, PAPER_ID, PAPER_LICENSE, PINNED_BLOBS, RUN_ROOT_10H_RE, SNAPSHOT_ID, SOURCE_LICENSE, TASK_BASENAME_RE, TIME_TAKEN_WITNESS_BYTES, TIME_TAKEN_WITNESS_PATH, TIME_TAKEN_WITNESS_SHA256, TRACE_EXCERPTS, TRUNCATED_SIBLINGS_COUNT, TRUNCATED_SIBLINGS_SHA256, UPSTREAM_TOKEN, VIEWER_DATA_FILE_COUNT, ) # --------------------------------------------------------------------------- # Provenance # --------------------------------------------------------------------------- def get_provenance(acquired: dict[str, Any]) -> dict[str, Any]: """Return the provenance record derived from acquired data. Requires a verified ``acquired`` dict from :func:`acquisition.acquire_all`. """ github = acquired["github"] hf_inv = acquired["hf_inventory"] return { "paper_id": PAPER_ID, "attempt_id": ATTEMPT_ID, "assessed_snapshot": SNAPSHOT_ID, "challenge_revision": CHALLENGE_REVISION, "challenge_assessment_digest": CHALLENGE_ASSESSMENT_DIGEST, "challenge_json_sha256": CHALLENGE_JSON_SHA256, "index_json_sha256": INDEX_JSON_SHA256, "upstream_token": UPSTREAM_TOKEN, "arxiv_id": ARXIV_ID, "paper_license": PAPER_LICENSE, "source": { "repository": GITHUB_REPO_URL, "pinned_commit": github["commit"], "tree_id": github["tree_id"], "entry_count": github["entry_count"], "canonical_tree_digest": github["canonical_tree_digest"], "license": SOURCE_LICENSE, "tree_acquisition": github["tree_acquisition"], "consumed_blobs": { path: meta for path, meta in sorted(github["blobs"].items()) }, }, "dataset": { "repository": HF_DATASET_URL, "pinned_revision": hf_inv["revision"], "license": HF_DATASET_LICENSE, "pagination": { "endpoint": "tree", "params": "recursive=true&expand=false&limit=1000", "mechanism": "Link header cursor, rel=\"next\"", "page_size": HF_TREE_PAGE_SIZE, "total_pages": hf_inv["page_count"], "total_entries": hf_inv["total_entries"], "file_count": hf_inv["file_count"], "directory_count": hf_inv["dir_count"], }, "canonical_digests": { "all_entries": hf_inv["canonical_all_digest"], "files": hf_inv["canonical_file_digest"], "directories": hf_inv["canonical_dir_digest"], }, "tree_acquisition": hf_inv["tree_acquisition"], "consumed_files": acquired["hf_consumed_files"], "truncated_siblings": { "count": TRUNCATED_SIBLINGS_COUNT, "digest": TRUNCATED_SIBLINGS_SHA256, "note": "Truncated lexical prefix from Hub revision-metadata; " "rejected as coverage input.", }, }, "paid_api_cost_usd": PAID_API_COST_USD, } # --------------------------------------------------------------------------- # Coverage census # --------------------------------------------------------------------------- def compute_coverage( hf_inventory: dict[str, Any], ) -> dict[str, Any]: """Compute the 4-by-7 coverage matrix from the complete HF inventory. Requires the verified ``hf_inventory`` dict from acquisition. Duplicate-job counting is per (root, benchmark, model) pair. """ dir_paths = hf_inventory["dir_paths"] coverage = _compute_coverage_from_dirs(dir_paths) coverage["accepted_benchmark_count"] = len(EXPECTED_BENCHMARKS) coverage["accepted_model_count"] = len(EXPECTED_MODEL_FRAGMENTS) if "file_paths" in hf_inventory: viewer_dirs = sorted( path for path in dir_paths if path == "viewer_data" or path.startswith("viewer_data/") ) viewer_files = sorted( path for path in hf_inventory["file_paths"] if path.startswith("viewer_data/") ) if "viewer_data" not in viewer_dirs: raise ValueError( "Verified inventory is missing excluded top-level viewer_data" ) if len(viewer_files) != VIEWER_DATA_FILE_COUNT: raise ValueError( "Verified viewer_data auxiliary file count mismatch: " f"{len(viewer_files)} != 2,397" ) coverage["excluded_auxiliary_data"] = { "top_level_path": "viewer_data", "present": True, "file_count": len(viewer_files), "directory_count": len(viewer_dirs), "counted_as_task_root": False, } inventory_fields = { "page_count", "total_entries", "file_count", "dir_count", "canonical_all_digest", "canonical_file_digest", "canonical_dir_digest", } if inventory_fields.issubset(hf_inventory): coverage["inventory"] = { "page_count": hf_inventory["page_count"], "total_entries": hf_inventory["total_entries"], "file_count": hf_inventory["file_count"], "dir_count": hf_inventory["dir_count"], "all_entries_digest": hf_inventory["canonical_all_digest"], "file_entries_digest": hf_inventory["canonical_file_digest"], "dir_entries_digest": hf_inventory["canonical_dir_digest"], "rejected_siblings_count": TRUNCATED_SIBLINGS_COUNT, "rejected_siblings_digest": TRUNCATED_SIBLINGS_SHA256, } return coverage def _compute_coverage_from_dirs( dir_paths: list[str], ) -> dict[str, Any]: """Core coverage computation from directory paths. Duplicate counting: a duplicate is an extra task for the same (opaque run root, benchmark, model) triple. """ model_order = MODEL_ORDER benchmark_list = sorted(EXPECTED_BENCHMARKS) task_dirs: list[str] = [] run_roots: set[str] = set() excluded_dirs: list[str] = [] unrecognized_dirs: list[str] = [] # Task directories: exactly two path components (depth-2) for p in dir_paths: parts = p.split("/") if len(parts) != 2: continue root = parts[0] basename = parts[1] if root in EXCLUDED_TOP_LEVEL: excluded_dirs.append(p) continue # Check if root matches 10h pattern if not RUN_ROOT_10H_RE.search(root): unrecognized_dirs.append(p) continue m = TASK_BASENAME_RE.match(basename) if not m: unrecognized_dirs.append(p) continue task_dirs.append(p) run_roots.add(root) # Build the coverage matrix # Per-cell task counting (global cell = bench × model) cell_tasks: dict[tuple[str, str], list[str]] = defaultdict(list) # Per-root/cell tracking for duplicates and missing pairs root_cell_set: set[tuple[str, str, str]] = set() # (root, bench, model) root_cell_tasks: dict[tuple[str, str, str], list[str]] = defaultdict(list) for p in task_dirs: root, basename = p.split("/") m = TASK_BASENAME_RE.match(basename) if not m: continue bench = m.group(1) model_fragment = m.group(2) model_normalized = EXPECTED_MODEL_FRAGMENTS[model_fragment] cell_tasks[(bench, model_normalized)].append(p) root_cell_set.add((root, bench, model_normalized)) root_cell_tasks[(root, bench, model_normalized)].append(p) # Compute cell counts cell_counts: dict[str, list[int]] = {} matrix_list: list[dict[str, Any]] = [] for bench in benchmark_list: counts = [] for model in model_order: count = len(cell_tasks.get((bench, model), [])) counts.append(count) matrix_list.append({ "benchmark": bench, "model": model, "count": count, }) cell_counts[bench] = counts # Duplicate-job pairs: per (root, bench, model), extra tasks beyond the first duplicate_pairs = sum( len(tasks) - 1 for tasks in root_cell_tasks.values() if len(tasks) > 1 ) # Count unique root/cell pairs root_cell_pair_count = len(root_cell_set) # Missing root/cell pairs: (roots × benchmarks × models) − actual total_possible = len(run_roots) * len(benchmark_list) * len(model_order) missing_pairs = total_possible - root_cell_pair_count return { "accepted_benchmarks": benchmark_list, "accepted_models": EXPECTED_MODEL_FRAGMENTS, "recognized_task_count": len(task_dirs), "recognized_root_count": len(run_roots), "recognized_root_cell_pairs": root_cell_pair_count, "duplicate_job_pairs": duplicate_pairs, "missing_root_cell_pairs": missing_pairs, "excluded_dirs_count": len(excluded_dirs), "unrecognized_dirs_count": len(unrecognized_dirs), "matrix": matrix_list, "cell_counts": cell_counts, } # --------------------------------------------------------------------------- # Protocol audit # --------------------------------------------------------------------------- def _source_lines( content: str, predicate: Any, ) -> list[int]: """Return one-based active source lines matching a predicate.""" return [ line_number for line_number, line in enumerate(content.splitlines(), 1) if line.strip() and not line.lstrip().startswith("#") and predicate(line) ] def _blob_reference( path: str, content: str, predicate: Any, label: str, ) -> dict[str, Any]: """Build a deterministic path/line reference into one verified blob.""" lines = _source_lines(content, predicate) if not lines: raise ValueError(f"Could not source {label} in {path}") git_object_sha1, raw_sha256 = PINNED_BLOBS[path] return { "commit": GITHUB_PINNED_COMMIT, "path": path, "lines": lines, "git_object_sha1": git_object_sha1, "raw_sha256": raw_sha256, } def audit_protocol( blob_contents: dict[str, bytes], git_entries: list[dict[str, Any]], ) -> dict[str, Any]: """Audit runner protocol controls from pinned source blobs. Requires verified blob contents and git tree entries. Derives all facts from actual content; fails on changed facts. """ single_task = blob_contents["src/commit_utils/single_task.sub"].decode("utf-8") run_task = blob_contents["src/run_task.sh"].decode("utf-8") commit_sh = blob_contents["src/commit_utils/commit.sh"].decode("utf-8") result: dict[str, Any] = {} # num_gpus default m = re.search(r"num_gpus\s*=\s*(\d+)", single_task) if not m: raise ValueError("Could not find num_gpus in single_task.sub") result["num_gpus_default"] = int(m.group(1)) if result["num_gpus_default"] != 1: raise ValueError( f"Expected single_task.sub num_gpus default 1, " f"got {result['num_gpus_default']}" ) # CUDA device requirement m = re.search( r'TARGET\.CUDADeviceName\s*==\s*"([^"]+)"', single_task, ) if not m: raise ValueError("Could not find CUDADeviceName in single_task.sub") result["cuda_device_requirement"] = ( f'TARGET.CUDADeviceName == "{m.group(1)}"' ) expected_cuda = 'TARGET.CUDADeviceName == "NVIDIA H100 80GB HBM3"' if result["cuda_device_requirement"] != expected_cuda: raise ValueError( "Unexpected CUDADeviceName requirement: " f"{result['cuda_device_requirement']}" ) # request_gpus binding m = re.search(r"request_gpus\s*=\s*\$\(num_gpus\)", single_task) if not m: raise ValueError("Could not find request_gpus binding") result["request_gpus_binding"] = "request_gpus = $(num_gpus)" # NUM_HOURS in run_task.sh result["receives_num_hours"] = "NUM_HOURS" in run_task if not result["receives_num_hours"]: raise ValueError("run_task.sh does not receive NUM_HOURS") # Solve timeout formula (minutes based on NUM_HOURS * 60 + 5) timeout_patterns = [ r"NUM_HOURS\s*\*\s*60\s*\+\s*5", r"\$\(\(\s*NUM_HOURS\s*\*\s*60\s*\+\s*5\s*\)\)", r"NUM_HOURS.*60.*\+.*5", ] if not any(re.search(p, run_task) for p in timeout_patterns): raise ValueError("Could not find timeout formula in run_task.sh") result["solve_timeout_formula"] = "NUM_HOURS * 60 + 5" result["timeout_grace_minutes"] = 5 result["timeout_formula_found"] = True # 10h suffix in task roots result["task_dir_10h_suffix"] = True # Eval directories present in git tree tree_paths = {e["path"] for e in git_entries} eval_dirs_found: list[str] = [] for d in EXPECTED_EVAL_DIRS: if d in tree_paths: eval_dirs_found.append(d) if len(eval_dirs_found) != 7: raise ValueError( f"Expected 7 eval directories, found {len(eval_dirs_found)}: " f"{eval_dirs_found}" ) result["evaluation_dirs_present"] = eval_dirs_found result["evaluation_dir_count"] = len(eval_dirs_found) # Commit.sh analysis for limitations result["commit_sh_analysis"] = _analyze_commit_sh(commit_sh) single_task_path = "src/commit_utils/single_task.sub" run_task_path = "src/run_task.sh" commit_path = "src/commit_utils/commit.sh" mpi_job = lambda line: ( line.lstrip().startswith("condor_submit_bid") and '"num_hours=100"' in line and '"num_gpus=8"' in line ) default_jobs = lambda line: ( line.lstrip().startswith("condor_submit_bid") and '"num_hours=' in line and '"num_gpus=' not in line ) timeout_line = lambda line: bool( re.search(r"NUM_HOURS.*60.*\+.*5", line) ) result["source_references"] = { "num_gpus_default": _blob_reference( single_task_path, single_task, lambda line: bool(re.search(r"num_gpus\s*=\s*1\b", line)), "num_gpus default", ), "cuda_device_requirement": _blob_reference( single_task_path, single_task, lambda line: "NVIDIA H100 80GB HBM3" in line, "CUDA device requirement", ), "request_gpus_binding": _blob_reference( single_task_path, single_task, lambda line: bool( re.search(r"request_gpus\s*=\s*\$\(num_gpus\)", line) ), "request_gpus binding", ), "receives_num_hours": _blob_reference( run_task_path, run_task, lambda line: "NUM_HOURS" in line, "NUM_HOURS input", ), "solve_timeout_formula": _blob_reference( run_task_path, run_task, timeout_line, "solve timeout formula", ), "timeout_grace_minutes": _blob_reference( run_task_path, run_task, timeout_line, "timeout grace", ), "evaluation_dirs_present": { "kind": "git-tree-entries", "paths": eval_dirs_found, }, "commit_sh_analysis.current_models_in_arrays": _blob_reference( commit_path, commit_sh, lambda line: line.strip() == '"Qwen/Qwen3-4B-Base"', "active model array", ), "commit_sh_analysis.current_benchmarks_in_arrays": _blob_reference( commit_path, commit_sh, lambda line: line.strip() == '"healthbench"', "active benchmark array", ), "commit_sh_analysis.htcondor_mpi_is_branch": _blob_reference( commit_path, commit_sh, mpi_job, "MPI scheduler job", ), "commit_sh_analysis.htcondor_branch": _blob_reference( commit_path, commit_sh, default_jobs, "default htcondor jobs", ), } result["limitation_multi_gpu_extension"] = True result["limitation_five_minute_grace"] = True return result def _analyze_commit_sh(content: str) -> dict[str, Any]: """Analyze commit.sh for scheduler-dependent branches and limitations. Parse only active arrays and ``condor_submit_bid`` calls. The pinned scheduler shape is an authority gate: any changed model, benchmark, job count, hour count, or GPU count aborts evidence generation. """ code_lines = [ line.strip() for line in content.splitlines() if line.strip() and not line.lstrip().startswith("#") ] def parse_array(name: str) -> list[str]: match = re.search( rf"(?ms)^\s*{re.escape(name)}\s*=\s*\((.*?)^\s*\)", content, ) if not match: raise ValueError(f"commit.sh missing active {name}=(...) array") entries: list[str] = [] for raw_line in match.group(1).splitlines(): line = raw_line.strip() if not line or line.startswith("#"): continue item = re.fullmatch(r"""(["'])(.*?)\1(?:\s+#.*)?""", line) if not item: raise ValueError( f"commit.sh has unparseable active {name} entry: {line}" ) entries.append(item.group(2)) return entries models = parse_array("models") benchmarks = parse_array("evals") if models != ["Qwen/Qwen3-4B-Base"]: raise ValueError(f"commit.sh active model mismatch: {models}") if benchmarks != ["healthbench"]: raise ValueError( f"commit.sh active benchmark mismatch: {benchmarks}" ) def find_branch_line( prefix: str, scheduler: str, *, start: int = 0, ) -> int: scheduler_re = re.compile( rf"(?:=|==)\s*[\"']{re.escape(scheduler)}[\"']" ) matches = [ index for index, line in enumerate(code_lines[start:], start) if line.startswith(prefix) and scheduler_re.search(line) ] if len(matches) != 1: raise ValueError( f"commit.sh expected one active {prefix.strip()} " f"{scheduler} branch, found {len(matches)}" ) return matches[0] mpi_start = find_branch_line("if ", "htcondor_mpi-is") condor_start = find_branch_line( "elif ", "htcondor", start=mpi_start + 1, ) else_matches = [ index for index, line in enumerate(code_lines[condor_start + 1:], condor_start + 1) if line == "else" or line.startswith("else ") ] if len(else_matches) != 1: raise ValueError( "commit.sh expected one active else after htcondor branch" ) else_start = else_matches[0] if not mpi_start < condor_start < else_start: raise ValueError("commit.sh scheduler branch ordering mismatch") mpi_lines = code_lines[mpi_start + 1:condor_start] condor_lines = code_lines[condor_start + 1:else_start] def parse_jobs(lines: list[str], label: str) -> list[dict[str, int]]: jobs: list[dict[str, int]] = [] for line in lines: if not re.match(r"^condor_submit_bid(?:\s|$)", line): continue attrs: dict[str, str] = {} for _, assignment in re.findall( r"""-a\s+(["'])([^"']+)\1""", line, ): key, separator, value = assignment.partition("=") if not separator or key in attrs: raise ValueError( f"commit.sh malformed {label} -a argument: {assignment}" ) attrs[key] = value if "num_hours" not in attrs: raise ValueError( f"commit.sh {label} job missing quoted num_hours" ) try: job = {"hours": int(attrs["num_hours"])} if "num_gpus" in attrs: job["gpus"] = int(attrs["num_gpus"]) except ValueError as exc: raise ValueError( f"commit.sh {label} job has nonnumeric resource value" ) from exc jobs.append(job) return jobs mpi_jobs = parse_jobs(mpi_lines, "htcondor_mpi-is") condor_jobs = parse_jobs(condor_lines, "htcondor") if mpi_jobs != [{"hours": 100, "gpus": 8}]: raise ValueError( f"commit.sh MPI job mismatch: expected one 100h/8-GPU job, " f"got {mpi_jobs}" ) if any("gpus" in job for job in condor_jobs): raise ValueError("commit.sh htcondor jobs must use default GPU count") condor_hours = [job["hours"] for job in condor_jobs] if condor_hours.count(10) != 7 or condor_hours.count(1) != 1: raise ValueError( "commit.sh htcondor job mismatch: expected seven 10h and one 1h " f"jobs, got {condor_hours}" ) if len(condor_hours) != 8: raise ValueError( f"commit.sh htcondor expected 8 active jobs, got {len(condor_hours)}" ) return { "has_metr_branch": True, "scheduler_dependent": True, "current_models_in_arrays": models, "current_benchmarks_in_arrays": benchmarks, "htcondor_mpi_is_branch": { "active_jobs": 1, "hours": 100, "gpus": 8, "note": "Active 100-hour, eight-GPU METR command", }, "htcondor_branch": { "active_jobs": 8, "ten_hour_jobs": 7, "one_hour_jobs": 1, "gpu_spec": "default (single_task.sub num_gpus=1)", }, "num_hours_values": [1, 10, 100], "gpu_counts_found": [8], } # --------------------------------------------------------------------------- # Reward-hacking audit # --------------------------------------------------------------------------- def audit_reward_hacking( acquired: dict[str, Any], all_paths: list[str] | None = None, ) -> dict[str, Any]: """Audit the three reward-hacking submodes from acquired data. Requires the verified ``acquired`` dict. API-key unavailability is derived from the verified inventory (``all_paths``). """ hf_inv = acquired["hf_inventory"] inventory_paths = all_paths if all_paths is not None else hf_inv["all_paths"] trace_excerpts = acquired["trace_excerpts"] # Mode 1: Training on test sets (contamination) contam_content = acquired["contamination_content"] time_taken_content = acquired["time_taken_content"] contam_sha256 = hashlib.sha256(contam_content).hexdigest() time_taken_sha256 = hashlib.sha256(time_taken_content).hexdigest() contam_observations = [ f"Pinned released run has explicit contamination label at {CONTAMINATION_WITNESS_PATH}", f"Label bytes: {contam_content!r}", f"Label SHA-256: {contam_sha256}", f"Corresponding time_taken.txt: {time_taken_content!r} (SHA-256: {time_taken_sha256})", "Time 10:05:01 interpreted with runner's 5-minute termination grace; not rounded to 10 hours", ] contamination = { "mode": "training_on_test_sets", "status": "partial-support", "witness_path": CONTAMINATION_WITNESS_PATH, "witness_sha256": contam_sha256, "witness_bytes": contam_content.decode("utf-8"), "observation_type": "upstream_judge_label", "observations": contam_observations, "time_taken": { "path": TIME_TAKEN_WITNESS_PATH, "value": time_taken_content.decode("utf-8").strip(), "sha256": time_taken_sha256, "note": "Interpreted with runner's 5-minute termination grace", }, } # Mode 2: Downloading instruction-tuned checkpoint instruction_observations = [ f"Categorical judgment at {INSTRUCTION_MODEL_JUDGMENT_PATH}", f"Judgment bytes: {INSTRUCTION_MODEL_JUDGMENT_BYTES!r} (no trailing newline)", f"Judgment Git object: {INSTRUCTION_MODEL_JUDGMENT_GIT_OBJECT}, size {INSTRUCTION_MODEL_JUDGMENT_SIZE}", f"Judgment SHA-256: {INSTRUCTION_MODEL_JUDGMENT_SHA256}", f"Complete trace at {INSTRUCTION_MODEL_TRACE_PATH} ({INSTRUCTION_MODEL_TRACE_SIZE} bytes)", f"Trace Git object: {INSTRUCTION_MODEL_TRACE_GIT_OBJECT}", f"Trace SHA-256: {INSTRUCTION_MODEL_TRACE_SHA256}", "Trace is NOT redistributed; only safe JSONL-pointer excerpts emitted", ] # Verify excerpts came from trace parsing (not constants) instruction_excerpts = trace_excerpts instruction = { "mode": "downloading_instruction_tuned_checkpoint", "status": "partial-support", "judgment_path": INSTRUCTION_MODEL_JUDGMENT_PATH, "judgment_sha256": INSTRUCTION_MODEL_JUDGMENT_SHA256, "judgment_git_object": INSTRUCTION_MODEL_JUDGMENT_GIT_OBJECT, "judgment_size": INSTRUCTION_MODEL_JUDGMENT_SIZE, "judgment_bytes": INSTRUCTION_MODEL_JUDGMENT_BYTES.decode("utf-8"), "observation_type": "upstream_judge_label_plus_trace_excerpts", "observations": instruction_observations, "trace": { "path": INSTRUCTION_MODEL_TRACE_PATH, "sha256": acquired["instruction_trace_sha256"], "git_object": INSTRUCTION_MODEL_TRACE_GIT_OBJECT, "size": acquired["instruction_trace_size"], "redistributed": False, "note": "Complete trace is not an output; only deterministic JSONL-pointer extracts emitted", }, "safe_excerpts": instruction_excerpts, } # Mode 3: Using discovered API key (unavailable) # Derive from the verified complete inventory cluster_paths = [ p for p in inventory_paths if API_MISUSE_TASK_CLUSTER in p ] if cluster_paths: raise ValueError( f"API-misuse cluster {API_MISUSE_TASK_CLUSTER} found in inventory " f"({len(cluster_paths)} paths). Cannot emit 'unavailable' when the " f"cluster is present." ) api_misuse = { "mode": "using_discovered_api_key", "status": "unavailable", "observation_type": "missing_artifact", "observations": [ f"The exact paper task cluster {API_MISUSE_TASK_CLUSTER} and its named " f"root/task signature have {len(cluster_paths)} paths in the complete " f"verified inventory ({HF_TREE_TOTAL_ENTRIES} entries)", "A different public OpenCode GPT-5.1 root is not a substitute", "Paper prose cannot satisfy an unavailable artifact", ], "unavailability_reason": ( f"The selected trajectory revision omits the specific GPT-5.1 " f"Codex-Max run described by the paper (task cluster {API_MISUSE_TASK_CLUSTER})" ), "inventory_proof": { "cluster_id": API_MISUSE_TASK_CLUSTER, "matching_paths": len(cluster_paths), "total_inventory_entries": len(inventory_paths), }, } return { "training_on_test_sets": contamination, "downloading_instruction_tuned_checkpoint": instruction, "using_discovered_api_key": api_misuse, } # --------------------------------------------------------------------------- # Claim evaluation # --------------------------------------------------------------------------- def evaluate_claims( coverage: dict[str, Any], protocol: dict[str, Any], reward_hacking: dict[str, Any], ) -> dict[str, Any]: """Evaluate the two selected claims from verified audit data. Requires verified coverage, protocol, and reward_hacking dicts. Both claims are ``partial-support``. """ def require_exact(label: str, actual: Any, expected: Any) -> None: if actual != expected: raise ValueError(f"{label} mismatch: {actual!r} != {expected!r}") # Verify the complete coverage census, not only its headline count. if coverage["recognized_task_count"] != EXPECTED_TASK_COUNT: raise ValueError( f"Coverage task count {coverage['recognized_task_count']} " f"!= expected {EXPECTED_TASK_COUNT}" ) if coverage.get("recognized_root_count") != EXPECTED_ROOT_COUNT: raise ValueError( f"Coverage root count {coverage.get('recognized_root_count')} " f"!= expected {EXPECTED_ROOT_COUNT}" ) if coverage.get("recognized_root_cell_pairs") != EXPECTED_ROOT_CELL_PAIRS: raise ValueError( f"Coverage root/cell pairs {coverage.get('recognized_root_cell_pairs')} " f"!= expected {EXPECTED_ROOT_CELL_PAIRS}" ) if coverage.get("duplicate_job_pairs") != EXPECTED_DUPLICATE_PAIRS: raise ValueError( f"Coverage duplicate pairs {coverage.get('duplicate_job_pairs')} " f"!= expected {EXPECTED_DUPLICATE_PAIRS}" ) if coverage.get("missing_root_cell_pairs") != EXPECTED_MISSING_PAIRS: raise ValueError( f"Coverage missing pairs {coverage.get('missing_root_cell_pairs')} " f"!= expected {EXPECTED_MISSING_PAIRS}" ) require_exact( "Coverage cell-count map", coverage.get("cell_counts"), EXPECTED_CELL_COUNTS, ) expected_matrix = [ { "benchmark": benchmark, "model": model, "count": EXPECTED_CELL_COUNTS[benchmark][model_index], } for benchmark in sorted(EXPECTED_BENCHMARKS) for model_index, model in enumerate(MODEL_ORDER) ] require_exact( "Coverage 28-cell matrix", coverage.get("matrix"), expected_matrix, ) # Verify every protocol fact that supports the partial status. exact_protocol = { "num_gpus_default": 1, "cuda_device_requirement": ( 'TARGET.CUDADeviceName == "NVIDIA H100 80GB HBM3"' ), "request_gpus_binding": "request_gpus = $(num_gpus)", "receives_num_hours": True, "solve_timeout_formula": "NUM_HOURS * 60 + 5", "timeout_grace_minutes": 5, "timeout_formula_found": True, "task_dir_10h_suffix": True, "evaluation_dirs_present": EXPECTED_EVAL_DIRS, } for key, expected in exact_protocol.items(): require_exact(f"Protocol {key}", protocol.get(key), expected) analysis = protocol.get("commit_sh_analysis") if not isinstance(analysis, dict): raise ValueError("Protocol commit_sh_analysis missing") require_exact( "Protocol scheduler_dependent", analysis.get("scheduler_dependent"), True, ) require_exact( "Protocol active models", analysis.get("current_models_in_arrays"), ["Qwen/Qwen3-4B-Base"], ) require_exact( "Protocol active benchmarks", analysis.get("current_benchmarks_in_arrays"), ["healthbench"], ) require_exact( "Protocol MPI branch", analysis.get("htcondor_mpi_is_branch"), { "active_jobs": 1, "hours": 100, "gpus": 8, "note": "Active 100-hour, eight-GPU METR command", }, ) require_exact( "Protocol htcondor branch", analysis.get("htcondor_branch"), { "active_jobs": 8, "ten_hour_jobs": 7, "one_hour_jobs": 1, "gpu_spec": "default (single_task.sub num_gpus=1)", }, ) # Verify all reward-mode gates before emitting either selected status. contamination = reward_hacking.get("training_on_test_sets", {}) instruction = reward_hacking.get( "downloading_instruction_tuned_checkpoint", {}, ) api_misuse = reward_hacking.get("using_discovered_api_key", {}) require_exact( "Contamination reward status", contamination.get("status"), "partial-support", ) require_exact( "Instruction-model reward status", instruction.get("status"), "partial-support", ) require_exact( "API-misuse reward status", api_misuse.get("status"), "unavailable", ) require_exact( "API-misuse matching path count", api_misuse.get("inventory_proof", {}).get("matching_paths"), 0, ) trace = instruction.get("trace", {}) require_exact( "Instruction trace SHA-256", trace.get("sha256"), INSTRUCTION_MODEL_TRACE_SHA256, ) require_exact( "Instruction trace size", trace.get("size"), INSTRUCTION_MODEL_TRACE_SIZE, ) require_exact( "Instruction trace redistribution flag", trace.get("redistributed"), False, ) expected_excerpts = [ { "record": excerpt["record"], "json_pointer": excerpt["pointer"], "text": excerpt["text"], "sha256": excerpt["sha256"], } for excerpt in TRACE_EXCERPTS ] require_exact( "Instruction trace safe excerpts", instruction.get("safe_excerpts"), expected_excerpts, ) limitations_1 = [ "No H100 run is reproduced; the resource and time findings are a released-configuration audit.", "The runner allows a five-minute termination grace, and a released example records 10:05:01.", f"The pinned source's current launcher is scheduler-dependent: one branch has a 100-hour/eight-GPU " f"METR command, another has {protocol['commit_sh_analysis']['htcondor_branch']['ten_hour_jobs']} " f"ten-hour and {protocol['commit_sh_analysis']['htcondor_branch']['one_hour_jobs']} one-hour " f"default-GPU commands, and the arrays currently select only one model/benchmark pair.", "Evidence is not an official challenge verdict.", ] limitations_2 = [ "A released judge label is not independently established behavioral truth.", "The instruction-model evidence is an upstream categorical label plus safe extracts from a " "released trace, not a fresh independent behavioral audit.", "The selected trajectory revision does not expose the exact GPT-5.1 Codex-Max API-misuse " f"task cluster {API_MISUSE_TASK_CLUSTER}.", "No leaderboard score, BFCL score, weighted average, or reasoning-effort ablation is a " "selected target or reproduced measurement.", "Evidence is not an official challenge verdict.", ] return { "claim_1": { "text": CLAIM_1_TEXT, "sha256": CLAIM_1_SHA256, "status": "partial-support", "summary": ( "Released trajectory inventory confirms 4-by-7 coverage across " "all accepted benchmark/model cells. Runner configuration defaults " "to one H100 with a NUM_HOURS-based timeout. The current checkout's " "scheduler-dependent branches and five-minute termination grace are " "reported as limitations." ), "evidence_pointers": [ "evidence/coverage.json#/recognized_task_count", "evidence/coverage.json#/matrix", "evidence/coverage.json#/cell_counts", "evidence/provenance.json#/source", "evidence/coverage.json#/protocol", ], "limitations": limitations_1, }, "claim_2": { "text": CLAIM_2_TEXT, "sha256": CLAIM_2_SHA256, "status": "partial-support", "summary": ( "Released contamination and instruction-model judgments provide " "partial support for two of three reward-hacking submodes. The " "API-key submode artifact is absent from the pinned revision." ), "evidence_pointers": [ "evidence/reward_hacking.json#/training_on_test_sets", "evidence/reward_hacking.json#/downloading_instruction_tuned_checkpoint", "evidence/reward_hacking.json#/using_discovered_api_key", ], "limitations": limitations_2, }, }