GitHub Actions Bot
deploy: auto-inject hf config & sync
30f75ba
Raw History Blame Contribute Delete
8.17 kB
"""Dataset pinning and deterministic, reference-free generation manifests."""
import hashlib
import json
import re
import urllib.request
from collections import Counter, defaultdict
from pathlib import Path
DATASET = "TIGER-Lab/SWE-QA-Pro-Bench"
UPSTREAM = "TIGER-AI-Lab/SWE-QA-Pro"
def digest(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def read_json(path):
return json.loads(Path(path).read_text())
def load_manifest(path):
manifest = read_json(path)
if manifest["manifest_hash"] != digest({k: v for k, v in manifest.items() if k != "manifest_hash"}):
raise ValueError("Manifest content hash mismatch")
return manifest
def write_json(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
tmp.replace(path)
def fetch(url):
request = urllib.request.Request(url, headers={"User-Agent": "RepoReaper-benchmark"})
with urllib.request.urlopen(request, timeout=60) as response:
return response.read()
def download_dataset(directory):
directory = Path(directory)
info = json.loads(fetch(f"https://huggingface.co/api/datasets/{DATASET}"))
revision = info["sha"]
candidates = [r["rfilename"] for r in info["siblings"] if r["rfilename"].endswith(".jsonl")]
if len(candidates) != 1:
raise ValueError(f"Expected exactly one benchmark JSONL, found {candidates}")
raw = fetch(f"https://huggingface.co/datasets/{DATASET}/resolve/{revision}/{candidates[0]}")
rows = [json.loads(line) for line in raw.decode().splitlines() if line.strip()]
upstream = json.loads(fetch(f"https://api.github.com/repos/{UPSTREAM}/commits/main"))["sha"]
prompt = fetch(f"https://raw.githubusercontent.com/{UPSTREAM}/{upstream}/eval/prompts/judge_prompt.txt")
write_json(directory / "dataset.json", rows)
(directory / "judge_prompt.txt").write_bytes(prompt)
metadata = {"dataset": DATASET, "revision": revision, "dataset_sha256": hashlib.sha256(raw).hexdigest(),
"upstream_revision": upstream, "judge_prompt_sha256": hashlib.sha256(prompt).hexdigest(),
"url": f"https://huggingface.co/datasets/{DATASET}/tree/{revision}"}
write_json(directory / "provenance.json", metadata)
return metadata
def prepare(rows, provenance, directory, per_repo=4, seed=42):
if per_repo not in (2, 4):
raise ValueError("formal samples per repository must be 2 or 4")
directory = Path(directory)
if (directory / "manifest.json").exists():
raise ValueError("Manifest already exists; create a new run directory")
groups = defaultdict(list)
references = {}
for row in rows:
repo, sha, question = row["repo"], row["commit_id"], row["question"].strip()
if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo):
raise ValueError("Invalid repository identifier")
if not re.fullmatch(r"[0-9a-f]{40}", sha) or not question or not row["answer"].strip():
raise ValueError("Invalid commit/question/reference")
sample_id = digest([repo, sha, question])
if sample_id in references:
raise ValueError("Duplicate benchmark question")
references[sample_id] = row["answer"]
# Whitelist fields: no reference answers or extra dataset fields cross
# the worker boundary. qa_type/cluster are reporting metadata only.
groups[repo].append({"id": sample_id, "repo": repo, "commit_id": sha,
"question": question, "qa_type": row.get("qa_type", {}),
"cluster": row.get("cluster", {})})
samples = []
for repo, records in sorted(groups.items()):
if len(records) < per_repo + 1 or len({r["commit_id"] for r in records}) != 1:
raise ValueError(f"Insufficient samples or multiple revisions: {repo}")
ordered = sorted(records, key=lambda r: digest([seed, r["id"]]))
for index, row in enumerate(ordered[:per_repo + 1]):
samples.append(dict(row, split="dev" if index == 0 else "test",
evidence_review=index == 1, repeat=index == 1))
selected_refs = {s["id"]: references[s["id"]] for s in samples}
manifest = {"protocol": "SWE-QA-Pro fixed subset / custom RepoReaper Agent",
"seed": seed, "per_repo": per_repo, "provenance": provenance, "samples": samples,
"reference_hash": digest(selected_refs),
"distribution": dict(Counter(str(s["qa_type"]) for s in samples if s["split"] == "test"))}
manifest["manifest_hash"] = digest(manifest)
write_json(directory / "manifest.json", manifest)
write_json(directory / "references.json", selected_refs)
write_json(directory / "evidence_labels.json", {
s["id"]: {"repo": s["repo"], "commit_id": s["commit_id"], "question": s["question"],
"required_files": [], "annotator": "", "verified_before_review": False}
for s in samples if s["evidence_review"]})
return manifest
def prepare_jit(rows, provenance, source_manifest, directory, repos, seed=42, *,
experiment="jev", excluded_manifests=()):
"""Freeze one previously unused question per requested repository."""
directory = Path(directory)
if (directory / "manifest.json").exists():
raise ValueError("Manifest already exists; create a new run directory")
revisions = {s["repo"]: s["commit_id"] for s in source_manifest["samples"]}
seen = {s["id"] for manifest in (source_manifest, *excluded_manifests) for s in manifest["samples"]}
variants = {"jev": ["C", "F", "G"], "exploration": ["F", "H", "I"],
"source": ["H", "K", "L"], "tools": ["K", "L", "M"]}[experiment]
manifest = {"protocol": "SWE-QA-Pro JIT pilot on shared sealed seeds", "variants": variants,
"seed": seed, "source_manifest_hash": source_manifest["manifest_hash"],
"provenance": provenance, "samples": []}
references = {}
for repo in sorted(set(repos)):
sha = revisions[repo]
candidates = {}
for row in rows:
if row["repo"] != repo or row["commit_id"] != sha:
continue
question = row["question"].strip()
sample_id = digest([repo, sha, question])
if sample_id not in seen and question and row["answer"].strip():
candidates[sample_id] = row
if not candidates:
raise ValueError(f"No unused questions at the sealed revision for {repo}")
sample_id = min(candidates, key=lambda candidate: digest([seed, candidate]))
row = candidates[sample_id]
references[sample_id] = row["answer"]
manifest["samples"].append({"id": sample_id, "repo": repo, "commit_id": sha,
"question": row["question"].strip(), "split": "dev",
"evidence_review": True, "repeat": False})
manifest["reference_hash"] = digest(references)
manifest["manifest_hash"] = digest(manifest)
write_json(directory / "manifest.json", manifest)
write_json(directory / "references.json", references)
return manifest
def jobs(manifest, split, variants=("A", "B", "C", "D"), repeats=False):
for index, sample in enumerate(manifest["samples"]):
if sample["split"] != split or (repeats and not sample["repeat"]):
continue
rotation = index % len(variants) if manifest.get("variants") else 0
for variant in (*variants[rotation:], *variants[:rotation]):
if variant not in ("A", "B", "C", "D", "E", "F", "G", "H", "I", "K", "L", "M"):
raise ValueError("Unknown variant")
for repeat in ((1, 2) if repeats else (0,)):
if repeats and variant not in ("B", "C"):
continue
yield {"job_id": f"{sample['id']}_{variant}_{repeat}", "sample": sample,
"variant": variant, "repeat": repeat}