Spaces:
Running
Running
Download evaluation/benchmark/data.py from realdexter/RepoReaper: direct link, hf CLI and curl.
- Browser
- Download file 8.17 kB
-
https://huggingface.co/spaces/realdexter/RepoReaper/resolve/main/evaluation/benchmark/data.py
- Command line
-
hf download hf://spaces/realdexter/RepoReaper/evaluation/benchmark/data.py
-
curl -L -o data.py https://huggingface.co/spaces/realdexter/RepoReaper/resolve/main/evaluation/benchmark/data.py
8.17 kB
| """Dataset pinning and deterministic, reference-free generation manifests.""" | |
| import hashlib | |
| import json | |
| import re | |
| import urllib.request | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| DATASET = "TIGER-Lab/SWE-QA-Pro-Bench" | |
| UPSTREAM = "TIGER-AI-Lab/SWE-QA-Pro" | |
| def digest(value): | |
| return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest() | |
| def read_json(path): | |
| return json.loads(Path(path).read_text()) | |
| def load_manifest(path): | |
| manifest = read_json(path) | |
| if manifest["manifest_hash"] != digest({k: v for k, v in manifest.items() if k != "manifest_hash"}): | |
| raise ValueError("Manifest content hash mismatch") | |
| return manifest | |
| def write_json(path, value): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = path.with_suffix(path.suffix + ".tmp") | |
| tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n") | |
| tmp.replace(path) | |
| def fetch(url): | |
| request = urllib.request.Request(url, headers={"User-Agent": "RepoReaper-benchmark"}) | |
| with urllib.request.urlopen(request, timeout=60) as response: | |
| return response.read() | |
| def download_dataset(directory): | |
| directory = Path(directory) | |
| info = json.loads(fetch(f"https://huggingface.co/api/datasets/{DATASET}")) | |
| revision = info["sha"] | |
| candidates = [r["rfilename"] for r in info["siblings"] if r["rfilename"].endswith(".jsonl")] | |
| if len(candidates) != 1: | |
| raise ValueError(f"Expected exactly one benchmark JSONL, found {candidates}") | |
| raw = fetch(f"https://huggingface.co/datasets/{DATASET}/resolve/{revision}/{candidates[0]}") | |
| rows = [json.loads(line) for line in raw.decode().splitlines() if line.strip()] | |
| upstream = json.loads(fetch(f"https://api.github.com/repos/{UPSTREAM}/commits/main"))["sha"] | |
| prompt = fetch(f"https://raw.githubusercontent.com/{UPSTREAM}/{upstream}/eval/prompts/judge_prompt.txt") | |
| write_json(directory / "dataset.json", rows) | |
| (directory / "judge_prompt.txt").write_bytes(prompt) | |
| metadata = {"dataset": DATASET, "revision": revision, "dataset_sha256": hashlib.sha256(raw).hexdigest(), | |
| "upstream_revision": upstream, "judge_prompt_sha256": hashlib.sha256(prompt).hexdigest(), | |
| "url": f"https://huggingface.co/datasets/{DATASET}/tree/{revision}"} | |
| write_json(directory / "provenance.json", metadata) | |
| return metadata | |
| def prepare(rows, provenance, directory, per_repo=4, seed=42): | |
| if per_repo not in (2, 4): | |
| raise ValueError("formal samples per repository must be 2 or 4") | |
| directory = Path(directory) | |
| if (directory / "manifest.json").exists(): | |
| raise ValueError("Manifest already exists; create a new run directory") | |
| groups = defaultdict(list) | |
| references = {} | |
| for row in rows: | |
| repo, sha, question = row["repo"], row["commit_id"], row["question"].strip() | |
| if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo): | |
| raise ValueError("Invalid repository identifier") | |
| if not re.fullmatch(r"[0-9a-f]{40}", sha) or not question or not row["answer"].strip(): | |
| raise ValueError("Invalid commit/question/reference") | |
| sample_id = digest([repo, sha, question]) | |
| if sample_id in references: | |
| raise ValueError("Duplicate benchmark question") | |
| references[sample_id] = row["answer"] | |
| # Whitelist fields: no reference answers or extra dataset fields cross | |
| # the worker boundary. qa_type/cluster are reporting metadata only. | |
| groups[repo].append({"id": sample_id, "repo": repo, "commit_id": sha, | |
| "question": question, "qa_type": row.get("qa_type", {}), | |
| "cluster": row.get("cluster", {})}) | |
| samples = [] | |
| for repo, records in sorted(groups.items()): | |
| if len(records) < per_repo + 1 or len({r["commit_id"] for r in records}) != 1: | |
| raise ValueError(f"Insufficient samples or multiple revisions: {repo}") | |
| ordered = sorted(records, key=lambda r: digest([seed, r["id"]])) | |
| for index, row in enumerate(ordered[:per_repo + 1]): | |
| samples.append(dict(row, split="dev" if index == 0 else "test", | |
| evidence_review=index == 1, repeat=index == 1)) | |
| selected_refs = {s["id"]: references[s["id"]] for s in samples} | |
| manifest = {"protocol": "SWE-QA-Pro fixed subset / custom RepoReaper Agent", | |
| "seed": seed, "per_repo": per_repo, "provenance": provenance, "samples": samples, | |
| "reference_hash": digest(selected_refs), | |
| "distribution": dict(Counter(str(s["qa_type"]) for s in samples if s["split"] == "test"))} | |
| manifest["manifest_hash"] = digest(manifest) | |
| write_json(directory / "manifest.json", manifest) | |
| write_json(directory / "references.json", selected_refs) | |
| write_json(directory / "evidence_labels.json", { | |
| s["id"]: {"repo": s["repo"], "commit_id": s["commit_id"], "question": s["question"], | |
| "required_files": [], "annotator": "", "verified_before_review": False} | |
| for s in samples if s["evidence_review"]}) | |
| return manifest | |
| def prepare_jit(rows, provenance, source_manifest, directory, repos, seed=42, *, | |
| experiment="jev", excluded_manifests=()): | |
| """Freeze one previously unused question per requested repository.""" | |
| directory = Path(directory) | |
| if (directory / "manifest.json").exists(): | |
| raise ValueError("Manifest already exists; create a new run directory") | |
| revisions = {s["repo"]: s["commit_id"] for s in source_manifest["samples"]} | |
| seen = {s["id"] for manifest in (source_manifest, *excluded_manifests) for s in manifest["samples"]} | |
| variants = {"jev": ["C", "F", "G"], "exploration": ["F", "H", "I"], | |
| "source": ["H", "K", "L"], "tools": ["K", "L", "M"]}[experiment] | |
| manifest = {"protocol": "SWE-QA-Pro JIT pilot on shared sealed seeds", "variants": variants, | |
| "seed": seed, "source_manifest_hash": source_manifest["manifest_hash"], | |
| "provenance": provenance, "samples": []} | |
| references = {} | |
| for repo in sorted(set(repos)): | |
| sha = revisions[repo] | |
| candidates = {} | |
| for row in rows: | |
| if row["repo"] != repo or row["commit_id"] != sha: | |
| continue | |
| question = row["question"].strip() | |
| sample_id = digest([repo, sha, question]) | |
| if sample_id not in seen and question and row["answer"].strip(): | |
| candidates[sample_id] = row | |
| if not candidates: | |
| raise ValueError(f"No unused questions at the sealed revision for {repo}") | |
| sample_id = min(candidates, key=lambda candidate: digest([seed, candidate])) | |
| row = candidates[sample_id] | |
| references[sample_id] = row["answer"] | |
| manifest["samples"].append({"id": sample_id, "repo": repo, "commit_id": sha, | |
| "question": row["question"].strip(), "split": "dev", | |
| "evidence_review": True, "repeat": False}) | |
| manifest["reference_hash"] = digest(references) | |
| manifest["manifest_hash"] = digest(manifest) | |
| write_json(directory / "manifest.json", manifest) | |
| write_json(directory / "references.json", references) | |
| return manifest | |
| def jobs(manifest, split, variants=("A", "B", "C", "D"), repeats=False): | |
| for index, sample in enumerate(manifest["samples"]): | |
| if sample["split"] != split or (repeats and not sample["repeat"]): | |
| continue | |
| rotation = index % len(variants) if manifest.get("variants") else 0 | |
| for variant in (*variants[rotation:], *variants[:rotation]): | |
| if variant not in ("A", "B", "C", "D", "E", "F", "G", "H", "I", "K", "L", "M"): | |
| raise ValueError("Unknown variant") | |
| for repeat in ((1, 2) if repeats else (0,)): | |
| if repeats and variant not in ("B", "C"): | |
| continue | |
| yield {"job_id": f"{sample['id']}_{variant}_{repeat}", "sample": sample, | |
| "variant": variant, "repeat": repeat} | |