| """Resolves commit dates for SecVulEval's (project_url, commit_id) pairs via |
| the GitHub REST API, to close the chronological-split date-coverage gap |
| documented in DECISIONS.md ("Chronological split date coverage"). |
| |
| Resumable: writes one JSON line per resolved commit to a cache file as it |
| goes (spec §1.5, "preprocessing writes intermediate artifacts so re-runs |
| resume rather than restart"); re-running skips commits already in the cache. |
| |
| Requires a GITHUB_TOKEN env var (personal access token, no special scopes |
| needed for public-repo reads) to get the 5000 req/hour authenticated rate |
| limit — the unauthenticated 60/hour limit makes ~6,050 distinct commits |
| impractical (100+ hours). |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import os |
| import re |
| import time |
| from pathlib import Path |
|
|
| import pandas as pd |
| import requests |
|
|
| CACHE_PATH = Path("data/processed/secvuleval_commit_dates.jsonl") |
| _OWNER_REPO_RE = re.compile(r"github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$") |
|
|
|
|
| def _owner_repo(project_url: str) -> tuple[str, str] | None: |
| m = _OWNER_REPO_RE.search(project_url or "") |
| if not m: |
| return None |
| return m.group(1), m.group(2) |
|
|
|
|
| def load_cache(path: Path = CACHE_PATH) -> dict[str, dict]: |
| cache: dict[str, dict] = {} |
| if path.exists(): |
| with open(path) as f: |
| for line in f: |
| line = line.strip() |
| if line: |
| rec = json.loads(line) |
| cache[rec["commit_id"]] = rec |
| return cache |
|
|
|
|
| def fetch_commit_date(session: requests.Session, owner: str, repo: str, commit_id: str) -> dict: |
| url = f"https://api.github.com/repos/{owner}/{repo}/commits/{commit_id}" |
| resp = session.get(url, timeout=15) |
| if resp.status_code == 200: |
| data = resp.json() |
| date = ( |
| data.get("commit", {}).get("committer", {}).get("date") |
| or data.get("commit", {}).get("author", {}).get("date") |
| ) |
| return {"commit_id": commit_id, "date": date, "status": "ok"} |
| return {"commit_id": commit_id, "date": None, "status": f"http_{resp.status_code}"} |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--secvuleval-parquet", default="data/raw/SecVulEval/secvuleval.parquet") |
| parser.add_argument("--cache", default=str(CACHE_PATH)) |
| parser.add_argument("--max-requests", type=int, default=10_000) |
| args = parser.parse_args() |
|
|
| token = os.environ.get("GITHUB_TOKEN") |
| if not token: |
| raise SystemExit("GITHUB_TOKEN env var required (pass inline: GITHUB_TOKEN=... python ...)") |
|
|
| cache_path = Path(args.cache) |
| cache_path.parent.mkdir(parents=True, exist_ok=True) |
| cache = load_cache(cache_path) |
| print(f"loaded {len(cache)} cached commit dates") |
|
|
| df = pd.read_parquet(args.secvuleval_parquet) |
| distinct = df[["commit_id", "project_url"]].drop_duplicates("commit_id") |
| todo = distinct[~distinct["commit_id"].isin(cache.keys())] |
| print(f"{len(distinct)} distinct commits, {len(todo)} not yet cached") |
|
|
| session = requests.Session() |
| session.headers.update({ |
| "Authorization": f"token {token}", |
| "Accept": "application/vnd.github+json", |
| }) |
|
|
| n_requests = 0 |
| n_ok = 0 |
| n_no_owner_repo = 0 |
| with open(cache_path, "a") as f: |
| for row in todo.itertuples(): |
| if n_requests >= args.max_requests: |
| print(f"hit --max-requests={args.max_requests}, stopping (resume later)") |
| break |
| owner_repo = _owner_repo(row.project_url) |
| if owner_repo is None: |
| n_no_owner_repo += 1 |
| rec = {"commit_id": row.commit_id, "date": None, "status": "no_owner_repo"} |
| f.write(json.dumps(rec) + "\n") |
| continue |
| owner, repo = owner_repo |
| rec = fetch_commit_date(session, owner, repo, row.commit_id) |
| n_requests += 1 |
| if rec["status"] == "ok": |
| n_ok += 1 |
| f.write(json.dumps(rec) + "\n") |
| f.flush() |
|
|
| remaining = session.headers.get("X-RateLimit-Remaining") |
| if n_requests % 200 == 0: |
| print(f"{n_requests} requests made, {n_ok} resolved so far") |
| |
| time.sleep(0.05) |
|
|
| print(f"done: {n_requests} requests, {n_ok} resolved, {n_no_owner_repo} unparseable project_url") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
