File size: 4,465 Bytes
5bfc214 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | """Resolves commit dates for SecVulEval's (project_url, commit_id) pairs via
the GitHub REST API, to close the chronological-split date-coverage gap
documented in DECISIONS.md ("Chronological split date coverage").
Resumable: writes one JSON line per resolved commit to a cache file as it
goes (spec §1.5, "preprocessing writes intermediate artifacts so re-runs
resume rather than restart"); re-running skips commits already in the cache.
Requires a GITHUB_TOKEN env var (personal access token, no special scopes
needed for public-repo reads) to get the 5000 req/hour authenticated rate
limit — the unauthenticated 60/hour limit makes ~6,050 distinct commits
impractical (100+ hours).
"""
from __future__ import annotations
import argparse
import json
import os
import re
import time
from pathlib import Path
import pandas as pd
import requests
CACHE_PATH = Path("data/processed/secvuleval_commit_dates.jsonl")
_OWNER_REPO_RE = re.compile(r"github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$")
def _owner_repo(project_url: str) -> tuple[str, str] | None:
m = _OWNER_REPO_RE.search(project_url or "")
if not m:
return None
return m.group(1), m.group(2)
def load_cache(path: Path = CACHE_PATH) -> dict[str, dict]:
cache: dict[str, dict] = {}
if path.exists():
with open(path) as f:
for line in f:
line = line.strip()
if line:
rec = json.loads(line)
cache[rec["commit_id"]] = rec
return cache
def fetch_commit_date(session: requests.Session, owner: str, repo: str, commit_id: str) -> dict:
url = f"https://api.github.com/repos/{owner}/{repo}/commits/{commit_id}"
resp = session.get(url, timeout=15)
if resp.status_code == 200:
data = resp.json()
date = (
data.get("commit", {}).get("committer", {}).get("date")
or data.get("commit", {}).get("author", {}).get("date")
)
return {"commit_id": commit_id, "date": date, "status": "ok"}
return {"commit_id": commit_id, "date": None, "status": f"http_{resp.status_code}"}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--secvuleval-parquet", default="data/raw/SecVulEval/secvuleval.parquet")
parser.add_argument("--cache", default=str(CACHE_PATH))
parser.add_argument("--max-requests", type=int, default=10_000)
args = parser.parse_args()
token = os.environ.get("GITHUB_TOKEN")
if not token:
raise SystemExit("GITHUB_TOKEN env var required (pass inline: GITHUB_TOKEN=... python ...)")
cache_path = Path(args.cache)
cache_path.parent.mkdir(parents=True, exist_ok=True)
cache = load_cache(cache_path)
print(f"loaded {len(cache)} cached commit dates")
df = pd.read_parquet(args.secvuleval_parquet)
distinct = df[["commit_id", "project_url"]].drop_duplicates("commit_id")
todo = distinct[~distinct["commit_id"].isin(cache.keys())]
print(f"{len(distinct)} distinct commits, {len(todo)} not yet cached")
session = requests.Session()
session.headers.update({
"Authorization": f"token {token}",
"Accept": "application/vnd.github+json",
})
n_requests = 0
n_ok = 0
n_no_owner_repo = 0
with open(cache_path, "a") as f:
for row in todo.itertuples():
if n_requests >= args.max_requests:
print(f"hit --max-requests={args.max_requests}, stopping (resume later)")
break
owner_repo = _owner_repo(row.project_url)
if owner_repo is None:
n_no_owner_repo += 1
rec = {"commit_id": row.commit_id, "date": None, "status": "no_owner_repo"}
f.write(json.dumps(rec) + "\n")
continue
owner, repo = owner_repo
rec = fetch_commit_date(session, owner, repo, row.commit_id)
n_requests += 1
if rec["status"] == "ok":
n_ok += 1
f.write(json.dumps(rec) + "\n")
f.flush()
remaining = session.headers.get("X-RateLimit-Remaining")
if n_requests % 200 == 0:
print(f"{n_requests} requests made, {n_ok} resolved so far")
# Be polite even though we have 5000/hr headroom.
time.sleep(0.05)
print(f"done: {n_requests} requests, {n_ok} resolved, {n_no_owner_repo} unparseable project_url")
if __name__ == "__main__":
main()
|