tracker script

tracker.py

@@ -0,0 +1,235 @@
+# /// script
+# dependencies = ["huggingface_hub>=0.35", "requests"]
+# ///
+"""Daily radar: transformers-related issues + monkey patches in downstream repos -> static Space.
+
+Run:      HF_TOKEN=... GITHUB_TOKEN=... uv run tracker.py            (DRY=1 to write index.html locally instead of pushing)
+Schedule: hf jobs scheduled uv run daily https://huggingface.co/spaces/$SPACE/resolve/main/tracker.py --secrets HF_TOKEN  (GITHUB_TOKEN optional: 3x faster search)
+"""
+
+import json
+import os
+import re
+import subprocess
+import tempfile
+import time
+from collections import defaultdict
+from datetime import date, timedelta
+from html import escape
+from pathlib import Path
+
+import requests
+from huggingface_hub import HfApi, InferenceClient, hf_hub_download
+
+# Heavy transformers users, ordered by GitHub stars (2026-11). Picked from stars >5k or pypi >300k/month.
+REPOS = [
+    "ggml-org/llama.cpp", "vllm-project/vllm", "unslothai/unsloth", "hiyouga/LlamaFactory",
+    "deepspeedai/DeepSpeed", "hpcaitech/ColossalAI", "sgl-project/sglang", "huggingface/diffusers",
+    "huggingface/lerobot", "huggingface/open-r1", "m-bain/whisperX", "verl-project/verl", "huggingface/peft",
+    "huggingface/trl", "kvcache-ai/ktransformers", "huggingface/sentence-transformers", "NVIDIA-NeMo/Automodel",
+    "modelscope/ms-swift", "dottxt-ai/outlines", "NVIDIA/TensorRT-LLM", "flairNLP/flair",
+    "EleutherAI/lm-evaluation-harness", "axolotl-ai-cloud/axolotl", "FlagOpen/FlagEmbedding",
+    "OpenRLHF/OpenRLHF", "huggingface/accelerate", "THUDM/slime", "bitsandbytes-foundation/bitsandbytes",
+    "ml-explore/mlx-lm", "arcee-ai/mergekit", "linkedin/Liger-Kernel", "Blaizzy/mlx-vlm",
+    "EvolvingLMMs-Lab/lmms-eval", "urchade/GLiNER", "vllm-project/llm-compressor", "allenai/open-instruct",
+    "huggingface/optimum", "embeddings-benchmark/mteb", "alibaba/ROLL", "PrimeIntellect-ai/prime-rl",
+    "intel/auto-round", "ModelCloud/GPTQModel", "huggingface/optimum-quanto", "unslothai/unsloth-zoo",
+]  # fmt: skip
+SPACE = os.environ.get("SPACE", "ArthurZ/transformers-downstream-tracker")
+MODEL = os.environ.get("MODEL", "Qwen/Qwen3-235B-A22B-Instruct-2507")
+DAYS = int(os.environ.get("DAYS", 2))  # ponytail: overlap between daily runs is fine
+DRY = os.environ.get("DRY") == "1"
+
+# ponytail: regex heuristic, catches `transformers.x.y = ...`, `modeling_llama.Foo.forward = ...`, setattr on
+# transformers modules, and `LlamaAttention.forward = ...`. Misses aliased imports; AST pass if that matters.
+PATCH_RE = re.compile(
+    r"^\s*(transformers(\.\w+)+|modeling_\w+(\.\w+)+"
+    r"|[A-Z]\w*(Attention|MLP|Model|ForCausalLM|DecoderLayer|RMSNorm|PreTrainedModel|Embedding)\w*\.\w+)\s*=[^=]"
+    r"|setattr\(\s*(transformers|modeling_)"
+)
+MODEL_CLS = re.compile(
+    r"^(?!PreTrainedModel)[A-Z]\w*?(?=(ForCausalLM|ForConditionalGeneration|SdpaAttention|FlashAttention2|Attention"
+    r"|MLP|DecoderLayer|RMSNorm|RotaryEmbedding|SparseMoeBlock|Experts|Model)\b)"
+)
+
+
+def issues():
+    # GitHub search, not GH Archive: the public events feed is heavily sampled (~40 of ~650 daily vllm events).
+    since = (date.today() - timedelta(days=DAYS)).isoformat()
+    headers = {"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"} if "GITHUB_TOKEN" in os.environ else {}
+    rows = []
+    for i, repo in enumerate(REPOS, 1):
+        q = f"repo:{repo} transformers in:title,body,comments updated:>={since}"
+        r = requests.get(
+            "https://api.github.com/search/issues", params={"q": q, "per_page": 100, "sort": "updated"},
+            headers=headers, timeout=60,
+        )  # fmt: skip
+        if not r.ok:  # renamed/private repo shouldn't kill the whole report
+            print(f"[{i}/{len(REPOS)}] {repo}: search failed {r.status_code}", flush=True)
+            continue
+        # ponytail: first 100 most recently updated only; vllm hits the cap, paginate if the tail matters
+        items = r.json()["items"]
+        rows += [
+            {"repo_name": repo, "number": it["number"], "t": it["title"], "b": it["body"], "comments": it["comments"],
+             "is_pr": "pull_request" in it}
+            for it in items
+        ]  # fmt: skip
+        print(f"[{i}/{len(REPOS)}] {repo}: {len(items)} threads", flush=True)
+        time.sleep(2.1 if headers else 6.5)  # search API: 30 req/min authenticated, 10 anonymous
+    return sorted(rows, key=lambda r: -r["comments"])
+
+
+def patches():
+    found = set()
+    for i, repo in enumerate(REPOS, 1):
+        with tempfile.TemporaryDirectory() as dst:  # one at a time, keeps job disk small
+            clone = ["git", "clone", "-q", "--depth", "1", "--filter=blob:limit=1m", f"https://github.com/{repo}", dst]
+            if subprocess.run(clone).returncode:
+                print(f"[{i}/{len(REPOS)}] {repo}: clone failed", flush=True)
+                continue
+            files = subprocess.run(["git", "ls-files", "*.py"], cwd=dst, capture_output=True, text=True).stdout
+            # key on file + stripped line so line-number churn doesn't count as a new patch
+            hits = {
+                f"{repo}:{f}: {line.strip()}"
+                for f in files.split()
+                if "test" not in f and Path(dst, f).exists()
+                for line in Path(dst, f).read_text(errors="ignore").splitlines()
+                if PATCH_RE.search(line)
+            }
+            found |= hits
+            print(f"[{i}/{len(REPOS)}] {repo}: {len(hits)} patches", flush=True)
+    return found
+
+
+def target(key):
+    """`repo:file: modeling_llama.LlamaAttention.forward = f` -> `*Attention.forward` (what transformers symbol is patched)."""
+    code = key.split(": ", 1)[1]
+    m = re.search(r"setattr\(\s*([\w.]+)\s*,\s*['\"](\w+)", code)
+    sym = f"{m[1]}.{m[2]}" if m else code.split("=")[0].strip()
+    sym = re.sub(r"^(transformers\.)?(models\.\w+\.)?(\w*modeling_\w+\.?)?", "", sym) or "<module>"
+    return MODEL_CLS.sub("*", sym)
+
+
+def triage(rows):
+    """LLM groups threads by root cause and says what transformers should do. Returns list of dicts or None."""
+    listing = "\n".join(
+        f"- {r['repo_name']}#{r['number']} ({'PR' if r['is_pr'] else 'issue'}, {r['comments']} comments): "
+        f"{r['t']}\n  {(r['b'] or '')[:300]!r}"
+        for r in rows[:200]
+    )
+    prompt = (
+        "You triage downstream breakage for the huggingface/transformers maintainers. Below are recent issues/PRs from "
+        "libraries that depend on transformers. Group them by root cause. Drop anything not caused by or about "
+        "transformers (env dumps that merely list a version do not count). Reply with ONLY a JSON array, most urgent "
+        'first, items: {"severity": "high|medium|low", "action": "fix in transformers|reply/guide users|watch", '
+        '"component": "e.g. tokenizers, generate, from_pretrained, attention, model:<name>, trainer", '
+        '"summary": "one line, in transformers terms", "refs": ["owner/repo#N", ...]}\n\n' + listing
+    )
+    try:
+        out = InferenceClient().chat_completion([{"role": "user", "content": prompt}], model=MODEL, max_tokens=4000)
+        return json.loads(re.search(r"\[.*\]", out.choices[0].message.content, re.S)[0])
+    except Exception as e:  # LLM is a nicety; raw table still rendered
+        print(f"triage failed: {e}", flush=True)
+        return None
+
+
+def link(ref):
+    repo, n = ref.split("#")
+    return f'<a href="https://github.com/{repo}/issues/{n}">{escape(ref)}</a>'
+
+
+def patch_link(key):
+    repo, rest = key.split(":", 1)
+    f, code = rest.split(": ", 1)
+    return f'<a href="https://github.com/{repo}/blob/HEAD/{f}">{escape(repo)}/{escape(f)}</a>', f"<code>{escape(code)}</code>"
+
+
+def table(head, rows):
+    th = "".join(f"<th>{h}</th>" for h in head)
+    return f"<table><tr>{th}</tr>" + "".join("<tr>" + "".join(f"<td>{c}</td>" for c in r) + "</tr>" for r in rows) + "</table>"
+
+
+def render(rows, groups, now, new, gone):
+    sev = {"high": "🔴", "medium": "🟠", "low": "⚪"}
+    h = [f"<h1>transformers downstream radar · {date.today()}</h1>",
+         f"<p>{len(REPOS)} repos · last {DAYS} days of GitHub activity · {len(now)} monkey patches tracked</p>"]  # fmt: skip
+
+    h.append("<h2>Act on these</h2>")
+    if groups is not None:
+        groups = sorted(groups, key=lambda g: ["high", "medium", "low"].index(g.get("severity", "low")))
+        h.append(table(["", "action", "component", "what", "threads"], [
+            [sev.get(g.get("severity"), ""), escape(g.get("action", "")), escape(g.get("component", "")),
+             escape(g.get("summary", "")), " ".join(link(r) for r in g.get("refs", []) if "#" in r)]
+            for g in groups if g.get("action") != "watch"
+        ]))  # fmt: skip
+    h.append(f"<details><summary>All {len(rows)} matching threads</summary>")
+    h.append(table(["thread", "comments", "title"], [
+        [link(f"{r['repo_name']}#{r['number']}"), r["comments"], escape(r["t"] or "")] for r in rows
+    ]))  # fmt: skip
+    h.append("</details>")
+
+    h.append(f"<h2>New monkey patches since last run ({len(new)})</h2>")
+    h.append("<p>Something changed on their side or ours: check whether a transformers change forced it.</p>")
+    h.append(table(["where", "patch"], [patch_link(k) for k in new]) if new else "<p>None.</p>")
+    if gone:
+        h.append(f"<details><summary>Removed since last run ({len(gone)})</summary>")
+        h.append(table(["where", "patch"], [patch_link(k) for k in gone]) + "</details>")
+
+    by_target = defaultdict(set)
+    for k in now:
+        by_target[target(k)].add(k.split(":", 1)[0])
+    top = sorted(by_target.items(), key=lambda kv: (-len(kv[1]), kv[0]))[:40]
+    h.append("<h2>Most-patched transformers internals</h2>")
+    h.append("<p>Patched by several repos = missing extension point (kernel hub / attention interface / hook).</p>")
+    h.append(table(["symbol", "#repos", "repos"], [
+        [f"<code>{escape(t)}</code>", len(rs), ", ".join(r.split("/")[1] for r in sorted(rs))] for t, rs in top
+    ]))  # fmt: skip
+
+    threads = defaultdict(int)
+    for r in rows:
+        threads[r["repo_name"]] += 1
+    count = lambda keys, repo: sum(k.startswith(repo + ":") for k in keys)
+    h.append("<h2>Per repo</h2>")
+    h.append(table(["repo", "threads", "patches", "new patches"], [
+        [f'<a href="https://github.com/{r}">{r}</a>', threads[r], count(now, r), count(new, r)] for r in REPOS
+    ]))  # fmt: skip
+    h.append(f"<details><summary>All {len(now)} patches</summary>")
+    h.append(table(["where", "patch"], [patch_link(k) for k in sorted(now)]) + "</details>")
+
+    css = (
+        "body{max-width:1200px;margin:auto;padding:1em;font-family:system-ui}table{border-collapse:collapse;width:100%}"
+        "td,th{border-bottom:1px solid #ddd;padding:4px 8px;text-align:left;vertical-align:top}code{font-size:85%}"
+    )
+    return f"<!doctype html><meta charset=utf-8><title>transformers radar</title><style>{css}</style>" + "\n".join(h)
+
+
+def main():
+    rows = issues()
+    print(f"{len(rows)} transformers-related threads in last {DAYS}d", flush=True)
+    groups = triage(rows) if rows else []
+    print(f"triaged into {len(groups) if groups is not None else '?'} groups", flush=True)
+    now = patches()
+    try:
+        before = set(json.load(open(hf_hub_download(SPACE, "patches.json", repo_type="space"))))
+    except Exception:  # first run: nothing to diff against
+        before = now
+    new, gone = sorted(now - before), sorted(before - now)
+    html = render(rows, groups, now, new, gone)
+
+    if DRY:
+        Path("index.html").write_text(html)
+        print("wrote index.html", flush=True)
+        return
+    api = HfApi()
+    api.create_repo(SPACE, repo_type="space", space_sdk="static", exist_ok=True)
+    with tempfile.TemporaryDirectory() as tmp:
+        Path(tmp, "index.html").write_text(html)
+        Path(tmp, "patches.json").write_text(json.dumps(sorted(now), indent=1))
+        Path(tmp, "reports").mkdir()
+        Path(tmp, f"reports/{date.today()}.json").write_text(json.dumps({"threads": rows, "groups": groups}, indent=1))
+        api.upload_folder(folder_path=tmp, repo_id=SPACE, repo_type="space", commit_message=f"radar {date.today()}")
+    print(f"pushed https://huggingface.co/spaces/{SPACE}", flush=True)
+
+
+if __name__ == "__main__":
+    main()