Blame for tracker.py

# /// script
# dependencies = ["huggingface_hub>=0.35", "requests"]
# ///
"""Daily radar: transformers-related issues + monkey patches in downstream repos -> static Space.

Run:      HF_TOKEN=... GITHUB_TOKEN=... uv run tracker.py            (DRY=1 to write index.html locally instead of pushing)
Schedule: hf jobs scheduled uv run daily https://huggingface.co/spaces/$SPACE/resolve/main/tracker.py --secrets HF_TOKEN  (GITHUB_TOKEN optional: 3x faster search)
"""

import json
import os
import re
import subprocess
import tempfile
import time
from collections import defaultdict
from datetime import date, timedelta
from html import escape
from pathlib import Path

import requests
from huggingface_hub import HfApi, InferenceClient, hf_hub_download

# Heavy transformers users, ordered by GitHub stars (2026-11). Picked from stars >5k or pypi >300k/month.
REPOS = [
    "ggml-org/llama.cpp", "vllm-project/vllm", "unslothai/unsloth", "hiyouga/LlamaFactory",
    "deepspeedai/DeepSpeed", "hpcaitech/ColossalAI", "sgl-project/sglang", "huggingface/diffusers",
    "huggingface/lerobot", "huggingface/open-r1", "m-bain/whisperX", "verl-project/verl", "huggingface/peft",
    "huggingface/trl", "kvcache-ai/ktransformers", "huggingface/sentence-transformers", "NVIDIA-NeMo/Automodel",
    "modelscope/ms-swift", "dottxt-ai/outlines", "NVIDIA/TensorRT-LLM", "flairNLP/flair",
    "EleutherAI/lm-evaluation-harness", "axolotl-ai-cloud/axolotl", "FlagOpen/FlagEmbedding",
    "OpenRLHF/OpenRLHF", "huggingface/accelerate", "THUDM/slime", "bitsandbytes-foundation/bitsandbytes",
    "ml-explore/mlx-lm", "arcee-ai/mergekit", "linkedin/Liger-Kernel", "Blaizzy/mlx-vlm",
    "EvolvingLMMs-Lab/lmms-eval", "urchade/GLiNER", "vllm-project/llm-compressor", "allenai/open-instruct",
    "huggingface/optimum", "embeddings-benchmark/mteb", "alibaba/ROLL", "PrimeIntellect-ai/prime-rl",
    "intel/auto-round", "ModelCloud/GPTQModel", "huggingface/optimum-quanto", "unslothai/unsloth-zoo",
]  # fmt: skip
SPACE = os.environ.get("SPACE", "ModelAdders/transformers-downstream-tracker")
MODEL = os.environ.get("MODEL", "Qwen/Qwen3-235B-A22B-Instruct-2507")
DAYS = int(os.environ.get("DAYS", 2))  # ponytail: overlap between daily runs is fine
DRY = os.environ.get("DRY") == "1"

# ponytail: regex heuristic, catches `transformers.x.y = ...`, `modeling_llama.Foo.forward = ...`, setattr on
# transformers modules, and `LlamaAttention.forward = ...`. Misses aliased imports; AST pass if that matters.
PATCH_RE = re.compile(
    r"^\s*(transformers(\.\w+)+|modeling_\w+(\.\w+)+"
    r"|[A-Z]\w*(Attention|MLP|Model|ForCausalLM|DecoderLayer|RMSNorm|PreTrainedModel|Embedding)\w*\.\w+)\s*=[^=]"
    r"|setattr\(\s*(transformers|modeling_)"
)
MODEL_CLS = re.compile(
    r"^(?!PreTrainedModel)[A-Z]\w*?(?=(ForCausalLM|ForConditionalGeneration|SdpaAttention|FlashAttention2|Attention"
    r"|MLP|DecoderLayer|RMSNorm|RotaryEmbedding|SparseMoeBlock|Experts|Model)\b)"
)


def issues():
    # GitHub search, not GH Archive: the public events feed is heavily sampled (~40 of ~650 daily vllm events).
    since = (date.today() - timedelta(days=DAYS)).isoformat()
    headers = {"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"} if "GITHUB_TOKEN" in os.environ else {}
    rows = []
    for i, repo in enumerate(REPOS, 1):
        q = f"repo:{repo} transformers in:title,body,comments updated:>={since}"
        r = requests.get(
            "https://api.github.com/search/issues", params={"q": q, "per_page": 100, "sort": "updated"},
            headers=headers, timeout=60,
        )  # fmt: skip
        if not r.ok:  # renamed/private repo shouldn't kill the whole report
            print(f"[{i}/{len(REPOS)}] {repo}: search failed {r.status_code}", flush=True)
            continue
        # ponytail: first 100 most recently updated only; vllm hits the cap, paginate if the tail matters
        items = r.json()["items"]
        rows += [
            {"repo_name": repo, "number": it["number"], "t": it["title"], "b": it["body"], "comments": it["comments"],
             "is_pr": "pull_request" in it}
            for it in items
        ]  # fmt: skip
        print(f"[{i}/{len(REPOS)}] {repo}: {len(items)} threads", flush=True)
        time.sleep(2.1 if headers else 6.5)  # search API: 30 req/min authenticated, 10 anonymous
    return sorted(rows, key=lambda r: -r["comments"])


def patches():
    found = set()
    for i, repo in enumerate(REPOS, 1):
        with tempfile.TemporaryDirectory() as dst:  # one at a time, keeps job disk small
            clone = ["git", "clone", "-q", "--depth", "1", "--filter=blob:limit=1m", f"https://github.com/{repo}", dst]
            if subprocess.run(clone).returncode:
                print(f"[{i}/{len(REPOS)}] {repo}: clone failed", flush=True)
                continue
            files = subprocess.run(["git", "ls-files", "*.py"], cwd=dst, capture_output=True, text=True).stdout
            # key on file + stripped line so line-number churn doesn't count as a new patch
            hits = {
                f"{repo}:{f}: {line.strip()}"
                for f in files.split()
                if "test" not in f and Path(dst, f).exists()
                for line in Path(dst, f).read_text(errors="ignore").splitlines()
                if PATCH_RE.search(line)
            }
            found |= hits
            print(f"[{i}/{len(REPOS)}] {repo}: {len(hits)} patches", flush=True)
    return found


def target(key):
    """`repo:file: modeling_llama.LlamaAttention.forward = f` -> `*Attention.forward` (what transformers symbol is patched)."""
    code = key.split(": ", 1)[1]
    m = re.search(r"setattr\(\s*([\w.]+)\s*,\s*['\"](\w+)", code)
    sym = f"{m[1]}.{m[2]}" if m else code.split("=")[0].strip()
    sym = re.sub(r"^(transformers\.)?(models\.\w+\.)?(\w*modeling_\w+\.?)?", "", sym) or "<module>"
    return MODEL_CLS.sub("*", sym)


def triage(rows):
    """LLM groups threads by root cause and says what transformers should do. Returns list of dicts or None."""
    listing = "\n".join(
        f"- {r['repo_name']}#{r['number']} ({'PR' if r['is_pr'] else 'issue'}, {r['comments']} comments): "
        f"{r['t']}\n  {(r['b'] or '')[:300]!r}"
        for r in rows[:200]
    )
    prompt = (
        "You triage downstream breakage for the huggingface/transformers maintainers. Below are recent issues/PRs from "
        "libraries that depend on transformers. Group them by root cause. Drop anything not caused by or about "
        "transformers (env dumps that merely list a version do not count). Reply with ONLY a JSON array, most urgent "
        'first, items: {"severity": "high|medium|low", "action": "fix in transformers|reply/guide users|watch", '
        '"component": "e.g. tokenizers, generate, from_pretrained, attention, model:<name>, trainer", '
        '"summary": "one line, in transformers terms", "refs": ["owner/repo#N", ...]}\n\n' + listing
    )
    try:
        out = InferenceClient().chat_completion([{"role": "user", "content": prompt}], model=MODEL, max_tokens=4000)
        return json.loads(re.search(r"\[.*\]", out.choices[0].message.content, re.S)[0])
    except Exception as e:  # LLM is a nicety; raw table still rendered
        print(f"triage failed: {e}", flush=True)
        return None


def link(ref):
    repo, n = ref.split("#")
    return f'<a href="https://github.com/{repo}/issues/{n}">{escape(ref)}</a>'


def patch_link(key):
    repo, rest = key.split(":", 1)
    f, code = rest.split(": ", 1)
    return f'<a href="https://github.com/{repo}/blob/HEAD/{f}">{escape(repo)}/{escape(f)}</a>', f"<code>{escape(code)}</code>"


def table(head, rows):
    th = "".join(f"<th>{h}</th>" for h in head)
    return f"<table><tr>{th}</tr>" + "".join("<tr>" + "".join(f"<td>{c}</td>" for c in r) + "</tr>" for r in rows) + "</table>"


def render(rows, groups, now, new, gone):
    sev = {"high": "🔴", "medium": "🟠", "low": "⚪"}
    h = [f"<h1>transformers downstream radar · {date.today()}</h1>",
         f"<p>{len(REPOS)} repos · last {DAYS} days of GitHub activity · {len(now)} monkey patches tracked</p>"]  # fmt: skip

    h.append("<h2>Act on these</h2>")
    if groups is not None:
        groups = sorted(groups, key=lambda g: ["high", "medium", "low"].index(g.get("severity", "low")))
        h.append(table(["", "action", "component", "what", "threads"], [
            [sev.get(g.get("severity"), ""), escape(g.get("action", "")), escape(g.get("component", "")),
             escape(g.get("summary", "")), " ".join(link(r) for r in g.get("refs", []) if "#" in r)]
            for g in groups if g.get("action") != "watch"
        ]))  # fmt: skip
    h.append(f"<details><summary>All {len(rows)} matching threads</summary>")
    h.append(table(["thread", "comments", "title"], [
        [link(f"{r['repo_name']}#{r['number']}"), r["comments"], escape(r["t"] or "")] for r in rows
    ]))  # fmt: skip
    h.append("</details>")

    h.append(f"<h2>New monkey patches since last run ({len(new)})</h2>")
    h.append("<p>Something changed on their side or ours: check whether a transformers change forced it.</p>")
    h.append(table(["where", "patch"], [patch_link(k) for k in new]) if new else "<p>None.</p>")
    if gone:
        h.append(f"<details><summary>Removed since last run ({len(gone)})</summary>")
        h.append(table(["where", "patch"], [patch_link(k) for k in gone]) + "</details>")

    by_target = defaultdict(set)
    for k in now:
        by_target[target(k)].add(k.split(":", 1)[0])
    top = sorted(by_target.items(), key=lambda kv: (-len(kv[1]), kv[0]))[:40]
    h.append("<h2>Most-patched transformers internals</h2>")
    h.append("<p>Patched by several repos = missing extension point (kernel hub / attention interface / hook).</p>")
    h.append(table(["symbol", "#repos", "repos"], [
        [f"<code>{escape(t)}</code>", len(rs), ", ".join(r.split("/")[1] for r in sorted(rs))] for t, rs in top
    ]))  # fmt: skip

    threads = defaultdict(int)
    for r in rows:
        threads[r["repo_name"]] += 1
    count = lambda keys, repo: sum(k.startswith(repo + ":") for k in keys)
    h.append("<h2>Per repo</h2>")
    h.append(table(["repo", "threads", "patches", "new patches"], [
        [f'<a href="https://github.com/{r}">{r}</a>', threads[r], count(now, r), count(new, r)] for r in REPOS
    ]))  # fmt: skip
    h.append(f"<details><summary>All {len(now)} patches</summary>")
    h.append(table(["where", "patch"], [patch_link(k) for k in sorted(now)]) + "</details>")

    css = (
        "body{max-width:1200px;margin:auto;padding:1em;font-family:system-ui}table{border-collapse:collapse;width:100%}"
        "td,th{border-bottom:1px solid #ddd;padding:4px 8px;text-align:left;vertical-align:top}code{font-size:85%}"
    )
    return f"<!doctype html><meta charset=utf-8><title>transformers radar</title><style>{css}</style>" + "\n".join(h)


def main():
    rows = issues()
    print(f"{len(rows)} transformers-related threads in last {DAYS}d", flush=True)
    groups = triage(rows) if rows else []
    print(f"triaged into {len(groups) if groups is not None else '?'} groups", flush=True)
    now = patches()
    try:
        before = set(json.load(open(hf_hub_download(SPACE, "patches.json", repo_type="space"))))
    except Exception:  # first run: nothing to diff against
        before = now
    new, gone = sorted(now - before), sorted(before - now)
    html = render(rows, groups, now, new, gone)

    if DRY:
        Path("index.html").write_text(html)
        print("wrote index.html", flush=True)
        return
    api = HfApi()
    api.create_repo(SPACE, repo_type="space", space_sdk="static", exist_ok=True)
    with tempfile.TemporaryDirectory() as tmp:
        Path(tmp, "index.html").write_text(html)
        Path(tmp, "patches.json").write_text(json.dumps(sorted(now), indent=1))
        Path(tmp, "reports").mkdir()
        Path(tmp, f"reports/{date.today()}.json").write_text(json.dumps({"threads": rows, "groups": groups}, indent=1))
        api.upload_folder(folder_path=tmp, repo_id=SPACE, repo_type="space", commit_message=f"radar {date.today()}")
    print(f"pushed https://huggingface.co/spaces/{SPACE}", flush=True)


if __name__ == "__main__":
    main()