| """Render the triage report to a single static HTML page. |
| |
| We hand-roll markdown → HTML rather than depending on a `markdown` library |
| to keep the Docker image lean and the output deterministic. CSS is inlined. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import datetime |
| import html |
| from collections import Counter, defaultdict |
|
|
|
|
| HEADER = """<!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <title>transformers integration-test failure triage</title> |
| <style> |
| body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, |
| Arial, sans-serif; max-width: 1200px; margin: 24px auto; color: #1f2328; |
| padding: 0 16px; } |
| h1 { border-bottom: 1px solid #d0d7de; padding-bottom: 8px; } |
| h2 { margin-top: 32px; border-bottom: 1px solid #eaeef2; padding-bottom: 4px; } |
| h3 { margin-top: 24px; } |
| table { border-collapse: collapse; width: 100%; font-size: 13px; margin: 8px 0 20px; } |
| th, td { border-bottom: 1px solid #eaeef2; padding: 6px 10px; text-align: left; |
| vertical-align: top; } |
| th { background: #f6f8fa; font-weight: 600; } |
| tr:hover td { background: #f6f8fa; } |
| code, pre { font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; |
| background: #f6f8fa; padding: 1px 4px; border-radius: 4px; font-size: 12px; } |
| pre { padding: 8px 12px; overflow-x: auto; } |
| .pill { display: inline-block; padding: 1px 8px; border-radius: 10px; |
| font-size: 12px; font-weight: 500; } |
| .pill-OOM { background: #fff8c5; color: #6e5500; } |
| .pill-output_mismatch { background: #ddf4ff; color: #054270; } |
| .pill-cuda_runtime { background: #ffdce0; color: #67050a; } |
| .pill-load_error { background: #fbecd0; color: #6e3a00; } |
| .pill-import_or_config { background: #f3e5f5; color: #4a148c; } |
| .pill-other { background: #eaeef2; color: #57606a; } |
| .meta { color: #57606a; font-size: 13px; } |
| details { margin: 4px 0; } |
| summary { cursor: pointer; } |
| .num { text-align: right; font-variant-numeric: tabular-nums; } |
| </style> |
| </head> |
| <body> |
| """ |
|
|
| FOOTER = """ |
| </body> |
| </html> |
| """ |
|
|
|
|
| def _pill(tag: str) -> str: |
| return f'<span class="pill pill-{tag}">{html.escape(tag)}</span>' |
|
|
|
|
| def _trace_excerpt(trace: str, max_chars: int = 220) -> str: |
| """Last non-empty line of trace, trimmed.""" |
| if not trace: |
| return "" |
| for line in reversed(trace.splitlines()): |
| line = line.strip() |
| if line: |
| return (line[: max_chars - 1] + "…") if len(line) > max_chars else line |
| return "" |
|
|
|
|
| def _table(headers: list[str], rows: list[list[str]]) -> str: |
| if not rows: |
| return "<p class='meta'><em>(none)</em></p>" |
| head = "".join(f"<th>{html.escape(h)}</th>" for h in headers) |
| body = "".join( |
| "<tr>" + "".join(f"<td>{cell}</td>" for cell in r) + "</tr>" for r in rows |
| ) |
| return f"<table><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>" |
|
|
|
|
| def _model_breakdown(failures: list[dict]) -> list[list[str]]: |
| """Group failures by model, return rows sorted by total desc.""" |
| by_model: dict[str, list[dict]] = defaultdict(list) |
| for f in failures: |
| by_model[f["model"]].append(f) |
| rows = [] |
| for model in sorted(by_model, key=lambda m: -len(by_model[m])): |
| items = by_model[model] |
| mode_counts = Counter(f["failure_mode"] for f in items) |
| mode_str = " ".join(f"{_pill(m)} {n}" for m, n in mode_counts.most_common()) |
| gpu_str = "/".join(sorted({f["gpu"] for f in items})) |
| rows.append([ |
| html.escape(model), |
| f'<span class="num">{len(items)}</span>', |
| html.escape(gpu_str), |
| mode_str, |
| ]) |
| return rows |
|
|
|
|
| def _cluster_block(cluster: dict, idx: int) -> str: |
| bc = cluster["bad_commit"] |
| pr = cluster.get("pr_number") |
| author = cluster.get("author") or "?" |
| merged_by = cluster.get("merged_by") or "?" |
| rows = [] |
| for f in sorted(cluster["failures"], key=lambda f: (f["model"], f["gpu"], f["test"])): |
| rows.append([ |
| html.escape(f["model"]), |
| html.escape(f["gpu"]), |
| f'<code>{html.escape(f["test"].split("::")[-1])}</code>', |
| _pill(f["failure_mode"]), |
| f'<span class="num">{f["days_seen"]}/7</span>', |
| f'<code>{html.escape(_trace_excerpt(f.get("latest_trace") or f.get("trace") or ""))}</code>', |
| ]) |
| pr_link = ( |
| f'<a href="https://github.com/huggingface/transformers/pull/{pr}">PR #{pr}</a>' |
| if pr else "<em>no PR</em>" |
| ) |
| commit_link = ( |
| f'<a href="https://github.com/huggingface/transformers/commit/{bc}">' |
| f'<code>{html.escape(bc[:12])}</code></a>' |
| ) |
| excerpt = cluster.get("failure_excerpt") or "" |
| excerpt_html = ( |
| f"<details><summary>CI trace from bad commit</summary><pre>{html.escape(excerpt[:2000])}</pre></details>" |
| if excerpt else "" |
| ) |
| return ( |
| f"<h3>Cluster {idx} · {commit_link} · {pr_link} · " |
| f"{html.escape(author)} / merged by {html.escape(merged_by)} · " |
| f"{len(cluster['failures'])} failures</h3>" |
| + _table( |
| ["model", "gpu", "test", "mode", "days", "trace excerpt"], |
| rows, |
| ) |
| + excerpt_html |
| ) |
|
|
|
|
| def _examples_for_unpinned(unpinned: list[dict], k_per_mode: int = 5) -> str: |
| by_mode: dict[str, list[dict]] = defaultdict(list) |
| for f in unpinned: |
| by_mode[f["failure_mode"]].append(f) |
| sections = [] |
| for mode in ("output_mismatch", "OOM", "load_error", "cuda_runtime", "import_or_config", "other"): |
| items = by_mode.get(mode, []) |
| if not items: |
| continue |
| |
| items = sorted(items, key=lambda f: (f.get("latest_seen") or "", f["model"]), reverse=True) |
| sample = items[:k_per_mode] |
| rows = [] |
| for f in sample: |
| rows.append([ |
| html.escape(f["model"]), |
| html.escape(f["gpu"]), |
| f'<code>{html.escape(f["test"].split("::")[-1])}</code>', |
| f'<span class="num">{f["days_seen"]}/7</span>', |
| f'<code>{html.escape(_trace_excerpt(f.get("latest_trace") or f.get("trace") or ""))}</code>', |
| ]) |
| sections.append( |
| f"<h3>{_pill(mode)} {len(items)} unpinned failures — sample of {len(sample)}</h3>" |
| + _table(["model", "gpu", "test", "days", "trace excerpt"], rows) |
| ) |
| return "".join(sections) |
|
|
|
|
| def _regression_day_block(report: dict) -> str: |
| """The killer table: how many tests regressed on each historical day, |
| sorted by size descending. Big buckets = candidate fleet-regressions.""" |
| buckets = report.get("regression_day_buckets") or {} |
| if not buckets: |
| return "" |
| out = ["<h2>Regression-day clustering (historical first-failure)</h2>"] |
| out.append( |
| '<p class="meta">For every persistent failure we walked the daily CI ' |
| "dataset backwards to find the first day it appeared as failing. The " |
| "table below groups failures by that day — large buckets are likely " |
| "<strong>fleet regressions</strong> from a single landed PR. Click a " |
| "date to see the commits merged in the 24h window before it.</p>" |
| ) |
| rows = [] |
| total = sum(buckets.values()) |
| for day, n in list(buckets.items())[:25]: |
| if day == "unknown": |
| link = day |
| else: |
| since = day |
| link = ( |
| f'<a href="https://github.com/huggingface/transformers/commits/main?' |
| f'until={day}T23:59:59Z">{day}</a>' |
| ) |
| pct = f"{(n / total) * 100:0.1f}%" if total else "—" |
| rows.append([link, f'<span class="num">{n}</span>', f'<span class="num">{pct}</span>']) |
| out.append(_table(["first-failure day", "failures", "share"], rows)) |
| return "".join(out) |
|
|
|
|
| def _regression_day_detail_blocks(report: dict, top_k: int = 5) -> str: |
| """For each of the top-k regression days, show which tests broke (table). |
| This gives the reader the "what got hit when" inline.""" |
| buckets = report.get("regression_day_buckets") or {} |
| if not buckets: |
| return "" |
| |
| all_failures = ( |
| [f for c in report["clusters"].values() for f in c["failures"]] |
| + report["flaky"] |
| + report["unpinned"] |
| ) |
| by_day: dict[str, list[dict]] = {} |
| for f in all_failures: |
| by_day.setdefault(f.get("first_failure_day") or "unknown", []).append(f) |
|
|
| out = ["<h2>Top regression days — failure breakdown</h2>"] |
| for day, _ in list(buckets.items())[:top_k]: |
| items = by_day.get(day, []) |
| if not items: |
| continue |
| mode_counts = Counter(f["failure_mode"] for f in items) |
| models = Counter(f["model"] for f in items) |
| out.append(f"<h3>{html.escape(day)} — {len(items)} failures</h3>") |
| out.append('<p class="meta">Failure-mode mix: ' |
| + " ".join(f"{_pill(m)} {n}" for m, n in mode_counts.most_common()) |
| + f' · {len(models)} distinct models touched. ' |
| f'<a href="https://github.com/huggingface/transformers/commits/main?' |
| f'until={day}T23:59:59Z">commit log around {day}</a></p>') |
| |
| rows = [] |
| for model, n in models.most_common(12): |
| sample = next(f for f in items if f["model"] == model) |
| rows.append([ |
| html.escape(model), |
| f'<span class="num">{n}</span>', |
| _pill(sample["failure_mode"]), |
| f'<code>{html.escape(_trace_excerpt(sample.get("latest_trace") or sample.get("trace") or "", 180))}</code>', |
| ]) |
| if len(models) > 12: |
| rows.append([ |
| f"<em>… and {len(models) - 12} more models</em>", "", "", "" |
| ]) |
| out.append(_table(["model", "failures", "sample mode", "sample trace excerpt"], rows)) |
|
|
| |
| |
| all_rows = [] |
| for f in sorted( |
| items, |
| key=lambda f: ( |
| f.get("failure_mode") or "", |
| f.get("model") or "", |
| f.get("gpu") or "", |
| f.get("test") or "", |
| ), |
| ): |
| all_rows.append([ |
| html.escape(f["model"]), |
| html.escape(f["gpu"]), |
| f'<code>{html.escape(f["test"].split("::")[-1])}</code>', |
| _pill(f["failure_mode"]), |
| f'<span class="num">{f["days_seen"]}/7</span>', |
| f'<code>{html.escape(_trace_excerpt(f.get("latest_trace") or f.get("trace") or ""))}</code>', |
| ]) |
| out.append( |
| f"<details><summary>Show all {len(items)} failures in this bucket</summary>" |
| + _table( |
| ["model", "gpu", "test", "failure_mode", "days_seen", "trace excerpt"], |
| all_rows, |
| ) |
| + "</details>" |
| ) |
| return "".join(out) |
|
|
|
|
| def render(report: dict, *, generated_at: datetime.datetime, dates_window: list[str]) -> str: |
| t = report["totals"] |
| out = [HEADER] |
| out.append(f"<h1>transformers · integration-test failure triage</h1>") |
| out.append( |
| f'<p class="meta">Generated <code>{generated_at.isoformat(timespec="seconds")}Z</code> · ' |
| f"window <code>{dates_window[0]}</code> → <code>{dates_window[-1]}</code> " |
| f"({len(dates_window)} daily runs, ≥5/7 intersection)</p>" |
| ) |
|
|
| |
| out.append("<h2>TL;DR</h2><ul>") |
| out.append(f"<li><strong>{t['total']}</strong> persistent integration-test failures (≥5/7 days)</li>") |
| out.append( |
| f"<li><strong>{t['in_clusters']}</strong> attributed to <strong>{t['clusters']}</strong> " |
| f"distinct bad commits (CI bisect)</li>" |
| ) |
| out.append(f"<li><strong>{t['flaky']}</strong> tagged flaky by CI</li>") |
| out.append( |
| f"<li><strong>{t['unpinned']}</strong> unpinned — CI bisect didn't converge " |
| "(usually because the regression predates CI's 7-day bisect window)</li>" |
| ) |
| buckets = report.get("regression_day_buckets") or {} |
| if buckets: |
| top_day, top_n = next(iter(buckets.items())) |
| out.append( |
| f"<li>Historical sweep finds <strong>{top_n}</strong> failures " |
| f"({(top_n / t['total']) * 100:0.0f}% of the total) first appeared on " |
| f"<code>{html.escape(top_day)}</code> — likely a single fleet-regression PR.</li>" |
| ) |
| out.append("</ul>") |
|
|
| out.append(_regression_day_block(report)) |
| out.append(_regression_day_detail_blocks(report, top_k=5)) |
|
|
| |
| if report["unpinned"]: |
| modes = Counter(f["failure_mode"] for f in report["unpinned"]) |
| out.append("<h2>Unpinned failure modes</h2>") |
| out.append(_table( |
| ["mode", "count"], |
| [[_pill(m), f'<span class="num">{n}</span>'] for m, n in modes.most_common()], |
| )) |
|
|
| |
| out.append("<h2>Per-model breakdown (all failures)</h2>") |
| all_failures = ( |
| [f for c in report["clusters"].values() for f in c["failures"]] |
| + report["flaky"] |
| + report["unpinned"] |
| ) |
| out.append(_table( |
| ["model", "failures", "gpu", "mode mix"], |
| _model_breakdown(all_failures), |
| )) |
|
|
| |
| out.append("<h2>Pinned clusters (CI bisect)</h2>") |
| if report["clusters"]: |
| for i, (_bc, c) in enumerate(report["clusters"].items(), start=1): |
| out.append(_cluster_block(c, i)) |
| else: |
| out.append("<p class='meta'><em>(none)</em></p>") |
|
|
| |
| out.append("<h2>Flaky (CI flagged)</h2>") |
| if report["flaky"]: |
| rows = [] |
| for f in report["flaky"]: |
| rows.append([ |
| html.escape(f["model"]), |
| html.escape(f["gpu"]), |
| f'<code>{html.escape(f["test"].split("::")[-1])}</code>', |
| _pill(f["failure_mode"]), |
| f'<span class="num">{f["days_seen"]}/7</span>', |
| ]) |
| out.append(_table(["model", "gpu", "test", "mode", "days"], rows)) |
| else: |
| out.append("<p class='meta'><em>(none)</em></p>") |
|
|
| |
| if report["unpinned"]: |
| out.append("<h2>Unpinned — samples per mode</h2>") |
| out.append( |
| "<p class='meta'>These failures persisted across the window but CI couldn't" |
| " attribute a bad commit. They likely regressed before the 7-day bisect window." |
| " Showing the most-recently-seen samples per failure mode.</p>" |
| ) |
| out.append(_examples_for_unpinned(report["unpinned"])) |
|
|
| out.append(FOOTER) |
| return "".join(out) |
|
|
|
|
| if __name__ == "__main__": |
| import sys |
|
|
| from classify import classify |
| from cluster import cluster_failures |
| from fetch import fetch_last_n |
| from filter import per_day_integration_failures, intersect_across_days |
|
|
| cache = sys.argv[1] if len(sys.argv) > 1 else None |
| out_path = sys.argv[2] if len(sys.argv) > 2 else "/tmp/integration_failures.html" |
|
|
| daily = fetch_last_n(7, cache_dir=cache) |
| per_day = per_day_integration_failures(daily) |
| kept = intersect_across_days(per_day, min_days=5) |
| latest = max(daily.keys()) |
| nf = daily[latest].get("new_failures") |
| report = cluster_failures(kept, nf, classify) |
|
|
| html_out = render( |
| report, |
| generated_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None), |
| dates_window=sorted(daily.keys()), |
| ) |
| with open(out_path, "w") as f: |
| f.write(html_out) |
| print(f"wrote {out_path} ({len(html_out)} bytes)") |
|
|
