"""Render the triage report to a single static HTML page.
We hand-roll markdown → HTML rather than depending on a `markdown` library
to keep the Docker image lean and the output deterministic. CSS is inlined.
"""
from __future__ import annotations
import datetime
import html
from collections import Counter, defaultdict
HEADER = """
transformers integration-test failure triage
"""
FOOTER = """
"""
def _pill(tag: str) -> str:
return f'{html.escape(tag)}'
def _trace_excerpt(trace: str, max_chars: int = 220) -> str:
"""Last non-empty line of trace, trimmed."""
if not trace:
return ""
for line in reversed(trace.splitlines()):
line = line.strip()
if line:
return (line[: max_chars - 1] + "…") if len(line) > max_chars else line
return ""
def _table(headers: list[str], rows: list[list[str]]) -> str:
if not rows:
return "(none)
"
head = "".join(f"{html.escape(h)} | " for h in headers)
body = "".join(
"" + "".join(f"| {cell} | " for cell in r) + "
" for r in rows
)
return f""
def _model_breakdown(failures: list[dict]) -> list[list[str]]:
"""Group failures by model, return rows sorted by total desc."""
by_model: dict[str, list[dict]] = defaultdict(list)
for f in failures:
by_model[f["model"]].append(f)
rows = []
for model in sorted(by_model, key=lambda m: -len(by_model[m])):
items = by_model[model]
mode_counts = Counter(f["failure_mode"] for f in items)
mode_str = " ".join(f"{_pill(m)} {n}" for m, n in mode_counts.most_common())
gpu_str = "/".join(sorted({f["gpu"] for f in items}))
rows.append([
html.escape(model),
f'{len(items)}',
html.escape(gpu_str),
mode_str,
])
return rows
def _cluster_block(cluster: dict, idx: int) -> str:
bc = cluster["bad_commit"]
pr = cluster.get("pr_number")
author = cluster.get("author") or "?"
merged_by = cluster.get("merged_by") or "?"
rows = []
for f in sorted(cluster["failures"], key=lambda f: (f["model"], f["gpu"], f["test"])):
rows.append([
html.escape(f["model"]),
html.escape(f["gpu"]),
f'{html.escape(f["test"].split("::")[-1])}',
_pill(f["failure_mode"]),
f'{f["days_seen"]}/7',
f'{html.escape(_trace_excerpt(f.get("latest_trace") or f.get("trace") or ""))}',
])
pr_link = (
f'PR #{pr}'
if pr else "no PR"
)
commit_link = (
f''
f'{html.escape(bc[:12])}'
)
excerpt = cluster.get("failure_excerpt") or ""
excerpt_html = (
f"CI trace from bad commit
{html.escape(excerpt[:2000])}Cluster {idx} · {commit_link} · {pr_link} · "
f"{html.escape(author)} / merged by {html.escape(merged_by)} · "
f"{len(cluster['failures'])} failures
"
+ _table(
["model", "gpu", "test", "mode", "days", "trace excerpt"],
rows,
)
+ excerpt_html
)
def _examples_for_unpinned(unpinned: list[dict], k_per_mode: int = 5) -> str:
by_mode: dict[str, list[dict]] = defaultdict(list)
for f in unpinned:
by_mode[f["failure_mode"]].append(f)
sections = []
for mode in ("output_mismatch", "OOM", "load_error", "cuda_runtime", "import_or_config", "other"):
items = by_mode.get(mode, [])
if not items:
continue
# Sort: most-recent first by latest_seen, then model.
items = sorted(items, key=lambda f: (f.get("latest_seen") or "", f["model"]), reverse=True)
sample = items[:k_per_mode]
rows = []
for f in sample:
rows.append([
html.escape(f["model"]),
html.escape(f["gpu"]),
f'{html.escape(f["test"].split("::")[-1])}',
f'{f["days_seen"]}/7',
f'{html.escape(_trace_excerpt(f.get("latest_trace") or f.get("trace") or ""))}',
])
sections.append(
f"{_pill(mode)} {len(items)} unpinned failures — sample of {len(sample)}
"
+ _table(["model", "gpu", "test", "days", "trace excerpt"], rows)
)
return "".join(sections)
def _regression_day_block(report: dict) -> str:
"""The killer table: how many tests regressed on each historical day,
sorted by size descending. Big buckets = candidate fleet-regressions."""
buckets = report.get("regression_day_buckets") or {}
if not buckets:
return ""
out = ["Regression-day clustering (historical first-failure)
"]
out.append(
'For every persistent failure we walked the daily CI '
"dataset backwards to find the first day it appeared as failing. The "
"table below groups failures by that day — large buckets are likely "
"fleet regressions from a single landed PR. Click a "
"date to see the commits merged in the 24h window before it.
"
)
rows = []
total = sum(buckets.values())
for day, n in list(buckets.items())[:25]:
if day == "unknown":
link = day
else:
since = day # before that day's CI run
link = (
f'{day}'
)
pct = f"{(n / total) * 100:0.1f}%" if total else "—"
rows.append([link, f'{n}', f'{pct}'])
out.append(_table(["first-failure day", "failures", "share"], rows))
return "".join(out)
def _regression_day_detail_blocks(report: dict, top_k: int = 5) -> str:
"""For each of the top-k regression days, show which tests broke (table).
This gives the reader the "what got hit when" inline."""
buckets = report.get("regression_day_buckets") or {}
if not buckets:
return ""
# All failures, with first_failure_day attached
all_failures = (
[f for c in report["clusters"].values() for f in c["failures"]]
+ report["flaky"]
+ report["unpinned"]
)
by_day: dict[str, list[dict]] = {}
for f in all_failures:
by_day.setdefault(f.get("first_failure_day") or "unknown", []).append(f)
out = ["Top regression days — failure breakdown
"]
for day, _ in list(buckets.items())[:top_k]:
items = by_day.get(day, [])
if not items:
continue
mode_counts = Counter(f["failure_mode"] for f in items)
models = Counter(f["model"] for f in items)
out.append(f"{html.escape(day)} — {len(items)} failures
")
out.append('Failure-mode mix: '
+ " ".join(f"{_pill(m)} {n}" for m, n in mode_counts.most_common())
+ f' · {len(models)} distinct models touched. '
f'commit log around {day}
')
# Top 12 affected models
rows = []
for model, n in models.most_common(12):
sample = next(f for f in items if f["model"] == model)
rows.append([
html.escape(model),
f'{n}',
_pill(sample["failure_mode"]),
f'{html.escape(_trace_excerpt(sample.get("latest_trace") or sample.get("trace") or "", 180))}',
])
if len(models) > 12:
rows.append([
f"… and {len(models) - 12} more models", "", "", ""
])
out.append(_table(["model", "failures", "sample mode", "sample trace excerpt"], rows))
# Full failure list (collapsed). Sort by (failure_mode, model, gpu, test)
# so visually similar failures cluster.
all_rows = []
for f in sorted(
items,
key=lambda f: (
f.get("failure_mode") or "",
f.get("model") or "",
f.get("gpu") or "",
f.get("test") or "",
),
):
all_rows.append([
html.escape(f["model"]),
html.escape(f["gpu"]),
f'{html.escape(f["test"].split("::")[-1])}',
_pill(f["failure_mode"]),
f'{f["days_seen"]}/7',
f'{html.escape(_trace_excerpt(f.get("latest_trace") or f.get("trace") or ""))}',
])
out.append(
f"Show all {len(items)} failures in this bucket
"
+ _table(
["model", "gpu", "test", "failure_mode", "days_seen", "trace excerpt"],
all_rows,
)
+ "transformers · integration-test failure triage
")
out.append(
f'Generated {generated_at.isoformat(timespec="seconds")}Z · '
f"window {dates_window[0]} → {dates_window[-1]} "
f"({len(dates_window)} daily runs, ≥5/7 intersection)
"
)
# TL;DR
out.append("TL;DR
")
out.append(f"- {t['total']} persistent integration-test failures (≥5/7 days)
")
out.append(
f"- {t['in_clusters']} attributed to {t['clusters']} "
f"distinct bad commits (CI bisect)
"
)
out.append(f"- {t['flaky']} tagged flaky by CI
")
out.append(
f"- {t['unpinned']} unpinned — CI bisect didn't converge "
"(usually because the regression predates CI's 7-day bisect window)
"
)
buckets = report.get("regression_day_buckets") or {}
if buckets:
top_day, top_n = next(iter(buckets.items()))
out.append(
f"- Historical sweep finds {top_n} failures "
f"({(top_n / t['total']) * 100:0.0f}% of the total) first appeared on "
f"
{html.escape(top_day)} — likely a single fleet-regression PR. "
)
out.append("
")
out.append(_regression_day_block(report))
out.append(_regression_day_detail_blocks(report, top_k=5))
# Failure-mode breakdown for unpinned
if report["unpinned"]:
modes = Counter(f["failure_mode"] for f in report["unpinned"])
out.append("Unpinned failure modes
")
out.append(_table(
["mode", "count"],
[[_pill(m), f'{n}'] for m, n in modes.most_common()],
))
# Per-model
out.append("Per-model breakdown (all failures)
")
all_failures = (
[f for c in report["clusters"].values() for f in c["failures"]]
+ report["flaky"]
+ report["unpinned"]
)
out.append(_table(
["model", "failures", "gpu", "mode mix"],
_model_breakdown(all_failures),
))
# Clusters
out.append("Pinned clusters (CI bisect)
")
if report["clusters"]:
for i, (_bc, c) in enumerate(report["clusters"].items(), start=1):
out.append(_cluster_block(c, i))
else:
out.append("(none)
")
# Flaky
out.append("Flaky (CI flagged)
")
if report["flaky"]:
rows = []
for f in report["flaky"]:
rows.append([
html.escape(f["model"]),
html.escape(f["gpu"]),
f'{html.escape(f["test"].split("::")[-1])}',
_pill(f["failure_mode"]),
f'{f["days_seen"]}/7',
])
out.append(_table(["model", "gpu", "test", "mode", "days"], rows))
else:
out.append("(none)
")
# Unpinned examples
if report["unpinned"]:
out.append("Unpinned — samples per mode
")
out.append(
"These failures persisted across the window but CI couldn't"
" attribute a bad commit. They likely regressed before the 7-day bisect window."
" Showing the most-recently-seen samples per failure mode.
"
)
out.append(_examples_for_unpinned(report["unpinned"]))
out.append(FOOTER)
return "".join(out)
if __name__ == "__main__":
import sys
from classify import classify
from cluster import cluster_failures
from fetch import fetch_last_n
from filter import per_day_integration_failures, intersect_across_days
cache = sys.argv[1] if len(sys.argv) > 1 else None
out_path = sys.argv[2] if len(sys.argv) > 2 else "/tmp/integration_failures.html"
daily = fetch_last_n(7, cache_dir=cache)
per_day = per_day_integration_failures(daily)
kept = intersect_across_days(per_day, min_days=5)
latest = max(daily.keys())
nf = daily[latest].get("new_failures")
report = cluster_failures(kept, nf, classify)
html_out = render(
report,
generated_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None),
dates_window=sorted(daily.keys()),
)
with open(out_path, "w") as f:
f.write(html_out)
print(f"wrote {out_path} ({len(html_out)} bytes)")