scripts/build_report_2026_06_10.py

"""Build the 2026-06-10 campaign report PDF:
  report/QUANT_REPORT_2026-06-10.md  (body)
  + analysis figures (outputs/quant_report_assets_0610/*.png, generated here)
  + APPENDIX: every new-axis cell's 8-probe teacher|quant montage (20 cells)
-> report/QUANT_REPORT_2026-06-10.pdf

Run: PYTHONPATH=. python3 scripts/build_report_2026_06_10.py
"""
import os

import markdown
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from PIL import Image, ImageDraw, ImageFont
from weasyprint import HTML

AST = "outputs/quant_report_assets_0610"
APP = f"{AST}/appendix"
os.makedirs(APP, exist_ok=True)

PROBES = ["storefront text", "mountain lake", "fisherman portrait", "neon street",
          "chalkboard 'FRESH COFFEE'", "breakfast flat-lay (3 eggs/2 bacon)",
          "hand / five fingers", "dewy spiderweb macro"]

# (dir, title, eval-loss, vel-relerr) — every new-axis cell with montages, report order
CELLS = [
    # W4A8 smoothed twin re-evals (baselines)
    ("recheck_r16_plain_refine",  "W4A8 r16 plain+refine α=0.5 (smoothed twin re-eval)", 0.0405, 0.1855),
    ("recheck_r32_plain_refine",  "W4A8 r32 plain+refine α=0.5 (smoothed twin re-eval)", 0.0362, 0.1753),
    ("recheck_r64_plain_refine",  "W4A8 r64 plain+refine α=0.5 (smoothed twin re-eval)", 0.0325, 0.1661),
    # W4A8 SMOOTH=0
    ("abl_c300_r16_plain_refine_nosmooth", "W4A8 r16 plain+refine SMOOTH=0", 0.0348, 0.1719),
    ("abl_c300_r32_plain_refine_nosmooth", "W4A8 r32 plain+refine SMOOTH=0", 0.0331, 0.1675),
    ("abl_c300_r64_plain_refine_nosmooth", "W4A8 r64 plain+refine SMOOTH=0 — ★ W4A8 CHAMPION", 0.0297, 0.1588),
    # W4A8 α sweep
    ("abl_c300_r64_plain_refine_a10", "W4A8 r64 plain+refine α=0.1 (U-shape worst)", 0.0380, 0.1797),
    ("abl_c300_r64_plain_refine_a25", "W4A8 r64 plain+refine α=0.25", 0.0317, 0.1640),
    ("abl_c300_r32_plain_refine_a10", "W4A8 r32 plain+refine α=0.1 (U-shape worst)", 0.0408, 0.1862),
    ("abl_c300_r32_plain_refine_a25", "W4A8 r32 plain+refine α=0.25", 0.0349, 0.1722),
    # W4A4 per-token
    ("abl_c300_r64_w4a4_plain_refine_nosmooth", "W4A4 per-token r64 SMOOTH=0 — catastrophic", 0.5103, 0.6582),
    ("abl_c300_r64_w4a4_plain_refine_a50",  "W4A4 per-token r64 α=0.5 (smoothing flips +24%)", 0.3885, 0.5743),
    ("abl_c300_r64_w4a4_plain_refine_a75",  "W4A4 per-token r64 α=0.75", 0.2819, 0.4892),
    ("abl_c300_r128_w4a4_plain_refine_a50", "W4A4 per-token r128 α=0.5", 0.3060, 0.5097),
    ("abl_c300_r128_w4a4_plain_refine_a75", "W4A4 per-token r128 α=0.75 (best per-token)", 0.2080, 0.4202),
    ("abl_c300_r128_w4a4_plain_refine_a100","W4A4 per-token r128 α=1.0 (curve turns)", 0.2397, 0.4511),
    # W4A4 per-group (the fix)
    ("abl_c300_r64_w4a4g64_nosmooth", "W4A4 per-GROUP g64 r64 SMOOTH=0 — the fix (−85%)", 0.0742, 0.2510),
    ("abl_c300_r64_w4a4g64_a50",  "W4A4 per-GROUP g64 r64 α=0.5", 0.0759, 0.2538),
    ("abl_c300_r128_w4a4g64_nosmooth", "W4A4 per-GROUP g64 r128 SMOOTH=0 — ★ W4A4 CHAMPION", 0.0610, 0.2276),
    ("abl_c300_r128_w4a4g64_a50", "W4A4 per-GROUP g64 r128 α=0.5", 0.0620, 0.2293),
]


def _font(sz):
    p = "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"
    try:
        return ImageFont.truetype(p, sz)
    except Exception:
        return ImageFont.load_default()


def cell_grid_montage(cell_dir, out_path, ncols=2, cell_w=560, label_h=26):
    pairs = []
    for i in range(8):
        p = f"outputs/{cell_dir}/eval/cmp_{i}.png"
        if os.path.exists(p):
            pairs.append((PROBES[i], Image.open(p)))
    if not pairs:
        return False
    scaled = [(lab, im.resize((cell_w, int(im.height * cell_w / im.width)))) for lab, im in pairs]
    row_h = max(s.height for _, s in scaled) + label_h
    nrows = (len(scaled) + ncols - 1) // ncols
    W = ncols * cell_w + (ncols + 1) * 8
    H = nrows * row_h + (nrows + 1) * 8
    canvas = Image.new("RGB", (W, H), "white")
    d = ImageDraw.Draw(canvas)
    f = _font(15)
    for idx, (lab, s) in enumerate(scaled):
        r, c = divmod(idx, ncols)
        x, y = 8 + c * (cell_w + 8), 8 + r * (row_h + 8)
        d.text((x + 2, y + 4), f"{idx}. {lab}   (teacher | quant)", fill="black", font=f)
        canvas.paste(s, (x, y + label_h))
    canvas.save(out_path)
    return True


# ----------------------------------------------------------------------------- figures
def fig_alpha_w4a8():
    al = [0.0, 0.1, 0.25, 0.5]
    r64 = [0.0297, 0.0380, 0.0317, 0.0325]
    r32 = [0.0331, 0.0408, 0.0349, 0.0362]
    fig, ax = plt.subplots(figsize=(7, 4))
    ax.plot(al, r64, "o-", label="rank 64", lw=2)
    ax.plot(al, r32, "s-", label="rank 32", lw=2)
    for x, y in zip(al, r64):
        ax.annotate(f"{y:.4f}", (x, y), textcoords="offset points", xytext=(0, 8), fontsize=8, ha="center")
    ax.set_xticks(al, ["off\n(s=1)", "0.1", "0.25", "0.5"])
    ax.set_xlabel("SmoothQuant α"); ax.set_ylabel("eval velocity-loss (lower = closer to teacher)")
    ax.set_title("W4A8: the α dial has NO good setting — replicated U-shape, off wins")
    ax.legend(); ax.grid(alpha=0.3)
    fig.tight_layout(); fig.savefig(f"{AST}/alpha_u_w4a8.png", dpi=140); plt.close(fig)


def fig_w4a4_arc():
    labels = ["per-token\nr64 ns", "per-token\nr64 α=.5", "per-token\nr64 α=.75",
              "per-token\nr128 α=.75", "per-group g64\nr64 ns", "per-group g64\nr128 ns"]
    vals = [0.5103, 0.3885, 0.2819, 0.2080, 0.0742, 0.0610]
    colors = ["#c0392b", "#d35400", "#e67e22", "#f39c12", "#27ae60", "#16a085"]
    fig, ax = plt.subplots(figsize=(8, 4.2))
    bars = ax.bar(labels, vals, color=colors)
    for b, v in zip(bars, vals):
        ax.text(b.get_x() + b.get_width() / 2, v + 0.006, f"{v:.4f}", ha="center", fontsize=9)
    ax.axhline(0.0297, color="#2c3e50", ls="--", lw=1.5)
    ax.text(0.02, 0.0297 + 0.006, "W4A8 champion 0.0297", fontsize=9, color="#2c3e50")
    ax.set_ylabel("eval velocity-loss"); ax.grid(axis="y", alpha=0.3)
    ax.set_title("W4A4 arc: smoothing+rank patch per-token poorly; per-group granularity FIXES it (−85%)")
    fig.tight_layout(); fig.savefig(f"{AST}/w4a4_arc.png", dpi=140); plt.close(fig)


def fig_group_ladder():
    gs = ["per-token\n(3072)", "256", "128", "64", "32", "16"]
    synth = [0.602, 0.233, 0.172, 0.129, 0.099, 0.077]
    real = {  # from scripts/test_act_group_quant.py --real (2026-06-10)
        "S0 qkv_mlp (real)": [0.469, 0.251, 0.209, 0.169, 0.137, 0.109],
        "S10 to_out (real)": [0.443, 0.211, 0.179, 0.152, 0.128, 0.104],
        "S19 qkv_mlp (real)": [0.366, 0.193, 0.160, 0.134, 0.113, 0.095],
    }
    fig, ax = plt.subplots(figsize=(7.5, 4.2))
    x = range(len(gs))
    ax.plot(x, synth, "o--", label="synthetic (3 planted outlier ch.)", color="#7f8c8d", lw=2)
    for (lab, v), c in zip(real.items(), ["#2980b9", "#27ae60", "#8e44ad"]):
        ax.plot(x, v, "o-", label=lab, color=c, lw=2)
    ax.set_xticks(list(x), gs)
    ax.axvspan(2.6, 3.4, color="#27ae60", alpha=0.10)
    ax.text(3, 0.55, "g64 = paper INT4 spec\n(used in our W4A4 fix)", ha="center", fontsize=8, color="#1e8449")
    ax.axvspan(4.6, 5.4, color="#16a085", alpha=0.10)
    ax.text(5, 0.55, "g16 = NVFP4\n(Blackwell-native)", ha="center", fontsize=8, color="#117a65")
    ax.set_xlabel("activation quant group size (channels per dynamic scale)")
    ax.set_ylabel("4-bit act quant rel-err  ‖Q(x)−x‖/‖x‖")
    ax.set_title("Granularity ladder (scripts/test_act_group_quant.py): ~−20-25% error per halving")
    ax.legend(fontsize=8); ax.grid(alpha=0.3)
    fig.tight_layout(); fig.savefig(f"{AST}/group_ladder.png", dpi=140); plt.close(fig)


def fig_smooth0_w4a8():
    ranks = ["16", "32", "64"]
    sm = [0.0405, 0.0362, 0.0325]
    ns = [0.0348, 0.0331, 0.0297]
    x = range(3)
    fig, ax = plt.subplots(figsize=(6.5, 4))
    w = 0.36
    b1 = ax.bar([i - w / 2 for i in x], sm, w, label="smoothed α=0.5 (twin re-eval)", color="#c0392b")
    b2 = ax.bar([i + w / 2 for i in x], ns, w, label="SMOOTH=0", color="#27ae60")
    for b, v in list(zip(b1, sm)) + list(zip(b2, ns)):
        ax.text(b.get_x() + b.get_width() / 2, v + 0.0006, f"{v:.4f}", ha="center", fontsize=8)
    deltas = ["−14.1%", "−8.6%", "−8.6%"]
    for i, dlt in enumerate(deltas):
        ax.text(i, max(sm[i], ns[i]) + 0.0035, dlt, ha="center", fontsize=10, weight="bold")
    ax.set_xticks(list(x), [f"rank {r}" for r in ranks])
    ax.set_ylabel("eval velocity-loss"); ax.legend(); ax.grid(axis="y", alpha=0.3)
    ax.set_title("W4A8: disabling SmoothQuant wins at every rank")
    fig.tight_layout(); fig.savefig(f"{AST}/smooth0_w4a8.png", dpi=140); plt.close(fig)


print("generating figures...")
fig_smooth0_w4a8(); fig_alpha_w4a8(); fig_w4a4_arc(); fig_group_ladder()

print("generating per-cell appendix montages...")
cell_imgs = []
for d, title, loss, rel in CELLS:
    op = f"{APP}/{d}.png"
    if cell_grid_montage(d, op):
        cell_imgs.append((op, title, loss, rel))
print(f"  wrote {len(cell_imgs)}/{len(CELLS)} montages")

# --------------------------------------------------------------------- assemble HTML
md_body = open("report/QUANT_REPORT_2026-06-10.md").read()
md_lines = [ln for ln in md_body.splitlines() if not ln.startswith("% ")]
title_html = ('<h1>FLUX.2 [klein] 4B SVDQuant — the SmoothQuant verdict, the W4A4 fix, and the '
              'converged recipe</h1><p><em>2026-06-10 campaign report · new box / new eval axis · '
              '1× RTX PRO 4500 Blackwell 32 GB · companion to QUANT_REPORT.md (2026-05-31, A100 era)'
              '</em></p>')
body_html = markdown.markdown("\n".join(md_lines), extensions=["tables", "fenced_code", "sane_lists"])

FIGS = [
    ("smooth0_w4a8.png", "Figure 1. W4A8 SMOOTH=0 ablation — disabling SmoothQuant wins at every rank (§2)."),
    ("alpha_u_w4a8.png", "Figure 2. W4A8 α sweep — replicated U-shape at both ranks; no α beats off; α=0.1 (the weight-equalizing extreme) is the worst point (§3)."),
    ("w4a4_arc.png", "Figure 3. The W4A4 arc — per-token activations are catastrophic and α/rank only patch them (0.51→0.21); per-group g64 granularity fixes them (0.0610) (§4–5)."),
    ("group_ladder.png", "Figure 4. Activation-quant granularity ladder on synthetic AND real hooked activations — ~−20-25% rel-err per halving of group size; g16 (NVFP4) is the queued next knob (§5, §7)."),
]
figs_html = "<h1>Figures</h1>\n"
for fn, cap in FIGS:
    figs_html += f'<figure class="fig"><img src="{AST}/{fn}"/><figcaption>{cap}</figcaption></figure>\n'

app_html = (f'<h1 class="appendix">Appendix A — Per-cell visual montages ({len(cell_imgs)} new-axis '
            'cells × 8 probes)</h1>\n'
            '<p>Left = teacher (bf16, 4-step), right = quantized student, same prompt and seed. '
            'All cells 300-img calib, plain SVD + refine(3). States deleted post-eval (quota); '
            'losses/configs/montages retained; every cell rebuilds deterministically.</p>\n')
for op, title, loss, rel in cell_imgs:
    app_html += (f'<div class="cellpage"><h3>{title}</h3>'
                 f'<p class="cellmeta">eval_vel_loss = <b>{loss:.4f}</b> · vel rel-err = {rel:.4f} · '
                 f'dir: <code>outputs/{os.path.basename(op)[:-4]}</code></p>'
                 f'<img src="{op}"/></div>\n')

css = open("report/style.css").read()
css += "\n.cellpage { page-break-before: always; } .fig { page-break-inside: avoid; } figcaption { font-size: 9px; color: #555; }"
html = f"<html><head><meta charset='utf-8'><style>{css}</style></head><body>{title_html}{body_html}{figs_html}{app_html}</body></html>"
HTML(string=html, base_url=".").write_pdf("report/QUANT_REPORT_2026-06-10.pdf")
print("DONE -> report/QUANT_REPORT_2026-06-10.pdf")