# -*- coding: utf-8 -*- # tools/report_pretty_0920.py import os, re, csv, math from pathlib import Path import argparse import torch import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt # ---------- 基础解析（沿用你现有 summary 逻辑，稍微更鲁棒） ---------- DATASET_KIND = { "scannet": "seg", "s3dis": "seg", "nuscenes": "seg", "modelnet": "cls", } def guess_dataset(run_dir: Path): name = run_dir.name.lower() for k in DATASET_KIND: if k in name: return k log = run_dir / "train.log" if log.exists(): t = log.read_text(errors="ignore").lower() for k in DATASET_KIND: if k in t: return k return "unknown" def parse_bits_and_mode(run_dir: Path): name = run_dir.name.lower() w = a = None m = re.search(r"w(\d+)a(\d+)", name) if m: w, a = int(m.group(1)), int(m.group(2)) mode = "quant" if m else ("fp32" if ("fp32" in name or "baseline" in name) else "unknown") # 日志兜底 log = run_dir / "train.log" if log.exists(): t = log.read_text(errors="ignore").lower() m1 = re.search(r"quant\d*\.?enable\s*=\s*(true|false)", t) if m1: mode = "quant" if m1.group(1) == "true" else "fp32" mw = re.search(r"quant\d*\.?w_bits\s*=\s*(\d+)", t) ma = re.search(r"quant\d*\.?a_bits\s*=\s*(\d+)", t) if mw: w = int(mw.group(1)) if ma: a = int(ma.group(1)) if mode == "fp32": w = w or 32; a = a or 32 return w or 2, a or 8, mode def parse_metrics(log_path: Path): res={} if not log_path.exists(): return res txt = log_path.read_text(errors="ignore") # mIoU mi=[] for line in txt.splitlines(): l=line.lower() if "miou" in l: mi += [float(x) for x in re.findall(r"[-+]?\d*\.\d+|\d+", l) if 0<=float(x)<=100] if mi: res["mIoU_best"]=max(mi) # Acc / OA acc=[] for key in ["overall acc","oa","accuracy","acc"]: for line in txt.splitlines(): l=line.lower() if key in l: acc += [float(x) for x in re.findall(r"[-+]?\d*\.\d+|\d+", l) if 0<=float(x)<=100] if acc: res["Acc_best"]=max(acc) return res def find_ckpt(run_dir: Path): cands=[] for ext in ("*.pth","*.pt"): cands += list(run_dir.rglob(ext)) scored=[] for p in cands: s=0; n=p.name.lower() if "best" in n: s+=100 if "last" in n or "latest" in n: s+=50 s += int(p.stat().st_mtime) scored.append((s,p)) scored.sort(reverse=True) return scored[0][1] if scored else None def load_state_dict_any(p: Path): try: try: obj = torch.load(p, map_location="cpu", weights_only=True) except TypeError: obj = torch.load(p, map_location="cpu") if isinstance(obj, dict): for k in ["state_dict","model","net","module","ema","model_state","model_ema"]: if k in obj and isinstance(obj[k], dict): return {kk:vv for kk,vv in obj[k].items() if torch.is_tensor(vv)} if all(isinstance(k, str) for k in obj.keys()): return {k:v for k,v in obj.items() if torch.is_tensor(v)} return {} except Exception: return {} def avg_weight_bits(sd, w_bits=2, exclude=("cls_head","embedding.stem","stem","head"), excl_norm_bias=True, force_fp32=False): total=qcnt=fpcnt=0 for name, t in sd.items(): if not torch.is_tensor(t): continue n=t.numel(); lname=name.lower() excl = any(h in lname for h in exclude) if excl_norm_bias and (".norm" in lname or "bn" in lname or lname.endswith(".bias")): excl=True total += n if excl or force_fp32: fpcnt += n else: qcnt += n if total==0: return float('nan'),0.0,0,0,0 avg = (qcnt*(32 if force_fp32 else w_bits) + fpcnt*32.0)/total return avg, qcnt/total, total, qcnt, fpcnt # ---------- 报告 ---------- def main(): ap=argparse.ArgumentParser() ap.add_argument("--exp-root", default="exp") ap.add_argument("--out-dir", default="exp/summary_0920") ap.add_argument("--plots-dir", default="exp/summary_0920/plots_0920_pretty") ap.add_argument("--exclude", default="cls_head,embedding.stem,stem,head") ap.add_argument("--no-exclude-norm-bias", action="store_true") args=ap.parse_args() exp_root=Path(args.exp_root) out_dir=Path(args.out_dir); out_dir.mkdir(parents=True, exist_ok=True) plots_dir=Path(args.plots_dir); plots_dir.mkdir(parents=True, exist_ok=True) exclude=[s.strip().lower() for s in args.exclude.split(",") if s.strip()] excl_norm_bias = not args.no_exclude_norm_bias runs=[p for p in exp_root.iterdir() if p.is_dir()] # 先记录每个数据集 FP32 最佳作为对比 best_fp32={} for r in runs: ds=guess_dataset(r) w,a,mode=parse_bits_and_mode(r) m=parse_metrics(r/"train.log") if mode=="fp32" and m: cur=best_fp32.get(ds,{}) if "mIoU_best" in m: cur["mIoU_best"]=max(m["mIoU_best"], cur.get("mIoU_best",-1)) if "Acc_best" in m: cur["Acc_best" ]=max(m["Acc_best" ], cur.get("Acc_best" ,-1)) best_fp32[ds]=cur # 收集每个 run rows=[] for r in runs: ds=guess_dataset(r) kind=DATASET_KIND.get(ds, "unknown") w,a,mode=parse_bits_and_mode(r) ckpt=find_ckpt(r) sd = load_state_dict_any(ckpt) if ckpt else {} if sd: avg, qratio, total, qcnt, fpcnt = ( avg_weight_bits(sd, w_bits=w, exclude=exclude, excl_norm_bias=excl_norm_bias, force_fp32=(mode=="fp32")) ) else: avg, qratio, total, qcnt, fpcnt = (float("nan"),0.0,0,0,0) met = parse_metrics(r/"train.log") miou = met.get("mIoU_best"); acc = met.get("Acc_best") base=best_fp32.get(ds,{}) d_miou = miou - base["mIoU_best"] if miou is not None and "mIoU_best" in base else None d_acc = acc - base["Acc_best" ] if acc is not None and "Acc_best" in base else None rows.append(dict( run=r.name, dataset=ds, kind=kind, mode=mode, w_bits=w, a_bits=a, avg_weight_bit=(None if math.isnan(avg) else round(avg,3)), quant_ratio=round(qratio*100,2), params_total=total, params_quant=qcnt, params_fp32=fpcnt, mIoU_best=miou, d_mIoU=d_miou, Acc_best=acc, d_Acc=d_acc, ckpt=str(ckpt) if ckpt else "" )) # 写 CSV csv_path = out_dir/"summary_pretty_0920.csv" with csv_path.open("w", newline="") as f: if rows: fieldnames=list(rows[0].keys()) w=csv.DictWriter(f, fieldnames=fieldnames); w.writeheader() for r in rows: w.writerow(r) print(f"[OK] CSV: {csv_path}") # ---- 画图：Seg（mIoU vs AvgBit） & Cls（Acc vs AvgBit） ---- def scatter_plot(items, xkey, ykey, title, xlabel, ylabel, save): if not items: return xs=[]; ys=[]; labs=[] for r in items: x=r.get(xkey); y=r.get(ykey) if x is None or y is None: continue xs.append(float(x)); ys.append(float(y)); labs.append(r["run"]) if not xs: return plt.figure(figsize=(7.5,4.5)) plt.scatter(xs, ys, s=45, alpha=0.9) for x,y,l in zip(xs,ys,labs): plt.annotate(l, (x,y), fontsize=8, xytext=(4,4), textcoords="offset points") plt.xlabel(xlabel); plt.ylabel(ylabel); plt.title(title) plt.grid(True, linestyle="--", alpha=0.35) plt.tight_layout(); plt.savefig(save, dpi=220); plt.close() print(f"[plot] {save}") seg_items=[r for r in rows if r["kind"]=="seg"] cls_items=[r for r in rows if r["kind"]=="cls"] scatter_plot(seg_items, "avg_weight_bit", "mIoU_best", "Semantic Segmentation: mIoU vs Avg Weight Bit (0920)", "Average Weight Bit", "mIoU (%)", plots_dir/"seg_miou_vs_avgbit_0920.png") scatter_plot(cls_items, "avg_weight_bit", "Acc_best", "Classification: Accuracy vs Avg Weight Bit (0920)", "Average Weight Bit", "Accuracy (%)", plots_dir/"cls_acc_vs_avgbit_0920.png") # ---- 漂亮的排行榜：每个数据集一张表（PNG） ---- import pandas as pd df = pd.DataFrame(rows) for ds in sorted(df["dataset"].dropna().unique()): sub = df[df["dataset"]==ds].copy() if sub.empty or ds=="unknown": continue # 选择显示列 cols = ["run","mode","w_bits","a_bits","avg_weight_bit","quant_ratio", "mIoU_best","d_mIoU","Acc_best","d_Acc"] show = sub[cols].sort_values(by=["mode","avg_weight_bit","mIoU_best","Acc_best"], ascending=[True,True,False,False]) # 用 pandas Styler 出 PNG（matplotlib 渲染） fig, ax = plt.subplots(figsize=(min(18, 1.1*len(cols)), min(0.6+0.35*len(show), 10))) ax.axis("off") the_table = ax.table(cellText=show.fillna("-").values, colLabels=show.columns, loc="center") the_table.auto_set_font_size(False) the_table.set_fontsize(9) the_table.scale(1.0, 1.2) ax.set_title(f"{ds.upper()} Leaderboard (0920)", fontsize=12, pad=10) out = plots_dir/f"{ds}_leaderboard_0920.png" plt.tight_layout(); plt.savefig(out, dpi=220); plt.close() print(f"[table] {out}") # ---- 总览柱状图：各 run 的 AvgBit（你已生成，这里再做个更紧凑版本） ---- ordered = [r for r in rows if r["avg_weight_bit"] is not None] ordered.sort(key=lambda x:(x["kind"], x["dataset"], x["avg_weight_bit"])) labs=[f'{r["dataset"]}:{r["run"]}' for r in ordered] vals=[r["avg_weight_bit"] for r in ordered] if vals: plt.figure(figsize=(max(9,0.22*len(vals)+4), 5.5)) plt.bar(range(len(vals)), vals) plt.xticks(range(len(vals)), labs, rotation=70, ha="right", fontsize=8) plt.ylabel("Average Weight Bit") plt.title("Avg Weight Bit by Run (0920)") plt.grid(axis="y", linestyle="--", alpha=0.35) out = plots_dir/"all_runs_avgbit_0920.png" plt.tight_layout(); plt.savefig(out, dpi=220); plt.close() print(f"[plot] {out}") if __name__ == "__main__": main()