| |
| |
| import os, re, csv, math |
| from pathlib import Path |
| import argparse |
| import torch |
| import matplotlib |
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
|
|
| TARGET_DATASETS = {"scannet":"seg", "nuscenes":"seg"} |
|
|
| def dataset_of(run: Path): |
| name = run.name.lower() |
| for k in TARGET_DATASETS: |
| if k in name: return k |
| log = run/"train.log" |
| if log.exists(): |
| t = log.read_text(errors="ignore").lower() |
| for k in TARGET_DATASETS: |
| if k in t: return k |
| return None |
|
|
| def find_ckpt(run: Path): |
| pats = ["best*.pth","model_best*.pth","*checkpoint*.pth","*ckpt*.pth", |
| "last*.pth","latest*.pth","*.pth","*.pt"] |
| cands=[] |
| for p in pats: cands += list(run.rglob(p)) |
| cands = [p for p in cands if p.is_file()] |
| if not cands: return None |
| scored=[] |
| for p in cands: |
| s=0; n=p.name.lower() |
| if "best" in n: s+=100 |
| if "last" in n or "latest" in n: s+=50 |
| s += int(p.stat().st_mtime) |
| scored.append((s,p)) |
| scored.sort(reverse=True) |
| return scored[0][1] |
|
|
| def load_state_dict_any(p: Path): |
| try: |
| try: obj=torch.load(p, map_location="cpu", weights_only=True) |
| except TypeError: obj=torch.load(p, map_location="cpu") |
| if isinstance(obj, dict): |
| for k in ["state_dict","model","net","module","ema","model_state","model_ema"]: |
| if k in obj and isinstance(obj[k], dict): |
| return {kk:vv for kk,vv in obj[k].items() if torch.is_tensor(vv)} |
| if all(isinstance(k,str) for k in obj.keys()): |
| return {k:v for k,v in obj.items() if torch.is_tensor(v)} |
| return {} |
| except Exception: |
| return {} |
|
|
| def avg_weight_bits(sd, w_bits=2, exclude=("cls_head","embedding.stem","stem","head"), |
| excl_norm_bias=True, force_fp32=False): |
| total=qcnt=fpcnt=0 |
| for name,t in sd.items(): |
| if not torch.is_tensor(t): continue |
| n=t.numel(); lname=name.lower() |
| excl = any(h in lname for h in exclude) |
| if excl_norm_bias and (".norm" in lname or "bn" in lname or lname.endswith(".bias")): |
| excl=True |
| total += n |
| if excl or force_fp32: fpcnt += n |
| else: qcnt += n |
| if total==0: return float("nan"),0.0,0,0,0 |
| avg=(qcnt*(32 if force_fp32 else w_bits) + fpcnt*32.0)/total |
| return avg, qcnt/total, total, qcnt, fpcnt |
|
|
| def parse_bits_and_mode(run: Path): |
| name=run.name.lower() |
| w=a=None |
| m=re.search(r"w(\d+)a(\d+)", name) |
| if m: w,a=int(m.group(1)), int(m.group(2)) |
| mode="quant" if m else ("fp32" if ("fp32" in name or "baseline" in name) else "unknown") |
| log=run/"train.log" |
| if log.exists(): |
| t=log.read_text(errors="ignore").lower() |
| m1=re.search(r"quant\d*\.?enable\s*=\s*(true|false)", t) |
| if m1: mode="quant" if m1.group(1)=="true" else "fp32" |
| mw=re.search(r"quant\d*\.?w_bits\s*=\s*(\d+)", t) |
| ma=re.search(r"quant\d*\.?a_bits\s*=\s*(\d+)", t) |
| if mw: w=int(mw.group(1)) |
| if ma: a=int(ma.group(1)) |
| if mode=="fp32": w=w or 32; a=a or 32 |
| return w or 2, a or 8, mode |
|
|
| def parse_metrics(log_path: Path): |
| res={} |
| if not log_path.exists(): return res |
| lines=log_path.read_text(errors="ignore").splitlines() |
| RE_MIOU=re.compile(r'(?:^|\b)(?:mIoU|miou|val_miou|mean\s*iou|iou\s*mean)\s*[:=]\s*([0-9]+(?:\.[0-9]+)?)', re.I) |
| RE_ACC =re.compile(r'(?:^|\b)(?:acc|accuracy|oa|top-1|top1)\s*[:=]\s*([0-9]+(?:\.[0-9]+)?)', re.I) |
| mi=[]; ac=[] |
| for l in lines: |
| l=l.strip() |
| m1=RE_MIOU.search(l); m2=RE_ACC.search(l) |
| if m1: |
| v=float(m1.group(1)) |
| if 0<=v<=100: mi.append(v) |
| if m2: |
| v=float(m2.group(1)) |
| if 0<=v<=100: ac.append(v) |
| if mi: res["mIoU_best"]=max(mi) |
| if ac: res["Acc_best"]=max(ac) |
| return res |
|
|
| def main(): |
| ap=argparse.ArgumentParser() |
| ap.add_argument("--exp-root", default="exp") |
| ap.add_argument("--out-csv", default="exp/summary_0920/focus_scannet_nusc_0920.csv") |
| ap.add_argument("--plots-dir", default="exp/summary_0920/plots_focus_0920") |
| ap.add_argument("--no-exclude-norm-bias", action="store_true") |
| args=ap.parse_args() |
|
|
| root=Path(args.exp_root) |
| plots=Path(args.plots_dir); plots.mkdir(parents=True, exist_ok=True) |
| excl_norm_bias = not args.no_exclude_norm_bias |
|
|
| runs=[p for p in root.iterdir() if p.is_dir()] |
| rows=[] |
|
|
| |
| for r in runs: |
| ds=dataset_of(r) |
| if ds is None: continue |
| ckpt=find_ckpt(r) |
| if ckpt is None: continue |
|
|
| w,a,mode=parse_bits_and_mode(r) |
| sd=load_state_dict_any(ckpt) |
| if sd: |
| avg, qratio, total, qcnt, fpcnt = avg_weight_bits(sd, w_bits=w, |
| exclude=("cls_head","embedding.stem","stem","head"), |
| excl_norm_bias=excl_norm_bias, force_fp32=(mode=="fp32")) |
| else: |
| avg, qratio, total, qcnt, fpcnt = (float("nan"),0.0,0,0,0) |
|
|
| |
| if mode=="unknown" and not math.isnan(avg): |
| mode="quant" if avg<31.9 else "fp32" |
|
|
| metr=parse_metrics(r/"train.log") |
| miou=metr.get("mIoU_best") |
|
|
| rows.append(dict( |
| run=r.name, dataset=ds, mode=mode, w_bits=w, a_bits=a, |
| avg_weight_bit=(None if math.isnan(avg) else round(avg,3)), |
| quant_ratio=round(qratio*100,2), |
| mIoU_best=miou, ckpt=str(ckpt) |
| )) |
|
|
| |
| best_fp32={} |
| for r in rows: |
| if r["dataset"] in TARGET_DATASETS and r["avg_weight_bit"] is not None and r["avg_weight_bit"]>=31.9 and r["mIoU_best"] is not None: |
| cur=best_fp32.get(r["dataset"], -1) |
| if r["mIoU_best"]>cur: best_fp32[r["dataset"]]=r["mIoU_best"] |
|
|
| |
| for r in rows: |
| base = best_fp32.get(r["dataset"]) |
| r["ΔmIoU_vs_FP32"] = (None if base is None or r["mIoU_best"] is None else round(r["mIoU_best"]-base,3)) |
|
|
| |
| out=Path(args.out_csv); out.parent.mkdir(parents=True, exist_ok=True) |
| with out.open("w", newline="") as f: |
| if rows: |
| import csv |
| cols=["dataset","run","mode","w_bits","a_bits","avg_weight_bit","quant_ratio","mIoU_best","ΔmIoU_vs_FP32","ckpt"] |
| w=csv.DictWriter(f, fieldnames=cols); w.writeheader() |
| for r in sorted(rows, key=lambda x:(x["dataset"], x["avg_weight_bit"] if x["avg_weight_bit"] is not None else 99)): |
| w.writerow({k:r.get(k,"") for k in cols}) |
| print(f"[OK] saved CSV: {out}") |
|
|
| |
| for ds in TARGET_DATASETS: |
| sub=[r for r in rows if r["dataset"]==ds and r["avg_weight_bit"] is not None and r["mIoU_best"] is not None] |
| if not sub: continue |
| xs=[r["avg_weight_bit"] for r in sub] |
| ys=[r["mIoU_best"] for r in sub] |
| labs=[r["run"] for r in sub] |
| plt.figure(figsize=(7.5,4.5)) |
| plt.scatter(xs, ys, s=50) |
| for x,y,l in zip(xs,ys,labs): |
| plt.annotate(l, (x,y), fontsize=8, xytext=(4,4), textcoords="offset points") |
| base = best_fp32.get(ds) |
| if base is not None: |
| plt.axhline(base, linestyle="--") |
| plt.xlabel("Average Weight Bit"); plt.ylabel("mIoU (%)") |
| plt.title(f"{ds.upper()} mIoU vs AvgBit (only ckpt) 0920") |
| p = plots/f"{ds}_miou_vs_avgbit_focus_0920.png" |
| plt.tight_layout(); plt.savefig(p, dpi=220); plt.close() |
| print(f"[plot] {p}") |
|
|
| if __name__=="__main__": |
| main() |
|
|
