| |
| |
| import os, re, csv, math |
| from pathlib import Path |
| import argparse |
| import torch |
| import matplotlib |
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
|
|
| |
| DATASET_KIND = { |
| "scannet": "seg", |
| "s3dis": "seg", |
| "nuscenes": "seg", |
| "modelnet": "cls", |
| } |
|
|
| def guess_dataset(run_dir: Path): |
| name = run_dir.name.lower() |
| for k in DATASET_KIND: |
| if k in name: return k |
| log = run_dir / "train.log" |
| if log.exists(): |
| t = log.read_text(errors="ignore").lower() |
| for k in DATASET_KIND: |
| if k in t: return k |
| return "unknown" |
|
|
| def parse_bits_and_mode(run_dir: Path): |
| name = run_dir.name.lower() |
| w = a = None |
| m = re.search(r"w(\d+)a(\d+)", name) |
| if m: w, a = int(m.group(1)), int(m.group(2)) |
| mode = "quant" if m else ("fp32" if ("fp32" in name or "baseline" in name) else "unknown") |
| |
| log = run_dir / "train.log" |
| if log.exists(): |
| t = log.read_text(errors="ignore").lower() |
| m1 = re.search(r"quant\d*\.?enable\s*=\s*(true|false)", t) |
| if m1: |
| mode = "quant" if m1.group(1) == "true" else "fp32" |
| mw = re.search(r"quant\d*\.?w_bits\s*=\s*(\d+)", t) |
| ma = re.search(r"quant\d*\.?a_bits\s*=\s*(\d+)", t) |
| if mw: w = int(mw.group(1)) |
| if ma: a = int(ma.group(1)) |
| if mode == "fp32": w = w or 32; a = a or 32 |
| return w or 2, a or 8, mode |
|
|
| def parse_metrics(log_path: Path): |
| res={} |
| if not log_path.exists(): return res |
| txt = log_path.read_text(errors="ignore") |
| |
| mi=[] |
| for line in txt.splitlines(): |
| l=line.lower() |
| if "miou" in l: |
| mi += [float(x) for x in re.findall(r"[-+]?\d*\.\d+|\d+", l) if 0<=float(x)<=100] |
| if mi: res["mIoU_best"]=max(mi) |
| |
| acc=[] |
| for key in ["overall acc","oa","accuracy","acc"]: |
| for line in txt.splitlines(): |
| l=line.lower() |
| if key in l: |
| acc += [float(x) for x in re.findall(r"[-+]?\d*\.\d+|\d+", l) if 0<=float(x)<=100] |
| if acc: res["Acc_best"]=max(acc) |
| return res |
|
|
| def find_ckpt(run_dir: Path): |
| cands=[] |
| for ext in ("*.pth","*.pt"): cands += list(run_dir.rglob(ext)) |
| scored=[] |
| for p in cands: |
| s=0; n=p.name.lower() |
| if "best" in n: s+=100 |
| if "last" in n or "latest" in n: s+=50 |
| s += int(p.stat().st_mtime) |
| scored.append((s,p)) |
| scored.sort(reverse=True) |
| return scored[0][1] if scored else None |
|
|
| def load_state_dict_any(p: Path): |
| try: |
| try: obj = torch.load(p, map_location="cpu", weights_only=True) |
| except TypeError: obj = torch.load(p, map_location="cpu") |
| if isinstance(obj, dict): |
| for k in ["state_dict","model","net","module","ema","model_state","model_ema"]: |
| if k in obj and isinstance(obj[k], dict): return {kk:vv for kk,vv in obj[k].items() if torch.is_tensor(vv)} |
| if all(isinstance(k, str) for k in obj.keys()): |
| return {k:v for k,v in obj.items() if torch.is_tensor(v)} |
| return {} |
| except Exception: |
| return {} |
|
|
| def avg_weight_bits(sd, w_bits=2, exclude=("cls_head","embedding.stem","stem","head"), excl_norm_bias=True, force_fp32=False): |
| total=qcnt=fpcnt=0 |
| for name, t in sd.items(): |
| if not torch.is_tensor(t): continue |
| n=t.numel(); lname=name.lower() |
| excl = any(h in lname for h in exclude) |
| if excl_norm_bias and (".norm" in lname or "bn" in lname or lname.endswith(".bias")): |
| excl=True |
| total += n |
| if excl or force_fp32: fpcnt += n |
| else: qcnt += n |
| if total==0: return float('nan'),0.0,0,0,0 |
| avg = (qcnt*(32 if force_fp32 else w_bits) + fpcnt*32.0)/total |
| return avg, qcnt/total, total, qcnt, fpcnt |
|
|
| |
| def main(): |
| ap=argparse.ArgumentParser() |
| ap.add_argument("--exp-root", default="exp") |
| ap.add_argument("--out-dir", default="exp/summary_0920") |
| ap.add_argument("--plots-dir", default="exp/summary_0920/plots_0920_pretty") |
| ap.add_argument("--exclude", default="cls_head,embedding.stem,stem,head") |
| ap.add_argument("--no-exclude-norm-bias", action="store_true") |
| args=ap.parse_args() |
|
|
| exp_root=Path(args.exp_root) |
| out_dir=Path(args.out_dir); out_dir.mkdir(parents=True, exist_ok=True) |
| plots_dir=Path(args.plots_dir); plots_dir.mkdir(parents=True, exist_ok=True) |
| exclude=[s.strip().lower() for s in args.exclude.split(",") if s.strip()] |
| excl_norm_bias = not args.no_exclude_norm_bias |
|
|
| runs=[p for p in exp_root.iterdir() if p.is_dir()] |
|
|
| |
| best_fp32={} |
| for r in runs: |
| ds=guess_dataset(r) |
| w,a,mode=parse_bits_and_mode(r) |
| m=parse_metrics(r/"train.log") |
| if mode=="fp32" and m: |
| cur=best_fp32.get(ds,{}) |
| if "mIoU_best" in m: cur["mIoU_best"]=max(m["mIoU_best"], cur.get("mIoU_best",-1)) |
| if "Acc_best" in m: cur["Acc_best" ]=max(m["Acc_best" ], cur.get("Acc_best" ,-1)) |
| best_fp32[ds]=cur |
|
|
| |
| rows=[] |
| for r in runs: |
| ds=guess_dataset(r) |
| kind=DATASET_KIND.get(ds, "unknown") |
| w,a,mode=parse_bits_and_mode(r) |
| ckpt=find_ckpt(r) |
| sd = load_state_dict_any(ckpt) if ckpt else {} |
| if sd: |
| avg, qratio, total, qcnt, fpcnt = ( |
| avg_weight_bits(sd, w_bits=w, exclude=exclude, |
| excl_norm_bias=excl_norm_bias, |
| force_fp32=(mode=="fp32")) |
| ) |
| else: |
| avg, qratio, total, qcnt, fpcnt = (float("nan"),0.0,0,0,0) |
| met = parse_metrics(r/"train.log") |
| miou = met.get("mIoU_best"); acc = met.get("Acc_best") |
| base=best_fp32.get(ds,{}) |
| d_miou = miou - base["mIoU_best"] if miou is not None and "mIoU_best" in base else None |
| d_acc = acc - base["Acc_best" ] if acc is not None and "Acc_best" in base else None |
| rows.append(dict( |
| run=r.name, dataset=ds, kind=kind, mode=mode, |
| w_bits=w, a_bits=a, |
| avg_weight_bit=(None if math.isnan(avg) else round(avg,3)), |
| quant_ratio=round(qratio*100,2), |
| params_total=total, params_quant=qcnt, params_fp32=fpcnt, |
| mIoU_best=miou, d_mIoU=d_miou, Acc_best=acc, d_Acc=d_acc, |
| ckpt=str(ckpt) if ckpt else "" |
| )) |
|
|
| |
| csv_path = out_dir/"summary_pretty_0920.csv" |
| with csv_path.open("w", newline="") as f: |
| if rows: |
| fieldnames=list(rows[0].keys()) |
| w=csv.DictWriter(f, fieldnames=fieldnames); w.writeheader() |
| for r in rows: w.writerow(r) |
| print(f"[OK] CSV: {csv_path}") |
|
|
| |
| def scatter_plot(items, xkey, ykey, title, xlabel, ylabel, save): |
| if not items: return |
| xs=[]; ys=[]; labs=[] |
| for r in items: |
| x=r.get(xkey); y=r.get(ykey) |
| if x is None or y is None: continue |
| xs.append(float(x)); ys.append(float(y)); labs.append(r["run"]) |
| if not xs: return |
| plt.figure(figsize=(7.5,4.5)) |
| plt.scatter(xs, ys, s=45, alpha=0.9) |
| for x,y,l in zip(xs,ys,labs): |
| plt.annotate(l, (x,y), fontsize=8, xytext=(4,4), textcoords="offset points") |
| plt.xlabel(xlabel); plt.ylabel(ylabel); plt.title(title) |
| plt.grid(True, linestyle="--", alpha=0.35) |
| plt.tight_layout(); plt.savefig(save, dpi=220); plt.close() |
| print(f"[plot] {save}") |
|
|
| seg_items=[r for r in rows if r["kind"]=="seg"] |
| cls_items=[r for r in rows if r["kind"]=="cls"] |
|
|
| scatter_plot(seg_items, "avg_weight_bit", "mIoU_best", |
| "Semantic Segmentation: mIoU vs Avg Weight Bit (0920)", |
| "Average Weight Bit", "mIoU (%)", |
| plots_dir/"seg_miou_vs_avgbit_0920.png") |
|
|
| scatter_plot(cls_items, "avg_weight_bit", "Acc_best", |
| "Classification: Accuracy vs Avg Weight Bit (0920)", |
| "Average Weight Bit", "Accuracy (%)", |
| plots_dir/"cls_acc_vs_avgbit_0920.png") |
|
|
| |
| import pandas as pd |
| df = pd.DataFrame(rows) |
| for ds in sorted(df["dataset"].dropna().unique()): |
| sub = df[df["dataset"]==ds].copy() |
| if sub.empty or ds=="unknown": continue |
| |
| cols = ["run","mode","w_bits","a_bits","avg_weight_bit","quant_ratio", |
| "mIoU_best","d_mIoU","Acc_best","d_Acc"] |
| show = sub[cols].sort_values(by=["mode","avg_weight_bit","mIoU_best","Acc_best"], |
| ascending=[True,True,False,False]) |
| |
| fig, ax = plt.subplots(figsize=(min(18, 1.1*len(cols)), min(0.6+0.35*len(show), 10))) |
| ax.axis("off") |
| the_table = ax.table(cellText=show.fillna("-").values, |
| colLabels=show.columns, |
| loc="center") |
| the_table.auto_set_font_size(False) |
| the_table.set_fontsize(9) |
| the_table.scale(1.0, 1.2) |
| ax.set_title(f"{ds.upper()} Leaderboard (0920)", fontsize=12, pad=10) |
| out = plots_dir/f"{ds}_leaderboard_0920.png" |
| plt.tight_layout(); plt.savefig(out, dpi=220); plt.close() |
| print(f"[table] {out}") |
|
|
| |
| ordered = [r for r in rows if r["avg_weight_bit"] is not None] |
| ordered.sort(key=lambda x:(x["kind"], x["dataset"], x["avg_weight_bit"])) |
| labs=[f'{r["dataset"]}:{r["run"]}' for r in ordered] |
| vals=[r["avg_weight_bit"] for r in ordered] |
| if vals: |
| plt.figure(figsize=(max(9,0.22*len(vals)+4), 5.5)) |
| plt.bar(range(len(vals)), vals) |
| plt.xticks(range(len(vals)), labs, rotation=70, ha="right", fontsize=8) |
| plt.ylabel("Average Weight Bit") |
| plt.title("Avg Weight Bit by Run (0920)") |
| plt.grid(axis="y", linestyle="--", alpha=0.35) |
| out = plots_dir/"all_runs_avgbit_0920.png" |
| plt.tight_layout(); plt.savefig(out, dpi=220); plt.close() |
| print(f"[plot] {out}") |
|
|
| if __name__ == "__main__": |
| main() |
|
|
