scripts/isolate_bottleneck.py

"""
Decisive isolation benchmark: WHERE do the 8.5 s/iteration actually go?

Reported inference is 1.5 ms/image, so a batch-32 fwd+bwd should cost ~150 ms,
yet a training iteration measures ~8500 ms (55x gap). Rather than guess, time
each stage separately:

  A. compute      -- fwd+bwd on synthetic tensors, no dataloader, no loss
  B. compute+loss -- adds the real semantic criterion (scatter ops on MPS)
  C. determinism  -- same as B with use_deterministic_algorithms(False)
  D. dataload     -- real dataset pipeline only, no model (workers 0 vs 8)
  E. scaling      -- fwd+bwd cost for n / l / x variants

Note: another GPU job may be running; treat absolute numbers as contended and
compare stages relative to each other within this run.
"""
import argparse, time

import numpy as np
import torch


def bench(fn, n=8, warmup=3):
    for _ in range(warmup):
        fn()
    torch.mps.synchronize() if torch.backends.mps.is_available() else None
    t0 = time.time()
    for _ in range(n):
        fn()
    if torch.backends.mps.is_available():
        torch.mps.synchronize()
    return (time.time() - t0) / n * 1000


def make_model(name, device):
    from ultralytics import YOLO
    m = YOLO(name).model.to(device).train()
    for p in m.parameters():          # ultralytics freezes params on checkpoint load
        p.requires_grad_(True)
    return m


def stage_compute(device, batch, imgsz, name="yolo26n-sem.pt"):
    m = make_model(name, device)
    opt = torch.optim.AdamW(m.parameters(), lr=1e-4)
    x = torch.rand(batch, 3, imgsz, imgsz, device=device)

    def collect(z, out):
        if torch.is_tensor(z):
            if z.requires_grad and z.is_floating_point():
                out.append(z.float().pow(2).mean())
        elif isinstance(z, (list, tuple)):
            for e in z:
                collect(e, out)
        elif isinstance(z, dict):
            for e in z.values():
                collect(e, out)

    def step():
        opt.zero_grad(set_to_none=True)
        terms = []
        collect(m(x), terms)
        if not terms:
            raise RuntimeError("no differentiable output from model forward")
        torch.stack(terms).sum().backward()   # trivial surrogate loss
        opt.step()
    return bench(step)


def stage_compute_loss(device, batch, imgsz, deterministic, name="yolo26n-sem.pt"):
    torch.use_deterministic_algorithms(deterministic, warn_only=True)
    from ultralytics import YOLO
    y = YOLO(name)
    m = y.model.to(device).train()
    for p in m.parameters():
        p.requires_grad_(True)
    opt = torch.optim.AdamW(m.parameters(), lr=1e-4)
    x = torch.rand(batch, 3, imgsz, imgsz, device=device)
    masks = torch.randint(0, 19, (batch, imgsz, imgsz), device=device, dtype=torch.long)
    b = {"img": x, "semantic_mask": masks}

    def step():
        opt.zero_grad(set_to_none=True)
        loss, _ = m.loss(b)
        (loss.sum() if loss.dim() else loss).backward()
        opt.step()
    try:
        ms = bench(step, n=6, warmup=2)
    except Exception as e:
        ms = float("nan")
        print("   (loss path failed:", repr(e)[:160], ")")
    torch.use_deterministic_algorithms(False, warn_only=True)
    return ms


def stage_dataload(workers, batch, imgsz, n_batches=8):
    from ultralytics.data.dataset import SemanticDataset
    from ultralytics.cfg import get_cfg
    from ultralytics.utils import DEFAULT_CFG
    from torch.utils.data import DataLoader
    import yaml

    dpath = "/Users/ari/FaceSegmentation/dataset_celebamaskhq_semantic"
    with open(f"{dpath}/data.yaml") as f:
        data = yaml.safe_load(f)
    data["nc"] = len(data["names"])
    data["channels"] = 3
    data["path"] = dpath
    args = get_cfg(DEFAULT_CFG)
    args.imgsz = imgsz
    args.mosaic = 0.5
    args.fliplr = 0.0
    args.degrees = 10.0
    args.scale = 0.5
    ds = SemanticDataset(img_path=f"{dpath}/images/train", imgsz=imgsz, batch_size=batch,
                         augment=True, hyp=args, data=data, task="semantic")
    dl = DataLoader(ds, batch_size=batch, shuffle=True, num_workers=workers,
                    collate_fn=getattr(ds, "collate_fn", None),
                    persistent_workers=workers > 0, prefetch_factor=4 if workers > 0 else None)
    it = iter(dl)
    next(it)  # warm
    t0 = time.time()
    got = 0
    for _ in range(n_batches):
        try:
            next(it)
            got += 1
        except StopIteration:
            break
    return (time.time() - t0) / max(got, 1) * 1000


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--batch", type=int, default=32)
    ap.add_argument("--imgsz", type=int, default=512)
    ap.add_argument("--device", default="mps")
    ap.add_argument("--stages", default="A,B,C,D,E")
    args = ap.parse_args()
    S = set(args.stages.split(","))
    dev = args.device
    B, R = args.batch, args.imgsz
    print(f"device={dev} batch={B} imgsz={R}\n")

    if "A" in S:
        ms = stage_compute(dev, B, R)
        print(f"A. compute only (fwd+bwd, synthetic, no loss) : {ms:8.0f} ms/iter")
    if "B" in S:
        ms = stage_compute_loss(dev, B, R, deterministic=True)
        print(f"B. compute + real loss, deterministic=True    : {ms:8.0f} ms/iter")
    if "C" in S:
        ms = stage_compute_loss(dev, B, R, deterministic=False)
        print(f"C. compute + real loss, deterministic=False   : {ms:8.0f} ms/iter")
    if "D" in S:
        for w in (0, 8):
            ms = stage_dataload(w, B, R)
            print(f"D. dataload only, workers={w}                  : {ms:8.0f} ms/batch")
    if "E" in S:
        for nm in ("yolo26n-sem.pt", "yolo26s-sem.pt", "yolo26m-sem.pt", "yolo26l-sem.pt", "yolo26x-sem.pt"):
            try:
                ms = stage_compute(dev, B, R, nm)
                print(f"E. compute {nm:16s}              : {ms:8.0f} ms/iter")
            except Exception as e:
                print(f"E. compute {nm:16s}              : FAILED {repr(e)[:90]}")


if __name__ == "__main__":
    main()