minhluanquach/unet-training-data / scripts /inspect_checkpoint.py
minhluanquach's picture
download
raw
2.95 kB
"""Quick checkpoint inspection script."""
import sys
import torch
path = sys.argv[1] if len(sys.argv) > 1 else "checkpoints/checkpoint_epoch_22.pt"
ckpt = torch.load(path, map_location="cpu", weights_only=False)
print("=== Checkpoint Keys ===")
for k in sorted(ckpt.keys()):
val = ckpt[k]
if isinstance(val, dict) and "state_dict" in k:
num = len(val)
print(f" {k}: {num} entries")
elif isinstance(val, (int, float)):
print(f" {k}: {val}")
elif isinstance(val, dict):
print(f" {k}: dict with {len(val)} keys")
else:
print(f" {k}: {type(val).__name__}")
print("\n=== Training State ===")
print(f" epoch: {ckpt.get('epoch', '?')}")
print(f" global_step: {ckpt.get('global_step', '?')}")
print(f" best_val_loss: {ckpt.get('best_val_loss', '?')}")
print(f" world_size: {ckpt.get('world_size', '?')}")
# Model weights check
sd = ckpt.get("model_state_dict", {})
print(f"\n=== Model Weights ({len(sd)} params) ===")
nan_count = sum(1 for v in sd.values() if torch.isnan(v).any())
inf_count = sum(1 for v in sd.values() if torch.isinf(v).any())
print(f" NaN params: {nan_count}, Inf params: {inf_count}")
for i, (name, tensor) in enumerate(sd.items()):
if i < 5:
print(f" {name}: shape={list(tensor.shape)}, mean={tensor.float().mean():.6f}, std={tensor.float().std():.6f}")
# Check if keys have module. prefix
has_module = any(k.startswith("module.") for k in sd.keys())
print(f" DDP prefix (module.): {has_module}")
# EMA check
if "ema_state_dict" in ckpt:
ema = ckpt["ema_state_dict"]
shadow = ema.get("shadow", {})
print(f"\n=== EMA Weights ({len(shadow)} params) ===")
print(f" EMA step: {ema.get('step', '?')}")
print(f" EMA decay: {ema.get('decay', '?')}")
nan_count = sum(1 for v in shadow.values() if torch.isnan(v).any())
inf_count = sum(1 for v in shadow.values() if torch.isinf(v).any())
print(f" NaN params: {nan_count}, Inf params: {inf_count}")
for i, (name, tensor) in enumerate(shadow.items()):
if i < 3:
print(f" {name}: mean={tensor.float().mean():.6f}, std={tensor.float().std():.6f}")
# Compare model vs EMA weights (are they very different?)
if shadow and sd:
diffs = []
for name in list(shadow.keys())[:20]:
model_key = name.replace("module.", "")
if model_key in sd:
diff = (shadow[name].float() - sd[model_key].float()).abs().mean().item()
diffs.append(diff)
if diffs:
print(f"\n Model vs EMA avg abs diff (first 20 params): {sum(diffs)/len(diffs):.6f}")
else:
print("\n No EMA weights in checkpoint")
# Config check
if "config" in ckpt:
cfg = ckpt["config"]
unet = cfg.get("model", {}).get("unet", {})
print(f"\n=== UNet Config ===")
for k, v in unet.items():
print(f" {k}: {v}")

Xet Storage Details

Size:
2.95 kB
·
Xet hash:
58d21c3b39934b4654a4497c842b9ef49661785c5597b0d3fae31ad63ae16a8c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.