Buckets:
| """Quick checkpoint inspection script.""" | |
| import sys | |
| import torch | |
| path = sys.argv[1] if len(sys.argv) > 1 else "checkpoints/checkpoint_epoch_22.pt" | |
| ckpt = torch.load(path, map_location="cpu", weights_only=False) | |
| print("=== Checkpoint Keys ===") | |
| for k in sorted(ckpt.keys()): | |
| val = ckpt[k] | |
| if isinstance(val, dict) and "state_dict" in k: | |
| num = len(val) | |
| print(f" {k}: {num} entries") | |
| elif isinstance(val, (int, float)): | |
| print(f" {k}: {val}") | |
| elif isinstance(val, dict): | |
| print(f" {k}: dict with {len(val)} keys") | |
| else: | |
| print(f" {k}: {type(val).__name__}") | |
| print("\n=== Training State ===") | |
| print(f" epoch: {ckpt.get('epoch', '?')}") | |
| print(f" global_step: {ckpt.get('global_step', '?')}") | |
| print(f" best_val_loss: {ckpt.get('best_val_loss', '?')}") | |
| print(f" world_size: {ckpt.get('world_size', '?')}") | |
| # Model weights check | |
| sd = ckpt.get("model_state_dict", {}) | |
| print(f"\n=== Model Weights ({len(sd)} params) ===") | |
| nan_count = sum(1 for v in sd.values() if torch.isnan(v).any()) | |
| inf_count = sum(1 for v in sd.values() if torch.isinf(v).any()) | |
| print(f" NaN params: {nan_count}, Inf params: {inf_count}") | |
| for i, (name, tensor) in enumerate(sd.items()): | |
| if i < 5: | |
| print(f" {name}: shape={list(tensor.shape)}, mean={tensor.float().mean():.6f}, std={tensor.float().std():.6f}") | |
| # Check if keys have module. prefix | |
| has_module = any(k.startswith("module.") for k in sd.keys()) | |
| print(f" DDP prefix (module.): {has_module}") | |
| # EMA check | |
| if "ema_state_dict" in ckpt: | |
| ema = ckpt["ema_state_dict"] | |
| shadow = ema.get("shadow", {}) | |
| print(f"\n=== EMA Weights ({len(shadow)} params) ===") | |
| print(f" EMA step: {ema.get('step', '?')}") | |
| print(f" EMA decay: {ema.get('decay', '?')}") | |
| nan_count = sum(1 for v in shadow.values() if torch.isnan(v).any()) | |
| inf_count = sum(1 for v in shadow.values() if torch.isinf(v).any()) | |
| print(f" NaN params: {nan_count}, Inf params: {inf_count}") | |
| for i, (name, tensor) in enumerate(shadow.items()): | |
| if i < 3: | |
| print(f" {name}: mean={tensor.float().mean():.6f}, std={tensor.float().std():.6f}") | |
| # Compare model vs EMA weights (are they very different?) | |
| if shadow and sd: | |
| diffs = [] | |
| for name in list(shadow.keys())[:20]: | |
| model_key = name.replace("module.", "") | |
| if model_key in sd: | |
| diff = (shadow[name].float() - sd[model_key].float()).abs().mean().item() | |
| diffs.append(diff) | |
| if diffs: | |
| print(f"\n Model vs EMA avg abs diff (first 20 params): {sum(diffs)/len(diffs):.6f}") | |
| else: | |
| print("\n No EMA weights in checkpoint") | |
| # Config check | |
| if "config" in ckpt: | |
| cfg = ckpt["config"] | |
| unet = cfg.get("model", {}).get("unet", {}) | |
| print(f"\n=== UNet Config ===") | |
| for k, v in unet.items(): | |
| print(f" {k}: {v}") | |
Xet Storage Details
- Size:
- 2.95 kB
- Xet hash:
- 58d21c3b39934b4654a4497c842b9ef49661785c5597b0d3fae31ad63ae16a8c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.