| |
| """#19 Hardware optimizations — throughput boost (accuracy-neutral). |
| |
| - torch.compile (PyTorch 2.0+) |
| - FlashAttention-3 (Dao 2024) |
| - Liger Kernel (LinkedIn 2024) |
| - bfloat16 mixed precision |
| - Gradient checkpointing |
| - Fused optimizers (Apex) |
| |
| Accuracy kazancı: 0 (sadece iteration hız artışı → daha fazla deney → dolaylı) |
| """ |
|
|
| OPTIMIZATION_RECIPE = { |
| "torch_compile": { |
| "enabled": True, |
| "mode": "max-autotune", |
| "gain": "1.5-2× throughput (backbone inference)", |
| "code": "model = torch.compile(model, mode='max-autotune')", |
| }, |
| "flash_attention_3": { |
| "enabled": True, |
| "gain": "2-3× attention speedup, ~30% memory save", |
| "install": "pip install flash-attn --no-build-isolation", |
| "note": "FlashAttention-3 H100-spesifik; A100'de FlashAttention-2" |
| }, |
| "liger_kernel": { |
| "enabled": True, |
| "gain": "~20% throughput on LLMs/ViTs (Linkedin 2024)", |
| "install": "pip install liger-kernel", |
| "modules": ["LigerRMSNorm", "LigerCrossEntropy", "LigerFusedLinearCrossEntropy"] |
| }, |
| "bf16": { |
| "enabled": True, |
| "gain": "50% memory reduce, 2× throughput on A100", |
| "code": "torch.set_float32_matmul_precision('medium')" |
| }, |
| "gradient_checkpointing": { |
| "enabled": False, |
| "note": "Sadece memory kısıtı varsa. Accuracy neutral." |
| }, |
| "fused_adamw": { |
| "enabled": True, |
| "gain": "~5% optimizer step speedup", |
| "code": "torch.optim.AdamW(fused=True)" |
| }, |
| "channels_last": { |
| "enabled": True, |
| "code": "model = model.to(memory_format=torch.channels_last)" |
| } |
| } |
|
|
| if __name__ == '__main__': |
| import json |
| from pathlib import Path |
| out = Path("/arf/scratch/stakan/hitit-proje/datasets/processed/hw_optimizations.json") |
| with open(out, 'w') as f: |
| json.dump(OPTIMIZATION_RECIPE, f, indent=2, ensure_ascii=False) |
| print(f"HW opt recipe yazıldı: {out}") |
| print("\nAktifleştirme:") |
| for k, v in OPTIMIZATION_RECIPE.items(): |
| if isinstance(v, dict) and v.get('enabled'): |
| print(f" ✓ {k}") |
|
|
