| { |
| "cache_key": { |
| "version": 1, |
| "kind": "gpu", |
| "gpu": { |
| "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "total_memory": 101975851008, |
| "capability": [ |
| 12, |
| 0 |
| ] |
| }, |
| "torch": "2.8.0+cu128", |
| "cuda": "12.8", |
| "python": "3.13.11", |
| "packages": { |
| "transformers": "4.57.1", |
| "numpy": "2.2.6", |
| "pyarrow": "21.0.0", |
| "huggingface_hub": "0.36.0" |
| }, |
| "settings": { |
| "context": 2048, |
| "global_batch": 90, |
| "microbatch_candidates": [ |
| 8, |
| 16, |
| 32, |
| 48, |
| 64, |
| 72, |
| 76, |
| 80, |
| 86, |
| 90 |
| ], |
| "batch_selection": "max_batch", |
| "gradient_checkpointing": false, |
| "compile_model": true, |
| "autotune_microbatch": true, |
| "attention_backend": "flash", |
| "vram_fraction": 0.95 |
| }, |
| "source": "e6d135a4487a9fd1907e12903cda79e75fd61305fad149fda7ec5281279dc18b", |
| "requested_microbatch_ceiling": 90 |
| }, |
| "cache_reused": true, |
| "attention": "PyTorch FlashAttention (forced; forward/backward verified)", |
| "selected": { |
| "microbatch": 86, |
| "mode": "compiled", |
| "microbatch_seconds": 0.8974938420000171, |
| "peak_gb": 85.39913082122803, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9533413040000482, |
| "input_tokens_per_second": 196244.2434228943 |
| }, |
| "measurements": [ |
| { |
| "microbatch": 8, |
| "mode": "eager", |
| "microbatch_seconds": 0.15782159200000478, |
| "peak_gb": 14.079077243804932, |
| "fits_budget": true, |
| "estimated_update_seconds": 1.793421506500053, |
| "input_tokens_per_second": 103813.42497165725 |
| }, |
| { |
| "microbatch": 16, |
| "mode": "eager", |
| "microbatch_seconds": 0.3450626039999918, |
| "peak_gb": 27.397329807281494, |
| "fits_budget": true, |
| "estimated_update_seconds": 1.9234990809999601, |
| "input_tokens_per_second": 94962.47817106481 |
| }, |
| { |
| "microbatch": 32, |
| "mode": "eager", |
| "microbatch_seconds": 0.7818640030000097, |
| "peak_gb": 54.14767265319824, |
| "fits_budget": true, |
| "estimated_update_seconds": 2.185234111500023, |
| "input_tokens_per_second": 83820.20370363462 |
| }, |
| { |
| "microbatch": 48, |
| "mode": "eager", |
| "microbatch_seconds": 1.1685763079999845, |
| "peak_gb": 80.94732475280762, |
| "fits_budget": true, |
| "estimated_update_seconds": 2.190914332499986, |
| "input_tokens_per_second": 84122.87612457851 |
| }, |
| { |
| "microbatch": 8, |
| "mode": "compiled", |
| "microbatch_seconds": 0.08803204299999834, |
| "peak_gb": 8.428553104400635, |
| "fits_budget": true, |
| "estimated_update_seconds": 1.0060347614999898, |
| "input_tokens_per_second": 186114.0494035826 |
| }, |
| { |
| "microbatch": 16, |
| "mode": "compiled", |
| "microbatch_seconds": 0.1704058820000114, |
| "peak_gb": 16.191233158111572, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9576335650000658, |
| "input_tokens_per_second": 192293.8317352086 |
| }, |
| { |
| "microbatch": 32, |
| "mode": "compiled", |
| "microbatch_seconds": 0.34082671699997036, |
| "peak_gb": 31.810625553131104, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9493586249999737, |
| "input_tokens_per_second": 192285.39527904938 |
| }, |
| { |
| "microbatch": 48, |
| "mode": "compiled", |
| "microbatch_seconds": 0.5061537779999981, |
| "peak_gb": 47.43971490859985, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9468006980000041, |
| "input_tokens_per_second": 194217.6553308279 |
| }, |
| { |
| "microbatch": 64, |
| "mode": "compiled", |
| "microbatch_seconds": 0.6658512800000267, |
| "peak_gb": 63.073320388793945, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9295456565000109, |
| "input_tokens_per_second": 196848.76178355434 |
| }, |
| { |
| "microbatch": 72, |
| "mode": "compiled", |
| "microbatch_seconds": 0.7430151979999664, |
| "peak_gb": 70.97307014465332, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9250642054999503, |
| "input_tokens_per_second": 198456.23669195347 |
| }, |
| { |
| "microbatch": 76, |
| "mode": "compiled", |
| "microbatch_seconds": 0.78068104700003, |
| "peak_gb": 74.84185981750488, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9241739715000392, |
| "input_tokens_per_second": 199374.63654089966 |
| }, |
| { |
| "microbatch": 80, |
| "mode": "compiled", |
| "microbatch_seconds": 0.8210239410000213, |
| "peak_gb": 78.72572612762451, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9229792625000073, |
| "input_tokens_per_second": 199555.69115370748 |
| }, |
| { |
| "microbatch": 86, |
| "mode": "compiled", |
| "microbatch_seconds": 0.8818629400000191, |
| "peak_gb": 84.78368663787842, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9349941874999956, |
| "input_tokens_per_second": 199722.64624250587 |
| }, |
| { |
| "microbatch": 90, |
| "mode": "compiled", |
| "microbatch_seconds": 0.9259720020000373, |
| "peak_gb": 88.6924238204956, |
| "fits_budget": false, |
| "estimated_update_seconds": 0.9259720020000373, |
| "input_tokens_per_second": 199055.69455867045 |
| } |
| ], |
| "failures": [ |
| { |
| "microbatch": 64, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 72, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 76, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 80, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 86, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 90, |
| "mode": "eager", |
| "reason": "oom" |
| } |
| ], |
| "selection_policy": "max_batch", |
| "budget_gb": 89.6964394569397, |
| "reserve_gb": 2.1791439056396484, |
| "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time.", |
| "legacy_seed_verified": false |
| } |
