| { | |
| "assumptions": "H100 SXM BF16 dense peak 989 TFLOP/s planning scenario; MFU .2/.35/.5 not measured; price $2/$3/$5 per GPU-hour are editable hypothetical inputs, not vendor quotes; excludes retries/people/tax/storage/power", | |
| "architectures": { | |
| "B_custom_120B_class": { | |
| "layers": 49, | |
| "hidden_size": 3072, | |
| "expert_intermediate": 2048, | |
| "experts": 128, | |
| "selected": 4, | |
| "shared": 1, | |
| "heads": 24, | |
| "kv_heads": 8, | |
| "head_dim": 128, | |
| "vocab": 131072, | |
| "attention_parameters": 1233125376, | |
| "expert_parameters": 119304880128, | |
| "router_parameters": 19267584, | |
| "norm_parameters": 304128, | |
| "tied_embedding_parameters": 402653184, | |
| "total_parameters": 120960230400, | |
| "active_parameters_upper_convention": 6279570432, | |
| "active_convention": "Counts full tied embedding/output matrix, all attention, router and selected/shared FFNs; actual sparse embedding lookup uses fewer elements", | |
| "kv_cache_GB_batch1": { | |
| "32768": 6.576668672, | |
| "65536": 13.153337344, | |
| "131072": 26.306674688, | |
| "262144": 52.613349376 | |
| } | |
| }, | |
| "C_custom_larger_MoE": { | |
| "layers": 80, | |
| "hidden_size": 4096, | |
| "expert_intermediate": 1536, | |
| "experts": 160, | |
| "selected": 4, | |
| "shared": 1, | |
| "heads": 32, | |
| "kv_heads": 8, | |
| "head_dim": 128, | |
| "vocab": 131072, | |
| "attention_parameters": 3355443200, | |
| "expert_parameters": 243101859840, | |
| "router_parameters": 52428800, | |
| "norm_parameters": 659456, | |
| "tied_embedding_parameters": 536870912, | |
| "total_parameters": 247047262208, | |
| "active_parameters_upper_convention": 11495149568, | |
| "active_convention": "Counts full tied embedding/output matrix, all attention, router and selected/shared FFNs; actual sparse embedding lookup uses fewer elements", | |
| "kv_cache_GB_batch1": { | |
| "32768": 10.73741824, | |
| "65536": 21.47483648, | |
| "131072": 42.94967296, | |
| "262144": 85.89934592 | |
| } | |
| } | |
| }, | |
| "estimates": { | |
| "A_dense_120B": { | |
| "low_cost": { | |
| "total_parameters": 120000000000.0, | |
| "active_parameters": 120000000000.0, | |
| "tokens": 2400000000000.0, | |
| "gpus": 1024, | |
| "peak_tflops": 989, | |
| "mfu": 0.5, | |
| "gpu_hour_usd": 2, | |
| "training_flops": 1.728e+24, | |
| "hours": 947.9271991911022, | |
| "gpu_hours": 970677.4519716887, | |
| "compute_usd": 1941354.9039433773, | |
| "weight_GB": { | |
| "bf16": 240.0, | |
| "fp8": 120.0, | |
| "int8": 120.0, | |
| "int4_ideal": 60.0 | |
| }, | |
| "adam_training_state_GB": 1920.0, | |
| "checkpoint_weights_master_moments_GB": 1680.0, | |
| "token_storage_uint32_GB": 9600.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "expected": { | |
| "total_parameters": 120000000000.0, | |
| "active_parameters": 120000000000.0, | |
| "tokens": 2400000000000.0, | |
| "gpus": 1024, | |
| "peak_tflops": 989, | |
| "mfu": 0.35, | |
| "gpu_hour_usd": 3, | |
| "training_flops": 1.728e+24, | |
| "hours": 1354.181713130146, | |
| "gpu_hours": 1386682.0742452694, | |
| "compute_usd": 4160046.222735808, | |
| "weight_GB": { | |
| "bf16": 240.0, | |
| "fp8": 120.0, | |
| "int8": 120.0, | |
| "int4_ideal": 60.0 | |
| }, | |
| "adam_training_state_GB": 1920.0, | |
| "checkpoint_weights_master_moments_GB": 1680.0, | |
| "token_storage_uint32_GB": 9600.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "high_cost": { | |
| "total_parameters": 120000000000.0, | |
| "active_parameters": 120000000000.0, | |
| "tokens": 2400000000000.0, | |
| "gpus": 1024, | |
| "peak_tflops": 989, | |
| "mfu": 0.2, | |
| "gpu_hour_usd": 5, | |
| "training_flops": 1.728e+24, | |
| "hours": 2369.8179979777556, | |
| "gpu_hours": 2426693.6299292217, | |
| "compute_usd": 12133468.149646109, | |
| "weight_GB": { | |
| "bf16": 240.0, | |
| "fp8": 120.0, | |
| "int8": 120.0, | |
| "int4_ideal": 60.0 | |
| }, | |
| "adam_training_state_GB": 1920.0, | |
| "checkpoint_weights_master_moments_GB": 1680.0, | |
| "token_storage_uint32_GB": 9600.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| } | |
| }, | |
| "B_custom_120B_class": { | |
| "low_cost": { | |
| "total_parameters": 120960230400, | |
| "active_parameters": 6279570432, | |
| "tokens": 2400000000000.0, | |
| "gpus": 256, | |
| "peak_tflops": 989, | |
| "mfu": 0.5, | |
| "gpu_hour_usd": 2, | |
| "training_flops": 9.04258142208e+22, | |
| "hours": 198.41918705763396, | |
| "gpu_hours": 50795.311886754294, | |
| "compute_usd": 101590.62377350859, | |
| "weight_GB": { | |
| "bf16": 241.9204608, | |
| "fp8": 120.9602304, | |
| "int8": 120.9602304, | |
| "int4_ideal": 60.4801152 | |
| }, | |
| "adam_training_state_GB": 1935.3636864, | |
| "checkpoint_weights_master_moments_GB": 1693.4432256, | |
| "token_storage_uint32_GB": 9600.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "expected": { | |
| "total_parameters": 120960230400, | |
| "active_parameters": 6279570432, | |
| "tokens": 2400000000000.0, | |
| "gpus": 256, | |
| "peak_tflops": 989, | |
| "mfu": 0.35, | |
| "gpu_hour_usd": 3, | |
| "training_flops": 9.04258142208e+22, | |
| "hours": 283.4559815109057, | |
| "gpu_hours": 72564.73126679186, | |
| "compute_usd": 217694.19380037556, | |
| "weight_GB": { | |
| "bf16": 241.9204608, | |
| "fp8": 120.9602304, | |
| "int8": 120.9602304, | |
| "int4_ideal": 60.4801152 | |
| }, | |
| "adam_training_state_GB": 1935.3636864, | |
| "checkpoint_weights_master_moments_GB": 1693.4432256, | |
| "token_storage_uint32_GB": 9600.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "high_cost": { | |
| "total_parameters": 120960230400, | |
| "active_parameters": 6279570432, | |
| "tokens": 2400000000000.0, | |
| "gpus": 256, | |
| "peak_tflops": 989, | |
| "mfu": 0.2, | |
| "gpu_hour_usd": 5, | |
| "training_flops": 9.04258142208e+22, | |
| "hours": 496.0479676440849, | |
| "gpu_hours": 126988.27971688574, | |
| "compute_usd": 634941.3985844287, | |
| "weight_GB": { | |
| "bf16": 241.9204608, | |
| "fp8": 120.9602304, | |
| "int8": 120.9602304, | |
| "int4_ideal": 60.4801152 | |
| }, | |
| "adam_training_state_GB": 1935.3636864, | |
| "checkpoint_weights_master_moments_GB": 1693.4432256, | |
| "token_storage_uint32_GB": 9600.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| } | |
| }, | |
| "C_custom_larger_MoE": { | |
| "low_cost": { | |
| "total_parameters": 247047262208, | |
| "active_parameters": 11495149568, | |
| "tokens": 3000000000000.0, | |
| "gpus": 512, | |
| "peak_tflops": 989, | |
| "mfu": 0.5, | |
| "gpu_hour_usd": 2, | |
| "training_flops": 2.06912692224e+23, | |
| "hours": 227.01176946410516, | |
| "gpu_hours": 116230.02596562184, | |
| "compute_usd": 232460.05193124368, | |
| "weight_GB": { | |
| "bf16": 494.094524416, | |
| "fp8": 247.047262208, | |
| "int8": 247.047262208, | |
| "int4_ideal": 123.523631104 | |
| }, | |
| "adam_training_state_GB": 3952.756195328, | |
| "checkpoint_weights_master_moments_GB": 3458.661670912, | |
| "token_storage_uint32_GB": 12000.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "expected": { | |
| "total_parameters": 247047262208, | |
| "active_parameters": 11495149568, | |
| "tokens": 3000000000000.0, | |
| "gpus": 512, | |
| "peak_tflops": 989, | |
| "mfu": 0.35, | |
| "gpu_hour_usd": 3, | |
| "training_flops": 2.06912692224e+23, | |
| "hours": 324.3025278058645, | |
| "gpu_hours": 166042.89423660262, | |
| "compute_usd": 498128.6827098079, | |
| "weight_GB": { | |
| "bf16": 494.094524416, | |
| "fp8": 247.047262208, | |
| "int8": 247.047262208, | |
| "int4_ideal": 123.523631104 | |
| }, | |
| "adam_training_state_GB": 3952.756195328, | |
| "checkpoint_weights_master_moments_GB": 3458.661670912, | |
| "token_storage_uint32_GB": 12000.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "high_cost": { | |
| "total_parameters": 247047262208, | |
| "active_parameters": 11495149568, | |
| "tokens": 3000000000000.0, | |
| "gpus": 512, | |
| "peak_tflops": 989, | |
| "mfu": 0.2, | |
| "gpu_hour_usd": 5, | |
| "training_flops": 2.06912692224e+23, | |
| "hours": 567.5294236602629, | |
| "gpu_hours": 290575.0649140546, | |
| "compute_usd": 1452875.324570273, | |
| "weight_GB": { | |
| "bf16": 494.094524416, | |
| "fp8": 247.047262208, | |
| "int8": 247.047262208, | |
| "int4_ideal": 123.523631104 | |
| }, | |
| "adam_training_state_GB": 3952.756195328, | |
| "checkpoint_weights_master_moments_GB": 3458.661670912, | |
| "token_storage_uint32_GB": 12000.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| } | |
| }, | |
| "D_30B_continue": { | |
| "low_cost": { | |
| "total_parameters": 30500000000.0, | |
| "active_parameters": 3300000000.0, | |
| "tokens": 10000000000.0, | |
| "gpus": 8, | |
| "peak_tflops": 989, | |
| "mfu": 0.5, | |
| "gpu_hour_usd": 2, | |
| "training_flops": 1.98e+20, | |
| "hours": 13.902932254802831, | |
| "gpu_hours": 111.22345803842265, | |
| "compute_usd": 222.4469160768453, | |
| "weight_GB": { | |
| "bf16": 61.0, | |
| "fp8": 30.5, | |
| "int8": 30.5, | |
| "int4_ideal": 15.25 | |
| }, | |
| "adam_training_state_GB": 488.0, | |
| "checkpoint_weights_master_moments_GB": 427.0, | |
| "token_storage_uint32_GB": 40.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "expected": { | |
| "total_parameters": 30500000000.0, | |
| "active_parameters": 3300000000.0, | |
| "tokens": 10000000000.0, | |
| "gpus": 8, | |
| "peak_tflops": 989, | |
| "mfu": 0.35, | |
| "gpu_hour_usd": 3, | |
| "training_flops": 1.98e+20, | |
| "hours": 19.861331792575474, | |
| "gpu_hours": 158.8906543406038, | |
| "compute_usd": 476.6719630218114, | |
| "weight_GB": { | |
| "bf16": 61.0, | |
| "fp8": 30.5, | |
| "int8": 30.5, | |
| "int4_ideal": 15.25 | |
| }, | |
| "adam_training_state_GB": 488.0, | |
| "checkpoint_weights_master_moments_GB": 427.0, | |
| "token_storage_uint32_GB": 40.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "high_cost": { | |
| "total_parameters": 30500000000.0, | |
| "active_parameters": 3300000000.0, | |
| "tokens": 10000000000.0, | |
| "gpus": 8, | |
| "peak_tflops": 989, | |
| "mfu": 0.2, | |
| "gpu_hour_usd": 5, | |
| "training_flops": 1.98e+20, | |
| "hours": 34.75733063700708, | |
| "gpu_hours": 278.05864509605664, | |
| "compute_usd": 1390.2932254802831, | |
| "weight_GB": { | |
| "bf16": 61.0, | |
| "fp8": 30.5, | |
| "int8": 30.5, | |
| "int4_ideal": 15.25 | |
| }, | |
| "adam_training_state_GB": 488.0, | |
| "checkpoint_weights_master_moments_GB": 427.0, | |
| "token_storage_uint32_GB": 40.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| } | |
| }, | |
| "E_4B_continue": { | |
| "low_cost": { | |
| "total_parameters": 4000000000.0, | |
| "active_parameters": 4000000000.0, | |
| "tokens": 1000000000.0, | |
| "gpus": 8, | |
| "peak_tflops": 989, | |
| "mfu": 0.5, | |
| "gpu_hour_usd": 2, | |
| "training_flops": 2.4e+19, | |
| "hours": 1.6852039096730704, | |
| "gpu_hours": 13.481631277384563, | |
| "compute_usd": 26.963262554769127, | |
| "weight_GB": { | |
| "bf16": 8.0, | |
| "fp8": 4.0, | |
| "int8": 4.0, | |
| "int4_ideal": 2.0 | |
| }, | |
| "adam_training_state_GB": 64.0, | |
| "checkpoint_weights_master_moments_GB": 56.0, | |
| "token_storage_uint32_GB": 4.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "expected": { | |
| "total_parameters": 4000000000.0, | |
| "active_parameters": 4000000000.0, | |
| "tokens": 1000000000.0, | |
| "gpus": 8, | |
| "peak_tflops": 989, | |
| "mfu": 0.35, | |
| "gpu_hour_usd": 3, | |
| "training_flops": 2.4e+19, | |
| "hours": 2.407434156675815, | |
| "gpu_hours": 19.25947325340652, | |
| "compute_usd": 57.778419760219556, | |
| "weight_GB": { | |
| "bf16": 8.0, | |
| "fp8": 4.0, | |
| "int8": 4.0, | |
| "int4_ideal": 2.0 | |
| }, | |
| "adam_training_state_GB": 64.0, | |
| "checkpoint_weights_master_moments_GB": 56.0, | |
| "token_storage_uint32_GB": 4.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| }, | |
| "high_cost": { | |
| "total_parameters": 4000000000.0, | |
| "active_parameters": 4000000000.0, | |
| "tokens": 1000000000.0, | |
| "gpus": 8, | |
| "peak_tflops": 989, | |
| "mfu": 0.2, | |
| "gpu_hour_usd": 5, | |
| "training_flops": 2.4e+19, | |
| "hours": 4.213009774182676, | |
| "gpu_hours": 33.704078193461406, | |
| "compute_usd": 168.52039096730704, | |
| "weight_GB": { | |
| "bf16": 8.0, | |
| "fp8": 4.0, | |
| "int8": 4.0, | |
| "int4_ideal": 2.0 | |
| }, | |
| "adam_training_state_GB": 64.0, | |
| "checkpoint_weights_master_moments_GB": 56.0, | |
| "token_storage_uint32_GB": 4.0, | |
| "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative" | |
| } | |
| } | |
| }, | |
| "topologies": [ | |
| { | |
| "world": 8, | |
| "tp": 1, | |
| "pp": 1, | |
| "cp": 1, | |
| "dp": 8, | |
| "ep": 1, | |
| "expert_data_parallel": 8, | |
| "note": "EP is a subgroup of DP here, not another world-size multiplier; framework-specific mapping must be validated" | |
| }, | |
| { | |
| "world": 64, | |
| "tp": 2, | |
| "pp": 2, | |
| "cp": 1, | |
| "dp": 16, | |
| "ep": 8, | |
| "expert_data_parallel": 2, | |
| "note": "EP is a subgroup of DP here, not another world-size multiplier; framework-specific mapping must be validated" | |
| }, | |
| { | |
| "world": 1024, | |
| "tp": 8, | |
| "pp": 8, | |
| "cp": 2, | |
| "dp": 8, | |
| "ep": 1, | |
| "expert_data_parallel": 8, | |
| "note": "EP is a subgroup of DP here, not another world-size multiplier; framework-specific mapping must be validated" | |
| } | |
| ], | |
| "storage_formula": "3 retained checkpoints * checkpoint_GB + 2 replicated copies of token shards + raw corpus + intermediate dedup data; add validation/holdouts and free-space headroom" | |
| } |