{
  "assumptions": "H100 SXM BF16 dense peak 989 TFLOP/s planning scenario; MFU .2/.35/.5 not measured; price $2/$3/$5 per GPU-hour are editable hypothetical inputs, not vendor quotes; excludes retries/people/tax/storage/power",
  "architectures": {
    "B_custom_120B_class": {
      "layers": 49,
      "hidden_size": 3072,
      "expert_intermediate": 2048,
      "experts": 128,
      "selected": 4,
      "shared": 1,
      "heads": 24,
      "kv_heads": 8,
      "head_dim": 128,
      "vocab": 131072,
      "attention_parameters": 1233125376,
      "expert_parameters": 119304880128,
      "router_parameters": 19267584,
      "norm_parameters": 304128,
      "tied_embedding_parameters": 402653184,
      "total_parameters": 120960230400,
      "active_parameters_upper_convention": 6279570432,
      "active_convention": "Counts full tied embedding/output matrix, all attention, router and selected/shared FFNs; actual sparse embedding lookup uses fewer elements",
      "kv_cache_GB_batch1": {
        "32768": 6.576668672,
        "65536": 13.153337344,
        "131072": 26.306674688,
        "262144": 52.613349376
      }
    },
    "C_custom_larger_MoE": {
      "layers": 80,
      "hidden_size": 4096,
      "expert_intermediate": 1536,
      "experts": 160,
      "selected": 4,
      "shared": 1,
      "heads": 32,
      "kv_heads": 8,
      "head_dim": 128,
      "vocab": 131072,
      "attention_parameters": 3355443200,
      "expert_parameters": 243101859840,
      "router_parameters": 52428800,
      "norm_parameters": 659456,
      "tied_embedding_parameters": 536870912,
      "total_parameters": 247047262208,
      "active_parameters_upper_convention": 11495149568,
      "active_convention": "Counts full tied embedding/output matrix, all attention, router and selected/shared FFNs; actual sparse embedding lookup uses fewer elements",
      "kv_cache_GB_batch1": {
        "32768": 10.73741824,
        "65536": 21.47483648,
        "131072": 42.94967296,
        "262144": 85.89934592
      }
    }
  },
  "estimates": {
    "A_dense_120B": {
      "low_cost": {
        "total_parameters": 120000000000.0,
        "active_parameters": 120000000000.0,
        "tokens": 2400000000000.0,
        "gpus": 1024,
        "peak_tflops": 989,
        "mfu": 0.5,
        "gpu_hour_usd": 2,
        "training_flops": 1.728e+24,
        "hours": 947.9271991911022,
        "gpu_hours": 970677.4519716887,
        "compute_usd": 1941354.9039433773,
        "weight_GB": {
          "bf16": 240.0,
          "fp8": 120.0,
          "int8": 120.0,
          "int4_ideal": 60.0
        },
        "adam_training_state_GB": 1920.0,
        "checkpoint_weights_master_moments_GB": 1680.0,
        "token_storage_uint32_GB": 9600.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "expected": {
        "total_parameters": 120000000000.0,
        "active_parameters": 120000000000.0,
        "tokens": 2400000000000.0,
        "gpus": 1024,
        "peak_tflops": 989,
        "mfu": 0.35,
        "gpu_hour_usd": 3,
        "training_flops": 1.728e+24,
        "hours": 1354.181713130146,
        "gpu_hours": 1386682.0742452694,
        "compute_usd": 4160046.222735808,
        "weight_GB": {
          "bf16": 240.0,
          "fp8": 120.0,
          "int8": 120.0,
          "int4_ideal": 60.0
        },
        "adam_training_state_GB": 1920.0,
        "checkpoint_weights_master_moments_GB": 1680.0,
        "token_storage_uint32_GB": 9600.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "high_cost": {
        "total_parameters": 120000000000.0,
        "active_parameters": 120000000000.0,
        "tokens": 2400000000000.0,
        "gpus": 1024,
        "peak_tflops": 989,
        "mfu": 0.2,
        "gpu_hour_usd": 5,
        "training_flops": 1.728e+24,
        "hours": 2369.8179979777556,
        "gpu_hours": 2426693.6299292217,
        "compute_usd": 12133468.149646109,
        "weight_GB": {
          "bf16": 240.0,
          "fp8": 120.0,
          "int8": 120.0,
          "int4_ideal": 60.0
        },
        "adam_training_state_GB": 1920.0,
        "checkpoint_weights_master_moments_GB": 1680.0,
        "token_storage_uint32_GB": 9600.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      }
    },
    "B_custom_120B_class": {
      "low_cost": {
        "total_parameters": 120960230400,
        "active_parameters": 6279570432,
        "tokens": 2400000000000.0,
        "gpus": 256,
        "peak_tflops": 989,
        "mfu": 0.5,
        "gpu_hour_usd": 2,
        "training_flops": 9.04258142208e+22,
        "hours": 198.41918705763396,
        "gpu_hours": 50795.311886754294,
        "compute_usd": 101590.62377350859,
        "weight_GB": {
          "bf16": 241.9204608,
          "fp8": 120.9602304,
          "int8": 120.9602304,
          "int4_ideal": 60.4801152
        },
        "adam_training_state_GB": 1935.3636864,
        "checkpoint_weights_master_moments_GB": 1693.4432256,
        "token_storage_uint32_GB": 9600.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "expected": {
        "total_parameters": 120960230400,
        "active_parameters": 6279570432,
        "tokens": 2400000000000.0,
        "gpus": 256,
        "peak_tflops": 989,
        "mfu": 0.35,
        "gpu_hour_usd": 3,
        "training_flops": 9.04258142208e+22,
        "hours": 283.4559815109057,
        "gpu_hours": 72564.73126679186,
        "compute_usd": 217694.19380037556,
        "weight_GB": {
          "bf16": 241.9204608,
          "fp8": 120.9602304,
          "int8": 120.9602304,
          "int4_ideal": 60.4801152
        },
        "adam_training_state_GB": 1935.3636864,
        "checkpoint_weights_master_moments_GB": 1693.4432256,
        "token_storage_uint32_GB": 9600.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "high_cost": {
        "total_parameters": 120960230400,
        "active_parameters": 6279570432,
        "tokens": 2400000000000.0,
        "gpus": 256,
        "peak_tflops": 989,
        "mfu": 0.2,
        "gpu_hour_usd": 5,
        "training_flops": 9.04258142208e+22,
        "hours": 496.0479676440849,
        "gpu_hours": 126988.27971688574,
        "compute_usd": 634941.3985844287,
        "weight_GB": {
          "bf16": 241.9204608,
          "fp8": 120.9602304,
          "int8": 120.9602304,
          "int4_ideal": 60.4801152
        },
        "adam_training_state_GB": 1935.3636864,
        "checkpoint_weights_master_moments_GB": 1693.4432256,
        "token_storage_uint32_GB": 9600.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      }
    },
    "C_custom_larger_MoE": {
      "low_cost": {
        "total_parameters": 247047262208,
        "active_parameters": 11495149568,
        "tokens": 3000000000000.0,
        "gpus": 512,
        "peak_tflops": 989,
        "mfu": 0.5,
        "gpu_hour_usd": 2,
        "training_flops": 2.06912692224e+23,
        "hours": 227.01176946410516,
        "gpu_hours": 116230.02596562184,
        "compute_usd": 232460.05193124368,
        "weight_GB": {
          "bf16": 494.094524416,
          "fp8": 247.047262208,
          "int8": 247.047262208,
          "int4_ideal": 123.523631104
        },
        "adam_training_state_GB": 3952.756195328,
        "checkpoint_weights_master_moments_GB": 3458.661670912,
        "token_storage_uint32_GB": 12000.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "expected": {
        "total_parameters": 247047262208,
        "active_parameters": 11495149568,
        "tokens": 3000000000000.0,
        "gpus": 512,
        "peak_tflops": 989,
        "mfu": 0.35,
        "gpu_hour_usd": 3,
        "training_flops": 2.06912692224e+23,
        "hours": 324.3025278058645,
        "gpu_hours": 166042.89423660262,
        "compute_usd": 498128.6827098079,
        "weight_GB": {
          "bf16": 494.094524416,
          "fp8": 247.047262208,
          "int8": 247.047262208,
          "int4_ideal": 123.523631104
        },
        "adam_training_state_GB": 3952.756195328,
        "checkpoint_weights_master_moments_GB": 3458.661670912,
        "token_storage_uint32_GB": 12000.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "high_cost": {
        "total_parameters": 247047262208,
        "active_parameters": 11495149568,
        "tokens": 3000000000000.0,
        "gpus": 512,
        "peak_tflops": 989,
        "mfu": 0.2,
        "gpu_hour_usd": 5,
        "training_flops": 2.06912692224e+23,
        "hours": 567.5294236602629,
        "gpu_hours": 290575.0649140546,
        "compute_usd": 1452875.324570273,
        "weight_GB": {
          "bf16": 494.094524416,
          "fp8": 247.047262208,
          "int8": 247.047262208,
          "int4_ideal": 123.523631104
        },
        "adam_training_state_GB": 3952.756195328,
        "checkpoint_weights_master_moments_GB": 3458.661670912,
        "token_storage_uint32_GB": 12000.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      }
    },
    "D_30B_continue": {
      "low_cost": {
        "total_parameters": 30500000000.0,
        "active_parameters": 3300000000.0,
        "tokens": 10000000000.0,
        "gpus": 8,
        "peak_tflops": 989,
        "mfu": 0.5,
        "gpu_hour_usd": 2,
        "training_flops": 1.98e+20,
        "hours": 13.902932254802831,
        "gpu_hours": 111.22345803842265,
        "compute_usd": 222.4469160768453,
        "weight_GB": {
          "bf16": 61.0,
          "fp8": 30.5,
          "int8": 30.5,
          "int4_ideal": 15.25
        },
        "adam_training_state_GB": 488.0,
        "checkpoint_weights_master_moments_GB": 427.0,
        "token_storage_uint32_GB": 40.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "expected": {
        "total_parameters": 30500000000.0,
        "active_parameters": 3300000000.0,
        "tokens": 10000000000.0,
        "gpus": 8,
        "peak_tflops": 989,
        "mfu": 0.35,
        "gpu_hour_usd": 3,
        "training_flops": 1.98e+20,
        "hours": 19.861331792575474,
        "gpu_hours": 158.8906543406038,
        "compute_usd": 476.6719630218114,
        "weight_GB": {
          "bf16": 61.0,
          "fp8": 30.5,
          "int8": 30.5,
          "int4_ideal": 15.25
        },
        "adam_training_state_GB": 488.0,
        "checkpoint_weights_master_moments_GB": 427.0,
        "token_storage_uint32_GB": 40.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "high_cost": {
        "total_parameters": 30500000000.0,
        "active_parameters": 3300000000.0,
        "tokens": 10000000000.0,
        "gpus": 8,
        "peak_tflops": 989,
        "mfu": 0.2,
        "gpu_hour_usd": 5,
        "training_flops": 1.98e+20,
        "hours": 34.75733063700708,
        "gpu_hours": 278.05864509605664,
        "compute_usd": 1390.2932254802831,
        "weight_GB": {
          "bf16": 61.0,
          "fp8": 30.5,
          "int8": 30.5,
          "int4_ideal": 15.25
        },
        "adam_training_state_GB": 488.0,
        "checkpoint_weights_master_moments_GB": 427.0,
        "token_storage_uint32_GB": 40.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      }
    },
    "E_4B_continue": {
      "low_cost": {
        "total_parameters": 4000000000.0,
        "active_parameters": 4000000000.0,
        "tokens": 1000000000.0,
        "gpus": 8,
        "peak_tflops": 989,
        "mfu": 0.5,
        "gpu_hour_usd": 2,
        "training_flops": 2.4e+19,
        "hours": 1.6852039096730704,
        "gpu_hours": 13.481631277384563,
        "compute_usd": 26.963262554769127,
        "weight_GB": {
          "bf16": 8.0,
          "fp8": 4.0,
          "int8": 4.0,
          "int4_ideal": 2.0
        },
        "adam_training_state_GB": 64.0,
        "checkpoint_weights_master_moments_GB": 56.0,
        "token_storage_uint32_GB": 4.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "expected": {
        "total_parameters": 4000000000.0,
        "active_parameters": 4000000000.0,
        "tokens": 1000000000.0,
        "gpus": 8,
        "peak_tflops": 989,
        "mfu": 0.35,
        "gpu_hour_usd": 3,
        "training_flops": 2.4e+19,
        "hours": 2.407434156675815,
        "gpu_hours": 19.25947325340652,
        "compute_usd": 57.778419760219556,
        "weight_GB": {
          "bf16": 8.0,
          "fp8": 4.0,
          "int8": 4.0,
          "int4_ideal": 2.0
        },
        "adam_training_state_GB": 64.0,
        "checkpoint_weights_master_moments_GB": 56.0,
        "token_storage_uint32_GB": 4.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      },
      "high_cost": {
        "total_parameters": 4000000000.0,
        "active_parameters": 4000000000.0,
        "tokens": 1000000000.0,
        "gpus": 8,
        "peak_tflops": 989,
        "mfu": 0.2,
        "gpu_hour_usd": 5,
        "training_flops": 2.4e+19,
        "hours": 4.213009774182676,
        "gpu_hours": 33.704078193461406,
        "compute_usd": 168.52039096730704,
        "weight_GB": {
          "bf16": 8.0,
          "fp8": 4.0,
          "int8": 4.0,
          "int4_ideal": 2.0
        },
        "adam_training_state_GB": 64.0,
        "checkpoint_weights_master_moments_GB": 56.0,
        "token_storage_uint32_GB": 4.0,
        "limitations": "6NT omits attention quadratic term, routing, communication and recomputation; quantization excludes scales/metadata; costs illustrative"
      }
    }
  },
  "topologies": [
    {
      "world": 8,
      "tp": 1,
      "pp": 1,
      "cp": 1,
      "dp": 8,
      "ep": 1,
      "expert_data_parallel": 8,
      "note": "EP is a subgroup of DP here, not another world-size multiplier; framework-specific mapping must be validated"
    },
    {
      "world": 64,
      "tp": 2,
      "pp": 2,
      "cp": 1,
      "dp": 16,
      "ep": 8,
      "expert_data_parallel": 2,
      "note": "EP is a subgroup of DP here, not another world-size multiplier; framework-specific mapping must be validated"
    },
    {
      "world": 1024,
      "tp": 8,
      "pp": 8,
      "cp": 2,
      "dp": 8,
      "ep": 1,
      "expert_data_parallel": 8,
      "note": "EP is a subgroup of DP here, not another world-size multiplier; framework-specific mapping must be validated"
    }
  ],
  "storage_formula": "3 retained checkpoints * checkpoint_GB + 2 replicated copies of token shards + raw corpus + intermediate dedup data; add validation/holdouts and free-space headroom"
}