{
  "schema": "rivetcoder-fast-fp8-serving",
  "schema_version": 1,
  "created_at": "2026-08-27T23:51:49+09:00",
  "hardware": {
    "gpu": "NVIDIA GeForce RTX 5070 Ti",
    "compute_capability": [12, 0],
    "vram_mib": 16303
  },
  "software": {
    "platform": "Windows 11",
    "python": "3.12.10",
    "torch": "2.12.0+cu130",
    "torchao": "0.15.0",
    "transformers": "5.16.1",
    "triton": "3.5.1",
    "compiler": "MSVC 19.44.35222"
  },
  "implementation": {
    "backend": "Triton grouped FP8",
    "packed_layers": 30,
    "packed_experts": 480,
    "direct_fp8_linears": 196,
    "top_k": 4,
    "expert_gemms_per_layer": 2,
    "gate_up_shape": [16, 4096, 2048],
    "down_shape": [16, 2048, 2048],
    "activation_scaling": "dynamic per routed expert",
    "weight_scaling": "checkpoint-preserving per projection",
    "kernel_reference": "kernels-community/finegrained-fp8",
    "kernel_license": "Apache-2.0"
  },
  "full_model_parity": {
    "baseline_seconds": 4.8937,
    "fast_seconds": 0.3044,
    "speedup": 16.08,
    "mean_absolute_logit_error": 0.0,
    "maximum_absolute_logit_error": 0.0,
    "top1_equal": true,
    "repeat_maximum_absolute_error": 0.0,
    "resident_vram_gib": 8.428,
    "packing_peak_vram_gib": 14.053
  },
  "microbatch_forward": [
    {"batch": 1, "seconds": 0.3374, "sequences_per_second": 2.9635},
    {"batch": 4, "seconds": 0.3162, "sequences_per_second": 12.6488},
    {"batch": 8, "seconds": 0.3404, "sequences_per_second": 23.5045},
    {"batch": 16, "seconds": 0.3094, "sequences_per_second": 51.7069}
  ],
  "server": {
    "protocol": "OpenAI-compatible chat completions",
    "batching": "queue-to-completion fixed microbatch",
    "default_max_batch_size": 16,
    "default_batch_wait_ms": 3.0,
    "streaming": false
  }
}