{ "checkpoint": "seekr-org/Inkling-2-Small-FP8-ROCm", "selected_variant": "both", "activation_override": { "expert_fp8_activation_scale_method": "power_of_two" }, "weights": { "values": 587068342272, "squared_error": 4.6084034011073527e-07, "squared_original": 169743711.65203604, "changed_values": 6987660, "max_absolute_error": 1.9073486328125e-06, "underflow_values": 870110, "relative_rms_error": 5.210487964523546e-08, "exact_value_fraction": 0.9999880973653374 }, "verification": { "expert_tensors_sampled": 96, "all_row_scales_positive_finite_power_of_two": true, "saved_samples_match_independent_cpu_reference": true, "nonexpert_tensors_bitwise_unchanged": 1418, "mtp_bitwise_unchanged": true }, "settings": { "gpus": "8 x MI300X", "max_model_len": 262144, "max_num_batched_tokens": 2048, "max_num_seqs": 8, "prefix_caching": false, "speculative_config": { "method": "dspark", "num_speculative_tokens": 7, "draft_sample_method": "probabilistic" } }, "throughput_workload": { "requests": 24, "domains": [ "math", "programming", "prose" ], "input_tokens": 2016, "output_tokens": 512, "temperature": 1 }, "accuracy_workload": { "gsm8k": 50, "humaneval": 20, "temperature": 0, "reasoning_effort": 0, "max_tokens": 2048, "concurrency": 8, "dataset_sha256": "c649cb2a467c1e1945e87c7b34c23818fb560d6374779043de85521ff5863629" }, "notes": [ "GSM8K raw scores retain a known incorrect official label and one ambiguous problem.", "Weight fidelity does not imply identical outputs. Accuracy samples are limited.", "Long-context comparison uses a common 128-token budget. The initial weight-only v2 trial at 32 tokens recalled the correct code in reasoning but exhausted its budget before a final answer.", "Throughput is output tokens including reasoning, and includes prefill time." ], "validation": { "gpu_kernel_tests": "3 passed", "renderer_tests": "54 passed", "ci_tests": "74 passed", "pre_commit": "passed", "wheel_source_files": 38, "installed_wheel_source_files": 38, "full_docker_build": "not run: no local container engine" }, "variants": { "v1-baseline": { "weight_scale_method": "amax", "activation_scale_method": "amax", "throughput": { "1": { "output_throughput": 244.61615730860646, "mean_ttft_ms": 127.79092920633653, "mean_tpot_ms": 3.844314277146573, "spec_decode_acceptance_length": 3.07525, "duration": 50.2338035851717, "completed": 24 }, "8": { "output_throughput": 820.7450492553916, "mean_ttft_ms": 378.7963572734346, "mean_tpot_ms": 8.024158996918079, "spec_decode_acceptance_length": 3.084545910687406, "duration": 14.97176256030798, "completed": 24 } }, "c1_by_domain": { "math": 315.68463696455007, "programming": 246.26104690128898, "prose": 198.7778069379256 }, "accuracy": { "gsm8k": { "passed": 48, "total": 50 }, "humaneval": { "passed": 20, "total": 20 } }, "prose": { "word_count_180_to_220": true, "Mara": true, "lighthouse": true, "map": true, "dialogue": true, "exact_ending": true }, "long_context": { "prompt_tokens": 261040, "seconds": 25.83570277504623, "output": "739124", "output_budget": 128, "usage": { "prompt_tokens": 261040, "total_tokens": 261046, "completion_tokens": 6, "completion_tokens_details": { "reasoning_tokens": 0 } } } }, "weights": { "weight_scale_method": "power_of_two", "activation_scale_method": "amax", "throughput": { "1": { "output_throughput": 237.7338231242387, "mean_ttft_ms": 128.2839255873114, "mean_tpot_ms": 3.963001193302524, "spec_decode_acceptance_length": 2.9929474708171204, "duration": 51.68805952183902, "completed": 24 }, "8": { "output_throughput": 814.3932974316863, "mean_ttft_ms": 388.1922477545838, "mean_tpot_ms": 8.122070970873981, "spec_decode_acceptance_length": 3.0490827962320277, "duration": 15.088532824069262, "completed": 24 } }, "c1_by_domain": { "math": 321.27982069232934, "programming": 236.86293104924414, "prose": 189.28481497302792 }, "accuracy": { "gsm8k": { "passed": 48, "total": 50 }, "humaneval": { "passed": 20, "total": 20 } }, "prose": { "word_count_180_to_220": false, "Mara": true, "lighthouse": true, "map": true, "dialogue": true, "exact_ending": true }, "long_context": { "prompt_tokens": 261040, "seconds": 26.27843856997788, "output": "739124", "output_budget": 128, "usage": { "prompt_tokens": 261040, "total_tokens": 261158, "completion_tokens": 118, "completion_tokens_details": { "reasoning_tokens": 109 } } }, "vision": { "arithmetic": true, "capital": true, "image_red": true, "image_blue": true, "image_ocr": true, "two_images": true }, "streaming": true }, "both": { "weight_scale_method": "power_of_two", "activation_scale_method": "power_of_two", "throughput": { "1": { "output_throughput": 244.0570869789804, "mean_ttft_ms": 128.37754484886923, "mean_tpot_ms": 3.853745164989997, "spec_decode_acceptance_length": 3.07275, "duration": 50.34887596219778, "completed": 24 }, "8": { "output_throughput": 807.6616718152346, "mean_ttft_ms": 397.71096043599147, "mean_tpot_ms": 8.082313082482854, "spec_decode_acceptance_length": 3.0662680617837568, "duration": 15.21429136581719, "completed": 24 } }, "c1_by_domain": { "math": 331.4881084713469, "programming": 247.23512247454568, "prose": 191.22525058557943 }, "accuracy": { "gsm8k": { "passed": 48, "total": 50 }, "humaneval": { "passed": 20, "total": 20 } }, "prose": { "word_count_180_to_220": true, "Mara": true, "lighthouse": true, "map": true, "dialogue": true, "exact_ending": true }, "long_context": { "prompt_tokens": 261040, "seconds": 26.639021350070834, "output": "739124", "output_budget": 128, "usage": { "prompt_tokens": 261040, "total_tokens": 261149, "completion_tokens": 109, "completion_tokens_details": { "reasoning_tokens": 100 } } }, "vision": { "arithmetic": true, "capital": true, "image_red": true, "image_blue": true, "image_ocr": true, "two_images": true }, "streaming": true } }, "published_activation_scale_method": "power_of_two" }