Inkling-2-Small-FP8-ROCm / validation_results.json
BalazsHoranyi's picture
Update to validated v2 power-of-two FP8 weights and activation scales
6eaa9b7 verified
Raw History Blame Contribute Delete
7.69 kB
{
"checkpoint": "seekr-org/Inkling-2-Small-FP8-ROCm",
"selected_variant": "both",
"activation_override": {
"expert_fp8_activation_scale_method": "power_of_two"
},
"weights": {
"values": 587068342272,
"squared_error": 4.6084034011073527e-07,
"squared_original": 169743711.65203604,
"changed_values": 6987660,
"max_absolute_error": 1.9073486328125e-06,
"underflow_values": 870110,
"relative_rms_error": 5.210487964523546e-08,
"exact_value_fraction": 0.9999880973653374
},
"verification": {
"expert_tensors_sampled": 96,
"all_row_scales_positive_finite_power_of_two": true,
"saved_samples_match_independent_cpu_reference": true,
"nonexpert_tensors_bitwise_unchanged": 1418,
"mtp_bitwise_unchanged": true
},
"settings": {
"gpus": "8 x MI300X",
"max_model_len": 262144,
"max_num_batched_tokens": 2048,
"max_num_seqs": 8,
"prefix_caching": false,
"speculative_config": {
"method": "dspark",
"num_speculative_tokens": 7,
"draft_sample_method": "probabilistic"
}
},
"throughput_workload": {
"requests": 24,
"domains": [
"math",
"programming",
"prose"
],
"input_tokens": 2016,
"output_tokens": 512,
"temperature": 1
},
"accuracy_workload": {
"gsm8k": 50,
"humaneval": 20,
"temperature": 0,
"reasoning_effort": 0,
"max_tokens": 2048,
"concurrency": 8,
"dataset_sha256": "c649cb2a467c1e1945e87c7b34c23818fb560d6374779043de85521ff5863629"
},
"notes": [
"GSM8K raw scores retain a known incorrect official label and one ambiguous problem.",
"Weight fidelity does not imply identical outputs. Accuracy samples are limited.",
"Long-context comparison uses a common 128-token budget. The initial weight-only v2 trial at 32 tokens recalled the correct code in reasoning but exhausted its budget before a final answer.",
"Throughput is output tokens including reasoning, and includes prefill time."
],
"validation": {
"gpu_kernel_tests": "3 passed",
"renderer_tests": "54 passed",
"ci_tests": "74 passed",
"pre_commit": "passed",
"wheel_source_files": 38,
"installed_wheel_source_files": 38,
"full_docker_build": "not run: no local container engine"
},
"variants": {
"v1-baseline": {
"weight_scale_method": "amax",
"activation_scale_method": "amax",
"throughput": {
"1": {
"output_throughput": 244.61615730860646,
"mean_ttft_ms": 127.79092920633653,
"mean_tpot_ms": 3.844314277146573,
"spec_decode_acceptance_length": 3.07525,
"duration": 50.2338035851717,
"completed": 24
},
"8": {
"output_throughput": 820.7450492553916,
"mean_ttft_ms": 378.7963572734346,
"mean_tpot_ms": 8.024158996918079,
"spec_decode_acceptance_length": 3.084545910687406,
"duration": 14.97176256030798,
"completed": 24
}
},
"c1_by_domain": {
"math": 315.68463696455007,
"programming": 246.26104690128898,
"prose": 198.7778069379256
},
"accuracy": {
"gsm8k": {
"passed": 48,
"total": 50
},
"humaneval": {
"passed": 20,
"total": 20
}
},
"prose": {
"word_count_180_to_220": true,
"Mara": true,
"lighthouse": true,
"map": true,
"dialogue": true,
"exact_ending": true
},
"long_context": {
"prompt_tokens": 261040,
"seconds": 25.83570277504623,
"output": "739124",
"output_budget": 128,
"usage": {
"prompt_tokens": 261040,
"total_tokens": 261046,
"completion_tokens": 6,
"completion_tokens_details": {
"reasoning_tokens": 0
}
}
}
},
"weights": {
"weight_scale_method": "power_of_two",
"activation_scale_method": "amax",
"throughput": {
"1": {
"output_throughput": 237.7338231242387,
"mean_ttft_ms": 128.2839255873114,
"mean_tpot_ms": 3.963001193302524,
"spec_decode_acceptance_length": 2.9929474708171204,
"duration": 51.68805952183902,
"completed": 24
},
"8": {
"output_throughput": 814.3932974316863,
"mean_ttft_ms": 388.1922477545838,
"mean_tpot_ms": 8.122070970873981,
"spec_decode_acceptance_length": 3.0490827962320277,
"duration": 15.088532824069262,
"completed": 24
}
},
"c1_by_domain": {
"math": 321.27982069232934,
"programming": 236.86293104924414,
"prose": 189.28481497302792
},
"accuracy": {
"gsm8k": {
"passed": 48,
"total": 50
},
"humaneval": {
"passed": 20,
"total": 20
}
},
"prose": {
"word_count_180_to_220": false,
"Mara": true,
"lighthouse": true,
"map": true,
"dialogue": true,
"exact_ending": true
},
"long_context": {
"prompt_tokens": 261040,
"seconds": 26.27843856997788,
"output": "739124",
"output_budget": 128,
"usage": {
"prompt_tokens": 261040,
"total_tokens": 261158,
"completion_tokens": 118,
"completion_tokens_details": {
"reasoning_tokens": 109
}
}
},
"vision": {
"arithmetic": true,
"capital": true,
"image_red": true,
"image_blue": true,
"image_ocr": true,
"two_images": true
},
"streaming": true
},
"both": {
"weight_scale_method": "power_of_two",
"activation_scale_method": "power_of_two",
"throughput": {
"1": {
"output_throughput": 244.0570869789804,
"mean_ttft_ms": 128.37754484886923,
"mean_tpot_ms": 3.853745164989997,
"spec_decode_acceptance_length": 3.07275,
"duration": 50.34887596219778,
"completed": 24
},
"8": {
"output_throughput": 807.6616718152346,
"mean_ttft_ms": 397.71096043599147,
"mean_tpot_ms": 8.082313082482854,
"spec_decode_acceptance_length": 3.0662680617837568,
"duration": 15.21429136581719,
"completed": 24
}
},
"c1_by_domain": {
"math": 331.4881084713469,
"programming": 247.23512247454568,
"prose": 191.22525058557943
},
"accuracy": {
"gsm8k": {
"passed": 48,
"total": 50
},
"humaneval": {
"passed": 20,
"total": 20
}
},
"prose": {
"word_count_180_to_220": true,
"Mara": true,
"lighthouse": true,
"map": true,
"dialogue": true,
"exact_ending": true
},
"long_context": {
"prompt_tokens": 261040,
"seconds": 26.639021350070834,
"output": "739124",
"output_budget": 128,
"usage": {
"prompt_tokens": 261040,
"total_tokens": 261149,
"completion_tokens": 109,
"completion_tokens_details": {
"reasoning_tokens": 100
}
}
},
"vision": {
"arithmetic": true,
"capital": true,
"image_red": true,
"image_blue": true,
"image_ocr": true,
"two_images": true
},
"streaming": true
}
},
"published_activation_scale_method": "power_of_two"
}