Publish final step-5780 weights and acceptance results

README.md

@@ -1,7 +1,7 @@
 ---
 pipeline_tag: text-generation
 base_model: moonshotai/Kimi-K3
-checkpoint_step: 4900
+checkpoint_step: 5780
 tags:
   - speculative-decoding
   - vllm
@@ -18,33 +18,42 @@ trained at `block_size=5`: one forward pass drafts the full five-token block.
 
 ## Training
 
-The draft is trained with [**speculators**](https://github.com/vllm-project/speculators),
-the speculative-decoding library from the [vLLM project](https://github.com/vllm-project),
-against Kimi-K3 hidden states streamed live from **vLLM**.
+The draft completed a four-epoch run with
+[**speculators**](https://github.com/vllm-project/speculators), the speculative-decoding
+library from the [vLLM project](https://github.com/vllm-project), against Kimi-K3 hidden
+states streamed live from **vLLM**.
 
 ## Acceptance
 
 Evaluated against Kimi-K3 on 8×B300 (TP8), drafting 5 tokens with probabilistic
 sampling and block rejection at temperature 1.0 and top-p 0.95. All 1,604 requests
-succeeded.
+succeeded. The aggregate is computed from summed counters, not an average of arm rates.
 
 | Workload | Requests | Acceptance | Mean accepted length (max 6) |
 | --- | ---: | ---: | ---: |
-| GSM8K | 256 | 69.22% | 4.461 |
-| HumanEval | 164 | 63.90% | 4.195 |
-| MBPP | 256 | 56.29% | 3.815 |
-| MATH-500 | 500 | 46.82% | 3.341 |
-| SWE-bench Pro | 128 | 37.51% | 2.875 |
-| MT-Bench | 80 | 33.04% | 2.652 |
-| AIME26 | 30 | 28.56% | 2.428 |
-| AA-LCR (~100K) | 100 | 40.15% | 3.008 |
-| BEAM 100K | 20 | 37.34% | 2.867 |
-| BEAM 500K | 35 | 34.57% | 2.728 |
-| BEAM 1M | 35 | 31.56% | 2.578 |
-| **Aggregate** | **1,604** | **45.63%** | **3.282** |
+| GSM8K | 256 | 71.62% | 4.581 |
+| HumanEval | 164 | 63.74% | 4.187 |
+| MBPP | 256 | 56.38% | 3.819 |
+| MATH-500 | 500 | 47.48% | 3.374 |
+| SWE-bench Pro | 128 | 38.09% | 2.904 |
+| MT-Bench | 80 | 33.19% | 2.659 |
+| AIME26 | 30 | 29.78% | 2.489 |
+| AA-LCR (~100K) | 100 | 41.04% | 3.052 |
+| BEAM 100K | 20 | 37.17% | 2.858 |
+| BEAM 500K | 35 | 30.26% | 2.513 |
+| BEAM 1M | 35 | 31.37% | 2.569 |
+| **Aggregate** | **1,604** | **46.10%** | **3.305** |
 
-**The full 1M context window is supported through YaRN**, validated on every conversation
-of the [BEAM](https://huggingface.co/datasets/Mohammadta/BEAM) 100K, 500K, and 1M splits above.
+The separate five-shot GSM8K accuracy check scored **96.51% exact match** on all 1,319
+examples with zero request errors.
+
+**The full 1M context window is supported through YaRN**, evaluated on every row of the
+[BEAM](https://huggingface.co/datasets/Mohammadta/BEAM) 100K, 500K, and 1M splits above.
+The fixed 1M acceptance corpus left-truncates oldest turns in 14 of 35 over-limit source
+conversations to keep prompt plus output within 1,048,576 tokens; its longest measured
+prompt is 1,039,871 tokens. Two successful completions were flagged as degenerate by the
+client (BEAM 500K conversation 4 and BEAM 1M conversation 18). Full counters,
+per-position rates, and quality flags are in `benchmark_results.json`.
 
 ## Serving
 
@@ -61,4 +70,4 @@ vllm serve moonshotai/Kimi-K3 \
     "draft_sample_method": "probabilistic",
     "rejection_sample_method": "block"
   }'
-```
+```

benchmark_results.json

@@ -0,0 +1,193 @@
+{
+  "schema_version": 1,
+  "checkpoint": {
+    "global_step": 5780,
+    "weights_sha256": "4182214cd0e9898c56ffb220dfababf1e7d8d9029b0a08f4ecc12be0ea6ded72",
+    "weights_bytes": 5707153178,
+    "config_sha256": "5fd7249ad8dbd8fd23947dacfe5cc49383b69f8083c3b88bc2c3971d670d616f",
+    "config_unchanged_from_hub_revision": "066131ff38a6925f745768b2ed3ee5ae05f40144"
+  },
+  "evaluation": {
+    "run": "k3_dspark_block5_step5780_final_20260825_031135",
+    "slurm_job_id": 9799,
+    "slurm_state": "COMPLETED",
+    "slurm_exit_code": "0:0",
+    "pipeline_duration_s": 9454.0,
+    "target_model": "moonshotai/Kimi-K3",
+    "target_revision": "cdd2e49a2c1cf8d4713b513955e415ed75405a72",
+    "hardware": "8x NVIDIA B300 (TP8 across 2 nodes)",
+    "num_speculative_tokens": 5,
+    "draft_sample_method": "probabilistic",
+    "rejection_sample_method": "block",
+    "temperature": 1.0,
+    "top_p": 0.95,
+    "max_model_len": 1048576,
+    "requests": 1604,
+    "failed_requests": 0
+  },
+  "aggregate": {
+    "accepted_tokens": 823451,
+    "draft_tokens": 1786225,
+    "drafts": 357245,
+    "acceptance_rate": 0.4610007137958544,
+    "mean_accepted_length": 3.305003568979272
+  },
+  "arms": [
+    {
+      "label": "accept_mtbench",
+      "requests": 80,
+      "failed_requests": 0,
+      "acceptance_rate": 0.33188085625047614,
+      "mean_accepted_length": 2.659404281252381,
+      "accepted_tokens": 43566,
+      "draft_tokens": 131270,
+      "drafts": 26254,
+      "per_position_acceptance": [0.6965414793936162, 0.4423325969376095, 0.26594042812523805, 0.16386074502932887, 0.09072903176658795]
+    },
+    {
+      "label": "accept_gsm8k",
+      "requests": 256,
+      "failed_requests": 0,
+      "acceptance_rate": 0.7161949021264611,
+      "mean_accepted_length": 4.580974510632306,
+      "accepted_tokens": 50857,
+      "draft_tokens": 71010,
+      "drafts": 14202,
+      "per_position_acceptance": [0.9294465568229827, 0.8285452753133361, 0.7165892127869314, 0.6059709900014083, 0.5004224757076469]
+    },
+    {
+      "label": "accept_math500",
+      "requests": 500,
+      "failed_requests": 0,
+      "acceptance_rate": 0.4747567821928452,
+      "mean_accepted_length": 3.373783910964226,
+      "accepted_tokens": 216914,
+      "draft_tokens": 456895,
+      "drafts": 91379,
+      "per_position_acceptance": [0.7832105844887776, 0.5922038980509745, 0.4407796101949025, 0.3229735497214896, 0.23461626850808173]
+    },
+    {
+      "label": "accept_humaneval",
+      "requests": 164,
+      "failed_requests": 0,
+      "acceptance_rate": 0.6373504652193177,
+      "mean_accepted_length": 4.186752326096588,
+      "accepted_tokens": 100695,
+      "draft_tokens": 157990,
+      "drafts": 31598,
+      "per_position_acceptance": [0.8817646686499145, 0.7481169694284449, 0.6228558769542376, 0.5135135135135135, 0.4205012975504779]
+    },
+    {
+      "label": "accept_mbpp",
+      "requests": 256,
+      "failed_requests": 0,
+      "acceptance_rate": 0.5638449795636973,
+      "mean_accepted_length": 3.8192248978184864,
+      "accepted_tokens": 171750,
+      "draft_tokens": 304605,
+      "drafts": 60921,
+      "per_position_acceptance": [0.8627895142889972, 0.6984291131137046, 0.5382052165919797, 0.4110569425977906, 0.30874411122601403]
+    },
+    {
+      "label": "accept_aime26",
+      "requests": 30,
+      "failed_requests": 0,
+      "acceptance_rate": 0.2977662636729994,
+      "mean_accepted_length": 2.488831318364997,
+      "accepted_tokens": 51722,
+      "draft_tokens": 173700,
+      "drafts": 34740,
+      "per_position_acceptance": [0.6478411053540587, 0.39833045480713875, 0.2360391479562464, 0.13457109959700633, 0.07204951065054692]
+    },
+    {
+      "label": "accept_swebench_pro",
+      "requests": 128,
+      "failed_requests": 0,
+      "acceptance_rate": 0.3808832202890003,
+      "mean_accepted_length": 2.904416101445001,
+      "accepted_tokens": 154989,
+      "draft_tokens": 406920,
+      "drafts": 81384,
+      "per_position_acceptance": [0.7383023690160228, 0.5001351617025459, 0.32659982306104396, 0.20828418362331663, 0.13109456404207215]
+    },
+    {
+      "label": "accept_aa_lcr",
+      "requests": 100,
+      "failed_requests": 0,
+      "acceptance_rate": 0.41044765075841655,
+      "mean_accepted_length": 3.052238253792083,
+      "accepted_tokens": 27736,
+      "draft_tokens": 67575,
+      "drafts": 13515,
+      "per_position_acceptance": [0.7897891231964483, 0.5861635220125786, 0.4015538290788013, 0.20621531631520532, 0.0685164631890492]
+    },
+    {
+      "label": "accept_beam_100k",
+      "requests": 20,
+      "failed_requests": 0,
+      "acceptance_rate": 0.3716840536512668,
+      "mean_accepted_length": 2.8584202682563338,
+      "accepted_tokens": 1247,
+      "draft_tokens": 3355,
+      "drafts": 671,
+      "per_position_acceptance": [0.789865871833085, 0.5380029806259314, 0.338301043219076, 0.15350223546944858, 0.038748137108792845]
+    },
+    {
+      "label": "accept_beam_500k",
+      "requests": 35,
+      "failed_requests": 0,
+      "acceptance_rate": 0.30257186081694404,
+      "mean_accepted_length": 2.51285930408472,
+      "accepted_tokens": 2000,
+      "draft_tokens": 6610,
+      "drafts": 1322,
+      "per_position_acceptance": [0.7291981845688351, 0.4523449319213313, 0.23373676248108927, 0.07564296520423601, 0.02193645990922844]
+    },
+    {
+      "label": "accept_beam_1m",
+      "requests": 35,
+      "failed_requests": 0,
+      "acceptance_rate": 0.31374106433677523,
+      "mean_accepted_length": 2.568705321683876,
+      "accepted_tokens": 1975,
+      "draft_tokens": 6295,
+      "drafts": 1259,
+      "per_position_acceptance": [0.7410643367752184, 0.4908657664813344, 0.2454328832406672, 0.07307386814932486, 0.018268467037331215]
+    }
+  ],
+  "accuracy": {
+    "task": "gsm8k",
+    "n": 1319,
+    "request_errors": 0,
+    "exact_match_strict": 0.9651,
+    "exact_match_flexible": 0.9651,
+    "shot_num": 5,
+    "temperature": 0.7,
+    "top_p": 0.95
+  },
+  "long_context_quality_flags": {
+    "beam_100k": {
+      "source_conversations_left_truncated": 0,
+      "degenerate_outputs": 0,
+      "unparsed_outputs": 0,
+      "no_output_requests": 0,
+      "maximum_prompt_tokens": 186117
+    },
+    "beam_500k": {
+      "source_conversations_left_truncated": 0,
+      "degenerate_outputs": 1,
+      "degenerate_conversation_ids": ["4"],
+      "unparsed_outputs": 0,
+      "no_output_requests": 0,
+      "maximum_prompt_tokens": 717727
+    },
+    "beam_1m": {
+      "source_conversations_left_truncated": 14,
+      "degenerate_outputs": 1,
+      "degenerate_conversation_ids": ["18"],
+      "unparsed_outputs": 0,
+      "no_output_requests": 0,
+      "maximum_prompt_tokens": 1039871
+    }
+  }
+}

export_manifest.json

@@ -1,12 +1,12 @@
 {
   "format_version": 1,
-  "created_at_utc": "2026-08-24T15:24:25.225600+00:00",
-  "source_checkpoint": "/home/inf-zixi/speculator-training/k3_dspark_3layers/export_staging/global4900",
+  "created_at_utc": "2026-08-25T03:09:45.012409+00:00",
+  "source_checkpoint": "/home/inf-zixi/speculator-training/k3_dspark_3layers/export_staging/global5780_final",
   "target_model": "/home/inf-zixi/.cache/huggingface/hub/models--moonshotai--Kimi-K3/snapshots/cdd2e49a2c1cf8d4713b513955e415ed75405a72",
   "source_config_sha256": "e906971eab49e77693bcb0c0e9c4fb73c19afa2a048d6f8c9d2c62d1ec436f35",
-  "source_weights_sha256": "5c6470d6de9f7e1d13213909a154fff6cd8dc29b8e3023657bee1d70da2f3cf0",
+  "source_weights_sha256": "0dea41a55dff6baafd2d00c035be14039a80dadaf81647fd8d63c6f8bbf7137a",
   "output_config_sha256": "d46a5c8ae79c8f85f90d554d8df9e4c23579ea2795102930a6505704b298223c",
-  "output_weights_sha256": "0d40f8be1d2c91f82296e3481cab02873893213601531499fd1ffaf0aa7f9993",
+  "output_weights_sha256": "4182214cd0e9898c56ffb220dfababf1e7d8d9029b0a08f4ecc12be0ea6ded72",
   "source_weight_bytes": 8055963450,
   "output_weight_bytes": 5707153178,
   "tensor_count": 44,
@@ -20,7 +20,7 @@
   ],
   "training_state": {
     "epoch": 3,
-    "local_step": 4496,
-    "global_step": 4900
+    "local_step": 0,
+    "global_step": 5780
   }
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d40f8be1d2c91f82296e3481cab02873893213601531499fd1ffaf0aa7f9993
+oid sha256:4182214cd0e9898c56ffb220dfababf1e7d8d9029b0a08f4ecc12be0ea6ded72
 size 5707153178

overlay_manifest.json

@@ -1,11 +1,11 @@
 {
   "format_version": 1,
-  "created_at_utc": "2026-08-24T15:24:42.547804+00:00",
+  "created_at_utc": "2026-08-25T03:10:02.617008+00:00",
   "kind": "config-only serving overlay",
-  "plain_export": "/home/inf-zixi/speculator-training/evals/hf_checkpoints/k3_dspark_3layers_step4900_plain",
+  "plain_export": "/home/inf-zixi/speculator-training/evals/hf_checkpoints/k3_dspark_3layers_step5780_plain",
   "weights_hard_linked": true,
   "weights_same_inode": true,
-  "weights_sha256": "0d40f8be1d2c91f82296e3481cab02873893213601531499fd1ffaf0aa7f9993",
+  "weights_sha256": "4182214cd0e9898c56ffb220dfababf1e7d8d9029b0a08f4ecc12be0ea6ded72",
   "weights_bytes": 5707153178,
   "changed_fields": {
     "max_position_embeddings": [
@@ -18,7 +18,7 @@
         "rope_type": "default"
       },
       {
-        "rope_type": "deepseek_yarn",
+        "rope_type": "yarn",
         "rope_theta": 1000000.0,
         "factor": 32.0,
         "original_max_position_embeddings": 32768,