| { |
| "_comment": "R22 DSpark draft \u2014 Solar-Open2-250B-W4A8. \uacf5\uac1c RadixArk/Inferact Kimi-K3-DSpark \ub808\uc2dc\ud53c \uc815\uc11d.", |
| "_design_rule": [ |
| "\u2605draft \uc544\ud0a4\ud14d\ucc98\ub294 target \uc5d0\uc11c \uc720\ub3c4\ud558\uc9c0 \uc54a\ub294\ub2e4 \u2014 \uacf5\uac1c\ub3c4 target(Kimi hidden 7168) \uacfc \ubb34\uad00\ud558\uac8c", |
| " draft hidden 4096/heads 32/kv 8/FFN 12288 \uc744 \uc37c\ub2e4. \uadf8 \ud615\uc0c1\uc744 \uadf8\ub300\ub85c \ubcf4\uc874\ud55c\ub2e4.", |
| "\u2605target-\uacb0\ud569 \ud544\ub4dc\ub9cc Solar \uc2e4\uce21\uac12\uc73c\ub85c \uad50\uccb4: vocab_size \u00b7 target_hidden_size \u00b7", |
| " target_num_hidden_layers \u00b7 target_layer_ids \u00b7 mask_token_id.", |
| "\u2605target_layer_ids = [4,12,24,36,44] \u2014 Solar gqa_layers=[0,4,8,...,44] \ub300\uc870\ub85c 5/5 \uc804\ubd80", |
| " full-attention \ud655\uc778([[project_spec_draft_aux_tap_must_be_full_attention]]).", |
| " \uc790\ub3d9\uc720\ub3c4(build_target_layer_ids(5,48)=[1,12,23,34,45])\ub294 KDA \uce35\uc744 \ubb3c\uc5b4 \uc4f0\uc9c0 \uc54a\ub294\ub2e4.", |
| "\u2605mask_token_id=4095=<|special_4095|> \u2014 \uc2e4\uc81c \ubbf8\uc0ac\uc6a9 \uc608\uc57d \ud1a0\ud070(added_tokens 4096\uac1c \uc911).", |
| " \u26a0\ufe0f\uad6c dspark_solar_open2.py \uc758 mask_token_id=2 \ub294 EOS \uc7ac\uc0ac\uc6a9 \ubc84\uadf8\uc600\ub2e4." |
| ], |
| "architectures": [ |
| "Qwen3DSparkModel" |
| ], |
| "model_type": "qwen3", |
| "hidden_size": 4096, |
| "intermediate_size": 12288, |
| "num_hidden_layers": 5, |
| "num_attention_heads": 32, |
| "num_key_value_heads": 8, |
| "rms_norm_eps": 1e-05, |
| "max_position_embeddings": 1048576, |
| "vocab_size": 196608, |
| "num_target_layers": 5, |
| "target_hidden_size": 4096, |
| "target_num_hidden_layers": 48, |
| "target_layer_ids": [ |
| 4, |
| 12, |
| 24, |
| 36, |
| 44 |
| ], |
| "mask_token_id": 4095, |
| "_serving_contract": [ |
| "\u2605\uc544\ub798 2\ud544\ub4dc\ub294 **\ud559\uc2b5\uc774 \uc544\ub2c8\ub77c vLLM 0.26.0 \uc11c\ube59**\uc774 \uc694\uad6c\ud55c\ub2e4(\uc18c\uc2a4 \uc2e4\uce21).", |
| "\u2460 dspark_block_size \u2014 speculative.py:1011 \uc774 draft hf_config \uc5d0\uc11c \uc77d\ub294\ub2e4. \uc5c6\uc73c\uba74", |
| " `num_speculative_tokens >= block_size` \uac00\ub4dc\uac00 **\ud1b5\uc9f8\ub85c \uc2a4\ud0b5**\ub418\uace0, \ubbf8\ub2ec \uc2dc vLLM \uc790\uccb4 \uc8fc\uc11d\ub300\ub85c", |
| " 'garbled output' \uc774 \uc870\uc6a9\ud788 \ub09c\ub2e4(accept \uc800\ud558\uac00 \uc544\ub2c8\ub77c \uc624\ub2f5).", |
| "\u2461 dflash_config \u2014 qwen3_dflash.py:356-357 \uc774 drafter_config = eagle_config \u222a dflash_config \ub85c \ub9cc\ub4e4\uace0", |
| " **mask_token_id \ub97c \uc5ec\uae30\uc11c\ub9cc** \uc77d\ub294\ub2e4(top-level \uc544\ub2d8). target_layer_ids \ub3c4 \uc5ec\uae30\uc11c fc \uc785\ub825\ud3ed\uc744 \uc815\ud55c\ub2e4.", |
| " (\uc5c6\uc5b4\ub3c4 num_hidden_layers=5 \ub85c \uc6b0\uc5f0\ud788 \ub9de\uc9c0\ub9cc, draft \uae4a\uc774\ub97c \ubc14\uafb8\uba74 \uc870\uc6a9\ud788 \uae68\uc9c4\ub2e4 \u2192 \uba85\uc2dc)", |
| "\u2605aux tap \ud574\uc11d \uc6b0\uc120\uc21c\uc704(eagle3_utils.py:41-55) \u2014 \uc774 \uc21c\uc11c\uac00 '+1 \uc774\uc911\uc801\uc6a9' \ud568\uc815\uc758 \uc815\uccb4\ub2e4:", |
| " 1) eagle_aux_hidden_state_layer_ids \u2192 **\uadf8\ub300\ub85c**(+1 \uc5c6\uc74c) \u2190 convert_to_hf.py:156 \uc774 \uc4f0\ub294 \ud0a4", |
| " 2) dflash_config.target_layer_ids \u2192 **+1**", |
| " 3) dspark_target_layer_ids \u2192 **+1**", |
| " 4) top-level target_layer_ids \u2192 **+1**", |
| " \u21d2 \ud559\uc2b5\uc774 \uc2e4\uc81c\ub85c \ubb38 vLLM \uce35 = [5,13,25,37,45]. 2/3/4 \uacbd\ub85c\uba74 [4,12,24,36,44] \ub97c \uc8fc\ub294 \uac8c \ub9de\uace0,", |
| " 1 \uacbd\ub85c\uba74 [5,13,25,37,45] \ub97c \uc918\uc57c \ud55c\ub2e4. \uc11e\uc73c\uba74 \ud615\uc0c1\uc774 \ub9de\uc544 crash \uc5c6\uc774 accept \ub9cc \uc8fd\ub294\ub2e4.", |
| " \u21d2 export \ud6c4 \ubc18\ub4dc\uc2dc `gate_export_aux.py` \ub85c **vLLM \uc2e4\uc81c \ud574\uc11d \ud568\uc218**\ub97c \ub3cc\ub824 \ud655\uc815\ud55c\ub2e4." |
| ], |
| "dspark_block_size": 4, |
| "dflash_config": { |
| "target_layer_ids": [ |
| 4, |
| 12, |
| 24, |
| 36, |
| 44 |
| ], |
| "mask_token_id": 4095, |
| "use_aux_hidden_state": true |
| }, |
| "markov_rank": 256, |
| "markov_head_type": "vanilla", |
| "enable_confidence_head": true, |
| "confidence_head_with_markov": true, |
| "tie_word_embeddings": false, |
| "head_dim": 128, |
| "layer_types": [ |
| "full_attention", |
| "full_attention", |
| "full_attention", |
| "full_attention", |
| "full_attention" |
| ], |
| "_v2_changes": [ |
| "max_position_embeddings: 40960 -> 1048576 (target=1048576. d1\uc758 40960\uc740 \uc2e4\ubd80\ud558 crash \uc6d0\uc778(D530))", |
| "rms_norm_eps: 1e-06 -> 1e-05 (target=1e-5. \uacf5\uac1c\ub3c4 target \uac12 \uc0c1\uc18d)", |
| "head_dim: None -> 128 (target=128. \ud604\uc7ac \uc720\ub3c4\uac12\uacfc \ub3d9\uc77c\ud558\ub098 \uc554\ubb35 \uc758\uc874 \uc81c\uac70\ub97c \uc704\ud574 \uba85\uc2dc)", |
| "layer_types: None -> ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention'] (\uacf5\uac1c\uac00 \uba85\uc2dc. \uc554\ubb35 \uae30\ubcf8\uac12(causal=False \uc720\ub3c4) \uc758\uc874 \uc81c\uac70)", |
| "rope: theta 1e6 (YaRN \uc5c6\uc74c) -> theta 10000, rope_scaling \uc5c6\uc74c (Solar target \uc744 \uadf8\ub300\ub85c \ubbf8\ub7ec\ub9c1. \u2605\uacf5\uac1c\uac00 YaRN \uc744 \uc4f4 \uac83\uc740 **Kimi target \uc774 YaRN \uc774\ub77c\uc11c**\uc774\uc9c0 DSpark \uace0\uc720 \uc694\uc18c\uac00 \uc544\ub2c8\ub2e4 \u2014 \uaddc\uce59\uc740 '\ub808\uc2dc\ud53c \ud615\ud0dc \ubcf5\uc0ac'\uac00 \uc544\ub2c8\ub77c 'target \uc0c1\uc18d'\uc774\ub2e4)" |
| ], |
| "_v2_note": [ |
| "d1(2026-08-01) \ub300\ube44 \uc7ac\ud559\uc2b5\uc6a9 v2. \uadfc\uac70 = \uacf5\uac1c \uc815\ubcf8 RadixArk/Kimi-K3-DSpark + Solar target \uc2e4\uce21(D530).", |
| "\uaddc\uce59: draft \uc758 attention \uae30\ud558\u00b7\uc815\uaddc\ud654\u00b7rope \ub294 **target \uc5d0\uc11c \uc0c1\uc18d**\ud55c\ub2e4.", |
| "", |
| "\u2605\u2605\uc77c\ubd80\ub7ec **\ubc14\uafb8\uc9c0 \uc54a\uc740** \uac83 \u2014 \uadfc\uac70\ub97c \ub0a8\uae34\ub2e4:", |
| "\u2460 num_attention_heads 32 \uc720\uc9c0(target/\uacf5\uac1c\ub294 64).", |
| " head_dim 128 \uc744 \uc720\uc9c0\ud55c \ucc44 heads \ub97c 64 \ub85c \uc62c\ub9ac\uba74 q/k/v \ud22c\uc601\uc774 8192 \ud3ed = **draft attention \uc5f0\uc0b0 2\ubc30**.", |
| " \uc6b0\ub9ac binding \uac8c\uc774\ud2b8\ub294 derTPOT(48ms vs 25ms) = **per_step \uc744 \uc904\uc5ec\uc57c \ud558\ub294 \uc0c1\ud669**\uc774\ub77c \uc5ed\ubc29\ud5a5\uc774\ub2e4.", |
| " \uacf5\uac1c\uac00 target heads \ub97c \ub530\ub978 \uac83\uc740 Kimi K3(93\uce35) \ub77c draft \ube44\uc6a9 \ube44\uc911\uc774 \uc791\uae30 \ub54c\ubb38 \u2014 \uc6b0\ub9ac\ub294 48\uce35\uc774\ub2e4.", |
| " \u21d2 **\ubcc4\ub3c4 OFAT \uc73c\ub85c \ubd84\ub9ac**. \uc774\ubc88 \uc7ac\ud559\uc2b5\uc740 rope\u00b7\ucee8\ud14d\uc2a4\ud2b8\u00b7\ubb3c\ub7c9 3\uac1c\uac00 \uc774\ubbf8 \ub3d9\uc2dc\uc5d0 \ubc14\ub010\ub2e4.", |
| "\u2461 intermediate_size 12288 \uc720\uc9c0(\uacf5\uac1c\ub294 2\u00d7hidden). \uac19\uc740 \uc774\uc720 \u2014 \ud55c \ubc88\uc5d0 \ud558\ub098\uc529.", |
| "\u2462 vocab 196608 \uc804\uccb4 \uc720\uc9c0(pruning \uc548 \ud568) \u2014 \uacf5\uac1c\ub3c4 target \uc804\uccb4 vocab. \uc6b4\uc601\uc790 \uacb0\uc815(D534).", |
| "\u2463 DSpark \uace0\uc720 \ud558\uc774\ud37c(block7/anchor512/markov256/loss/confidence) \uc804\ubd80 \uacf5\uac1c\uac12 \uc720\uc9c0.", |
| "", |
| "\u2605\u2605rope \ud0a4 \ud568\uc815(\uc2e4\uce21): TorchSpec `config/utils.py:118-119` \ub294 **`rope_theta`\u00b7`rope_scaling`** \ub9cc \uc77d\ub294\ub2e4. \uacf5\uac1c HF \uce74\ub4dc\uc758 `rope_parameters`(transformers 5.x \uc2e0\ud615\uc2dd)\ub97c \uadf8\ub300\ub85c \uc4f0\uba74 **\uc870\uc6a9\ud788 \ubb34\uc2dc**\ub418\uace0 \uae30\ubcf8\uac12\uc73c\ub85c 10\uc2dc\uac04\uc744 \ub3cc\uac8c \ub41c\ub2e4. \uac19\uc740 \ud30c\uc77c `generate_draft_model_config` \uac00 target \uc758 \ub450 \ud0a4\ub97c draft \ub85c \ubcf5\uc0ac\ud558\ub3c4\ub85d \uc124\uacc4\ub3fc \uc788\ub2e4 = 'target \uc0c1\uc18d' \uaddc\uce59\uc758 \ucf54\ub4dc \uadfc\uac70." |
| ], |
| "rope_theta": 10000.0, |
| "draft_vocab_size": 196608, |
| "_torchspec_version": "0.1.0", |
| "torch_dtype": "bfloat16" |
| } |
