Hugging Face's logo

cveavy
/
train

Safetensors
Qwen3-TTS
Model card Files
xet
 Community
cveavy commited on
Commit
df3722f
·
verified ·
1 Parent(s): c099e1e

Upload folder using huggingface_hub

Browse files
Files changed (15) hide show
  1. chute_config.yml +27 -0
  2. config.json +163 -0
  3. generation_config.json +12 -0
  4. merge_log.json +4325 -0
  5. merges.txt +0 -0
  6. miner.py +160 -0
  7. model.safetensors +3 -0
  8. preprocessor_config.json +6 -0
  9. speech_tokenizer/config.json +94 -0
  10. speech_tokenizer/configuration.json +1 -0
  11. speech_tokenizer/model.safetensors +3 -0
  12. speech_tokenizer/preprocessor_config.json +10 -0
  13. tokenizer_config.json +316 -0
  14. vocab.json +0 -0
  15. vocence_config.yaml +22 -0
chute_config.yml ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Image + node + Chute for Vocence deploy. Required in the HF repo at build time.
2
+ # Keep deps minimal — anything imported by miner.py must not be in the validator's
3
+ # banned list (requests/urllib/httpx/aiohttp/socket/huggingface_hub/importlib/torch.hub).
4
+ # huggingface_hub is intentionally NOT installed so an accidental future import
5
+ # in miner.py fails fast at runtime instead of silently being available.
6
+
7
+ Image:
8
+ from_base: parachutes/base-python:3.12.9
9
+ run_command:
10
+ - pip install torch torchaudio transformers==4.57.3 accelerate pyyaml soundfile
11
+ - pip install -U qwen-tts
12
+ set_workdir: /app
13
+
14
+ NodeSelector:
15
+ gpu_count: 1
16
+ min_vram_gb_per_gpu: 24
17
+ include: ["pro_6000"]
18
+ exclude: []
19
+
20
+ Chute:
21
+ tagline: vocence qwen3-tts miner
22
+ readme: vocence chute serving qwen3-tts via miner.py (weights pinned in repo)
23
+ shutdown_after_seconds: 86400
24
+ concurrency: 1
25
+ max_instances: 1
26
+ scaling_threshold: 0.5
27
+ tee: true
config.json ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3TTSForConditionalGeneration"
4
+ ],
5
+ "assistant_token_id": 77091,
6
+ "im_end_token_id": 151645,
7
+ "im_start_token_id": 151644,
8
+ "tts_bos_token_id": 151672,
9
+ "tts_eos_token_id": 151673,
10
+ "tts_pad_token_id": 151671,
11
+ "model_type": "qwen3_tts",
12
+ "tokenizer_type": "qwen3_tts_tokenizer_12hz",
13
+ "tts_model_size": "1b7",
14
+ "tts_model_type": "voice_design",
15
+ "talker_config": {
16
+ "attention_bias": false,
17
+ "attention_dropout": 0,
18
+ "code_predictor_config": {
19
+ "_name_or_path": "",
20
+ "add_cross_attention": false,
21
+ "architectures": null,
22
+ "attention_bias": false,
23
+ "attention_dropout": 0,
24
+ "bad_words_ids": null,
25
+ "begin_suppress_tokens": null,
26
+ "bos_token_id": null,
27
+ "chunk_size_feed_forward": 0,
28
+ "cross_attention_hidden_size": null,
29
+ "decoder_start_token_id": null,
30
+ "diversity_penalty": 0.0,
31
+ "do_sample": false,
32
+ "early_stopping": false,
33
+ "encoder_no_repeat_ngram_size": 0,
34
+ "eos_token_id": null,
35
+ "exponential_decay_length_penalty": null,
36
+ "finetuning_task": null,
37
+ "forced_bos_token_id": null,
38
+ "forced_eos_token_id": null,
39
+ "head_dim": 128,
40
+ "hidden_act": "silu",
41
+ "hidden_size": 1024,
42
+ "id2label": {
43
+ "0": "LABEL_0",
44
+ "1": "LABEL_1"
45
+ },
46
+ "initializer_range": 0.02,
47
+ "intermediate_size": 3072,
48
+ "is_decoder": false,
49
+ "is_encoder_decoder": false,
50
+ "label2id": {
51
+ "LABEL_0": 0,
52
+ "LABEL_1": 1
53
+ },
54
+ "layer_types": [
55
+ "full_attention",
56
+ "full_attention",
57
+ "full_attention",
58
+ "full_attention",
59
+ "full_attention"
60
+ ],
61
+ "length_penalty": 1.0,
62
+ "max_length": 20,
63
+ "max_position_embeddings": 65536,
64
+ "max_window_layers": 28,
65
+ "min_length": 0,
66
+ "model_type": "qwen3_tts_talker_code_predictor",
67
+ "no_repeat_ngram_size": 0,
68
+ "num_attention_heads": 16,
69
+ "num_beam_groups": 1,
70
+ "num_beams": 1,
71
+ "num_code_groups": 16,
72
+ "num_hidden_layers": 5,
73
+ "num_key_value_heads": 8,
74
+ "num_return_sequences": 1,
75
+ "output_attentions": false,
76
+ "output_hidden_states": false,
77
+ "output_scores": false,
78
+ "pad_token_id": null,
79
+ "prefix": null,
80
+ "problem_type": null,
81
+ "pruned_heads": {},
82
+ "remove_invalid_values": false,
83
+ "repetition_penalty": 1.0,
84
+ "return_dict": true,
85
+ "return_dict_in_generate": false,
86
+ "rms_norm_eps": 1e-06,
87
+ "rope_scaling": null,
88
+ "rope_theta": 1000000,
89
+ "sep_token_id": null,
90
+ "sliding_window": null,
91
+ "suppress_tokens": null,
92
+ "task_specific_params": null,
93
+ "temperature": 1.0,
94
+ "tf_legacy_loss": false,
95
+ "tie_encoder_decoder": false,
96
+ "tie_word_embeddings": false,
97
+ "tokenizer_class": null,
98
+ "top_k": 50,
99
+ "top_p": 1.0,
100
+ "dtype": null,
101
+ "torchscript": false,
102
+ "typical_p": 1.0,
103
+ "use_bfloat16": false,
104
+ "use_cache": true,
105
+ "use_sliding_window": false,
106
+ "vocab_size": 2048
107
+ },
108
+ "codec_bos_id": 2149,
109
+ "codec_eos_token_id": 2150,
110
+ "codec_think_id": 2154,
111
+ "codec_language_id": {
112
+ "chinese": 2055,
113
+ "english": 2050,
114
+ "german": 2053,
115
+ "italian": 2070,
116
+ "portuguese": 2071,
117
+ "spanish": 2054,
118
+ "japanese": 2058,
119
+ "korean": 2064,
120
+ "french": 2061,
121
+ "russian": 2069
122
+ },
123
+ "codec_nothink_id": 2155,
124
+ "codec_pad_id": 2148,
125
+ "codec_think_bos_id": 2156,
126
+ "codec_think_eos_id": 2157,
127
+ "spk_id": {
128
+ },
129
+ "spk_is_dialect": {
130
+ },
131
+ "head_dim": 128,
132
+ "hidden_act": "silu",
133
+ "hidden_size": 2048,
134
+ "initializer_range": 0.02,
135
+ "intermediate_size": 6144,
136
+ "max_position_embeddings": 32768,
137
+ "model_type": "qwen3_tts_talker",
138
+ "num_attention_heads": 16,
139
+ "num_code_groups": 16,
140
+ "num_hidden_layers": 28,
141
+ "num_key_value_heads": 8,
142
+ "position_id_per_seconds": 13,
143
+ "rms_norm_eps": 1e-06,
144
+ "rope_scaling": {
145
+ "interleaved": true,
146
+ "mrope_section": [
147
+ 24,
148
+ 20,
149
+ 20
150
+ ],
151
+ "rope_type": "default",
152
+ "type": "default"
153
+ },
154
+ "rope_theta": 1000000,
155
+ "sliding_window": null,
156
+ "text_hidden_size": 2048,
157
+ "text_vocab_size": 151936,
158
+ "use_cache": true,
159
+ "use_sliding_window": false,
160
+ "vocab_size": 3072
161
+ },
162
+ "transformers_version": "4.57.3"
163
+ }
generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "repetition_penalty": 1.05,
4
+ "temperature": 0.9,
5
+ "top_p": 1.0,
6
+ "top_k": 50,
7
+ "subtalker_dosample": true,
8
+ "subtalker_temperature": 0.9,
9
+ "subtalker_top_p": 1.0,
10
+ "subtalker_top_k": 50,
11
+ "max_new_tokens": 8192
12
+ }
merge_log.json ADDED
@@ -0,0 +1,4325 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_a": "./base_model1",
3
+ "model_b": "./base_model4",
4
+ "output": "./merged_model_v6",
5
+ "total_keys": 404,
6
+ "alpha_config": {
7
+ "lm_head": 1.0,
8
+ "layernorm": 1.0,
9
+ "embedding": 1.0,
10
+ "other_frozen": 1.0,
11
+ "cp_mlp": 0.75,
12
+ "cp_attn": 0.62,
13
+ "talker_mlp": 0.35,
14
+ "talker_attn": 0.35
15
+ },
16
+ "layers": [
17
+ {
18
+ "key": "talker.code_predictor.lm_head.0.weight",
19
+ "group": "lm_head",
20
+ "alpha_a": 1.0,
21
+ "alpha_b": 0.0,
22
+ "shape": [
23
+ 2048,
24
+ 1024
25
+ ],
26
+ "dtype": "torch.bfloat16"
27
+ },
28
+ {
29
+ "key": "talker.code_predictor.lm_head.1.weight",
30
+ "group": "lm_head",
31
+ "alpha_a": 1.0,
32
+ "alpha_b": 0.0,
33
+ "shape": [
34
+ 2048,
35
+ 1024
36
+ ],
37
+ "dtype": "torch.bfloat16"
38
+ },
39
+ {
40
+ "key": "talker.code_predictor.lm_head.10.weight",
41
+ "group": "lm_head",
42
+ "alpha_a": 1.0,
43
+ "alpha_b": 0.0,
44
+ "shape": [
45
+ 2048,
46
+ 1024
47
+ ],
48
+ "dtype": "torch.bfloat16"
49
+ },
50
+ {
51
+ "key": "talker.code_predictor.lm_head.11.weight",
52
+ "group": "lm_head",
53
+ "alpha_a": 1.0,
54
+ "alpha_b": 0.0,
55
+ "shape": [
56
+ 2048,
57
+ 1024
58
+ ],
59
+ "dtype": "torch.bfloat16"
60
+ },
61
+ {
62
+ "key": "talker.code_predictor.lm_head.12.weight",
63
+ "group": "lm_head",
64
+ "alpha_a": 1.0,
65
+ "alpha_b": 0.0,
66
+ "shape": [
67
+ 2048,
68
+ 1024
69
+ ],
70
+ "dtype": "torch.bfloat16"
71
+ },
72
+ {
73
+ "key": "talker.code_predictor.lm_head.13.weight",
74
+ "group": "lm_head",
75
+ "alpha_a": 1.0,
76
+ "alpha_b": 0.0,
77
+ "shape": [
78
+ 2048,
79
+ 1024
80
+ ],
81
+ "dtype": "torch.bfloat16"
82
+ },
83
+ {
84
+ "key": "talker.code_predictor.lm_head.14.weight",
85
+ "group": "lm_head",
86
+ "alpha_a": 1.0,
87
+ "alpha_b": 0.0,
88
+ "shape": [
89
+ 2048,
90
+ 1024
91
+ ],
92
+ "dtype": "torch.bfloat16"
93
+ },
94
+ {
95
+ "key": "talker.code_predictor.lm_head.2.weight",
96
+ "group": "lm_head",
97
+ "alpha_a": 1.0,
98
+ "alpha_b": 0.0,
99
+ "shape": [
100
+ 2048,
101
+ 1024
102
+ ],
103
+ "dtype": "torch.bfloat16"
104
+ },
105
+ {
106
+ "key": "talker.code_predictor.lm_head.3.weight",
107
+ "group": "lm_head",
108
+ "alpha_a": 1.0,
109
+ "alpha_b": 0.0,
110
+ "shape": [
111
+ 2048,
112
+ 1024
113
+ ],
114
+ "dtype": "torch.bfloat16"
115
+ },
116
+ {
117
+ "key": "talker.code_predictor.lm_head.4.weight",
118
+ "group": "lm_head",
119
+ "alpha_a": 1.0,
120
+ "alpha_b": 0.0,
121
+ "shape": [
122
+ 2048,
123
+ 1024
124
+ ],
125
+ "dtype": "torch.bfloat16"
126
+ },
127
+ {
128
+ "key": "talker.code_predictor.lm_head.5.weight",
129
+ "group": "lm_head",
130
+ "alpha_a": 1.0,
131
+ "alpha_b": 0.0,
132
+ "shape": [
133
+ 2048,
134
+ 1024
135
+ ],
136
+ "dtype": "torch.bfloat16"
137
+ },
138
+ {
139
+ "key": "talker.code_predictor.lm_head.6.weight",
140
+ "group": "lm_head",
141
+ "alpha_a": 1.0,
142
+ "alpha_b": 0.0,
143
+ "shape": [
144
+ 2048,
145
+ 1024
146
+ ],
147
+ "dtype": "torch.bfloat16"
148
+ },
149
+ {
150
+ "key": "talker.code_predictor.lm_head.7.weight",
151
+ "group": "lm_head",
152
+ "alpha_a": 1.0,
153
+ "alpha_b": 0.0,
154
+ "shape": [
155
+ 2048,
156
+ 1024
157
+ ],
158
+ "dtype": "torch.bfloat16"
159
+ },
160
+ {
161
+ "key": "talker.code_predictor.lm_head.8.weight",
162
+ "group": "lm_head",
163
+ "alpha_a": 1.0,
164
+ "alpha_b": 0.0,
165
+ "shape": [
166
+ 2048,
167
+ 1024
168
+ ],
169
+ "dtype": "torch.bfloat16"
170
+ },
171
+ {
172
+ "key": "talker.code_predictor.lm_head.9.weight",
173
+ "group": "lm_head",
174
+ "alpha_a": 1.0,
175
+ "alpha_b": 0.0,
176
+ "shape": [
177
+ 2048,
178
+ 1024
179
+ ],
180
+ "dtype": "torch.bfloat16"
181
+ },
182
+ {
183
+ "key": "talker.code_predictor.model.codec_embedding.0.weight",
184
+ "group": "other_frozen",
185
+ "alpha_a": 1.0,
186
+ "alpha_b": 0.0,
187
+ "shape": [
188
+ 2048,
189
+ 2048
190
+ ],
191
+ "dtype": "torch.bfloat16"
192
+ },
193
+ {
194
+ "key": "talker.code_predictor.model.codec_embedding.1.weight",
195
+ "group": "other_frozen",
196
+ "alpha_a": 1.0,
197
+ "alpha_b": 0.0,
198
+ "shape": [
199
+ 2048,
200
+ 2048
201
+ ],
202
+ "dtype": "torch.bfloat16"
203
+ },
204
+ {
205
+ "key": "talker.code_predictor.model.codec_embedding.10.weight",
206
+ "group": "other_frozen",
207
+ "alpha_a": 1.0,
208
+ "alpha_b": 0.0,
209
+ "shape": [
210
+ 2048,
211
+ 2048
212
+ ],
213
+ "dtype": "torch.bfloat16"
214
+ },
215
+ {
216
+ "key": "talker.code_predictor.model.codec_embedding.11.weight",
217
+ "group": "other_frozen",
218
+ "alpha_a": 1.0,
219
+ "alpha_b": 0.0,
220
+ "shape": [
221
+ 2048,
222
+ 2048
223
+ ],
224
+ "dtype": "torch.bfloat16"
225
+ },
226
+ {
227
+ "key": "talker.code_predictor.model.codec_embedding.12.weight",
228
+ "group": "other_frozen",
229
+ "alpha_a": 1.0,
230
+ "alpha_b": 0.0,
231
+ "shape": [
232
+ 2048,
233
+ 2048
234
+ ],
235
+ "dtype": "torch.bfloat16"
236
+ },
237
+ {
238
+ "key": "talker.code_predictor.model.codec_embedding.13.weight",
239
+ "group": "other_frozen",
240
+ "alpha_a": 1.0,
241
+ "alpha_b": 0.0,
242
+ "shape": [
243
+ 2048,
244
+ 2048
245
+ ],
246
+ "dtype": "torch.bfloat16"
247
+ },
248
+ {
249
+ "key": "talker.code_predictor.model.codec_embedding.14.weight",
250
+ "group": "other_frozen",
251
+ "alpha_a": 1.0,
252
+ "alpha_b": 0.0,
253
+ "shape": [
254
+ 2048,
255
+ 2048
256
+ ],
257
+ "dtype": "torch.bfloat16"
258
+ },
259
+ {
260
+ "key": "talker.code_predictor.model.codec_embedding.2.weight",
261
+ "group": "other_frozen",
262
+ "alpha_a": 1.0,
263
+ "alpha_b": 0.0,
264
+ "shape": [
265
+ 2048,
266
+ 2048
267
+ ],
268
+ "dtype": "torch.bfloat16"
269
+ },
270
+ {
271
+ "key": "talker.code_predictor.model.codec_embedding.3.weight",
272
+ "group": "other_frozen",
273
+ "alpha_a": 1.0,
274
+ "alpha_b": 0.0,
275
+ "shape": [
276
+ 2048,
277
+ 2048
278
+ ],
279
+ "dtype": "torch.bfloat16"
280
+ },
281
+ {
282
+ "key": "talker.code_predictor.model.codec_embedding.4.weight",
283
+ "group": "other_frozen",
284
+ "alpha_a": 1.0,
285
+ "alpha_b": 0.0,
286
+ "shape": [
287
+ 2048,
288
+ 2048
289
+ ],
290
+ "dtype": "torch.bfloat16"
291
+ },
292
+ {
293
+ "key": "talker.code_predictor.model.codec_embedding.5.weight",
294
+ "group": "other_frozen",
295
+ "alpha_a": 1.0,
296
+ "alpha_b": 0.0,
297
+ "shape": [
298
+ 2048,
299
+ 2048
300
+ ],
301
+ "dtype": "torch.bfloat16"
302
+ },
303
+ {
304
+ "key": "talker.code_predictor.model.codec_embedding.6.weight",
305
+ "group": "other_frozen",
306
+ "alpha_a": 1.0,
307
+ "alpha_b": 0.0,
308
+ "shape": [
309
+ 2048,
310
+ 2048
311
+ ],
312
+ "dtype": "torch.bfloat16"
313
+ },
314
+ {
315
+ "key": "talker.code_predictor.model.codec_embedding.7.weight",
316
+ "group": "other_frozen",
317
+ "alpha_a": 1.0,
318
+ "alpha_b": 0.0,
319
+ "shape": [
320
+ 2048,
321
+ 2048
322
+ ],
323
+ "dtype": "torch.bfloat16"
324
+ },
325
+ {
326
+ "key": "talker.code_predictor.model.codec_embedding.8.weight",
327
+ "group": "other_frozen",
328
+ "alpha_a": 1.0,
329
+ "alpha_b": 0.0,
330
+ "shape": [
331
+ 2048,
332
+ 2048
333
+ ],
334
+ "dtype": "torch.bfloat16"
335
+ },
336
+ {
337
+ "key": "talker.code_predictor.model.codec_embedding.9.weight",
338
+ "group": "other_frozen",
339
+ "alpha_a": 1.0,
340
+ "alpha_b": 0.0,
341
+ "shape": [
342
+ 2048,
343
+ 2048
344
+ ],
345
+ "dtype": "torch.bfloat16"
346
+ },
347
+ {
348
+ "key": "talker.code_predictor.model.layers.0.input_layernorm.weight",
349
+ "group": "layernorm",
350
+ "alpha_a": 1.0,
351
+ "alpha_b": 0.0,
352
+ "shape": [
353
+ 1024
354
+ ],
355
+ "dtype": "torch.bfloat16"
356
+ },
357
+ {
358
+ "key": "talker.code_predictor.model.layers.0.mlp.down_proj.weight",
359
+ "group": "cp_mlp",
360
+ "alpha_a": 0.75,
361
+ "alpha_b": 0.25,
362
+ "shape": [
363
+ 1024,
364
+ 3072
365
+ ],
366
+ "dtype": "torch.bfloat16"
367
+ },
368
+ {
369
+ "key": "talker.code_predictor.model.layers.0.mlp.gate_proj.weight",
370
+ "group": "cp_mlp",
371
+ "alpha_a": 0.75,
372
+ "alpha_b": 0.25,
373
+ "shape": [
374
+ 3072,
375
+ 1024
376
+ ],
377
+ "dtype": "torch.bfloat16"
378
+ },
379
+ {
380
+ "key": "talker.code_predictor.model.layers.0.mlp.up_proj.weight",
381
+ "group": "cp_mlp",
382
+ "alpha_a": 0.75,
383
+ "alpha_b": 0.25,
384
+ "shape": [
385
+ 3072,
386
+ 1024
387
+ ],
388
+ "dtype": "torch.bfloat16"
389
+ },
390
+ {
391
+ "key": "talker.code_predictor.model.layers.0.post_attention_layernorm.weight",
392
+ "group": "layernorm",
393
+ "alpha_a": 1.0,
394
+ "alpha_b": 0.0,
395
+ "shape": [
396
+ 1024
397
+ ],
398
+ "dtype": "torch.bfloat16"
399
+ },
400
+ {
401
+ "key": "talker.code_predictor.model.layers.0.self_attn.k_norm.weight",
402
+ "group": "layernorm",
403
+ "alpha_a": 1.0,
404
+ "alpha_b": 0.0,
405
+ "shape": [
406
+ 128
407
+ ],
408
+ "dtype": "torch.bfloat16"
409
+ },
410
+ {
411
+ "key": "talker.code_predictor.model.layers.0.self_attn.k_proj.weight",
412
+ "group": "cp_attn",
413
+ "alpha_a": 0.62,
414
+ "alpha_b": 0.38,
415
+ "shape": [
416
+ 1024,
417
+ 1024
418
+ ],
419
+ "dtype": "torch.bfloat16"
420
+ },
421
+ {
422
+ "key": "talker.code_predictor.model.layers.0.self_attn.o_proj.weight",
423
+ "group": "cp_attn",
424
+ "alpha_a": 0.62,
425
+ "alpha_b": 0.38,
426
+ "shape": [
427
+ 1024,
428
+ 2048
429
+ ],
430
+ "dtype": "torch.bfloat16"
431
+ },
432
+ {
433
+ "key": "talker.code_predictor.model.layers.0.self_attn.q_norm.weight",
434
+ "group": "layernorm",
435
+ "alpha_a": 1.0,
436
+ "alpha_b": 0.0,
437
+ "shape": [
438
+ 128
439
+ ],
440
+ "dtype": "torch.bfloat16"
441
+ },
442
+ {
443
+ "key": "talker.code_predictor.model.layers.0.self_attn.q_proj.weight",
444
+ "group": "cp_attn",
445
+ "alpha_a": 0.62,
446
+ "alpha_b": 0.38,
447
+ "shape": [
448
+ 2048,
449
+ 1024
450
+ ],
451
+ "dtype": "torch.bfloat16"
452
+ },
453
+ {
454
+ "key": "talker.code_predictor.model.layers.0.self_attn.v_proj.weight",
455
+ "group": "cp_attn",
456
+ "alpha_a": 0.62,
457
+ "alpha_b": 0.38,
458
+ "shape": [
459
+ 1024,
460
+ 1024
461
+ ],
462
+ "dtype": "torch.bfloat16"
463
+ },
464
+ {
465
+ "key": "talker.code_predictor.model.layers.1.input_layernorm.weight",
466
+ "group": "layernorm",
467
+ "alpha_a": 1.0,
468
+ "alpha_b": 0.0,
469
+ "shape": [
470
+ 1024
471
+ ],
472
+ "dtype": "torch.bfloat16"
473
+ },
474
+ {
475
+ "key": "talker.code_predictor.model.layers.1.mlp.down_proj.weight",
476
+ "group": "cp_mlp",
477
+ "alpha_a": 0.75,
478
+ "alpha_b": 0.25,
479
+ "shape": [
480
+ 1024,
481
+ 3072
482
+ ],
483
+ "dtype": "torch.bfloat16"
484
+ },
485
+ {
486
+ "key": "talker.code_predictor.model.layers.1.mlp.gate_proj.weight",
487
+ "group": "cp_mlp",
488
+ "alpha_a": 0.75,
489
+ "alpha_b": 0.25,
490
+ "shape": [
491
+ 3072,
492
+ 1024
493
+ ],
494
+ "dtype": "torch.bfloat16"
495
+ },
496
+ {
497
+ "key": "talker.code_predictor.model.layers.1.mlp.up_proj.weight",
498
+ "group": "cp_mlp",
499
+ "alpha_a": 0.75,
500
+ "alpha_b": 0.25,
501
+ "shape": [
502
+ 3072,
503
+ 1024
504
+ ],
505
+ "dtype": "torch.bfloat16"
506
+ },
507
+ {
508
+ "key": "talker.code_predictor.model.layers.1.post_attention_layernorm.weight",
509
+ "group": "layernorm",
510
+ "alpha_a": 1.0,
511
+ "alpha_b": 0.0,
512
+ "shape": [
513
+ 1024
514
+ ],
515
+ "dtype": "torch.bfloat16"
516
+ },
517
+ {
518
+ "key": "talker.code_predictor.model.layers.1.self_attn.k_norm.weight",
519
+ "group": "layernorm",
520
+ "alpha_a": 1.0,
521
+ "alpha_b": 0.0,
522
+ "shape": [
523
+ 128
524
+ ],
525
+ "dtype": "torch.bfloat16"
526
+ },
527
+ {
528
+ "key": "talker.code_predictor.model.layers.1.self_attn.k_proj.weight",
529
+ "group": "cp_attn",
530
+ "alpha_a": 0.62,
531
+ "alpha_b": 0.38,
532
+ "shape": [
533
+ 1024,
534
+ 1024
535
+ ],
536
+ "dtype": "torch.bfloat16"
537
+ },
538
+ {
539
+ "key": "talker.code_predictor.model.layers.1.self_attn.o_proj.weight",
540
+ "group": "cp_attn",
541
+ "alpha_a": 0.62,
542
+ "alpha_b": 0.38,
543
+ "shape": [
544
+ 1024,
545
+ 2048
546
+ ],
547
+ "dtype": "torch.bfloat16"
548
+ },
549
+ {
550
+ "key": "talker.code_predictor.model.layers.1.self_attn.q_norm.weight",
551
+ "group": "layernorm",
552
+ "alpha_a": 1.0,
553
+ "alpha_b": 0.0,
554
+ "shape": [
555
+ 128
556
+ ],
557
+ "dtype": "torch.bfloat16"
558
+ },
559
+ {
560
+ "key": "talker.code_predictor.model.layers.1.self_attn.q_proj.weight",
561
+ "group": "cp_attn",
562
+ "alpha_a": 0.62,
563
+ "alpha_b": 0.38,
564
+ "shape": [
565
+ 2048,
566
+ 1024
567
+ ],
568
+ "dtype": "torch.bfloat16"
569
+ },
570
+ {
571
+ "key": "talker.code_predictor.model.layers.1.self_attn.v_proj.weight",
572
+ "group": "cp_attn",
573
+ "alpha_a": 0.62,
574
+ "alpha_b": 0.38,
575
+ "shape": [
576
+ 1024,
577
+ 1024
578
+ ],
579
+ "dtype": "torch.bfloat16"
580
+ },
581
+ {
582
+ "key": "talker.code_predictor.model.layers.2.input_layernorm.weight",
583
+ "group": "layernorm",
584
+ "alpha_a": 1.0,
585
+ "alpha_b": 0.0,
586
+ "shape": [
587
+ 1024
588
+ ],
589
+ "dtype": "torch.bfloat16"
590
+ },
591
+ {
592
+ "key": "talker.code_predictor.model.layers.2.mlp.down_proj.weight",
593
+ "group": "cp_mlp",
594
+ "alpha_a": 0.75,
595
+ "alpha_b": 0.25,
596
+ "shape": [
597
+ 1024,
598
+ 3072
599
+ ],
600
+ "dtype": "torch.bfloat16"
601
+ },
602
+ {
603
+ "key": "talker.code_predictor.model.layers.2.mlp.gate_proj.weight",
604
+ "group": "cp_mlp",
605
+ "alpha_a": 0.75,
606
+ "alpha_b": 0.25,
607
+ "shape": [
608
+ 3072,
609
+ 1024
610
+ ],
611
+ "dtype": "torch.bfloat16"
612
+ },
613
+ {
614
+ "key": "talker.code_predictor.model.layers.2.mlp.up_proj.weight",
615
+ "group": "cp_mlp",
616
+ "alpha_a": 0.75,
617
+ "alpha_b": 0.25,
618
+ "shape": [
619
+ 3072,
620
+ 1024
621
+ ],
622
+ "dtype": "torch.bfloat16"
623
+ },
624
+ {
625
+ "key": "talker.code_predictor.model.layers.2.post_attention_layernorm.weight",
626
+ "group": "layernorm",
627
+ "alpha_a": 1.0,
628
+ "alpha_b": 0.0,
629
+ "shape": [
630
+ 1024
631
+ ],
632
+ "dtype": "torch.bfloat16"
633
+ },
634
+ {
635
+ "key": "talker.code_predictor.model.layers.2.self_attn.k_norm.weight",
636
+ "group": "layernorm",
637
+ "alpha_a": 1.0,
638
+ "alpha_b": 0.0,
639
+ "shape": [
640
+ 128
641
+ ],
642
+ "dtype": "torch.bfloat16"
643
+ },
644
+ {
645
+ "key": "talker.code_predictor.model.layers.2.self_attn.k_proj.weight",
646
+ "group": "cp_attn",
647
+ "alpha_a": 0.62,
648
+ "alpha_b": 0.38,
649
+ "shape": [
650
+ 1024,
651
+ 1024
652
+ ],
653
+ "dtype": "torch.bfloat16"
654
+ },
655
+ {
656
+ "key": "talker.code_predictor.model.layers.2.self_attn.o_proj.weight",
657
+ "group": "cp_attn",
658
+ "alpha_a": 0.62,
659
+ "alpha_b": 0.38,
660
+ "shape": [
661
+ 1024,
662
+ 2048
663
+ ],
664
+ "dtype": "torch.bfloat16"
665
+ },
666
+ {
667
+ "key": "talker.code_predictor.model.layers.2.self_attn.q_norm.weight",
668
+ "group": "layernorm",
669
+ "alpha_a": 1.0,
670
+ "alpha_b": 0.0,
671
+ "shape": [
672
+ 128
673
+ ],
674
+ "dtype": "torch.bfloat16"
675
+ },
676
+ {
677
+ "key": "talker.code_predictor.model.layers.2.self_attn.q_proj.weight",
678
+ "group": "cp_attn",
679
+ "alpha_a": 0.62,
680
+ "alpha_b": 0.38,
681
+ "shape": [
682
+ 2048,
683
+ 1024
684
+ ],
685
+ "dtype": "torch.bfloat16"
686
+ },
687
+ {
688
+ "key": "talker.code_predictor.model.layers.2.self_attn.v_proj.weight",
689
+ "group": "cp_attn",
690
+ "alpha_a": 0.62,
691
+ "alpha_b": 0.38,
692
+ "shape": [
693
+ 1024,
694
+ 1024
695
+ ],
696
+ "dtype": "torch.bfloat16"
697
+ },
698
+ {
699
+ "key": "talker.code_predictor.model.layers.3.input_layernorm.weight",
700
+ "group": "layernorm",
701
+ "alpha_a": 1.0,
702
+ "alpha_b": 0.0,
703
+ "shape": [
704
+ 1024
705
+ ],
706
+ "dtype": "torch.bfloat16"
707
+ },
708
+ {
709
+ "key": "talker.code_predictor.model.layers.3.mlp.down_proj.weight",
710
+ "group": "cp_mlp",
711
+ "alpha_a": 0.75,
712
+ "alpha_b": 0.25,
713
+ "shape": [
714
+ 1024,
715
+ 3072
716
+ ],
717
+ "dtype": "torch.bfloat16"
718
+ },
719
+ {
720
+ "key": "talker.code_predictor.model.layers.3.mlp.gate_proj.weight",
721
+ "group": "cp_mlp",
722
+ "alpha_a": 0.75,
723
+ "alpha_b": 0.25,
724
+ "shape": [
725
+ 3072,
726
+ 1024
727
+ ],
728
+ "dtype": "torch.bfloat16"
729
+ },
730
+ {
731
+ "key": "talker.code_predictor.model.layers.3.mlp.up_proj.weight",
732
+ "group": "cp_mlp",
733
+ "alpha_a": 0.75,
734
+ "alpha_b": 0.25,
735
+ "shape": [
736
+ 3072,
737
+ 1024
738
+ ],
739
+ "dtype": "torch.bfloat16"
740
+ },
741
+ {
742
+ "key": "talker.code_predictor.model.layers.3.post_attention_layernorm.weight",
743
+ "group": "layernorm",
744
+ "alpha_a": 1.0,
745
+ "alpha_b": 0.0,
746
+ "shape": [
747
+ 1024
748
+ ],
749
+ "dtype": "torch.bfloat16"
750
+ },
751
+ {
752
+ "key": "talker.code_predictor.model.layers.3.self_attn.k_norm.weight",
753
+ "group": "layernorm",
754
+ "alpha_a": 1.0,
755
+ "alpha_b": 0.0,
756
+ "shape": [
757
+ 128
758
+ ],
759
+ "dtype": "torch.bfloat16"
760
+ },
761
+ {
762
+ "key": "talker.code_predictor.model.layers.3.self_attn.k_proj.weight",
763
+ "group": "cp_attn",
764
+ "alpha_a": 0.62,
765
+ "alpha_b": 0.38,
766
+ "shape": [
767
+ 1024,
768
+ 1024
769
+ ],
770
+ "dtype": "torch.bfloat16"
771
+ },
772
+ {
773
+ "key": "talker.code_predictor.model.layers.3.self_attn.o_proj.weight",
774
+ "group": "cp_attn",
775
+ "alpha_a": 0.62,
776
+ "alpha_b": 0.38,
777
+ "shape": [
778
+ 1024,
779
+ 2048
780
+ ],
781
+ "dtype": "torch.bfloat16"
782
+ },
783
+ {
784
+ "key": "talker.code_predictor.model.layers.3.self_attn.q_norm.weight",
785
+ "group": "layernorm",
786
+ "alpha_a": 1.0,
787
+ "alpha_b": 0.0,
788
+ "shape": [
789
+ 128
790
+ ],
791
+ "dtype": "torch.bfloat16"
792
+ },
793
+ {
794
+ "key": "talker.code_predictor.model.layers.3.self_attn.q_proj.weight",
795
+ "group": "cp_attn",
796
+ "alpha_a": 0.62,
797
+ "alpha_b": 0.38,
798
+ "shape": [
799
+ 2048,
800
+ 1024
801
+ ],
802
+ "dtype": "torch.bfloat16"
803
+ },
804
+ {
805
+ "key": "talker.code_predictor.model.layers.3.self_attn.v_proj.weight",
806
+ "group": "cp_attn",
807
+ "alpha_a": 0.62,
808
+ "alpha_b": 0.38,
809
+ "shape": [
810
+ 1024,
811
+ 1024
812
+ ],
813
+ "dtype": "torch.bfloat16"
814
+ },
815
+ {
816
+ "key": "talker.code_predictor.model.layers.4.input_layernorm.weight",
817
+ "group": "layernorm",
818
+ "alpha_a": 1.0,
819
+ "alpha_b": 0.0,
820
+ "shape": [
821
+ 1024
822
+ ],
823
+ "dtype": "torch.bfloat16"
824
+ },
825
+ {
826
+ "key": "talker.code_predictor.model.layers.4.mlp.down_proj.weight",
827
+ "group": "cp_mlp",
828
+ "alpha_a": 0.75,
829
+ "alpha_b": 0.25,
830
+ "shape": [
831
+ 1024,
832
+ 3072
833
+ ],
834
+ "dtype": "torch.bfloat16"
835
+ },
836
+ {
837
+ "key": "talker.code_predictor.model.layers.4.mlp.gate_proj.weight",
838
+ "group": "cp_mlp",
839
+ "alpha_a": 0.75,
840
+ "alpha_b": 0.25,
841
+ "shape": [
842
+ 3072,
843
+ 1024
844
+ ],
845
+ "dtype": "torch.bfloat16"
846
+ },
847
+ {
848
+ "key": "talker.code_predictor.model.layers.4.mlp.up_proj.weight",
849
+ "group": "cp_mlp",
850
+ "alpha_a": 0.75,
851
+ "alpha_b": 0.25,
852
+ "shape": [
853
+ 3072,
854
+ 1024
855
+ ],
856
+ "dtype": "torch.bfloat16"
857
+ },
858
+ {
859
+ "key": "talker.code_predictor.model.layers.4.post_attention_layernorm.weight",
860
+ "group": "layernorm",
861
+ "alpha_a": 1.0,
862
+ "alpha_b": 0.0,
863
+ "shape": [
864
+ 1024
865
+ ],
866
+ "dtype": "torch.bfloat16"
867
+ },
868
+ {
869
+ "key": "talker.code_predictor.model.layers.4.self_attn.k_norm.weight",
870
+ "group": "layernorm",
871
+ "alpha_a": 1.0,
872
+ "alpha_b": 0.0,
873
+ "shape": [
874
+ 128
875
+ ],
876
+ "dtype": "torch.bfloat16"
877
+ },
878
+ {
879
+ "key": "talker.code_predictor.model.layers.4.self_attn.k_proj.weight",
880
+ "group": "cp_attn",
881
+ "alpha_a": 0.62,
882
+ "alpha_b": 0.38,
883
+ "shape": [
884
+ 1024,
885
+ 1024
886
+ ],
887
+ "dtype": "torch.bfloat16"
888
+ },
889
+ {
890
+ "key": "talker.code_predictor.model.layers.4.self_attn.o_proj.weight",
891
+ "group": "cp_attn",
892
+ "alpha_a": 0.62,
893
+ "alpha_b": 0.38,
894
+ "shape": [
895
+ 1024,
896
+ 2048
897
+ ],
898
+ "dtype": "torch.bfloat16"
899
+ },
900
+ {
901
+ "key": "talker.code_predictor.model.layers.4.self_attn.q_norm.weight",
902
+ "group": "layernorm",
903
+ "alpha_a": 1.0,
904
+ "alpha_b": 0.0,
905
+ "shape": [
906
+ 128
907
+ ],
908
+ "dtype": "torch.bfloat16"
909
+ },
910
+ {
911
+ "key": "talker.code_predictor.model.layers.4.self_attn.q_proj.weight",
912
+ "group": "cp_attn",
913
+ "alpha_a": 0.62,
914
+ "alpha_b": 0.38,
915
+ "shape": [
916
+ 2048,
917
+ 1024
918
+ ],
919
+ "dtype": "torch.bfloat16"
920
+ },
921
+ {
922
+ "key": "talker.code_predictor.model.layers.4.self_attn.v_proj.weight",
923
+ "group": "cp_attn",
924
+ "alpha_a": 0.62,
925
+ "alpha_b": 0.38,
926
+ "shape": [
927
+ 1024,
928
+ 1024
929
+ ],
930
+ "dtype": "torch.bfloat16"
931
+ },
932
+ {
933
+ "key": "talker.code_predictor.model.norm.weight",
934
+ "group": "layernorm",
935
+ "alpha_a": 1.0,
936
+ "alpha_b": 0.0,
937
+ "shape": [
938
+ 1024
939
+ ],
940
+ "dtype": "torch.bfloat16"
941
+ },
942
+ {
943
+ "key": "talker.code_predictor.small_to_mtp_projection.bias",
944
+ "group": "other_frozen",
945
+ "alpha_a": 1.0,
946
+ "alpha_b": 0.0,
947
+ "shape": [
948
+ 1024
949
+ ],
950
+ "dtype": "torch.bfloat16"
951
+ },
952
+ {
953
+ "key": "talker.code_predictor.small_to_mtp_projection.weight",
954
+ "group": "other_frozen",
955
+ "alpha_a": 1.0,
956
+ "alpha_b": 0.0,
957
+ "shape": [
958
+ 1024,
959
+ 2048
960
+ ],
961
+ "dtype": "torch.bfloat16"
962
+ },
963
+ {
964
+ "key": "talker.codec_head.weight",
965
+ "group": "other_frozen",
966
+ "alpha_a": 1.0,
967
+ "alpha_b": 0.0,
968
+ "shape": [
969
+ 3072,
970
+ 2048
971
+ ],
972
+ "dtype": "torch.bfloat16"
973
+ },
974
+ {
975
+ "key": "talker.model.codec_embedding.weight",
976
+ "group": "embedding",
977
+ "alpha_a": 1.0,
978
+ "alpha_b": 0.0,
979
+ "shape": [
980
+ 3072,
981
+ 2048
982
+ ],
983
+ "dtype": "torch.bfloat16"
984
+ },
985
+ {
986
+ "key": "talker.model.layers.0.input_layernorm.weight",
987
+ "group": "layernorm",
988
+ "alpha_a": 1.0,
989
+ "alpha_b": 0.0,
990
+ "shape": [
991
+ 2048
992
+ ],
993
+ "dtype": "torch.bfloat16"
994
+ },
995
+ {
996
+ "key": "talker.model.layers.0.mlp.down_proj.weight",
997
+ "group": "talker_mlp",
998
+ "alpha_a": 0.35,
999
+ "alpha_b": 0.65,
1000
+ "shape": [
1001
+ 2048,
1002
+ 6144
1003
+ ],
1004
+ "dtype": "torch.bfloat16"
1005
+ },
1006
+ {
1007
+ "key": "talker.model.layers.0.mlp.gate_proj.weight",
1008
+ "group": "talker_mlp",
1009
+ "alpha_a": 0.35,
1010
+ "alpha_b": 0.65,
1011
+ "shape": [
1012
+ 6144,
1013
+ 2048
1014
+ ],
1015
+ "dtype": "torch.bfloat16"
1016
+ },
1017
+ {
1018
+ "key": "talker.model.layers.0.mlp.up_proj.weight",
1019
+ "group": "talker_mlp",
1020
+ "alpha_a": 0.35,
1021
+ "alpha_b": 0.65,
1022
+ "shape": [
1023
+ 6144,
1024
+ 2048
1025
+ ],
1026
+ "dtype": "torch.bfloat16"
1027
+ },
1028
+ {
1029
+ "key": "talker.model.layers.0.post_attention_layernorm.weight",
1030
+ "group": "layernorm",
1031
+ "alpha_a": 1.0,
1032
+ "alpha_b": 0.0,
1033
+ "shape": [
1034
+ 2048
1035
+ ],
1036
+ "dtype": "torch.bfloat16"
1037
+ },
1038
+ {
1039
+ "key": "talker.model.layers.0.self_attn.k_norm.weight",
1040
+ "group": "layernorm",
1041
+ "alpha_a": 1.0,
1042
+ "alpha_b": 0.0,
1043
+ "shape": [
1044
+ 128
1045
+ ],
1046
+ "dtype": "torch.bfloat16"
1047
+ },
1048
+ {
1049
+ "key": "talker.model.layers.0.self_attn.k_proj.weight",
1050
+ "group": "talker_attn",
1051
+ "alpha_a": 0.35,
1052
+ "alpha_b": 0.65,
1053
+ "shape": [
1054
+ 1024,
1055
+ 2048
1056
+ ],
1057
+ "dtype": "torch.bfloat16"
1058
+ },
1059
+ {
1060
+ "key": "talker.model.layers.0.self_attn.o_proj.weight",
1061
+ "group": "talker_attn",
1062
+ "alpha_a": 0.35,
1063
+ "alpha_b": 0.65,
1064
+ "shape": [
1065
+ 2048,
1066
+ 2048
1067
+ ],
1068
+ "dtype": "torch.bfloat16"
1069
+ },
1070
+ {
1071
+ "key": "talker.model.layers.0.self_attn.q_norm.weight",
1072
+ "group": "layernorm",
1073
+ "alpha_a": 1.0,
1074
+ "alpha_b": 0.0,
1075
+ "shape": [
1076
+ 128
1077
+ ],
1078
+ "dtype": "torch.bfloat16"
1079
+ },
1080
+ {
1081
+ "key": "talker.model.layers.0.self_attn.q_proj.weight",
1082
+ "group": "talker_attn",
1083
+ "alpha_a": 0.35,
1084
+ "alpha_b": 0.65,
1085
+ "shape": [
1086
+ 2048,
1087
+ 2048
1088
+ ],
1089
+ "dtype": "torch.bfloat16"
1090
+ },
1091
+ {
1092
+ "key": "talker.model.layers.0.self_attn.v_proj.weight",
1093
+ "group": "talker_attn",
1094
+ "alpha_a": 0.35,
1095
+ "alpha_b": 0.65,
1096
+ "shape": [
1097
+ 1024,
1098
+ 2048
1099
+ ],
1100
+ "dtype": "torch.bfloat16"
1101
+ },
1102
+ {
1103
+ "key": "talker.model.layers.1.input_layernorm.weight",
1104
+ "group": "layernorm",
1105
+ "alpha_a": 1.0,
1106
+ "alpha_b": 0.0,
1107
+ "shape": [
1108
+ 2048
1109
+ ],
1110
+ "dtype": "torch.bfloat16"
1111
+ },
1112
+ {
1113
+ "key": "talker.model.layers.1.mlp.down_proj.weight",
1114
+ "group": "talker_mlp",
1115
+ "alpha_a": 0.35,
1116
+ "alpha_b": 0.65,
1117
+ "shape": [
1118
+ 2048,
1119
+ 6144
1120
+ ],
1121
+ "dtype": "torch.bfloat16"
1122
+ },
1123
+ {
1124
+ "key": "talker.model.layers.1.mlp.gate_proj.weight",
1125
+ "group": "talker_mlp",
1126
+ "alpha_a": 0.35,
1127
+ "alpha_b": 0.65,
1128
+ "shape": [
1129
+ 6144,
1130
+ 2048
1131
+ ],
1132
+ "dtype": "torch.bfloat16"
1133
+ },
1134
+ {
1135
+ "key": "talker.model.layers.1.mlp.up_proj.weight",
1136
+ "group": "talker_mlp",
1137
+ "alpha_a": 0.35,
1138
+ "alpha_b": 0.65,
1139
+ "shape": [
1140
+ 6144,
1141
+ 2048
1142
+ ],
1143
+ "dtype": "torch.bfloat16"
1144
+ },
1145
+ {
1146
+ "key": "talker.model.layers.1.post_attention_layernorm.weight",
1147
+ "group": "layernorm",
1148
+ "alpha_a": 1.0,
1149
+ "alpha_b": 0.0,
1150
+ "shape": [
1151
+ 2048
1152
+ ],
1153
+ "dtype": "torch.bfloat16"
1154
+ },
1155
+ {
1156
+ "key": "talker.model.layers.1.self_attn.k_norm.weight",
1157
+ "group": "layernorm",
1158
+ "alpha_a": 1.0,
1159
+ "alpha_b": 0.0,
1160
+ "shape": [
1161
+ 128
1162
+ ],
1163
+ "dtype": "torch.bfloat16"
1164
+ },
1165
+ {
1166
+ "key": "talker.model.layers.1.self_attn.k_proj.weight",
1167
+ "group": "talker_attn",
1168
+ "alpha_a": 0.35,
1169
+ "alpha_b": 0.65,
1170
+ "shape": [
1171
+ 1024,
1172
+ 2048
1173
+ ],
1174
+ "dtype": "torch.bfloat16"
1175
+ },
1176
+ {
1177
+ "key": "talker.model.layers.1.self_attn.o_proj.weight",
1178
+ "group": "talker_attn",
1179
+ "alpha_a": 0.35,
1180
+ "alpha_b": 0.65,
1181
+ "shape": [
1182
+ 2048,
1183
+ 2048
1184
+ ],
1185
+ "dtype": "torch.bfloat16"
1186
+ },
1187
+ {
1188
+ "key": "talker.model.layers.1.self_attn.q_norm.weight",
1189
+ "group": "layernorm",
1190
+ "alpha_a": 1.0,
1191
+ "alpha_b": 0.0,
1192
+ "shape": [
1193
+ 128
1194
+ ],
1195
+ "dtype": "torch.bfloat16"
1196
+ },
1197
+ {
1198
+ "key": "talker.model.layers.1.self_attn.q_proj.weight",
1199
+ "group": "talker_attn",
1200
+ "alpha_a": 0.35,
1201
+ "alpha_b": 0.65,
1202
+ "shape": [
1203
+ 2048,
1204
+ 2048
1205
+ ],
1206
+ "dtype": "torch.bfloat16"
1207
+ },
1208
+ {
1209
+ "key": "talker.model.layers.1.self_attn.v_proj.weight",
1210
+ "group": "talker_attn",
1211
+ "alpha_a": 0.35,
1212
+ "alpha_b": 0.65,
1213
+ "shape": [
1214
+ 1024,
1215
+ 2048
1216
+ ],
1217
+ "dtype": "torch.bfloat16"
1218
+ },
1219
+ {
1220
+ "key": "talker.model.layers.10.input_layernorm.weight",
1221
+ "group": "layernorm",
1222
+ "alpha_a": 1.0,
1223
+ "alpha_b": 0.0,
1224
+ "shape": [
1225
+ 2048
1226
+ ],
1227
+ "dtype": "torch.bfloat16"
1228
+ },
1229
+ {
1230
+ "key": "talker.model.layers.10.mlp.down_proj.weight",
1231
+ "group": "talker_mlp",
1232
+ "alpha_a": 0.35,
1233
+ "alpha_b": 0.65,
1234
+ "shape": [
1235
+ 2048,
1236
+ 6144
1237
+ ],
1238
+ "dtype": "torch.bfloat16"
1239
+ },
1240
+ {
1241
+ "key": "talker.model.layers.10.mlp.gate_proj.weight",
1242
+ "group": "talker_mlp",
1243
+ "alpha_a": 0.35,
1244
+ "alpha_b": 0.65,
1245
+ "shape": [
1246
+ 6144,
1247
+ 2048
1248
+ ],
1249
+ "dtype": "torch.bfloat16"
1250
+ },
1251
+ {
1252
+ "key": "talker.model.layers.10.mlp.up_proj.weight",
1253
+ "group": "talker_mlp",
1254
+ "alpha_a": 0.35,
1255
+ "alpha_b": 0.65,
1256
+ "shape": [
1257
+ 6144,
1258
+ 2048
1259
+ ],
1260
+ "dtype": "torch.bfloat16"
1261
+ },
1262
+ {
1263
+ "key": "talker.model.layers.10.post_attention_layernorm.weight",
1264
+ "group": "layernorm",
1265
+ "alpha_a": 1.0,
1266
+ "alpha_b": 0.0,
1267
+ "shape": [
1268
+ 2048
1269
+ ],
1270
+ "dtype": "torch.bfloat16"
1271
+ },
1272
+ {
1273
+ "key": "talker.model.layers.10.self_attn.k_norm.weight",
1274
+ "group": "layernorm",
1275
+ "alpha_a": 1.0,
1276
+ "alpha_b": 0.0,
1277
+ "shape": [
1278
+ 128
1279
+ ],
1280
+ "dtype": "torch.bfloat16"
1281
+ },
1282
+ {
1283
+ "key": "talker.model.layers.10.self_attn.k_proj.weight",
1284
+ "group": "talker_attn",
1285
+ "alpha_a": 0.35,
1286
+ "alpha_b": 0.65,
1287
+ "shape": [
1288
+ 1024,
1289
+ 2048
1290
+ ],
1291
+ "dtype": "torch.bfloat16"
1292
+ },
1293
+ {
1294
+ "key": "talker.model.layers.10.self_attn.o_proj.weight",
1295
+ "group": "talker_attn",
1296
+ "alpha_a": 0.35,
1297
+ "alpha_b": 0.65,
1298
+ "shape": [
1299
+ 2048,
1300
+ 2048
1301
+ ],
1302
+ "dtype": "torch.bfloat16"
1303
+ },
1304
+ {
1305
+ "key": "talker.model.layers.10.self_attn.q_norm.weight",
1306
+ "group": "layernorm",
1307
+ "alpha_a": 1.0,
1308
+ "alpha_b": 0.0,
1309
+ "shape": [
1310
+ 128
1311
+ ],
1312
+ "dtype": "torch.bfloat16"
1313
+ },
1314
+ {
1315
+ "key": "talker.model.layers.10.self_attn.q_proj.weight",
1316
+ "group": "talker_attn",
1317
+ "alpha_a": 0.35,
1318
+ "alpha_b": 0.65,
1319
+ "shape": [
1320
+ 2048,
1321
+ 2048
1322
+ ],
1323
+ "dtype": "torch.bfloat16"
1324
+ },
1325
+ {
1326
+ "key": "talker.model.layers.10.self_attn.v_proj.weight",
1327
+ "group": "talker_attn",
1328
+ "alpha_a": 0.35,
1329
+ "alpha_b": 0.65,
1330
+ "shape": [
1331
+ 1024,
1332
+ 2048
1333
+ ],
1334
+ "dtype": "torch.bfloat16"
1335
+ },
1336
+ {
1337
+ "key": "talker.model.layers.11.input_layernorm.weight",
1338
+ "group": "layernorm",
1339
+ "alpha_a": 1.0,
1340
+ "alpha_b": 0.0,
1341
+ "shape": [
1342
+ 2048
1343
+ ],
1344
+ "dtype": "torch.bfloat16"
1345
+ },
1346
+ {
1347
+ "key": "talker.model.layers.11.mlp.down_proj.weight",
1348
+ "group": "talker_mlp",
1349
+ "alpha_a": 0.35,
1350
+ "alpha_b": 0.65,
1351
+ "shape": [
1352
+ 2048,
1353
+ 6144
1354
+ ],
1355
+ "dtype": "torch.bfloat16"
1356
+ },
1357
+ {
1358
+ "key": "talker.model.layers.11.mlp.gate_proj.weight",
1359
+ "group": "talker_mlp",
1360
+ "alpha_a": 0.35,
1361
+ "alpha_b": 0.65,
1362
+ "shape": [
1363
+ 6144,
1364
+ 2048
1365
+ ],
1366
+ "dtype": "torch.bfloat16"
1367
+ },
1368
+ {
1369
+ "key": "talker.model.layers.11.mlp.up_proj.weight",
1370
+ "group": "talker_mlp",
1371
+ "alpha_a": 0.35,
1372
+ "alpha_b": 0.65,
1373
+ "shape": [
1374
+ 6144,
1375
+ 2048
1376
+ ],
1377
+ "dtype": "torch.bfloat16"
1378
+ },
1379
+ {
1380
+ "key": "talker.model.layers.11.post_attention_layernorm.weight",
1381
+ "group": "layernorm",
1382
+ "alpha_a": 1.0,
1383
+ "alpha_b": 0.0,
1384
+ "shape": [
1385
+ 2048
1386
+ ],
1387
+ "dtype": "torch.bfloat16"
1388
+ },
1389
+ {
1390
+ "key": "talker.model.layers.11.self_attn.k_norm.weight",
1391
+ "group": "layernorm",
1392
+ "alpha_a": 1.0,
1393
+ "alpha_b": 0.0,
1394
+ "shape": [
1395
+ 128
1396
+ ],
1397
+ "dtype": "torch.bfloat16"
1398
+ },
1399
+ {
1400
+ "key": "talker.model.layers.11.self_attn.k_proj.weight",
1401
+ "group": "talker_attn",
1402
+ "alpha_a": 0.35,
1403
+ "alpha_b": 0.65,
1404
+ "shape": [
1405
+ 1024,
1406
+ 2048
1407
+ ],
1408
+ "dtype": "torch.bfloat16"
1409
+ },
1410
+ {
1411
+ "key": "talker.model.layers.11.self_attn.o_proj.weight",
1412
+ "group": "talker_attn",
1413
+ "alpha_a": 0.35,
1414
+ "alpha_b": 0.65,
1415
+ "shape": [
1416
+ 2048,
1417
+ 2048
1418
+ ],
1419
+ "dtype": "torch.bfloat16"
1420
+ },
1421
+ {
1422
+ "key": "talker.model.layers.11.self_attn.q_norm.weight",
1423
+ "group": "layernorm",
1424
+ "alpha_a": 1.0,
1425
+ "alpha_b": 0.0,
1426
+ "shape": [
1427
+ 128
1428
+ ],
1429
+ "dtype": "torch.bfloat16"
1430
+ },
1431
+ {
1432
+ "key": "talker.model.layers.11.self_attn.q_proj.weight",
1433
+ "group": "talker_attn",
1434
+ "alpha_a": 0.35,
1435
+ "alpha_b": 0.65,
1436
+ "shape": [
1437
+ 2048,
1438
+ 2048
1439
+ ],
1440
+ "dtype": "torch.bfloat16"
1441
+ },
1442
+ {
1443
+ "key": "talker.model.layers.11.self_attn.v_proj.weight",
1444
+ "group": "talker_attn",
1445
+ "alpha_a": 0.35,
1446
+ "alpha_b": 0.65,
1447
+ "shape": [
1448
+ 1024,
1449
+ 2048
1450
+ ],
1451
+ "dtype": "torch.bfloat16"
1452
+ },
1453
+ {
1454
+ "key": "talker.model.layers.12.input_layernorm.weight",
1455
+ "group": "layernorm",
1456
+ "alpha_a": 1.0,
1457
+ "alpha_b": 0.0,
1458
+ "shape": [
1459
+ 2048
1460
+ ],
1461
+ "dtype": "torch.bfloat16"
1462
+ },
1463
+ {
1464
+ "key": "talker.model.layers.12.mlp.down_proj.weight",
1465
+ "group": "talker_mlp",
1466
+ "alpha_a": 0.35,
1467
+ "alpha_b": 0.65,
1468
+ "shape": [
1469
+ 2048,
1470
+ 6144
1471
+ ],
1472
+ "dtype": "torch.bfloat16"
1473
+ },
1474
+ {
1475
+ "key": "talker.model.layers.12.mlp.gate_proj.weight",
1476
+ "group": "talker_mlp",
1477
+ "alpha_a": 0.35,
1478
+ "alpha_b": 0.65,
1479
+ "shape": [
1480
+ 6144,
1481
+ 2048
1482
+ ],
1483
+ "dtype": "torch.bfloat16"
1484
+ },
1485
+ {
1486
+ "key": "talker.model.layers.12.mlp.up_proj.weight",
1487
+ "group": "talker_mlp",
1488
+ "alpha_a": 0.35,
1489
+ "alpha_b": 0.65,
1490
+ "shape": [
1491
+ 6144,
1492
+ 2048
1493
+ ],
1494
+ "dtype": "torch.bfloat16"
1495
+ },
1496
+ {
1497
+ "key": "talker.model.layers.12.post_attention_layernorm.weight",
1498
+ "group": "layernorm",
1499
+ "alpha_a": 1.0,
1500
+ "alpha_b": 0.0,
1501
+ "shape": [
1502
+ 2048
1503
+ ],
1504
+ "dtype": "torch.bfloat16"
1505
+ },
1506
+ {
1507
+ "key": "talker.model.layers.12.self_attn.k_norm.weight",
1508
+ "group": "layernorm",
1509
+ "alpha_a": 1.0,
1510
+ "alpha_b": 0.0,
1511
+ "shape": [
1512
+ 128
1513
+ ],
1514
+ "dtype": "torch.bfloat16"
1515
+ },
1516
+ {
1517
+ "key": "talker.model.layers.12.self_attn.k_proj.weight",
1518
+ "group": "talker_attn",
1519
+ "alpha_a": 0.35,
1520
+ "alpha_b": 0.65,
1521
+ "shape": [
1522
+ 1024,
1523
+ 2048
1524
+ ],
1525
+ "dtype": "torch.bfloat16"
1526
+ },
1527
+ {
1528
+ "key": "talker.model.layers.12.self_attn.o_proj.weight",
1529
+ "group": "talker_attn",
1530
+ "alpha_a": 0.35,
1531
+ "alpha_b": 0.65,
1532
+ "shape": [
1533
+ 2048,
1534
+ 2048
1535
+ ],
1536
+ "dtype": "torch.bfloat16"
1537
+ },
1538
+ {
1539
+ "key": "talker.model.layers.12.self_attn.q_norm.weight",
1540
+ "group": "layernorm",
1541
+ "alpha_a": 1.0,
1542
+ "alpha_b": 0.0,
1543
+ "shape": [
1544
+ 128
1545
+ ],
1546
+ "dtype": "torch.bfloat16"
1547
+ },
1548
+ {
1549
+ "key": "talker.model.layers.12.self_attn.q_proj.weight",
1550
+ "group": "talker_attn",
1551
+ "alpha_a": 0.35,
1552
+ "alpha_b": 0.65,
1553
+ "shape": [
1554
+ 2048,
1555
+ 2048
1556
+ ],
1557
+ "dtype": "torch.bfloat16"
1558
+ },
1559
+ {
1560
+ "key": "talker.model.layers.12.self_attn.v_proj.weight",
1561
+ "group": "talker_attn",
1562
+ "alpha_a": 0.35,
1563
+ "alpha_b": 0.65,
1564
+ "shape": [
1565
+ 1024,
1566
+ 2048
1567
+ ],
1568
+ "dtype": "torch.bfloat16"
1569
+ },
1570
+ {
1571
+ "key": "talker.model.layers.13.input_layernorm.weight",
1572
+ "group": "layernorm",
1573
+ "alpha_a": 1.0,
1574
+ "alpha_b": 0.0,
1575
+ "shape": [
1576
+ 2048
1577
+ ],
1578
+ "dtype": "torch.bfloat16"
1579
+ },
1580
+ {
1581
+ "key": "talker.model.layers.13.mlp.down_proj.weight",
1582
+ "group": "talker_mlp",
1583
+ "alpha_a": 0.35,
1584
+ "alpha_b": 0.65,
1585
+ "shape": [
1586
+ 2048,
1587
+ 6144
1588
+ ],
1589
+ "dtype": "torch.bfloat16"
1590
+ },
1591
+ {
1592
+ "key": "talker.model.layers.13.mlp.gate_proj.weight",
1593
+ "group": "talker_mlp",
1594
+ "alpha_a": 0.35,
1595
+ "alpha_b": 0.65,
1596
+ "shape": [
1597
+ 6144,
1598
+ 2048
1599
+ ],
1600
+ "dtype": "torch.bfloat16"
1601
+ },
1602
+ {
1603
+ "key": "talker.model.layers.13.mlp.up_proj.weight",
1604
+ "group": "talker_mlp",
1605
+ "alpha_a": 0.35,
1606
+ "alpha_b": 0.65,
1607
+ "shape": [
1608
+ 6144,
1609
+ 2048
1610
+ ],
1611
+ "dtype": "torch.bfloat16"
1612
+ },
1613
+ {
1614
+ "key": "talker.model.layers.13.post_attention_layernorm.weight",
1615
+ "group": "layernorm",
1616
+ "alpha_a": 1.0,
1617
+ "alpha_b": 0.0,
1618
+ "shape": [
1619
+ 2048
1620
+ ],
1621
+ "dtype": "torch.bfloat16"
1622
+ },
1623
+ {
1624
+ "key": "talker.model.layers.13.self_attn.k_norm.weight",
1625
+ "group": "layernorm",
1626
+ "alpha_a": 1.0,
1627
+ "alpha_b": 0.0,
1628
+ "shape": [
1629
+ 128
1630
+ ],
1631
+ "dtype": "torch.bfloat16"
1632
+ },
1633
+ {
1634
+ "key": "talker.model.layers.13.self_attn.k_proj.weight",
1635
+ "group": "talker_attn",
1636
+ "alpha_a": 0.35,
1637
+ "alpha_b": 0.65,
1638
+ "shape": [
1639
+ 1024,
1640
+ 2048
1641
+ ],
1642
+ "dtype": "torch.bfloat16"
1643
+ },
1644
+ {
1645
+ "key": "talker.model.layers.13.self_attn.o_proj.weight",
1646
+ "group": "talker_attn",
1647
+ "alpha_a": 0.35,
1648
+ "alpha_b": 0.65,
1649
+ "shape": [
1650
+ 2048,
1651
+ 2048
1652
+ ],
1653
+ "dtype": "torch.bfloat16"
1654
+ },
1655
+ {
1656
+ "key": "talker.model.layers.13.self_attn.q_norm.weight",
1657
+ "group": "layernorm",
1658
+ "alpha_a": 1.0,
1659
+ "alpha_b": 0.0,
1660
+ "shape": [
1661
+ 128
1662
+ ],
1663
+ "dtype": "torch.bfloat16"
1664
+ },
1665
+ {
1666
+ "key": "talker.model.layers.13.self_attn.q_proj.weight",
1667
+ "group": "talker_attn",
1668
+ "alpha_a": 0.35,
1669
+ "alpha_b": 0.65,
1670
+ "shape": [
1671
+ 2048,
1672
+ 2048
1673
+ ],
1674
+ "dtype": "torch.bfloat16"
1675
+ },
1676
+ {
1677
+ "key": "talker.model.layers.13.self_attn.v_proj.weight",
1678
+ "group": "talker_attn",
1679
+ "alpha_a": 0.35,
1680
+ "alpha_b": 0.65,
1681
+ "shape": [
1682
+ 1024,
1683
+ 2048
1684
+ ],
1685
+ "dtype": "torch.bfloat16"
1686
+ },
1687
+ {
1688
+ "key": "talker.model.layers.14.input_layernorm.weight",
1689
+ "group": "layernorm",
1690
+ "alpha_a": 1.0,
1691
+ "alpha_b": 0.0,
1692
+ "shape": [
1693
+ 2048
1694
+ ],
1695
+ "dtype": "torch.bfloat16"
1696
+ },
1697
+ {
1698
+ "key": "talker.model.layers.14.mlp.down_proj.weight",
1699
+ "group": "talker_mlp",
1700
+ "alpha_a": 0.35,
1701
+ "alpha_b": 0.65,
1702
+ "shape": [
1703
+ 2048,
1704
+ 6144
1705
+ ],
1706
+ "dtype": "torch.bfloat16"
1707
+ },
1708
+ {
1709
+ "key": "talker.model.layers.14.mlp.gate_proj.weight",
1710
+ "group": "talker_mlp",
1711
+ "alpha_a": 0.35,
1712
+ "alpha_b": 0.65,
1713
+ "shape": [
1714
+ 6144,
1715
+ 2048
1716
+ ],
1717
+ "dtype": "torch.bfloat16"
1718
+ },
1719
+ {
1720
+ "key": "talker.model.layers.14.mlp.up_proj.weight",
1721
+ "group": "talker_mlp",
1722
+ "alpha_a": 0.35,
1723
+ "alpha_b": 0.65,
1724
+ "shape": [
1725
+ 6144,
1726
+ 2048
1727
+ ],
1728
+ "dtype": "torch.bfloat16"
1729
+ },
1730
+ {
1731
+ "key": "talker.model.layers.14.post_attention_layernorm.weight",
1732
+ "group": "layernorm",
1733
+ "alpha_a": 1.0,
1734
+ "alpha_b": 0.0,
1735
+ "shape": [
1736
+ 2048
1737
+ ],
1738
+ "dtype": "torch.bfloat16"
1739
+ },
1740
+ {
1741
+ "key": "talker.model.layers.14.self_attn.k_norm.weight",
1742
+ "group": "layernorm",
1743
+ "alpha_a": 1.0,
1744
+ "alpha_b": 0.0,
1745
+ "shape": [
1746
+ 128
1747
+ ],
1748
+ "dtype": "torch.bfloat16"
1749
+ },
1750
+ {
1751
+ "key": "talker.model.layers.14.self_attn.k_proj.weight",
1752
+ "group": "talker_attn",
1753
+ "alpha_a": 0.35,
1754
+ "alpha_b": 0.65,
1755
+ "shape": [
1756
+ 1024,
1757
+ 2048
1758
+ ],
1759
+ "dtype": "torch.bfloat16"
1760
+ },
1761
+ {
1762
+ "key": "talker.model.layers.14.self_attn.o_proj.weight",
1763
+ "group": "talker_attn",
1764
+ "alpha_a": 0.35,
1765
+ "alpha_b": 0.65,
1766
+ "shape": [
1767
+ 2048,
1768
+ 2048
1769
+ ],
1770
+ "dtype": "torch.bfloat16"
1771
+ },
1772
+ {
1773
+ "key": "talker.model.layers.14.self_attn.q_norm.weight",
1774
+ "group": "layernorm",
1775
+ "alpha_a": 1.0,
1776
+ "alpha_b": 0.0,
1777
+ "shape": [
1778
+ 128
1779
+ ],
1780
+ "dtype": "torch.bfloat16"
1781
+ },
1782
+ {
1783
+ "key": "talker.model.layers.14.self_attn.q_proj.weight",
1784
+ "group": "talker_attn",
1785
+ "alpha_a": 0.35,
1786
+ "alpha_b": 0.65,
1787
+ "shape": [
1788
+ 2048,
1789
+ 2048
1790
+ ],
1791
+ "dtype": "torch.bfloat16"
1792
+ },
1793
+ {
1794
+ "key": "talker.model.layers.14.self_attn.v_proj.weight",
1795
+ "group": "talker_attn",
1796
+ "alpha_a": 0.35,
1797
+ "alpha_b": 0.65,
1798
+ "shape": [
1799
+ 1024,
1800
+ 2048
1801
+ ],
1802
+ "dtype": "torch.bfloat16"
1803
+ },
1804
+ {
1805
+ "key": "talker.model.layers.15.input_layernorm.weight",
1806
+ "group": "layernorm",
1807
+ "alpha_a": 1.0,
1808
+ "alpha_b": 0.0,
1809
+ "shape": [
1810
+ 2048
1811
+ ],
1812
+ "dtype": "torch.bfloat16"
1813
+ },
1814
+ {
1815
+ "key": "talker.model.layers.15.mlp.down_proj.weight",
1816
+ "group": "talker_mlp",
1817
+ "alpha_a": 0.35,
1818
+ "alpha_b": 0.65,
1819
+ "shape": [
1820
+ 2048,
1821
+ 6144
1822
+ ],
1823
+ "dtype": "torch.bfloat16"
1824
+ },
1825
+ {
1826
+ "key": "talker.model.layers.15.mlp.gate_proj.weight",
1827
+ "group": "talker_mlp",
1828
+ "alpha_a": 0.35,
1829
+ "alpha_b": 0.65,
1830
+ "shape": [
1831
+ 6144,
1832
+ 2048
1833
+ ],
1834
+ "dtype": "torch.bfloat16"
1835
+ },
1836
+ {
1837
+ "key": "talker.model.layers.15.mlp.up_proj.weight",
1838
+ "group": "talker_mlp",
1839
+ "alpha_a": 0.35,
1840
+ "alpha_b": 0.65,
1841
+ "shape": [
1842
+ 6144,
1843
+ 2048
1844
+ ],
1845
+ "dtype": "torch.bfloat16"
1846
+ },
1847
+ {
1848
+ "key": "talker.model.layers.15.post_attention_layernorm.weight",
1849
+ "group": "layernorm",
1850
+ "alpha_a": 1.0,
1851
+ "alpha_b": 0.0,
1852
+ "shape": [
1853
+ 2048
1854
+ ],
1855
+ "dtype": "torch.bfloat16"
1856
+ },
1857
+ {
1858
+ "key": "talker.model.layers.15.self_attn.k_norm.weight",
1859
+ "group": "layernorm",
1860
+ "alpha_a": 1.0,
1861
+ "alpha_b": 0.0,
1862
+ "shape": [
1863
+ 128
1864
+ ],
1865
+ "dtype": "torch.bfloat16"
1866
+ },
1867
+ {
1868
+ "key": "talker.model.layers.15.self_attn.k_proj.weight",
1869
+ "group": "talker_attn",
1870
+ "alpha_a": 0.35,
1871
+ "alpha_b": 0.65,
1872
+ "shape": [
1873
+ 1024,
1874
+ 2048
1875
+ ],
1876
+ "dtype": "torch.bfloat16"
1877
+ },
1878
+ {
1879
+ "key": "talker.model.layers.15.self_attn.o_proj.weight",
1880
+ "group": "talker_attn",
1881
+ "alpha_a": 0.35,
1882
+ "alpha_b": 0.65,
1883
+ "shape": [
1884
+ 2048,
1885
+ 2048
1886
+ ],
1887
+ "dtype": "torch.bfloat16"
1888
+ },
1889
+ {
1890
+ "key": "talker.model.layers.15.self_attn.q_norm.weight",
1891
+ "group": "layernorm",
1892
+ "alpha_a": 1.0,
1893
+ "alpha_b": 0.0,
1894
+ "shape": [
1895
+ 128
1896
+ ],
1897
+ "dtype": "torch.bfloat16"
1898
+ },
1899
+ {
1900
+ "key": "talker.model.layers.15.self_attn.q_proj.weight",
1901
+ "group": "talker_attn",
1902
+ "alpha_a": 0.35,
1903
+ "alpha_b": 0.65,
1904
+ "shape": [
1905
+ 2048,
1906
+ 2048
1907
+ ],
1908
+ "dtype": "torch.bfloat16"
1909
+ },
1910
+ {
1911
+ "key": "talker.model.layers.15.self_attn.v_proj.weight",
1912
+ "group": "talker_attn",
1913
+ "alpha_a": 0.35,
1914
+ "alpha_b": 0.65,
1915
+ "shape": [
1916
+ 1024,
1917
+ 2048
1918
+ ],
1919
+ "dtype": "torch.bfloat16"
1920
+ },
1921
+ {
1922
+ "key": "talker.model.layers.16.input_layernorm.weight",
1923
+ "group": "layernorm",
1924
+ "alpha_a": 1.0,
1925
+ "alpha_b": 0.0,
1926
+ "shape": [
1927
+ 2048
1928
+ ],
1929
+ "dtype": "torch.bfloat16"
1930
+ },
1931
+ {
1932
+ "key": "talker.model.layers.16.mlp.down_proj.weight",
1933
+ "group": "talker_mlp",
1934
+ "alpha_a": 0.35,
1935
+ "alpha_b": 0.65,
1936
+ "shape": [
1937
+ 2048,
1938
+ 6144
1939
+ ],
1940
+ "dtype": "torch.bfloat16"
1941
+ },
1942
+ {
1943
+ "key": "talker.model.layers.16.mlp.gate_proj.weight",
1944
+ "group": "talker_mlp",
1945
+ "alpha_a": 0.35,
1946
+ "alpha_b": 0.65,
1947
+ "shape": [
1948
+ 6144,
1949
+ 2048
1950
+ ],
1951
+ "dtype": "torch.bfloat16"
1952
+ },
1953
+ {
1954
+ "key": "talker.model.layers.16.mlp.up_proj.weight",
1955
+ "group": "talker_mlp",
1956
+ "alpha_a": 0.35,
1957
+ "alpha_b": 0.65,
1958
+ "shape": [
1959
+ 6144,
1960
+ 2048
1961
+ ],
1962
+ "dtype": "torch.bfloat16"
1963
+ },
1964
+ {
1965
+ "key": "talker.model.layers.16.post_attention_layernorm.weight",
1966
+ "group": "layernorm",
1967
+ "alpha_a": 1.0,
1968
+ "alpha_b": 0.0,
1969
+ "shape": [
1970
+ 2048
1971
+ ],
1972
+ "dtype": "torch.bfloat16"
1973
+ },
1974
+ {
1975
+ "key": "talker.model.layers.16.self_attn.k_norm.weight",
1976
+ "group": "layernorm",
1977
+ "alpha_a": 1.0,
1978
+ "alpha_b": 0.0,
1979
+ "shape": [
1980
+ 128
1981
+ ],
1982
+ "dtype": "torch.bfloat16"
1983
+ },
1984
+ {
1985
+ "key": "talker.model.layers.16.self_attn.k_proj.weight",
1986
+ "group": "talker_attn",
1987
+ "alpha_a": 0.35,
1988
+ "alpha_b": 0.65,
1989
+ "shape": [
1990
+ 1024,
1991
+ 2048
1992
+ ],
1993
+ "dtype": "torch.bfloat16"
1994
+ },
1995
+ {
1996
+ "key": "talker.model.layers.16.self_attn.o_proj.weight",
1997
+ "group": "talker_attn",
1998
+ "alpha_a": 0.35,
1999
+ "alpha_b": 0.65,
2000
+ "shape": [
2001
+ 2048,
2002
+ 2048
2003
+ ],
2004
+ "dtype": "torch.bfloat16"
2005
+ },
2006
+ {
2007
+ "key": "talker.model.layers.16.self_attn.q_norm.weight",
2008
+ "group": "layernorm",
2009
+ "alpha_a": 1.0,
2010
+ "alpha_b": 0.0,
2011
+ "shape": [
2012
+ 128
2013
+ ],
2014
+ "dtype": "torch.bfloat16"
2015
+ },
2016
+ {
2017
+ "key": "talker.model.layers.16.self_attn.q_proj.weight",
2018
+ "group": "talker_attn",
2019
+ "alpha_a": 0.35,
2020
+ "alpha_b": 0.65,
2021
+ "shape": [
2022
+ 2048,
2023
+ 2048
2024
+ ],
2025
+ "dtype": "torch.bfloat16"
2026
+ },
2027
+ {
2028
+ "key": "talker.model.layers.16.self_attn.v_proj.weight",
2029
+ "group": "talker_attn",
2030
+ "alpha_a": 0.35,
2031
+ "alpha_b": 0.65,
2032
+ "shape": [
2033
+ 1024,
2034
+ 2048
2035
+ ],
2036
+ "dtype": "torch.bfloat16"
2037
+ },
2038
+ {
2039
+ "key": "talker.model.layers.17.input_layernorm.weight",
2040
+ "group": "layernorm",
2041
+ "alpha_a": 1.0,
2042
+ "alpha_b": 0.0,
2043
+ "shape": [
2044
+ 2048
2045
+ ],
2046
+ "dtype": "torch.bfloat16"
2047
+ },
2048
+ {
2049
+ "key": "talker.model.layers.17.mlp.down_proj.weight",
2050
+ "group": "talker_mlp",
2051
+ "alpha_a": 0.35,
2052
+ "alpha_b": 0.65,
2053
+ "shape": [
2054
+ 2048,
2055
+ 6144
2056
+ ],
2057
+ "dtype": "torch.bfloat16"
2058
+ },
2059
+ {
2060
+ "key": "talker.model.layers.17.mlp.gate_proj.weight",
2061
+ "group": "talker_mlp",
2062
+ "alpha_a": 0.35,
2063
+ "alpha_b": 0.65,
2064
+ "shape": [
2065
+ 6144,
2066
+ 2048
2067
+ ],
2068
+ "dtype": "torch.bfloat16"
2069
+ },
2070
+ {
2071
+ "key": "talker.model.layers.17.mlp.up_proj.weight",
2072
+ "group": "talker_mlp",
2073
+ "alpha_a": 0.35,
2074
+ "alpha_b": 0.65,
2075
+ "shape": [
2076
+ 6144,
2077
+ 2048
2078
+ ],
2079
+ "dtype": "torch.bfloat16"
2080
+ },
2081
+ {
2082
+ "key": "talker.model.layers.17.post_attention_layernorm.weight",
2083
+ "group": "layernorm",
2084
+ "alpha_a": 1.0,
2085
+ "alpha_b": 0.0,
2086
+ "shape": [
2087
+ 2048
2088
+ ],
2089
+ "dtype": "torch.bfloat16"
2090
+ },
2091
+ {
2092
+ "key": "talker.model.layers.17.self_attn.k_norm.weight",
2093
+ "group": "layernorm",
2094
+ "alpha_a": 1.0,
2095
+ "alpha_b": 0.0,
2096
+ "shape": [
2097
+ 128
2098
+ ],
2099
+ "dtype": "torch.bfloat16"
2100
+ },
2101
+ {
2102
+ "key": "talker.model.layers.17.self_attn.k_proj.weight",
2103
+ "group": "talker_attn",
2104
+ "alpha_a": 0.35,
2105
+ "alpha_b": 0.65,
2106
+ "shape": [
2107
+ 1024,
2108
+ 2048
2109
+ ],
2110
+ "dtype": "torch.bfloat16"
2111
+ },
2112
+ {
2113
+ "key": "talker.model.layers.17.self_attn.o_proj.weight",
2114
+ "group": "talker_attn",
2115
+ "alpha_a": 0.35,
2116
+ "alpha_b": 0.65,
2117
+ "shape": [
2118
+ 2048,
2119
+ 2048
2120
+ ],
2121
+ "dtype": "torch.bfloat16"
2122
+ },
2123
+ {
2124
+ "key": "talker.model.layers.17.self_attn.q_norm.weight",
2125
+ "group": "layernorm",
2126
+ "alpha_a": 1.0,
2127
+ "alpha_b": 0.0,
2128
+ "shape": [
2129
+ 128
2130
+ ],
2131
+ "dtype": "torch.bfloat16"
2132
+ },
2133
+ {
2134
+ "key": "talker.model.layers.17.self_attn.q_proj.weight",
2135
+ "group": "talker_attn",
2136
+ "alpha_a": 0.35,
2137
+ "alpha_b": 0.65,
2138
+ "shape": [
2139
+ 2048,
2140
+ 2048
2141
+ ],
2142
+ "dtype": "torch.bfloat16"
2143
+ },
2144
+ {
2145
+ "key": "talker.model.layers.17.self_attn.v_proj.weight",
2146
+ "group": "talker_attn",
2147
+ "alpha_a": 0.35,
2148
+ "alpha_b": 0.65,
2149
+ "shape": [
2150
+ 1024,
2151
+ 2048
2152
+ ],
2153
+ "dtype": "torch.bfloat16"
2154
+ },
2155
+ {
2156
+ "key": "talker.model.layers.18.input_layernorm.weight",
2157
+ "group": "layernorm",
2158
+ "alpha_a": 1.0,
2159
+ "alpha_b": 0.0,
2160
+ "shape": [
2161
+ 2048
2162
+ ],
2163
+ "dtype": "torch.bfloat16"
2164
+ },
2165
+ {
2166
+ "key": "talker.model.layers.18.mlp.down_proj.weight",
2167
+ "group": "talker_mlp",
2168
+ "alpha_a": 0.35,
2169
+ "alpha_b": 0.65,
2170
+ "shape": [
2171
+ 2048,
2172
+ 6144
2173
+ ],
2174
+ "dtype": "torch.bfloat16"
2175
+ },
2176
+ {
2177
+ "key": "talker.model.layers.18.mlp.gate_proj.weight",
2178
+ "group": "talker_mlp",
2179
+ "alpha_a": 0.35,
2180
+ "alpha_b": 0.65,
2181
+ "shape": [
2182
+ 6144,
2183
+ 2048
2184
+ ],
2185
+ "dtype": "torch.bfloat16"
2186
+ },
2187
+ {
2188
+ "key": "talker.model.layers.18.mlp.up_proj.weight",
2189
+ "group": "talker_mlp",
2190
+ "alpha_a": 0.35,
2191
+ "alpha_b": 0.65,
2192
+ "shape": [
2193
+ 6144,
2194
+ 2048
2195
+ ],
2196
+ "dtype": "torch.bfloat16"
2197
+ },
2198
+ {
2199
+ "key": "talker.model.layers.18.post_attention_layernorm.weight",
2200
+ "group": "layernorm",
2201
+ "alpha_a": 1.0,
2202
+ "alpha_b": 0.0,
2203
+ "shape": [
2204
+ 2048
2205
+ ],
2206
+ "dtype": "torch.bfloat16"
2207
+ },
2208
+ {
2209
+ "key": "talker.model.layers.18.self_attn.k_norm.weight",
2210
+ "group": "layernorm",
2211
+ "alpha_a": 1.0,
2212
+ "alpha_b": 0.0,
2213
+ "shape": [
2214
+ 128
2215
+ ],
2216
+ "dtype": "torch.bfloat16"
2217
+ },
2218
+ {
2219
+ "key": "talker.model.layers.18.self_attn.k_proj.weight",
2220
+ "group": "talker_attn",
2221
+ "alpha_a": 0.35,
2222
+ "alpha_b": 0.65,
2223
+ "shape": [
2224
+ 1024,
2225
+ 2048
2226
+ ],
2227
+ "dtype": "torch.bfloat16"
2228
+ },
2229
+ {
2230
+ "key": "talker.model.layers.18.self_attn.o_proj.weight",
2231
+ "group": "talker_attn",
2232
+ "alpha_a": 0.35,
2233
+ "alpha_b": 0.65,
2234
+ "shape": [
2235
+ 2048,
2236
+ 2048
2237
+ ],
2238
+ "dtype": "torch.bfloat16"
2239
+ },
2240
+ {
2241
+ "key": "talker.model.layers.18.self_attn.q_norm.weight",
2242
+ "group": "layernorm",
2243
+ "alpha_a": 1.0,
2244
+ "alpha_b": 0.0,
2245
+ "shape": [
2246
+ 128
2247
+ ],
2248
+ "dtype": "torch.bfloat16"
2249
+ },
2250
+ {
2251
+ "key": "talker.model.layers.18.self_attn.q_proj.weight",
2252
+ "group": "talker_attn",
2253
+ "alpha_a": 0.35,
2254
+ "alpha_b": 0.65,
2255
+ "shape": [
2256
+ 2048,
2257
+ 2048
2258
+ ],
2259
+ "dtype": "torch.bfloat16"
2260
+ },
2261
+ {
2262
+ "key": "talker.model.layers.18.self_attn.v_proj.weight",
2263
+ "group": "talker_attn",
2264
+ "alpha_a": 0.35,
2265
+ "alpha_b": 0.65,
2266
+ "shape": [
2267
+ 1024,
2268
+ 2048
2269
+ ],
2270
+ "dtype": "torch.bfloat16"
2271
+ },
2272
+ {
2273
+ "key": "talker.model.layers.19.input_layernorm.weight",
2274
+ "group": "layernorm",
2275
+ "alpha_a": 1.0,
2276
+ "alpha_b": 0.0,
2277
+ "shape": [
2278
+ 2048
2279
+ ],
2280
+ "dtype": "torch.bfloat16"
2281
+ },
2282
+ {
2283
+ "key": "talker.model.layers.19.mlp.down_proj.weight",
2284
+ "group": "talker_mlp",
2285
+ "alpha_a": 0.35,
2286
+ "alpha_b": 0.65,
2287
+ "shape": [
2288
+ 2048,
2289
+ 6144
2290
+ ],
2291
+ "dtype": "torch.bfloat16"
2292
+ },
2293
+ {
2294
+ "key": "talker.model.layers.19.mlp.gate_proj.weight",
2295
+ "group": "talker_mlp",
2296
+ "alpha_a": 0.35,
2297
+ "alpha_b": 0.65,
2298
+ "shape": [
2299
+ 6144,
2300
+ 2048
2301
+ ],
2302
+ "dtype": "torch.bfloat16"
2303
+ },
2304
+ {
2305
+ "key": "talker.model.layers.19.mlp.up_proj.weight",
2306
+ "group": "talker_mlp",
2307
+ "alpha_a": 0.35,
2308
+ "alpha_b": 0.65,
2309
+ "shape": [
2310
+ 6144,
2311
+ 2048
2312
+ ],
2313
+ "dtype": "torch.bfloat16"
2314
+ },
2315
+ {
2316
+ "key": "talker.model.layers.19.post_attention_layernorm.weight",
2317
+ "group": "layernorm",
2318
+ "alpha_a": 1.0,
2319
+ "alpha_b": 0.0,
2320
+ "shape": [
2321
+ 2048
2322
+ ],
2323
+ "dtype": "torch.bfloat16"
2324
+ },
2325
+ {
2326
+ "key": "talker.model.layers.19.self_attn.k_norm.weight",
2327
+ "group": "layernorm",
2328
+ "alpha_a": 1.0,
2329
+ "alpha_b": 0.0,
2330
+ "shape": [
2331
+ 128
2332
+ ],
2333
+ "dtype": "torch.bfloat16"
2334
+ },
2335
+ {
2336
+ "key": "talker.model.layers.19.self_attn.k_proj.weight",
2337
+ "group": "talker_attn",
2338
+ "alpha_a": 0.35,
2339
+ "alpha_b": 0.65,
2340
+ "shape": [
2341
+ 1024,
2342
+ 2048
2343
+ ],
2344
+ "dtype": "torch.bfloat16"
2345
+ },
2346
+ {
2347
+ "key": "talker.model.layers.19.self_attn.o_proj.weight",
2348
+ "group": "talker_attn",
2349
+ "alpha_a": 0.35,
2350
+ "alpha_b": 0.65,
2351
+ "shape": [
2352
+ 2048,
2353
+ 2048
2354
+ ],
2355
+ "dtype": "torch.bfloat16"
2356
+ },
2357
+ {
2358
+ "key": "talker.model.layers.19.self_attn.q_norm.weight",
2359
+ "group": "layernorm",
2360
+ "alpha_a": 1.0,
2361
+ "alpha_b": 0.0,
2362
+ "shape": [
2363
+ 128
2364
+ ],
2365
+ "dtype": "torch.bfloat16"
2366
+ },
2367
+ {
2368
+ "key": "talker.model.layers.19.self_attn.q_proj.weight",
2369
+ "group": "talker_attn",
2370
+ "alpha_a": 0.35,
2371
+ "alpha_b": 0.65,
2372
+ "shape": [
2373
+ 2048,
2374
+ 2048
2375
+ ],
2376
+ "dtype": "torch.bfloat16"
2377
+ },
2378
+ {
2379
+ "key": "talker.model.layers.19.self_attn.v_proj.weight",
2380
+ "group": "talker_attn",
2381
+ "alpha_a": 0.35,
2382
+ "alpha_b": 0.65,
2383
+ "shape": [
2384
+ 1024,
2385
+ 2048
2386
+ ],
2387
+ "dtype": "torch.bfloat16"
2388
+ },
2389
+ {
2390
+ "key": "talker.model.layers.2.input_layernorm.weight",
2391
+ "group": "layernorm",
2392
+ "alpha_a": 1.0,
2393
+ "alpha_b": 0.0,
2394
+ "shape": [
2395
+ 2048
2396
+ ],
2397
+ "dtype": "torch.bfloat16"
2398
+ },
2399
+ {
2400
+ "key": "talker.model.layers.2.mlp.down_proj.weight",
2401
+ "group": "talker_mlp",
2402
+ "alpha_a": 0.35,
2403
+ "alpha_b": 0.65,
2404
+ "shape": [
2405
+ 2048,
2406
+ 6144
2407
+ ],
2408
+ "dtype": "torch.bfloat16"
2409
+ },
2410
+ {
2411
+ "key": "talker.model.layers.2.mlp.gate_proj.weight",
2412
+ "group": "talker_mlp",
2413
+ "alpha_a": 0.35,
2414
+ "alpha_b": 0.65,
2415
+ "shape": [
2416
+ 6144,
2417
+ 2048
2418
+ ],
2419
+ "dtype": "torch.bfloat16"
2420
+ },
2421
+ {
2422
+ "key": "talker.model.layers.2.mlp.up_proj.weight",
2423
+ "group": "talker_mlp",
2424
+ "alpha_a": 0.35,
2425
+ "alpha_b": 0.65,
2426
+ "shape": [
2427
+ 6144,
2428
+ 2048
2429
+ ],
2430
+ "dtype": "torch.bfloat16"
2431
+ },
2432
+ {
2433
+ "key": "talker.model.layers.2.post_attention_layernorm.weight",
2434
+ "group": "layernorm",
2435
+ "alpha_a": 1.0,
2436
+ "alpha_b": 0.0,
2437
+ "shape": [
2438
+ 2048
2439
+ ],
2440
+ "dtype": "torch.bfloat16"
2441
+ },
2442
+ {
2443
+ "key": "talker.model.layers.2.self_attn.k_norm.weight",
2444
+ "group": "layernorm",
2445
+ "alpha_a": 1.0,
2446
+ "alpha_b": 0.0,
2447
+ "shape": [
2448
+ 128
2449
+ ],
2450
+ "dtype": "torch.bfloat16"
2451
+ },
2452
+ {
2453
+ "key": "talker.model.layers.2.self_attn.k_proj.weight",
2454
+ "group": "talker_attn",
2455
+ "alpha_a": 0.35,
2456
+ "alpha_b": 0.65,
2457
+ "shape": [
2458
+ 1024,
2459
+ 2048
2460
+ ],
2461
+ "dtype": "torch.bfloat16"
2462
+ },
2463
+ {
2464
+ "key": "talker.model.layers.2.self_attn.o_proj.weight",
2465
+ "group": "talker_attn",
2466
+ "alpha_a": 0.35,
2467
+ "alpha_b": 0.65,
2468
+ "shape": [
2469
+ 2048,
2470
+ 2048
2471
+ ],
2472
+ "dtype": "torch.bfloat16"
2473
+ },
2474
+ {
2475
+ "key": "talker.model.layers.2.self_attn.q_norm.weight",
2476
+ "group": "layernorm",
2477
+ "alpha_a": 1.0,
2478
+ "alpha_b": 0.0,
2479
+ "shape": [
2480
+ 128
2481
+ ],
2482
+ "dtype": "torch.bfloat16"
2483
+ },
2484
+ {
2485
+ "key": "talker.model.layers.2.self_attn.q_proj.weight",
2486
+ "group": "talker_attn",
2487
+ "alpha_a": 0.35,
2488
+ "alpha_b": 0.65,
2489
+ "shape": [
2490
+ 2048,
2491
+ 2048
2492
+ ],
2493
+ "dtype": "torch.bfloat16"
2494
+ },
2495
+ {
2496
+ "key": "talker.model.layers.2.self_attn.v_proj.weight",
2497
+ "group": "talker_attn",
2498
+ "alpha_a": 0.35,
2499
+ "alpha_b": 0.65,
2500
+ "shape": [
2501
+ 1024,
2502
+ 2048
2503
+ ],
2504
+ "dtype": "torch.bfloat16"
2505
+ },
2506
+ {
2507
+ "key": "talker.model.layers.20.input_layernorm.weight",
2508
+ "group": "layernorm",
2509
+ "alpha_a": 1.0,
2510
+ "alpha_b": 0.0,
2511
+ "shape": [
2512
+ 2048
2513
+ ],
2514
+ "dtype": "torch.bfloat16"
2515
+ },
2516
+ {
2517
+ "key": "talker.model.layers.20.mlp.down_proj.weight",
2518
+ "group": "talker_mlp",
2519
+ "alpha_a": 0.35,
2520
+ "alpha_b": 0.65,
2521
+ "shape": [
2522
+ 2048,
2523
+ 6144
2524
+ ],
2525
+ "dtype": "torch.bfloat16"
2526
+ },
2527
+ {
2528
+ "key": "talker.model.layers.20.mlp.gate_proj.weight",
2529
+ "group": "talker_mlp",
2530
+ "alpha_a": 0.35,
2531
+ "alpha_b": 0.65,
2532
+ "shape": [
2533
+ 6144,
2534
+ 2048
2535
+ ],
2536
+ "dtype": "torch.bfloat16"
2537
+ },
2538
+ {
2539
+ "key": "talker.model.layers.20.mlp.up_proj.weight",
2540
+ "group": "talker_mlp",
2541
+ "alpha_a": 0.35,
2542
+ "alpha_b": 0.65,
2543
+ "shape": [
2544
+ 6144,
2545
+ 2048
2546
+ ],
2547
+ "dtype": "torch.bfloat16"
2548
+ },
2549
+ {
2550
+ "key": "talker.model.layers.20.post_attention_layernorm.weight",
2551
+ "group": "layernorm",
2552
+ "alpha_a": 1.0,
2553
+ "alpha_b": 0.0,
2554
+ "shape": [
2555
+ 2048
2556
+ ],
2557
+ "dtype": "torch.bfloat16"
2558
+ },
2559
+ {
2560
+ "key": "talker.model.layers.20.self_attn.k_norm.weight",
2561
+ "group": "layernorm",
2562
+ "alpha_a": 1.0,
2563
+ "alpha_b": 0.0,
2564
+ "shape": [
2565
+ 128
2566
+ ],
2567
+ "dtype": "torch.bfloat16"
2568
+ },
2569
+ {
2570
+ "key": "talker.model.layers.20.self_attn.k_proj.weight",
2571
+ "group": "talker_attn",
2572
+ "alpha_a": 0.35,
2573
+ "alpha_b": 0.65,
2574
+ "shape": [
2575
+ 1024,
2576
+ 2048
2577
+ ],
2578
+ "dtype": "torch.bfloat16"
2579
+ },
2580
+ {
2581
+ "key": "talker.model.layers.20.self_attn.o_proj.weight",
2582
+ "group": "talker_attn",
2583
+ "alpha_a": 0.35,
2584
+ "alpha_b": 0.65,
2585
+ "shape": [
2586
+ 2048,
2587
+ 2048
2588
+ ],
2589
+ "dtype": "torch.bfloat16"
2590
+ },
2591
+ {
2592
+ "key": "talker.model.layers.20.self_attn.q_norm.weight",
2593
+ "group": "layernorm",
2594
+ "alpha_a": 1.0,
2595
+ "alpha_b": 0.0,
2596
+ "shape": [
2597
+ 128
2598
+ ],
2599
+ "dtype": "torch.bfloat16"
2600
+ },
2601
+ {
2602
+ "key": "talker.model.layers.20.self_attn.q_proj.weight",
2603
+ "group": "talker_attn",
2604
+ "alpha_a": 0.35,
2605
+ "alpha_b": 0.65,
2606
+ "shape": [
2607
+ 2048,
2608
+ 2048
2609
+ ],
2610
+ "dtype": "torch.bfloat16"
2611
+ },
2612
+ {
2613
+ "key": "talker.model.layers.20.self_attn.v_proj.weight",
2614
+ "group": "talker_attn",
2615
+ "alpha_a": 0.35,
2616
+ "alpha_b": 0.65,
2617
+ "shape": [
2618
+ 1024,
2619
+ 2048
2620
+ ],
2621
+ "dtype": "torch.bfloat16"
2622
+ },
2623
+ {
2624
+ "key": "talker.model.layers.21.input_layernorm.weight",
2625
+ "group": "layernorm",
2626
+ "alpha_a": 1.0,
2627
+ "alpha_b": 0.0,
2628
+ "shape": [
2629
+ 2048
2630
+ ],
2631
+ "dtype": "torch.bfloat16"
2632
+ },
2633
+ {
2634
+ "key": "talker.model.layers.21.mlp.down_proj.weight",
2635
+ "group": "talker_mlp",
2636
+ "alpha_a": 0.35,
2637
+ "alpha_b": 0.65,
2638
+ "shape": [
2639
+ 2048,
2640
+ 6144
2641
+ ],
2642
+ "dtype": "torch.bfloat16"
2643
+ },
2644
+ {
2645
+ "key": "talker.model.layers.21.mlp.gate_proj.weight",
2646
+ "group": "talker_mlp",
2647
+ "alpha_a": 0.35,
2648
+ "alpha_b": 0.65,
2649
+ "shape": [
2650
+ 6144,
2651
+ 2048
2652
+ ],
2653
+ "dtype": "torch.bfloat16"
2654
+ },
2655
+ {
2656
+ "key": "talker.model.layers.21.mlp.up_proj.weight",
2657
+ "group": "talker_mlp",
2658
+ "alpha_a": 0.35,
2659
+ "alpha_b": 0.65,
2660
+ "shape": [
2661
+ 6144,
2662
+ 2048
2663
+ ],
2664
+ "dtype": "torch.bfloat16"
2665
+ },
2666
+ {
2667
+ "key": "talker.model.layers.21.post_attention_layernorm.weight",
2668
+ "group": "layernorm",
2669
+ "alpha_a": 1.0,
2670
+ "alpha_b": 0.0,
2671
+ "shape": [
2672
+ 2048
2673
+ ],
2674
+ "dtype": "torch.bfloat16"
2675
+ },
2676
+ {
2677
+ "key": "talker.model.layers.21.self_attn.k_norm.weight",
2678
+ "group": "layernorm",
2679
+ "alpha_a": 1.0,
2680
+ "alpha_b": 0.0,
2681
+ "shape": [
2682
+ 128
2683
+ ],
2684
+ "dtype": "torch.bfloat16"
2685
+ },
2686
+ {
2687
+ "key": "talker.model.layers.21.self_attn.k_proj.weight",
2688
+ "group": "talker_attn",
2689
+ "alpha_a": 0.35,
2690
+ "alpha_b": 0.65,
2691
+ "shape": [
2692
+ 1024,
2693
+ 2048
2694
+ ],
2695
+ "dtype": "torch.bfloat16"
2696
+ },
2697
+ {
2698
+ "key": "talker.model.layers.21.self_attn.o_proj.weight",
2699
+ "group": "talker_attn",
2700
+ "alpha_a": 0.35,
2701
+ "alpha_b": 0.65,
2702
+ "shape": [
2703
+ 2048,
2704
+ 2048
2705
+ ],
2706
+ "dtype": "torch.bfloat16"
2707
+ },
2708
+ {
2709
+ "key": "talker.model.layers.21.self_attn.q_norm.weight",
2710
+ "group": "layernorm",
2711
+ "alpha_a": 1.0,
2712
+ "alpha_b": 0.0,
2713
+ "shape": [
2714
+ 128
2715
+ ],
2716
+ "dtype": "torch.bfloat16"
2717
+ },
2718
+ {
2719
+ "key": "talker.model.layers.21.self_attn.q_proj.weight",
2720
+ "group": "talker_attn",
2721
+ "alpha_a": 0.35,
2722
+ "alpha_b": 0.65,
2723
+ "shape": [
2724
+ 2048,
2725
+ 2048
2726
+ ],
2727
+ "dtype": "torch.bfloat16"
2728
+ },
2729
+ {
2730
+ "key": "talker.model.layers.21.self_attn.v_proj.weight",
2731
+ "group": "talker_attn",
2732
+ "alpha_a": 0.35,
2733
+ "alpha_b": 0.65,
2734
+ "shape": [
2735
+ 1024,
2736
+ 2048
2737
+ ],
2738
+ "dtype": "torch.bfloat16"
2739
+ },
2740
+ {
2741
+ "key": "talker.model.layers.22.input_layernorm.weight",
2742
+ "group": "layernorm",
2743
+ "alpha_a": 1.0,
2744
+ "alpha_b": 0.0,
2745
+ "shape": [
2746
+ 2048
2747
+ ],
2748
+ "dtype": "torch.bfloat16"
2749
+ },
2750
+ {
2751
+ "key": "talker.model.layers.22.mlp.down_proj.weight",
2752
+ "group": "talker_mlp",
2753
+ "alpha_a": 0.35,
2754
+ "alpha_b": 0.65,
2755
+ "shape": [
2756
+ 2048,
2757
+ 6144
2758
+ ],
2759
+ "dtype": "torch.bfloat16"
2760
+ },
2761
+ {
2762
+ "key": "talker.model.layers.22.mlp.gate_proj.weight",
2763
+ "group": "talker_mlp",
2764
+ "alpha_a": 0.35,
2765
+ "alpha_b": 0.65,
2766
+ "shape": [
2767
+ 6144,
2768
+ 2048
2769
+ ],
2770
+ "dtype": "torch.bfloat16"
2771
+ },
2772
+ {
2773
+ "key": "talker.model.layers.22.mlp.up_proj.weight",
2774
+ "group": "talker_mlp",
2775
+ "alpha_a": 0.35,
2776
+ "alpha_b": 0.65,
2777
+ "shape": [
2778
+ 6144,
2779
+ 2048
2780
+ ],
2781
+ "dtype": "torch.bfloat16"
2782
+ },
2783
+ {
2784
+ "key": "talker.model.layers.22.post_attention_layernorm.weight",
2785
+ "group": "layernorm",
2786
+ "alpha_a": 1.0,
2787
+ "alpha_b": 0.0,
2788
+ "shape": [
2789
+ 2048
2790
+ ],
2791
+ "dtype": "torch.bfloat16"
2792
+ },
2793
+ {
2794
+ "key": "talker.model.layers.22.self_attn.k_norm.weight",
2795
+ "group": "layernorm",
2796
+ "alpha_a": 1.0,
2797
+ "alpha_b": 0.0,
2798
+ "shape": [
2799
+ 128
2800
+ ],
2801
+ "dtype": "torch.bfloat16"
2802
+ },
2803
+ {
2804
+ "key": "talker.model.layers.22.self_attn.k_proj.weight",
2805
+ "group": "talker_attn",
2806
+ "alpha_a": 0.35,
2807
+ "alpha_b": 0.65,
2808
+ "shape": [
2809
+ 1024,
2810
+ 2048
2811
+ ],
2812
+ "dtype": "torch.bfloat16"
2813
+ },
2814
+ {
2815
+ "key": "talker.model.layers.22.self_attn.o_proj.weight",
2816
+ "group": "talker_attn",
2817
+ "alpha_a": 0.35,
2818
+ "alpha_b": 0.65,
2819
+ "shape": [
2820
+ 2048,
2821
+ 2048
2822
+ ],
2823
+ "dtype": "torch.bfloat16"
2824
+ },
2825
+ {
2826
+ "key": "talker.model.layers.22.self_attn.q_norm.weight",
2827
+ "group": "layernorm",
2828
+ "alpha_a": 1.0,
2829
+ "alpha_b": 0.0,
2830
+ "shape": [
2831
+ 128
2832
+ ],
2833
+ "dtype": "torch.bfloat16"
2834
+ },
2835
+ {
2836
+ "key": "talker.model.layers.22.self_attn.q_proj.weight",
2837
+ "group": "talker_attn",
2838
+ "alpha_a": 0.35,
2839
+ "alpha_b": 0.65,
2840
+ "shape": [
2841
+ 2048,
2842
+ 2048
2843
+ ],
2844
+ "dtype": "torch.bfloat16"
2845
+ },
2846
+ {
2847
+ "key": "talker.model.layers.22.self_attn.v_proj.weight",
2848
+ "group": "talker_attn",
2849
+ "alpha_a": 0.35,
2850
+ "alpha_b": 0.65,
2851
+ "shape": [
2852
+ 1024,
2853
+ 2048
2854
+ ],
2855
+ "dtype": "torch.bfloat16"
2856
+ },
2857
+ {
2858
+ "key": "talker.model.layers.23.input_layernorm.weight",
2859
+ "group": "layernorm",
2860
+ "alpha_a": 1.0,
2861
+ "alpha_b": 0.0,
2862
+ "shape": [
2863
+ 2048
2864
+ ],
2865
+ "dtype": "torch.bfloat16"
2866
+ },
2867
+ {
2868
+ "key": "talker.model.layers.23.mlp.down_proj.weight",
2869
+ "group": "talker_mlp",
2870
+ "alpha_a": 0.35,
2871
+ "alpha_b": 0.65,
2872
+ "shape": [
2873
+ 2048,
2874
+ 6144
2875
+ ],
2876
+ "dtype": "torch.bfloat16"
2877
+ },
2878
+ {
2879
+ "key": "talker.model.layers.23.mlp.gate_proj.weight",
2880
+ "group": "talker_mlp",
2881
+ "alpha_a": 0.35,
2882
+ "alpha_b": 0.65,
2883
+ "shape": [
2884
+ 6144,
2885
+ 2048
2886
+ ],
2887
+ "dtype": "torch.bfloat16"
2888
+ },
2889
+ {
2890
+ "key": "talker.model.layers.23.mlp.up_proj.weight",
2891
+ "group": "talker_mlp",
2892
+ "alpha_a": 0.35,
2893
+ "alpha_b": 0.65,
2894
+ "shape": [
2895
+ 6144,
2896
+ 2048
2897
+ ],
2898
+ "dtype": "torch.bfloat16"
2899
+ },
2900
+ {
2901
+ "key": "talker.model.layers.23.post_attention_layernorm.weight",
2902
+ "group": "layernorm",
2903
+ "alpha_a": 1.0,
2904
+ "alpha_b": 0.0,
2905
+ "shape": [
2906
+ 2048
2907
+ ],
2908
+ "dtype": "torch.bfloat16"
2909
+ },
2910
+ {
2911
+ "key": "talker.model.layers.23.self_attn.k_norm.weight",
2912
+ "group": "layernorm",
2913
+ "alpha_a": 1.0,
2914
+ "alpha_b": 0.0,
2915
+ "shape": [
2916
+ 128
2917
+ ],
2918
+ "dtype": "torch.bfloat16"
2919
+ },
2920
+ {
2921
+ "key": "talker.model.layers.23.self_attn.k_proj.weight",
2922
+ "group": "talker_attn",
2923
+ "alpha_a": 0.35,
2924
+ "alpha_b": 0.65,
2925
+ "shape": [
2926
+ 1024,
2927
+ 2048
2928
+ ],
2929
+ "dtype": "torch.bfloat16"
2930
+ },
2931
+ {
2932
+ "key": "talker.model.layers.23.self_attn.o_proj.weight",
2933
+ "group": "talker_attn",
2934
+ "alpha_a": 0.35,
2935
+ "alpha_b": 0.65,
2936
+ "shape": [
2937
+ 2048,
2938
+ 2048
2939
+ ],
2940
+ "dtype": "torch.bfloat16"
2941
+ },
2942
+ {
2943
+ "key": "talker.model.layers.23.self_attn.q_norm.weight",
2944
+ "group": "layernorm",
2945
+ "alpha_a": 1.0,
2946
+ "alpha_b": 0.0,
2947
+ "shape": [
2948
+ 128
2949
+ ],
2950
+ "dtype": "torch.bfloat16"
2951
+ },
2952
+ {
2953
+ "key": "talker.model.layers.23.self_attn.q_proj.weight",
2954
+ "group": "talker_attn",
2955
+ "alpha_a": 0.35,
2956
+ "alpha_b": 0.65,
2957
+ "shape": [
2958
+ 2048,
2959
+ 2048
2960
+ ],
2961
+ "dtype": "torch.bfloat16"
2962
+ },
2963
+ {
2964
+ "key": "talker.model.layers.23.self_attn.v_proj.weight",
2965
+ "group": "talker_attn",
2966
+ "alpha_a": 0.35,
2967
+ "alpha_b": 0.65,
2968
+ "shape": [
2969
+ 1024,
2970
+ 2048
2971
+ ],
2972
+ "dtype": "torch.bfloat16"
2973
+ },
2974
+ {
2975
+ "key": "talker.model.layers.24.input_layernorm.weight",
2976
+ "group": "layernorm",
2977
+ "alpha_a": 1.0,
2978
+ "alpha_b": 0.0,
2979
+ "shape": [
2980
+ 2048
2981
+ ],
2982
+ "dtype": "torch.bfloat16"
2983
+ },
2984
+ {
2985
+ "key": "talker.model.layers.24.mlp.down_proj.weight",
2986
+ "group": "talker_mlp",
2987
+ "alpha_a": 0.35,
2988
+ "alpha_b": 0.65,
2989
+ "shape": [
2990
+ 2048,
2991
+ 6144
2992
+ ],
2993
+ "dtype": "torch.bfloat16"
2994
+ },
2995
+ {
2996
+ "key": "talker.model.layers.24.mlp.gate_proj.weight",
2997
+ "group": "talker_mlp",
2998
+ "alpha_a": 0.35,
2999
+ "alpha_b": 0.65,
3000
+ "shape": [
3001
+ 6144,
3002
+ 2048
3003
+ ],
3004
+ "dtype": "torch.bfloat16"
3005
+ },
3006
+ {
3007
+ "key": "talker.model.layers.24.mlp.up_proj.weight",
3008
+ "group": "talker_mlp",
3009
+ "alpha_a": 0.35,
3010
+ "alpha_b": 0.65,
3011
+ "shape": [
3012
+ 6144,
3013
+ 2048
3014
+ ],
3015
+ "dtype": "torch.bfloat16"
3016
+ },
3017
+ {
3018
+ "key": "talker.model.layers.24.post_attention_layernorm.weight",
3019
+ "group": "layernorm",
3020
+ "alpha_a": 1.0,
3021
+ "alpha_b": 0.0,
3022
+ "shape": [
3023
+ 2048
3024
+ ],
3025
+ "dtype": "torch.bfloat16"
3026
+ },
3027
+ {
3028
+ "key": "talker.model.layers.24.self_attn.k_norm.weight",
3029
+ "group": "layernorm",
3030
+ "alpha_a": 1.0,
3031
+ "alpha_b": 0.0,
3032
+ "shape": [
3033
+ 128
3034
+ ],
3035
+ "dtype": "torch.bfloat16"
3036
+ },
3037
+ {
3038
+ "key": "talker.model.layers.24.self_attn.k_proj.weight",
3039
+ "group": "talker_attn",
3040
+ "alpha_a": 0.35,
3041
+ "alpha_b": 0.65,
3042
+ "shape": [
3043
+ 1024,
3044
+ 2048
3045
+ ],
3046
+ "dtype": "torch.bfloat16"
3047
+ },
3048
+ {
3049
+ "key": "talker.model.layers.24.self_attn.o_proj.weight",
3050
+ "group": "talker_attn",
3051
+ "alpha_a": 0.35,
3052
+ "alpha_b": 0.65,
3053
+ "shape": [
3054
+ 2048,
3055
+ 2048
3056
+ ],
3057
+ "dtype": "torch.bfloat16"
3058
+ },
3059
+ {
3060
+ "key": "talker.model.layers.24.self_attn.q_norm.weight",
3061
+ "group": "layernorm",
3062
+ "alpha_a": 1.0,
3063
+ "alpha_b": 0.0,
3064
+ "shape": [
3065
+ 128
3066
+ ],
3067
+ "dtype": "torch.bfloat16"
3068
+ },
3069
+ {
3070
+ "key": "talker.model.layers.24.self_attn.q_proj.weight",
3071
+ "group": "talker_attn",
3072
+ "alpha_a": 0.35,
3073
+ "alpha_b": 0.65,
3074
+ "shape": [
3075
+ 2048,
3076
+ 2048
3077
+ ],
3078
+ "dtype": "torch.bfloat16"
3079
+ },
3080
+ {
3081
+ "key": "talker.model.layers.24.self_attn.v_proj.weight",
3082
+ "group": "talker_attn",
3083
+ "alpha_a": 0.35,
3084
+ "alpha_b": 0.65,
3085
+ "shape": [
3086
+ 1024,
3087
+ 2048
3088
+ ],
3089
+ "dtype": "torch.bfloat16"
3090
+ },
3091
+ {
3092
+ "key": "talker.model.layers.25.input_layernorm.weight",
3093
+ "group": "layernorm",
3094
+ "alpha_a": 1.0,
3095
+ "alpha_b": 0.0,
3096
+ "shape": [
3097
+ 2048
3098
+ ],
3099
+ "dtype": "torch.bfloat16"
3100
+ },
3101
+ {
3102
+ "key": "talker.model.layers.25.mlp.down_proj.weight",
3103
+ "group": "talker_mlp",
3104
+ "alpha_a": 0.35,
3105
+ "alpha_b": 0.65,
3106
+ "shape": [
3107
+ 2048,
3108
+ 6144
3109
+ ],
3110
+ "dtype": "torch.bfloat16"
3111
+ },
3112
+ {
3113
+ "key": "talker.model.layers.25.mlp.gate_proj.weight",
3114
+ "group": "talker_mlp",
3115
+ "alpha_a": 0.35,
3116
+ "alpha_b": 0.65,
3117
+ "shape": [
3118
+ 6144,
3119
+ 2048
3120
+ ],
3121
+ "dtype": "torch.bfloat16"
3122
+ },
3123
+ {
3124
+ "key": "talker.model.layers.25.mlp.up_proj.weight",
3125
+ "group": "talker_mlp",
3126
+ "alpha_a": 0.35,
3127
+ "alpha_b": 0.65,
3128
+ "shape": [
3129
+ 6144,
3130
+ 2048
3131
+ ],
3132
+ "dtype": "torch.bfloat16"
3133
+ },
3134
+ {
3135
+ "key": "talker.model.layers.25.post_attention_layernorm.weight",
3136
+ "group": "layernorm",
3137
+ "alpha_a": 1.0,
3138
+ "alpha_b": 0.0,
3139
+ "shape": [
3140
+ 2048
3141
+ ],
3142
+ "dtype": "torch.bfloat16"
3143
+ },
3144
+ {
3145
+ "key": "talker.model.layers.25.self_attn.k_norm.weight",
3146
+ "group": "layernorm",
3147
+ "alpha_a": 1.0,
3148
+ "alpha_b": 0.0,
3149
+ "shape": [
3150
+ 128
3151
+ ],
3152
+ "dtype": "torch.bfloat16"
3153
+ },
3154
+ {
3155
+ "key": "talker.model.layers.25.self_attn.k_proj.weight",
3156
+ "group": "talker_attn",
3157
+ "alpha_a": 0.35,
3158
+ "alpha_b": 0.65,
3159
+ "shape": [
3160
+ 1024,
3161
+ 2048
3162
+ ],
3163
+ "dtype": "torch.bfloat16"
3164
+ },
3165
+ {
3166
+ "key": "talker.model.layers.25.self_attn.o_proj.weight",
3167
+ "group": "talker_attn",
3168
+ "alpha_a": 0.35,
3169
+ "alpha_b": 0.65,
3170
+ "shape": [
3171
+ 2048,
3172
+ 2048
3173
+ ],
3174
+ "dtype": "torch.bfloat16"
3175
+ },
3176
+ {
3177
+ "key": "talker.model.layers.25.self_attn.q_norm.weight",
3178
+ "group": "layernorm",
3179
+ "alpha_a": 1.0,
3180
+ "alpha_b": 0.0,
3181
+ "shape": [
3182
+ 128
3183
+ ],
3184
+ "dtype": "torch.bfloat16"
3185
+ },
3186
+ {
3187
+ "key": "talker.model.layers.25.self_attn.q_proj.weight",
3188
+ "group": "talker_attn",
3189
+ "alpha_a": 0.35,
3190
+ "alpha_b": 0.65,
3191
+ "shape": [
3192
+ 2048,
3193
+ 2048
3194
+ ],
3195
+ "dtype": "torch.bfloat16"
3196
+ },
3197
+ {
3198
+ "key": "talker.model.layers.25.self_attn.v_proj.weight",
3199
+ "group": "talker_attn",
3200
+ "alpha_a": 0.35,
3201
+ "alpha_b": 0.65,
3202
+ "shape": [
3203
+ 1024,
3204
+ 2048
3205
+ ],
3206
+ "dtype": "torch.bfloat16"
3207
+ },
3208
+ {
3209
+ "key": "talker.model.layers.26.input_layernorm.weight",
3210
+ "group": "layernorm",
3211
+ "alpha_a": 1.0,
3212
+ "alpha_b": 0.0,
3213
+ "shape": [
3214
+ 2048
3215
+ ],
3216
+ "dtype": "torch.bfloat16"
3217
+ },
3218
+ {
3219
+ "key": "talker.model.layers.26.mlp.down_proj.weight",
3220
+ "group": "talker_mlp",
3221
+ "alpha_a": 0.35,
3222
+ "alpha_b": 0.65,
3223
+ "shape": [
3224
+ 2048,
3225
+ 6144
3226
+ ],
3227
+ "dtype": "torch.bfloat16"
3228
+ },
3229
+ {
3230
+ "key": "talker.model.layers.26.mlp.gate_proj.weight",
3231
+ "group": "talker_mlp",
3232
+ "alpha_a": 0.35,
3233
+ "alpha_b": 0.65,
3234
+ "shape": [
3235
+ 6144,
3236
+ 2048
3237
+ ],
3238
+ "dtype": "torch.bfloat16"
3239
+ },
3240
+ {
3241
+ "key": "talker.model.layers.26.mlp.up_proj.weight",
3242
+ "group": "talker_mlp",
3243
+ "alpha_a": 0.35,
3244
+ "alpha_b": 0.65,
3245
+ "shape": [
3246
+ 6144,
3247
+ 2048
3248
+ ],
3249
+ "dtype": "torch.bfloat16"
3250
+ },
3251
+ {
3252
+ "key": "talker.model.layers.26.post_attention_layernorm.weight",
3253
+ "group": "layernorm",
3254
+ "alpha_a": 1.0,
3255
+ "alpha_b": 0.0,
3256
+ "shape": [
3257
+ 2048
3258
+ ],
3259
+ "dtype": "torch.bfloat16"
3260
+ },
3261
+ {
3262
+ "key": "talker.model.layers.26.self_attn.k_norm.weight",
3263
+ "group": "layernorm",
3264
+ "alpha_a": 1.0,
3265
+ "alpha_b": 0.0,
3266
+ "shape": [
3267
+ 128
3268
+ ],
3269
+ "dtype": "torch.bfloat16"
3270
+ },
3271
+ {
3272
+ "key": "talker.model.layers.26.self_attn.k_proj.weight",
3273
+ "group": "talker_attn",
3274
+ "alpha_a": 0.35,
3275
+ "alpha_b": 0.65,
3276
+ "shape": [
3277
+ 1024,
3278
+ 2048
3279
+ ],
3280
+ "dtype": "torch.bfloat16"
3281
+ },
3282
+ {
3283
+ "key": "talker.model.layers.26.self_attn.o_proj.weight",
3284
+ "group": "talker_attn",
3285
+ "alpha_a": 0.35,
3286
+ "alpha_b": 0.65,
3287
+ "shape": [
3288
+ 2048,
3289
+ 2048
3290
+ ],
3291
+ "dtype": "torch.bfloat16"
3292
+ },
3293
+ {
3294
+ "key": "talker.model.layers.26.self_attn.q_norm.weight",
3295
+ "group": "layernorm",
3296
+ "alpha_a": 1.0,
3297
+ "alpha_b": 0.0,
3298
+ "shape": [
3299
+ 128
3300
+ ],
3301
+ "dtype": "torch.bfloat16"
3302
+ },
3303
+ {
3304
+ "key": "talker.model.layers.26.self_attn.q_proj.weight",
3305
+ "group": "talker_attn",
3306
+ "alpha_a": 0.35,
3307
+ "alpha_b": 0.65,
3308
+ "shape": [
3309
+ 2048,
3310
+ 2048
3311
+ ],
3312
+ "dtype": "torch.bfloat16"
3313
+ },
3314
+ {
3315
+ "key": "talker.model.layers.26.self_attn.v_proj.weight",
3316
+ "group": "talker_attn",
3317
+ "alpha_a": 0.35,
3318
+ "alpha_b": 0.65,
3319
+ "shape": [
3320
+ 1024,
3321
+ 2048
3322
+ ],
3323
+ "dtype": "torch.bfloat16"
3324
+ },
3325
+ {
3326
+ "key": "talker.model.layers.27.input_layernorm.weight",
3327
+ "group": "layernorm",
3328
+ "alpha_a": 1.0,
3329
+ "alpha_b": 0.0,
3330
+ "shape": [
3331
+ 2048
3332
+ ],
3333
+ "dtype": "torch.bfloat16"
3334
+ },
3335
+ {
3336
+ "key": "talker.model.layers.27.mlp.down_proj.weight",
3337
+ "group": "talker_mlp",
3338
+ "alpha_a": 0.35,
3339
+ "alpha_b": 0.65,
3340
+ "shape": [
3341
+ 2048,
3342
+ 6144
3343
+ ],
3344
+ "dtype": "torch.bfloat16"
3345
+ },
3346
+ {
3347
+ "key": "talker.model.layers.27.mlp.gate_proj.weight",
3348
+ "group": "talker_mlp",
3349
+ "alpha_a": 0.35,
3350
+ "alpha_b": 0.65,
3351
+ "shape": [
3352
+ 6144,
3353
+ 2048
3354
+ ],
3355
+ "dtype": "torch.bfloat16"
3356
+ },
3357
+ {
3358
+ "key": "talker.model.layers.27.mlp.up_proj.weight",
3359
+ "group": "talker_mlp",
3360
+ "alpha_a": 0.35,
3361
+ "alpha_b": 0.65,
3362
+ "shape": [
3363
+ 6144,
3364
+ 2048
3365
+ ],
3366
+ "dtype": "torch.bfloat16"
3367
+ },
3368
+ {
3369
+ "key": "talker.model.layers.27.post_attention_layernorm.weight",
3370
+ "group": "layernorm",
3371
+ "alpha_a": 1.0,
3372
+ "alpha_b": 0.0,
3373
+ "shape": [
3374
+ 2048
3375
+ ],
3376
+ "dtype": "torch.bfloat16"
3377
+ },
3378
+ {
3379
+ "key": "talker.model.layers.27.self_attn.k_norm.weight",
3380
+ "group": "layernorm",
3381
+ "alpha_a": 1.0,
3382
+ "alpha_b": 0.0,
3383
+ "shape": [
3384
+ 128
3385
+ ],
3386
+ "dtype": "torch.bfloat16"
3387
+ },
3388
+ {
3389
+ "key": "talker.model.layers.27.self_attn.k_proj.weight",
3390
+ "group": "talker_attn",
3391
+ "alpha_a": 0.35,
3392
+ "alpha_b": 0.65,
3393
+ "shape": [
3394
+ 1024,
3395
+ 2048
3396
+ ],
3397
+ "dtype": "torch.bfloat16"
3398
+ },
3399
+ {
3400
+ "key": "talker.model.layers.27.self_attn.o_proj.weight",
3401
+ "group": "talker_attn",
3402
+ "alpha_a": 0.35,
3403
+ "alpha_b": 0.65,
3404
+ "shape": [
3405
+ 2048,
3406
+ 2048
3407
+ ],
3408
+ "dtype": "torch.bfloat16"
3409
+ },
3410
+ {
3411
+ "key": "talker.model.layers.27.self_attn.q_norm.weight",
3412
+ "group": "layernorm",
3413
+ "alpha_a": 1.0,
3414
+ "alpha_b": 0.0,
3415
+ "shape": [
3416
+ 128
3417
+ ],
3418
+ "dtype": "torch.bfloat16"
3419
+ },
3420
+ {
3421
+ "key": "talker.model.layers.27.self_attn.q_proj.weight",
3422
+ "group": "talker_attn",
3423
+ "alpha_a": 0.35,
3424
+ "alpha_b": 0.65,
3425
+ "shape": [
3426
+ 2048,
3427
+ 2048
3428
+ ],
3429
+ "dtype": "torch.bfloat16"
3430
+ },
3431
+ {
3432
+ "key": "talker.model.layers.27.self_attn.v_proj.weight",
3433
+ "group": "talker_attn",
3434
+ "alpha_a": 0.35,
3435
+ "alpha_b": 0.65,
3436
+ "shape": [
3437
+ 1024,
3438
+ 2048
3439
+ ],
3440
+ "dtype": "torch.bfloat16"
3441
+ },
3442
+ {
3443
+ "key": "talker.model.layers.3.input_layernorm.weight",
3444
+ "group": "layernorm",
3445
+ "alpha_a": 1.0,
3446
+ "alpha_b": 0.0,
3447
+ "shape": [
3448
+ 2048
3449
+ ],
3450
+ "dtype": "torch.bfloat16"
3451
+ },
3452
+ {
3453
+ "key": "talker.model.layers.3.mlp.down_proj.weight",
3454
+ "group": "talker_mlp",
3455
+ "alpha_a": 0.35,
3456
+ "alpha_b": 0.65,
3457
+ "shape": [
3458
+ 2048,
3459
+ 6144
3460
+ ],
3461
+ "dtype": "torch.bfloat16"
3462
+ },
3463
+ {
3464
+ "key": "talker.model.layers.3.mlp.gate_proj.weight",
3465
+ "group": "talker_mlp",
3466
+ "alpha_a": 0.35,
3467
+ "alpha_b": 0.65,
3468
+ "shape": [
3469
+ 6144,
3470
+ 2048
3471
+ ],
3472
+ "dtype": "torch.bfloat16"
3473
+ },
3474
+ {
3475
+ "key": "talker.model.layers.3.mlp.up_proj.weight",
3476
+ "group": "talker_mlp",
3477
+ "alpha_a": 0.35,
3478
+ "alpha_b": 0.65,
3479
+ "shape": [
3480
+ 6144,
3481
+ 2048
3482
+ ],
3483
+ "dtype": "torch.bfloat16"
3484
+ },
3485
+ {
3486
+ "key": "talker.model.layers.3.post_attention_layernorm.weight",
3487
+ "group": "layernorm",
3488
+ "alpha_a": 1.0,
3489
+ "alpha_b": 0.0,
3490
+ "shape": [
3491
+ 2048
3492
+ ],
3493
+ "dtype": "torch.bfloat16"
3494
+ },
3495
+ {
3496
+ "key": "talker.model.layers.3.self_attn.k_norm.weight",
3497
+ "group": "layernorm",
3498
+ "alpha_a": 1.0,
3499
+ "alpha_b": 0.0,
3500
+ "shape": [
3501
+ 128
3502
+ ],
3503
+ "dtype": "torch.bfloat16"
3504
+ },
3505
+ {
3506
+ "key": "talker.model.layers.3.self_attn.k_proj.weight",
3507
+ "group": "talker_attn",
3508
+ "alpha_a": 0.35,
3509
+ "alpha_b": 0.65,
3510
+ "shape": [
3511
+ 1024,
3512
+ 2048
3513
+ ],
3514
+ "dtype": "torch.bfloat16"
3515
+ },
3516
+ {
3517
+ "key": "talker.model.layers.3.self_attn.o_proj.weight",
3518
+ "group": "talker_attn",
3519
+ "alpha_a": 0.35,
3520
+ "alpha_b": 0.65,
3521
+ "shape": [
3522
+ 2048,
3523
+ 2048
3524
+ ],
3525
+ "dtype": "torch.bfloat16"
3526
+ },
3527
+ {
3528
+ "key": "talker.model.layers.3.self_attn.q_norm.weight",
3529
+ "group": "layernorm",
3530
+ "alpha_a": 1.0,
3531
+ "alpha_b": 0.0,
3532
+ "shape": [
3533
+ 128
3534
+ ],
3535
+ "dtype": "torch.bfloat16"
3536
+ },
3537
+ {
3538
+ "key": "talker.model.layers.3.self_attn.q_proj.weight",
3539
+ "group": "talker_attn",
3540
+ "alpha_a": 0.35,
3541
+ "alpha_b": 0.65,
3542
+ "shape": [
3543
+ 2048,
3544
+ 2048
3545
+ ],
3546
+ "dtype": "torch.bfloat16"
3547
+ },
3548
+ {
3549
+ "key": "talker.model.layers.3.self_attn.v_proj.weight",
3550
+ "group": "talker_attn",
3551
+ "alpha_a": 0.35,
3552
+ "alpha_b": 0.65,
3553
+ "shape": [
3554
+ 1024,
3555
+ 2048
3556
+ ],
3557
+ "dtype": "torch.bfloat16"
3558
+ },
3559
+ {
3560
+ "key": "talker.model.layers.4.input_layernorm.weight",
3561
+ "group": "layernorm",
3562
+ "alpha_a": 1.0,
3563
+ "alpha_b": 0.0,
3564
+ "shape": [
3565
+ 2048
3566
+ ],
3567
+ "dtype": "torch.bfloat16"
3568
+ },
3569
+ {
3570
+ "key": "talker.model.layers.4.mlp.down_proj.weight",
3571
+ "group": "talker_mlp",
3572
+ "alpha_a": 0.35,
3573
+ "alpha_b": 0.65,
3574
+ "shape": [
3575
+ 2048,
3576
+ 6144
3577
+ ],
3578
+ "dtype": "torch.bfloat16"
3579
+ },
3580
+ {
3581
+ "key": "talker.model.layers.4.mlp.gate_proj.weight",
3582
+ "group": "talker_mlp",
3583
+ "alpha_a": 0.35,
3584
+ "alpha_b": 0.65,
3585
+ "shape": [
3586
+ 6144,
3587
+ 2048
3588
+ ],
3589
+ "dtype": "torch.bfloat16"
3590
+ },
3591
+ {
3592
+ "key": "talker.model.layers.4.mlp.up_proj.weight",
3593
+ "group": "talker_mlp",
3594
+ "alpha_a": 0.35,
3595
+ "alpha_b": 0.65,
3596
+ "shape": [
3597
+ 6144,
3598
+ 2048
3599
+ ],
3600
+ "dtype": "torch.bfloat16"
3601
+ },
3602
+ {
3603
+ "key": "talker.model.layers.4.post_attention_layernorm.weight",
3604
+ "group": "layernorm",
3605
+ "alpha_a": 1.0,
3606
+ "alpha_b": 0.0,
3607
+ "shape": [
3608
+ 2048
3609
+ ],
3610
+ "dtype": "torch.bfloat16"
3611
+ },
3612
+ {
3613
+ "key": "talker.model.layers.4.self_attn.k_norm.weight",
3614
+ "group": "layernorm",
3615
+ "alpha_a": 1.0,
3616
+ "alpha_b": 0.0,
3617
+ "shape": [
3618
+ 128
3619
+ ],
3620
+ "dtype": "torch.bfloat16"
3621
+ },
3622
+ {
3623
+ "key": "talker.model.layers.4.self_attn.k_proj.weight",
3624
+ "group": "talker_attn",
3625
+ "alpha_a": 0.35,
3626
+ "alpha_b": 0.65,
3627
+ "shape": [
3628
+ 1024,
3629
+ 2048
3630
+ ],
3631
+ "dtype": "torch.bfloat16"
3632
+ },
3633
+ {
3634
+ "key": "talker.model.layers.4.self_attn.o_proj.weight",
3635
+ "group": "talker_attn",
3636
+ "alpha_a": 0.35,
3637
+ "alpha_b": 0.65,
3638
+ "shape": [
3639
+ 2048,
3640
+ 2048
3641
+ ],
3642
+ "dtype": "torch.bfloat16"
3643
+ },
3644
+ {
3645
+ "key": "talker.model.layers.4.self_attn.q_norm.weight",
3646
+ "group": "layernorm",
3647
+ "alpha_a": 1.0,
3648
+ "alpha_b": 0.0,
3649
+ "shape": [
3650
+ 128
3651
+ ],
3652
+ "dtype": "torch.bfloat16"
3653
+ },
3654
+ {
3655
+ "key": "talker.model.layers.4.self_attn.q_proj.weight",
3656
+ "group": "talker_attn",
3657
+ "alpha_a": 0.35,
3658
+ "alpha_b": 0.65,
3659
+ "shape": [
3660
+ 2048,
3661
+ 2048
3662
+ ],
3663
+ "dtype": "torch.bfloat16"
3664
+ },
3665
+ {
3666
+ "key": "talker.model.layers.4.self_attn.v_proj.weight",
3667
+ "group": "talker_attn",
3668
+ "alpha_a": 0.35,
3669
+ "alpha_b": 0.65,
3670
+ "shape": [
3671
+ 1024,
3672
+ 2048
3673
+ ],
3674
+ "dtype": "torch.bfloat16"
3675
+ },
3676
+ {
3677
+ "key": "talker.model.layers.5.input_layernorm.weight",
3678
+ "group": "layernorm",
3679
+ "alpha_a": 1.0,
3680
+ "alpha_b": 0.0,
3681
+ "shape": [
3682
+ 2048
3683
+ ],
3684
+ "dtype": "torch.bfloat16"
3685
+ },
3686
+ {
3687
+ "key": "talker.model.layers.5.mlp.down_proj.weight",
3688
+ "group": "talker_mlp",
3689
+ "alpha_a": 0.35,
3690
+ "alpha_b": 0.65,
3691
+ "shape": [
3692
+ 2048,
3693
+ 6144
3694
+ ],
3695
+ "dtype": "torch.bfloat16"
3696
+ },
3697
+ {
3698
+ "key": "talker.model.layers.5.mlp.gate_proj.weight",
3699
+ "group": "talker_mlp",
3700
+ "alpha_a": 0.35,
3701
+ "alpha_b": 0.65,
3702
+ "shape": [
3703
+ 6144,
3704
+ 2048
3705
+ ],
3706
+ "dtype": "torch.bfloat16"
3707
+ },
3708
+ {
3709
+ "key": "talker.model.layers.5.mlp.up_proj.weight",
3710
+ "group": "talker_mlp",
3711
+ "alpha_a": 0.35,
3712
+ "alpha_b": 0.65,
3713
+ "shape": [
3714
+ 6144,
3715
+ 2048
3716
+ ],
3717
+ "dtype": "torch.bfloat16"
3718
+ },
3719
+ {
3720
+ "key": "talker.model.layers.5.post_attention_layernorm.weight",
3721
+ "group": "layernorm",
3722
+ "alpha_a": 1.0,
3723
+ "alpha_b": 0.0,
3724
+ "shape": [
3725
+ 2048
3726
+ ],
3727
+ "dtype": "torch.bfloat16"
3728
+ },
3729
+ {
3730
+ "key": "talker.model.layers.5.self_attn.k_norm.weight",
3731
+ "group": "layernorm",
3732
+ "alpha_a": 1.0,
3733
+ "alpha_b": 0.0,
3734
+ "shape": [
3735
+ 128
3736
+ ],
3737
+ "dtype": "torch.bfloat16"
3738
+ },
3739
+ {
3740
+ "key": "talker.model.layers.5.self_attn.k_proj.weight",
3741
+ "group": "talker_attn",
3742
+ "alpha_a": 0.35,
3743
+ "alpha_b": 0.65,
3744
+ "shape": [
3745
+ 1024,
3746
+ 2048
3747
+ ],
3748
+ "dtype": "torch.bfloat16"
3749
+ },
3750
+ {
3751
+ "key": "talker.model.layers.5.self_attn.o_proj.weight",
3752
+ "group": "talker_attn",
3753
+ "alpha_a": 0.35,
3754
+ "alpha_b": 0.65,
3755
+ "shape": [
3756
+ 2048,
3757
+ 2048
3758
+ ],
3759
+ "dtype": "torch.bfloat16"
3760
+ },
3761
+ {
3762
+ "key": "talker.model.layers.5.self_attn.q_norm.weight",
3763
+ "group": "layernorm",
3764
+ "alpha_a": 1.0,
3765
+ "alpha_b": 0.0,
3766
+ "shape": [
3767
+ 128
3768
+ ],
3769
+ "dtype": "torch.bfloat16"
3770
+ },
3771
+ {
3772
+ "key": "talker.model.layers.5.self_attn.q_proj.weight",
3773
+ "group": "talker_attn",
3774
+ "alpha_a": 0.35,
3775
+ "alpha_b": 0.65,
3776
+ "shape": [
3777
+ 2048,
3778
+ 2048
3779
+ ],
3780
+ "dtype": "torch.bfloat16"
3781
+ },
3782
+ {
3783
+ "key": "talker.model.layers.5.self_attn.v_proj.weight",
3784
+ "group": "talker_attn",
3785
+ "alpha_a": 0.35,
3786
+ "alpha_b": 0.65,
3787
+ "shape": [
3788
+ 1024,
3789
+ 2048
3790
+ ],
3791
+ "dtype": "torch.bfloat16"
3792
+ },
3793
+ {
3794
+ "key": "talker.model.layers.6.input_layernorm.weight",
3795
+ "group": "layernorm",
3796
+ "alpha_a": 1.0,
3797
+ "alpha_b": 0.0,
3798
+ "shape": [
3799
+ 2048
3800
+ ],
3801
+ "dtype": "torch.bfloat16"
3802
+ },
3803
+ {
3804
+ "key": "talker.model.layers.6.mlp.down_proj.weight",
3805
+ "group": "talker_mlp",
3806
+ "alpha_a": 0.35,
3807
+ "alpha_b": 0.65,
3808
+ "shape": [
3809
+ 2048,
3810
+ 6144
3811
+ ],
3812
+ "dtype": "torch.bfloat16"
3813
+ },
3814
+ {
3815
+ "key": "talker.model.layers.6.mlp.gate_proj.weight",
3816
+ "group": "talker_mlp",
3817
+ "alpha_a": 0.35,
3818
+ "alpha_b": 0.65,
3819
+ "shape": [
3820
+ 6144,
3821
+ 2048
3822
+ ],
3823
+ "dtype": "torch.bfloat16"
3824
+ },
3825
+ {
3826
+ "key": "talker.model.layers.6.mlp.up_proj.weight",
3827
+ "group": "talker_mlp",
3828
+ "alpha_a": 0.35,
3829
+ "alpha_b": 0.65,
3830
+ "shape": [
3831
+ 6144,
3832
+ 2048
3833
+ ],
3834
+ "dtype": "torch.bfloat16"
3835
+ },
3836
+ {
3837
+ "key": "talker.model.layers.6.post_attention_layernorm.weight",
3838
+ "group": "layernorm",
3839
+ "alpha_a": 1.0,
3840
+ "alpha_b": 0.0,
3841
+ "shape": [
3842
+ 2048
3843
+ ],
3844
+ "dtype": "torch.bfloat16"
3845
+ },
3846
+ {
3847
+ "key": "talker.model.layers.6.self_attn.k_norm.weight",
3848
+ "group": "layernorm",
3849
+ "alpha_a": 1.0,
3850
+ "alpha_b": 0.0,
3851
+ "shape": [
3852
+ 128
3853
+ ],
3854
+ "dtype": "torch.bfloat16"
3855
+ },
3856
+ {
3857
+ "key": "talker.model.layers.6.self_attn.k_proj.weight",
3858
+ "group": "talker_attn",
3859
+ "alpha_a": 0.35,
3860
+ "alpha_b": 0.65,
3861
+ "shape": [
3862
+ 1024,
3863
+ 2048
3864
+ ],
3865
+ "dtype": "torch.bfloat16"
3866
+ },
3867
+ {
3868
+ "key": "talker.model.layers.6.self_attn.o_proj.weight",
3869
+ "group": "talker_attn",
3870
+ "alpha_a": 0.35,
3871
+ "alpha_b": 0.65,
3872
+ "shape": [
3873
+ 2048,
3874
+ 2048
3875
+ ],
3876
+ "dtype": "torch.bfloat16"
3877
+ },
3878
+ {
3879
+ "key": "talker.model.layers.6.self_attn.q_norm.weight",
3880
+ "group": "layernorm",
3881
+ "alpha_a": 1.0,
3882
+ "alpha_b": 0.0,
3883
+ "shape": [
3884
+ 128
3885
+ ],
3886
+ "dtype": "torch.bfloat16"
3887
+ },
3888
+ {
3889
+ "key": "talker.model.layers.6.self_attn.q_proj.weight",
3890
+ "group": "talker_attn",
3891
+ "alpha_a": 0.35,
3892
+ "alpha_b": 0.65,
3893
+ "shape": [
3894
+ 2048,
3895
+ 2048
3896
+ ],
3897
+ "dtype": "torch.bfloat16"
3898
+ },
3899
+ {
3900
+ "key": "talker.model.layers.6.self_attn.v_proj.weight",
3901
+ "group": "talker_attn",
3902
+ "alpha_a": 0.35,
3903
+ "alpha_b": 0.65,
3904
+ "shape": [
3905
+ 1024,
3906
+ 2048
3907
+ ],
3908
+ "dtype": "torch.bfloat16"
3909
+ },
3910
+ {
3911
+ "key": "talker.model.layers.7.input_layernorm.weight",
3912
+ "group": "layernorm",
3913
+ "alpha_a": 1.0,
3914
+ "alpha_b": 0.0,
3915
+ "shape": [
3916
+ 2048
3917
+ ],
3918
+ "dtype": "torch.bfloat16"
3919
+ },
3920
+ {
3921
+ "key": "talker.model.layers.7.mlp.down_proj.weight",
3922
+ "group": "talker_mlp",
3923
+ "alpha_a": 0.35,
3924
+ "alpha_b": 0.65,
3925
+ "shape": [
3926
+ 2048,
3927
+ 6144
3928
+ ],
3929
+ "dtype": "torch.bfloat16"
3930
+ },
3931
+ {
3932
+ "key": "talker.model.layers.7.mlp.gate_proj.weight",
3933
+ "group": "talker_mlp",
3934
+ "alpha_a": 0.35,
3935
+ "alpha_b": 0.65,
3936
+ "shape": [
3937
+ 6144,
3938
+ 2048
3939
+ ],
3940
+ "dtype": "torch.bfloat16"
3941
+ },
3942
+ {
3943
+ "key": "talker.model.layers.7.mlp.up_proj.weight",
3944
+ "group": "talker_mlp",
3945
+ "alpha_a": 0.35,
3946
+ "alpha_b": 0.65,
3947
+ "shape": [
3948
+ 6144,
3949
+ 2048
3950
+ ],
3951
+ "dtype": "torch.bfloat16"
3952
+ },
3953
+ {
3954
+ "key": "talker.model.layers.7.post_attention_layernorm.weight",
3955
+ "group": "layernorm",
3956
+ "alpha_a": 1.0,
3957
+ "alpha_b": 0.0,
3958
+ "shape": [
3959
+ 2048
3960
+ ],
3961
+ "dtype": "torch.bfloat16"
3962
+ },
3963
+ {
3964
+ "key": "talker.model.layers.7.self_attn.k_norm.weight",
3965
+ "group": "layernorm",
3966
+ "alpha_a": 1.0,
3967
+ "alpha_b": 0.0,
3968
+ "shape": [
3969
+ 128
3970
+ ],
3971
+ "dtype": "torch.bfloat16"
3972
+ },
3973
+ {
3974
+ "key": "talker.model.layers.7.self_attn.k_proj.weight",
3975
+ "group": "talker_attn",
3976
+ "alpha_a": 0.35,
3977
+ "alpha_b": 0.65,
3978
+ "shape": [
3979
+ 1024,
3980
+ 2048
3981
+ ],
3982
+ "dtype": "torch.bfloat16"
3983
+ },
3984
+ {
3985
+ "key": "talker.model.layers.7.self_attn.o_proj.weight",
3986
+ "group": "talker_attn",
3987
+ "alpha_a": 0.35,
3988
+ "alpha_b": 0.65,
3989
+ "shape": [
3990
+ 2048,
3991
+ 2048
3992
+ ],
3993
+ "dtype": "torch.bfloat16"
3994
+ },
3995
+ {
3996
+ "key": "talker.model.layers.7.self_attn.q_norm.weight",
3997
+ "group": "layernorm",
3998
+ "alpha_a": 1.0,
3999
+ "alpha_b": 0.0,
4000
+ "shape": [
4001
+ 128
4002
+ ],
4003
+ "dtype": "torch.bfloat16"
4004
+ },
4005
+ {
4006
+ "key": "talker.model.layers.7.self_attn.q_proj.weight",
4007
+ "group": "talker_attn",
4008
+ "alpha_a": 0.35,
4009
+ "alpha_b": 0.65,
4010
+ "shape": [
4011
+ 2048,
4012
+ 2048
4013
+ ],
4014
+ "dtype": "torch.bfloat16"
4015
+ },
4016
+ {
4017
+ "key": "talker.model.layers.7.self_attn.v_proj.weight",
4018
+ "group": "talker_attn",
4019
+ "alpha_a": 0.35,
4020
+ "alpha_b": 0.65,
4021
+ "shape": [
4022
+ 1024,
4023
+ 2048
4024
+ ],
4025
+ "dtype": "torch.bfloat16"
4026
+ },
4027
+ {
4028
+ "key": "talker.model.layers.8.input_layernorm.weight",
4029
+ "group": "layernorm",
4030
+ "alpha_a": 1.0,
4031
+ "alpha_b": 0.0,
4032
+ "shape": [
4033
+ 2048
4034
+ ],
4035
+ "dtype": "torch.bfloat16"
4036
+ },
4037
+ {
4038
+ "key": "talker.model.layers.8.mlp.down_proj.weight",
4039
+ "group": "talker_mlp",
4040
+ "alpha_a": 0.35,
4041
+ "alpha_b": 0.65,
4042
+ "shape": [
4043
+ 2048,
4044
+ 6144
4045
+ ],
4046
+ "dtype": "torch.bfloat16"
4047
+ },
4048
+ {
4049
+ "key": "talker.model.layers.8.mlp.gate_proj.weight",
4050
+ "group": "talker_mlp",
4051
+ "alpha_a": 0.35,
4052
+ "alpha_b": 0.65,
4053
+ "shape": [
4054
+ 6144,
4055
+ 2048
4056
+ ],
4057
+ "dtype": "torch.bfloat16"
4058
+ },
4059
+ {
4060
+ "key": "talker.model.layers.8.mlp.up_proj.weight",
4061
+ "group": "talker_mlp",
4062
+ "alpha_a": 0.35,
4063
+ "alpha_b": 0.65,
4064
+ "shape": [
4065
+ 6144,
4066
+ 2048
4067
+ ],
4068
+ "dtype": "torch.bfloat16"
4069
+ },
4070
+ {
4071
+ "key": "talker.model.layers.8.post_attention_layernorm.weight",
4072
+ "group": "layernorm",
4073
+ "alpha_a": 1.0,
4074
+ "alpha_b": 0.0,
4075
+ "shape": [
4076
+ 2048
4077
+ ],
4078
+ "dtype": "torch.bfloat16"
4079
+ },
4080
+ {
4081
+ "key": "talker.model.layers.8.self_attn.k_norm.weight",
4082
+ "group": "layernorm",
4083
+ "alpha_a": 1.0,
4084
+ "alpha_b": 0.0,
4085
+ "shape": [
4086
+ 128
4087
+ ],
4088
+ "dtype": "torch.bfloat16"
4089
+ },
4090
+ {
4091
+ "key": "talker.model.layers.8.self_attn.k_proj.weight",
4092
+ "group": "talker_attn",
4093
+ "alpha_a": 0.35,
4094
+ "alpha_b": 0.65,
4095
+ "shape": [
4096
+ 1024,
4097
+ 2048
4098
+ ],
4099
+ "dtype": "torch.bfloat16"
4100
+ },
4101
+ {
4102
+ "key": "talker.model.layers.8.self_attn.o_proj.weight",
4103
+ "group": "talker_attn",
4104
+ "alpha_a": 0.35,
4105
+ "alpha_b": 0.65,
4106
+ "shape": [
4107
+ 2048,
4108
+ 2048
4109
+ ],
4110
+ "dtype": "torch.bfloat16"
4111
+ },
4112
+ {
4113
+ "key": "talker.model.layers.8.self_attn.q_norm.weight",
4114
+ "group": "layernorm",
4115
+ "alpha_a": 1.0,
4116
+ "alpha_b": 0.0,
4117
+ "shape": [
4118
+ 128
4119
+ ],
4120
+ "dtype": "torch.bfloat16"
4121
+ },
4122
+ {
4123
+ "key": "talker.model.layers.8.self_attn.q_proj.weight",
4124
+ "group": "talker_attn",
4125
+ "alpha_a": 0.35,
4126
+ "alpha_b": 0.65,
4127
+ "shape": [
4128
+ 2048,
4129
+ 2048
4130
+ ],
4131
+ "dtype": "torch.bfloat16"
4132
+ },
4133
+ {
4134
+ "key": "talker.model.layers.8.self_attn.v_proj.weight",
4135
+ "group": "talker_attn",
4136
+ "alpha_a": 0.35,
4137
+ "alpha_b": 0.65,
4138
+ "shape": [
4139
+ 1024,
4140
+ 2048
4141
+ ],
4142
+ "dtype": "torch.bfloat16"
4143
+ },
4144
+ {
4145
+ "key": "talker.model.layers.9.input_layernorm.weight",
4146
+ "group": "layernorm",
4147
+ "alpha_a": 1.0,
4148
+ "alpha_b": 0.0,
4149
+ "shape": [
4150
+ 2048
4151
+ ],
4152
+ "dtype": "torch.bfloat16"
4153
+ },
4154
+ {
4155
+ "key": "talker.model.layers.9.mlp.down_proj.weight",
4156
+ "group": "talker_mlp",
4157
+ "alpha_a": 0.35,
4158
+ "alpha_b": 0.65,
4159
+ "shape": [
4160
+ 2048,
4161
+ 6144
4162
+ ],
4163
+ "dtype": "torch.bfloat16"
4164
+ },
4165
+ {
4166
+ "key": "talker.model.layers.9.mlp.gate_proj.weight",
4167
+ "group": "talker_mlp",
4168
+ "alpha_a": 0.35,
4169
+ "alpha_b": 0.65,
4170
+ "shape": [
4171
+ 6144,
4172
+ 2048
4173
+ ],
4174
+ "dtype": "torch.bfloat16"
4175
+ },
4176
+ {
4177
+ "key": "talker.model.layers.9.mlp.up_proj.weight",
4178
+ "group": "talker_mlp",
4179
+ "alpha_a": 0.35,
4180
+ "alpha_b": 0.65,
4181
+ "shape": [
4182
+ 6144,
4183
+ 2048
4184
+ ],
4185
+ "dtype": "torch.bfloat16"
4186
+ },
4187
+ {
4188
+ "key": "talker.model.layers.9.post_attention_layernorm.weight",
4189
+ "group": "layernorm",
4190
+ "alpha_a": 1.0,
4191
+ "alpha_b": 0.0,
4192
+ "shape": [
4193
+ 2048
4194
+ ],
4195
+ "dtype": "torch.bfloat16"
4196
+ },
4197
+ {
4198
+ "key": "talker.model.layers.9.self_attn.k_norm.weight",
4199
+ "group": "layernorm",
4200
+ "alpha_a": 1.0,
4201
+ "alpha_b": 0.0,
4202
+ "shape": [
4203
+ 128
4204
+ ],
4205
+ "dtype": "torch.bfloat16"
4206
+ },
4207
+ {
4208
+ "key": "talker.model.layers.9.self_attn.k_proj.weight",
4209
+ "group": "talker_attn",
4210
+ "alpha_a": 0.35,
4211
+ "alpha_b": 0.65,
4212
+ "shape": [
4213
+ 1024,
4214
+ 2048
4215
+ ],
4216
+ "dtype": "torch.bfloat16"
4217
+ },
4218
+ {
4219
+ "key": "talker.model.layers.9.self_attn.o_proj.weight",
4220
+ "group": "talker_attn",
4221
+ "alpha_a": 0.35,
4222
+ "alpha_b": 0.65,
4223
+ "shape": [
4224
+ 2048,
4225
+ 2048
4226
+ ],
4227
+ "dtype": "torch.bfloat16"
4228
+ },
4229
+ {
4230
+ "key": "talker.model.layers.9.self_attn.q_norm.weight",
4231
+ "group": "layernorm",
4232
+ "alpha_a": 1.0,
4233
+ "alpha_b": 0.0,
4234
+ "shape": [
4235
+ 128
4236
+ ],
4237
+ "dtype": "torch.bfloat16"
4238
+ },
4239
+ {
4240
+ "key": "talker.model.layers.9.self_attn.q_proj.weight",
4241
+ "group": "talker_attn",
4242
+ "alpha_a": 0.35,
4243
+ "alpha_b": 0.65,
4244
+ "shape": [
4245
+ 2048,
4246
+ 2048
4247
+ ],
4248
+ "dtype": "torch.bfloat16"
4249
+ },
4250
+ {
4251
+ "key": "talker.model.layers.9.self_attn.v_proj.weight",
4252
+ "group": "talker_attn",
4253
+ "alpha_a": 0.35,
4254
+ "alpha_b": 0.65,
4255
+ "shape": [
4256
+ 1024,
4257
+ 2048
4258
+ ],
4259
+ "dtype": "torch.bfloat16"
4260
+ },
4261
+ {
4262
+ "key": "talker.model.norm.weight",
4263
+ "group": "layernorm",
4264
+ "alpha_a": 1.0,
4265
+ "alpha_b": 0.0,
4266
+ "shape": [
4267
+ 2048
4268
+ ],
4269
+ "dtype": "torch.bfloat16"
4270
+ },
4271
+ {
4272
+ "key": "talker.model.text_embedding.weight",
4273
+ "group": "embedding",
4274
+ "alpha_a": 1.0,
4275
+ "alpha_b": 0.0,
4276
+ "shape": [
4277
+ 151936,
4278
+ 2048
4279
+ ],
4280
+ "dtype": "torch.bfloat16"
4281
+ },
4282
+ {
4283
+ "key": "talker.text_projection.linear_fc1.bias",
4284
+ "group": "other_frozen",
4285
+ "alpha_a": 1.0,
4286
+ "alpha_b": 0.0,
4287
+ "shape": [
4288
+ 2048
4289
+ ],
4290
+ "dtype": "torch.bfloat16"
4291
+ },
4292
+ {
4293
+ "key": "talker.text_projection.linear_fc1.weight",
4294
+ "group": "other_frozen",
4295
+ "alpha_a": 1.0,
4296
+ "alpha_b": 0.0,
4297
+ "shape": [
4298
+ 2048,
4299
+ 2048
4300
+ ],
4301
+ "dtype": "torch.bfloat16"
4302
+ },
4303
+ {
4304
+ "key": "talker.text_projection.linear_fc2.bias",
4305
+ "group": "other_frozen",
4306
+ "alpha_a": 1.0,
4307
+ "alpha_b": 0.0,
4308
+ "shape": [
4309
+ 2048
4310
+ ],
4311
+ "dtype": "torch.bfloat16"
4312
+ },
4313
+ {
4314
+ "key": "talker.text_projection.linear_fc2.weight",
4315
+ "group": "other_frozen",
4316
+ "alpha_a": 1.0,
4317
+ "alpha_b": 0.0,
4318
+ "shape": [
4319
+ 2048,
4320
+ 2048
4321
+ ],
4322
+ "dtype": "torch.bfloat16"
4323
+ }
4324
+ ]
4325
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
miner.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Vocence TTS engine: Qwen3 12Hz checkpoint in the HF repo snapshot.
3
+
4
+ The chute snapshot is the only weight source: nothing is pulled from an external
5
+ model id at inference time. Optional vocence_config.yaml tweaks device, dtype,
6
+ attention, and language defaults.
7
+
8
+ Model load: Miner.__init__ -> _instantiate_qwen() -> Qwen3TTSModel.from_pretrained(repo_path).
9
+
10
+ Contract (Vocence):
11
+ Miner(path_hf_repo: Path)
12
+ warmup() -> None
13
+ generate_wav(instruction: str, text: str) -> tuple[np.ndarray, int]
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import threading
18
+ from pathlib import Path
19
+ from typing import Any, Mapping
20
+
21
+ import numpy as np
22
+
23
+ _CONFIG_NAME = "config.json"
24
+ _VOCENCE_YAML = "vocence_config.yaml"
25
+
26
+
27
+ def _merge_vocence_yaml(repo: Path) -> dict[str, Any]:
28
+ path = repo / _VOCENCE_YAML
29
+ if not path.is_file():
30
+ return {}
31
+ from yaml import safe_load
32
+
33
+ with path.open("r", encoding="utf-8") as fh:
34
+ data = safe_load(fh)
35
+ return data if isinstance(data, Mapping) else {}
36
+
37
+
38
+ def _ensure_repo_checkpoint(repo: Path) -> Path:
39
+ repo = repo.resolve()
40
+ marker = repo / _CONFIG_NAME
41
+ if not marker.is_file():
42
+ raise FileNotFoundError(
43
+ f"Model snapshot incomplete: {marker} missing. "
44
+ "Host the full Qwen3-TTS weights (checkpoint + tokenizers) in this repository."
45
+ )
46
+ return repo
47
+
48
+
49
+ def _resolve_compute_device(prefer_cuda: bool) -> str:
50
+ import torch
51
+
52
+ if prefer_cuda and torch.cuda.is_available():
53
+ return "cuda:0"
54
+ return "cpu"
55
+
56
+
57
+ def _resolve_torch_dtype(torch, prefer_bf16: bool):
58
+ if prefer_bf16 and torch.cuda.is_available():
59
+ return torch.bfloat16
60
+ return torch.float32
61
+
62
+
63
+ def _instantiate_qwen(checkpoint_dir: str, device_map: str, torch_dtype, use_flash2: bool):
64
+ """Load Qwen3TTSModel weights from the local repo directory (HF snapshot path)."""
65
+ from qwen_tts import Qwen3TTSModel
66
+
67
+ attn = "flash_attention_2" if use_flash2 else "sdpa"
68
+ common = dict(
69
+ pretrained_model_name_or_path=checkpoint_dir,
70
+ device_map=device_map,
71
+ dtype=torch_dtype,
72
+ attn_implementation=attn,
73
+ )
74
+ try:
75
+ return Qwen3TTSModel.from_pretrained(**common)
76
+ except Exception:
77
+ common["attn_implementation"] = "sdpa"
78
+ return Qwen3TTSModel.from_pretrained(**common)
79
+
80
+
81
+ def _to_mono_f32(segment: np.ndarray) -> np.ndarray:
82
+ x = np.asarray(segment, dtype=np.float32)
83
+ if x.ndim > 1:
84
+ x = x.mean(axis=1)
85
+ return x
86
+
87
+
88
+ class Miner:
89
+ """
90
+ Loads the checkpoint from the Hugging Face repo directory Chutes downloaded.
91
+ Synthesis uses natural-language instruction + text (qwen-tts API).
92
+ """
93
+
94
+ def __init__(self, path_hf_repo: Path) -> None:
95
+ self._root = _ensure_repo_checkpoint(Path(path_hf_repo))
96
+ self._cfg = _merge_vocence_yaml(self._root)
97
+ rt = self._cfg.get("runtime") or {}
98
+ gen = self._cfg.get("generation") or {}
99
+ lim = self._cfg.get("limits") or {}
100
+
101
+ self._language = str(lim.get("default_language") or rt.get("default_language", "English"))
102
+ self._output_sr = int(gen.get("sample_rate", 24000))
103
+ self._cap_instruction = int(lim.get("max_instruction_chars", 600))
104
+ self._cap_text = int(lim.get("max_text_chars", 2000))
105
+
106
+ prefer_cuda = str(rt.get("device_preference", "cuda")).lower() == "cuda"
107
+ want_bf16 = str(rt.get("dtype", "bfloat16")).lower() == "bfloat16"
108
+ flash = bool(rt.get("use_flash_attention_2", False))
109
+
110
+ import torch
111
+
112
+ device_map = _resolve_compute_device(prefer_cuda)
113
+ torch_dtype = _resolve_torch_dtype(torch, want_bf16)
114
+ ckpt = str(self._root)
115
+
116
+ self._tts = _instantiate_qwen(ckpt, device_map, torch_dtype, flash)
117
+ # Qwen3TTSModel is a thin wrapper, not nn.Module — no .eval()
118
+ print("Qwen3-TTS checkpoint ready (loaded from repo snapshot).")
119
+
120
+ def __repr__(self) -> str:
121
+ return "Miner(qwen3-tts-local, local_snapshot=True)"
122
+
123
+ def warmup(self) -> None:
124
+ """Force one cheap synthesis on a background thread (startup SLAs)."""
125
+ status: dict[str, object] = {"done": False, "error": None}
126
+
127
+ def _once() -> None:
128
+ try:
129
+ self.generate_wav(
130
+ instruction="Clear, neutral delivery.",
131
+ text="Warmup.",
132
+ )
133
+ status["done"] = True
134
+ except Exception as exc: # noqa: BLE001 — surface to host
135
+ status["error"] = str(exc)
136
+
137
+ worker = threading.Thread(target=_once, daemon=True)
138
+ worker.start()
139
+ worker.join(timeout=180.0)
140
+ if not status["done"]:
141
+ raise RuntimeError(status["error"] or "warmup exceeded 180s")
142
+
143
+ def generate_wav(self, instruction: str, text: str) -> tuple[np.ndarray, int]:
144
+ if self._cap_instruction > 0:
145
+ instruction = instruction[: self._cap_instruction]
146
+ if self._cap_text > 0:
147
+ text = text[: self._cap_text]
148
+
149
+ # Upstream qwen-tts method name (instruct + text -> waveform).
150
+ waves, sr = self._tts.generate_voice_design(
151
+ text=text,
152
+ language=self._language,
153
+ instruct=instruction,
154
+ )
155
+ if not waves:
156
+ raise ValueError("TTS generation returned no audio")
157
+ first = waves[0]
158
+ if first is None:
159
+ raise ValueError("TTS generation returned empty channel")
160
+ return _to_mono_f32(first), int(sr)
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c234efa58331b59d969aa48b410954fcee99ebb085195baeca31d7f17e7e204c
3
+ size 3833402520
preprocessor_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "padding_side": "left",
3
+ "padding_value": 0.0,
4
+ "processor_class": "Qwen3TTSProcessor",
5
+ "return_attention_mask": true
6
+ }
speech_tokenizer/config.json ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3TTSTokenizerV2Model"
4
+ ],
5
+ "model_type": "qwen3_tts_tokenizer_12hz",
6
+ "encoder_valid_num_quantizers": 16,
7
+ "input_sample_rate": 24000,
8
+ "output_sample_rate": 24000,
9
+ "decode_upsample_rate": 1920,
10
+ "encode_downsample_rate": 1920,
11
+ "decoder_config": {
12
+ "attention_bias": false,
13
+ "attention_dropout": 0.0,
14
+ "latent_dim": 1024,
15
+ "codebook_dim": 512,
16
+ "codebook_size": 2048,
17
+ "decoder_dim": 1536,
18
+ "hidden_act": "silu",
19
+ "hidden_size": 512,
20
+ "intermediate_size": 1024,
21
+ "layer_scale_initial_scale": 0.01,
22
+ "max_position_embeddings": 8000,
23
+ "head_dim": 64,
24
+ "num_attention_heads": 16,
25
+ "num_hidden_layers": 8,
26
+ "num_key_value_heads": 16,
27
+ "num_quantizers": 16,
28
+ "num_semantic_quantizers": 1,
29
+ "rms_norm_eps": 1e-05,
30
+ "rope_theta": 10000,
31
+ "semantic_codebook_size": 4096,
32
+ "sliding_window": 72,
33
+ "upsample_rates": [
34
+ 8,
35
+ 5,
36
+ 4,
37
+ 3
38
+ ],
39
+ "upsampling_ratios": [
40
+ 2,
41
+ 2
42
+ ],
43
+ "vector_quantization_hidden_dimension": 512
44
+ },
45
+ "encoder_config": {
46
+ "_frame_rate": 12.5,
47
+ "attention_bias": false,
48
+ "attention_dropout": 0.0,
49
+ "audio_channels": 1,
50
+ "codebook_dim": 256,
51
+ "codebook_size": 2048,
52
+ "compress": 2,
53
+ "dilation_growth_rate": 2,
54
+ "dtype": "float32",
55
+ "head_dim": 64,
56
+ "hidden_act": "gelu",
57
+ "hidden_size": 512,
58
+ "initializer_range": 0.02,
59
+ "intermediate_size": 2048,
60
+ "kernel_size": 7,
61
+ "last_kernel_size": 3,
62
+ "layer_scale_initial_scale": 0.01,
63
+ "max_position_embeddings": 8000,
64
+ "norm_eps": 1e-05,
65
+ "normalize": false,
66
+ "num_attention_heads": 8,
67
+ "num_filters": 64,
68
+ "num_hidden_layers": 8,
69
+ "num_key_value_heads": 8,
70
+ "num_quantizers": 32,
71
+ "num_residual_layers": 1,
72
+ "num_semantic_quantizers": 1,
73
+ "pad_mode": "constant",
74
+ "residual_kernel_size": 3,
75
+ "rope_theta": 10000.0,
76
+ "sampling_rate": 24000,
77
+ "sliding_window": 250,
78
+ "transformers_version": "4.57.0.dev0",
79
+ "trim_right_ratio": 1.0,
80
+ "upsample_groups": 512,
81
+ "upsampling_ratios": [
82
+ 8,
83
+ 6,
84
+ 5,
85
+ 4
86
+ ],
87
+ "use_cache": false,
88
+ "use_causal_conv": true,
89
+ "use_conv_shortcut": false,
90
+ "use_streaming": false,
91
+ "vector_quantization_hidden_dimension": 256
92
+ },
93
+ "transformers_version": "4.57.3"
94
+ }
speech_tokenizer/configuration.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"framework": "pytorch", "task": "feature-extraction", "allow_remote": true}
speech_tokenizer/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258
3
+ size 682293092
speech_tokenizer/preprocessor_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "chunk_length_s": null,
3
+ "feature_extractor_type": "EncodecFeatureExtractor",
4
+ "feature_size": 1,
5
+ "overlap": null,
6
+ "padding_side": "right",
7
+ "padding_value": 0.0,
8
+ "return_attention_mask": true,
9
+ "sampling_rate": 24000
10
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,316 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ },
181
+ "151665": {
182
+ "content": "<tool_response>",
183
+ "lstrip": false,
184
+ "normalized": false,
185
+ "rstrip": false,
186
+ "single_word": false,
187
+ "special": false
188
+ },
189
+ "151666": {
190
+ "content": "</tool_response>",
191
+ "lstrip": false,
192
+ "normalized": false,
193
+ "rstrip": false,
194
+ "single_word": false,
195
+ "special": false
196
+ },
197
+ "151667": {
198
+ "content": "<think>",
199
+ "lstrip": false,
200
+ "normalized": false,
201
+ "rstrip": false,
202
+ "single_word": false,
203
+ "special": false
204
+ },
205
+ "151668": {
206
+ "content": "</think>",
207
+ "lstrip": false,
208
+ "normalized": false,
209
+ "rstrip": false,
210
+ "single_word": false,
211
+ "special": false
212
+ },
213
+ "151669": {
214
+ "content": "<|audio_start|>",
215
+ "lstrip": false,
216
+ "normalized": false,
217
+ "rstrip": false,
218
+ "single_word": false,
219
+ "special": true
220
+ },
221
+ "151670": {
222
+ "content": "<|audio_end|>",
223
+ "lstrip": false,
224
+ "normalized": false,
225
+ "rstrip": false,
226
+ "single_word": false,
227
+ "special": true
228
+ },
229
+ "151671": {
230
+ "content": "<tts_pad>",
231
+ "lstrip": false,
232
+ "normalized": false,
233
+ "rstrip": false,
234
+ "single_word": false,
235
+ "special": true
236
+ },
237
+ "151672": {
238
+ "content": "<tts_text_bos>",
239
+ "lstrip": false,
240
+ "normalized": false,
241
+ "rstrip": false,
242
+ "single_word": false,
243
+ "special": true
244
+ },
245
+ "151673": {
246
+ "content": "<tts_text_eod>",
247
+ "lstrip": false,
248
+ "normalized": false,
249
+ "rstrip": false,
250
+ "single_word": false,
251
+ "special": true
252
+ },
253
+ "151674": {
254
+ "content": "<tts_text_bos_single>",
255
+ "lstrip": false,
256
+ "normalized": false,
257
+ "rstrip": false,
258
+ "single_word": false,
259
+ "special": true
260
+ },
261
+ "151675": {
262
+ "content": "<|audio_pad|>",
263
+ "lstrip": false,
264
+ "normalized": false,
265
+ "rstrip": false,
266
+ "single_word": false,
267
+ "special": true
268
+ }
269
+ },
270
+ "additional_special_tokens": [
271
+ "<|im_start|>",
272
+ "<|im_end|>",
273
+ "<|object_ref_start|>",
274
+ "<|object_ref_end|>",
275
+ "<|box_start|>",
276
+ "<|box_end|>",
277
+ "<|quad_start|>",
278
+ "<|quad_end|>",
279
+ "<|vision_start|>",
280
+ "<|vision_end|>",
281
+ "<|vision_pad|>",
282
+ "<|image_pad|>",
283
+ "<|video_pad|>",
284
+ "<|audio_start|>",
285
+ "<|audio_end|>",
286
+ "<tts_pad>",
287
+ "<tts_text_bos>",
288
+ "<tts_text_bos_single>",
289
+ "<|audio_pad|>"
290
+ ],
291
+ "extra_special_tokens": {
292
+ "image_token": "<|image_pad|>",
293
+ "audio_token": "<|audio_pad|>",
294
+ "video_token": "<|video_pad|>",
295
+ "vision_bos_token": "<|vision_start|>",
296
+ "vision_eos_token": "<|vision_end|>",
297
+ "audio_bos_token": "<|audio_start|>",
298
+ "audio_eos_token": "<|audio_end|>"
299
+ },
300
+ "bos_token": null,
301
+ "clean_up_tokenization_spaces": false,
302
+ "eos_token": "<|im_end|>",
303
+ "errors": "replace",
304
+ "model_max_length": 131072,
305
+ "pad_token": "<|endoftext|>",
306
+ "split_special_tokens": false,
307
+ "tokenizer_class": "Qwen2Tokenizer",
308
+ "unk_token": null,
309
+ "image_token": "<|image_pad|>",
310
+ "audio_token": "<|audio_pad|>",
311
+ "video_token": "<|video_pad|>",
312
+ "vision_bos_token": "<|vision_start|>",
313
+ "vision_eos_token": "<|vision_end|>",
314
+ "audio_bos_token": "<|audio_start|>",
315
+ "audio_eos_token": "<|audio_end|>"
316
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
vocence_config.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Vocence config. model_name MUST equal the on-chain HF repo id (and VOCENCE_REPO in the
2
+ # chute runner). The wrapper refuses to start otherwise. miner.py loads from_pretrained(model_name).
3
+ model_name: "cveavy/train"
4
+
5
+ runtime:
6
+ adapter: "qwen3-tts"
7
+ device_preference: "cuda"
8
+ dtype: "bfloat16"
9
+ use_flash_attention_2: false
10
+ default_language: "English"
11
+
12
+ generation:
13
+ sample_rate: 24000
14
+ max_seconds: 30
15
+ guidance_scale: 1.0
16
+
17
+ io:
18
+ output_format: "wav"
19
+
20
+ limits:
21
+ max_text_chars: 2000
22
+ max_instruction_chars: 600