chenbhao commited on
Commit
75ec370
·
1 Parent(s): f1a17e1

feat: add VAM T2A config variant using omni-v initialization

Browse files
configs/vam/vam_t2a_all_mini_omni-v.yaml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # VAM SFT 快速验证配置(Stage 1: T2A, mode=all, 从 omni-v 初始化)
2
+ # 从 checkpoint/omni-v/omni-v.pth 加载(已具备视觉能力),继续训练音频
3
+ model:
4
+ hidden_size: 768
5
+ num_hidden_layers: 8
6
+ use_moe: 0
7
+ vocab_size: 6400
8
+ max_seq_len: 768
9
+ num_attention_heads: 8
10
+ num_key_value_heads: 4
11
+ dropout: 0.0
12
+ # Talker(语音生成)相关
13
+ num_talker_hidden_layers: 4
14
+ talker_hidden_size: 768
15
+ # 音频相关
16
+ audio_special_token: "<|audio_pad|>"
17
+ audio_ids: [16]
18
+ audio_hidden_size: 512
19
+ audio_vocab_size: 2112
20
+ audio_pad_token: 2049
21
+ audio_stop_token: 2050
22
+ audio_spk_token: 2051
23
+ spk_emb_size: 192
24
+ think_end_ids: [26, 234, 234]
25
+ # 视觉相关
26
+ image_special_token: "<|image_pad|>"
27
+ image_ids: [12]
28
+ image_hidden_size: 768
29
+ image_token_len: 64
30
+ bridge_layer: 3
31
+
32
+ train:
33
+ epochs: 1
34
+ batch_size: 4
35
+ learning_rate: 5.0e-4
36
+ accumulation_steps: 1
37
+ grad_clip: 1.0
38
+ dtype: bfloat16
39
+ num_workers: 0
40
+ save_interval: 5000
41
+ log_interval: 50
42
+ from_weight: omni-v
43
+ model_dir: checkpoint/omni-v # 加载 checkpoint/omni-v/omni-v.pth(已有 vision_proj)
44
+ from_resume: 0
45
+ freeze_backbone: none
46
+ mode: all
47
+ max_samples: 2000
48
+
49
+ paths:
50
+ save_dir: checkpoint/vam_t2a_all_mini_omni-v
51
+ data_path: dataset/vam/sft_t2a_mini.parquet
52
+ audio_encoder_dir: checkpoint/sensevoice
53
+ vision_dir: checkpoint/siglip
54
+ tokenizer_dir: checkpoint/omni/native_hf