chenbhao commited on
Commit
f1a17e1
·
1 Parent(s): 7036629

feat: rename & add VAM mini pipeline configs

Browse files

- configs/vam/vam_t2a_all_mini.yaml (Stage 1: T2A, mode=all)
- configs/vam/vam_a2a_audio_proj_mini.yaml (Stage 2: A2A, mode=audio_proj)
- configs/vam/vam_a2a_all_mini.yaml (Stage 3: A2A, mode=all, low LR)

configs/vam/vam_a2a_all_mini.yaml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # VAM SFT 快速验证配置(Stage 3: A2A, mode=all, low LR)
2
+ # 参考 minimind-o mini pipeline: mode=all, from_weight=sft_omni, lr=2e-5, max_seq_len=768
3
+ model:
4
+ hidden_size: 768
5
+ num_hidden_layers: 8
6
+ use_moe: 0
7
+ vocab_size: 6400
8
+ max_seq_len: 768
9
+ num_attention_heads: 8
10
+ num_key_value_heads: 4
11
+ dropout: 0.0
12
+ # Talker(语音生成)相关
13
+ num_talker_hidden_layers: 4
14
+ talker_hidden_size: 768
15
+ # 音频相关
16
+ audio_special_token: "<|audio_pad|>"
17
+ audio_ids: [16]
18
+ audio_hidden_size: 512
19
+ audio_vocab_size: 2112
20
+ audio_pad_token: 2049
21
+ audio_stop_token: 2050
22
+ audio_spk_token: 2051
23
+ spk_emb_size: 192
24
+ think_end_ids: [26, 234, 234]
25
+ # 视觉相关
26
+ image_special_token: "<|image_pad|>"
27
+ image_ids: [12]
28
+ image_hidden_size: 768
29
+ image_token_len: 64
30
+ bridge_layer: 3
31
+
32
+ train:
33
+ epochs: 1
34
+ batch_size: 4
35
+ learning_rate: 2.0e-5
36
+ accumulation_steps: 1
37
+ grad_clip: 1.0
38
+ dtype: bfloat16
39
+ num_workers: 0
40
+ save_interval: 2000
41
+ log_interval: 50
42
+ from_weight: sft_omni
43
+ model_dir: checkpoint/vam_a2a_audio_proj_mini # 加载 Stage 2 输出
44
+ from_resume: 0
45
+ freeze_backbone: none
46
+ mode: all
47
+ max_samples: 2000
48
+
49
+ paths:
50
+ save_dir: checkpoint/vam_a2a_all_mini
51
+ data_path: dataset/vam/sft_a2a_mini.parquet
52
+ audio_encoder_dir: checkpoint/sensevoice
53
+ vision_dir: checkpoint/siglip
54
+ tokenizer_dir: checkpoint/omni/native_hf
configs/vam/vam_a2a_audio_proj_mini.yaml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # VAM SFT 快速验证配置(Stage 2: A2A, mode=audio_proj)
2
+ # 参考 minimind-o mini pipeline: mode=audio_proj, from_weight=sft_omni, lr=5e-4, max_seq_len=640
3
+ model:
4
+ hidden_size: 768
5
+ num_hidden_layers: 8
6
+ use_moe: 0
7
+ vocab_size: 6400
8
+ max_seq_len: 640
9
+ num_attention_heads: 8
10
+ num_key_value_heads: 4
11
+ dropout: 0.0
12
+ # Talker(语音生成)相关
13
+ num_talker_hidden_layers: 4
14
+ talker_hidden_size: 768
15
+ # 音频相关
16
+ audio_special_token: "<|audio_pad|>"
17
+ audio_ids: [16]
18
+ audio_hidden_size: 512
19
+ audio_vocab_size: 2112
20
+ audio_pad_token: 2049
21
+ audio_stop_token: 2050
22
+ audio_spk_token: 2051
23
+ spk_emb_size: 192
24
+ think_end_ids: [26, 234, 234]
25
+ # 视觉相关
26
+ image_special_token: "<|image_pad|>"
27
+ image_ids: [12]
28
+ image_hidden_size: 768
29
+ image_token_len: 64
30
+ bridge_layer: 3
31
+
32
+ train:
33
+ epochs: 1
34
+ batch_size: 8
35
+ learning_rate: 5.0e-4
36
+ accumulation_steps: 1
37
+ grad_clip: 1.0
38
+ dtype: bfloat16
39
+ num_workers: 0
40
+ save_interval: 2000
41
+ log_interval: 50
42
+ from_weight: sft_omni
43
+ model_dir: checkpoint/vam_t2a_all_mini # 加载 Stage 1 输出
44
+ from_resume: 0
45
+ freeze_backbone: none
46
+ mode: audio_proj # 仅训练 audio_proj
47
+ max_samples: 2000
48
+
49
+ paths:
50
+ save_dir: checkpoint/vam_a2a_audio_proj_mini
51
+ data_path: dataset/vam/sft_a2a_mini.parquet
52
+ audio_encoder_dir: checkpoint/sensevoice
53
+ vision_dir: checkpoint/siglip
54
+ tokenizer_dir: checkpoint/omni/native_hf
configs/vam/{vam_sft_mini.yaml → vam_t2a_all_mini.yaml} RENAMED
@@ -1,4 +1,4 @@
1
- # VAM SFT 快速验证配置(Stage 1: T2A)
2
  # 参考 minimind-o mini pipeline: mode=all, from_weight=llm, lr=5e-4, max_seq_len=512
3
  model:
4
  hidden_size: 768
@@ -47,7 +47,7 @@ train:
47
  max_samples: 2000
48
 
49
  paths:
50
- save_dir: checkpoint/vam_sft_mini
51
  data_path: dataset/vam/sft_t2a_mini.parquet
52
  audio_encoder_dir: checkpoint/sensevoice
53
  vision_dir: checkpoint/siglip
 
1
+ # VAM SFT 快速验证配置(Stage 1: T2A, mode=all
2
  # 参考 minimind-o mini pipeline: mode=all, from_weight=llm, lr=5e-4, max_seq_len=512
3
  model:
4
  hidden_size: 768
 
47
  max_samples: 2000
48
 
49
  paths:
50
+ save_dir: checkpoint/vam_t2a_all_mini
51
  data_path: dataset/vam/sft_t2a_mini.parquet
52
  audio_encoder_dir: checkpoint/sensevoice
53
  vision_dir: checkpoint/siglip