feat: rename & add VAM mini pipeline configs
Browse files- configs/vam/vam_t2a_all_mini.yaml (Stage 1: T2A, mode=all)
- configs/vam/vam_a2a_audio_proj_mini.yaml (Stage 2: A2A, mode=audio_proj)
- configs/vam/vam_a2a_all_mini.yaml (Stage 3: A2A, mode=all, low LR)
configs/vam/vam_a2a_all_mini.yaml
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# VAM SFT 快速验证配置(Stage 3: A2A, mode=all, low LR)
|
| 2 |
+
# 参考 minimind-o mini pipeline: mode=all, from_weight=sft_omni, lr=2e-5, max_seq_len=768
|
| 3 |
+
model:
|
| 4 |
+
hidden_size: 768
|
| 5 |
+
num_hidden_layers: 8
|
| 6 |
+
use_moe: 0
|
| 7 |
+
vocab_size: 6400
|
| 8 |
+
max_seq_len: 768
|
| 9 |
+
num_attention_heads: 8
|
| 10 |
+
num_key_value_heads: 4
|
| 11 |
+
dropout: 0.0
|
| 12 |
+
# Talker(语音生成)相关
|
| 13 |
+
num_talker_hidden_layers: 4
|
| 14 |
+
talker_hidden_size: 768
|
| 15 |
+
# 音频相关
|
| 16 |
+
audio_special_token: "<|audio_pad|>"
|
| 17 |
+
audio_ids: [16]
|
| 18 |
+
audio_hidden_size: 512
|
| 19 |
+
audio_vocab_size: 2112
|
| 20 |
+
audio_pad_token: 2049
|
| 21 |
+
audio_stop_token: 2050
|
| 22 |
+
audio_spk_token: 2051
|
| 23 |
+
spk_emb_size: 192
|
| 24 |
+
think_end_ids: [26, 234, 234]
|
| 25 |
+
# 视觉相关
|
| 26 |
+
image_special_token: "<|image_pad|>"
|
| 27 |
+
image_ids: [12]
|
| 28 |
+
image_hidden_size: 768
|
| 29 |
+
image_token_len: 64
|
| 30 |
+
bridge_layer: 3
|
| 31 |
+
|
| 32 |
+
train:
|
| 33 |
+
epochs: 1
|
| 34 |
+
batch_size: 4
|
| 35 |
+
learning_rate: 2.0e-5
|
| 36 |
+
accumulation_steps: 1
|
| 37 |
+
grad_clip: 1.0
|
| 38 |
+
dtype: bfloat16
|
| 39 |
+
num_workers: 0
|
| 40 |
+
save_interval: 2000
|
| 41 |
+
log_interval: 50
|
| 42 |
+
from_weight: sft_omni
|
| 43 |
+
model_dir: checkpoint/vam_a2a_audio_proj_mini # 加载 Stage 2 输出
|
| 44 |
+
from_resume: 0
|
| 45 |
+
freeze_backbone: none
|
| 46 |
+
mode: all
|
| 47 |
+
max_samples: 2000
|
| 48 |
+
|
| 49 |
+
paths:
|
| 50 |
+
save_dir: checkpoint/vam_a2a_all_mini
|
| 51 |
+
data_path: dataset/vam/sft_a2a_mini.parquet
|
| 52 |
+
audio_encoder_dir: checkpoint/sensevoice
|
| 53 |
+
vision_dir: checkpoint/siglip
|
| 54 |
+
tokenizer_dir: checkpoint/omni/native_hf
|
configs/vam/vam_a2a_audio_proj_mini.yaml
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# VAM SFT 快速验证配置(Stage 2: A2A, mode=audio_proj)
|
| 2 |
+
# 参考 minimind-o mini pipeline: mode=audio_proj, from_weight=sft_omni, lr=5e-4, max_seq_len=640
|
| 3 |
+
model:
|
| 4 |
+
hidden_size: 768
|
| 5 |
+
num_hidden_layers: 8
|
| 6 |
+
use_moe: 0
|
| 7 |
+
vocab_size: 6400
|
| 8 |
+
max_seq_len: 640
|
| 9 |
+
num_attention_heads: 8
|
| 10 |
+
num_key_value_heads: 4
|
| 11 |
+
dropout: 0.0
|
| 12 |
+
# Talker(语音生成)相关
|
| 13 |
+
num_talker_hidden_layers: 4
|
| 14 |
+
talker_hidden_size: 768
|
| 15 |
+
# 音频相关
|
| 16 |
+
audio_special_token: "<|audio_pad|>"
|
| 17 |
+
audio_ids: [16]
|
| 18 |
+
audio_hidden_size: 512
|
| 19 |
+
audio_vocab_size: 2112
|
| 20 |
+
audio_pad_token: 2049
|
| 21 |
+
audio_stop_token: 2050
|
| 22 |
+
audio_spk_token: 2051
|
| 23 |
+
spk_emb_size: 192
|
| 24 |
+
think_end_ids: [26, 234, 234]
|
| 25 |
+
# 视觉相关
|
| 26 |
+
image_special_token: "<|image_pad|>"
|
| 27 |
+
image_ids: [12]
|
| 28 |
+
image_hidden_size: 768
|
| 29 |
+
image_token_len: 64
|
| 30 |
+
bridge_layer: 3
|
| 31 |
+
|
| 32 |
+
train:
|
| 33 |
+
epochs: 1
|
| 34 |
+
batch_size: 8
|
| 35 |
+
learning_rate: 5.0e-4
|
| 36 |
+
accumulation_steps: 1
|
| 37 |
+
grad_clip: 1.0
|
| 38 |
+
dtype: bfloat16
|
| 39 |
+
num_workers: 0
|
| 40 |
+
save_interval: 2000
|
| 41 |
+
log_interval: 50
|
| 42 |
+
from_weight: sft_omni
|
| 43 |
+
model_dir: checkpoint/vam_t2a_all_mini # 加载 Stage 1 输出
|
| 44 |
+
from_resume: 0
|
| 45 |
+
freeze_backbone: none
|
| 46 |
+
mode: audio_proj # 仅训练 audio_proj
|
| 47 |
+
max_samples: 2000
|
| 48 |
+
|
| 49 |
+
paths:
|
| 50 |
+
save_dir: checkpoint/vam_a2a_audio_proj_mini
|
| 51 |
+
data_path: dataset/vam/sft_a2a_mini.parquet
|
| 52 |
+
audio_encoder_dir: checkpoint/sensevoice
|
| 53 |
+
vision_dir: checkpoint/siglip
|
| 54 |
+
tokenizer_dir: checkpoint/omni/native_hf
|
configs/vam/{vam_sft_mini.yaml → vam_t2a_all_mini.yaml}
RENAMED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# VAM SFT 快速验证配置(Stage 1: T2A)
|
| 2 |
# 参考 minimind-o mini pipeline: mode=all, from_weight=llm, lr=5e-4, max_seq_len=512
|
| 3 |
model:
|
| 4 |
hidden_size: 768
|
|
@@ -47,7 +47,7 @@ train:
|
|
| 47 |
max_samples: 2000
|
| 48 |
|
| 49 |
paths:
|
| 50 |
-
save_dir: checkpoint/
|
| 51 |
data_path: dataset/vam/sft_t2a_mini.parquet
|
| 52 |
audio_encoder_dir: checkpoint/sensevoice
|
| 53 |
vision_dir: checkpoint/siglip
|
|
|
|
| 1 |
+
# VAM SFT 快速验证配置(Stage 1: T2A, mode=all)
|
| 2 |
# 参考 minimind-o mini pipeline: mode=all, from_weight=llm, lr=5e-4, max_seq_len=512
|
| 3 |
model:
|
| 4 |
hidden_size: 768
|
|
|
|
| 47 |
max_samples: 2000
|
| 48 |
|
| 49 |
paths:
|
| 50 |
+
save_dir: checkpoint/vam_t2a_all_mini
|
| 51 |
data_path: dataset/vam/sft_t2a_mini.parquet
|
| 52 |
audio_encoder_dir: checkpoint/sensevoice
|
| 53 |
vision_dir: checkpoint/siglip
|