Upload folder using huggingface_hub

audiostory_3b/config.yaml

@@ -0,0 +1,114 @@
+seedx_dit_model_cfg:
+  _target_: src.models.mllm.modeling_audiostory_unified.AudioStory_unified
+  rec_loss_type: mse
+  model_dims: 2048
+  lm_loss_scale: 5.0
+  rec_loss_scale: 10.0
+  dit_loss_scale: 1.0
+agent_model:
+  _target_: src.models.mllm.modeling_audiostory_llm.AudioStory_llm.from_pretrained
+  input_resampler: None
+  output_resampler: None
+  whisper_resampler_llava: None
+  mse: true
+  t5_feature_scale: 10
+  audio_feature_scale: 10.0
+  lm_loss_scale: 1.0
+  t5_rec_loss_scale: 5.0
+  target_audio_type: T5
+  model_dims: 2048
+  pretrained_model_path: audioseed_ckpt/seed_omni_t5_multi_audio_duration/seed_omni_qwen_3b_t5_multi_audio_unav_scale10_5e5_loss0105_bz8_genpretrain_withinst_duration_begin0/checkpoint-15000/pytorch_model.bin
+llm_model:
+  _target_: src.models.mllm.peft_models.get_peft_model_with_resize_embedding
+  model:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: ckpt/Qwen2.5-3B-Instruct
+  peft_config:
+    _target_: peft.LoraConfig
+    _convert_: object
+    r: 32
+    lora_alpha: 32
+    modules_to_save:
+    - input_layernorm
+    - post_attention_layernorm
+    - norm
+    target_modules:
+    - q_proj
+    - v_proj
+    - k_proj
+    - o_proj
+    - gate_proj
+    - down_proj
+    - up_proj
+    task_type: CAUSAL_LM
+    lora_dropout: 0.05
+  vocab_size: 152277
+tokenizer:
+  _target_: src.models.tokenizer.init_qwen_tokenizer_special_token.init_tokenizer
+  pretrained_model_path: tokenizer
+  add_tokens_path: tokenizer/added_tokens.json
+train_dataset:
+  _target_: src.data.sft_clm_audio_multi_audio_unav_tomjerry_cotrain.build_multi_datapipes
+  _recursive_: false
+  datapipes:
+  - _target_: src.data.sft_clm_audio_multi_audio_unav_tomjerry_cotrain.build_t2t_Flant5_audiotoken_json_datapipes_qwen_reasoning_captionloss_multi_audio_cotrain_duration
+    data_dir: datasets_audio_json/audio_seedomni_UnAV_multi_audio_generation_instruction_duration_chunk
+    audio_dir: ''
+    max_length: 1300
+    batch_size: 1
+    add_boi_token: false
+    add_gen_prompt: false
+    instruction_prompt: '<|im_start|>user
+
+      {instruction}<|im_end|>
+
+      '
+    assistant_template: '<|im_start|>assistant
+
+      {gen_prompt_response}'
+    system_message: '<|im_start|>system
+
+      You are a helpful assistant.<|im_end|>
+
+      '
+    reasoning_template: <|think|>{reasoning}<|/think|>
+    aud_first_ratio: -1
+    num_t5_in_tokens: 64
+    num_t5_out_tokens: 64
+    num_aud_in_tokens: 8
+    num_aud_out_tokens: 8
+    audio_max_length: 30.0
+    assure_text: true
+    cycle_count: 50
+    multi_resolution: false
+    dataset_name: wavcaps_clotho_audiocaps
+train_args:
+  output_dir: audioseed_ckpt/seed_omni_t5_multi_audio_duration/audiostory_qwen_3b_t5_multi_audio_unav_scale10_1e4_loss0105_bz8_genpretrain_withinst_t5_aud_attn_cotrain_with_mhattn_weight_detokenizer_full_open_1opt_coscale_8token_duration_begin0_new
+  resume_from_checkpoint: null
+  resume_steps: null
+  batch_size: 8
+  learning_rate: 0.0001
+  weight_decay: 0.0001
+  adam_beta1: 0.9
+  adam_beta2: 0.98
+  adam_epsilon: 0.0002
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  mixed_precision: bf16
+  num_train_epochs: 60
+  max_steps: 12000
+  save_steps: 4000
+  lr_scheduler_type: cosine
+  warmup_steps: 300
+  min_lr_ratio: 0.05
+  dataloader_num_workers: 8
+  project_name: ContinuousVLM
+  expr_name: audiostory_qwen_3b_t5_multi_audio_unav_scale10_1e4_loss0105_bz8_genpretrain_withinst_t5_aud_attn_cotrain_with_mhattn_weight_detokenizer_full_open_1opt_coscale_8token_duration_begin0_new
+  unfreeze_agent_model_part: lora
+  freeze_dit: false
+  dit_open_type: full_open
+  use_whisper: true
+  use_detokenizer: true
+  load_pretrained_model: audioseed_ckpt/seed_omni_t5_multi_audio_duration/seed_omni_qwen_3b_t5_multi_audio_unav_scale10_5e5_loss0105_bz8_genpretrain_withinst_duration_begin0/checkpoint-15000
+  pretrain: true
+  zero_attn_last_layer: true

audiostory_3b/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f15dbd66ea12c30f18e9589eb424fddf76599255959155b668bee23ff9426794
+size 9833703234