train_msp_fusion.json

{
    "modality": "msp",
    "model_type": "msp",
    "model_name_or_path": null,
    "processor_name_or_path": null,
    "tokenizer_name_or_path": "facebook/wav2vec2-large-robust-ft-libri-960h",
    "freeze_feature_encoder": false,
    "freeze_base_model": false,
    "freeze_audio_branch": true,
    "freeze_visual_branch": true,
    "trust_remote_code": true,
    "ctc_loss_audio_weight": 0.0,
    "ctc_loss_visual_weight": 0.0,
    "ctc_loss_msp_weight": 1.0,
    "modality_dropout_prob": 0.2,
    "audio_dropout_prob": 0.5,
    "visual_dropout_prob": 0.5,
    "streaming_dataset": true,
    "data_cache_dir": "workspace/streaming",
    "include_mcorec": false,
    "remove_unused_columns": false,
    "output_dir": "/workspace/checkpoints-av/msp_fusion",
    "run_name": "msp_fusion_ctc",
    "report_to": [
        "tensorboard",
        "wandb"
    ],
    "max_steps": 10000,
    "per_device_train_batch_size": 32,
    "per_device_eval_batch_size": 32,
    "gradient_accumulation_steps": 1,
    "learning_rate": 0.0002,
    "lr_scheduler_type": "cosine",
    "warmup_steps": 500,
    "weight_decay": 0.01,
    "fp16": false,
    "bf16": true,
    "gradient_checkpointing": false,
    "dataloader_num_workers": 4,
    "dataloader_pin_memory": true,
    "dataloader_drop_last": true,
    "ignore_data_skip": true,
    "eval_strategy": "steps",
    "eval_steps": 500,
    "save_strategy": "steps",
    "save_steps": 500,
    "save_total_limit": 3,
    "load_best_model_at_end": true,
    "metric_for_best_model": "wer",
    "greater_is_better": false,
    "logging_steps": 25,
    "push_to_hub": true,
    "hub_strategy": "checkpoint",
    "hub_model_id": "MahmoodAnaam/MSP-Fusion",
    "resume_from_checkpoint": null
}