train_msp_visual.json

{
  "modality": "msp_visual",
  "model_type": "msp_visual",
  "model_name_or_path": null,
  "processor_name_or_path": null,
  "tokenizer_name_or_path": "facebook/wav2vec2-large-robust-ft-libri-960h",
  "freeze_feature_encoder": true,
  "freeze_base_model": false,
  "freeze_audio_branch": false,
  "freeze_visual_branch": false,
  "trust_remote_code": true,
  "streaming_dataset": false,
  "data_cache_dir": "/workspace/data/cache",
  "include_mcorec": false,
  "remove_unused_columns": false,
  "output_dir": "/workspace/checkpoints-av/msp_visual",
  "run_name": "msp_visual_ctc",
  "report_to": [
    "tensorboard",
    "wandb"
  ],
  "max_steps": 20000,
  "per_device_train_batch_size": 32,
  "per_device_eval_batch_size": 32,
  "gradient_accumulation_steps": 1,
  "learning_rate": 1e-4,
  "lr_scheduler_type": "cosine",
  "warmup_steps": 1000.0,
  "weight_decay": 0.005,
  "max_grad_norm": 1.0,
  "fp16": false,
  "bf16": true,
  "gradient_checkpointing": false,
  "dataloader_num_workers": 4,
  "dataloader_pin_memory": true,
  "ignore_data_skip": true,
  "eval_strategy": "steps",
  "eval_steps": 1000,
  "save_strategy": "steps",
  "save_steps": 1000,
  "save_total_limit": 3,
  "load_best_model_at_end": true,
  "metric_for_best_model": "wer",
  "greater_is_better": false,
  "logging_steps": 25,
  "push_to_hub": true,
  "hub_strategy": "checkpoint",
  "hub_model_id": "MahmoodAnaam/MSP-Visual",
  "resume_from_checkpoint": null
}