training_args.json

{
  "data_dir": "/home/ubuntu/HeAR/distillation/data/laion_audio_lake/decay",
  "shards_glob": "shard-*.tar",
  "streams_glob": "stream-*",
  "out": "/home/ubuntu/HeAR/distillation/checkpoints/hear_vit_s_lake/decay_phase",
  "max_steps": 440000,
  "batch_size": 48,
  "grad_accum": 1,
  "num_workers": 4,
  "device": "cuda",
  "lr": 0.000310138180725,
  "lr_schedule": "linear",
  "lr_schedule_start_step": 400000,
  "lr_warmup_steps": 0,
  "lr_min_ratio": 0.1,
  "auto_warmup": false,
  "auto_warmup_init_lr": 0.0,
  "auto_warmup_probe_batch_size": 0,
  "auto_warmup_steps": 1000,
  "auto_warmup_metric_every": 5,
  "auto_warmup_lr_safety_frac": 0.8,
  "auto_warmup_ema_beta": 0.9,
  "auto_warmup_cbs_ema_beta": 0.995,
  "gns_batch_window": 9,
  "gns_batch_target_utility": 0.5,
  "auto_warmup_lr_outlier_factor": 4.0,
  "auto_warmup_cbs_outlier_factor": 4.0,
  "auto_warmup_batch_round_to": 8,
  "auto_warmup_max_lr": 0.0,
  "auto_warmup_max_batch_size": 0,
  "lr_gns_mode": "sqrt",
  "lr_gns_adapt": false,
  "lr_gns_ema_beta": 0.9,
  "lr_gns_min_samples": 100,
  "lr_gns_update_every": 500,
  "lr_gns_min_factor": 0.01,
  "lr_gns_max_factor": 1.0,
  "lr_gns_ref_batch": 0.0,
  "weight_decay": 0.05,
  "compile_teacher": true,
  "compile_student": true,
  "compile_mode": "default",
  "compile_dynamic": true,
  "fused_adamw": true,
  "optimizer_mode": "teacher-superbatch",
  "optimizer_log_every": 50,
  "teacher_batch_factor": 128,
  "teacher_max_batch": 8192,
  "log_every": 10,
  "save_every": 1000,
  "max_checkpoints": 20,
  "resume_from": "/home/ubuntu/HeAR/distillation/checkpoints/hear_vit_s_lake/ckpt_stable_final.pt",
  "resume_latest": false,
  "resume_require_optim": true,
  "gns_every": 5,
  "gns_param_sample": 200000,
  "teacher_id": "google/hear-pytorch",
  "clip_seconds": 2.0,
  "sample_rate": 16000,
  "shuffle_shards": true,
  "repeat": true,
  "live_shard_refresh": false,
  "shard_refresh_sec": 30.0,
  "amp": true,
  "seed": 1337,
  "loss_mse_weight": 1.0,
  "loss_contrastive_weight": 0.5,
  "loss_relational_weight": 0.5,
  "contrastive_temp": 0.07,
  "canon": true,
  "canon_2d": true,
  "canon_kernel": 4,
  "canon_a": false,
  "canon_b": false,
  "canon_b_qkv": false,
  "canon_c": false,
  "canon_d": false,
  "canon_abcd": true,
  "canon_no_pos_enc": true,
  "canon_pre": false,
  "canon_post": false,
  "canon_causal": false,
  "val_fraction": 0.0,
  "val_manifest": "/home/ubuntu/HeAR/distillation/data/laion_audio_lake/manifests/val_active.json",
  "val_live_refresh": true,
  "val_shard_refresh_sec": 30.0,
  "val_shards_file": null,
  "val_target_clips": 0,
  "val_defer_start_steps": 0,
  "val_defer_check_every": 50,
  "val_every": 250,
  "val_batches": 5,
  "wandb": true,
  "wandb_project": "hear-distill",
  "wandb_entity": null,
  "wandb_run_name": null,
  "wandb_tags": null
}