File size: 2,384 Bytes
9358c80 bdd5c2a 9358c80 4413037 9358c80 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | {
"model_type": "audio-jepa",
"architecture": "VisionTransformer",
"citation": "Tuncay, Labbé, Benetos, Pellegrini. Audio-JEPA: Joint-Embedding Predictive Architecture for Audio Representation Learning. ICME 2025 (arXiv 2507.02915)",
"audio": {
"sample_rate": 32000,
"clip_length_s": 10,
"n_mels": 128,
"target_time_bins": 256,
"mel_spec_backend": "torchaudio.compliance.kaldi.fbank",
"f_min": 20,
"f_max": null,
"log_fbank": true,
"window": "hanning"
},
"patch_embed": {
"patch_size": [16, 16],
"in_chans": 1,
"num_patches": 128,
"grid_time": 8,
"grid_freq": 16
},
"encoder": {
"embed_dim": 768,
"depth": 12,
"num_heads": 12,
"mlp_ratio": 4.0,
"qkv_bias": true,
"use_flash_attn": true,
"norm_layer": "LayerNorm",
"cls_token": false,
"pos_embed": "2d_sincos"
},
"output": {
"shape_per_10s_clip": [128, 768],
"temporal_positions_per_second": 1.6,
"tokens_per_second": 12.8,
"note": "Each patch spans 16 mel bins x 16 time bins ≈ 625 ms. The 12.8 Hz rate counts 8 temporal x 16 frequency patches per second; effective temporal resolution is 1.6 positions/s."
},
"training": {
"framework": "PyTorch Lightning + Hydra",
"pretraining_corpus": "AudioSet unlabeled (~5.3k h after silence filtering)",
"target_module": "target_encoder (EMA of encoder)",
"predictor": {
"type": "VisionTransformerPredictor",
"depth": 6,
"num_heads": 12,
"mlp_ratio": 4.0
},
"loss": "norm_mse (with norm_pix_loss=True)",
"note": "Only the encoder is used downstream. Predictor + target encoder are discarded at inference."
},
"known_speech_gap": {
"description": "This model is designed for generic audio representation. The 16x16 patch shape reflects a compromise across speech, music, and environmental sounds. Speech-only downstream performance is lower than speech-specific SSL models (wav2vec2, HuBERT, Whisper). See the paper's X-ARES tables and the follow-up BEST-RQ-2 (arXiv 2606.30700) for a quantitative comparison.",
"suggested_alternative_for_speech": "For speech-heavy downstream tasks, consider BEST-RQ-2 (same author, arXiv 2606.30700, MIT weights at ltuncay/BEST-RQ-2) which shares the encoder architecture, or wait for the upcoming BEST-RQ-2.1 with explicit speech-oriented improvements."
}
}
|