{
  "model_type": "audio-jepa",
  "architecture": "VisionTransformer",
  "citation": "Tuncay, Labbé, Benetos, Pellegrini. Audio-JEPA: Joint-Embedding Predictive Architecture for Audio Representation Learning. ICME 2025 (arXiv 2507.02915)",
  "audio": {
    "sample_rate": 32000,
    "clip_length_s": 10,
    "n_mels": 128,
    "target_time_bins": 256,
    "mel_spec_backend": "torchaudio.compliance.kaldi.fbank",
    "f_min": 20,
    "f_max": null,
    "log_fbank": true,
    "window": "hanning"
  },
  "patch_embed": {
    "patch_size": [16, 16],
    "in_chans": 1,
    "num_patches": 128,
    "grid_time": 8,
    "grid_freq": 16
  },
  "encoder": {
    "embed_dim": 768,
    "depth": 12,
    "num_heads": 12,
    "mlp_ratio": 4.0,
    "qkv_bias": true,
    "use_flash_attn": true,
    "norm_layer": "LayerNorm",
    "cls_token": false,
    "pos_embed": "2d_sincos"
  },
  "output": {
    "shape_per_10s_clip": [128, 768],
    "temporal_positions_per_second": 1.6,
    "tokens_per_second": 12.8,
    "note": "Each patch spans 16 mel bins x 16 time bins ≈ 625 ms. The 12.8 Hz rate counts 8 temporal x 16 frequency patches per second; effective temporal resolution is 1.6 positions/s."
  },
  "training": {
    "framework": "PyTorch Lightning + Hydra",
    "pretraining_corpus": "AudioSet unlabeled (~5.3k h after silence filtering)",
    "target_module": "target_encoder (EMA of encoder)",
    "predictor": {
      "type": "VisionTransformerPredictor",
      "depth": 6,
      "num_heads": 12,
      "mlp_ratio": 4.0
    },
    "loss": "norm_mse (with norm_pix_loss=True)",
    "note": "Only the encoder is used downstream. Predictor + target encoder are discarded at inference."
  },
  "known_speech_gap": {
    "description": "This model is designed for generic audio representation. The 16x16 patch shape reflects a compromise across speech, music, and environmental sounds. Speech-only downstream performance is lower than speech-specific SSL models (wav2vec2, HuBERT, Whisper). See the paper's X-ARES tables and the follow-up BEST-RQ-2 (arXiv 2606.30700) for a quantitative comparison.",
    "suggested_alternative_for_speech": "For speech-heavy downstream tasks, consider BEST-RQ-2 (same author, arXiv 2606.30700, MIT weights at ltuncay/BEST-RQ-2) which shares the encoder architecture, or wait for the upcoming BEST-RQ-2.1 with explicit speech-oriented improvements."
  }
}