BEST-RQ-2.2-base-2M / evaluation_results.json

Release verified 2M-step BEST-RQ-2.2 encoder with benchmark results and paper badge

47b2c06 verified 8 days ago

7.36 kB

{
  "model": "ltuncay/BEST-RQ-2.2-base-2M",
  "training_run": "https://wandb.ai/tuncay-ludovic/audio%20embeddings/runs/sq96rg53",
  "training_steps": 2000000,
  "checkpoint_sha256": "a0dbb85c931c5576445aec6c4f8afe4a02f071b21237a1440372398c7ffcb51d",
  "xares": [
    {
      "run_id": "cw0rrt7o",
      "url": "https://wandb.ai/tuncay-ludovic/X-ARES/runs/cw0rrt7o",
      "extraction": "original evaluation",
      "score_definition": "Combined: equal-weight average of LinearProbing and kNN aggregates; missing tasks excluded separately per method.",
      "scores": {
        "speech": 0.5862191134171868,
        "music": 0.6573524663823312,
        "environment": 0.6179871805579857,
        "global": 0.587723631056599,
        "mean_of_means": 0.6205195867858346
      },
      "summary": {
        "Speech": {
          "LinearProbing": 0.7636170055788533,
          "kNN": 0.4088212212555204,
          "Combined": 0.5862191134171868
        },
        "Environment": {
          "LinearProbing": 0.5684945892123604,
          "kNN": 0.667479771903611,
          "Combined": 0.6179871805579857
        },
        "Music": {
          "LinearProbing": 0.6799432296191538,
          "kNN": 0.6347617031455087,
          "Combined": 0.6573524663823312
        },
        "Global (task mean)": {
          "LinearProbing": 0.6919298816722935,
          "kNN": 0.48351738044090453,
          "Combined": 0.587723631056599
        },
        "Mean of means": {
          "LinearProbing": 0.6706849414701225,
          "kNN": 0.5703542321015468,
          "Combined": 0.6205195867858346
        }
      }
    },
    {
      "run_id": "a7a3fy9z",
      "url": "https://wandb.ai/tuncay-ludovic/X-ARES/runs/a7a3fy9z",
      "extraction": "overlap50",
      "score_definition": "Combined: equal-weight average of LinearProbing and kNN aggregates; missing tasks excluded separately per method.",
      "scores": {
        "speech": 0.6086589548747638,
        "music": 0.6578351827475715,
        "environment": 0.6590215908535326,
        "global": 0.6112992962777977,
        "mean_of_means": 0.6418385761586227
      },
      "summary": {
        "Speech": {
          "LinearProbing": 0.7734610140006535,
          "kNN": 0.4438568957488741,
          "Combined": 0.6086589548747638
        },
        "Environment": {
          "LinearProbing": 0.6150207318167585,
          "kNN": 0.7030224498903067,
          "Combined": 0.6590215908535326
        },
        "Music": {
          "LinearProbing": 0.6768795405793181,
          "kNN": 0.6387908249158248,
          "Combined": 0.6578351827475715
        },
        "Global (task mean)": {
          "LinearProbing": 0.7097958908202388,
          "kNN": 0.5128027017353565,
          "Combined": 0.6112992962777977
        },
        "Mean of means": {
          "LinearProbing": 0.6884537621322434,
          "kNN": 0.5952233901850019,
          "Combined": 0.6418385761586227
        }
      }
    }
  ],
  "hear": {
    "url": "https://wandb.ai/tuncay-ludovic/HEAR/runs/hear-v2-aa56c55deb6099f6",
    "extraction_preset": "overlap50_two_phase",
    "scores": {
      "speech": 0.7017774299097067,
      "music": 0.8537939808535793,
      "environment": 0.8182406907256897,
      "global": 0.7964326252587012,
      "mean_of_means": 0.7912707004963252
    },
    "completed_tasks": 19,
    "total_tasks": 19,
    "probe": {
      "batch_size": 1024,
      "max_epochs": 500,
      "grid_points": 8,
      "protocol_commit": "855964977238e89dfc76394aa11c37010edb6f20"
    },
    "task_scores": {
      "task/beehive_states/normalized/aucroc": 0.798388069058642,
      "task/beehive_states/raw/aucroc": 0.798388069058642,
      "task/beijing_opera/normalized/top1_acc": 0.961790780141844,
      "task/beijing_opera/raw/top1_acc": 0.961790780141844,
      "task/crema_d/normalized/top1_acc": 0.7873092789841711,
      "task/crema_d/raw/top1_acc": 0.7873092789841711,
      "task/dcase2016_task2/normalized/event_onset_200ms_fms": 0.9637526652452026,
      "task/dcase2016_task2/raw/event_onset_200ms_fms": 0.9637526652452026,
      "task/esc50/normalized/top1_acc": 0.8695,
      "task/esc50/raw/top1_acc": 0.8695,
      "task/fsd50k/normalized/mAP": 0.5518246240865085,
      "task/fsd50k/raw/mAP": 0.5518246240865085,
      "task/gtzan_genre/normalized/top1_acc": 0.806,
      "task/gtzan_genre/raw/top1_acc": 0.806,
      "task/gtzan_music_speech/normalized/top1_acc": 0.9685897435897436,
      "task/gtzan_music_speech/raw/top1_acc": 0.9685897435897436,
      "task/gunshot_triangulation/normalized/top1_acc": 0.9077380952380952,
      "task/gunshot_triangulation/raw/top1_acc": 0.9077380952380952,
      "task/libricount/normalized/top1_acc": 0.7618881118881119,
      "task/libricount/raw/top1_acc": 0.7618881118881119,
      "task/maestro_5h/normalized/event_onset_50ms_fms": 0.4899327848021202,
      "task/maestro_5h/raw/event_onset_50ms_fms": 0.4899327848021202,
      "task/mridangam_stroke/normalized/top1_acc": 0.9764953630957884,
      "task/mridangam_stroke/raw/top1_acc": 0.9764953630957884,
      "task/mridangam_tonic/normalized/top1_acc": 0.97792741165234,
      "task/mridangam_tonic/raw/top1_acc": 0.97792741165234,
      "task/nsynth_pitch_50h/normalized/pitch_acc": 0.875615763546798,
      "task/nsynth_pitch_50h/raw/pitch_acc": 0.875615763546798,
      "task/nsynth_pitch_5h/normalized/pitch_acc": 0.774,
      "task/nsynth_pitch_5h/raw/pitch_acc": 0.774,
      "task/speech_commands_5h/normalized/top1_acc": 0.938036809815951,
      "task/speech_commands_5h/raw/top1_acc": 0.938036809815951,
      "task/speech_commands_full/normalized/top1_acc": 0.9541922290388548,
      "task/speech_commands_full/raw/top1_acc": 0.9541922290388548,
      "task/vocal_imitation/normalized/mAP": 0.17866453091010995,
      "task/vocal_imitation/raw/mAP": 0.17866453091010995,
      "task/vox_lingua_top10/normalized/top1_acc": 0.5905736188210414,
      "task/vox_lingua_top10/raw/top1_acc": 0.5905736188210414
    },
    "historical_comparison_source": "W&B evaluation screenshot, 2026-09-19. All four runs cover all 19 tasks. Full run IDs are not visible.",
    "historical_comparison": [
      {
        "model": "Audio-JEPA",
        "speech": 0.36527,
        "music": 0.74195,
        "environment": 0.51852,
        "global": 0.5642,
        "mean_of_means": 0.54191,
        "recipe": "audio_jepa/audioset/default.yaml",
        "completed_tasks": 19,
        "total_tasks": 19
      },
      {
        "model": "BEST-RQ-2",
        "speech": 0.50737,
        "music": 0.79461,
        "environment": 0.69211,
        "global": 0.67693,
        "mean_of_means": 0.6647,
        "recipe": "best_rq_2/default.yaml",
        "completed_tasks": 19,
        "total_tasks": 19
      },
      {
        "model": "BEST-RQ-2.1",
        "speech": 0.64741,
        "music": 0.86473,
        "environment": 0.70832,
        "global": 0.75495,
        "mean_of_means": 0.74016,
        "recipe": "best_rq_2_1/masking/80.yaml",
        "completed_tasks": 19,
        "total_tasks": 19
      },
      {
        "model": "BEST-RQ-2.2",
        "speech": 0.65756,
        "music": 0.85127,
        "environment": 0.76186,
        "global": 0.76657,
        "mean_of_means": 0.7569,
        "recipe": "best_rq_2_2/masking/80.yaml",
        "completed_tasks": 19,
        "total_tasks": 19
      }
    ]
  }
}

	{
	"model": "ltuncay/BEST-RQ-2.2-base-2M",
	"training_run": "https://wandb.ai/tuncay-ludovic/audio%20embeddings/runs/sq96rg53",
	"training_steps": 2000000,
	"checkpoint_sha256": "a0dbb85c931c5576445aec6c4f8afe4a02f071b21237a1440372398c7ffcb51d",
	"xares": [
	{
	"run_id": "cw0rrt7o",
	"url": "https://wandb.ai/tuncay-ludovic/X-ARES/runs/cw0rrt7o",
	"extraction": "original evaluation",
	"score_definition": "Combined: equal-weight average of LinearProbing and kNN aggregates; missing tasks excluded separately per method.",
	"scores": {
	"speech": 0.5862191134171868,
	"music": 0.6573524663823312,
	"environment": 0.6179871805579857,
	"global": 0.587723631056599,
	"mean_of_means": 0.6205195867858346
	},
	"summary": {
	"Speech": {
	"LinearProbing": 0.7636170055788533,
	"kNN": 0.4088212212555204,
	"Combined": 0.5862191134171868
	},
	"Environment": {
	"LinearProbing": 0.5684945892123604,
	"kNN": 0.667479771903611,
	"Combined": 0.6179871805579857
	},
	"Music": {
	"LinearProbing": 0.6799432296191538,
	"kNN": 0.6347617031455087,
	"Combined": 0.6573524663823312
	},
	"Global (task mean)": {
	"LinearProbing": 0.6919298816722935,
	"kNN": 0.48351738044090453,
	"Combined": 0.587723631056599
	},
	"Mean of means": {
	"LinearProbing": 0.6706849414701225,
	"kNN": 0.5703542321015468,
	"Combined": 0.6205195867858346
	}
	}
	},
	{
	"run_id": "a7a3fy9z",
	"url": "https://wandb.ai/tuncay-ludovic/X-ARES/runs/a7a3fy9z",
	"extraction": "overlap50",
	"score_definition": "Combined: equal-weight average of LinearProbing and kNN aggregates; missing tasks excluded separately per method.",
	"scores": {
	"speech": 0.6086589548747638,
	"music": 0.6578351827475715,
	"environment": 0.6590215908535326,
	"global": 0.6112992962777977,
	"mean_of_means": 0.6418385761586227
	},
	"summary": {
	"Speech": {
	"LinearProbing": 0.7734610140006535,
	"kNN": 0.4438568957488741,
	"Combined": 0.6086589548747638
	},
	"Environment": {
	"LinearProbing": 0.6150207318167585,
	"kNN": 0.7030224498903067,
	"Combined": 0.6590215908535326
	},
	"Music": {
	"LinearProbing": 0.6768795405793181,
	"kNN": 0.6387908249158248,
	"Combined": 0.6578351827475715
	},
	"Global (task mean)": {
	"LinearProbing": 0.7097958908202388,
	"kNN": 0.5128027017353565,
	"Combined": 0.6112992962777977
	},
	"Mean of means": {
	"LinearProbing": 0.6884537621322434,
	"kNN": 0.5952233901850019,
	"Combined": 0.6418385761586227
	}
	}
	}
	],
	"hear": {
	"url": "https://wandb.ai/tuncay-ludovic/HEAR/runs/hear-v2-aa56c55deb6099f6",
	"extraction_preset": "overlap50_two_phase",
	"scores": {
	"speech": 0.7017774299097067,
	"music": 0.8537939808535793,
	"environment": 0.8182406907256897,
	"global": 0.7964326252587012,
	"mean_of_means": 0.7912707004963252
	},
	"completed_tasks": 19,
	"total_tasks": 19,
	"probe": {
	"batch_size": 1024,
	"max_epochs": 500,
	"grid_points": 8,
	"protocol_commit": "855964977238e89dfc76394aa11c37010edb6f20"
	},
	"task_scores": {
	"task/beehive_states/normalized/aucroc": 0.798388069058642,
	"task/beehive_states/raw/aucroc": 0.798388069058642,
	"task/beijing_opera/normalized/top1_acc": 0.961790780141844,
	"task/beijing_opera/raw/top1_acc": 0.961790780141844,
	"task/crema_d/normalized/top1_acc": 0.7873092789841711,
	"task/crema_d/raw/top1_acc": 0.7873092789841711,
	"task/dcase2016_task2/normalized/event_onset_200ms_fms": 0.9637526652452026,
	"task/dcase2016_task2/raw/event_onset_200ms_fms": 0.9637526652452026,
	"task/esc50/normalized/top1_acc": 0.8695,
	"task/esc50/raw/top1_acc": 0.8695,
	"task/fsd50k/normalized/mAP": 0.5518246240865085,
	"task/fsd50k/raw/mAP": 0.5518246240865085,
	"task/gtzan_genre/normalized/top1_acc": 0.806,
	"task/gtzan_genre/raw/top1_acc": 0.806,
	"task/gtzan_music_speech/normalized/top1_acc": 0.9685897435897436,
	"task/gtzan_music_speech/raw/top1_acc": 0.9685897435897436,
	"task/gunshot_triangulation/normalized/top1_acc": 0.9077380952380952,
	"task/gunshot_triangulation/raw/top1_acc": 0.9077380952380952,
	"task/libricount/normalized/top1_acc": 0.7618881118881119,
	"task/libricount/raw/top1_acc": 0.7618881118881119,
	"task/maestro_5h/normalized/event_onset_50ms_fms": 0.4899327848021202,
	"task/maestro_5h/raw/event_onset_50ms_fms": 0.4899327848021202,
	"task/mridangam_stroke/normalized/top1_acc": 0.9764953630957884,
	"task/mridangam_stroke/raw/top1_acc": 0.9764953630957884,
	"task/mridangam_tonic/normalized/top1_acc": 0.97792741165234,
	"task/mridangam_tonic/raw/top1_acc": 0.97792741165234,
	"task/nsynth_pitch_50h/normalized/pitch_acc": 0.875615763546798,
	"task/nsynth_pitch_50h/raw/pitch_acc": 0.875615763546798,
	"task/nsynth_pitch_5h/normalized/pitch_acc": 0.774,
	"task/nsynth_pitch_5h/raw/pitch_acc": 0.774,
	"task/speech_commands_5h/normalized/top1_acc": 0.938036809815951,
	"task/speech_commands_5h/raw/top1_acc": 0.938036809815951,
	"task/speech_commands_full/normalized/top1_acc": 0.9541922290388548,
	"task/speech_commands_full/raw/top1_acc": 0.9541922290388548,
	"task/vocal_imitation/normalized/mAP": 0.17866453091010995,
	"task/vocal_imitation/raw/mAP": 0.17866453091010995,
	"task/vox_lingua_top10/normalized/top1_acc": 0.5905736188210414,
	"task/vox_lingua_top10/raw/top1_acc": 0.5905736188210414
	},
	"historical_comparison_source": "W&B evaluation screenshot, 2026-09-19. All four runs cover all 19 tasks. Full run IDs are not visible.",
	"historical_comparison": [
	{
	"model": "Audio-JEPA",
	"speech": 0.36527,
	"music": 0.74195,
	"environment": 0.51852,
	"global": 0.5642,
	"mean_of_means": 0.54191,
	"recipe": "audio_jepa/audioset/default.yaml",
	"completed_tasks": 19,
	"total_tasks": 19
	},
	{
	"model": "BEST-RQ-2",
	"speech": 0.50737,
	"music": 0.79461,
	"environment": 0.69211,
	"global": 0.67693,
	"mean_of_means": 0.6647,
	"recipe": "best_rq_2/default.yaml",
	"completed_tasks": 19,
	"total_tasks": 19
	},
	{
	"model": "BEST-RQ-2.1",
	"speech": 0.64741,
	"music": 0.86473,
	"environment": 0.70832,
	"global": 0.75495,
	"mean_of_means": 0.74016,
	"recipe": "best_rq_2_1/masking/80.yaml",
	"completed_tasks": 19,
	"total_tasks": 19
	},
	{
	"model": "BEST-RQ-2.2",
	"speech": 0.65756,
	"music": 0.85127,
	"environment": 0.76186,
	"global": 0.76657,
	"mean_of_means": 0.7569,
	"recipe": "best_rq_2_2/masking/80.yaml",
	"completed_tasks": 19,
	"total_tasks": 19
	}
	]
	}
	}