updated eval testing

backend/evaluation/generate_full_test_set.py

@@ -0,0 +1,66 @@
+"""
+Generate full 8-piece MAESTRO test set.
+"""
+
+from pathlib import Path
+import json
+
+# Test cases from prepare_maestro.py
+MAESTRO_SUBSET = [
+    # Easy - Simple classical pieces
+    ("2004", "test", "MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_03_Track03_wav", "classical", "easy"),
+    ("2004", "test", "MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_05_Track05_wav", "classical", "easy"),
+
+    # Medium - Chopin, moderate tempo
+    ("2004", "test", "MIDI-Unprocessed_XP_14_R1_2004_01-04_ORIG_MID--AUDIO_14_R1_2004_04_Track04_wav", "classical", "medium"),
+    ("2006", "test", "MIDI-Unprocessed_07_R1_2006_01-09_ORIG_MID--AUDIO_07_R1_2006_04_Track04_wav", "classical", "medium"),
+    ("2008", "test", "MIDI-Unprocessed_11_R1_2008_01-04_ORIG_MID--AUDIO_11_R1_2008_02_Track02_wav", "classical", "medium"),
+
+    # Hard - Fast passages, complex harmony
+    ("2009", "test", "MIDI-Unprocessed_16_R1_2009_01-04_ORIG_MID--AUDIO_16_R1_2009_16_R1_2009_02_WAV", "classical", "hard"),
+    ("2011", "test", "MIDI-Unprocessed_03_R1_2011_MID--AUDIO_03_R1_2011_03_R1_2011_02_WAV", "classical", "hard"),
+    ("2013", "test", "MIDI-Unprocessed_20_R1_2013_MID--AUDIO_20_R1_2013_20_R1_2013_02_WAV", "classical", "hard"),
+]
+
+
+def generate_test_cases(maestro_dir: str = "../../data/maestro-v3.0.0"):
+    """Generate test_videos.json with all 8 MAESTRO pieces."""
+
+    test_cases = []
+
+    for year, split, filename_prefix, genre, difficulty in MAESTRO_SUBSET:
+        # Construct paths
+        audio_path = f"{maestro_dir}/{year}/{filename_prefix}.wav"
+        midi_path = f"{maestro_dir}/{year}/{filename_prefix}.midi"
+
+        # Create test case
+        test_case = {
+            "name": filename_prefix,
+            "audio_path": audio_path,
+            "ground_truth_midi": midi_path,
+            "genre": genre,
+            "difficulty": difficulty,
+            "duration": None
+        }
+
+        test_cases.append(test_case)
+
+    return test_cases
+
+
+if __name__ == "__main__":
+    # Generate test cases
+    test_cases = generate_test_cases()
+
+    # Save to JSON
+    output_file = Path(__file__).parent / "test_videos.json"
+
+    with open(output_file, 'w') as f:
+        json.dump(test_cases, f, indent=2)
+
+    print(f"✅ Generated {len(test_cases)} test cases")
+    print(f"📝 Saved to: {output_file}")
+    print("\nBreakdown:")
+    print(f"  - Easy: {sum(1 for tc in test_cases if tc['difficulty'] == 'easy')}")
+    print(f"  - Medium: {sum(1 for tc in test_cases if tc['difficulty'] == 'medium')}")
+    print(f"  - Hard: {sum(1 for tc in test_cases if tc['difficulty'] == 'hard')}")

backend/evaluation/test_videos.json

@@ -1,10 +1,66 @@
 [
   {
-    "name": "MAESTRO_2004_AUDIO 02 R1 2004 05 Track05 wav",
+    "name": "MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_03_Track03_wav",
+    "audio_path": "../../data/maestro-v3.0.0/2004/MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_03_Track03_wav.wav",
+    "ground_truth_midi": "../../data/maestro-v3.0.0/2004/MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_03_Track03_wav.midi",
+    "genre": "classical",
+    "difficulty": "easy",
+    "duration": null
+  },
+  {
+    "name": "MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_05_Track05_wav",
     "audio_path": "../../data/maestro-v3.0.0/2004/MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_05_Track05_wav.wav",
     "ground_truth_midi": "../../data/maestro-v3.0.0/2004/MIDI-Unprocessed_SMF_02_R1_2004_01-05_ORIG_MID--AUDIO_02_R1_2004_05_Track05_wav.midi",
     "genre": "classical",
     "difficulty": "easy",
     "duration": null
+  },
+  {
+    "name": "MIDI-Unprocessed_XP_14_R1_2004_01-04_ORIG_MID--AUDIO_14_R1_2004_04_Track04_wav",
+    "audio_path": "../../data/maestro-v3.0.0/2004/MIDI-Unprocessed_XP_14_R1_2004_01-04_ORIG_MID--AUDIO_14_R1_2004_04_Track04_wav.wav",
+    "ground_truth_midi": "../../data/maestro-v3.0.0/2004/MIDI-Unprocessed_XP_14_R1_2004_01-04_ORIG_MID--AUDIO_14_R1_2004_04_Track04_wav.midi",
+    "genre": "classical",
+    "difficulty": "medium",
+    "duration": null
+  },
+  {
+    "name": "MIDI-Unprocessed_07_R1_2006_01-09_ORIG_MID--AUDIO_07_R1_2006_04_Track04_wav",
+    "audio_path": "../../data/maestro-v3.0.0/2006/MIDI-Unprocessed_07_R1_2006_01-09_ORIG_MID--AUDIO_07_R1_2006_04_Track04_wav.wav",
+    "ground_truth_midi": "../../data/maestro-v3.0.0/2006/MIDI-Unprocessed_07_R1_2006_01-09_ORIG_MID--AUDIO_07_R1_2006_04_Track04_wav.midi",
+    "genre": "classical",
+    "difficulty": "medium",
+    "duration": null
+  },
+  {
+    "name": "MIDI-Unprocessed_11_R1_2008_01-04_ORIG_MID--AUDIO_11_R1_2008_02_Track02_wav",
+    "audio_path": "../../data/maestro-v3.0.0/2008/MIDI-Unprocessed_11_R1_2008_01-04_ORIG_MID--AUDIO_11_R1_2008_02_Track02_wav.wav",
+    "ground_truth_midi": "../../data/maestro-v3.0.0/2008/MIDI-Unprocessed_11_R1_2008_01-04_ORIG_MID--AUDIO_11_R1_2008_02_Track02_wav.midi",
+    "genre": "classical",
+    "difficulty": "medium",
+    "duration": null
+  },
+  {
+    "name": "MIDI-Unprocessed_16_R1_2009_01-04_ORIG_MID--AUDIO_16_R1_2009_16_R1_2009_02_WAV",
+    "audio_path": "../../data/maestro-v3.0.0/2009/MIDI-Unprocessed_16_R1_2009_01-04_ORIG_MID--AUDIO_16_R1_2009_16_R1_2009_02_WAV.wav",
+    "ground_truth_midi": "../../data/maestro-v3.0.0/2009/MIDI-Unprocessed_16_R1_2009_01-04_ORIG_MID--AUDIO_16_R1_2009_16_R1_2009_02_WAV.midi",
+    "genre": "classical",
+    "difficulty": "hard",
+    "duration": null
+  },
+  {
+    "name": "MIDI-Unprocessed_03_R1_2011_MID--AUDIO_03_R1_2011_03_R1_2011_02_WAV",
+    "audio_path": "../../data/maestro-v3.0.0/2011/MIDI-Unprocessed_03_R1_2011_MID--AUDIO_03_R1_2011_03_R1_2011_02_WAV.wav",
+    "ground_truth_midi": "../../data/maestro-v3.0.0/2011/MIDI-Unprocessed_03_R1_2011_MID--AUDIO_03_R1_2011_03_R1_2011_02_WAV.midi",
+    "genre": "classical",
+    "difficulty": "hard",
+    "duration": null
+  },
+  {
+    "name": "MIDI-Unprocessed_20_R1_2013_MID--AUDIO_20_R1_2013_20_R1_2013_02_WAV",
+    "audio_path": "../../data/maestro-v3.0.0/2013/MIDI-Unprocessed_20_R1_2013_MID--AUDIO_20_R1_2013_20_R1_2013_02_WAV.wav",
+    "ground_truth_midi": "../../data/maestro-v3.0.0/2013/MIDI-Unprocessed_20_R1_2013_MID--AUDIO_20_R1_2013_20_R1_2013_02_WAV.midi",
+    "genre": "classical",
+    "difficulty": "hard",
+    "duration": null
   }
 ]