""" Music-to-Sheet-Music: an interactive exploration of three architectural fixes. This is a Marimo notebook. Run it locally: pip install marimo basic-pitch music21 librosa verovio cairosvg soundfile marimo run notebook.py The notebook reproduces the student's transcription pipeline (Basic Pitch -> music21 -> MusicXML -> sheet music), then lets you toggle three architectural fixes on and off to see what each one repairs. The three fixes correspond to the three failure modes the student's paper documents at the seam between pitch detection and notation. """ import marimo __generated_with = "0.23.8" app = marimo.App(width="medium") @app.cell def setup(): import marimo as mo return (mo,) @app.cell def intro(mo): mo.md( r""" # Music-to-Sheet-Music: Where the Map Loses the City The student's research paper makes a precise claim: when a lightweight AI transcription pipeline turns audio into sheet music, the failure is not a few wrong notes. It is *a cascade across every dimension of musical information at once* — pitch (overtones detected as separate notes), rhythm (no beat tracking), time signature (defaults to 4/4), tempo (defaults to ♩ = 120). This notebook reproduces the pipeline and lets you turn three architectural fixes on and off, one at a time, to see what each one actually repairs. It is the interactive version of the static comparison on the capstone page. - **Fix 1 — Octave-overtone filter.** Drop any pitch that is exactly one octave above another note and starts within ~50 ms of it. These are harmonics, not real notes. - **Fix 2 — Beat tracking with librosa.** Find the actual tempo from the audio before quantizing. Default music21 quantization assumes a fixed grid and produces nonsense durations when the tempo is wrong. - **Fix 3 — Tempo-aware quantization.** Quantize each note's onset and duration to the detected beat grid, not to a default sixteenth-note quarter-length grid. Each fix is small. Together they close most of the gap the paper documents — without touching the audio model. """ ) return @app.cell def imports(): import os import tempfile import numpy as np import soundfile as sf from music21 import ( stream, note, meter, key, clef, tempo as m21_tempo, instrument, ) return ( clef, instrument, key, m21_tempo, meter, np, note, os, sf, stream, tempfile, ) @app.cell def constants(): SAMPLE_RATE = 22050 NOTE_FREQS = { "C4": 261.63, "D4": 293.66, "E4": 329.63, "F4": 349.23, "G4": 392.00, "A4": 440.00, "B4": 493.88, "C5": 523.25, } # The same Ode to Joy used on the static capstone page. ODE_TO_JOY = [ ("E4", 1), ("E4", 1), ("F4", 1), ("G4", 1), ("G4", 1), ("F4", 1), ("E4", 1), ("D4", 1), ("C4", 1), ("C4", 1), ("D4", 1), ("E4", 1), ("E4", 1.5), ("D4", 0.5), ("D4", 2.0), ] TEMPO_BPM = 92 SECS_PER_BEAT = 60.0 / TEMPO_BPM return NOTE_FREQS, ODE_TO_JOY, SAMPLE_RATE, SECS_PER_BEAT, TEMPO_BPM @app.cell def synth_audio_fn(NOTE_FREQS, SAMPLE_RATE, SECS_PER_BEAT, np): def synth_note(freq, duration_sec, sample_rate=SAMPLE_RATE): n_samples = int(duration_sec * sample_rate) t = np.linspace(0, duration_sec, n_samples, endpoint=False) # Fundamental + harmonics. The octave overtone here is what causes # Basic Pitch to hallucinate phantom pitches one octave above the # real note. wave = ( 1.00 * np.sin(2 * np.pi * freq * t) + 0.50 * np.sin(2 * np.pi * 2 * freq * t) + 0.25 * np.sin(2 * np.pi * 3 * freq * t) + 0.10 * np.sin(2 * np.pi * 4 * freq * t) ) attack_n = int(0.005 * sample_rate) decay_constant = duration_sec * 0.4 env = np.exp(-t / decay_constant) if attack_n > 0: env[:attack_n] *= np.linspace(0, 1, attack_n) wave *= env return wave def synth_melody(melody, secs_per_beat=SECS_PER_BEAT): chunks = [] for pitch, dur_beats in melody: chunks.append(synth_note(NOTE_FREQS[pitch], dur_beats * secs_per_beat)) full = np.concatenate(chunks) full = full / np.max(np.abs(full)) * 0.7 return full return synth_melody, synth_note @app.cell def step1_synthesize_audio(ODE_TO_JOY, SAMPLE_RATE, mo, os, sf, synth_melody, tempfile): audio = synth_melody(ODE_TO_JOY) # Save to a temp WAV so Basic Pitch can read it and so the player can play it. audio_path = os.path.join(tempfile.gettempdir(), "ode_to_joy_input.wav") sf.write(audio_path, audio, SAMPLE_RATE) mo.md( f""" ## Step 1 — Synthesized input audio 15 notes of *Ode to Joy*, synthesized as a piano-like tone (fundamental plus three harmonics, sharp attack, exponential decay). This is the audio Basic Pitch will see. Listen — the overtones are what cause the phantom pitches. """ ) return audio, audio_path @app.cell def audio_player(audio_path, mo): mo.audio(audio_path) return @app.cell def step2_run_basic_pitch(audio_path, mo): import basic_pitch from basic_pitch.inference import predict onnx_path = os.path.join( os.path.dirname(basic_pitch.__file__), "saved_models", "icassp_2022", "nmp.onnx", ) # Run Basic Pitch on the synthesized audio. _, midi_data, note_events = predict( audio_path, model_or_model_path=onnx_path ) # note_events is a list of (start_sec, end_sec, pitch_midi, amplitude, pitch_bends) mo.md( f""" ## Step 2 — Basic Pitch transcription Basic Pitch ran on the audio above and detected **{len(note_events)}** notes. The ground truth is 15 notes (one for each note in *Ode to Joy*). Anything beyond 15 is either a hallucination or a fragmented duplicate. """ ) return midi_data, note_events, onnx_path, predict @app.cell def show_raw_note_events(mo, note_events): rows = [] for _start, _end, _pitch, _amp, _bends in note_events: rows.append( { "start_sec": round(_start, 3), "end_sec": round(_end, 3), "midi_pitch": int(_pitch), "amplitude": round(_amp, 3), } ) mo.md("### Raw note events from Basic Pitch (before any fixes)") return (rows,) @app.cell def raw_table(mo, rows): mo.ui.table(rows, page_size=10) return @app.cell def fix_controls(mo): overtone_toggle = mo.ui.switch(value=False, label="Fix 1 — Octave overtone filter") beat_track_toggle = mo.ui.switch(value=False, label="Fix 2 — Beat tracking with librosa") quantize_toggle = mo.ui.switch(value=False, label="Fix 3 — Tempo-aware quantization") mo.md( f""" ## Step 3 — Three architectural fixes Toggle each fix on or off. The notebook will reprocess the Basic Pitch output and show the resulting sheet music and metrics below. {mo.vstack([overtone_toggle, beat_track_toggle, quantize_toggle])} """ ) return beat_track_toggle, overtone_toggle, quantize_toggle @app.cell def fix_functions(np): def fix_overtones(note_events, octave_window_sec=0.05): """Drop notes that are exactly 12 semitones above another note starting within octave_window_sec. These are almost always harmonic overtones the model detected as separate notes.""" # Sort by start time so we can scan a small window events = sorted(note_events, key=lambda e: e[0]) keep = [True] * len(events) for i, (s_i, _, p_i, _, _) in enumerate(events): for j, (s_j, _, p_j, _, _) in enumerate(events): if i == j or not keep[i]: continue if abs(s_i - s_j) <= octave_window_sec and p_i == p_j + 12: keep[i] = False break return [e for e, k in zip(events, keep) if k] def detect_tempo_with_librosa(audio_array, sample_rate): """Use librosa to detect the tempo from the audio. Returns BPM.""" import librosa tempo, _ = librosa.beat.beat_track(y=audio_array, sr=sample_rate) # librosa returns a numpy array sometimes; coerce to float return float(np.atleast_1d(tempo)[0]) def tempo_aware_quantize(note_events, bpm, subdivision=4): """Quantize each note's onset and duration to the beat grid implied by bpm. Default subdivision=4 means quarter-note grid (snap to sixteenths within a beat).""" sec_per_beat = 60.0 / bpm sec_per_grid = sec_per_beat / subdivision quantized = [] for s, e, p, a, pb in note_events: s_q = round(s / sec_per_grid) * sec_per_grid dur = e - s dur_q = max(sec_per_grid, round(dur / sec_per_grid) * sec_per_grid) quantized.append((s_q, s_q + dur_q, p, a, pb)) return quantized return detect_tempo_with_librosa, fix_overtones, tempo_aware_quantize @app.cell def apply_fixes( audio, beat_track_toggle, detect_tempo_with_librosa, fix_overtones, note_events, overtone_toggle, quantize_toggle, SAMPLE_RATE, tempo_aware_quantize, ): processed = list(note_events) detected_bpm = None if overtone_toggle.value: processed = fix_overtones(processed) if beat_track_toggle.value: detected_bpm = detect_tempo_with_librosa(audio, SAMPLE_RATE) if quantize_toggle.value: # If beat tracking is also on, use its detected BPM; otherwise # quantize against the default music21 assumption of 120 BPM. bpm = detected_bpm if detected_bpm is not None else 120.0 processed = tempo_aware_quantize(processed, bpm) return detected_bpm, processed @app.cell def show_metrics( beat_track_toggle, detected_bpm, mo, note_events, overtone_toggle, processed, quantize_toggle, ): GROUND_TRUTH_COUNT = 15 GROUND_TRUTH_TEMPO = 92.0 GROUND_TRUTH_PITCHES = {60, 62, 64, 65, 67} # C4, D4, E4, F4, G4 (MIDI numbers) raw_count = len(note_events) processed_count = len(processed) processed_pitches = {int(p) for (_, _, p, _, _) in processed} phantom = processed_pitches - GROUND_TRUTH_PITCHES missing = GROUND_TRUTH_PITCHES - processed_pitches tempo_line = ( f"**Detected tempo:** ♩ = {detected_bpm:.1f} BPM (ground truth: ♩ = {GROUND_TRUTH_TEMPO} BPM)" if detected_bpm is not None else "**Detected tempo:** — *(beat tracking off)*" ) mo.md( f""" ### How the fixes compare to ground truth | Dimension | Ground truth | This run | | --- | --- | --- | | Note count | {GROUND_TRUTH_COUNT} | {processed_count} ({processed_count - GROUND_TRUTH_COUNT:+d}) | | Unique pitches | C4, D4, E4, F4, G4 | {", ".join(sorted_pitch_names(processed_pitches))} | | Phantom pitches (false positives) | none | {", ".join(sorted_pitch_names(phantom)) or "none"} | | Missed pitches (false negatives) | none | {", ".join(sorted_pitch_names(missing)) or "none"} | {tempo_line} **Fixes currently applied:** {fixes_summary(overtone_toggle, beat_track_toggle, quantize_toggle)} """ ) return GROUND_TRUTH_COUNT, GROUND_TRUTH_PITCHES, GROUND_TRUTH_TEMPO @app.cell def helper_fns(): def sorted_pitch_names(midi_set): names = [] for n in sorted(midi_set): # MIDI 60 = C4 octave = n // 12 - 1 note_name = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"][n % 12] names.append(f"{note_name}{octave}") return names def fixes_summary(overtone_toggle, beat_track_toggle, quantize_toggle): active = [] if overtone_toggle.value: active.append("overtone filter") if beat_track_toggle.value: active.append("beat tracking") if quantize_toggle.value: active.append("tempo-aware quantize") return ", ".join(active) if active else "*none — this is the raw Basic Pitch output*" return fixes_summary, sorted_pitch_names @app.cell def render_score( GROUND_TRUTH_TEMPO, clef, detected_bpm, instrument, key, m21_tempo, meter, mo, note, os, processed, stream, tempfile, ): """Render the processed note events as sheet music using music21 + Verovio.""" # Build a music21 stream from the processed note events s = stream.Score() p = stream.Part() p.append(instrument.Piano()) p.append(clef.TrebleClef()) p.append(key.KeySignature(0)) p.append(meter.TimeSignature("4/4")) bpm_for_render = detected_bpm if detected_bpm is not None else 120.0 p.append(m21_tempo.MetronomeMark(number=int(bpm_for_render))) # Each note: (start_sec, end_sec, midi_pitch, amplitude, pitch_bends) sec_per_beat = 60.0 / bpm_for_render for s_sec, e_sec, midi_p, _, _ in sorted(processed, key=lambda x: x[0]): dur_beats = max(0.0625, (e_sec - s_sec) / sec_per_beat) n = note.Note(midi=int(midi_p), quarterLength=round(dur_beats * 16) / 16) p.append(n) s.append(p) # Write to a temp MusicXML file tmpdir = tempfile.gettempdir() xml_path = os.path.join(tmpdir, "marimo_processed.musicxml") s.write("musicxml", fp=xml_path) # Render with Verovio to SVG import verovio tk = verovio.toolkit() tk.setOptions( { "scale": 50, "pageWidth": 2200, "pageHeight": 600, "pageMarginLeft": 40, "pageMarginRight": 40, "pageMarginTop": 30, "pageMarginBottom": 30, "footer": "none", "header": "none", "breaks": "auto", } ) tk.loadFile(xml_path) svg = tk.renderToSVG(1) mo.md( f""" ### Resulting sheet music Tempo: ♩ = {bpm_for_render:.0f} BPM. Ground truth was ♩ = {GROUND_TRUTH_TEMPO}. {mo.Html(svg)} """ ) return svg @app.cell def closing(mo): mo.md( r""" ## What this notebook is for This is the interactive sibling of the static comparison on the capstone page. The static page shows *what the failure looks like* on one example. This notebook shows *what each fix does to that failure*, one variable at a time. ### Suggested experiments 1. **Turn on Fix 1 (overtone filter) only.** Watch the four phantom pitches (C5, D5, E5, G5) disappear. The note count should drop toward 15. 2. **Turn on Fix 2 (beat tracking) only.** The detected tempo should be close to 92 BPM (the ground truth). This alone doesn't fix the rendered score because the durations are still wrong — but the tempo mark above the staff is now correct. 3. **Turn on Fix 3 (tempo-aware quantize) only.** The durations snap to a 16th-note grid using the default 120 BPM (since beat tracking is off). The rhythm doesn't get better, just different. 4. **Turn on Fix 2 + Fix 3 together.** Now the quantization uses the *detected* tempo. The durations should start looking closer to actual quarter notes. 5. **Turn on all three.** This is the architectural fix the paper proposes in Section 6. Note count, pitches, tempo, and rhythm should all improve simultaneously. ### What the notebook doesn't do Three honest limitations to keep in mind: - It runs on one synthesized example. Real piano recordings will have more overtones, more rhythmic complexity, and more places for each fix to fail in interesting ways. - It treats the AMT Report Card score as a small metrics table, not the full rubric. The companion AMT Report Card Space has the full scoring. - The Verovio renderer is not the same as LilyPond. Engraving conventions differ slightly. The visual evidence of the gap is identical either way. ### Next step If you want to add these fixes to the actual Music to Sheet Music Space, open `summer-prompt.md` in this folder. It is a long prompt that already knows your project and the code change. Paste it into Claude or Codex and follow what it says. """ ) return if __name__ == "__main__": app.run()