notebook.py

"""
Music-to-Sheet-Music: an interactive exploration of three architectural fixes.

This is a Marimo notebook. Run it locally:

    pip install marimo basic-pitch music21 librosa verovio cairosvg soundfile
    marimo run notebook.py

The notebook reproduces the student's transcription pipeline (Basic Pitch ->
music21 -> MusicXML -> sheet music), then lets you toggle three architectural
fixes on and off to see what each one repairs. The three fixes correspond to
the three failure modes the student's paper documents at the seam between
pitch detection and notation.
"""

import marimo

__generated_with = "0.23.8"
app = marimo.App(width="medium")


@app.cell
def setup():
    import marimo as mo
    return (mo,)


@app.cell
def intro(mo):
    mo.md(
        r"""
        # Music-to-Sheet-Music: Where the Map Loses the City

        The student's research paper makes a precise claim: when a lightweight
        AI transcription pipeline turns audio into sheet music, the failure is
        not a few wrong notes. It is *a cascade across every dimension of
        musical information at once* — pitch (overtones detected as separate
        notes), rhythm (no beat tracking), time signature (defaults to 4/4),
        tempo (defaults to ♩ = 120).

        This notebook reproduces the pipeline and lets you turn three
        architectural fixes on and off, one at a time, to see what each one
        actually repairs. It is the interactive version of the static
        comparison on the capstone page.

        - **Fix 1 — Octave-overtone filter.** Drop any pitch that is exactly
          one octave above another note and starts within ~50 ms of it. These
          are harmonics, not real notes.
        - **Fix 2 — Beat tracking with librosa.** Find the actual tempo from
          the audio before quantizing. Default music21 quantization assumes
          a fixed grid and produces nonsense durations when the tempo is
          wrong.
        - **Fix 3 — Tempo-aware quantization.** Quantize each note's onset
          and duration to the detected beat grid, not to a default
          sixteenth-note quarter-length grid.

        Each fix is small. Together they close most of the gap the paper
        documents — without touching the audio model.
        """
    )
    return


@app.cell
def imports():
    import os
    import tempfile
    import numpy as np
    import soundfile as sf
    from music21 import (
        stream,
        note,
        meter,
        key,
        clef,
        tempo as m21_tempo,
        instrument,
    )
    return (
        clef,
        instrument,
        key,
        m21_tempo,
        meter,
        np,
        note,
        os,
        sf,
        stream,
        tempfile,
    )


@app.cell
def constants():
    SAMPLE_RATE = 22050
    NOTE_FREQS = {
        "C4": 261.63,
        "D4": 293.66,
        "E4": 329.63,
        "F4": 349.23,
        "G4": 392.00,
        "A4": 440.00,
        "B4": 493.88,
        "C5": 523.25,
    }

    # The same Ode to Joy used on the static capstone page.
    ODE_TO_JOY = [
        ("E4", 1),
        ("E4", 1),
        ("F4", 1),
        ("G4", 1),
        ("G4", 1),
        ("F4", 1),
        ("E4", 1),
        ("D4", 1),
        ("C4", 1),
        ("C4", 1),
        ("D4", 1),
        ("E4", 1),
        ("E4", 1.5),
        ("D4", 0.5),
        ("D4", 2.0),
    ]

    TEMPO_BPM = 92
    SECS_PER_BEAT = 60.0 / TEMPO_BPM
    return NOTE_FREQS, ODE_TO_JOY, SAMPLE_RATE, SECS_PER_BEAT, TEMPO_BPM


@app.cell
def synth_audio_fn(NOTE_FREQS, SAMPLE_RATE, SECS_PER_BEAT, np):
    def synth_note(freq, duration_sec, sample_rate=SAMPLE_RATE):
        n_samples = int(duration_sec * sample_rate)
        t = np.linspace(0, duration_sec, n_samples, endpoint=False)
        # Fundamental + harmonics. The octave overtone here is what causes
        # Basic Pitch to hallucinate phantom pitches one octave above the
        # real note.
        wave = (
            1.00 * np.sin(2 * np.pi * freq * t)
            + 0.50 * np.sin(2 * np.pi * 2 * freq * t)
            + 0.25 * np.sin(2 * np.pi * 3 * freq * t)
            + 0.10 * np.sin(2 * np.pi * 4 * freq * t)
        )
        attack_n = int(0.005 * sample_rate)
        decay_constant = duration_sec * 0.4
        env = np.exp(-t / decay_constant)
        if attack_n > 0:
            env[:attack_n] *= np.linspace(0, 1, attack_n)
        wave *= env
        return wave

    def synth_melody(melody, secs_per_beat=SECS_PER_BEAT):
        chunks = []
        for pitch, dur_beats in melody:
            chunks.append(synth_note(NOTE_FREQS[pitch], dur_beats * secs_per_beat))
        full = np.concatenate(chunks)
        full = full / np.max(np.abs(full)) * 0.7
        return full

    return synth_melody, synth_note


@app.cell
def step1_synthesize_audio(ODE_TO_JOY, SAMPLE_RATE, mo, os, sf, synth_melody, tempfile):
    audio = synth_melody(ODE_TO_JOY)
    # Save to a temp WAV so Basic Pitch can read it and so the player can play it.
    audio_path = os.path.join(tempfile.gettempdir(), "ode_to_joy_input.wav")
    sf.write(audio_path, audio, SAMPLE_RATE)

    mo.md(
        f"""
        ## Step 1 — Synthesized input audio

        15 notes of *Ode to Joy*, synthesized as a piano-like tone (fundamental
        plus three harmonics, sharp attack, exponential decay). This is the
        audio Basic Pitch will see. Listen — the overtones are what cause
        the phantom pitches.
        """
    )
    return audio, audio_path


@app.cell
def audio_player(audio_path, mo):
    mo.audio(audio_path)
    return


@app.cell
def step2_run_basic_pitch(audio_path, mo):
    import basic_pitch
    from basic_pitch.inference import predict

    onnx_path = os.path.join(
        os.path.dirname(basic_pitch.__file__),
        "saved_models",
        "icassp_2022",
        "nmp.onnx",
    )

    # Run Basic Pitch on the synthesized audio.
    _, midi_data, note_events = predict(
        audio_path, model_or_model_path=onnx_path
    )

    # note_events is a list of (start_sec, end_sec, pitch_midi, amplitude, pitch_bends)
    mo.md(
        f"""
        ## Step 2 — Basic Pitch transcription

        Basic Pitch ran on the audio above and detected **{len(note_events)}**
        notes. The ground truth is 15 notes (one for each note in *Ode to Joy*).
        Anything beyond 15 is either a hallucination or a fragmented duplicate.
        """
    )
    return midi_data, note_events, onnx_path, predict


@app.cell
def show_raw_note_events(mo, note_events):
    rows = []
    for _start, _end, _pitch, _amp, _bends in note_events:
        rows.append(
            {
                "start_sec": round(_start, 3),
                "end_sec": round(_end, 3),
                "midi_pitch": int(_pitch),
                "amplitude": round(_amp, 3),
            }
        )

    mo.md("### Raw note events from Basic Pitch (before any fixes)")
    return (rows,)


@app.cell
def raw_table(mo, rows):
    mo.ui.table(rows, page_size=10)
    return


@app.cell
def fix_controls(mo):
    overtone_toggle = mo.ui.switch(value=False, label="Fix 1 — Octave overtone filter")
    beat_track_toggle = mo.ui.switch(value=False, label="Fix 2 — Beat tracking with librosa")
    quantize_toggle = mo.ui.switch(value=False, label="Fix 3 — Tempo-aware quantization")

    mo.md(
        f"""
        ## Step 3 — Three architectural fixes

        Toggle each fix on or off. The notebook will reprocess the Basic
        Pitch output and show the resulting sheet music and metrics below.

        {mo.vstack([overtone_toggle, beat_track_toggle, quantize_toggle])}
        """
    )
    return beat_track_toggle, overtone_toggle, quantize_toggle


@app.cell
def fix_functions(np):
    def fix_overtones(note_events, octave_window_sec=0.05):
        """Drop notes that are exactly 12 semitones above another note
        starting within octave_window_sec.

        These are almost always harmonic overtones the model detected as
        separate notes."""
        # Sort by start time so we can scan a small window
        events = sorted(note_events, key=lambda e: e[0])
        keep = [True] * len(events)
        for i, (s_i, _, p_i, _, _) in enumerate(events):
            for j, (s_j, _, p_j, _, _) in enumerate(events):
                if i == j or not keep[i]:
                    continue
                if abs(s_i - s_j) <= octave_window_sec and p_i == p_j + 12:
                    keep[i] = False
                    break
        return [e for e, k in zip(events, keep) if k]

    def detect_tempo_with_librosa(audio_array, sample_rate):
        """Use librosa to detect the tempo from the audio. Returns BPM."""
        import librosa

        tempo, _ = librosa.beat.beat_track(y=audio_array, sr=sample_rate)
        # librosa returns a numpy array sometimes; coerce to float
        return float(np.atleast_1d(tempo)[0])

    def tempo_aware_quantize(note_events, bpm, subdivision=4):
        """Quantize each note's onset and duration to the beat grid implied
        by bpm. Default subdivision=4 means quarter-note grid (snap to
        sixteenths within a beat)."""
        sec_per_beat = 60.0 / bpm
        sec_per_grid = sec_per_beat / subdivision
        quantized = []
        for s, e, p, a, pb in note_events:
            s_q = round(s / sec_per_grid) * sec_per_grid
            dur = e - s
            dur_q = max(sec_per_grid, round(dur / sec_per_grid) * sec_per_grid)
            quantized.append((s_q, s_q + dur_q, p, a, pb))
        return quantized

    return detect_tempo_with_librosa, fix_overtones, tempo_aware_quantize


@app.cell
def apply_fixes(
    audio,
    beat_track_toggle,
    detect_tempo_with_librosa,
    fix_overtones,
    note_events,
    overtone_toggle,
    quantize_toggle,
    SAMPLE_RATE,
    tempo_aware_quantize,
):
    processed = list(note_events)
    detected_bpm = None

    if overtone_toggle.value:
        processed = fix_overtones(processed)

    if beat_track_toggle.value:
        detected_bpm = detect_tempo_with_librosa(audio, SAMPLE_RATE)

    if quantize_toggle.value:
        # If beat tracking is also on, use its detected BPM; otherwise
        # quantize against the default music21 assumption of 120 BPM.
        bpm = detected_bpm if detected_bpm is not None else 120.0
        processed = tempo_aware_quantize(processed, bpm)
    return detected_bpm, processed


@app.cell
def show_metrics(
    beat_track_toggle,
    detected_bpm,
    mo,
    note_events,
    overtone_toggle,
    processed,
    quantize_toggle,
):
    GROUND_TRUTH_COUNT = 15
    GROUND_TRUTH_TEMPO = 92.0
    GROUND_TRUTH_PITCHES = {60, 62, 64, 65, 67}  # C4, D4, E4, F4, G4 (MIDI numbers)

    raw_count = len(note_events)
    processed_count = len(processed)
    processed_pitches = {int(p) for (_, _, p, _, _) in processed}
    phantom = processed_pitches - GROUND_TRUTH_PITCHES
    missing = GROUND_TRUTH_PITCHES - processed_pitches

    tempo_line = (
        f"**Detected tempo:** ♩ = {detected_bpm:.1f} BPM  (ground truth: ♩ = {GROUND_TRUTH_TEMPO} BPM)"
        if detected_bpm is not None
        else "**Detected tempo:** —  *(beat tracking off)*"
    )

    mo.md(
        f"""
        ### How the fixes compare to ground truth

        | Dimension | Ground truth | This run |
        | --- | --- | --- |
        | Note count | {GROUND_TRUTH_COUNT} | {processed_count}  ({processed_count - GROUND_TRUTH_COUNT:+d}) |
        | Unique pitches | C4, D4, E4, F4, G4 | {", ".join(sorted_pitch_names(processed_pitches))} |
        | Phantom pitches (false positives) | none | {", ".join(sorted_pitch_names(phantom)) or "none"} |
        | Missed pitches (false negatives) | none | {", ".join(sorted_pitch_names(missing)) or "none"} |

        {tempo_line}

        **Fixes currently applied:** {fixes_summary(overtone_toggle, beat_track_toggle, quantize_toggle)}
        """
    )
    return GROUND_TRUTH_COUNT, GROUND_TRUTH_PITCHES, GROUND_TRUTH_TEMPO


@app.cell
def helper_fns():
    def sorted_pitch_names(midi_set):
        names = []
        for n in sorted(midi_set):
            # MIDI 60 = C4
            octave = n // 12 - 1
            note_name = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"][n % 12]
            names.append(f"{note_name}{octave}")
        return names

    def fixes_summary(overtone_toggle, beat_track_toggle, quantize_toggle):
        active = []
        if overtone_toggle.value:
            active.append("overtone filter")
        if beat_track_toggle.value:
            active.append("beat tracking")
        if quantize_toggle.value:
            active.append("tempo-aware quantize")
        return ", ".join(active) if active else "*none — this is the raw Basic Pitch output*"

    return fixes_summary, sorted_pitch_names


@app.cell
def render_score(
    GROUND_TRUTH_TEMPO,
    clef,
    detected_bpm,
    instrument,
    key,
    m21_tempo,
    meter,
    mo,
    note,
    os,
    processed,
    stream,
    tempfile,
):
    """Render the processed note events as sheet music using music21 + Verovio."""
    # Build a music21 stream from the processed note events
    s = stream.Score()
    p = stream.Part()
    p.append(instrument.Piano())
    p.append(clef.TrebleClef())
    p.append(key.KeySignature(0))
    p.append(meter.TimeSignature("4/4"))

    bpm_for_render = detected_bpm if detected_bpm is not None else 120.0
    p.append(m21_tempo.MetronomeMark(number=int(bpm_for_render)))

    # Each note: (start_sec, end_sec, midi_pitch, amplitude, pitch_bends)
    sec_per_beat = 60.0 / bpm_for_render
    for s_sec, e_sec, midi_p, _, _ in sorted(processed, key=lambda x: x[0]):
        dur_beats = max(0.0625, (e_sec - s_sec) / sec_per_beat)
        n = note.Note(midi=int(midi_p), quarterLength=round(dur_beats * 16) / 16)
        p.append(n)

    s.append(p)

    # Write to a temp MusicXML file
    tmpdir = tempfile.gettempdir()
    xml_path = os.path.join(tmpdir, "marimo_processed.musicxml")
    s.write("musicxml", fp=xml_path)

    # Render with Verovio to SVG
    import verovio

    tk = verovio.toolkit()
    tk.setOptions(
        {
            "scale": 50,
            "pageWidth": 2200,
            "pageHeight": 600,
            "pageMarginLeft": 40,
            "pageMarginRight": 40,
            "pageMarginTop": 30,
            "pageMarginBottom": 30,
            "footer": "none",
            "header": "none",
            "breaks": "auto",
        }
    )
    tk.loadFile(xml_path)
    svg = tk.renderToSVG(1)

    mo.md(
        f"""
        ### Resulting sheet music

        Tempo: ♩ = {bpm_for_render:.0f} BPM. Ground truth was ♩ = {GROUND_TRUTH_TEMPO}.

        {mo.Html(svg)}
        """
    )
    return svg


@app.cell
def closing(mo):
    mo.md(
        r"""
        ## What this notebook is for

        This is the interactive sibling of the static comparison on the
        capstone page. The static page shows *what the failure looks like*
        on one example. This notebook shows *what each fix does to that
        failure*, one variable at a time.

        ### Suggested experiments

        1. **Turn on Fix 1 (overtone filter) only.** Watch the four phantom
           pitches (C5, D5, E5, G5) disappear. The note count should drop
           toward 15.

        2. **Turn on Fix 2 (beat tracking) only.** The detected tempo
           should be close to 92 BPM (the ground truth). This alone
           doesn't fix the rendered score because the durations are still
           wrong — but the tempo mark above the staff is now correct.

        3. **Turn on Fix 3 (tempo-aware quantize) only.** The durations
           snap to a 16th-note grid using the default 120 BPM (since beat
           tracking is off). The rhythm doesn't get better, just different.

        4. **Turn on Fix 2 + Fix 3 together.** Now the quantization uses
           the *detected* tempo. The durations should start looking
           closer to actual quarter notes.

        5. **Turn on all three.** This is the architectural fix the paper
           proposes in Section 6. Note count, pitches, tempo, and rhythm
           should all improve simultaneously.

        ### What the notebook doesn't do

        Three honest limitations to keep in mind:

        - It runs on one synthesized example. Real piano recordings will
          have more overtones, more rhythmic complexity, and more places
          for each fix to fail in interesting ways.
        - It treats the AMT Report Card score as a small metrics table,
          not the full rubric. The companion AMT Report Card Space has the
          full scoring.
        - The Verovio renderer is not the same as LilyPond. Engraving
          conventions differ slightly. The visual evidence of the gap is
          identical either way.

        ### Next step

        If you want to add these fixes to the actual Music to Sheet Music
        Space, open `summer-prompt.md` in this folder. It is a long
        prompt that already knows your project and the code change. Paste
        it into Claude or Codex and follow what it says.
        """
    )
    return


if __name__ == "__main__":
    app.run()