import spaces # MUST come before any torch / CUDA-touching import import torch import gradio as gr import tempfile import os import time import sys import io import json import re import base64 from pathlib import Path from safetensors.torch import load_file from huggingface_hub import hf_hub_download from muscriptor.models.lm import LMModel, TorchAutocast from muscriptor.modules.conditioners import ( MelSpectrogramConditioner, ClassConditioner, ConditioningProvider, ) from muscriptor.tokenizer.mt3 import MT3Tokenizer, MT3_FULL_PLUS_GROUP_NAMES from muscriptor.transcription_model import ( TranscriptionModel, _resolve_source, _resolve_config, _remap_single_codebook_keys, _build_model, ) _SAMPLE_RATE = 16000 def load_model_zerogpu(): """Load the MuScriptor model in a ZeroGPU-compatible way. On ZeroGPU, safetensors.load_file(device="cuda") fails because there's no real GPU at module scope. However, .to("cuda") is intercepted by the spaces hijack. So we: 1. Build the model with device="cuda" (conditioners store this as self.device and use it at runtime to route tensors). 2. Load safetensors weights on CPU. 3. Load_state_dict into the model (weights land on CPU because the model's tensors are still fake-CUDA via the hijack). 4. Call model.to("cuda") — ZeroGPU intercepts this and packs weights to disk, streaming them into VRAM on the first @spaces.GPU call. """ device = torch.device("cuda") # ZeroGPU intercepts this source = _resolve_source("large") weights_path = Path(hf_hub_download( repo_id="MuScriptor/muscriptor-large", filename="model.safetensors", )) cfg = _resolve_config(source, weights_path) # Build the model with device="cuda" — conditioners store this and use it # at runtime to route tensors. ZeroGPU intercepts .to("cuda") calls inside. model = _build_model(device, cfg) model.eval() # Load weights on CPU (safetensors can't load to fake CUDA) state_dict = load_file(str(weights_path), device="cpu") state_dict = _remap_single_codebook_keys(state_dict) # Load state dict — the model's parameters are fake-CUDA (ZeroGPU), # but load_state_dict copies CPU data into them, which is fine. model.load_state_dict(state_dict) # Move everything to "cuda" — ZeroGPU intercepts this and packs to disk model.to("cuda") tokenizer = MT3Tokenizer( instrument_vocabulary="MT3_FULL_PLUS", max_shift_steps=1001, ) return TranscriptionModel(model=model, tokenizer=tokenizer, device=device) print("[muscriptor-space] Loading model...", file=sys.stderr, flush=True) t0 = time.perf_counter() model = load_model_zerogpu() print(f"[muscriptor-space] Model loaded in {time.perf_counter() - t0:.1f}s", file=sys.stderr, flush=True) # Build the instrument choices list (sorted by group ID for stable ordering) INSTRUMENT_CHOICES = sorted(MT3_FULL_PLUS_GROUP_NAMES.keys(), key=lambda k: MT3_FULL_PLUS_GROUP_NAMES[k]) CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } midi-player { width: 100%; display: block; margin-bottom: 8px; } midi-visualizer { width: 100%; display: block; overflow: auto; } """ # The html-midi-player web components ( / ) rely # on Magenta.js + Tone.js. Following Gradio's official custom-HTML-components # guide (https://gradio.app/guides/custom-HTML-components), the library is # loaded via the gr.HTML `head` parameter instead of embedding ' ) # Static markup for the player/visualizer. js_on_load wires the transcribed # MIDI (a base64 data URI passed as the component value) into the # and links it to the so both mount and render. _MIDI_PLAYER_TEMPLATE = ( '
' '' '' "
" ) # Runs once when the component loads. `element` is the component's DOM node, # `props.value` is the base64 data URI (or "" when there's nothing yet). # `watch('value', ...)` re-runs the sync every time the server pushes a new # value (i.e. after each transcription), so the web components actually update. _MIDI_PLAYER_JS = """ function syncPlayer() { const player = element.querySelector('midi-player'); const container = element.querySelector('.midi-player-container'); if (!player || !container) { return; } if (props.value) { container.style.display = 'block'; player.src = props.value; } else { container.style.display = 'none'; player.removeAttribute('src'); } } syncPlayer(); watch('value', syncPlayer); """ def _midi_player_src(midi_bytes: bytes | None) -> str: """Return the transcribed MIDI as a base64 data URI for the player. The MIDI data is embedded directly as a base64 data URI so it needs no separate file-serving route. The / web components consume this via the gr.HTML custom component's js_on_load. """ if not midi_bytes: return "" b64 = base64.b64encode(midi_bytes).decode("ascii") return f"data:audio/midi;base64,{b64}" @spaces.GPU(duration=120) def transcribe_audio( audio_path: str, instruments: list[str] | None, use_sampling: bool, temperature: float, progress=gr.Progress(track_tqdm=True), ): """Transcribe an audio recording into a downloadable MIDI file. Upload any music recording (multi-instrument, any genre) and MuScriptor will convert it into a MIDI file with per-note onset, offset, pitch, and instrument information. Args: audio_path: Path to the uploaded audio file (wav, mp3, flac, etc.). instruments: Optional list of instrument group names to condition the transcription (improves coherence when you know which instruments are present). Leave empty for automatic (unconditioned) transcription. use_sampling: If True, use stochastic sampling instead of greedy decoding. temperature: Sampling temperature (only used when use_sampling is True). """ if audio_path is None: return None, "Please upload an audio file first.", "" t0 = time.perf_counter() # Run the transcription — returns MIDI bytes try: midi_bytes = model.transcribe_to_midi( audio_path, instruments=instruments if instruments else None, use_sampling=use_sampling, temperature=temperature, sliding_hop=1.0 ) except Exception as e: return None, f"Transcription failed: {e}", "" elapsed = time.perf_counter() - t0 # Write to a temporary file for download tmp = tempfile.NamedTemporaryFile(suffix=".mid", delete=False) tmp.write(midi_bytes) tmp.close() # Also collect note statistics for the summary note_count = 0 instrument_set = set() try: from mido import MidiFile midi = MidiFile(tmp.name) for track in midi.tracks: for msg in track: if msg.type == "note_on" and msg.velocity > 0: note_count += 1 instrument_set.add(msg.program if not msg.is_meta else 0) except Exception: pass summary = ( f"Transcription complete in {elapsed:.1f}s. " f"Found {note_count} notes across {len(instrument_set)} instrument program(s). " f"Play it back below or download the MIDI file." ) player_src = _midi_player_src(midi_bytes) return tmp.name, summary, player_src with gr.Blocks() as demo: with gr.Column(elem_id="col-container"): gr.Markdown( """ # MuScriptor — Music Transcription (Audio → MIDI) Upload a music recording and get a downloadable MIDI file with transcribed notes. MuScriptor is a ~1.3B parameter multi-instrument automatic music transcription model developed by [Mirelo](https://www.mirelo.ai/) x [Kyutai](https://kyutai.org/). [Model card](https://huggingface.co/MuScriptor/muscriptor-large) · [Code](https://github.com/muscriptor/muscriptor) · [Audio samples](https://muscriptor.github.io) """ ) with gr.Row(): with gr.Column(): audio_input = gr.Audio( label="Upload or record audio", type="filepath", sources=["upload", "microphone"], ) with gr.Accordion("Advanced settings", open=False): instrument_checkbox = gr.CheckboxGroup( choices=INSTRUMENT_CHOICES, value=[], label="Instrument conditioning (optional)", info="Select instruments present in the audio to improve transcription accuracy. Leave empty for automatic detection.", ) use_sampling = gr.Checkbox( label="Use sampling (stochastic decoding)", value=False, info="If enabled, uses temperature-based sampling instead of greedy decoding.", ) temperature = gr.Slider( label="Temperature", minimum=0.1, maximum=2.0, value=1.0, step=0.1, info="Sampling temperature (only used when sampling is enabled).", ) transcribe_btn = gr.Button("Transcribe", variant="primary") with gr.Column(): midi_player = gr.HTML( value="", label="MIDI playback & visualization", head=_MIDI_PLAYER_HEAD, html_template=_MIDI_PLAYER_TEMPLATE, js_on_load=_MIDI_PLAYER_JS, ) midi_output = gr.File(label="Download MIDI file") summary_output = gr.Textbox(label="Summary", interactive=False, visible=False) transcribe_btn.click( fn=transcribe_audio, inputs=[audio_input, instrument_checkbox, use_sampling, temperature], outputs=[midi_output, summary_output, midi_player], api_name="transcribe", ) gr.Examples( examples=[ ["example_medicine.mp3", [], False, 1.0], ["example_organic_flow.mp3", [], False, 1.0], ["example_water_afro_pop.mp3", [], False, 1.0], ], inputs=[audio_input, instrument_checkbox, use_sampling, temperature], outputs=[midi_output, summary_output, midi_player], fn=transcribe_audio, cache_examples=True, cache_mode="lazy", ) demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)